@forzalabs/remora 1.0.22 ā 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/automap.js +26 -42
- package/actions/compile.js +27 -43
- package/actions/create_consumer.js +24 -40
- package/actions/create_producer.js +16 -32
- package/actions/debug.js +18 -34
- package/actions/deploy.js +30 -46
- package/actions/discover.js +13 -29
- package/actions/init.js +29 -45
- package/actions/mock.js +16 -32
- package/actions/run.js +34 -52
- package/actions/sample.js +42 -58
- package/index.js +109 -119
- package/package.json +4 -4
- package/workers/ExecutorWorker.js +18 -34
- package/Constants.js +0 -34
- package/core/Affirm.js +0 -42
- package/core/Algo.js +0 -160
- package/core/dste/DSTE.js +0 -113
- package/core/logger/DebugLogService.js +0 -48
- package/core/logger/DevelopmentLogService.js +0 -70
- package/core/logger/LocalLogService.js +0 -70
- package/core/logger/Logger.js +0 -54
- package/database/DatabaseEngine.js +0 -149
- package/database/DatabaseStructure.js +0 -27
- package/definitions/DatasetDefinitions.js +0 -2
- package/definitions/ExecutorDefinitions.js +0 -2
- package/definitions/ProcessENV.js +0 -169
- package/definitions/agents/DestinationDriver.js +0 -2
- package/definitions/agents/SourceDriver.js +0 -2
- package/definitions/cli.js +0 -2
- package/definitions/database/ApiKeys.js +0 -2
- package/definitions/database/Stored.js +0 -7
- package/definitions/database/UsageStat.js +0 -2
- package/definitions/database/User.js +0 -2
- package/definitions/json_schemas/consumer-schema.json +0 -1226
- package/definitions/json_schemas/producer-schema.json +0 -308
- package/definitions/json_schemas/project-schema.json +0 -100
- package/definitions/json_schemas/source-schema.json +0 -249
- package/definitions/requests/ConsumerRequest.js +0 -2
- package/definitions/requests/Developer.js +0 -2
- package/definitions/requests/Mapping.js +0 -2
- package/definitions/requests/ProducerRequest.js +0 -2
- package/definitions/requests/Request.js +0 -2
- package/definitions/resources/Compiled.js +0 -2
- package/definitions/resources/Consumer.js +0 -2
- package/definitions/resources/Environment.js +0 -2
- package/definitions/resources/Library.js +0 -2
- package/definitions/resources/Producer.js +0 -2
- package/definitions/resources/Project.js +0 -2
- package/definitions/resources/Schema.js +0 -2
- package/definitions/resources/Source.js +0 -2
- package/definitions/temp.js +0 -2
- package/definitions/transform/Transformations.js +0 -2
- package/drivers/DeltaShareDriver.js +0 -186
- package/drivers/DriverFactory.js +0 -72
- package/drivers/DriverHelper.js +0 -248
- package/drivers/HttpApiDriver.js +0 -208
- package/drivers/RedshiftDriver.js +0 -184
- package/drivers/files/LocalDestinationDriver.js +0 -146
- package/drivers/files/LocalSourceDriver.js +0 -405
- package/drivers/s3/S3DestinationDriver.js +0 -197
- package/drivers/s3/S3SourceDriver.js +0 -495
- package/engines/CryptoEngine.js +0 -75
- package/engines/Environment.js +0 -170
- package/engines/ProcessENVManager.js +0 -112
- package/engines/RandomEngine.js +0 -47
- package/engines/SecretManager.js +0 -23
- package/engines/UserManager.js +0 -66
- package/engines/ai/AutoMapperEngine.js +0 -37
- package/engines/ai/DeveloperEngine.js +0 -497
- package/engines/ai/LLM.js +0 -255
- package/engines/consumer/ConsumerManager.js +0 -218
- package/engines/consumer/ConsumerOnFinishManager.js +0 -202
- package/engines/dataset/Dataset.js +0 -824
- package/engines/dataset/DatasetManager.js +0 -211
- package/engines/dataset/DatasetRecord.js +0 -120
- package/engines/dataset/DatasetRecordPool.js +0 -77
- package/engines/execution/RequestExecutor.js +0 -67
- package/engines/parsing/CSVParser.js +0 -60
- package/engines/parsing/LineParser.js +0 -71
- package/engines/parsing/ParseCompression.js +0 -101
- package/engines/parsing/ParseHelper.js +0 -18
- package/engines/parsing/ParseManager.js +0 -54
- package/engines/parsing/XLSParser.js +0 -87
- package/engines/parsing/XMLParser.js +0 -115
- package/engines/producer/ProducerEngine.js +0 -127
- package/engines/producer/ProducerManager.js +0 -43
- package/engines/scheduler/CronScheduler.js +0 -222
- package/engines/scheduler/QueueManager.js +0 -314
- package/engines/schema/SchemaValidator.js +0 -67
- package/engines/transform/JoinEngine.js +0 -232
- package/engines/transform/TransformationEngine.js +0 -277
- package/engines/transform/TypeCaster.js +0 -59
- package/engines/usage/DataframeManager.js +0 -55
- package/engines/usage/UsageDataManager.js +0 -151
- package/engines/usage/UsageManager.js +0 -65
- package/engines/validation/Validator.js +0 -216
- package/executors/ConsumerExecutor.js +0 -280
- package/executors/Executor.js +0 -177
- package/executors/ExecutorOrchestrator.js +0 -331
- package/executors/ExecutorPerformance.js +0 -17
- package/executors/ExecutorProgress.js +0 -54
- package/executors/ExecutorScope.js +0 -52
- package/executors/OutputExecutor.js +0 -118
- package/executors/ProducerExecutor.js +0 -108
- package/helper/Helper.js +0 -149
- package/helper/Logger.js +0 -84
- package/helper/Runtime.js +0 -20
- package/helper/Settings.js +0 -13
- package/licencing/LicenceManager.js +0 -64
- package/settings.js +0 -12
package/actions/automap.js
CHANGED
|
@@ -1,77 +1,61 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.automap = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const ora_1 = __importDefault(require("ora"));
|
|
18
|
-
const compile_1 = require("./compile");
|
|
19
|
-
const AutoMapperEngine_1 = __importDefault(require("../engines/ai/AutoMapperEngine"));
|
|
20
|
-
const fs_1 = __importDefault(require("fs"));
|
|
21
|
-
const path_1 = __importDefault(require("path"));
|
|
22
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
23
|
-
const ProducerEngine_1 = __importDefault(require("../engines/producer/ProducerEngine"));
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { compile } from './compile';
|
|
4
|
+
import { AutoMapperEngine } from '@remora/app-engines';
|
|
5
|
+
import fs from 'fs';
|
|
6
|
+
import path from 'path';
|
|
7
|
+
import { Environment } from '@remora/app-common';
|
|
8
|
+
import { ProducerEngine } from '@remora/app-engines';
|
|
24
9
|
/**
|
|
25
10
|
* e.g. npm run automap -- myclaims Claim
|
|
26
11
|
*/
|
|
27
|
-
const automap = (producerName, schemaNames) =>
|
|
12
|
+
export const automap = async (producerName, schemaNames) => {
|
|
28
13
|
try {
|
|
29
|
-
|
|
30
|
-
const spinner = (
|
|
14
|
+
compile();
|
|
15
|
+
const spinner = ora(chalk.blue('Auto-mapping producer data...\n')).start();
|
|
31
16
|
// Get the producer
|
|
32
|
-
const producer =
|
|
17
|
+
const producer = Environment.getProducer(producerName);
|
|
33
18
|
if (!producer) {
|
|
34
19
|
throw new Error(`Producer ${producerName} not found`);
|
|
35
20
|
}
|
|
36
|
-
const source =
|
|
21
|
+
const source = Environment.getSource(producer.source);
|
|
37
22
|
if (!source) {
|
|
38
23
|
throw new Error(`Source ${producer.source} not found`);
|
|
39
24
|
}
|
|
40
25
|
// Get the specified schemas
|
|
41
26
|
const schemas = [];
|
|
42
27
|
for (const schemaName of schemaNames) {
|
|
43
|
-
const schema =
|
|
28
|
+
const schema = Environment.getSchema(schemaName);
|
|
44
29
|
if (!schema) {
|
|
45
30
|
throw new Error(`Schema ${schemaName} not found`);
|
|
46
31
|
}
|
|
47
32
|
schemas.push(schema);
|
|
48
33
|
}
|
|
49
34
|
// Read and convert sample data
|
|
50
|
-
const sampleData =
|
|
35
|
+
const sampleData = await ProducerEngine.readSampleData(producer);
|
|
51
36
|
// Convert sample data to strings for AutoMapperEngine
|
|
52
37
|
const sampleStrings = sampleData.map(item => JSON.stringify(item));
|
|
53
38
|
// Call the automapper
|
|
54
|
-
const mapResult =
|
|
39
|
+
const mapResult = await AutoMapperEngine.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
|
|
55
40
|
// Create the producers based on the mapping
|
|
56
41
|
for (const producer of mapResult.producers) {
|
|
57
|
-
const producerPath =
|
|
58
|
-
|
|
59
|
-
console.log(
|
|
42
|
+
const producerPath = path.join('remora/producers', `${producer.name}.json`);
|
|
43
|
+
fs.writeFileSync(producerPath, JSON.stringify(producer, null, 4));
|
|
44
|
+
console.log(chalk.blue(`Created producer: ${producer.name}`));
|
|
60
45
|
}
|
|
61
46
|
// Create the consumers based on the mapping
|
|
62
47
|
for (const consumer of mapResult.consumers) {
|
|
63
|
-
const consumerPath =
|
|
64
|
-
|
|
65
|
-
console.log(
|
|
48
|
+
const consumerPath = path.join('remora/consumers', `${consumer.name}.json`);
|
|
49
|
+
fs.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
|
|
50
|
+
console.log(chalk.blue(`Created consumer: ${consumer.name}`));
|
|
66
51
|
}
|
|
67
52
|
spinner.succeed('Producer has been successfully mapped');
|
|
68
|
-
console.log(
|
|
69
|
-
console.log(
|
|
53
|
+
console.log(chalk.green(`\nā
Created ${mapResult.producers.length} producers!`));
|
|
54
|
+
console.log(chalk.green(`ā
Created ${mapResult.consumers.length} consumers!`));
|
|
70
55
|
process.exit(0);
|
|
71
56
|
}
|
|
72
57
|
catch (err) {
|
|
73
|
-
console.error(
|
|
58
|
+
console.error(chalk.red.bold('\nā Unexpected error during automapping:'), err instanceof Error ? err.message : String(err));
|
|
74
59
|
process.exit(1);
|
|
75
60
|
}
|
|
76
|
-
}
|
|
77
|
-
exports.automap = automap;
|
|
61
|
+
};
|
package/actions/compile.js
CHANGED
|
@@ -1,57 +1,41 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.compile = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
18
|
-
const ora_1 = __importDefault(require("ora"));
|
|
19
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
20
|
-
const compile = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs-extra';
|
|
3
|
+
import ora from 'ora';
|
|
4
|
+
import { Environment } from '@remora/app-common';
|
|
5
|
+
export const compile = async () => {
|
|
21
6
|
// TODO compile single file
|
|
22
7
|
let errors = [];
|
|
23
8
|
try {
|
|
24
|
-
const spinner = (
|
|
25
|
-
|
|
26
|
-
if (!
|
|
27
|
-
errors.push(
|
|
28
|
-
if (!
|
|
29
|
-
errors.push(
|
|
30
|
-
if (!
|
|
31
|
-
errors.push(
|
|
32
|
-
if (!
|
|
33
|
-
errors.push(
|
|
34
|
-
if (!
|
|
35
|
-
errors.push(
|
|
36
|
-
if (!
|
|
37
|
-
errors.push(
|
|
38
|
-
const envErrors =
|
|
9
|
+
const spinner = ora(chalk.blue('Validating project structure...')).start();
|
|
10
|
+
Environment.load('');
|
|
11
|
+
if (!fs.existsSync('./remora'))
|
|
12
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora'));
|
|
13
|
+
if (!fs.existsSync('./remora/consumers'))
|
|
14
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora/consumers'));
|
|
15
|
+
if (!fs.existsSync('./remora/producers'))
|
|
16
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora/producers'));
|
|
17
|
+
if (!fs.existsSync('./remora/producers'))
|
|
18
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora/producers'));
|
|
19
|
+
if (!fs.existsSync('./remora/sources'))
|
|
20
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora/sources'));
|
|
21
|
+
if (!fs.existsSync('./remora/schemas'))
|
|
22
|
+
errors.push(chalk.red('Missing directory: ') + chalk.yellow('./remora/schemas'));
|
|
23
|
+
const envErrors = Environment.validate();
|
|
39
24
|
errors = [...errors, ...envErrors];
|
|
40
25
|
if (errors.length === 0) {
|
|
41
|
-
spinner.succeed(
|
|
42
|
-
console.log(
|
|
26
|
+
spinner.succeed(chalk.green('Project structure validated successfully'));
|
|
27
|
+
console.log(chalk.blueBright.bold('ā
Compilation complete!'));
|
|
43
28
|
}
|
|
44
29
|
else {
|
|
45
|
-
spinner.fail(
|
|
30
|
+
spinner.fail(chalk.red('Validation failed with errors:'));
|
|
46
31
|
console.log('');
|
|
47
|
-
errors.forEach(error => console.log(
|
|
48
|
-
console.log(
|
|
32
|
+
errors.forEach(error => console.log(chalk.red('⢠' + error)));
|
|
33
|
+
console.log(chalk.red.bold('\nā Compilation aborted due to errors.'));
|
|
49
34
|
process.exit(1);
|
|
50
35
|
}
|
|
51
36
|
}
|
|
52
37
|
catch (err) {
|
|
53
|
-
console.error(
|
|
38
|
+
console.error(chalk.red.bold('\nā Unexpected error during validation:'), err instanceof Error ? err.message : String(err));
|
|
54
39
|
process.exit(1);
|
|
55
40
|
}
|
|
56
|
-
}
|
|
57
|
-
exports.compile = compile;
|
|
41
|
+
};
|
|
@@ -1,37 +1,22 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.create_consumer = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const fs_1 = __importDefault(require("fs"));
|
|
18
|
-
const path_1 = __importDefault(require("path"));
|
|
19
|
-
const create_consumer = (name, producerName) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
export const create_consumer = async (name, producerName) => {
|
|
20
5
|
try {
|
|
21
|
-
if (!
|
|
22
|
-
throw new Error(
|
|
6
|
+
if (!fs.existsSync('./remora/consumers')) {
|
|
7
|
+
throw new Error(chalk.red('Missing directory: ') + chalk.yellow('./remora/consumers'));
|
|
23
8
|
}
|
|
24
9
|
// Read the default consumer template
|
|
25
|
-
const defaultConsumerTemplate =
|
|
10
|
+
const defaultConsumerTemplate = fs.readFileSync(path.join(__dirname, '../documentation/default_resources/consumer.json'), 'utf-8');
|
|
26
11
|
const defaultConsumer = JSON.parse(defaultConsumerTemplate);
|
|
27
12
|
defaultConsumer.name = name;
|
|
28
13
|
// If a producer is specified, set up a one-to-one mapping
|
|
29
14
|
if (producerName) {
|
|
30
|
-
const producerPath =
|
|
31
|
-
if (!
|
|
32
|
-
throw new Error(
|
|
15
|
+
const producerPath = path.join('remora/producers', `${producerName}.json`);
|
|
16
|
+
if (!fs.existsSync(producerPath)) {
|
|
17
|
+
throw new Error(chalk.red('Producer not found: ') + chalk.yellow(producerPath));
|
|
33
18
|
}
|
|
34
|
-
const producerConfig = JSON.parse(
|
|
19
|
+
const producerConfig = JSON.parse(fs.readFileSync(producerPath, 'utf-8'));
|
|
35
20
|
// Set up producer reference
|
|
36
21
|
defaultConsumer.producers = [{ name: producerName }];
|
|
37
22
|
// Set up fields to include all producer fields
|
|
@@ -51,25 +36,24 @@ const create_consumer = (name, producerName) => __awaiter(void 0, void 0, void 0
|
|
|
51
36
|
defaultConsumer.filters = undefined;
|
|
52
37
|
defaultConsumer.metadata = undefined;
|
|
53
38
|
}
|
|
54
|
-
const consumerPath =
|
|
55
|
-
|
|
56
|
-
console.log(
|
|
39
|
+
const consumerPath = path.join('remora/consumers', `${name}.json`);
|
|
40
|
+
fs.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
|
|
41
|
+
console.log(chalk.green(`ā
Created consumer config at ${consumerPath}`));
|
|
57
42
|
if (!producerName) {
|
|
58
|
-
console.log(
|
|
59
|
-
console.log(
|
|
60
|
-
console.log(
|
|
61
|
-
console.log(
|
|
43
|
+
console.log(chalk.blue('Remember to:'));
|
|
44
|
+
console.log(chalk.blue('1. Set the correct producer name'));
|
|
45
|
+
console.log(chalk.blue('2. Configure the fields you want to include'));
|
|
46
|
+
console.log(chalk.blue('3. Set up the desired output formats'));
|
|
62
47
|
}
|
|
63
48
|
else {
|
|
64
|
-
console.log(
|
|
65
|
-
console.log(
|
|
66
|
-
console.log(
|
|
67
|
-
console.log(
|
|
49
|
+
console.log(chalk.blue('Consumer is configured to match all fields from producer:'), chalk.yellow(producerName));
|
|
50
|
+
console.log(chalk.blue('Remember to:'));
|
|
51
|
+
console.log(chalk.blue('1. Review the included fields'));
|
|
52
|
+
console.log(chalk.blue('2. Set up the desired output formats'));
|
|
68
53
|
}
|
|
69
54
|
}
|
|
70
55
|
catch (error) {
|
|
71
|
-
console.error(
|
|
56
|
+
console.error(chalk.red('ā Error creating consumer:'), error instanceof Error ? error.message : String(error));
|
|
72
57
|
process.exit(1);
|
|
73
58
|
}
|
|
74
|
-
}
|
|
75
|
-
exports.create_consumer = create_consumer;
|
|
59
|
+
};
|
|
@@ -1,41 +1,25 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.create_producer = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const fs_1 = __importDefault(require("fs"));
|
|
18
|
-
const path_1 = __importDefault(require("path"));
|
|
19
|
-
const create_producer = (name) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
export const create_producer = async (name) => {
|
|
20
5
|
try {
|
|
21
|
-
if (!
|
|
22
|
-
throw new Error(
|
|
6
|
+
if (!fs.existsSync('./remora/producers')) {
|
|
7
|
+
throw new Error(chalk.red('Missing directory: ') + chalk.yellow('./remora/producers'));
|
|
23
8
|
}
|
|
24
9
|
// Read the default producer template
|
|
25
|
-
const defaultProducerTemplate =
|
|
10
|
+
const defaultProducerTemplate = fs.readFileSync(path.join(__dirname, '../documentation/default_resources/producer.json'), 'utf-8');
|
|
26
11
|
const defaultProducer = JSON.parse(defaultProducerTemplate);
|
|
27
12
|
defaultProducer.name = name;
|
|
28
|
-
const producerPath =
|
|
29
|
-
|
|
30
|
-
console.log(
|
|
31
|
-
console.log(
|
|
32
|
-
console.log(
|
|
33
|
-
console.log(
|
|
34
|
-
console.log(
|
|
13
|
+
const producerPath = path.join('remora/producers', `${name}.json`);
|
|
14
|
+
fs.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
|
|
15
|
+
console.log(chalk.green(`ā
Created producer config at ${producerPath}`));
|
|
16
|
+
console.log(chalk.blue('Remember to:'));
|
|
17
|
+
console.log(chalk.blue('1. Set the correct source name'));
|
|
18
|
+
console.log(chalk.blue('2. Configure the dimensions and measures (or use the "discover" command)'));
|
|
19
|
+
console.log(chalk.blue('3. Update the settings as needed'));
|
|
35
20
|
}
|
|
36
21
|
catch (error) {
|
|
37
|
-
console.error(
|
|
22
|
+
console.error(chalk.red('ā Error creating producer:'), error instanceof Error ? error.message : String(error));
|
|
38
23
|
process.exit(1);
|
|
39
24
|
}
|
|
40
|
-
}
|
|
41
|
-
exports.create_producer = create_producer;
|
|
25
|
+
};
|
package/actions/debug.js
CHANGED
|
@@ -1,41 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.debug = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const ora_1 = __importDefault(require("ora"));
|
|
18
|
-
const DriverFactory_1 = __importDefault(require("../drivers/DriverFactory"));
|
|
19
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
20
|
-
const compile_1 = require("./compile");
|
|
21
|
-
const debug = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { DriverFactory } from '@remora/app-drivers';
|
|
4
|
+
import { Environment } from '@remora/app-common';
|
|
5
|
+
import { compile } from './compile';
|
|
6
|
+
export const debug = async (options) => {
|
|
22
7
|
void options;
|
|
23
8
|
try {
|
|
24
|
-
|
|
9
|
+
compile();
|
|
25
10
|
console.log('\n');
|
|
26
|
-
const spinner = (
|
|
27
|
-
const producers =
|
|
11
|
+
const spinner = ora({ text: 'ā³ Checking producers connections...', color: 'blue' }).start();
|
|
12
|
+
const producers = Environment._env.producers;
|
|
28
13
|
const failures = [];
|
|
29
14
|
for (const producer of producers) {
|
|
30
15
|
spinner.text = `š Testing producer "${producer.name}" (source: ${producer.source})`;
|
|
31
|
-
const source =
|
|
32
|
-
const driver =
|
|
33
|
-
const canConnect =
|
|
16
|
+
const source = Environment.getSource(producer.source);
|
|
17
|
+
const driver = await DriverFactory.instantiateSource(source);
|
|
18
|
+
const canConnect = await driver.exist(producer);
|
|
34
19
|
if (canConnect) {
|
|
35
20
|
spinner.succeed(`Producer "${producer.name}" connected successfully.`);
|
|
36
21
|
}
|
|
37
22
|
else {
|
|
38
|
-
spinner.fail(
|
|
23
|
+
spinner.fail(chalk.red(`ā Producer "${producer.name}" failed to connect.`));
|
|
39
24
|
failures.push(producer.name);
|
|
40
25
|
}
|
|
41
26
|
// Avoid restarting the spinner unless needed
|
|
@@ -44,19 +29,18 @@ const debug = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
44
29
|
}
|
|
45
30
|
}
|
|
46
31
|
if (failures.length > 0) {
|
|
47
|
-
console.log(
|
|
48
|
-
failures.forEach(name => console.log(
|
|
32
|
+
console.log(chalk.red.bold(`\nā¼ ${failures.length} producer(s) could not connect:`));
|
|
33
|
+
failures.forEach(name => console.log(chalk.yellow(` ⢠${name}`)));
|
|
49
34
|
process.exit(1);
|
|
50
35
|
}
|
|
51
36
|
else {
|
|
52
37
|
// Stop spinner silently (no ā line)
|
|
53
38
|
spinner.stop();
|
|
54
|
-
console.log(
|
|
39
|
+
console.log(chalk.green.bold('\nš All producers are up and running!'));
|
|
55
40
|
}
|
|
56
41
|
}
|
|
57
42
|
catch (err) {
|
|
58
|
-
console.error(
|
|
43
|
+
console.error(chalk.red.bold('\nā Unexpected error during debug:'), err instanceof Error ? err.message : String(err));
|
|
59
44
|
process.exit(1);
|
|
60
45
|
}
|
|
61
|
-
}
|
|
62
|
-
exports.debug = debug;
|
|
46
|
+
};
|
package/actions/deploy.js
CHANGED
|
@@ -1,67 +1,52 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.deploy = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const fs_1 = __importDefault(require("fs"));
|
|
18
|
-
const adm_zip_1 = __importDefault(require("adm-zip"));
|
|
19
|
-
const path_1 = __importDefault(require("path"));
|
|
20
|
-
const Constants_1 = __importDefault(require("../Constants"));
|
|
21
|
-
const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
|
|
22
|
-
const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
|
-
console.log(chalk_1.default.blue.bold(`š Deploying to ${options.env}...`));
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import AdmZip from 'adm-zip';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { CONSTANTS } from '@remora/app-constants';
|
|
6
|
+
import { ProcessENVManager } from '@remora/app-common';
|
|
7
|
+
export const deploy = async (options) => {
|
|
8
|
+
console.log(chalk.blue.bold(`š Deploying to ${options.env}...`));
|
|
24
9
|
try {
|
|
25
10
|
const rootDir = './remora';
|
|
26
|
-
if (!
|
|
27
|
-
throw new Error(
|
|
28
|
-
const zip = new
|
|
11
|
+
if (!fs.existsSync('./remora'))
|
|
12
|
+
throw new Error(chalk.red('Missing directory: ') + chalk.yellow('./remora'));
|
|
13
|
+
const zip = new AdmZip();
|
|
29
14
|
// Function to recursively add files to zip
|
|
30
15
|
const addDirectoryToZip = (directoryPath, zipPath = '') => {
|
|
31
|
-
const files =
|
|
16
|
+
const files = fs.readdirSync(directoryPath);
|
|
32
17
|
files.forEach(file => {
|
|
33
|
-
const fullPath =
|
|
34
|
-
const stat =
|
|
18
|
+
const fullPath = path.join(directoryPath, file);
|
|
19
|
+
const stat = fs.statSync(fullPath);
|
|
35
20
|
if (file === 'temp')
|
|
36
21
|
return;
|
|
37
22
|
if (stat.isDirectory()) {
|
|
38
|
-
addDirectoryToZip(fullPath,
|
|
23
|
+
addDirectoryToZip(fullPath, path.join(zipPath, file));
|
|
39
24
|
}
|
|
40
25
|
else {
|
|
41
|
-
const fileContent =
|
|
42
|
-
zip.addFile(
|
|
26
|
+
const fileContent = fs.readFileSync(fullPath);
|
|
27
|
+
zip.addFile(path.join(zipPath, file), fileContent);
|
|
43
28
|
}
|
|
44
29
|
});
|
|
45
30
|
};
|
|
46
31
|
addDirectoryToZip(rootDir);
|
|
47
32
|
// Write the zip file to a temporary location
|
|
48
|
-
const tempZipPath =
|
|
33
|
+
const tempZipPath = path.join(process.cwd(), 'temp_deployment.zip');
|
|
49
34
|
zip.writeZipPromise(tempZipPath)
|
|
50
|
-
.then(() =>
|
|
35
|
+
.then(async () => {
|
|
51
36
|
try {
|
|
52
37
|
// Read the zip file as a buffer
|
|
53
|
-
const zipBuffer =
|
|
54
|
-
const host =
|
|
55
|
-
const version =
|
|
38
|
+
const zipBuffer = fs.readFileSync(tempZipPath);
|
|
39
|
+
const host = ProcessENVManager.getEnvVariable('REMORA_WORKER_HOST');
|
|
40
|
+
const version = CONSTANTS.workerVersion;
|
|
56
41
|
const workerAPI = `${host}/cli/v${version}/uploaddeployment`;
|
|
57
42
|
const formData = new FormData();
|
|
58
43
|
// @ts-ignore
|
|
59
44
|
const blob = new Blob([zipBuffer], { type: 'application/zip' });
|
|
60
45
|
formData.append('remora_config', blob, 'temp_deployment.zip'); // Updated to match the actual file name
|
|
61
|
-
const apiKey =
|
|
46
|
+
const apiKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
|
|
62
47
|
if (!apiKey)
|
|
63
48
|
throw new Error('REMORA_LICENCE_KEY environment variable is not set');
|
|
64
|
-
const response =
|
|
49
|
+
const response = await fetch(workerAPI, {
|
|
65
50
|
method: 'POST',
|
|
66
51
|
headers: {
|
|
67
52
|
Authorization: `Bearer ${apiKey}`
|
|
@@ -69,15 +54,15 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
69
54
|
body: formData
|
|
70
55
|
});
|
|
71
56
|
if (!response.ok) {
|
|
72
|
-
throw new Error(`Upload failed with status ${response.status}: ${
|
|
57
|
+
throw new Error(`Upload failed with status ${response.status}: ${await response.text()}`);
|
|
73
58
|
}
|
|
74
|
-
console.log(
|
|
59
|
+
console.log(chalk.green('ā
Deployment package uploaded successfully'));
|
|
75
60
|
}
|
|
76
61
|
finally {
|
|
77
62
|
// Clean up the temporary zip file
|
|
78
|
-
|
|
63
|
+
fs.unlinkSync(tempZipPath);
|
|
79
64
|
}
|
|
80
|
-
})
|
|
65
|
+
})
|
|
81
66
|
.catch(error => {
|
|
82
67
|
throw error;
|
|
83
68
|
});
|
|
@@ -89,8 +74,7 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
89
74
|
// create consumers tath require creation ( view, materialized view )
|
|
90
75
|
}
|
|
91
76
|
catch (error) {
|
|
92
|
-
console.error(
|
|
77
|
+
console.error(chalk.red('ā Deployment error:'), error instanceof Error ? error.message : String(error));
|
|
93
78
|
process.exit(1);
|
|
94
79
|
}
|
|
95
|
-
}
|
|
96
|
-
exports.deploy = deploy;
|
|
80
|
+
};
|
package/actions/discover.js
CHANGED
|
@@ -1,36 +1,20 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.discover = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const ora_1 = __importDefault(require("ora"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const compile_1 = require("./compile");
|
|
20
|
-
const DeveloperEngine_1 = __importDefault(require("../engines/ai/DeveloperEngine"));
|
|
21
|
-
const discover = (producerName) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { Environment } from '@remora/app-common';
|
|
4
|
+
import { compile } from './compile';
|
|
5
|
+
import { DeveloperEngine } from '@remora/app-engines';
|
|
6
|
+
export const discover = async (producerName) => {
|
|
22
7
|
try {
|
|
23
|
-
|
|
24
|
-
const spinner = (
|
|
25
|
-
const producer =
|
|
26
|
-
|
|
8
|
+
compile();
|
|
9
|
+
const spinner = ora(chalk.blue('Discovering producer...\n')).start();
|
|
10
|
+
const producer = Environment.getProducer(producerName);
|
|
11
|
+
await DeveloperEngine.discover(producer);
|
|
27
12
|
spinner.succeed('Producer has been successfully discovered');
|
|
28
|
-
console.log(
|
|
13
|
+
console.log(chalk.green('\nā
Discovery complete!'));
|
|
29
14
|
process.exit(1);
|
|
30
15
|
}
|
|
31
16
|
catch (err) {
|
|
32
|
-
console.error(
|
|
17
|
+
console.error(chalk.red.bold('\nā Unexpected error during discover:'), err instanceof Error ? err.message : String(err));
|
|
33
18
|
process.exit(1);
|
|
34
19
|
}
|
|
35
|
-
}
|
|
36
|
-
exports.discover = discover;
|
|
20
|
+
};
|