@forzalabs/remora 1.0.21 ā 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/actions/automap.js +26 -42
- package/actions/compile.js +27 -43
- package/actions/create_consumer.js +24 -40
- package/actions/create_producer.js +16 -32
- package/actions/debug.js +18 -34
- package/actions/deploy.js +30 -46
- package/actions/discover.js +13 -29
- package/actions/init.js +29 -45
- package/actions/mock.js +16 -32
- package/actions/run.js +34 -52
- package/actions/sample.js +42 -58
- package/index.js +38 -43
- package/package.json +4 -4
- package/workers/ExecutorWorker.js +18 -32
- package/Constants.js +0 -34
- package/core/Affirm.js +0 -42
- package/core/Algo.js +0 -160
- package/core/dste/DSTE.js +0 -113
- package/core/logger/DebugLogService.js +0 -48
- package/core/logger/DevelopmentLogService.js +0 -70
- package/core/logger/LocalLogService.js +0 -70
- package/core/logger/Logger.js +0 -54
- package/database/DatabaseEngine.js +0 -149
- package/database/DatabaseStructure.js +0 -27
- package/definitions/DatasetDefinitions.js +0 -2
- package/definitions/ExecutorDefinitions.js +0 -2
- package/definitions/ProcessENV.js +0 -2
- package/definitions/agents/DestinationDriver.js +0 -2
- package/definitions/agents/SourceDriver.js +0 -2
- package/definitions/cli.js +0 -2
- package/definitions/database/ApiKeys.js +0 -2
- package/definitions/database/Stored.js +0 -7
- package/definitions/database/UsageStat.js +0 -2
- package/definitions/database/User.js +0 -2
- package/definitions/json_schemas/consumer-schema.json +0 -1226
- package/definitions/json_schemas/producer-schema.json +0 -308
- package/definitions/json_schemas/project-schema.json +0 -100
- package/definitions/json_schemas/source-schema.json +0 -249
- package/definitions/requests/ConsumerRequest.js +0 -2
- package/definitions/requests/Developer.js +0 -2
- package/definitions/requests/Mapping.js +0 -2
- package/definitions/requests/ProducerRequest.js +0 -2
- package/definitions/requests/Request.js +0 -2
- package/definitions/resources/Compiled.js +0 -2
- package/definitions/resources/Consumer.js +0 -2
- package/definitions/resources/Environment.js +0 -2
- package/definitions/resources/Library.js +0 -2
- package/definitions/resources/Producer.js +0 -2
- package/definitions/resources/Project.js +0 -2
- package/definitions/resources/Schema.js +0 -2
- package/definitions/resources/Source.js +0 -2
- package/definitions/temp.js +0 -2
- package/definitions/transform/Transformations.js +0 -2
- package/drivers/DeltaShareDriver.js +0 -186
- package/drivers/DriverFactory.js +0 -72
- package/drivers/DriverHelper.js +0 -248
- package/drivers/HttpApiDriver.js +0 -208
- package/drivers/RedshiftDriver.js +0 -184
- package/drivers/files/LocalDestinationDriver.js +0 -146
- package/drivers/files/LocalSourceDriver.js +0 -405
- package/drivers/s3/S3DestinationDriver.js +0 -197
- package/drivers/s3/S3SourceDriver.js +0 -495
- package/engines/CryptoEngine.js +0 -75
- package/engines/Environment.js +0 -170
- package/engines/ProcessENVManager.js +0 -83
- package/engines/RandomEngine.js +0 -47
- package/engines/SecretManager.js +0 -23
- package/engines/UserManager.js +0 -66
- package/engines/ai/AutoMapperEngine.js +0 -37
- package/engines/ai/DeveloperEngine.js +0 -497
- package/engines/ai/LLM.js +0 -255
- package/engines/consumer/ConsumerManager.js +0 -218
- package/engines/consumer/ConsumerOnFinishManager.js +0 -202
- package/engines/dataset/Dataset.js +0 -824
- package/engines/dataset/DatasetManager.js +0 -211
- package/engines/dataset/DatasetRecord.js +0 -120
- package/engines/dataset/DatasetRecordPool.js +0 -77
- package/engines/execution/RequestExecutor.js +0 -67
- package/engines/parsing/CSVParser.js +0 -60
- package/engines/parsing/LineParser.js +0 -71
- package/engines/parsing/ParseCompression.js +0 -101
- package/engines/parsing/ParseHelper.js +0 -18
- package/engines/parsing/ParseManager.js +0 -54
- package/engines/parsing/XLSParser.js +0 -87
- package/engines/parsing/XMLParser.js +0 -115
- package/engines/producer/ProducerEngine.js +0 -127
- package/engines/producer/ProducerManager.js +0 -43
- package/engines/scheduler/CronScheduler.js +0 -222
- package/engines/scheduler/QueueManager.js +0 -314
- package/engines/schema/SchemaValidator.js +0 -67
- package/engines/transform/JoinEngine.js +0 -232
- package/engines/transform/TransformationEngine.js +0 -277
- package/engines/transform/TypeCaster.js +0 -59
- package/engines/usage/DataframeManager.js +0 -55
- package/engines/usage/UsageDataManager.js +0 -151
- package/engines/usage/UsageManager.js +0 -65
- package/engines/validation/Validator.js +0 -216
- package/executors/ConsumerExecutor.js +0 -280
- package/executors/Executor.js +0 -177
- package/executors/ExecutorOrchestrator.js +0 -331
- package/executors/ExecutorPerformance.js +0 -17
- package/executors/ExecutorProgress.js +0 -54
- package/executors/ExecutorScope.js +0 -52
- package/executors/OutputExecutor.js +0 -118
- package/executors/ProducerExecutor.js +0 -108
- package/helper/Helper.js +0 -149
- package/helper/Logger.js +0 -84
- package/helper/Runtime.js +0 -20
- package/helper/Settings.js +0 -13
- package/licencing/LicenceManager.js +0 -64
- package/settings.js +0 -12
package/actions/init.js
CHANGED
|
@@ -1,27 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.init = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
18
|
-
const ora_1 = __importDefault(require("ora"));
|
|
19
|
-
const path_1 = __importDefault(require("path"));
|
|
20
|
-
const init = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import fs from 'fs-extra';
|
|
3
|
+
import ora from 'ora';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
export const init = async () => {
|
|
21
6
|
// TODO init single folder
|
|
22
|
-
console.log(
|
|
7
|
+
console.log(chalk.blue.bold('š¦ Initializing your application...'));
|
|
23
8
|
try {
|
|
24
|
-
const spinner = (
|
|
9
|
+
const spinner = ora('Creating configuration files...').start();
|
|
25
10
|
const directories = [
|
|
26
11
|
'remora',
|
|
27
12
|
'remora/consumers',
|
|
@@ -29,12 +14,12 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
29
14
|
'remora/schemas',
|
|
30
15
|
'remora/sources'
|
|
31
16
|
];
|
|
32
|
-
const defaultSource =
|
|
33
|
-
const defaultConsumer =
|
|
34
|
-
const defaultProducer =
|
|
17
|
+
const defaultSource = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/source.json`), 'utf-8');
|
|
18
|
+
const defaultConsumer = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/consumer.json`), 'utf-8');
|
|
19
|
+
const defaultProducer = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/producer.json`), 'utf-8');
|
|
35
20
|
// const defaultSchema = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8')
|
|
36
|
-
const defaultRemoraProject =
|
|
37
|
-
const readme =
|
|
21
|
+
const defaultRemoraProject = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/project.json`), 'utf-8');
|
|
22
|
+
const readme = fs.readFileSync(path.join(__dirname, `../documentation/README.md`), 'utf-8');
|
|
38
23
|
const files = [
|
|
39
24
|
{ path: 'remora/sources/.gitkeep', content: '' },
|
|
40
25
|
{ path: 'remora/sources/default-source.json', content: defaultSource },
|
|
@@ -48,30 +33,29 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
48
33
|
];
|
|
49
34
|
for (let i = 0; i < directories.length; i++) {
|
|
50
35
|
const dir = directories[i];
|
|
51
|
-
|
|
36
|
+
await fs.ensureDir(dir);
|
|
52
37
|
}
|
|
53
38
|
for (let i = 0; i < files.length; i++) {
|
|
54
39
|
const file = files[i];
|
|
55
|
-
|
|
56
|
-
|
|
40
|
+
await fs.ensureFile(file.path);
|
|
41
|
+
await fs.writeFile(file.path, file.content);
|
|
57
42
|
}
|
|
58
43
|
spinner.succeed('Remora configuration files successfully created.');
|
|
59
|
-
console.log(
|
|
60
|
-
console.log(
|
|
61
|
-
console.log(
|
|
62
|
-
console.log(
|
|
63
|
-
console.log(
|
|
64
|
-
console.log(
|
|
65
|
-
console.log(
|
|
66
|
-
console.log(
|
|
67
|
-
console.log(
|
|
68
|
-
console.log(
|
|
69
|
-
console.log(
|
|
70
|
-
console.log(
|
|
44
|
+
console.log(chalk.green('ā
Initialization complete!'));
|
|
45
|
+
console.log(chalk.green('š A new folder named ') + chalk.cyan('remora/') + chalk.green(' has been created in your project.'));
|
|
46
|
+
console.log(chalk.green('āļø Customize the generated configuration files inside this folder to setup Remora.'));
|
|
47
|
+
console.log(chalk.green('\nš£ Next steps: configure your environment variables.'));
|
|
48
|
+
console.log(chalk.green('\nš§ Local development (add these to your .env file):'));
|
|
49
|
+
console.log(chalk.blue('⢠REMORA_WORKER_HOST') + chalk.gray(' ā URL of your Remora Worker instance'));
|
|
50
|
+
console.log(chalk.blue('⢠REMORA_LICENCE_KEY') + chalk.gray(' ā Your valid Remora license key'));
|
|
51
|
+
console.log(chalk.green('\nš Production environment (store these in your Secret Manager):'));
|
|
52
|
+
console.log(chalk.yellow('⢠ROOT_TEMP_PASSWORD') + chalk.gray(' ā Temporary root password for first-time setup'));
|
|
53
|
+
console.log(chalk.yellow('⢠INSTALLATION_ID') + chalk.gray(' ā Unique ID for this installation instance'));
|
|
54
|
+
console.log(chalk.yellow('⢠ADMIN_JWT_SECRET_FILE') + chalk.gray(' ā JWT secret for admin-level access'));
|
|
55
|
+
console.log(chalk.yellow('⢠JWT_SECRET_FILE') + chalk.gray(' ā JWT secret for regular user authentication'));
|
|
71
56
|
}
|
|
72
57
|
catch (error) {
|
|
73
|
-
console.error(
|
|
58
|
+
console.error(chalk.red('ā Error during initialization:'), error instanceof Error ? error.message : String(error));
|
|
74
59
|
process.exit(1);
|
|
75
60
|
}
|
|
76
|
-
}
|
|
77
|
-
exports.init = init;
|
|
61
|
+
};
|
package/actions/mock.js
CHANGED
|
@@ -1,43 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.mock = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const ora_1 = __importDefault(require("ora"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const DeveloperEngine_1 = __importDefault(require("../engines/ai/DeveloperEngine"));
|
|
20
|
-
const compile_1 = require("./compile");
|
|
21
|
-
const mock = (producerName, records) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { Environment } from '@remora/app-common';
|
|
4
|
+
import { DeveloperEngine } from '@remora/app-engines';
|
|
5
|
+
import { compile } from './compile';
|
|
6
|
+
export const mock = async (producerName, records) => {
|
|
22
7
|
try {
|
|
23
|
-
|
|
8
|
+
compile();
|
|
24
9
|
console.log(); // needed for newline
|
|
25
|
-
const spinner = (
|
|
26
|
-
const producer =
|
|
10
|
+
const spinner = ora(chalk.blue(`Generating ${records} mock records for producer "${producerName}"...`)).start();
|
|
11
|
+
const producer = Environment.getProducer(producerName);
|
|
27
12
|
if (!producer) {
|
|
28
|
-
spinner.fail(
|
|
13
|
+
spinner.fail(chalk.red(`Producer "${producerName}" not found.`));
|
|
29
14
|
process.exit(1);
|
|
30
15
|
}
|
|
31
|
-
const result =
|
|
16
|
+
const result = await DeveloperEngine.createMockData(producer, records);
|
|
32
17
|
spinner.succeed(`Mock data generated successfully`);
|
|
33
|
-
console.log(
|
|
34
|
-
console.log(
|
|
35
|
-
console.log(
|
|
18
|
+
console.log(chalk.green('\nā
Mock data generation complete!'));
|
|
19
|
+
console.log(chalk.blue('š File path: ') + chalk.cyan(result.filePath));
|
|
20
|
+
console.log(chalk.blue('š Records generated: ') + chalk.cyan(result.recordCount.toString()));
|
|
36
21
|
process.exit(0);
|
|
37
22
|
}
|
|
38
23
|
catch (err) {
|
|
39
|
-
console.error(
|
|
24
|
+
console.error(chalk.red.bold('\nā Unexpected error during mock data generation:'), err instanceof Error ? err.message : String(err));
|
|
40
25
|
process.exit(1);
|
|
41
26
|
}
|
|
42
|
-
}
|
|
43
|
-
exports.mock = mock;
|
|
27
|
+
};
|
package/actions/run.js
CHANGED
|
@@ -1,62 +1,47 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.run = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
18
|
-
const compile_1 = require("./compile");
|
|
19
|
-
const Helper_1 = __importDefault(require("../helper/Helper"));
|
|
20
|
-
const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
|
|
21
|
-
const ExecutorOrchestrator_1 = __importDefault(require("../executors/ExecutorOrchestrator"));
|
|
22
|
-
const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
|
|
23
|
-
const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { compile } from './compile';
|
|
3
|
+
import { Environment } from '@remora/app-common';
|
|
4
|
+
import { Helper } from '@remora/app-helper';
|
|
5
|
+
import { LicenceManager } from '@remora/app-auth';
|
|
6
|
+
import { ExecutorOrchestrator } from '@remora/app-executors';
|
|
7
|
+
import { ProcessENVManager } from '@remora/app-common';
|
|
8
|
+
export const run = async (consumerName, options) => {
|
|
24
9
|
try {
|
|
25
|
-
|
|
10
|
+
compile();
|
|
26
11
|
// needed for newline
|
|
27
12
|
console.log();
|
|
28
13
|
const consumersToExecute = [];
|
|
29
14
|
if (consumerName && consumerName.length > 0) {
|
|
30
|
-
const cons =
|
|
15
|
+
const cons = Environment.getConsumer(consumerName);
|
|
31
16
|
if (!cons)
|
|
32
17
|
throw new Error(`Consumer with name "${consumerName}" was not found.`);
|
|
33
18
|
consumersToExecute.push(cons);
|
|
34
19
|
}
|
|
35
|
-
else if (
|
|
36
|
-
const projectConsumers =
|
|
20
|
+
else if (options?.project && options.project.length > 0) {
|
|
21
|
+
const projectConsumers = Environment._env.consumers.filter(c => c.project?.toLowerCase() === options.project.toLowerCase());
|
|
37
22
|
if (projectConsumers.length === 0) {
|
|
38
23
|
throw new Error(`No consumers found for project "${options.project}".`);
|
|
39
24
|
}
|
|
40
25
|
consumersToExecute.push(...projectConsumers);
|
|
41
26
|
}
|
|
42
27
|
else {
|
|
43
|
-
consumersToExecute.push(...
|
|
28
|
+
consumersToExecute.push(...Environment._env.consumers);
|
|
44
29
|
}
|
|
45
30
|
if (consumersToExecute.length === 1)
|
|
46
|
-
console.log(
|
|
31
|
+
console.log(chalk.blue(`Running consumer ${consumersToExecute[0].name}...\n`));
|
|
47
32
|
else
|
|
48
|
-
console.log(
|
|
33
|
+
console.log(chalk.blue(`Running consumers ${consumersToExecute.map(x => x.name).join(', ')}...\n`));
|
|
49
34
|
const results = [];
|
|
50
35
|
for (let i = 0; i < consumersToExecute.length; i++) {
|
|
51
36
|
const consumer = consumersToExecute[i];
|
|
52
37
|
try {
|
|
53
|
-
const remoraLicenceKey =
|
|
54
|
-
const check =
|
|
38
|
+
const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
|
|
39
|
+
const check = LicenceManager.validate(remoraLicenceKey);
|
|
55
40
|
if (!check.valid) {
|
|
56
41
|
console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
|
|
57
42
|
process.exit(1);
|
|
58
43
|
}
|
|
59
|
-
const response =
|
|
44
|
+
const response = await ExecutorOrchestrator.launch({
|
|
60
45
|
consumer,
|
|
61
46
|
details: {
|
|
62
47
|
invokedBy: 'CLI',
|
|
@@ -67,9 +52,9 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
|
|
|
67
52
|
results.push({ success: true, consumer, response });
|
|
68
53
|
}
|
|
69
54
|
catch (error) {
|
|
70
|
-
const myErr =
|
|
55
|
+
const myErr = Helper.asError(error);
|
|
71
56
|
results.push({ success: false, consumer, error: myErr.message });
|
|
72
|
-
if (
|
|
57
|
+
if (Helper.isDev())
|
|
73
58
|
console.log(myErr.stack);
|
|
74
59
|
}
|
|
75
60
|
}
|
|
@@ -77,40 +62,37 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
|
|
|
77
62
|
if (success) {
|
|
78
63
|
const { elapsedMS, outputCount, resultUri } = response;
|
|
79
64
|
const rowCount = outputCount;
|
|
80
|
-
const duration =
|
|
81
|
-
const performanceInfo =
|
|
65
|
+
const duration = Helper.formatDuration(elapsedMS);
|
|
66
|
+
const performanceInfo = chalk.gray(` (${rowCount} rows, ${duration})`);
|
|
82
67
|
if (resultUri)
|
|
83
|
-
console.log(
|
|
68
|
+
console.log(chalk.green(`⢠Consumer ${consumer.name} -> ${resultUri}`) + performanceInfo);
|
|
84
69
|
else
|
|
85
|
-
console.log(
|
|
70
|
+
console.log(chalk.green(`⢠Consumer ${consumer.name} `) + performanceInfo);
|
|
86
71
|
}
|
|
87
72
|
else {
|
|
88
|
-
console.log(
|
|
73
|
+
console.log(chalk.red(`⢠Consumer ${consumer.name} -> Failed: ${error}`));
|
|
89
74
|
}
|
|
90
75
|
});
|
|
91
76
|
// Calculate totals for successful runs
|
|
92
77
|
const successfulResults = results.filter(x => x.success);
|
|
93
78
|
const totalRows = successfulResults.reduce((sum, result) => {
|
|
94
|
-
|
|
95
|
-
return sum + ((_b = (_a = result.response) === null || _a === void 0 ? void 0 : _a.outputCount) !== null && _b !== void 0 ? _b : 0);
|
|
79
|
+
return sum + (result.response?.outputCount ?? 0);
|
|
96
80
|
}, 0);
|
|
97
81
|
const totalDuration = successfulResults.reduce((sum, result) => {
|
|
98
|
-
|
|
99
|
-
return sum + (((_a = result.response) === null || _a === void 0 ? void 0 : _a.elapsedMS) || 0);
|
|
82
|
+
return sum + (result.response?.elapsedMS || 0);
|
|
100
83
|
}, 0);
|
|
101
|
-
const totalsInfo =
|
|
84
|
+
const totalsInfo = chalk.gray(` (${totalRows} rows, ${Helper.formatDuration(totalDuration)})`);
|
|
102
85
|
if (results.some(x => !x.success))
|
|
103
|
-
console.log(
|
|
86
|
+
console.log(chalk.blueBright('\nā¹ļø Run completed with errors') + totalsInfo);
|
|
104
87
|
else
|
|
105
|
-
console.log(
|
|
88
|
+
console.log(chalk.green('\nā
Run complete!') + totalsInfo);
|
|
106
89
|
process.exit(1);
|
|
107
90
|
}
|
|
108
91
|
catch (err) {
|
|
109
|
-
const myErr =
|
|
110
|
-
console.error(
|
|
111
|
-
if (
|
|
92
|
+
const myErr = Helper.asError(err);
|
|
93
|
+
console.error(chalk.red.bold('\nā Unexpected error during run:'), myErr.message);
|
|
94
|
+
if (Helper.isDev())
|
|
112
95
|
console.log(myErr.stack);
|
|
113
96
|
process.exit(1);
|
|
114
97
|
}
|
|
115
|
-
}
|
|
116
|
-
exports.run = run;
|
|
98
|
+
};
|
package/actions/sample.js
CHANGED
|
@@ -1,88 +1,72 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
-
};
|
|
14
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
-
exports.sample = void 0;
|
|
16
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
17
|
-
const ora_1 = __importDefault(require("ora"));
|
|
18
|
-
const Environment_1 = __importDefault(require("../engines/Environment"));
|
|
19
|
-
const ProducerEngine_1 = __importDefault(require("../engines/producer/ProducerEngine"));
|
|
20
|
-
const DatasetRecord_1 = __importDefault(require("../engines/dataset/DatasetRecord"));
|
|
21
|
-
const compile_1 = require("./compile");
|
|
22
|
-
const Helper_1 = __importDefault(require("../helper/Helper"));
|
|
23
|
-
const sample = (resourceName_1, ...args_1) => __awaiter(void 0, [resourceName_1, ...args_1], void 0, function* (resourceName, sampleSize = 10) {
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import ora from 'ora';
|
|
3
|
+
import { Environment } from '@remora/app-common';
|
|
4
|
+
import { ProducerEngine } from '@remora/app-engines';
|
|
5
|
+
import { DatasetRecord } from '@remora/app-engines';
|
|
6
|
+
import { compile } from './compile';
|
|
7
|
+
import { Helper } from '@remora/app-helper';
|
|
8
|
+
export const sample = async (resourceName, sampleSize = 10) => {
|
|
24
9
|
try {
|
|
25
|
-
|
|
10
|
+
compile();
|
|
26
11
|
console.log(); // needed for newline
|
|
27
|
-
const spinner = (
|
|
12
|
+
const spinner = ora(chalk.blue('Sampling dataset...')).start();
|
|
28
13
|
// Try to find the resource as a producer first, then as a consumer
|
|
29
|
-
const producer =
|
|
30
|
-
const consumer =
|
|
14
|
+
const producer = Environment.getProducer(resourceName);
|
|
15
|
+
const consumer = Environment.getConsumer(resourceName);
|
|
31
16
|
if (!producer && !consumer) {
|
|
32
|
-
spinner.fail(
|
|
17
|
+
spinner.fail(chalk.red(`Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`));
|
|
33
18
|
process.exit(1);
|
|
34
19
|
}
|
|
35
20
|
let sampleData;
|
|
36
21
|
let resourceType;
|
|
37
22
|
if (producer) {
|
|
38
23
|
resourceType = 'Producer';
|
|
39
|
-
spinner.text =
|
|
40
|
-
sampleData =
|
|
24
|
+
spinner.text = chalk.blue(`Sampling from producer "${resourceName}"...`);
|
|
25
|
+
sampleData = await ProducerEngine.readSampleData(producer, sampleSize, false);
|
|
41
26
|
}
|
|
42
27
|
else {
|
|
43
28
|
resourceType = 'Consumer';
|
|
44
|
-
spinner.text =
|
|
45
|
-
sampleData =
|
|
29
|
+
spinner.text = chalk.blue(`Sampling from consumer "${resourceName}"...`);
|
|
30
|
+
sampleData = await sampleFromConsumer(consumer, sampleSize);
|
|
46
31
|
}
|
|
47
|
-
spinner.succeed(
|
|
32
|
+
spinner.succeed(chalk.green(`Sample data retrieved from ${resourceType.toLowerCase()} "${resourceName}"`));
|
|
48
33
|
if (sampleData.length === 0) {
|
|
49
|
-
console.log(
|
|
34
|
+
console.log(chalk.yellow('No data found in the dataset.'));
|
|
50
35
|
return;
|
|
51
36
|
}
|
|
52
37
|
// Display the sample data
|
|
53
|
-
console.log(
|
|
54
|
-
console.log(
|
|
38
|
+
console.log(chalk.cyan(`\nš Sample Data (showing ${sampleData.length} rows):`));
|
|
39
|
+
console.log(chalk.gray('ā'.repeat(80)));
|
|
55
40
|
displayDataAsTable(sampleData);
|
|
56
|
-
console.log(
|
|
57
|
-
console.log(
|
|
41
|
+
console.log(chalk.gray('ā'.repeat(80)));
|
|
42
|
+
console.log(chalk.green(`ā
Successfully sampled ${sampleData.length} rows from ${resourceType.toLowerCase()} "${resourceName}"`));
|
|
58
43
|
}
|
|
59
44
|
catch (err) {
|
|
60
|
-
const myErr =
|
|
61
|
-
console.error(
|
|
62
|
-
if (
|
|
45
|
+
const myErr = Helper.asError(err);
|
|
46
|
+
console.error(chalk.red.bold('\nā Error during sampling:'), myErr.message);
|
|
47
|
+
if (Helper.isDev())
|
|
63
48
|
console.log(myErr.stack);
|
|
64
49
|
process.exit(1);
|
|
65
50
|
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, void 0, function* () {
|
|
51
|
+
};
|
|
52
|
+
const sampleFromConsumer = async (consumer, sampleSize) => {
|
|
69
53
|
// For consumers, we need to get sample data from the first producer
|
|
70
54
|
// and then apply the consumer's field mappings to show what the output would look like
|
|
71
55
|
const firstProducerRef = consumer.producers[0];
|
|
72
56
|
if (!firstProducerRef) {
|
|
73
57
|
throw new Error(`Consumer "${consumer.name}" has no producers configured`);
|
|
74
58
|
}
|
|
75
|
-
const producer =
|
|
59
|
+
const producer = Environment.getProducer(firstProducerRef.name);
|
|
76
60
|
if (!producer) {
|
|
77
|
-
const subConsumer =
|
|
61
|
+
const subConsumer = Environment.getConsumer(firstProducerRef.name);
|
|
78
62
|
if (!subConsumer) {
|
|
79
63
|
throw new Error(`Producer or consumer "${firstProducerRef.name}" not found for consumer "${consumer.name}"`);
|
|
80
64
|
}
|
|
81
65
|
// If it's a consumer that references another consumer, sample from that consumer
|
|
82
|
-
return
|
|
66
|
+
return await sampleFromConsumer(subConsumer, sampleSize);
|
|
83
67
|
}
|
|
84
68
|
// Get raw sample data from the producer
|
|
85
|
-
const rawSampleData =
|
|
69
|
+
const rawSampleData = await ProducerEngine.readSampleData(producer, sampleSize, false);
|
|
86
70
|
// For consumers with wildcard fields ("*"), return all data as-is
|
|
87
71
|
const hasWildcard = consumer.fields.some(field => field.key === '*');
|
|
88
72
|
if (hasWildcard) {
|
|
@@ -91,7 +75,7 @@ const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, v
|
|
|
91
75
|
// For consumers with specific field mappings, show only the mapped fields
|
|
92
76
|
// This gives users a preview of what the consumer output would look like
|
|
93
77
|
const mappedData = rawSampleData.map(record => {
|
|
94
|
-
const mappedRecord = new
|
|
78
|
+
const mappedRecord = new DatasetRecord('', [], record._delimiter);
|
|
95
79
|
consumer.fields.forEach(field => {
|
|
96
80
|
if (field.key !== '*') {
|
|
97
81
|
const sourceValue = record.getValue(field.key);
|
|
@@ -102,7 +86,7 @@ const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, v
|
|
|
102
86
|
return mappedRecord;
|
|
103
87
|
});
|
|
104
88
|
return mappedData;
|
|
105
|
-
}
|
|
89
|
+
};
|
|
106
90
|
const displayDataAsTable = (data) => {
|
|
107
91
|
if (data.length === 0)
|
|
108
92
|
return;
|
|
@@ -132,7 +116,7 @@ const displayDataAsTable = (data) => {
|
|
|
132
116
|
columnWidths[field] = Math.min(columnWidths[field], maxColumnWidth);
|
|
133
117
|
});
|
|
134
118
|
// Print header
|
|
135
|
-
const headerRow = fields.map(field =>
|
|
119
|
+
const headerRow = fields.map(field => chalk.bold(field.padEnd(columnWidths[field]))).join(' ā ');
|
|
136
120
|
console.log('ā ' + headerRow + ' ā');
|
|
137
121
|
// Print separator
|
|
138
122
|
const separator = fields.map(field => 'ā'.repeat(columnWidths[field])).join('āā¼ā');
|
|
@@ -149,28 +133,28 @@ const displayDataAsTable = (data) => {
|
|
|
149
133
|
}).join(' ā ');
|
|
150
134
|
// Alternate row colors for better readability
|
|
151
135
|
if (index % 2 === 0) {
|
|
152
|
-
console.log('ā ' +
|
|
136
|
+
console.log('ā ' + chalk.white(dataRow) + ' ā');
|
|
153
137
|
}
|
|
154
138
|
else {
|
|
155
|
-
console.log('ā ' +
|
|
139
|
+
console.log('ā ' + chalk.gray(dataRow) + ' ā');
|
|
156
140
|
}
|
|
157
141
|
});
|
|
158
142
|
};
|
|
159
143
|
const formatValue = (value) => {
|
|
160
144
|
if (value === null || value === undefined) {
|
|
161
|
-
return
|
|
145
|
+
return chalk.dim('null');
|
|
162
146
|
}
|
|
163
147
|
if (typeof value === 'string') {
|
|
164
148
|
return value;
|
|
165
149
|
}
|
|
166
150
|
if (typeof value === 'number') {
|
|
167
|
-
return
|
|
151
|
+
return chalk.cyan(value.toString());
|
|
168
152
|
}
|
|
169
153
|
if (typeof value === 'boolean') {
|
|
170
|
-
return
|
|
154
|
+
return chalk.yellow(value.toString());
|
|
171
155
|
}
|
|
172
156
|
if (value instanceof Date) {
|
|
173
|
-
return
|
|
157
|
+
return chalk.magenta(value.toISOString());
|
|
174
158
|
}
|
|
175
|
-
return
|
|
159
|
+
return chalk.dim(JSON.stringify(value));
|
|
176
160
|
};
|