@forzalabs/remora 1.0.21 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/actions/automap.js +26 -42
  2. package/actions/compile.js +27 -43
  3. package/actions/create_consumer.js +24 -40
  4. package/actions/create_producer.js +16 -32
  5. package/actions/debug.js +18 -34
  6. package/actions/deploy.js +30 -46
  7. package/actions/discover.js +13 -29
  8. package/actions/init.js +29 -45
  9. package/actions/mock.js +16 -32
  10. package/actions/run.js +34 -52
  11. package/actions/sample.js +42 -58
  12. package/index.js +38 -43
  13. package/package.json +4 -4
  14. package/workers/ExecutorWorker.js +18 -32
  15. package/Constants.js +0 -34
  16. package/core/Affirm.js +0 -42
  17. package/core/Algo.js +0 -160
  18. package/core/dste/DSTE.js +0 -113
  19. package/core/logger/DebugLogService.js +0 -48
  20. package/core/logger/DevelopmentLogService.js +0 -70
  21. package/core/logger/LocalLogService.js +0 -70
  22. package/core/logger/Logger.js +0 -54
  23. package/database/DatabaseEngine.js +0 -149
  24. package/database/DatabaseStructure.js +0 -27
  25. package/definitions/DatasetDefinitions.js +0 -2
  26. package/definitions/ExecutorDefinitions.js +0 -2
  27. package/definitions/ProcessENV.js +0 -2
  28. package/definitions/agents/DestinationDriver.js +0 -2
  29. package/definitions/agents/SourceDriver.js +0 -2
  30. package/definitions/cli.js +0 -2
  31. package/definitions/database/ApiKeys.js +0 -2
  32. package/definitions/database/Stored.js +0 -7
  33. package/definitions/database/UsageStat.js +0 -2
  34. package/definitions/database/User.js +0 -2
  35. package/definitions/json_schemas/consumer-schema.json +0 -1226
  36. package/definitions/json_schemas/producer-schema.json +0 -308
  37. package/definitions/json_schemas/project-schema.json +0 -100
  38. package/definitions/json_schemas/source-schema.json +0 -249
  39. package/definitions/requests/ConsumerRequest.js +0 -2
  40. package/definitions/requests/Developer.js +0 -2
  41. package/definitions/requests/Mapping.js +0 -2
  42. package/definitions/requests/ProducerRequest.js +0 -2
  43. package/definitions/requests/Request.js +0 -2
  44. package/definitions/resources/Compiled.js +0 -2
  45. package/definitions/resources/Consumer.js +0 -2
  46. package/definitions/resources/Environment.js +0 -2
  47. package/definitions/resources/Library.js +0 -2
  48. package/definitions/resources/Producer.js +0 -2
  49. package/definitions/resources/Project.js +0 -2
  50. package/definitions/resources/Schema.js +0 -2
  51. package/definitions/resources/Source.js +0 -2
  52. package/definitions/temp.js +0 -2
  53. package/definitions/transform/Transformations.js +0 -2
  54. package/drivers/DeltaShareDriver.js +0 -186
  55. package/drivers/DriverFactory.js +0 -72
  56. package/drivers/DriverHelper.js +0 -248
  57. package/drivers/HttpApiDriver.js +0 -208
  58. package/drivers/RedshiftDriver.js +0 -184
  59. package/drivers/files/LocalDestinationDriver.js +0 -146
  60. package/drivers/files/LocalSourceDriver.js +0 -405
  61. package/drivers/s3/S3DestinationDriver.js +0 -197
  62. package/drivers/s3/S3SourceDriver.js +0 -495
  63. package/engines/CryptoEngine.js +0 -75
  64. package/engines/Environment.js +0 -170
  65. package/engines/ProcessENVManager.js +0 -83
  66. package/engines/RandomEngine.js +0 -47
  67. package/engines/SecretManager.js +0 -23
  68. package/engines/UserManager.js +0 -66
  69. package/engines/ai/AutoMapperEngine.js +0 -37
  70. package/engines/ai/DeveloperEngine.js +0 -497
  71. package/engines/ai/LLM.js +0 -255
  72. package/engines/consumer/ConsumerManager.js +0 -218
  73. package/engines/consumer/ConsumerOnFinishManager.js +0 -202
  74. package/engines/dataset/Dataset.js +0 -824
  75. package/engines/dataset/DatasetManager.js +0 -211
  76. package/engines/dataset/DatasetRecord.js +0 -120
  77. package/engines/dataset/DatasetRecordPool.js +0 -77
  78. package/engines/execution/RequestExecutor.js +0 -67
  79. package/engines/parsing/CSVParser.js +0 -60
  80. package/engines/parsing/LineParser.js +0 -71
  81. package/engines/parsing/ParseCompression.js +0 -101
  82. package/engines/parsing/ParseHelper.js +0 -18
  83. package/engines/parsing/ParseManager.js +0 -54
  84. package/engines/parsing/XLSParser.js +0 -87
  85. package/engines/parsing/XMLParser.js +0 -115
  86. package/engines/producer/ProducerEngine.js +0 -127
  87. package/engines/producer/ProducerManager.js +0 -43
  88. package/engines/scheduler/CronScheduler.js +0 -222
  89. package/engines/scheduler/QueueManager.js +0 -314
  90. package/engines/schema/SchemaValidator.js +0 -67
  91. package/engines/transform/JoinEngine.js +0 -232
  92. package/engines/transform/TransformationEngine.js +0 -277
  93. package/engines/transform/TypeCaster.js +0 -59
  94. package/engines/usage/DataframeManager.js +0 -55
  95. package/engines/usage/UsageDataManager.js +0 -151
  96. package/engines/usage/UsageManager.js +0 -65
  97. package/engines/validation/Validator.js +0 -216
  98. package/executors/ConsumerExecutor.js +0 -280
  99. package/executors/Executor.js +0 -177
  100. package/executors/ExecutorOrchestrator.js +0 -331
  101. package/executors/ExecutorPerformance.js +0 -17
  102. package/executors/ExecutorProgress.js +0 -54
  103. package/executors/ExecutorScope.js +0 -52
  104. package/executors/OutputExecutor.js +0 -118
  105. package/executors/ProducerExecutor.js +0 -108
  106. package/helper/Helper.js +0 -149
  107. package/helper/Logger.js +0 -84
  108. package/helper/Runtime.js +0 -20
  109. package/helper/Settings.js +0 -13
  110. package/licencing/LicenceManager.js +0 -64
  111. package/settings.js +0 -12
package/actions/init.js CHANGED
@@ -1,27 +1,12 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.init = void 0;
16
- const chalk_1 = __importDefault(require("chalk"));
17
- const fs_extra_1 = __importDefault(require("fs-extra"));
18
- const ora_1 = __importDefault(require("ora"));
19
- const path_1 = __importDefault(require("path"));
20
- const init = () => __awaiter(void 0, void 0, void 0, function* () {
1
+ import chalk from 'chalk';
2
+ import fs from 'fs-extra';
3
+ import ora from 'ora';
4
+ import path from 'path';
5
+ export const init = async () => {
21
6
  // TODO init single folder
22
- console.log(chalk_1.default.blue.bold('šŸ“¦ Initializing your application...'));
7
+ console.log(chalk.blue.bold('šŸ“¦ Initializing your application...'));
23
8
  try {
24
- const spinner = (0, ora_1.default)('Creating configuration files...').start();
9
+ const spinner = ora('Creating configuration files...').start();
25
10
  const directories = [
26
11
  'remora',
27
12
  'remora/consumers',
@@ -29,12 +14,12 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
29
14
  'remora/schemas',
30
15
  'remora/sources'
31
16
  ];
32
- const defaultSource = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/source.json`), 'utf-8');
33
- const defaultConsumer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/consumer.json`), 'utf-8');
34
- const defaultProducer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/producer.json`), 'utf-8');
17
+ const defaultSource = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/source.json`), 'utf-8');
18
+ const defaultConsumer = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/consumer.json`), 'utf-8');
19
+ const defaultProducer = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/producer.json`), 'utf-8');
35
20
  // const defaultSchema = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8')
36
- const defaultRemoraProject = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/project.json`), 'utf-8');
37
- const readme = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/README.md`), 'utf-8');
21
+ const defaultRemoraProject = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/project.json`), 'utf-8');
22
+ const readme = fs.readFileSync(path.join(__dirname, `../documentation/README.md`), 'utf-8');
38
23
  const files = [
39
24
  { path: 'remora/sources/.gitkeep', content: '' },
40
25
  { path: 'remora/sources/default-source.json', content: defaultSource },
@@ -48,30 +33,29 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
48
33
  ];
49
34
  for (let i = 0; i < directories.length; i++) {
50
35
  const dir = directories[i];
51
- yield fs_extra_1.default.ensureDir(dir);
36
+ await fs.ensureDir(dir);
52
37
  }
53
38
  for (let i = 0; i < files.length; i++) {
54
39
  const file = files[i];
55
- yield fs_extra_1.default.ensureFile(file.path);
56
- yield fs_extra_1.default.writeFile(file.path, file.content);
40
+ await fs.ensureFile(file.path);
41
+ await fs.writeFile(file.path, file.content);
57
42
  }
58
43
  spinner.succeed('Remora configuration files successfully created.');
59
- console.log(chalk_1.default.green('āœ… Initialization complete!'));
60
- console.log(chalk_1.default.green('šŸ“ A new folder named ') + chalk_1.default.cyan('remora/') + chalk_1.default.green(' has been created in your project.'));
61
- console.log(chalk_1.default.green('āœļø Customize the generated configuration files inside this folder to setup Remora.'));
62
- console.log(chalk_1.default.green('\nšŸ‘£ Next steps: configure your environment variables.'));
63
- console.log(chalk_1.default.green('\nšŸ”§ Local development (add these to your .env file):'));
64
- console.log(chalk_1.default.blue('• REMORA_WORKER_HOST') + chalk_1.default.gray(' → URL of your Remora Worker instance'));
65
- console.log(chalk_1.default.blue('• REMORA_LICENCE_KEY') + chalk_1.default.gray(' → Your valid Remora license key'));
66
- console.log(chalk_1.default.green('\nšŸ” Production environment (store these in your Secret Manager):'));
67
- console.log(chalk_1.default.yellow('• ROOT_TEMP_PASSWORD_FILE') + chalk_1.default.gray(' → Temporary root password for first-time setup'));
68
- console.log(chalk_1.default.yellow('• INSTALLATION_ID') + chalk_1.default.gray(' → Unique ID for this installation instance'));
69
- console.log(chalk_1.default.yellow('• ADMIN_JWT_SECRET_FILE') + chalk_1.default.gray(' → JWT secret for admin-level access'));
70
- console.log(chalk_1.default.yellow('• JWT_SECRET_FILE') + chalk_1.default.gray(' → JWT secret for regular user authentication'));
44
+ console.log(chalk.green('āœ… Initialization complete!'));
45
+ console.log(chalk.green('šŸ“ A new folder named ') + chalk.cyan('remora/') + chalk.green(' has been created in your project.'));
46
+ console.log(chalk.green('āœļø Customize the generated configuration files inside this folder to setup Remora.'));
47
+ console.log(chalk.green('\nšŸ‘£ Next steps: configure your environment variables.'));
48
+ console.log(chalk.green('\nšŸ”§ Local development (add these to your .env file):'));
49
+ console.log(chalk.blue('• REMORA_WORKER_HOST') + chalk.gray(' → URL of your Remora Worker instance'));
50
+ console.log(chalk.blue('• REMORA_LICENCE_KEY') + chalk.gray(' → Your valid Remora license key'));
51
+ console.log(chalk.green('\nšŸ” Production environment (store these in your Secret Manager):'));
52
+ console.log(chalk.yellow('• ROOT_TEMP_PASSWORD') + chalk.gray(' → Temporary root password for first-time setup'));
53
+ console.log(chalk.yellow('• INSTALLATION_ID') + chalk.gray(' → Unique ID for this installation instance'));
54
+ console.log(chalk.yellow('• ADMIN_JWT_SECRET_FILE') + chalk.gray(' → JWT secret for admin-level access'));
55
+ console.log(chalk.yellow('• JWT_SECRET_FILE') + chalk.gray(' → JWT secret for regular user authentication'));
71
56
  }
72
57
  catch (error) {
73
- console.error(chalk_1.default.red('āŒ Error during initialization:'), error instanceof Error ? error.message : String(error));
58
+ console.error(chalk.red('āŒ Error during initialization:'), error instanceof Error ? error.message : String(error));
74
59
  process.exit(1);
75
60
  }
76
- });
77
- exports.init = init;
61
+ };
package/actions/mock.js CHANGED
@@ -1,43 +1,27 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.mock = void 0;
16
- const chalk_1 = __importDefault(require("chalk"));
17
- const ora_1 = __importDefault(require("ora"));
18
- const Environment_1 = __importDefault(require("../engines/Environment"));
19
- const DeveloperEngine_1 = __importDefault(require("../engines/ai/DeveloperEngine"));
20
- const compile_1 = require("./compile");
21
- const mock = (producerName, records) => __awaiter(void 0, void 0, void 0, function* () {
1
+ import chalk from 'chalk';
2
+ import ora from 'ora';
3
+ import { Environment } from '@remora/app-common';
4
+ import { DeveloperEngine } from '@remora/app-engines';
5
+ import { compile } from './compile';
6
+ export const mock = async (producerName, records) => {
22
7
  try {
23
- (0, compile_1.compile)();
8
+ compile();
24
9
  console.log(); // needed for newline
25
- const spinner = (0, ora_1.default)(chalk_1.default.blue(`Generating ${records} mock records for producer "${producerName}"...`)).start();
26
- const producer = Environment_1.default.getProducer(producerName);
10
+ const spinner = ora(chalk.blue(`Generating ${records} mock records for producer "${producerName}"...`)).start();
11
+ const producer = Environment.getProducer(producerName);
27
12
  if (!producer) {
28
- spinner.fail(chalk_1.default.red(`Producer "${producerName}" not found.`));
13
+ spinner.fail(chalk.red(`Producer "${producerName}" not found.`));
29
14
  process.exit(1);
30
15
  }
31
- const result = yield DeveloperEngine_1.default.createMockData(producer, records);
16
+ const result = await DeveloperEngine.createMockData(producer, records);
32
17
  spinner.succeed(`Mock data generated successfully`);
33
- console.log(chalk_1.default.green('\nāœ… Mock data generation complete!'));
34
- console.log(chalk_1.default.blue('šŸ“ File path: ') + chalk_1.default.cyan(result.filePath));
35
- console.log(chalk_1.default.blue('šŸ“Š Records generated: ') + chalk_1.default.cyan(result.recordCount.toString()));
18
+ console.log(chalk.green('\nāœ… Mock data generation complete!'));
19
+ console.log(chalk.blue('šŸ“ File path: ') + chalk.cyan(result.filePath));
20
+ console.log(chalk.blue('šŸ“Š Records generated: ') + chalk.cyan(result.recordCount.toString()));
36
21
  process.exit(0);
37
22
  }
38
23
  catch (err) {
39
- console.error(chalk_1.default.red.bold('\nāŒ Unexpected error during mock data generation:'), err instanceof Error ? err.message : String(err));
24
+ console.error(chalk.red.bold('\nāŒ Unexpected error during mock data generation:'), err instanceof Error ? err.message : String(err));
40
25
  process.exit(1);
41
26
  }
42
- });
43
- exports.mock = mock;
27
+ };
package/actions/run.js CHANGED
@@ -1,62 +1,47 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.run = void 0;
16
- const chalk_1 = __importDefault(require("chalk"));
17
- const Environment_1 = __importDefault(require("../engines/Environment"));
18
- const compile_1 = require("./compile");
19
- const Helper_1 = __importDefault(require("../helper/Helper"));
20
- const LicenceManager_1 = __importDefault(require("../licencing/LicenceManager"));
21
- const ExecutorOrchestrator_1 = __importDefault(require("../executors/ExecutorOrchestrator"));
22
- const ProcessENVManager_1 = __importDefault(require("../engines/ProcessENVManager"));
23
- const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, function* () {
1
+ import chalk from 'chalk';
2
+ import { compile } from './compile';
3
+ import { Environment } from '@remora/app-common';
4
+ import { Helper } from '@remora/app-helper';
5
+ import { LicenceManager } from '@remora/app-auth';
6
+ import { ExecutorOrchestrator } from '@remora/app-executors';
7
+ import { ProcessENVManager } from '@remora/app-common';
8
+ export const run = async (consumerName, options) => {
24
9
  try {
25
- (0, compile_1.compile)();
10
+ compile();
26
11
  // needed for newline
27
12
  console.log();
28
13
  const consumersToExecute = [];
29
14
  if (consumerName && consumerName.length > 0) {
30
- const cons = Environment_1.default.getConsumer(consumerName);
15
+ const cons = Environment.getConsumer(consumerName);
31
16
  if (!cons)
32
17
  throw new Error(`Consumer with name "${consumerName}" was not found.`);
33
18
  consumersToExecute.push(cons);
34
19
  }
35
- else if ((options === null || options === void 0 ? void 0 : options.project) && options.project.length > 0) {
36
- const projectConsumers = Environment_1.default._env.consumers.filter(c => { var _a; return ((_a = c.project) === null || _a === void 0 ? void 0 : _a.toLowerCase()) === options.project.toLowerCase(); });
20
+ else if (options?.project && options.project.length > 0) {
21
+ const projectConsumers = Environment._env.consumers.filter(c => c.project?.toLowerCase() === options.project.toLowerCase());
37
22
  if (projectConsumers.length === 0) {
38
23
  throw new Error(`No consumers found for project "${options.project}".`);
39
24
  }
40
25
  consumersToExecute.push(...projectConsumers);
41
26
  }
42
27
  else {
43
- consumersToExecute.push(...Environment_1.default._env.consumers);
28
+ consumersToExecute.push(...Environment._env.consumers);
44
29
  }
45
30
  if (consumersToExecute.length === 1)
46
- console.log(chalk_1.default.blue(`Running consumer ${consumersToExecute[0].name}...\n`));
31
+ console.log(chalk.blue(`Running consumer ${consumersToExecute[0].name}...\n`));
47
32
  else
48
- console.log(chalk_1.default.blue(`Running consumers ${consumersToExecute.map(x => x.name).join(', ')}...\n`));
33
+ console.log(chalk.blue(`Running consumers ${consumersToExecute.map(x => x.name).join(', ')}...\n`));
49
34
  const results = [];
50
35
  for (let i = 0; i < consumersToExecute.length; i++) {
51
36
  const consumer = consumersToExecute[i];
52
37
  try {
53
- const remoraLicenceKey = ProcessENVManager_1.default.getEnvVariable('REMORA_LICENCE_KEY');
54
- const check = LicenceManager_1.default.validate(remoraLicenceKey);
38
+ const remoraLicenceKey = ProcessENVManager.getEnvVariable('REMORA_LICENCE_KEY');
39
+ const check = LicenceManager.validate(remoraLicenceKey);
55
40
  if (!check.valid) {
56
41
  console.error(`Invalid Remora licence key, the product is not active: remember to set "REMORA_LICENCE_KEY" environment variable.`);
57
42
  process.exit(1);
58
43
  }
59
- const response = yield ExecutorOrchestrator_1.default.launch({
44
+ const response = await ExecutorOrchestrator.launch({
60
45
  consumer,
61
46
  details: {
62
47
  invokedBy: 'CLI',
@@ -67,9 +52,9 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
67
52
  results.push({ success: true, consumer, response });
68
53
  }
69
54
  catch (error) {
70
- const myErr = Helper_1.default.asError(error);
55
+ const myErr = Helper.asError(error);
71
56
  results.push({ success: false, consumer, error: myErr.message });
72
- if (Helper_1.default.isDev())
57
+ if (Helper.isDev())
73
58
  console.log(myErr.stack);
74
59
  }
75
60
  }
@@ -77,40 +62,37 @@ const run = (consumerName, options) => __awaiter(void 0, void 0, void 0, functio
77
62
  if (success) {
78
63
  const { elapsedMS, outputCount, resultUri } = response;
79
64
  const rowCount = outputCount;
80
- const duration = Helper_1.default.formatDuration(elapsedMS);
81
- const performanceInfo = chalk_1.default.gray(` (${rowCount} rows, ${duration})`);
65
+ const duration = Helper.formatDuration(elapsedMS);
66
+ const performanceInfo = chalk.gray(` (${rowCount} rows, ${duration})`);
82
67
  if (resultUri)
83
- console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${resultUri}`) + performanceInfo);
68
+ console.log(chalk.green(`• Consumer ${consumer.name} -> ${resultUri}`) + performanceInfo);
84
69
  else
85
- console.log(chalk_1.default.green(`• Consumer ${consumer.name} `) + performanceInfo);
70
+ console.log(chalk.green(`• Consumer ${consumer.name} `) + performanceInfo);
86
71
  }
87
72
  else {
88
- console.log(chalk_1.default.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
73
+ console.log(chalk.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
89
74
  }
90
75
  });
91
76
  // Calculate totals for successful runs
92
77
  const successfulResults = results.filter(x => x.success);
93
78
  const totalRows = successfulResults.reduce((sum, result) => {
94
- var _a, _b;
95
- return sum + ((_b = (_a = result.response) === null || _a === void 0 ? void 0 : _a.outputCount) !== null && _b !== void 0 ? _b : 0);
79
+ return sum + (result.response?.outputCount ?? 0);
96
80
  }, 0);
97
81
  const totalDuration = successfulResults.reduce((sum, result) => {
98
- var _a;
99
- return sum + (((_a = result.response) === null || _a === void 0 ? void 0 : _a.elapsedMS) || 0);
82
+ return sum + (result.response?.elapsedMS || 0);
100
83
  }, 0);
101
- const totalsInfo = chalk_1.default.gray(` (${totalRows} rows, ${Helper_1.default.formatDuration(totalDuration)})`);
84
+ const totalsInfo = chalk.gray(` (${totalRows} rows, ${Helper.formatDuration(totalDuration)})`);
102
85
  if (results.some(x => !x.success))
103
- console.log(chalk_1.default.blueBright('\nā„¹ļø Run completed with errors') + totalsInfo);
86
+ console.log(chalk.blueBright('\nā„¹ļø Run completed with errors') + totalsInfo);
104
87
  else
105
- console.log(chalk_1.default.green('\nāœ… Run complete!') + totalsInfo);
88
+ console.log(chalk.green('\nāœ… Run complete!') + totalsInfo);
106
89
  process.exit(1);
107
90
  }
108
91
  catch (err) {
109
- const myErr = Helper_1.default.asError(err);
110
- console.error(chalk_1.default.red.bold('\nāŒ Unexpected error during run:'), myErr.message);
111
- if (Helper_1.default.isDev())
92
+ const myErr = Helper.asError(err);
93
+ console.error(chalk.red.bold('\nāŒ Unexpected error during run:'), myErr.message);
94
+ if (Helper.isDev())
112
95
  console.log(myErr.stack);
113
96
  process.exit(1);
114
97
  }
115
- });
116
- exports.run = run;
98
+ };
package/actions/sample.js CHANGED
@@ -1,88 +1,72 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.sample = void 0;
16
- const chalk_1 = __importDefault(require("chalk"));
17
- const ora_1 = __importDefault(require("ora"));
18
- const Environment_1 = __importDefault(require("../engines/Environment"));
19
- const ProducerEngine_1 = __importDefault(require("../engines/producer/ProducerEngine"));
20
- const DatasetRecord_1 = __importDefault(require("../engines/dataset/DatasetRecord"));
21
- const compile_1 = require("./compile");
22
- const Helper_1 = __importDefault(require("../helper/Helper"));
23
- const sample = (resourceName_1, ...args_1) => __awaiter(void 0, [resourceName_1, ...args_1], void 0, function* (resourceName, sampleSize = 10) {
1
+ import chalk from 'chalk';
2
+ import ora from 'ora';
3
+ import { Environment } from '@remora/app-common';
4
+ import { ProducerEngine } from '@remora/app-engines';
5
+ import { DatasetRecord } from '@remora/app-engines';
6
+ import { compile } from './compile';
7
+ import { Helper } from '@remora/app-helper';
8
+ export const sample = async (resourceName, sampleSize = 10) => {
24
9
  try {
25
- (0, compile_1.compile)();
10
+ compile();
26
11
  console.log(); // needed for newline
27
- const spinner = (0, ora_1.default)(chalk_1.default.blue('Sampling dataset...')).start();
12
+ const spinner = ora(chalk.blue('Sampling dataset...')).start();
28
13
  // Try to find the resource as a producer first, then as a consumer
29
- const producer = Environment_1.default.getProducer(resourceName);
30
- const consumer = Environment_1.default.getConsumer(resourceName);
14
+ const producer = Environment.getProducer(resourceName);
15
+ const consumer = Environment.getConsumer(resourceName);
31
16
  if (!producer && !consumer) {
32
- spinner.fail(chalk_1.default.red(`Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`));
17
+ spinner.fail(chalk.red(`Resource "${resourceName}" not found. Please check if it exists as a producer or consumer.`));
33
18
  process.exit(1);
34
19
  }
35
20
  let sampleData;
36
21
  let resourceType;
37
22
  if (producer) {
38
23
  resourceType = 'Producer';
39
- spinner.text = chalk_1.default.blue(`Sampling from producer "${resourceName}"...`);
40
- sampleData = yield ProducerEngine_1.default.readSampleData(producer, sampleSize, false);
24
+ spinner.text = chalk.blue(`Sampling from producer "${resourceName}"...`);
25
+ sampleData = await ProducerEngine.readSampleData(producer, sampleSize, false);
41
26
  }
42
27
  else {
43
28
  resourceType = 'Consumer';
44
- spinner.text = chalk_1.default.blue(`Sampling from consumer "${resourceName}"...`);
45
- sampleData = yield sampleFromConsumer(consumer, sampleSize);
29
+ spinner.text = chalk.blue(`Sampling from consumer "${resourceName}"...`);
30
+ sampleData = await sampleFromConsumer(consumer, sampleSize);
46
31
  }
47
- spinner.succeed(chalk_1.default.green(`Sample data retrieved from ${resourceType.toLowerCase()} "${resourceName}"`));
32
+ spinner.succeed(chalk.green(`Sample data retrieved from ${resourceType.toLowerCase()} "${resourceName}"`));
48
33
  if (sampleData.length === 0) {
49
- console.log(chalk_1.default.yellow('No data found in the dataset.'));
34
+ console.log(chalk.yellow('No data found in the dataset.'));
50
35
  return;
51
36
  }
52
37
  // Display the sample data
53
- console.log(chalk_1.default.cyan(`\nšŸ“Š Sample Data (showing ${sampleData.length} rows):`));
54
- console.log(chalk_1.default.gray('─'.repeat(80)));
38
+ console.log(chalk.cyan(`\nšŸ“Š Sample Data (showing ${sampleData.length} rows):`));
39
+ console.log(chalk.gray('─'.repeat(80)));
55
40
  displayDataAsTable(sampleData);
56
- console.log(chalk_1.default.gray('─'.repeat(80)));
57
- console.log(chalk_1.default.green(`āœ… Successfully sampled ${sampleData.length} rows from ${resourceType.toLowerCase()} "${resourceName}"`));
41
+ console.log(chalk.gray('─'.repeat(80)));
42
+ console.log(chalk.green(`āœ… Successfully sampled ${sampleData.length} rows from ${resourceType.toLowerCase()} "${resourceName}"`));
58
43
  }
59
44
  catch (err) {
60
- const myErr = Helper_1.default.asError(err);
61
- console.error(chalk_1.default.red.bold('\nāŒ Error during sampling:'), myErr.message);
62
- if (Helper_1.default.isDev())
45
+ const myErr = Helper.asError(err);
46
+ console.error(chalk.red.bold('\nāŒ Error during sampling:'), myErr.message);
47
+ if (Helper.isDev())
63
48
  console.log(myErr.stack);
64
49
  process.exit(1);
65
50
  }
66
- });
67
- exports.sample = sample;
68
- const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, void 0, function* () {
51
+ };
52
+ const sampleFromConsumer = async (consumer, sampleSize) => {
69
53
  // For consumers, we need to get sample data from the first producer
70
54
  // and then apply the consumer's field mappings to show what the output would look like
71
55
  const firstProducerRef = consumer.producers[0];
72
56
  if (!firstProducerRef) {
73
57
  throw new Error(`Consumer "${consumer.name}" has no producers configured`);
74
58
  }
75
- const producer = Environment_1.default.getProducer(firstProducerRef.name);
59
+ const producer = Environment.getProducer(firstProducerRef.name);
76
60
  if (!producer) {
77
- const subConsumer = Environment_1.default.getConsumer(firstProducerRef.name);
61
+ const subConsumer = Environment.getConsumer(firstProducerRef.name);
78
62
  if (!subConsumer) {
79
63
  throw new Error(`Producer or consumer "${firstProducerRef.name}" not found for consumer "${consumer.name}"`);
80
64
  }
81
65
  // If it's a consumer that references another consumer, sample from that consumer
82
- return yield sampleFromConsumer(subConsumer, sampleSize);
66
+ return await sampleFromConsumer(subConsumer, sampleSize);
83
67
  }
84
68
  // Get raw sample data from the producer
85
- const rawSampleData = yield ProducerEngine_1.default.readSampleData(producer, sampleSize, false);
69
+ const rawSampleData = await ProducerEngine.readSampleData(producer, sampleSize, false);
86
70
  // For consumers with wildcard fields ("*"), return all data as-is
87
71
  const hasWildcard = consumer.fields.some(field => field.key === '*');
88
72
  if (hasWildcard) {
@@ -91,7 +75,7 @@ const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, v
91
75
  // For consumers with specific field mappings, show only the mapped fields
92
76
  // This gives users a preview of what the consumer output would look like
93
77
  const mappedData = rawSampleData.map(record => {
94
- const mappedRecord = new DatasetRecord_1.default('', [], record._delimiter);
78
+ const mappedRecord = new DatasetRecord('', [], record._delimiter);
95
79
  consumer.fields.forEach(field => {
96
80
  if (field.key !== '*') {
97
81
  const sourceValue = record.getValue(field.key);
@@ -102,7 +86,7 @@ const sampleFromConsumer = (consumer, sampleSize) => __awaiter(void 0, void 0, v
102
86
  return mappedRecord;
103
87
  });
104
88
  return mappedData;
105
- });
89
+ };
106
90
  const displayDataAsTable = (data) => {
107
91
  if (data.length === 0)
108
92
  return;
@@ -132,7 +116,7 @@ const displayDataAsTable = (data) => {
132
116
  columnWidths[field] = Math.min(columnWidths[field], maxColumnWidth);
133
117
  });
134
118
  // Print header
135
- const headerRow = fields.map(field => chalk_1.default.bold(field.padEnd(columnWidths[field]))).join(' │ ');
119
+ const headerRow = fields.map(field => chalk.bold(field.padEnd(columnWidths[field]))).join(' │ ');
136
120
  console.log('│ ' + headerRow + ' │');
137
121
  // Print separator
138
122
  const separator = fields.map(field => '─'.repeat(columnWidths[field])).join('─┼─');
@@ -149,28 +133,28 @@ const displayDataAsTable = (data) => {
149
133
  }).join(' │ ');
150
134
  // Alternate row colors for better readability
151
135
  if (index % 2 === 0) {
152
- console.log('│ ' + chalk_1.default.white(dataRow) + ' │');
136
+ console.log('│ ' + chalk.white(dataRow) + ' │');
153
137
  }
154
138
  else {
155
- console.log('│ ' + chalk_1.default.gray(dataRow) + ' │');
139
+ console.log('│ ' + chalk.gray(dataRow) + ' │');
156
140
  }
157
141
  });
158
142
  };
159
143
  const formatValue = (value) => {
160
144
  if (value === null || value === undefined) {
161
- return chalk_1.default.dim('null');
145
+ return chalk.dim('null');
162
146
  }
163
147
  if (typeof value === 'string') {
164
148
  return value;
165
149
  }
166
150
  if (typeof value === 'number') {
167
- return chalk_1.default.cyan(value.toString());
151
+ return chalk.cyan(value.toString());
168
152
  }
169
153
  if (typeof value === 'boolean') {
170
- return chalk_1.default.yellow(value.toString());
154
+ return chalk.yellow(value.toString());
171
155
  }
172
156
  if (value instanceof Date) {
173
- return chalk_1.default.magenta(value.toISOString());
157
+ return chalk.magenta(value.toISOString());
174
158
  }
175
- return chalk_1.default.dim(JSON.stringify(value));
159
+ return chalk.dim(JSON.stringify(value));
176
160
  };