@forzalabs/remora 0.1.4-nasco.3 → 0.1.5-nasco.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Constants.js +1 -1
- package/definitions/json_schemas/consumer-schema.json +6 -2
- package/definitions/json_schemas/producer-schema.json +2 -1
- package/definitions/json_schemas/source-schema.json +14 -1
- package/documentation/README.md +1 -0
- package/documentation/default_resources/consumer.json +7 -7
- package/drivers/DeltaShareDriver.js +178 -0
- package/drivers/DriverFactory.js +6 -0
- package/drivers/DriverHelper.js +15 -0
- package/engines/ai/DeveloperEngine.js +90 -1
- package/engines/consumer/ConsumerEngine.js +1 -1
- package/engines/consumer/PostProcessor.js +22 -15
- package/engines/dataset/Dataset.js +18 -7
- package/engines/dataset/DatasetManager.js +58 -12
- package/engines/dataset/DatasetRecord.js +17 -4
- package/engines/dataset/ParallelDataset.js +16 -6
- package/engines/execution/ExecutionEnvironment.js +13 -4
- package/engines/execution/ExecutionPlanner.js +2 -1
- package/engines/file/FileCompiler.js +2 -1
- package/engines/file/FileExporter.js +12 -3
- package/engines/parsing/ParseManager.js +7 -2
- package/engines/producer/ProducerEngine.js +4 -2
- package/engines/transform/JoinEngine.js +10 -6
- package/engines/transform/TransformationEngine.js +31 -2
- package/engines/usage/UsageDataManager.js +110 -0
- package/package.json +2 -1
- package/workers/FilterWorker.js +3 -3
- package/workers/ProjectionWorker.js +3 -3
- package/workers/TransformWorker.js +3 -3
package/workers/FilterWorker.js
CHANGED
|
@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
|
|
|
22
22
|
const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
23
|
Environment_1.default.load('./');
|
|
24
24
|
try {
|
|
25
|
-
const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, filterData: filter } = workerData;
|
|
25
|
+
const { datasetName, fromLine, toLine, workerId, executionId, datasetFile, datasetDimensions, datasetDelimiter, filterData: filter } = workerData;
|
|
26
26
|
Affirm_1.default.hasValue(fromLine, `Invalid from line`);
|
|
27
27
|
Affirm_1.default.hasValue(toLine, `Invalid to line`);
|
|
28
28
|
(0, Affirm_1.default)(datasetName, `Invalid dataset name`);
|
|
@@ -31,9 +31,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
31
31
|
(0, Affirm_1.default)(datasetDimensions, `Invalid dataset dimensions`);
|
|
32
32
|
(0, Affirm_1.default)(filter, `Invalid filter data`);
|
|
33
33
|
(0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
|
|
34
|
-
const dataset = new Dataset_1.default(datasetName, datasetFile);
|
|
34
|
+
const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
|
|
35
35
|
dataset
|
|
36
|
-
.
|
|
36
|
+
.setDimensions(datasetDimensions)
|
|
37
37
|
.setDelimiter(datasetDelimiter);
|
|
38
38
|
const outputPath = dataset['_tempPath'] + workerId;
|
|
39
39
|
const rules = filter.rules.map(x => x.rule);
|
|
@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
|
|
|
22
22
|
const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
23
|
Environment_1.default.load('./');
|
|
24
24
|
try {
|
|
25
|
-
const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, projectionData } = workerData;
|
|
25
|
+
const { datasetName, fromLine, toLine, workerId, datasetFile, executionId, datasetDimensions, datasetDelimiter, projectionData } = workerData;
|
|
26
26
|
Affirm_1.default.hasValue(fromLine, `Invalid from line`);
|
|
27
27
|
Affirm_1.default.hasValue(toLine, `Invalid to line`);
|
|
28
28
|
(0, Affirm_1.default)(datasetName, `Invalid dataset name`);
|
|
@@ -33,9 +33,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
33
33
|
(0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
|
|
34
34
|
const consumer = Environment_1.default.getConsumer(projectionData.consumerName);
|
|
35
35
|
(0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${projectionData.consumerName}" not found.`);
|
|
36
|
-
const dataset = new Dataset_1.default(datasetName, datasetFile);
|
|
36
|
+
const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
|
|
37
37
|
dataset
|
|
38
|
-
.
|
|
38
|
+
.setDimensions(datasetDimensions)
|
|
39
39
|
.setDelimiter(datasetDelimiter);
|
|
40
40
|
const outputPath = dataset['_tempPath'] + workerId;
|
|
41
41
|
yield PostProcessor_1.default.doProjection(consumer, dataset, { outputPath, range: { fromLine, toLine } });
|
|
@@ -22,7 +22,7 @@ dotenv_1.default.configDotenv();
|
|
|
22
22
|
const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
23
|
Environment_1.default.load('./');
|
|
24
24
|
try {
|
|
25
|
-
const { datasetName, fromLine, toLine, workerId, datasetFile, datasetDimensions, datasetDelimiter, transformData } = workerData;
|
|
25
|
+
const { datasetName, fromLine, toLine, workerId, executionId, datasetFile, datasetDimensions, datasetDelimiter, transformData } = workerData;
|
|
26
26
|
Affirm_1.default.hasValue(fromLine, `Invalid from line`);
|
|
27
27
|
Affirm_1.default.hasValue(toLine, `Invalid to line`);
|
|
28
28
|
(0, Affirm_1.default)(datasetName, `Invalid dataset name`);
|
|
@@ -33,9 +33,9 @@ const run = (workerData) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
33
33
|
(0, Affirm_1.default)(datasetDelimiter, `Invalid dataset delimter`);
|
|
34
34
|
const consumer = Environment_1.default.getConsumer(transformData.consumerName);
|
|
35
35
|
(0, Affirm_1.default)(consumer, `Wrong consumer name sent to projection worker: "${transformData.consumerName}" not found.`);
|
|
36
|
-
const dataset = new Dataset_1.default(datasetName, datasetFile);
|
|
36
|
+
const dataset = new Dataset_1.default(datasetName, datasetFile, undefined, executionId);
|
|
37
37
|
dataset
|
|
38
|
-
.
|
|
38
|
+
.setDimensions(datasetDimensions)
|
|
39
39
|
.setDelimiter(datasetDelimiter);
|
|
40
40
|
const outputPath = dataset['_tempPath'] + workerId;
|
|
41
41
|
yield TransformationEngine_1.default.apply(consumer, dataset, { outputPath, range: { fromLine, toLine } });
|