@forzalabs/remora 0.0.20 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Constants.js +1 -1
- package/actions/automap.js +5 -1
- package/actions/deploy.js +0 -1
- package/actions/run.js +1 -1
- package/auth/JWTManager.js +1 -1
- package/database/DatabaseEngine.js +13 -3
- package/definitions/json_schemas/consumer-schema.json +392 -0
- package/definitions/json_schemas/producer-schema.json +4 -0
- package/definitions/transform/Transformations.js +2 -0
- package/drivers/LocalDriver.js +28 -8
- package/drivers/RedshiftDriver.js +1 -0
- package/drivers/S3SourceDriver.js +12 -10
- package/engines/DataframeManager.js +55 -0
- package/engines/Environment.js +1 -1
- package/engines/ParseManager.js +25 -8
- package/engines/ProducerEngine.js +1 -1
- package/engines/UsageDataManager.js +110 -0
- package/engines/UserManager.js +2 -2
- package/engines/ai/AutoMapperEngine.js +2 -2
- package/engines/ai/LLM.js +42 -56
- package/engines/consumer/ConsumerManager.js +1 -1
- package/engines/consumer/PostProcessor.js +21 -4
- package/engines/execution/ExecutionEnvironment.js +8 -1
- package/engines/execution/ExecutionPlanner.js +6 -2
- package/engines/file/FileCompiler.js +1 -1
- package/engines/transform/TransformationEngine.js +223 -0
- package/engines/transform/TypeCaster.js +34 -0
- package/engines/validation/Validator.js +22 -5
- package/helper/Helper.js +7 -0
- package/index.js +7 -0
- package/licencing/LicenceManager.js +64 -0
- package/package.json +1 -1
package/Constants.js
CHANGED
package/actions/automap.js
CHANGED
|
@@ -33,6 +33,10 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
|
|
|
33
33
|
if (!producer) {
|
|
34
34
|
throw new Error(`Producer ${producerName} not found`);
|
|
35
35
|
}
|
|
36
|
+
const source = Environment_1.default.getSource(producer.source);
|
|
37
|
+
if (!source) {
|
|
38
|
+
throw new Error(`Source ${producer.source} not found`);
|
|
39
|
+
}
|
|
36
40
|
// Get the specified schemas
|
|
37
41
|
const schemas = [];
|
|
38
42
|
for (const schemaName of schemaNames) {
|
|
@@ -47,7 +51,7 @@ const automap = (producerName, schemaNames) => __awaiter(void 0, void 0, void 0,
|
|
|
47
51
|
// Convert sample data to strings for AutoMapperEngine
|
|
48
52
|
const sampleStrings = sampleData.map(item => JSON.stringify(item));
|
|
49
53
|
// Call the automapper
|
|
50
|
-
const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas);
|
|
54
|
+
const mapResult = yield AutoMapperEngine_1.default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
|
|
51
55
|
// Create the producers based on the mapping
|
|
52
56
|
for (const producer of mapResult.producers) {
|
|
53
57
|
const producerPath = path_1.default.join('remora/producers', `${producer.name}.json`);
|
package/actions/deploy.js
CHANGED
|
@@ -59,7 +59,6 @@ const deploy = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
59
59
|
const apiKey = process.env.REMORA_LICENCE_KEY;
|
|
60
60
|
if (!apiKey)
|
|
61
61
|
throw new Error('REMORA_LICENCE_KEY environment variable is not set');
|
|
62
|
-
console.log('workerAPI', workerAPI);
|
|
63
62
|
const response = yield fetch(workerAPI, {
|
|
64
63
|
method: 'POST',
|
|
65
64
|
headers: {
|
package/actions/run.js
CHANGED
|
@@ -49,7 +49,7 @@ const run = (consumerName) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
49
49
|
if (response.fileUri)
|
|
50
50
|
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.fileUri}`));
|
|
51
51
|
else
|
|
52
|
-
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> ${response.data.slice(0, 5).join('\n')}
|
|
52
|
+
console.log(chalk_1.default.green(`• Consumer ${consumer.name} -> \n${response.data.slice(0, 5).map(x => JSON.stringify(x)).join('\n')}\n and ${response.data.length - 5} more...`));
|
|
53
53
|
}
|
|
54
54
|
else {
|
|
55
55
|
console.log(chalk_1.default.red(`• Consumer ${consumer.name} -> Failed: ${error}`));
|
package/auth/JWTManager.js
CHANGED
|
@@ -46,7 +46,7 @@ class JWTManagerClass {
|
|
|
46
46
|
this.verifyCLI = (token) => {
|
|
47
47
|
(0, Affirm_1.default)(token, 'Invalid token');
|
|
48
48
|
// ⚠️ ATTENTION!! The remoteSecretKey is saved in clear text because we are studying a way to not make it visible to the customer. ATTENTION!!
|
|
49
|
-
const REMORA_KEY_SECRET =
|
|
49
|
+
const REMORA_KEY_SECRET = process.env.REMORA_KEY_SECRET;
|
|
50
50
|
(0, Affirm_1.default)(REMORA_KEY_SECRET, 'Invalid CLI JWT SECRET');
|
|
51
51
|
return jsonwebtoken_1.default.verify(token, REMORA_KEY_SECRET);
|
|
52
52
|
};
|
|
@@ -13,8 +13,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
13
13
|
};
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
const mongodb_1 = require("mongodb");
|
|
16
|
-
const Settings_1 = __importDefault(require("../helper/Settings"));
|
|
17
16
|
const Helper_1 = __importDefault(require("../helper/Helper"));
|
|
17
|
+
const Settings_1 = __importDefault(require("../helper/Settings"));
|
|
18
18
|
class DatabaseEngineClass {
|
|
19
19
|
constructor() {
|
|
20
20
|
this.MAX_TRY_CONNECTION = 3;
|
|
@@ -47,10 +47,10 @@ class DatabaseEngineClass {
|
|
|
47
47
|
console.error('Error disconnecting from MongoDB:', error);
|
|
48
48
|
}
|
|
49
49
|
});
|
|
50
|
-
this.query = (collectionName,
|
|
50
|
+
this.query = (collectionName, filter, options) => __awaiter(this, void 0, void 0, function* () {
|
|
51
51
|
try {
|
|
52
52
|
const collection = this._db.collection(collectionName);
|
|
53
|
-
const result = yield collection.find(
|
|
53
|
+
const result = yield collection.find(filter, options).toArray();
|
|
54
54
|
return result;
|
|
55
55
|
}
|
|
56
56
|
catch (error) {
|
|
@@ -58,6 +58,16 @@ class DatabaseEngineClass {
|
|
|
58
58
|
throw error;
|
|
59
59
|
}
|
|
60
60
|
});
|
|
61
|
+
this.aggregate = (collectionName, aggregation) => __awaiter(this, void 0, void 0, function* () {
|
|
62
|
+
try {
|
|
63
|
+
const collection = this._db.collection(collectionName);
|
|
64
|
+
return yield collection.aggregate(aggregation).toArray();
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
console.error('Error aggregate documents:', error);
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
});
|
|
61
71
|
this.get = (collectionName, id) => __awaiter(this, void 0, void 0, function* () {
|
|
62
72
|
try {
|
|
63
73
|
const collection = this._db.collection(collectionName);
|
|
@@ -127,6 +127,52 @@
|
|
|
127
127
|
"subFields"
|
|
128
128
|
],
|
|
129
129
|
"additionalProperties": false
|
|
130
|
+
},
|
|
131
|
+
"transform": {
|
|
132
|
+
"description": "Optional list of transformations to apply to the dataset before returning the data",
|
|
133
|
+
"oneOf": [
|
|
134
|
+
{
|
|
135
|
+
"$ref": "#/definitions/singleTransformation"
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"type": "array",
|
|
139
|
+
"items": {
|
|
140
|
+
"$ref": "#/definitions/singleTransformation"
|
|
141
|
+
},
|
|
142
|
+
"description": "A list of transformations to be applied in sequence"
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
},
|
|
146
|
+
"validate": {
|
|
147
|
+
"type": "object",
|
|
148
|
+
"description": "Rules to check field value compliance and data quality",
|
|
149
|
+
"properties": {
|
|
150
|
+
"min": {
|
|
151
|
+
"type": "number",
|
|
152
|
+
"description": "Minimum value for numeric fields"
|
|
153
|
+
},
|
|
154
|
+
"max": {
|
|
155
|
+
"type": "number",
|
|
156
|
+
"description": "Maximum value for numeric fields"
|
|
157
|
+
},
|
|
158
|
+
"regex": {
|
|
159
|
+
"type": "string",
|
|
160
|
+
"description": "Regular expression pattern to validate string fields"
|
|
161
|
+
},
|
|
162
|
+
"required": {
|
|
163
|
+
"type": "boolean",
|
|
164
|
+
"description": "Whether the field is required"
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
"additionalProperties": false
|
|
168
|
+
},
|
|
169
|
+
"onError": {
|
|
170
|
+
"type": "string",
|
|
171
|
+
"description": "Action to take if an error occurs during transformations or validation",
|
|
172
|
+
"enum": ["set_default", "skip", "fail"]
|
|
173
|
+
},
|
|
174
|
+
"default": {
|
|
175
|
+
"description": "Default value of the field if it is missing (or on error if specified)"
|
|
130
176
|
}
|
|
131
177
|
},
|
|
132
178
|
"required": [
|
|
@@ -325,12 +371,358 @@
|
|
|
325
371
|
"subFields"
|
|
326
372
|
],
|
|
327
373
|
"additionalProperties": false
|
|
374
|
+
},
|
|
375
|
+
"transform": {
|
|
376
|
+
"description": "Optional list of transformations to apply to the dataset before returning the data",
|
|
377
|
+
"oneOf": [
|
|
378
|
+
{
|
|
379
|
+
"$ref": "#/definitions/singleTransformation"
|
|
380
|
+
},
|
|
381
|
+
{
|
|
382
|
+
"type": "array",
|
|
383
|
+
"items": {
|
|
384
|
+
"$ref": "#/definitions/singleTransformation"
|
|
385
|
+
},
|
|
386
|
+
"description": "A list of transformations to be applied in sequence"
|
|
387
|
+
}
|
|
388
|
+
]
|
|
389
|
+
},
|
|
390
|
+
"validate": {
|
|
391
|
+
"type": "object",
|
|
392
|
+
"description": "Rules to check field value compliance and data quality",
|
|
393
|
+
"properties": {
|
|
394
|
+
"min": {
|
|
395
|
+
"type": "number",
|
|
396
|
+
"description": "Minimum value for numeric fields"
|
|
397
|
+
},
|
|
398
|
+
"max": {
|
|
399
|
+
"type": "number",
|
|
400
|
+
"description": "Maximum value for numeric fields"
|
|
401
|
+
},
|
|
402
|
+
"regex": {
|
|
403
|
+
"type": "string",
|
|
404
|
+
"description": "Regular expression pattern to validate string fields"
|
|
405
|
+
},
|
|
406
|
+
"required": {
|
|
407
|
+
"type": "boolean",
|
|
408
|
+
"description": "Whether the field is required"
|
|
409
|
+
}
|
|
410
|
+
},
|
|
411
|
+
"additionalProperties": false
|
|
412
|
+
},
|
|
413
|
+
"onError": {
|
|
414
|
+
"type": "string",
|
|
415
|
+
"description": "Action to take if an error occurs during transformations or validation",
|
|
416
|
+
"enum": ["set_default", "skip", "fail"]
|
|
417
|
+
},
|
|
418
|
+
"default": {
|
|
419
|
+
"description": "Default value of the field if it is missing (or on error if specified)"
|
|
328
420
|
}
|
|
329
421
|
},
|
|
330
422
|
"required": [
|
|
331
423
|
"key"
|
|
332
424
|
],
|
|
333
425
|
"additionalProperties": false
|
|
426
|
+
},
|
|
427
|
+
"singleTransformation": {
|
|
428
|
+
"oneOf": [
|
|
429
|
+
{
|
|
430
|
+
"type": "object",
|
|
431
|
+
"properties": {
|
|
432
|
+
"cast": {
|
|
433
|
+
"type": "string",
|
|
434
|
+
"description": "Cast the value to a specific type",
|
|
435
|
+
"enum": ["string", "number", "date", "boolean"]
|
|
436
|
+
}
|
|
437
|
+
},
|
|
438
|
+
"required": ["cast"],
|
|
439
|
+
"additionalProperties": false
|
|
440
|
+
},
|
|
441
|
+
{
|
|
442
|
+
"type": "object",
|
|
443
|
+
"properties": {
|
|
444
|
+
"multiply": {
|
|
445
|
+
"type": "number",
|
|
446
|
+
"description": "Multiply the numeric value by this factor"
|
|
447
|
+
}
|
|
448
|
+
},
|
|
449
|
+
"required": ["multiply"],
|
|
450
|
+
"additionalProperties": false
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
"type": "object",
|
|
454
|
+
"properties": {
|
|
455
|
+
"add": {
|
|
456
|
+
"type": "number",
|
|
457
|
+
"description": "Add this number to the numeric value"
|
|
458
|
+
}
|
|
459
|
+
},
|
|
460
|
+
"required": ["add"],
|
|
461
|
+
"additionalProperties": false
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
"type": "object",
|
|
465
|
+
"properties": {
|
|
466
|
+
"extract": {
|
|
467
|
+
"type": "string",
|
|
468
|
+
"enum": ["year", "month", "day", "hour", "minute"],
|
|
469
|
+
"description": "Extract a component from a date value"
|
|
470
|
+
}
|
|
471
|
+
},
|
|
472
|
+
"required": ["extract"],
|
|
473
|
+
"additionalProperties": false
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
"type": "object",
|
|
477
|
+
"properties": {
|
|
478
|
+
"concat": {
|
|
479
|
+
"type": "object",
|
|
480
|
+
"properties": {
|
|
481
|
+
"separator": {
|
|
482
|
+
"type": "string",
|
|
483
|
+
"description": "The separator to use when joining array elements"
|
|
484
|
+
}
|
|
485
|
+
},
|
|
486
|
+
"required": ["separator"],
|
|
487
|
+
"additionalProperties": false
|
|
488
|
+
}
|
|
489
|
+
},
|
|
490
|
+
"required": ["concat"],
|
|
491
|
+
"additionalProperties": false
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
"type": "object",
|
|
495
|
+
"properties": {
|
|
496
|
+
"split": {
|
|
497
|
+
"type": "object",
|
|
498
|
+
"properties": {
|
|
499
|
+
"separator": {
|
|
500
|
+
"type": "string",
|
|
501
|
+
"description": "The separator to use when splitting the string"
|
|
502
|
+
},
|
|
503
|
+
"index": {
|
|
504
|
+
"type": "number",
|
|
505
|
+
"description": "The index of the split part to keep"
|
|
506
|
+
}
|
|
507
|
+
},
|
|
508
|
+
"required": ["separator", "index"],
|
|
509
|
+
"additionalProperties": false
|
|
510
|
+
}
|
|
511
|
+
},
|
|
512
|
+
"required": ["split"],
|
|
513
|
+
"additionalProperties": false
|
|
514
|
+
},
|
|
515
|
+
{
|
|
516
|
+
"type": "object",
|
|
517
|
+
"properties": {
|
|
518
|
+
"regex_match": {
|
|
519
|
+
"type": "object",
|
|
520
|
+
"properties": {
|
|
521
|
+
"pattern": {
|
|
522
|
+
"type": "string",
|
|
523
|
+
"description": "The regex pattern to test against"
|
|
524
|
+
},
|
|
525
|
+
"flags": {
|
|
526
|
+
"type": "string",
|
|
527
|
+
"description": "Regex flags (e.g., 'i' for case-insensitive)"
|
|
528
|
+
}
|
|
529
|
+
},
|
|
530
|
+
"required": ["pattern"],
|
|
531
|
+
"additionalProperties": false
|
|
532
|
+
}
|
|
533
|
+
},
|
|
534
|
+
"required": ["regex_match"],
|
|
535
|
+
"additionalProperties": false
|
|
536
|
+
},
|
|
537
|
+
{
|
|
538
|
+
"type": "object",
|
|
539
|
+
"properties": {
|
|
540
|
+
"regex_replace": {
|
|
541
|
+
"type": "object",
|
|
542
|
+
"properties": {
|
|
543
|
+
"pattern": {
|
|
544
|
+
"type": "string",
|
|
545
|
+
"description": "The regex pattern to match"
|
|
546
|
+
},
|
|
547
|
+
"replacement": {
|
|
548
|
+
"type": "string",
|
|
549
|
+
"description": "The replacement string"
|
|
550
|
+
},
|
|
551
|
+
"flags": {
|
|
552
|
+
"type": "string",
|
|
553
|
+
"description": "Regex flags (e.g., 'g' for global)"
|
|
554
|
+
}
|
|
555
|
+
},
|
|
556
|
+
"required": ["pattern", "replacement"],
|
|
557
|
+
"additionalProperties": false
|
|
558
|
+
}
|
|
559
|
+
},
|
|
560
|
+
"required": ["regex_replace"],
|
|
561
|
+
"additionalProperties": false
|
|
562
|
+
},
|
|
563
|
+
{
|
|
564
|
+
"type": "object",
|
|
565
|
+
"properties": {
|
|
566
|
+
"regex_extract": {
|
|
567
|
+
"type": "object",
|
|
568
|
+
"properties": {
|
|
569
|
+
"pattern": {
|
|
570
|
+
"type": "string",
|
|
571
|
+
"description": "The regex pattern to extract with"
|
|
572
|
+
},
|
|
573
|
+
"group": {
|
|
574
|
+
"type": "number",
|
|
575
|
+
"description": "The capture group index to extract (0 for full match)"
|
|
576
|
+
},
|
|
577
|
+
"flags": {
|
|
578
|
+
"type": "string",
|
|
579
|
+
"description": "Regex flags (e.g., 'i' for case-insensitive)"
|
|
580
|
+
}
|
|
581
|
+
},
|
|
582
|
+
"required": ["pattern", "group"],
|
|
583
|
+
"additionalProperties": false
|
|
584
|
+
}
|
|
585
|
+
},
|
|
586
|
+
"required": ["regex_extract"],
|
|
587
|
+
"additionalProperties": false
|
|
588
|
+
},
|
|
589
|
+
{
|
|
590
|
+
"type": "object",
|
|
591
|
+
"properties": {
|
|
592
|
+
"trim": {
|
|
593
|
+
"type": "boolean",
|
|
594
|
+
"description": "Trim whitespace from both ends of the string"
|
|
595
|
+
}
|
|
596
|
+
},
|
|
597
|
+
"required": ["trim"],
|
|
598
|
+
"additionalProperties": false
|
|
599
|
+
},
|
|
600
|
+
{
|
|
601
|
+
"type": "object",
|
|
602
|
+
"properties": {
|
|
603
|
+
"to_lowercase": {
|
|
604
|
+
"type": "boolean",
|
|
605
|
+
"description": "Convert string to lowercase"
|
|
606
|
+
}
|
|
607
|
+
},
|
|
608
|
+
"required": ["to_lowercase"],
|
|
609
|
+
"additionalProperties": false
|
|
610
|
+
},
|
|
611
|
+
{
|
|
612
|
+
"type": "object",
|
|
613
|
+
"properties": {
|
|
614
|
+
"to_uppercase": {
|
|
615
|
+
"type": "boolean",
|
|
616
|
+
"description": "Convert string to uppercase"
|
|
617
|
+
}
|
|
618
|
+
},
|
|
619
|
+
"required": ["to_uppercase"],
|
|
620
|
+
"additionalProperties": false
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
"type": "object",
|
|
624
|
+
"properties": {
|
|
625
|
+
"capitalize": {
|
|
626
|
+
"type": "boolean",
|
|
627
|
+
"description": "Capitalize the first letter of the string"
|
|
628
|
+
}
|
|
629
|
+
},
|
|
630
|
+
"required": ["capitalize"],
|
|
631
|
+
"additionalProperties": false
|
|
632
|
+
},
|
|
633
|
+
{
|
|
634
|
+
"type": "object",
|
|
635
|
+
"properties": {
|
|
636
|
+
"substring": {
|
|
637
|
+
"type": "object",
|
|
638
|
+
"properties": {
|
|
639
|
+
"start": {
|
|
640
|
+
"type": "number",
|
|
641
|
+
"description": "Starting position of substring"
|
|
642
|
+
},
|
|
643
|
+
"end": {
|
|
644
|
+
"type": "number",
|
|
645
|
+
"description": "Optional ending position of substring"
|
|
646
|
+
}
|
|
647
|
+
},
|
|
648
|
+
"required": ["start"],
|
|
649
|
+
"additionalProperties": false
|
|
650
|
+
}
|
|
651
|
+
},
|
|
652
|
+
"required": ["substring"],
|
|
653
|
+
"additionalProperties": false
|
|
654
|
+
},
|
|
655
|
+
{
|
|
656
|
+
"type": "object",
|
|
657
|
+
"properties": {
|
|
658
|
+
"pad_start": {
|
|
659
|
+
"type": "object",
|
|
660
|
+
"properties": {
|
|
661
|
+
"length": {
|
|
662
|
+
"type": "number",
|
|
663
|
+
"description": "Desired string length after padding"
|
|
664
|
+
},
|
|
665
|
+
"char": {
|
|
666
|
+
"type": "string",
|
|
667
|
+
"description": "Character to pad with",
|
|
668
|
+
"minLength": 1,
|
|
669
|
+
"maxLength": 1
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
"required": ["length", "char"],
|
|
673
|
+
"additionalProperties": false
|
|
674
|
+
}
|
|
675
|
+
},
|
|
676
|
+
"required": ["pad_start"],
|
|
677
|
+
"additionalProperties": false
|
|
678
|
+
},
|
|
679
|
+
{
|
|
680
|
+
"type": "object",
|
|
681
|
+
"properties": {
|
|
682
|
+
"pad_end": {
|
|
683
|
+
"type": "object",
|
|
684
|
+
"properties": {
|
|
685
|
+
"length": {
|
|
686
|
+
"type": "number",
|
|
687
|
+
"description": "Desired string length after padding"
|
|
688
|
+
},
|
|
689
|
+
"char": {
|
|
690
|
+
"type": "string",
|
|
691
|
+
"description": "Character to pad with",
|
|
692
|
+
"minLength": 1,
|
|
693
|
+
"maxLength": 1
|
|
694
|
+
}
|
|
695
|
+
},
|
|
696
|
+
"required": ["length", "char"],
|
|
697
|
+
"additionalProperties": false
|
|
698
|
+
}
|
|
699
|
+
},
|
|
700
|
+
"required": ["pad_end"],
|
|
701
|
+
"additionalProperties": false
|
|
702
|
+
},
|
|
703
|
+
{
|
|
704
|
+
"type": "object",
|
|
705
|
+
"properties": {
|
|
706
|
+
"prepend": {
|
|
707
|
+
"type": "string",
|
|
708
|
+
"description": "String to add at the beginning"
|
|
709
|
+
}
|
|
710
|
+
},
|
|
711
|
+
"required": ["prepend"],
|
|
712
|
+
"additionalProperties": false
|
|
713
|
+
},
|
|
714
|
+
{
|
|
715
|
+
"type": "object",
|
|
716
|
+
"properties": {
|
|
717
|
+
"append": {
|
|
718
|
+
"type": "string",
|
|
719
|
+
"description": "String to add at the end"
|
|
720
|
+
}
|
|
721
|
+
},
|
|
722
|
+
"required": ["append"],
|
|
723
|
+
"additionalProperties": false
|
|
724
|
+
}
|
|
725
|
+
]
|
|
334
726
|
}
|
|
335
727
|
},
|
|
336
728
|
"examples": [
|
|
@@ -135,6 +135,10 @@
|
|
|
135
135
|
"CSV"
|
|
136
136
|
],
|
|
137
137
|
"description": "The type of file to read"
|
|
138
|
+
},
|
|
139
|
+
"delimiter": {
|
|
140
|
+
"type": "string",
|
|
141
|
+
"description": "The column delimiter for the CSV file if different from the default (,)."
|
|
138
142
|
}
|
|
139
143
|
},
|
|
140
144
|
"additionalProperties": false
|
package/drivers/LocalDriver.js
CHANGED
|
@@ -68,15 +68,35 @@ class LocalDriver {
|
|
|
68
68
|
return this;
|
|
69
69
|
});
|
|
70
70
|
this.download = (request) => __awaiter(this, void 0, void 0, function* () {
|
|
71
|
+
var _a, e_1, _b, _c;
|
|
71
72
|
(0, Affirm_1.default)(this._path, `Invalid path`);
|
|
72
73
|
(0, Affirm_1.default)(request, `Invalid download request`);
|
|
73
74
|
(0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
|
|
74
75
|
const fileUrl = path_1.default.join(this._path, request.fileKey);
|
|
75
|
-
const
|
|
76
|
-
|
|
76
|
+
const stream = fs.createReadStream(fileUrl);
|
|
77
|
+
const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
|
|
78
|
+
const lines = [];
|
|
79
|
+
try {
|
|
80
|
+
for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
|
|
81
|
+
_c = reader_1_1.value;
|
|
82
|
+
_d = false;
|
|
83
|
+
const line = _c;
|
|
84
|
+
lines.push(line);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
88
|
+
finally {
|
|
89
|
+
try {
|
|
90
|
+
if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
|
|
91
|
+
}
|
|
92
|
+
finally { if (e_1) throw e_1.error; }
|
|
93
|
+
}
|
|
94
|
+
reader.close();
|
|
95
|
+
stream.close();
|
|
96
|
+
return lines;
|
|
77
97
|
});
|
|
78
98
|
this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
|
|
79
|
-
var _a,
|
|
99
|
+
var _a, e_2, _b, _c;
|
|
80
100
|
(0, Affirm_1.default)(this._path, `Invalid path`);
|
|
81
101
|
(0, Affirm_1.default)(readOptions, 'Invalid read options');
|
|
82
102
|
(0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
|
|
@@ -87,8 +107,8 @@ class LocalDriver {
|
|
|
87
107
|
const lines = [];
|
|
88
108
|
let lineCounter = 0;
|
|
89
109
|
try {
|
|
90
|
-
for (var _d = true,
|
|
91
|
-
_c =
|
|
110
|
+
for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
|
|
111
|
+
_c = reader_2_1.value;
|
|
92
112
|
_d = false;
|
|
93
113
|
const line = _c;
|
|
94
114
|
if (lineCounter >= lineFrom && lineCounter < lineTo) {
|
|
@@ -99,12 +119,12 @@ class LocalDriver {
|
|
|
99
119
|
break;
|
|
100
120
|
}
|
|
101
121
|
}
|
|
102
|
-
catch (
|
|
122
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
103
123
|
finally {
|
|
104
124
|
try {
|
|
105
|
-
if (!_d && !_a && (_b =
|
|
125
|
+
if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
|
|
106
126
|
}
|
|
107
|
-
finally { if (
|
|
127
|
+
finally { if (e_2) throw e_2.error; }
|
|
108
128
|
}
|
|
109
129
|
reader.close();
|
|
110
130
|
stream.close();
|
|
@@ -53,23 +53,25 @@ class S3SourceDriver {
|
|
|
53
53
|
}));
|
|
54
54
|
(0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
|
|
55
55
|
const stream = response.Body;
|
|
56
|
-
const
|
|
56
|
+
const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
|
|
57
|
+
const lines = [];
|
|
57
58
|
try {
|
|
58
|
-
for (var _d = true,
|
|
59
|
-
_c =
|
|
59
|
+
for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
|
|
60
|
+
_c = reader_1_1.value;
|
|
60
61
|
_d = false;
|
|
61
|
-
const
|
|
62
|
-
|
|
62
|
+
const line = _c;
|
|
63
|
+
lines.push(line);
|
|
63
64
|
}
|
|
64
65
|
}
|
|
65
66
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
66
67
|
finally {
|
|
67
68
|
try {
|
|
68
|
-
if (!_d && !_a && (_b =
|
|
69
|
+
if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
|
|
69
70
|
}
|
|
70
71
|
finally { if (e_1) throw e_1.error; }
|
|
71
72
|
}
|
|
72
|
-
|
|
73
|
+
reader.close();
|
|
74
|
+
return lines;
|
|
73
75
|
});
|
|
74
76
|
this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
|
|
75
77
|
var _a, e_2, _b, _c;
|
|
@@ -87,8 +89,8 @@ class S3SourceDriver {
|
|
|
87
89
|
const lines = [];
|
|
88
90
|
let lineCounter = 0;
|
|
89
91
|
try {
|
|
90
|
-
for (var _d = true,
|
|
91
|
-
_c =
|
|
92
|
+
for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
|
|
93
|
+
_c = reader_2_1.value;
|
|
92
94
|
_d = false;
|
|
93
95
|
const line = _c;
|
|
94
96
|
if (lineCounter >= lineFrom && lineCounter < lineTo)
|
|
@@ -101,7 +103,7 @@ class S3SourceDriver {
|
|
|
101
103
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
102
104
|
finally {
|
|
103
105
|
try {
|
|
104
|
-
if (!_d && !_a && (_b =
|
|
106
|
+
if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
|
|
105
107
|
}
|
|
106
108
|
finally { if (e_2) throw e_2.error; }
|
|
107
109
|
}
|