@forzalabs/remora 0.0.21 → 0.0.22

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.21',
+    cliVersion: '0.0.22',
     lambdaVersion: 1,
     port: 5069,
     defaults: {

package/drivers/LocalDriver.js

@@ -68,15 +68,35 @@ class LocalDriver {
             return this;
         });
         this.download = (request) => __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
             (0, Affirm_1.default)(this._path, `Invalid path`);
             (0, Affirm_1.default)(request, `Invalid download request`);
             (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
             const fileUrl = path_1.default.join(this._path, request.fileKey);
-            const data = yield fs.promises.readFile(fileUrl, 'utf-8');
-            return data;
+            const stream = fs.createReadStream(fileUrl);
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
+            try {
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    lines.push(line);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            reader.close();
+            stream.close();
+            return lines;
         });
         this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
+            var _a, e_2, _b, _c;
             (0, Affirm_1.default)(this._path, `Invalid path`);
             (0, Affirm_1.default)(readOptions, 'Invalid read options');
             (0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
@@ -87,8 +107,8 @@ class LocalDriver {
             const lines = [];
             let lineCounter = 0;
             try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
+                for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
+                    _c = reader_2_1.value;
                     _d = false;
                     const line = _c;
                     if (lineCounter >= lineFrom && lineCounter < lineTo) {
@@ -99,12 +119,12 @@ class LocalDriver {
                         break;
                 }
             }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                    if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
                 }
-                finally { if (e_1) throw e_1.error; }
+                finally { if (e_2) throw e_2.error; }
             }
             reader.close();
             stream.close();

package/drivers/RedshiftDriver.js

@@ -96,6 +96,7 @@ class RedshiftDriver {
                 return true;
             }
             catch (e) {
+                console.error(e);
                 return false;
             }
         });

package/drivers/S3SourceDriver.js

@@ -53,23 +53,25 @@ class S3SourceDriver {
             }));
             (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
             const stream = response.Body;
-            const chunks = [];
+            const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
+            const lines = [];
             try {
-                for (var _d = true, stream_1 = __asyncValues(stream), stream_1_1; stream_1_1 = yield stream_1.next(), _a = stream_1_1.done, !_a; _d = true) {
-                    _c = stream_1_1.value;
+                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
+                    _c = reader_1_1.value;
                     _d = false;
-                    const chunk = _c;
-                    chunks.push(chunk);
+                    const line = _c;
+                    lines.push(line);
                 }
             }
             catch (e_1_1) { e_1 = { error: e_1_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = stream_1.return)) yield _b.call(stream_1);
+                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
                 }
                 finally { if (e_1) throw e_1.error; }
             }
-            return Buffer.concat(chunks).toString('utf-8');
+            reader.close();
+            return lines;
         });
         this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
             var _a, e_2, _b, _c;
@@ -87,8 +89,8 @@ class S3SourceDriver {
             const lines = [];
             let lineCounter = 0;
             try {
-                for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
-                    _c = reader_1_1.value;
+                for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
+                    _c = reader_2_1.value;
                     _d = false;
                     const line = _c;
                     if (lineCounter >= lineFrom && lineCounter < lineTo)
@@ -101,7 +103,7 @@ class S3SourceDriver {
             catch (e_2_1) { e_2 = { error: e_2_1 }; }
             finally {
                 try {
-                    if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
+                    if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
                 }
                 finally { if (e_2) throw e_2.error; }
             }

package/engines/ParseManager.js

@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Affirm_1 = __importDefault(require("../core/Affirm"));
+const Environment_1 = __importDefault(require("./Environment"));
+const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
 class ParseManagerClass {
     constructor() {
         this.csvToJson = (csv, producer) => {
@@ -17,21 +19,36 @@ class ParseManagerClass {
             (0, Affirm_1.default)(lines, 'Invalid csv lines');
             Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
             const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
-            const fileRows = lines;
-            const columns = fileRows[0].split(delimiterChar).map(x => x.trim());
-            const rows = fileRows.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
+            const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
+            const headerColumns = this._extractHeader(lines[0], delimiterChar, producer);
             const result = [];
-            for (let i = 0; i < rows.length; i++) {
-                const row = rows[i];
+            for (const row of rows) {
                 const rowObject = {};
-                for (let j = 0; j < columns.length; j++) {
-                    const column = columns[j];
-                    rowObject[column] = row[j];
+                for (let i = 0; i < headerColumns.length; i++) {
+                    const column = headerColumns[i];
+                    rowObject[column.saveAs] = row[column.index];
                 }
                 result.push(rowObject);
             }
             return result;
         };
+        this._extractHeader = (headerLine, delimiter, producer) => {
+            var _a;
+            (0, Affirm_1.default)(headerLine, 'Invalid CSV header line');
+            (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
+            (0, Affirm_1.default)(producer, 'Invalid producer');
+            const source = Environment_1.default.getSource(producer.source);
+            const columns = FileCompiler_1.default.compileProducer(producer, source);
+            const headerColumns = headerLine.split(delimiter).map(x => x.trim());
+            const csvColumns = [];
+            for (const pColumn of columns) {
+                const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
+                const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
+                (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
+                csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
+            }
+            return csvColumns;
+        };
     }
 }
 const ParseManager = new ParseManagerClass();

package/engines/ProducerEngine.js

@@ -95,7 +95,7 @@ class ProducerEngineClass {
             if (options.readmode === 'lines')
                 lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
             else
-                lines = [(yield driver.download({ fileKey: producer.settings.fileKey }))];
+                lines = yield driver.download({ fileKey: producer.settings.fileKey });
             switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
                 case 'CSV': {
                     return { data: lines, dataType: 'lines-of-text' };

package/engines/ai/LLM.js

@@ -263,46 +263,7 @@ class LLM {
             };
             const res = yield this._client.beta.chat.completions.parse(item);
             const msg = res.choices[0].message;
-            const firstDraft = msg.parsed;
-            const qaSystemPrompt = baseQASystemPrompt.replace('{{consumers}}', firstDraft.consumers.map(x => `- ${JSON.stringify(x)}`).join('\n'));
-            const res2 = yield this._client.beta.chat.completions.parse({
-                model: 'gpt-4o',
-                messages: [
-                    { role: 'system', content: qaSystemPrompt }
-                ],
-                response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
-                    consumers: zod_2.z.array(zod_2.z.object({
-                        $schema: zod_2.z.string().describe('The schema of the consumer. This should always be the same.'),
-                        name: zod_2.z.string(),
-                        description: zod_2.z.string(),
-                        producers: zod_2.z.array(zod_2.z.object({
-                            name: zod_2.z.string().describe('References one of the producers. Must be unique, there can\'t be two entry with the same name.'),
-                            joins: zod_2.z.array(zod_2.z.object({
-                                otherName: zod_2.z.string(),
-                                relationship: zod_2.z.enum(['one-to-one', 'one-to-many', 'many-to-one']),
-                                sql: zod_2.z.string()
-                            })).optional().describe('Which other producer to join this one with. Omit if empty.')
-                        })),
-                        fields: zod_2.z.array(zod_2.z.object({
-                            key: zod_2.z.string(),
-                            from: zod_2.z.string().optional(),
-                            alias: zod_2.z.string().optional()
-                            // grouping: z.object({
-                            //     groupingKey: z.string().optional(),
-                            //     subFields: z.array(z.lazy(() => z.object({
-                            //         key: z.string(),
-                            //         from: z.string().optional()
-                            //     }))).optional()
-                            // }).optional()
-                        })),
-                        outputs: zod_2.z.array(zod_2.z.object({
-                            format: zod_2.z.enum(['SQL', 'API', 'CSV', 'PARQUET', 'JSON'])
-                        }))
-                    }))
-                }), 'environment')
-            });
-            const msg2 = res2.choices[0].message;
-            const finalDraft = msg2.parsed;
+            const finalDraft = msg.parsed;
             // Do some manual adjustments cause some things still don't work...
             if (finalDraft && finalDraft.consumers) {
                 for (const cons of finalDraft.consumers) {

package/engines/consumer/ConsumerManager.js

@@ -113,7 +113,6 @@ class ConsumerManagerClass {
             else {
                 const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
                 (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
-                // TODO: CHECK THIS FIX IS GOOD
                 expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
             }
             return expandedFields;

package/engines/consumer/PostProcessor.js

@@ -8,12 +8,15 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
 const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
 const Environment_1 = __importDefault(require("../Environment"));
 const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
+const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
 const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
 class PostProcessorClass {
     constructor() {
         /**
-         * Get an array of objects and projects it to another array of objects but with different shape
-         * Basically does a field mapping with some extra logic (like grouping and creating nested fields)
+         * Maps an array of objects and projects it to another array of objects but with different shape:
+         * - grouping (creates nested fields)
+         * - type casting
+         * - default field values
          */
         this.doProjection = (consumer, data) => {
             (0, Affirm_1.default)(consumer, 'Invalid consumer');
@@ -52,13 +55,18 @@ class PostProcessorClass {
                 }
                 else {
                     return items.map(x => {
-                        var _a;
+                        var _a, _b, _c;
                         const projected = {};
                         for (const field of allFields) {
                             const { key, alias } = field.cField;
                             const fieldKey = alias !== null && alias !== void 0 ? alias : key;
                             const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
-                            projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[key]);
+                            const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
+                            const fieldValue = this._getFieldValue(x, field);
+                            if (Algo_1.default.hasVal(maskType))
+                                projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue);
+                            else
+                                projected[fieldKey] = TypeCaster_1.default.cast(fieldValue, fieldType);
                         }
                         return projected;
                     });
@@ -136,6 +144,15 @@ class PostProcessorClass {
             const unpackedData = data.flatMap(x => unpackSingle(x));
             return unpackedData;
         };
+        this._getFieldValue = (record, field) => {
+            const fieldValue = record[field.cField.key];
+            if (Algo_1.default.hasVal(fieldValue))
+                return fieldValue;
+            else if (!Algo_1.default.hasVal(fieldValue) && Algo_1.default.hasVal(field.cField.default))
+                return field.cField.default;
+            else
+                return fieldValue;
+        };
     }
 }
 const PostProcessor = new PostProcessorClass();

package/engines/file/FileCompiler.js

@@ -17,7 +17,7 @@ class FileCompilerClass {
                 nameInProducer: x.name,
                 consumerAlias: null,
                 consumerKey: null,
-                owner: x.name,
+                owner: producer.name,
                 dimension: x
             }));
             return columns;

package/engines/transform/TransformationEngine.js

@@ -24,7 +24,7 @@ class TransformationEngineClass {
                     else if (!Algo_1.default.hasVal(value))
                         continue;
                     try {
-                        record[field.key] = this.applyTransformations(value, field.transform, field.key);
+                        record[field.key] = this.applyTransformations(value, field.transform, field);
                     }
                     catch (error) {
                         switch (field.onError) {
@@ -42,36 +42,39 @@ class TransformationEngineClass {
             }
             return data;
         };
-        this.applyTransformations = (value, transformations, fieldName) => {
+        this.applyTransformations = (value, transformations, field) => {
             var _a;
             if (Array.isArray(transformations)) {
                 // Process array transformations without creating intermediate arrays
                 let result = value;
                 for (const transform of transformations) {
-                    result = this.applyTransformations(result, transform, fieldName);
+                    result = this.applyTransformations(result, transform, field);
                 }
                 return result;
             }
             // Single transformation
             if ('cast' in transformations) {
-                return TypeCaster_1.default.cast(value, transformations.cast);
+                const casted = TypeCaster_1.default.cast(value, transformations.cast);
+                if (isNaN(casted) && transformations.cast === 'number')
+                    throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
+                return casted;
             }
             if ('multiply' in transformations) {
                 const num = TypeCaster_1.default.cast(value, 'number');
                 if (isNaN(num))
-                    throw new Error(`Cannot multiply non-numeric value in field '${fieldName}'`);
+                    throw new Error(`Cannot multiply non-numeric value in field '${field.key}'`);
                 return num * transformations.multiply;
             }
             if ('add' in transformations) {
                 const num = TypeCaster_1.default.cast(value, 'number');
                 if (isNaN(num))
-                    throw new Error(`Cannot add to non-numeric value in field '${fieldName}'`);
+                    throw new Error(`Cannot add to non-numeric value in field '${field.key}'`);
                 return num + transformations.add;
             }
             if ('extract' in transformations) {
                 const date = TypeCaster_1.default.cast(value, 'date');
                 if (isNaN(date.getTime()))
-                    throw new Error(`Invalid date for extraction in field '${fieldName}'`);
+                    throw new Error(`Invalid date for extraction in field '${field.key}'`);
                 switch (transformations.extract) {
                     case 'year': return date.getFullYear();
                     case 'month': return date.getMonth() + 1; // 1-based month
@@ -82,43 +85,43 @@ class TransformationEngineClass {
             }
             if ('concat' in transformations) {
                 if (!Array.isArray(value))
-                    throw new Error(`Cannot concat non-array value in field '${fieldName}'`);
+                    throw new Error(`Cannot concat non-array value in field '${field.key}'`);
                 return value.join(transformations.concat.separator);
             }
             if ('split' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot split non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot split non-string value in field '${field.key}'`);
                 const parts = value.split(transformations.split.separator);
                 if (transformations.split.index >= parts.length) {
-                    throw new Error(`Split index ${transformations.split.index} out of bounds in field '${fieldName}'`);
+                    throw new Error(`Split index ${transformations.split.index} out of bounds in field '${field.key}'`);
                 }
                 return parts[transformations.split.index];
             }
             if ('regex_match' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot apply regex_match to non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot apply regex_match to non-string value in field '${field.key}'`);
                 try {
                     const regex = new RegExp(transformations.regex_match.pattern, transformations.regex_match.flags);
                     return regex.test(value);
                 }
                 catch (error) {
-                    throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
+                    throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
                 }
             }
             if ('regex_replace' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot apply regex_replace to non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot apply regex_replace to non-string value in field '${field.key}'`);
                 try {
                     const regex = new RegExp(transformations.regex_replace.pattern, transformations.regex_replace.flags);
                     return value.replace(regex, transformations.regex_replace.replacement);
                 }
                 catch (error) {
-                    throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
+                    throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
                 }
             }
             if ('regex_extract' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot apply regex_extract to non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot apply regex_extract to non-string value in field '${field.key}'`);
                 try {
                     const regex = new RegExp(transformations.regex_extract.pattern, transformations.regex_extract.flags);
                     const matches = value.match(regex);
@@ -128,49 +131,49 @@ class TransformationEngineClass {
                     return (_a = matches[groupIndex]) !== null && _a !== void 0 ? _a : null;
                 }
                 catch (error) {
-                    throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
+                    throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
                 }
             }
             if ('trim' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot trim non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot trim non-string value in field '${field.key}'`);
                 return value.trim();
             }
             if ('to_lowercase' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot convert non-string value to lowercase in field '${fieldName}'`);
+                    throw new Error(`Cannot convert non-string value to lowercase in field '${field.key}'`);
                 return value.toLowerCase();
             }
             if ('to_uppercase' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot convert non-string value to uppercase in field '${fieldName}'`);
+                    throw new Error(`Cannot convert non-string value to uppercase in field '${field.key}'`);
                 return value.toUpperCase();
             }
             if ('capitalize' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot capitalize non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot capitalize non-string value in field '${field.key}'`);
                 return value.charAt(0).toUpperCase() + value.slice(1);
             }
             if ('substring' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot take substring of non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot take substring of non-string value in field '${field.key}'`);
                 const { start, end } = transformations.substring;
                 return end !== undefined ? value.substring(start, end) : value.substring(start);
             }
             if ('pad_start' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot pad non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot pad non-string value in field '${field.key}'`);
                 const { length, char } = transformations.pad_start;
                 if (char.length !== 1)
-                    throw new Error(`Pad character must be exactly one character in field '${fieldName}'`);
+                    throw new Error(`Pad character must be exactly one character in field '${field.key}'`);
                 return value.padStart(length, char);
             }
             if ('pad_end' in transformations) {
                 if (typeof value !== 'string')
-                    throw new Error(`Cannot pad non-string value in field '${fieldName}'`);
+                    throw new Error(`Cannot pad non-string value in field '${field.key}'`);
                 const { length, char } = transformations.pad_end;
                 if (char.length !== 1)
-                    throw new Error(`Pad character must be exactly one character in field '${fieldName}'`);
+                    throw new Error(`Pad character must be exactly one character in field '${field.key}'`);
                 return value.padEnd(length, char);
             }
             if ('prepend' in transformations)

package/engines/transform/TypeCaster.js

@@ -12,6 +12,7 @@ class TypeCasterClass {
                 else
                     return Boolean(value);
             }
+            case 'datetime':
             case 'date':
                 return new Date(value);
             case 'number': {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.21",
+    "version": "0.0.22",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,