@forzalabs/remora 0.0.21 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/Constants.js CHANGED
@@ -1,7 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  const CONSTANTS = {
4
- cliVersion: '0.0.21',
4
+ cliVersion: '0.0.23',
5
5
  lambdaVersion: 1,
6
6
  port: 5069,
7
7
  defaults: {
@@ -68,15 +68,35 @@ class LocalDriver {
68
68
  return this;
69
69
  });
70
70
  this.download = (request) => __awaiter(this, void 0, void 0, function* () {
71
+ var _a, e_1, _b, _c;
71
72
  (0, Affirm_1.default)(this._path, `Invalid path`);
72
73
  (0, Affirm_1.default)(request, `Invalid download request`);
73
74
  (0, Affirm_1.default)(request.fileKey, `Invalid file key for download request`);
74
75
  const fileUrl = path_1.default.join(this._path, request.fileKey);
75
- const data = yield fs.promises.readFile(fileUrl, 'utf-8');
76
- return data;
76
+ const stream = fs.createReadStream(fileUrl);
77
+ const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
78
+ const lines = [];
79
+ try {
80
+ for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
81
+ _c = reader_1_1.value;
82
+ _d = false;
83
+ const line = _c;
84
+ lines.push(line);
85
+ }
86
+ }
87
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
88
+ finally {
89
+ try {
90
+ if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
91
+ }
92
+ finally { if (e_1) throw e_1.error; }
93
+ }
94
+ reader.close();
95
+ stream.close();
96
+ return lines;
77
97
  });
78
98
  this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
79
- var _a, e_1, _b, _c;
99
+ var _a, e_2, _b, _c;
80
100
  (0, Affirm_1.default)(this._path, `Invalid path`);
81
101
  (0, Affirm_1.default)(readOptions, 'Invalid read options');
82
102
  (0, Affirm_1.default)(readOptions.fileKey, 'Invalid file key');
@@ -87,8 +107,8 @@ class LocalDriver {
87
107
  const lines = [];
88
108
  let lineCounter = 0;
89
109
  try {
90
- for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
91
- _c = reader_1_1.value;
110
+ for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
111
+ _c = reader_2_1.value;
92
112
  _d = false;
93
113
  const line = _c;
94
114
  if (lineCounter >= lineFrom && lineCounter < lineTo) {
@@ -99,12 +119,12 @@ class LocalDriver {
99
119
  break;
100
120
  }
101
121
  }
102
- catch (e_1_1) { e_1 = { error: e_1_1 }; }
122
+ catch (e_2_1) { e_2 = { error: e_2_1 }; }
103
123
  finally {
104
124
  try {
105
- if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
125
+ if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
106
126
  }
107
- finally { if (e_1) throw e_1.error; }
127
+ finally { if (e_2) throw e_2.error; }
108
128
  }
109
129
  reader.close();
110
130
  stream.close();
@@ -96,6 +96,7 @@ class RedshiftDriver {
96
96
  return true;
97
97
  }
98
98
  catch (e) {
99
+ console.error(e);
99
100
  return false;
100
101
  }
101
102
  });
@@ -53,23 +53,25 @@ class S3SourceDriver {
53
53
  }));
54
54
  (0, Affirm_1.default)(response.Body, 'Failed to fetch object from S3');
55
55
  const stream = response.Body;
56
- const chunks = [];
56
+ const reader = readline_1.default.createInterface({ input: stream, crlfDelay: Infinity });
57
+ const lines = [];
57
58
  try {
58
- for (var _d = true, stream_1 = __asyncValues(stream), stream_1_1; stream_1_1 = yield stream_1.next(), _a = stream_1_1.done, !_a; _d = true) {
59
- _c = stream_1_1.value;
59
+ for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
60
+ _c = reader_1_1.value;
60
61
  _d = false;
61
- const chunk = _c;
62
- chunks.push(chunk);
62
+ const line = _c;
63
+ lines.push(line);
63
64
  }
64
65
  }
65
66
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
66
67
  finally {
67
68
  try {
68
- if (!_d && !_a && (_b = stream_1.return)) yield _b.call(stream_1);
69
+ if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
69
70
  }
70
71
  finally { if (e_1) throw e_1.error; }
71
72
  }
72
- return Buffer.concat(chunks).toString('utf-8');
73
+ reader.close();
74
+ return lines;
73
75
  });
74
76
  this.readLinesInRange = (readOptions) => __awaiter(this, void 0, void 0, function* () {
75
77
  var _a, e_2, _b, _c;
@@ -87,8 +89,8 @@ class S3SourceDriver {
87
89
  const lines = [];
88
90
  let lineCounter = 0;
89
91
  try {
90
- for (var _d = true, reader_1 = __asyncValues(reader), reader_1_1; reader_1_1 = yield reader_1.next(), _a = reader_1_1.done, !_a; _d = true) {
91
- _c = reader_1_1.value;
92
+ for (var _d = true, reader_2 = __asyncValues(reader), reader_2_1; reader_2_1 = yield reader_2.next(), _a = reader_2_1.done, !_a; _d = true) {
93
+ _c = reader_2_1.value;
92
94
  _d = false;
93
95
  const line = _c;
94
96
  if (lineCounter >= lineFrom && lineCounter < lineTo)
@@ -101,7 +103,7 @@ class S3SourceDriver {
101
103
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
102
104
  finally {
103
105
  try {
104
- if (!_d && !_a && (_b = reader_1.return)) yield _b.call(reader_1);
106
+ if (!_d && !_a && (_b = reader_2.return)) yield _b.call(reader_2);
105
107
  }
106
108
  finally { if (e_2) throw e_2.error; }
107
109
  }
@@ -4,6 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
4
4
  };
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  const Affirm_1 = __importDefault(require("../core/Affirm"));
7
+ const Environment_1 = __importDefault(require("./Environment"));
8
+ const FileCompiler_1 = __importDefault(require("./file/FileCompiler"));
7
9
  class ParseManagerClass {
8
10
  constructor() {
9
11
  this.csvToJson = (csv, producer) => {
@@ -17,21 +19,36 @@ class ParseManagerClass {
17
19
  (0, Affirm_1.default)(lines, 'Invalid csv lines');
18
20
  Affirm_1.default.hasValue(lines.length, 'Invalid csv lines length');
19
21
  const delimiterChar = (_a = producer.settings.delimiter) !== null && _a !== void 0 ? _a : ',';
20
- const fileRows = lines;
21
- const columns = fileRows[0].split(delimiterChar).map(x => x.trim());
22
- const rows = fileRows.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
22
+ const rows = lines.slice(1).map(x => x.split(delimiterChar).map(k => k.trim()));
23
+ const headerColumns = this._extractHeader(lines[0], delimiterChar, producer);
23
24
  const result = [];
24
- for (let i = 0; i < rows.length; i++) {
25
- const row = rows[i];
25
+ for (const row of rows) {
26
26
  const rowObject = {};
27
- for (let j = 0; j < columns.length; j++) {
28
- const column = columns[j];
29
- rowObject[column] = row[j];
27
+ for (let i = 0; i < headerColumns.length; i++) {
28
+ const column = headerColumns[i];
29
+ rowObject[column.saveAs] = row[column.index];
30
30
  }
31
31
  result.push(rowObject);
32
32
  }
33
33
  return result;
34
34
  };
35
+ this._extractHeader = (headerLine, delimiter, producer) => {
36
+ var _a;
37
+ (0, Affirm_1.default)(headerLine, 'Invalid CSV header line');
38
+ (0, Affirm_1.default)(delimiter, 'Invalid CSV delimiter');
39
+ (0, Affirm_1.default)(producer, 'Invalid producer');
40
+ const source = Environment_1.default.getSource(producer.source);
41
+ const columns = FileCompiler_1.default.compileProducer(producer, source);
42
+ const headerColumns = headerLine.split(delimiter).map(x => x.trim());
43
+ const csvColumns = [];
44
+ for (const pColumn of columns) {
45
+ const columnKey = (_a = pColumn.aliasInProducer) !== null && _a !== void 0 ? _a : pColumn.nameInProducer;
46
+ const csvColumnIndex = headerColumns.findIndex(x => x === columnKey);
47
+ (0, Affirm_1.default)(csvColumnIndex > -1, `The column "${pColumn.nameInProducer}" (with key "${columnKey}") of producer "${producer.name}" doesn't exist in the underlying dataset.`);
48
+ csvColumns.push({ index: csvColumnIndex, name: columnKey, saveAs: pColumn.nameInProducer });
49
+ }
50
+ return csvColumns;
51
+ };
35
52
  }
36
53
  }
37
54
  const ParseManager = new ParseManagerClass();
@@ -95,7 +95,7 @@ class ProducerEngineClass {
95
95
  if (options.readmode === 'lines')
96
96
  lines = yield driver.readLinesInRange({ fileKey: producer.settings.fileKey, lineFrom: options.lines.from, lineTo: options.lines.to });
97
97
  else
98
- lines = [(yield driver.download({ fileKey: producer.settings.fileKey }))];
98
+ lines = yield driver.download({ fileKey: producer.settings.fileKey });
99
99
  switch ((_a = producer.settings.fileType) === null || _a === void 0 ? void 0 : _a.toUpperCase()) {
100
100
  case 'CSV': {
101
101
  return { data: lines, dataType: 'lines-of-text' };
package/engines/ai/LLM.js CHANGED
@@ -263,46 +263,7 @@ class LLM {
263
263
  };
264
264
  const res = yield this._client.beta.chat.completions.parse(item);
265
265
  const msg = res.choices[0].message;
266
- const firstDraft = msg.parsed;
267
- const qaSystemPrompt = baseQASystemPrompt.replace('{{consumers}}', firstDraft.consumers.map(x => `- ${JSON.stringify(x)}`).join('\n'));
268
- const res2 = yield this._client.beta.chat.completions.parse({
269
- model: 'gpt-4o',
270
- messages: [
271
- { role: 'system', content: qaSystemPrompt }
272
- ],
273
- response_format: (0, zod_1.zodResponseFormat)(zod_2.z.object({
274
- consumers: zod_2.z.array(zod_2.z.object({
275
- $schema: zod_2.z.string().describe('The schema of the consumer. This should always be the same.'),
276
- name: zod_2.z.string(),
277
- description: zod_2.z.string(),
278
- producers: zod_2.z.array(zod_2.z.object({
279
- name: zod_2.z.string().describe('References one of the producers. Must be unique, there can\'t be two entry with the same name.'),
280
- joins: zod_2.z.array(zod_2.z.object({
281
- otherName: zod_2.z.string(),
282
- relationship: zod_2.z.enum(['one-to-one', 'one-to-many', 'many-to-one']),
283
- sql: zod_2.z.string()
284
- })).optional().describe('Which other producer to join this one with. Omit if empty.')
285
- })),
286
- fields: zod_2.z.array(zod_2.z.object({
287
- key: zod_2.z.string(),
288
- from: zod_2.z.string().optional(),
289
- alias: zod_2.z.string().optional()
290
- // grouping: z.object({
291
- // groupingKey: z.string().optional(),
292
- // subFields: z.array(z.lazy(() => z.object({
293
- // key: z.string(),
294
- // from: z.string().optional()
295
- // }))).optional()
296
- // }).optional()
297
- })),
298
- outputs: zod_2.z.array(zod_2.z.object({
299
- format: zod_2.z.enum(['SQL', 'API', 'CSV', 'PARQUET', 'JSON'])
300
- }))
301
- }))
302
- }), 'environment')
303
- });
304
- const msg2 = res2.choices[0].message;
305
- const finalDraft = msg2.parsed;
266
+ const finalDraft = msg.parsed;
306
267
  // Do some manual adjustments cause some things still don't work...
307
268
  if (finalDraft && finalDraft.consumers) {
308
269
  for (const cons of finalDraft.consumers) {
@@ -113,7 +113,6 @@ class ConsumerManagerClass {
113
113
  else {
114
114
  const col = ConsumerManager.searchFieldInColumns(field.cField, availableColumns, consumer);
115
115
  (0, Affirm_1.default)(col, `Consumer "${consumer.name}" misconfiguration: the requested field "${field.cField.key}" is not found in any of the specified producers ("${consumer.producers.map(x => x.name).join(', ')}")`);
116
- // TODO: CHECK THIS FIX IS GOOD
117
116
  expandedFields.push(Object.assign(Object.assign({}, field), { dimension: col.dimension, measure: col.measure }));
118
117
  }
119
118
  return expandedFields;
@@ -8,12 +8,15 @@ const Algo_1 = __importDefault(require("../../core/Algo"));
8
8
  const CryptoEngine_1 = __importDefault(require("../CryptoEngine"));
9
9
  const Environment_1 = __importDefault(require("../Environment"));
10
10
  const FileCompiler_1 = __importDefault(require("../file/FileCompiler"));
11
+ const TypeCaster_1 = __importDefault(require("../transform/TypeCaster"));
11
12
  const ConsumerManager_1 = __importDefault(require("./ConsumerManager"));
12
13
  class PostProcessorClass {
13
14
  constructor() {
14
15
  /**
15
- * Get an array of objects and projects it to another array of objects but with different shape
16
- * Basically does a field mapping with some extra logic (like grouping and creating nested fields)
16
+ * Maps an array of objects and projects it to another array of objects but with different shape:
17
+ * - grouping (creates nested fields)
18
+ * - type casting
19
+ * - default field values
17
20
  */
18
21
  this.doProjection = (consumer, data) => {
19
22
  (0, Affirm_1.default)(consumer, 'Invalid consumer');
@@ -52,13 +55,18 @@ class PostProcessorClass {
52
55
  }
53
56
  else {
54
57
  return items.map(x => {
55
- var _a;
58
+ var _a, _b, _c;
56
59
  const projected = {};
57
60
  for (const field of allFields) {
58
61
  const { key, alias } = field.cField;
59
62
  const fieldKey = alias !== null && alias !== void 0 ? alias : key;
60
63
  const maskType = (_a = field.dimension) === null || _a === void 0 ? void 0 : _a.mask;
61
- projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, x[key]);
64
+ const fieldType = (_c = (_b = field.dimension) === null || _b === void 0 ? void 0 : _b.type) !== null && _c !== void 0 ? _c : 'string';
65
+ const fieldValue = this._getFieldValue(x, field);
66
+ if (Algo_1.default.hasVal(maskType))
67
+ projected[fieldKey] = CryptoEngine_1.default.hashValue(maskType, fieldValue);
68
+ else
69
+ projected[fieldKey] = TypeCaster_1.default.cast(fieldValue, fieldType);
62
70
  }
63
71
  return projected;
64
72
  });
@@ -136,6 +144,15 @@ class PostProcessorClass {
136
144
  const unpackedData = data.flatMap(x => unpackSingle(x));
137
145
  return unpackedData;
138
146
  };
147
+ this._getFieldValue = (record, field) => {
148
+ const fieldValue = record[field.cField.key];
149
+ if (Algo_1.default.hasVal(fieldValue))
150
+ return fieldValue;
151
+ else if (!Algo_1.default.hasVal(fieldValue) && Algo_1.default.hasVal(field.cField.default))
152
+ return field.cField.default;
153
+ else
154
+ return fieldValue;
155
+ };
139
156
  }
140
157
  }
141
158
  const PostProcessor = new PostProcessorClass();
@@ -17,7 +17,7 @@ class FileCompilerClass {
17
17
  nameInProducer: x.name,
18
18
  consumerAlias: null,
19
19
  consumerKey: null,
20
- owner: x.name,
20
+ owner: producer.name,
21
21
  dimension: x
22
22
  }));
23
23
  return columns;
@@ -24,7 +24,7 @@ class TransformationEngineClass {
24
24
  else if (!Algo_1.default.hasVal(value))
25
25
  continue;
26
26
  try {
27
- record[field.key] = this.applyTransformations(value, field.transform, field.key);
27
+ record[field.key] = this.applyTransformations(value, field.transform, field);
28
28
  }
29
29
  catch (error) {
30
30
  switch (field.onError) {
@@ -42,36 +42,39 @@ class TransformationEngineClass {
42
42
  }
43
43
  return data;
44
44
  };
45
- this.applyTransformations = (value, transformations, fieldName) => {
45
+ this.applyTransformations = (value, transformations, field) => {
46
46
  var _a;
47
47
  if (Array.isArray(transformations)) {
48
48
  // Process array transformations without creating intermediate arrays
49
49
  let result = value;
50
50
  for (const transform of transformations) {
51
- result = this.applyTransformations(result, transform, fieldName);
51
+ result = this.applyTransformations(result, transform, field);
52
52
  }
53
53
  return result;
54
54
  }
55
55
  // Single transformation
56
56
  if ('cast' in transformations) {
57
- return TypeCaster_1.default.cast(value, transformations.cast);
57
+ const casted = TypeCaster_1.default.cast(value, transformations.cast);
58
+ if (isNaN(casted) && transformations.cast === 'number')
59
+ throw new Error(`Cannot cast non-numeric value in field '${field.key}'`);
60
+ return casted;
58
61
  }
59
62
  if ('multiply' in transformations) {
60
63
  const num = TypeCaster_1.default.cast(value, 'number');
61
64
  if (isNaN(num))
62
- throw new Error(`Cannot multiply non-numeric value in field '${fieldName}'`);
65
+ throw new Error(`Cannot multiply non-numeric value in field '${field.key}'`);
63
66
  return num * transformations.multiply;
64
67
  }
65
68
  if ('add' in transformations) {
66
69
  const num = TypeCaster_1.default.cast(value, 'number');
67
70
  if (isNaN(num))
68
- throw new Error(`Cannot add to non-numeric value in field '${fieldName}'`);
71
+ throw new Error(`Cannot add to non-numeric value in field '${field.key}'`);
69
72
  return num + transformations.add;
70
73
  }
71
74
  if ('extract' in transformations) {
72
75
  const date = TypeCaster_1.default.cast(value, 'date');
73
76
  if (isNaN(date.getTime()))
74
- throw new Error(`Invalid date for extraction in field '${fieldName}'`);
77
+ throw new Error(`Invalid date for extraction in field '${field.key}'`);
75
78
  switch (transformations.extract) {
76
79
  case 'year': return date.getFullYear();
77
80
  case 'month': return date.getMonth() + 1; // 1-based month
@@ -82,43 +85,43 @@ class TransformationEngineClass {
82
85
  }
83
86
  if ('concat' in transformations) {
84
87
  if (!Array.isArray(value))
85
- throw new Error(`Cannot concat non-array value in field '${fieldName}'`);
88
+ throw new Error(`Cannot concat non-array value in field '${field.key}'`);
86
89
  return value.join(transformations.concat.separator);
87
90
  }
88
91
  if ('split' in transformations) {
89
92
  if (typeof value !== 'string')
90
- throw new Error(`Cannot split non-string value in field '${fieldName}'`);
93
+ throw new Error(`Cannot split non-string value in field '${field.key}'`);
91
94
  const parts = value.split(transformations.split.separator);
92
95
  if (transformations.split.index >= parts.length) {
93
- throw new Error(`Split index ${transformations.split.index} out of bounds in field '${fieldName}'`);
96
+ throw new Error(`Split index ${transformations.split.index} out of bounds in field '${field.key}'`);
94
97
  }
95
98
  return parts[transformations.split.index];
96
99
  }
97
100
  if ('regex_match' in transformations) {
98
101
  if (typeof value !== 'string')
99
- throw new Error(`Cannot apply regex_match to non-string value in field '${fieldName}'`);
102
+ throw new Error(`Cannot apply regex_match to non-string value in field '${field.key}'`);
100
103
  try {
101
104
  const regex = new RegExp(transformations.regex_match.pattern, transformations.regex_match.flags);
102
105
  return regex.test(value);
103
106
  }
104
107
  catch (error) {
105
- throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
108
+ throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
106
109
  }
107
110
  }
108
111
  if ('regex_replace' in transformations) {
109
112
  if (typeof value !== 'string')
110
- throw new Error(`Cannot apply regex_replace to non-string value in field '${fieldName}'`);
113
+ throw new Error(`Cannot apply regex_replace to non-string value in field '${field.key}'`);
111
114
  try {
112
115
  const regex = new RegExp(transformations.regex_replace.pattern, transformations.regex_replace.flags);
113
116
  return value.replace(regex, transformations.regex_replace.replacement);
114
117
  }
115
118
  catch (error) {
116
- throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
119
+ throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
117
120
  }
118
121
  }
119
122
  if ('regex_extract' in transformations) {
120
123
  if (typeof value !== 'string')
121
- throw new Error(`Cannot apply regex_extract to non-string value in field '${fieldName}'`);
124
+ throw new Error(`Cannot apply regex_extract to non-string value in field '${field.key}'`);
122
125
  try {
123
126
  const regex = new RegExp(transformations.regex_extract.pattern, transformations.regex_extract.flags);
124
127
  const matches = value.match(regex);
@@ -128,49 +131,49 @@ class TransformationEngineClass {
128
131
  return (_a = matches[groupIndex]) !== null && _a !== void 0 ? _a : null;
129
132
  }
130
133
  catch (error) {
131
- throw new Error(`Invalid regex pattern in field '${fieldName}': ${error.message}`);
134
+ throw new Error(`Invalid regex pattern in field '${field.key}': ${error.message}`);
132
135
  }
133
136
  }
134
137
  if ('trim' in transformations) {
135
138
  if (typeof value !== 'string')
136
- throw new Error(`Cannot trim non-string value in field '${fieldName}'`);
139
+ throw new Error(`Cannot trim non-string value in field '${field.key}'`);
137
140
  return value.trim();
138
141
  }
139
142
  if ('to_lowercase' in transformations) {
140
143
  if (typeof value !== 'string')
141
- throw new Error(`Cannot convert non-string value to lowercase in field '${fieldName}'`);
144
+ throw new Error(`Cannot convert non-string value to lowercase in field '${field.key}'`);
142
145
  return value.toLowerCase();
143
146
  }
144
147
  if ('to_uppercase' in transformations) {
145
148
  if (typeof value !== 'string')
146
- throw new Error(`Cannot convert non-string value to uppercase in field '${fieldName}'`);
149
+ throw new Error(`Cannot convert non-string value to uppercase in field '${field.key}'`);
147
150
  return value.toUpperCase();
148
151
  }
149
152
  if ('capitalize' in transformations) {
150
153
  if (typeof value !== 'string')
151
- throw new Error(`Cannot capitalize non-string value in field '${fieldName}'`);
154
+ throw new Error(`Cannot capitalize non-string value in field '${field.key}'`);
152
155
  return value.charAt(0).toUpperCase() + value.slice(1);
153
156
  }
154
157
  if ('substring' in transformations) {
155
158
  if (typeof value !== 'string')
156
- throw new Error(`Cannot take substring of non-string value in field '${fieldName}'`);
159
+ throw new Error(`Cannot take substring of non-string value in field '${field.key}'`);
157
160
  const { start, end } = transformations.substring;
158
161
  return end !== undefined ? value.substring(start, end) : value.substring(start);
159
162
  }
160
163
  if ('pad_start' in transformations) {
161
164
  if (typeof value !== 'string')
162
- throw new Error(`Cannot pad non-string value in field '${fieldName}'`);
165
+ throw new Error(`Cannot pad non-string value in field '${field.key}'`);
163
166
  const { length, char } = transformations.pad_start;
164
167
  if (char.length !== 1)
165
- throw new Error(`Pad character must be exactly one character in field '${fieldName}'`);
168
+ throw new Error(`Pad character must be exactly one character in field '${field.key}'`);
166
169
  return value.padStart(length, char);
167
170
  }
168
171
  if ('pad_end' in transformations) {
169
172
  if (typeof value !== 'string')
170
- throw new Error(`Cannot pad non-string value in field '${fieldName}'`);
173
+ throw new Error(`Cannot pad non-string value in field '${field.key}'`);
171
174
  const { length, char } = transformations.pad_end;
172
175
  if (char.length !== 1)
173
- throw new Error(`Pad character must be exactly one character in field '${fieldName}'`);
176
+ throw new Error(`Pad character must be exactly one character in field '${field.key}'`);
174
177
  return value.padEnd(length, char);
175
178
  }
176
179
  if ('prepend' in transformations)
@@ -12,6 +12,7 @@ class TypeCasterClass {
12
12
  else
13
13
  return Boolean(value);
14
14
  }
15
+ case 'datetime':
15
16
  case 'date':
16
17
  return new Date(value);
17
18
  case 'number': {
@@ -130,9 +130,16 @@ class ValidatorClass {
130
130
  const errors = [];
131
131
  const trxsFields = fields.filter(x => x.transform);
132
132
  for (const field of trxsFields) {
133
- const transformations = Object.keys(field.transform);
134
- if (transformations.length !== 1)
135
- errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${transformations.length}`);
133
+ const trxToValidate = [];
134
+ if (Array.isArray(field.transform))
135
+ trxToValidate.push(...field.transform);
136
+ else
137
+ trxToValidate.push(field.transform);
138
+ for (const trans of trxToValidate) {
139
+ const trxKeys = Object.keys(trans);
140
+ if (trxKeys.length !== 1)
141
+ errors.push(`There can only be 1 transformation type in your transformation pipeline. Field "${field.key}" got ${trxKeys.length}`);
142
+ }
136
143
  }
137
144
  return errors;
138
145
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@forzalabs/remora",
3
- "version": "0.0.21",
3
+ "version": "0.0.23",
4
4
  "description": "A powerful CLI tool for seamless data translation.",
5
5
  "main": "index.js",
6
6
  "private": false,