@forzalabs/remora 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/index.js +9815 -9365
- package/json_schemas/producer-schema.json +132 -47
- package/package.json +5 -8
- package/workers/ExecutorWorker.js +14708 -14269
|
@@ -48,11 +48,6 @@
|
|
|
48
48
|
"type": "string",
|
|
49
49
|
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
50
50
|
},
|
|
51
|
-
"decimals": {
|
|
52
|
-
"type": "integer",
|
|
53
|
-
"minimum": 0,
|
|
54
|
-
"description": "How many decimal places generated number values should have. Used only by synthetic-data generation ('synth'): 0 forces whole integers (so values load into an integer column); a positive value rounds to that many decimals. Omit to keep the generator's own value."
|
|
55
|
-
},
|
|
56
51
|
"alias": {
|
|
57
52
|
"type": "string",
|
|
58
53
|
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
@@ -94,68 +89,119 @@
|
|
|
94
89
|
"type": "boolean",
|
|
95
90
|
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
96
91
|
},
|
|
97
|
-
"
|
|
98
|
-
"type": "
|
|
99
|
-
"
|
|
100
|
-
"
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
92
|
+
"position": {
|
|
93
|
+
"type": "object",
|
|
94
|
+
"description": "Where this dimension's value sits inside the line, for FIXED (fixed-width) files. Required on every dimension of a FIXED producer and only valid there: a fixed-width file has no header and no delimiter, so the position IS the mapping. Coordinates are 1-based and inclusive, like a file layout spec ('columns 21-30').",
|
|
95
|
+
"additionalProperties": false,
|
|
96
|
+
"required": ["start"],
|
|
97
|
+
"properties": {
|
|
98
|
+
"start": {
|
|
99
|
+
"type": "integer",
|
|
100
|
+
"minimum": 1,
|
|
101
|
+
"description": "1-based position of the field's first character in the line."
|
|
102
|
+
},
|
|
103
|
+
"length": {
|
|
104
|
+
"type": "integer",
|
|
105
|
+
"minimum": 1,
|
|
106
|
+
"description": "How many characters the field spans. Mutually exclusive with 'end'."
|
|
107
|
+
},
|
|
108
|
+
"end": {
|
|
109
|
+
"type": "integer",
|
|
110
|
+
"minimum": 1,
|
|
111
|
+
"description": "1-based position of the field's last character, inclusive. Mutually exclusive with 'length'."
|
|
112
|
+
}
|
|
113
|
+
}
|
|
104
114
|
},
|
|
105
|
-
"
|
|
106
|
-
"
|
|
107
|
-
"
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
115
|
+
"synth": {
|
|
116
|
+
"type": "object",
|
|
117
|
+
"description": "Synthetic-data configuration for this dimension, used only by the 'synth' command. Nothing in here affects a real producer read. 'allowedValues', 'codeSet' and 'references' are mutually exclusive ways to declare where a value comes from.",
|
|
118
|
+
"additionalProperties": false,
|
|
119
|
+
"properties": {
|
|
120
|
+
"decimals": {
|
|
121
|
+
"type": "integer",
|
|
122
|
+
"minimum": 0,
|
|
123
|
+
"description": "How many decimal places generated number values should have: 0 forces whole integers (so values load into an integer column); a positive value rounds to that many decimals. Omit to keep the generator's own value."
|
|
124
|
+
},
|
|
125
|
+
"allowedValues": {
|
|
126
|
+
"type": "array",
|
|
127
|
+
"minItems": 1,
|
|
128
|
+
"items": {
|
|
129
|
+
"type": ["string", "number", "boolean"]
|
|
130
|
+
},
|
|
131
|
+
"description": "Inline closed set of allowed values: one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
132
|
+
},
|
|
133
|
+
"codeSet": {
|
|
134
|
+
"description": "Reference to a named code set. A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
135
|
+
"oneOf": [
|
|
136
|
+
{
|
|
137
|
+
"type": "string",
|
|
138
|
+
"minLength": 1
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"type": "object",
|
|
142
|
+
"required": ["name", "field"],
|
|
143
|
+
"additionalProperties": false,
|
|
144
|
+
"properties": {
|
|
145
|
+
"name": {
|
|
146
|
+
"type": "string",
|
|
147
|
+
"minLength": 1,
|
|
148
|
+
"description": "Name of a structured (records) code set."
|
|
149
|
+
},
|
|
150
|
+
"field": {
|
|
151
|
+
"type": "string",
|
|
152
|
+
"minLength": 1,
|
|
153
|
+
"description": "Field of the picked record to emit for this dimension."
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
]
|
|
111
158
|
},
|
|
112
|
-
{
|
|
159
|
+
"references": {
|
|
113
160
|
"type": "object",
|
|
114
|
-
"
|
|
161
|
+
"description": "Declares this dimension as a foreign key referencing another producer's field. Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
162
|
+
"required": ["producer", "field"],
|
|
115
163
|
"additionalProperties": false,
|
|
116
164
|
"properties": {
|
|
117
|
-
"
|
|
165
|
+
"producer": {
|
|
118
166
|
"type": "string",
|
|
119
167
|
"minLength": 1,
|
|
120
|
-
"description": "Name of
|
|
168
|
+
"description": "Name of the parent producer whose field this dimension references."
|
|
121
169
|
},
|
|
122
170
|
"field": {
|
|
123
171
|
"type": "string",
|
|
124
172
|
"minLength": 1,
|
|
125
|
-
"description": "
|
|
173
|
+
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
126
174
|
}
|
|
127
175
|
}
|
|
128
176
|
}
|
|
177
|
+
},
|
|
178
|
+
"allOf": [
|
|
179
|
+
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
180
|
+
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
181
|
+
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
129
182
|
]
|
|
130
183
|
},
|
|
184
|
+
"decimals": {
|
|
185
|
+
"not": {},
|
|
186
|
+
"description": "MOVED: use \"synth\": { \"decimals\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
187
|
+
},
|
|
188
|
+
"allowedValues": {
|
|
189
|
+
"not": {},
|
|
190
|
+
"description": "MOVED: use \"synth\": { \"allowedValues\": [...] } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
191
|
+
},
|
|
192
|
+
"codeSet": {
|
|
193
|
+
"not": {},
|
|
194
|
+
"description": "MOVED: use \"synth\": { \"codeSet\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
195
|
+
},
|
|
131
196
|
"references": {
|
|
132
|
-
"
|
|
133
|
-
"description": "
|
|
134
|
-
"required": ["producer", "field"],
|
|
135
|
-
"additionalProperties": false,
|
|
136
|
-
"properties": {
|
|
137
|
-
"producer": {
|
|
138
|
-
"type": "string",
|
|
139
|
-
"minLength": 1,
|
|
140
|
-
"description": "Name of the parent producer whose field this dimension references."
|
|
141
|
-
},
|
|
142
|
-
"field": {
|
|
143
|
-
"type": "string",
|
|
144
|
-
"minLength": 1,
|
|
145
|
-
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
146
|
-
}
|
|
147
|
-
}
|
|
197
|
+
"not": {},
|
|
198
|
+
"description": "MOVED: use \"synth\": { \"references\": { \"producer\", \"field\" } } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
148
199
|
}
|
|
149
200
|
},
|
|
150
201
|
"required": [
|
|
151
202
|
"name",
|
|
152
203
|
"type"
|
|
153
204
|
],
|
|
154
|
-
"allOf": [
|
|
155
|
-
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
156
|
-
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
157
|
-
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
158
|
-
],
|
|
159
205
|
"additionalProperties": false
|
|
160
206
|
},
|
|
161
207
|
"minItems": 1
|
|
@@ -216,21 +262,60 @@
|
|
|
216
262
|
"XLS",
|
|
217
263
|
"XLSX",
|
|
218
264
|
"XML",
|
|
219
|
-
"PARQUET"
|
|
265
|
+
"PARQUET",
|
|
266
|
+
"FIXED"
|
|
220
267
|
],
|
|
221
|
-
"description": "The type of file to read."
|
|
268
|
+
"description": "The type of file to read. FIXED is a fixed-width (positional) flat file: no delimiter and no header, every dimension declares its 'position' in the line."
|
|
222
269
|
},
|
|
223
270
|
"delimiter": {
|
|
224
271
|
"type": "string",
|
|
225
272
|
"description": "The column delimiter for CSV or TXT files if different from the default (,)."
|
|
226
273
|
},
|
|
274
|
+
"recordSelector": {
|
|
275
|
+
"type": "object",
|
|
276
|
+
"description": "For FIXED files that interleave record types (e.g. H header / D detail / T trailer): reads only the lines whose marker at the given position matches, so this producer describes exactly one record layout. If omitted, every line is read.",
|
|
277
|
+
"additionalProperties": false,
|
|
278
|
+
"required": ["position", "equals"],
|
|
279
|
+
"properties": {
|
|
280
|
+
"position": {
|
|
281
|
+
"type": "object",
|
|
282
|
+
"description": "Where the record-type marker sits in the line. Same 1-based, inclusive coordinates as a dimension position.",
|
|
283
|
+
"additionalProperties": false,
|
|
284
|
+
"required": ["start"],
|
|
285
|
+
"properties": {
|
|
286
|
+
"start": {
|
|
287
|
+
"type": "integer",
|
|
288
|
+
"minimum": 1,
|
|
289
|
+
"description": "1-based position of the marker's first character in the line."
|
|
290
|
+
},
|
|
291
|
+
"length": {
|
|
292
|
+
"type": "integer",
|
|
293
|
+
"minimum": 1,
|
|
294
|
+
"description": "How many characters the marker spans. Mutually exclusive with 'end'."
|
|
295
|
+
},
|
|
296
|
+
"end": {
|
|
297
|
+
"type": "integer",
|
|
298
|
+
"minimum": 1,
|
|
299
|
+
"description": "1-based position of the marker's last character, inclusive. Mutually exclusive with 'length'."
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
"equals": {
|
|
304
|
+
"description": "The marker value(s) to keep, compared case-sensitively against the trimmed marker.",
|
|
305
|
+
"oneOf": [
|
|
306
|
+
{ "type": "string", "minLength": 1 },
|
|
307
|
+
{ "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
|
|
308
|
+
]
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
},
|
|
227
312
|
"disableHistory": {
|
|
228
313
|
"type": "boolean",
|
|
229
314
|
"description": "For Delta Share producers, skips sending the current table version in the query request body."
|
|
230
315
|
},
|
|
231
316
|
"hasHeaderRow": {
|
|
232
317
|
"type": "boolean",
|
|
233
|
-
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
|
|
318
|
+
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true. For FIXED files it defaults to false — set it to true only if the fixed-width file starts with a descriptive line that must be skipped."
|
|
234
319
|
},
|
|
235
320
|
"sheetName": {
|
|
236
321
|
"type": "string",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -43,20 +43,18 @@
|
|
|
43
43
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
44
44
|
"@aws-sdk/client-s3": "^3.701.0",
|
|
45
45
|
"@aws-sdk/client-sqs": "^3.886.0",
|
|
46
|
-
"adm-zip": "^0.
|
|
46
|
+
"adm-zip": "^0.6.0",
|
|
47
47
|
"ajv": "^8.17.1",
|
|
48
48
|
"ajv-formats": "^3.0.1",
|
|
49
|
-
"bcrypt": "^5.1.1",
|
|
50
49
|
"bcryptjs": "^3.0.2",
|
|
51
50
|
"chalk": "^4.1.2",
|
|
52
51
|
"commander": "^10.0.0",
|
|
53
52
|
"cors": "^2.8.5",
|
|
54
53
|
"cross-env": "^7.0.3",
|
|
55
54
|
"dayjs": "^1.11.13",
|
|
56
|
-
"decompress": "^4.2.1",
|
|
57
55
|
"dotenv": "^16.0.3",
|
|
58
56
|
"express": "^4.21.2",
|
|
59
|
-
"fast-xml-parser": "^5.
|
|
57
|
+
"fast-xml-parser": "^5.3.5",
|
|
60
58
|
"fs-extra": "^11.1.0",
|
|
61
59
|
"hyparquet": "^1.17.4",
|
|
62
60
|
"inquirer": "^8.2.5",
|
|
@@ -64,8 +62,7 @@
|
|
|
64
62
|
"jsonwebtoken": "^9.0.2",
|
|
65
63
|
"knex": "^2.4.2",
|
|
66
64
|
"mongodb": "^6.15.0",
|
|
67
|
-
"multer": "^
|
|
68
|
-
"next": "^13.4.1",
|
|
65
|
+
"multer": "^2.0.0",
|
|
69
66
|
"node-cron": "^4.2.1",
|
|
70
67
|
"openai": "^6.0.0",
|
|
71
68
|
"ora": "^5.4.1",
|
|
@@ -75,7 +72,7 @@
|
|
|
75
72
|
"seedrandom": "^3.0.5",
|
|
76
73
|
"uuid": "^13.0.0",
|
|
77
74
|
"workerpool": "^9.3.3",
|
|
78
|
-
"xlsx": "^0.
|
|
75
|
+
"@e965/xlsx": "^0.20.3",
|
|
79
76
|
"zod": "^3.24.2"
|
|
80
77
|
}
|
|
81
78
|
}
|