@forzalabs/remora 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/index.js +9825 -9365
- package/json_schemas/producer-schema.json +132 -42
- package/package.json +5 -8
- package/workers/ExecutorWorker.js +14719 -14270
|
@@ -89,68 +89,119 @@
|
|
|
89
89
|
"type": "boolean",
|
|
90
90
|
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
91
91
|
},
|
|
92
|
-
"
|
|
93
|
-
"type": "
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
92
|
+
"position": {
|
|
93
|
+
"type": "object",
|
|
94
|
+
"description": "Where this dimension's value sits inside the line, for FIXED (fixed-width) files. Required on every dimension of a FIXED producer and only valid there: a fixed-width file has no header and no delimiter, so the position IS the mapping. Coordinates are 1-based and inclusive, like a file layout spec ('columns 21-30').",
|
|
95
|
+
"additionalProperties": false,
|
|
96
|
+
"required": ["start"],
|
|
97
|
+
"properties": {
|
|
98
|
+
"start": {
|
|
99
|
+
"type": "integer",
|
|
100
|
+
"minimum": 1,
|
|
101
|
+
"description": "1-based position of the field's first character in the line."
|
|
102
|
+
},
|
|
103
|
+
"length": {
|
|
104
|
+
"type": "integer",
|
|
105
|
+
"minimum": 1,
|
|
106
|
+
"description": "How many characters the field spans. Mutually exclusive with 'end'."
|
|
107
|
+
},
|
|
108
|
+
"end": {
|
|
109
|
+
"type": "integer",
|
|
110
|
+
"minimum": 1,
|
|
111
|
+
"description": "1-based position of the field's last character, inclusive. Mutually exclusive with 'length'."
|
|
112
|
+
}
|
|
113
|
+
}
|
|
99
114
|
},
|
|
100
|
-
"
|
|
101
|
-
"
|
|
102
|
-
"
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
115
|
+
"synth": {
|
|
116
|
+
"type": "object",
|
|
117
|
+
"description": "Synthetic-data configuration for this dimension, used only by the 'synth' command. Nothing in here affects a real producer read. 'allowedValues', 'codeSet' and 'references' are mutually exclusive ways to declare where a value comes from.",
|
|
118
|
+
"additionalProperties": false,
|
|
119
|
+
"properties": {
|
|
120
|
+
"decimals": {
|
|
121
|
+
"type": "integer",
|
|
122
|
+
"minimum": 0,
|
|
123
|
+
"description": "How many decimal places generated number values should have: 0 forces whole integers (so values load into an integer column); a positive value rounds to that many decimals. Omit to keep the generator's own value."
|
|
124
|
+
},
|
|
125
|
+
"allowedValues": {
|
|
126
|
+
"type": "array",
|
|
127
|
+
"minItems": 1,
|
|
128
|
+
"items": {
|
|
129
|
+
"type": ["string", "number", "boolean"]
|
|
130
|
+
},
|
|
131
|
+
"description": "Inline closed set of allowed values: one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
106
132
|
},
|
|
107
|
-
{
|
|
133
|
+
"codeSet": {
|
|
134
|
+
"description": "Reference to a named code set. A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
135
|
+
"oneOf": [
|
|
136
|
+
{
|
|
137
|
+
"type": "string",
|
|
138
|
+
"minLength": 1
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"type": "object",
|
|
142
|
+
"required": ["name", "field"],
|
|
143
|
+
"additionalProperties": false,
|
|
144
|
+
"properties": {
|
|
145
|
+
"name": {
|
|
146
|
+
"type": "string",
|
|
147
|
+
"minLength": 1,
|
|
148
|
+
"description": "Name of a structured (records) code set."
|
|
149
|
+
},
|
|
150
|
+
"field": {
|
|
151
|
+
"type": "string",
|
|
152
|
+
"minLength": 1,
|
|
153
|
+
"description": "Field of the picked record to emit for this dimension."
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
]
|
|
158
|
+
},
|
|
159
|
+
"references": {
|
|
108
160
|
"type": "object",
|
|
109
|
-
"
|
|
161
|
+
"description": "Declares this dimension as a foreign key referencing another producer's field. Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
162
|
+
"required": ["producer", "field"],
|
|
110
163
|
"additionalProperties": false,
|
|
111
164
|
"properties": {
|
|
112
|
-
"
|
|
165
|
+
"producer": {
|
|
113
166
|
"type": "string",
|
|
114
167
|
"minLength": 1,
|
|
115
|
-
"description": "Name of
|
|
168
|
+
"description": "Name of the parent producer whose field this dimension references."
|
|
116
169
|
},
|
|
117
170
|
"field": {
|
|
118
171
|
"type": "string",
|
|
119
172
|
"minLength": 1,
|
|
120
|
-
"description": "
|
|
173
|
+
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
121
174
|
}
|
|
122
175
|
}
|
|
123
176
|
}
|
|
177
|
+
},
|
|
178
|
+
"allOf": [
|
|
179
|
+
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
180
|
+
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
181
|
+
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
124
182
|
]
|
|
125
183
|
},
|
|
184
|
+
"decimals": {
|
|
185
|
+
"not": {},
|
|
186
|
+
"description": "MOVED: use \"synth\": { \"decimals\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
187
|
+
},
|
|
188
|
+
"allowedValues": {
|
|
189
|
+
"not": {},
|
|
190
|
+
"description": "MOVED: use \"synth\": { \"allowedValues\": [...] } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
191
|
+
},
|
|
192
|
+
"codeSet": {
|
|
193
|
+
"not": {},
|
|
194
|
+
"description": "MOVED: use \"synth\": { \"codeSet\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
195
|
+
},
|
|
126
196
|
"references": {
|
|
127
|
-
"
|
|
128
|
-
"description": "
|
|
129
|
-
"required": ["producer", "field"],
|
|
130
|
-
"additionalProperties": false,
|
|
131
|
-
"properties": {
|
|
132
|
-
"producer": {
|
|
133
|
-
"type": "string",
|
|
134
|
-
"minLength": 1,
|
|
135
|
-
"description": "Name of the parent producer whose field this dimension references."
|
|
136
|
-
},
|
|
137
|
-
"field": {
|
|
138
|
-
"type": "string",
|
|
139
|
-
"minLength": 1,
|
|
140
|
-
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
141
|
-
}
|
|
142
|
-
}
|
|
197
|
+
"not": {},
|
|
198
|
+
"description": "MOVED: use \"synth\": { \"references\": { \"producer\", \"field\" } } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
143
199
|
}
|
|
144
200
|
},
|
|
145
201
|
"required": [
|
|
146
202
|
"name",
|
|
147
203
|
"type"
|
|
148
204
|
],
|
|
149
|
-
"allOf": [
|
|
150
|
-
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
151
|
-
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
152
|
-
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
153
|
-
],
|
|
154
205
|
"additionalProperties": false
|
|
155
206
|
},
|
|
156
207
|
"minItems": 1
|
|
@@ -211,21 +262,60 @@
|
|
|
211
262
|
"XLS",
|
|
212
263
|
"XLSX",
|
|
213
264
|
"XML",
|
|
214
|
-
"PARQUET"
|
|
265
|
+
"PARQUET",
|
|
266
|
+
"FIXED"
|
|
215
267
|
],
|
|
216
|
-
"description": "The type of file to read."
|
|
268
|
+
"description": "The type of file to read. FIXED is a fixed-width (positional) flat file: no delimiter and no header, every dimension declares its 'position' in the line."
|
|
217
269
|
},
|
|
218
270
|
"delimiter": {
|
|
219
271
|
"type": "string",
|
|
220
272
|
"description": "The column delimiter for CSV or TXT files if different from the default (,)."
|
|
221
273
|
},
|
|
274
|
+
"recordSelector": {
|
|
275
|
+
"type": "object",
|
|
276
|
+
"description": "For FIXED files that interleave record types (e.g. H header / D detail / T trailer): reads only the lines whose marker at the given position matches, so this producer describes exactly one record layout. If omitted, every line is read.",
|
|
277
|
+
"additionalProperties": false,
|
|
278
|
+
"required": ["position", "equals"],
|
|
279
|
+
"properties": {
|
|
280
|
+
"position": {
|
|
281
|
+
"type": "object",
|
|
282
|
+
"description": "Where the record-type marker sits in the line. Same 1-based, inclusive coordinates as a dimension position.",
|
|
283
|
+
"additionalProperties": false,
|
|
284
|
+
"required": ["start"],
|
|
285
|
+
"properties": {
|
|
286
|
+
"start": {
|
|
287
|
+
"type": "integer",
|
|
288
|
+
"minimum": 1,
|
|
289
|
+
"description": "1-based position of the marker's first character in the line."
|
|
290
|
+
},
|
|
291
|
+
"length": {
|
|
292
|
+
"type": "integer",
|
|
293
|
+
"minimum": 1,
|
|
294
|
+
"description": "How many characters the marker spans. Mutually exclusive with 'end'."
|
|
295
|
+
},
|
|
296
|
+
"end": {
|
|
297
|
+
"type": "integer",
|
|
298
|
+
"minimum": 1,
|
|
299
|
+
"description": "1-based position of the marker's last character, inclusive. Mutually exclusive with 'length'."
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
"equals": {
|
|
304
|
+
"description": "The marker value(s) to keep, compared case-sensitively against the trimmed marker.",
|
|
305
|
+
"oneOf": [
|
|
306
|
+
{ "type": "string", "minLength": 1 },
|
|
307
|
+
{ "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
|
|
308
|
+
]
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
},
|
|
222
312
|
"disableHistory": {
|
|
223
313
|
"type": "boolean",
|
|
224
314
|
"description": "For Delta Share producers, skips sending the current table version in the query request body."
|
|
225
315
|
},
|
|
226
316
|
"hasHeaderRow": {
|
|
227
317
|
"type": "boolean",
|
|
228
|
-
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
|
|
318
|
+
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true. For FIXED files it defaults to false — set it to true only if the fixed-width file starts with a descriptive line that must be skipped."
|
|
229
319
|
},
|
|
230
320
|
"sheetName": {
|
|
231
321
|
"type": "string",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.7.0",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -43,20 +43,18 @@
|
|
|
43
43
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
44
44
|
"@aws-sdk/client-s3": "^3.701.0",
|
|
45
45
|
"@aws-sdk/client-sqs": "^3.886.0",
|
|
46
|
-
"adm-zip": "^0.
|
|
46
|
+
"adm-zip": "^0.6.0",
|
|
47
47
|
"ajv": "^8.17.1",
|
|
48
48
|
"ajv-formats": "^3.0.1",
|
|
49
|
-
"bcrypt": "^5.1.1",
|
|
50
49
|
"bcryptjs": "^3.0.2",
|
|
51
50
|
"chalk": "^4.1.2",
|
|
52
51
|
"commander": "^10.0.0",
|
|
53
52
|
"cors": "^2.8.5",
|
|
54
53
|
"cross-env": "^7.0.3",
|
|
55
54
|
"dayjs": "^1.11.13",
|
|
56
|
-
"decompress": "^4.2.1",
|
|
57
55
|
"dotenv": "^16.0.3",
|
|
58
56
|
"express": "^4.21.2",
|
|
59
|
-
"fast-xml-parser": "^5.
|
|
57
|
+
"fast-xml-parser": "^5.3.5",
|
|
60
58
|
"fs-extra": "^11.1.0",
|
|
61
59
|
"hyparquet": "^1.17.4",
|
|
62
60
|
"inquirer": "^8.2.5",
|
|
@@ -64,8 +62,7 @@
|
|
|
64
62
|
"jsonwebtoken": "^9.0.2",
|
|
65
63
|
"knex": "^2.4.2",
|
|
66
64
|
"mongodb": "^6.15.0",
|
|
67
|
-
"multer": "^
|
|
68
|
-
"next": "^13.4.1",
|
|
65
|
+
"multer": "^2.0.0",
|
|
69
66
|
"node-cron": "^4.2.1",
|
|
70
67
|
"openai": "^6.0.0",
|
|
71
68
|
"ora": "^5.4.1",
|
|
@@ -75,7 +72,7 @@
|
|
|
75
72
|
"seedrandom": "^3.0.5",
|
|
76
73
|
"uuid": "^13.0.0",
|
|
77
74
|
"workerpool": "^9.3.3",
|
|
78
|
-
"xlsx": "^0.
|
|
75
|
+
"@e965/xlsx": "^0.20.3",
|
|
79
76
|
"zod": "^3.24.2"
|
|
80
77
|
}
|
|
81
78
|
}
|