@forzalabs/remora 1.8.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/documentation/README.md +1 -1
- package/index.js +6448 -1752
- package/json_schemas/consumer-schema.json +20 -9
- package/json_schemas/producer-schema.json +323 -210
- package/json_schemas/project-schema.json +6 -1
- package/package.json +6 -1
- package/workers/ExecutorWorker.js +26762 -22277
|
@@ -23,195 +23,7 @@
|
|
|
23
23
|
"dimensions": {
|
|
24
24
|
"type": "array",
|
|
25
25
|
"description": "Dimensions for the producer",
|
|
26
|
-
"items": {
|
|
27
|
-
"type": "object",
|
|
28
|
-
"properties": {
|
|
29
|
-
"name": {
|
|
30
|
-
"type": "string",
|
|
31
|
-
"description": "The name of the dimension. This is the output name of this dimension."
|
|
32
|
-
},
|
|
33
|
-
"description": {
|
|
34
|
-
"type": "string",
|
|
35
|
-
"description": "Optional description of the dimension"
|
|
36
|
-
},
|
|
37
|
-
"type": {
|
|
38
|
-
"type": "string",
|
|
39
|
-
"enum": [
|
|
40
|
-
"string",
|
|
41
|
-
"number",
|
|
42
|
-
"datetime",
|
|
43
|
-
"boolean"
|
|
44
|
-
],
|
|
45
|
-
"description": "The data type of the dimension"
|
|
46
|
-
},
|
|
47
|
-
"format": {
|
|
48
|
-
"type": "string",
|
|
49
|
-
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
50
|
-
},
|
|
51
|
-
"alias": {
|
|
52
|
-
"type": "string",
|
|
53
|
-
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
54
|
-
},
|
|
55
|
-
"pk": {
|
|
56
|
-
"type": "boolean",
|
|
57
|
-
"description": "Whether this is a primary key column or not"
|
|
58
|
-
},
|
|
59
|
-
"nullable": {
|
|
60
|
-
"type": "boolean",
|
|
61
|
-
"description": "Declares whether this dimension is allowed to hold null/missing values. Descriptive metadata only: it documents intent for consumers, schema generation and AI mappers, and does not change runtime behaviour (nulls are already passed through untouched). Use a consumer field 'required' validation to enforce presence."
|
|
62
|
-
},
|
|
63
|
-
"classification": {
|
|
64
|
-
"type": "array",
|
|
65
|
-
"items": {
|
|
66
|
-
"type": "string",
|
|
67
|
-
"enum": [
|
|
68
|
-
"PHI",
|
|
69
|
-
"PII",
|
|
70
|
-
"GDPR"
|
|
71
|
-
],
|
|
72
|
-
"description": "Classification categories for this dimension"
|
|
73
|
-
}
|
|
74
|
-
},
|
|
75
|
-
"mask": {
|
|
76
|
-
"type": "string",
|
|
77
|
-
"description": "Masking type to apply to this dimension. Predefined values: 'hash' (replaces with a hashed value), 'mask' (replaces characters with a mask character), 'crypt' (encrypts the value), 'random' (replaces with a random value), 'seeded-random' (replaces with a random value generated from a seed), 'none' (no masking). You can also use environment variables by using the {your-env-var} notation or any custom string value.",
|
|
78
|
-
"examples": [
|
|
79
|
-
"hash",
|
|
80
|
-
"mask",
|
|
81
|
-
"crypt",
|
|
82
|
-
"random",
|
|
83
|
-
"seeded-random",
|
|
84
|
-
"none",
|
|
85
|
-
"{REMORA_MASK_IN_DEV}"
|
|
86
|
-
]
|
|
87
|
-
},
|
|
88
|
-
"sourceFilename": {
|
|
89
|
-
"type": "boolean",
|
|
90
|
-
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
91
|
-
},
|
|
92
|
-
"position": {
|
|
93
|
-
"type": "object",
|
|
94
|
-
"description": "Where this dimension's value sits inside the line, for FIXED (fixed-width) files. Required on every dimension of a FIXED producer and only valid there: a fixed-width file has no header and no delimiter, so the position IS the mapping. Coordinates are 1-based and inclusive, like a file layout spec ('columns 21-30').",
|
|
95
|
-
"additionalProperties": false,
|
|
96
|
-
"required": ["start"],
|
|
97
|
-
"properties": {
|
|
98
|
-
"start": {
|
|
99
|
-
"type": "integer",
|
|
100
|
-
"minimum": 1,
|
|
101
|
-
"description": "1-based position of the field's first character in the line."
|
|
102
|
-
},
|
|
103
|
-
"length": {
|
|
104
|
-
"type": "integer",
|
|
105
|
-
"minimum": 1,
|
|
106
|
-
"description": "How many characters the field spans. Mutually exclusive with 'end'."
|
|
107
|
-
},
|
|
108
|
-
"end": {
|
|
109
|
-
"type": "integer",
|
|
110
|
-
"minimum": 1,
|
|
111
|
-
"description": "1-based position of the field's last character, inclusive. Mutually exclusive with 'length'."
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
},
|
|
115
|
-
"synth": {
|
|
116
|
-
"type": "object",
|
|
117
|
-
"description": "Synthetic-data configuration for this dimension, used only by the 'synth' command. Nothing in here affects a real producer read. 'allowedValues', 'codeSet' and 'references' are mutually exclusive ways to declare where a value comes from.",
|
|
118
|
-
"additionalProperties": false,
|
|
119
|
-
"properties": {
|
|
120
|
-
"decimals": {
|
|
121
|
-
"type": "integer",
|
|
122
|
-
"minimum": 0,
|
|
123
|
-
"description": "How many decimal places generated number values should have: 0 forces whole integers (so values load into an integer column); a positive value rounds to that many decimals. Omit to keep the generator's own value."
|
|
124
|
-
},
|
|
125
|
-
"min": {
|
|
126
|
-
"type": ["number", "string"],
|
|
127
|
-
"description": "Inclusive lower bound for generated values, so the column stays inside its real-world range. A number for a 'number' dimension; a date string for a 'datetime' dimension (read with the dimension's own 'format' when it has one, ISO otherwise). Only for 'number' and 'datetime' dimensions, and mutually exclusive with 'allowedValues', 'codeSet' and 'references'."
|
|
128
|
-
},
|
|
129
|
-
"max": {
|
|
130
|
-
"type": ["number", "string"],
|
|
131
|
-
"description": "Inclusive upper bound for generated values, so the column stays inside its real-world range. A number for a 'number' dimension; a date string for a 'datetime' dimension (read with the dimension's own 'format' when it has one, ISO otherwise). Only for 'number' and 'datetime' dimensions, and mutually exclusive with 'allowedValues', 'codeSet' and 'references'."
|
|
132
|
-
},
|
|
133
|
-
"allowedValues": {
|
|
134
|
-
"type": "array",
|
|
135
|
-
"minItems": 1,
|
|
136
|
-
"items": {
|
|
137
|
-
"type": ["string", "number", "boolean"]
|
|
138
|
-
},
|
|
139
|
-
"description": "Inline closed set of allowed values: one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
140
|
-
},
|
|
141
|
-
"codeSet": {
|
|
142
|
-
"description": "Reference to a named code set. A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
143
|
-
"oneOf": [
|
|
144
|
-
{
|
|
145
|
-
"type": "string",
|
|
146
|
-
"minLength": 1
|
|
147
|
-
},
|
|
148
|
-
{
|
|
149
|
-
"type": "object",
|
|
150
|
-
"required": ["name", "field"],
|
|
151
|
-
"additionalProperties": false,
|
|
152
|
-
"properties": {
|
|
153
|
-
"name": {
|
|
154
|
-
"type": "string",
|
|
155
|
-
"minLength": 1,
|
|
156
|
-
"description": "Name of a structured (records) code set."
|
|
157
|
-
},
|
|
158
|
-
"field": {
|
|
159
|
-
"type": "string",
|
|
160
|
-
"minLength": 1,
|
|
161
|
-
"description": "Field of the picked record to emit for this dimension."
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
]
|
|
166
|
-
},
|
|
167
|
-
"references": {
|
|
168
|
-
"type": "object",
|
|
169
|
-
"description": "Declares this dimension as a foreign key referencing another producer's field. Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
170
|
-
"required": ["producer", "field"],
|
|
171
|
-
"additionalProperties": false,
|
|
172
|
-
"properties": {
|
|
173
|
-
"producer": {
|
|
174
|
-
"type": "string",
|
|
175
|
-
"minLength": 1,
|
|
176
|
-
"description": "Name of the parent producer whose field this dimension references."
|
|
177
|
-
},
|
|
178
|
-
"field": {
|
|
179
|
-
"type": "string",
|
|
180
|
-
"minLength": 1,
|
|
181
|
-
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
},
|
|
186
|
-
"allOf": [
|
|
187
|
-
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
188
|
-
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
189
|
-
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
190
|
-
]
|
|
191
|
-
},
|
|
192
|
-
"decimals": {
|
|
193
|
-
"not": {},
|
|
194
|
-
"description": "MOVED: use \"synth\": { \"decimals\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
195
|
-
},
|
|
196
|
-
"allowedValues": {
|
|
197
|
-
"not": {},
|
|
198
|
-
"description": "MOVED: use \"synth\": { \"allowedValues\": [...] } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
199
|
-
},
|
|
200
|
-
"codeSet": {
|
|
201
|
-
"not": {},
|
|
202
|
-
"description": "MOVED: use \"synth\": { \"codeSet\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
203
|
-
},
|
|
204
|
-
"references": {
|
|
205
|
-
"not": {},
|
|
206
|
-
"description": "MOVED: use \"synth\": { \"references\": { \"producer\", \"field\" } } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
207
|
-
}
|
|
208
|
-
},
|
|
209
|
-
"required": [
|
|
210
|
-
"name",
|
|
211
|
-
"type"
|
|
212
|
-
],
|
|
213
|
-
"additionalProperties": false
|
|
214
|
-
},
|
|
26
|
+
"items": { "$ref": "#/definitions/dimension" },
|
|
215
27
|
"minItems": 1
|
|
216
28
|
},
|
|
217
29
|
"measures": {
|
|
@@ -286,27 +98,8 @@
|
|
|
286
98
|
"required": ["position", "equals"],
|
|
287
99
|
"properties": {
|
|
288
100
|
"position": {
|
|
289
|
-
"
|
|
290
|
-
"description": "Where the record-type marker sits in the line. Same 1-based, inclusive coordinates as a dimension position."
|
|
291
|
-
"additionalProperties": false,
|
|
292
|
-
"required": ["start"],
|
|
293
|
-
"properties": {
|
|
294
|
-
"start": {
|
|
295
|
-
"type": "integer",
|
|
296
|
-
"minimum": 1,
|
|
297
|
-
"description": "1-based position of the marker's first character in the line."
|
|
298
|
-
},
|
|
299
|
-
"length": {
|
|
300
|
-
"type": "integer",
|
|
301
|
-
"minimum": 1,
|
|
302
|
-
"description": "How many characters the marker spans. Mutually exclusive with 'end'."
|
|
303
|
-
},
|
|
304
|
-
"end": {
|
|
305
|
-
"type": "integer",
|
|
306
|
-
"minimum": 1,
|
|
307
|
-
"description": "1-based position of the marker's last character, inclusive. Mutually exclusive with 'length'."
|
|
308
|
-
}
|
|
309
|
-
}
|
|
101
|
+
"allOf": [{ "$ref": "#/definitions/position" }],
|
|
102
|
+
"description": "Where the record-type marker sits in the line. Same 1-based, inclusive coordinates as a dimension position."
|
|
310
103
|
},
|
|
311
104
|
"equals": {
|
|
312
105
|
"description": "The marker value(s) to keep, compared case-sensitively against the trimmed marker.",
|
|
@@ -314,6 +107,16 @@
|
|
|
314
107
|
{ "type": "string", "minLength": 1 },
|
|
315
108
|
{ "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
|
|
316
109
|
]
|
|
110
|
+
},
|
|
111
|
+
"minLength": {
|
|
112
|
+
"type": "integer",
|
|
113
|
+
"minimum": 1,
|
|
114
|
+
"description": "Shortest line this record layout can validly be, in characters. A matching marker says nothing about whether the rest of the line is there, so a truncated record would otherwise parse into garbage — its fields past the truncation reading as null instead of failing. Checked only on lines the marker already matched, so a short header or trailer is unaffected."
|
|
115
|
+
},
|
|
116
|
+
"onLengthMismatch": {
|
|
117
|
+
"type": "string",
|
|
118
|
+
"enum": ["skip", "fail"],
|
|
119
|
+
"description": "What to do with a matching line shorter than 'minLength': 'skip' it, logging at debug level (default), or 'fail' the run."
|
|
317
120
|
}
|
|
318
121
|
}
|
|
319
122
|
},
|
|
@@ -321,6 +124,10 @@
|
|
|
321
124
|
"type": "boolean",
|
|
322
125
|
"description": "For Delta Share producers, skips sending the current table version in the query request body."
|
|
323
126
|
},
|
|
127
|
+
"disablePushdown": {
|
|
128
|
+
"type": "boolean",
|
|
129
|
+
"description": "Stops the run's filters and record limit from being sent to the source as query hints (Delta Share). Both are still applied here, so this only costs bandwidth."
|
|
130
|
+
},
|
|
324
131
|
"hasHeaderRow": {
|
|
325
132
|
"type": "boolean",
|
|
326
133
|
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true. For FIXED files it defaults to false — set it to true only if the fixed-width file starts with a descriptive line that must be skipped."
|
|
@@ -365,6 +172,312 @@
|
|
|
365
172
|
"description": "Version number of the producer configuration"
|
|
366
173
|
}
|
|
367
174
|
},
|
|
175
|
+
"definitions": {
|
|
176
|
+
"position": {
|
|
177
|
+
"type": "object",
|
|
178
|
+
"description": "A character range inside a fixed-width line. Coordinates are 1-based and inclusive, like a file layout spec ('columns 21-30'). Give the width as either 'length' or 'end', never both. Give the start as either an absolute 'start' or an 'after' anchor.",
|
|
179
|
+
"additionalProperties": false,
|
|
180
|
+
"properties": {
|
|
181
|
+
"start": {
|
|
182
|
+
"type": "integer",
|
|
183
|
+
"minimum": 1,
|
|
184
|
+
"description": "1-based position of the field's first character in the line — or, with 'after', inside the anchored region, where it defaults to 1. Required unless 'after' is set."
|
|
185
|
+
},
|
|
186
|
+
"length": {
|
|
187
|
+
"type": "integer",
|
|
188
|
+
"minimum": 1,
|
|
189
|
+
"description": "How many characters the field spans. Mutually exclusive with 'end'."
|
|
190
|
+
},
|
|
191
|
+
"end": {
|
|
192
|
+
"type": "integer",
|
|
193
|
+
"minimum": 1,
|
|
194
|
+
"description": "1-based position of the field's last character, inclusive. Mutually exclusive with 'length'."
|
|
195
|
+
},
|
|
196
|
+
"after": {
|
|
197
|
+
"type": "string",
|
|
198
|
+
"minLength": 1,
|
|
199
|
+
"description": "Anchors this position to the end of the named group's ACTUAL occurrences on the line instead of to the start of the line, so it follows a group whose 'occurs' varies. Must name an EARLIER group dimension of the same producer. 'start' becomes an offset inside that region, default 1 (immediately after)."
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
"occursWhile": {
|
|
204
|
+
"type": "object",
|
|
205
|
+
"description": "Marker test that ends a variable group: occurrences are taken while the marker matches, and the first occurrence that fails it ends the block.",
|
|
206
|
+
"additionalProperties": false,
|
|
207
|
+
"required": ["position", "equals"],
|
|
208
|
+
"properties": {
|
|
209
|
+
"position": {
|
|
210
|
+
"allOf": [{ "$ref": "#/definitions/position" }],
|
|
211
|
+
"description": "Where the marker sits INSIDE one occurrence — relative, like a group child's position."
|
|
212
|
+
},
|
|
213
|
+
"equals": {
|
|
214
|
+
"oneOf": [
|
|
215
|
+
{ "type": "string" },
|
|
216
|
+
{ "type": "array", "minItems": 1, "items": { "type": "string" } }
|
|
217
|
+
],
|
|
218
|
+
"description": "The marker value(s) that mean 'this occurrence exists'. Compared against the trimmed marker."
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
},
|
|
222
|
+
"dimension": {
|
|
223
|
+
"type": "object",
|
|
224
|
+
"properties": {
|
|
225
|
+
"name": {
|
|
226
|
+
"type": "string",
|
|
227
|
+
"description": "The name of the dimension. This is the output name of this dimension."
|
|
228
|
+
},
|
|
229
|
+
"description": {
|
|
230
|
+
"type": "string",
|
|
231
|
+
"description": "Optional description of the dimension"
|
|
232
|
+
},
|
|
233
|
+
"type": {
|
|
234
|
+
"type": "string",
|
|
235
|
+
"enum": [
|
|
236
|
+
"string",
|
|
237
|
+
"number",
|
|
238
|
+
"datetime",
|
|
239
|
+
"boolean"
|
|
240
|
+
],
|
|
241
|
+
"description": "The data type of the dimension"
|
|
242
|
+
},
|
|
243
|
+
"format": {
|
|
244
|
+
"type": "string",
|
|
245
|
+
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
246
|
+
},
|
|
247
|
+
"alias": {
|
|
248
|
+
"type": "string",
|
|
249
|
+
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
250
|
+
},
|
|
251
|
+
"pk": {
|
|
252
|
+
"type": "boolean",
|
|
253
|
+
"description": "Whether this is a primary key column or not"
|
|
254
|
+
},
|
|
255
|
+
"nullable": {
|
|
256
|
+
"type": "boolean",
|
|
257
|
+
"description": "Declares whether this dimension is allowed to hold null/missing values. Descriptive metadata only: it documents intent for consumers, schema generation and AI mappers, and does not change runtime behaviour (nulls are already passed through untouched). Use a consumer field 'required' validation to enforce presence."
|
|
258
|
+
},
|
|
259
|
+
"classification": {
|
|
260
|
+
"type": "array",
|
|
261
|
+
"items": {
|
|
262
|
+
"type": "string",
|
|
263
|
+
"enum": [
|
|
264
|
+
"PHI",
|
|
265
|
+
"PII",
|
|
266
|
+
"GDPR"
|
|
267
|
+
],
|
|
268
|
+
"description": "Classification categories for this dimension"
|
|
269
|
+
}
|
|
270
|
+
},
|
|
271
|
+
"mask": {
|
|
272
|
+
"type": "string",
|
|
273
|
+
"description": "Masking type to apply to this dimension. Predefined values: 'hash' (replaces with a hashed value), 'mask' (replaces characters with a mask character), 'crypt' (encrypts the value), 'random' (replaces with a random value), 'seeded-random' (replaces with a random value generated from a seed), 'none' (no masking). You can also use environment variables by using the {your-env-var} notation or any custom string value.",
|
|
274
|
+
"examples": [
|
|
275
|
+
"hash",
|
|
276
|
+
"mask",
|
|
277
|
+
"crypt",
|
|
278
|
+
"random",
|
|
279
|
+
"seeded-random",
|
|
280
|
+
"none",
|
|
281
|
+
"{REMORA_MASK_IN_DEV}"
|
|
282
|
+
]
|
|
283
|
+
},
|
|
284
|
+
"sourceFilename": {
|
|
285
|
+
"type": "boolean",
|
|
286
|
+
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
287
|
+
},
|
|
288
|
+
"position": {
|
|
289
|
+
"allOf": [{ "$ref": "#/definitions/position" }],
|
|
290
|
+
"description": "Where this dimension's value sits inside the line, for FIXED (fixed-width) files. Required on every dimension of a FIXED producer and only valid there: a fixed-width file has no header and no delimiter, so the position IS the mapping. Coordinates are 1-based and inclusive, like a file layout spec ('columns 21-30'). On a group dimension (one declaring nested 'dimensions') this is the extent of ONE occurrence, so 'length' is the bytes per occurrence."
|
|
291
|
+
},
|
|
292
|
+
"dimensions": {
|
|
293
|
+
"type": "array",
|
|
294
|
+
"minItems": 1,
|
|
295
|
+
"items": { "$ref": "#/definitions/dimension" },
|
|
296
|
+
"description": "Makes this a repeating GROUP of fields (a COBOL OCCURS segment): the fields that repeat, each 'position' RELATIVE to one occurrence. The group's own 'position' is the extent of ONE occurrence, so its 'length' is the bytes per occurrence, not the whole group's span. Expanded into one dimension per (occurrence x child) when the config loads, so nothing downstream sees a group. A group declares no 'type' and produces no column of its own. FIXED producers only; nested groups are not supported yet."
|
|
297
|
+
},
|
|
298
|
+
"occurs": {
|
|
299
|
+
"description": "How many times a group's fields repeat. An integer when every line has the same count — default 1, so a group is also usable purely to structure a non-repeating segment. An object when the count VARIES line by line: the group still generates 'max' columns (a line with fewer reads them as null), and everything after it must be anchored with 'position.after' because the group's span differs per line.",
|
|
300
|
+
"oneOf": [
|
|
301
|
+
{
|
|
302
|
+
"type": "integer",
|
|
303
|
+
"minimum": 1,
|
|
304
|
+
"maximum": 1000
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
"type": "object",
|
|
308
|
+
"additionalProperties": false,
|
|
309
|
+
"required": ["max"],
|
|
310
|
+
"properties": {
|
|
311
|
+
"max": {
|
|
312
|
+
"type": "integer",
|
|
313
|
+
"minimum": 1,
|
|
314
|
+
"maximum": 1000,
|
|
315
|
+
"description": "Hard upper bound on the occurrences, and the number of columns the group generates. Mandatory: the output column set has to stay statically knowable and a per-line scan needs a limit."
|
|
316
|
+
},
|
|
317
|
+
"min": {
|
|
318
|
+
"type": "integer",
|
|
319
|
+
"minimum": 0,
|
|
320
|
+
"description": "Occurrences always present, whatever the count rule reports. Default 0."
|
|
321
|
+
},
|
|
322
|
+
"while": {
|
|
323
|
+
"allOf": [{ "$ref": "#/definitions/occursWhile" }],
|
|
324
|
+
"description": "Keep taking occurrences while this marker matches inside the occurrence. Mutually exclusive with 'from'."
|
|
325
|
+
},
|
|
326
|
+
"from": {
|
|
327
|
+
"type": "string",
|
|
328
|
+
"minLength": 2,
|
|
329
|
+
"pattern": "^\\$",
|
|
330
|
+
"description": "Reference to an earlier dimension of this producer holding the count, e.g. '$slotCount'. Its value is read per line, floored at 'min' and capped at 'max'. Mutually exclusive with 'while'.",
|
|
331
|
+
"examples": ["$slotCount"]
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
]
|
|
336
|
+
},
|
|
337
|
+
"indexFrom": {
|
|
338
|
+
"type": "integer",
|
|
339
|
+
"enum": [0, 1],
|
|
340
|
+
"description": "First index used in the dimension names a group generates. Default 0."
|
|
341
|
+
},
|
|
342
|
+
"emit": {
|
|
343
|
+
"type": "string",
|
|
344
|
+
"enum": ["columns", "rows"],
|
|
345
|
+
"description": "What a group's occurrences become. 'columns' (default) generates one dimension per (occurrence x child), so one line stays one record. 'rows' fans the line out to one record PER OCCURRENCE: the children keep their own names, every dimension outside the group repeats on each record, and a generated field carries the occurrence index. At most one group per producer may declare it."
|
|
346
|
+
},
|
|
347
|
+
"skipEmptyOccurrences": {
|
|
348
|
+
"type": "boolean",
|
|
349
|
+
"description": "On an 'emit': 'rows' group: drop the occurrences whose fields are all empty instead of emitting a record of nulls for each. Default true."
|
|
350
|
+
},
|
|
351
|
+
"occurrenceField": {
|
|
352
|
+
"type": "string",
|
|
353
|
+
"minLength": 1,
|
|
354
|
+
"description": "On an 'emit': 'rows' group: the name of the generated field holding the occurrence index. Default '{group}Occurrence'.",
|
|
355
|
+
"examples": ["addr_occurrence"]
|
|
356
|
+
},
|
|
357
|
+
"naming": {
|
|
358
|
+
"type": "string",
|
|
359
|
+
"minLength": 1,
|
|
360
|
+
"description": "Template for the dimension names generated by a group or by 'flags'. On a group the tokens are {group}, {index}, {field} and {Field} (capitalized), and {index} is required unless 'occurs' is 1; default '{group}{index}{Field}'. On a 'flags' dimension they are {field}, {Field}, {flag} and {Flag}, one flag token required; default '{field}{Flag}'.",
|
|
361
|
+
"examples": ["{group}{index}{Field}", "{group}_{index}_{field}", "{field}_{flag}"]
|
|
362
|
+
},
|
|
363
|
+
"flags": {
|
|
364
|
+
"type": "array",
|
|
365
|
+
"minItems": 1,
|
|
366
|
+
"uniqueItems": true,
|
|
367
|
+
"items": { "type": "string", "minLength": 1 },
|
|
368
|
+
"description": "Decodes a per-character bitmap into one dimension per character: this dimension fans out to flags.length 1-character dimensions, the first at position.start, named after each flag. 'flags' must have exactly one entry per character the dimension spans. Each character is SLICED, never interpreted, so a non-binary character survives as itself. FIXED producers only, on a 'string' dimension.",
|
|
369
|
+
"examples": [["mailing", "billing", "physical"]]
|
|
370
|
+
},
|
|
371
|
+
"synth": {
|
|
372
|
+
"type": "object",
|
|
373
|
+
"description": "Synthetic-data configuration for this dimension, used only by the 'synth' command. Nothing in here affects a real producer read. 'allowedValues', 'codeSet' and 'references' are mutually exclusive ways to declare where a value comes from.",
|
|
374
|
+
"additionalProperties": false,
|
|
375
|
+
"properties": {
|
|
376
|
+
"decimals": {
|
|
377
|
+
"type": "integer",
|
|
378
|
+
"minimum": 0,
|
|
379
|
+
"description": "How many decimal places generated number values should have: 0 forces whole integers (so values load into an integer column); a positive value rounds to that many decimals. Omit to keep the generator's own value."
|
|
380
|
+
},
|
|
381
|
+
"unique": {
|
|
382
|
+
"type": "boolean",
|
|
383
|
+
"description": "Emit a distinct value in every row, for a column that carries a uniqueness constraint without being the primary key (a natural key, an external reference, an account number). A name-based generator cannot promise uniqueness, so a unique column is served from the coordinated key pool the way a primary key is ('external_ref_1', 'external_ref_2', ...), with 'min'/'max' sizing it as they do for any key. Only for 'string' and 'number' dimensions, and mutually exclusive with 'allowedValues', 'codeSet' and 'references'."
|
|
384
|
+
},
|
|
385
|
+
"min": {
|
|
386
|
+
"type": ["number", "string"],
|
|
387
|
+
"description": "Inclusive lower bound for generated values, so the column stays inside its real-world range. A number for a 'number' dimension; a date string for a 'datetime' dimension (read with the dimension's own 'format' when it has one, ISO otherwise); a whole number of characters for a 'string' dimension, where it is the minimum length and a shorter generated value is filled out with letters. On a key/ID column the pool of values is generated AT the declared width instead of being clamped to it, so every key stays distinct. Not for 'boolean' dimensions, and mutually exclusive with 'allowedValues', 'codeSet' and 'references'."
|
|
388
|
+
},
|
|
389
|
+
"max": {
|
|
390
|
+
"type": ["number", "string"],
|
|
391
|
+
"description": "Inclusive upper bound for generated values, so the column stays inside its real-world range. A number for a 'number' dimension; a date string for a 'datetime' dimension (read with the dimension's own 'format' when it has one, ISO otherwise); a whole number of characters for a 'string' dimension, where it is the maximum length and a longer generated value is cut to it. On a key/ID column the pool of values is generated AT the declared width instead of being clamped to it, so every key stays distinct. Not for 'boolean' dimensions, and mutually exclusive with 'allowedValues', 'codeSet' and 'references'."
|
|
392
|
+
},
|
|
393
|
+
"allowedValues": {
|
|
394
|
+
"type": "array",
|
|
395
|
+
"minItems": 1,
|
|
396
|
+
"items": {
|
|
397
|
+
"type": ["string", "number", "boolean"]
|
|
398
|
+
},
|
|
399
|
+
"description": "Inline closed set of allowed values: one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
400
|
+
},
|
|
401
|
+
"codeSet": {
|
|
402
|
+
"description": "Reference to a named code set. A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
403
|
+
"oneOf": [
|
|
404
|
+
{
|
|
405
|
+
"type": "string",
|
|
406
|
+
"minLength": 1
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
"type": "object",
|
|
410
|
+
"required": ["name", "field"],
|
|
411
|
+
"additionalProperties": false,
|
|
412
|
+
"properties": {
|
|
413
|
+
"name": {
|
|
414
|
+
"type": "string",
|
|
415
|
+
"minLength": 1,
|
|
416
|
+
"description": "Name of a structured (records) code set."
|
|
417
|
+
},
|
|
418
|
+
"field": {
|
|
419
|
+
"type": "string",
|
|
420
|
+
"minLength": 1,
|
|
421
|
+
"description": "Field of the picked record to emit for this dimension."
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
]
|
|
426
|
+
},
|
|
427
|
+
"references": {
|
|
428
|
+
"type": "object",
|
|
429
|
+
"description": "Declares this dimension as a foreign key referencing another producer's field. Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
430
|
+
"required": ["producer", "field"],
|
|
431
|
+
"additionalProperties": false,
|
|
432
|
+
"properties": {
|
|
433
|
+
"producer": {
|
|
434
|
+
"type": "string",
|
|
435
|
+
"minLength": 1,
|
|
436
|
+
"description": "Name of the parent producer whose field this dimension references."
|
|
437
|
+
},
|
|
438
|
+
"field": {
|
|
439
|
+
"type": "string",
|
|
440
|
+
"minLength": 1,
|
|
441
|
+
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
},
|
|
446
|
+
"allOf": [
|
|
447
|
+
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
448
|
+
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
449
|
+
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
450
|
+
]
|
|
451
|
+
},
|
|
452
|
+
"decimals": {
|
|
453
|
+
"not": {},
|
|
454
|
+
"description": "MOVED: use \"synth\": { \"decimals\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
455
|
+
},
|
|
456
|
+
"allowedValues": {
|
|
457
|
+
"not": {},
|
|
458
|
+
"description": "MOVED: use \"synth\": { \"allowedValues\": [...] } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
459
|
+
},
|
|
460
|
+
"codeSet": {
|
|
461
|
+
"not": {},
|
|
462
|
+
"description": "MOVED: use \"synth\": { \"codeSet\": ... } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
463
|
+
},
|
|
464
|
+
"references": {
|
|
465
|
+
"not": {},
|
|
466
|
+
"description": "MOVED: use \"synth\": { \"references\": { \"producer\", \"field\" } } instead. All synthetic-data properties now live under the dimension's \"synth\" object."
|
|
467
|
+
}
|
|
468
|
+
},
|
|
469
|
+
"required": ["name"],
|
|
470
|
+
"additionalProperties": false,
|
|
471
|
+
"allOf": [
|
|
472
|
+
{
|
|
473
|
+
"description": "A group dimension produces no column of its own, so it declares no 'type'; every other dimension must declare one.",
|
|
474
|
+
"if": { "required": ["dimensions"], "properties": { "dimensions": {} } },
|
|
475
|
+
"then": { "not": { "required": ["type"], "properties": { "type": {} } } },
|
|
476
|
+
"else": { "required": ["type"], "properties": { "type": {} } }
|
|
477
|
+
}
|
|
478
|
+
]
|
|
479
|
+
}
|
|
480
|
+
},
|
|
368
481
|
"required": [
|
|
369
482
|
"name",
|
|
370
483
|
"source",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"title": "Project Schema",
|
|
4
4
|
"description": "Schema for defining remora project configuration",
|
|
5
5
|
"type": "object",
|
|
6
|
-
"required": ["name", "version", "consumers", "producers", "sources", "
|
|
6
|
+
"required": ["name", "version", "consumers", "producers", "sources", "settings"],
|
|
7
7
|
"properties": {
|
|
8
8
|
"$schema": {
|
|
9
9
|
"type": "string",
|
|
@@ -80,6 +80,11 @@
|
|
|
80
80
|
"type": "string",
|
|
81
81
|
"minLength": 1,
|
|
82
82
|
"description": "Name of an aws-sqs source used as an alternative execute-consumer API"
|
|
83
|
+
},
|
|
84
|
+
"deadLetterQueue": {
|
|
85
|
+
"type": "string",
|
|
86
|
+
"minLength": 1,
|
|
87
|
+
"description": "Dead-letter queue URL, or queue name resolved against the API queue's region and accountId. A command that failed is moved here instead of being deleted, so it can be inspected and replayed. A command always leaves the queue once it has run, successfully or not, so there is no delivery count to bound here"
|
|
83
88
|
}
|
|
84
89
|
}
|
|
85
90
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -16,6 +16,8 @@
|
|
|
16
16
|
"sync:internal": "cd ../.. && npm run sync:internal",
|
|
17
17
|
"watch:internal": "cd ../.. && npm run watch:internal",
|
|
18
18
|
"ts-check": "npx tsc --noemit",
|
|
19
|
+
"perf": "npm run sync:internal && npm run fast-build && npm run copy-static-file && npx tsx ./scripts/perf/RunPerfBaseline.ts",
|
|
20
|
+
"perf:quick": "npx tsx ./scripts/perf/RunPerfBaseline.ts",
|
|
19
21
|
"init": "npm run sync:internal && npx tsx ./src/index.ts init",
|
|
20
22
|
"version": "npx tsx ./src/index.ts -v",
|
|
21
23
|
"run": "npm run sync:internal && npx tsx ./src/index.ts run",
|
|
@@ -23,6 +25,8 @@
|
|
|
23
25
|
"synth": "npm run sync:internal && npx tsx ./src/index.ts synth",
|
|
24
26
|
"sample": "npm run sync:internal && npx tsx ./src/index.ts sample",
|
|
25
27
|
"compile": "npm run sync:internal && npx tsx ./src/index.ts compile",
|
|
28
|
+
"graph": "npm run sync:internal && npx tsx ./src/index.ts graph",
|
|
29
|
+
"ui-preview": "npm run sync:internal && npx tsx ./src/index.ts ui-preview",
|
|
26
30
|
"deploy": "npm run sync:internal && npx tsx ./src/index.ts deploy",
|
|
27
31
|
"verify": "npm run sync:internal && npx tsx ./src/index.ts verify",
|
|
28
32
|
"create-producer": "npm run sync:internal && npx tsx ./src/index.ts create-producer",
|
|
@@ -43,6 +47,7 @@
|
|
|
43
47
|
"@aws-sdk/client-redshift-data": "^3.699.0",
|
|
44
48
|
"@aws-sdk/client-s3": "^3.701.0",
|
|
45
49
|
"@aws-sdk/client-sqs": "^3.886.0",
|
|
50
|
+
"@forzalabs/remora-preview": "^1.0.0",
|
|
46
51
|
"adm-zip": "^0.6.0",
|
|
47
52
|
"ajv": "^8.17.1",
|
|
48
53
|
"ajv-formats": "^3.0.1",
|