@forzalabs/remora 1.2.12 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/documentation/README.md +3 -2
- package/documentation/default_resources/project.json +1 -3
- package/index.js +13545 -9923
- package/json_schemas/code-set-schema.json +60 -0
- package/json_schemas/consumer-schema.json +740 -297
- package/json_schemas/producer-schema.json +97 -17
- package/json_schemas/project-schema.json +8 -0
- package/json_schemas/source-schema.json +15 -21
- package/package.json +2 -1
- package/workers/ExecutorWorker.js +13313 -9859
|
@@ -44,6 +44,10 @@
|
|
|
44
44
|
],
|
|
45
45
|
"description": "The data type of the dimension"
|
|
46
46
|
},
|
|
47
|
+
"format": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
50
|
+
},
|
|
47
51
|
"alias": {
|
|
48
52
|
"type": "string",
|
|
49
53
|
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
@@ -52,6 +56,10 @@
|
|
|
52
56
|
"type": "boolean",
|
|
53
57
|
"description": "Whether this is a primary key column or not"
|
|
54
58
|
},
|
|
59
|
+
"nullable": {
|
|
60
|
+
"type": "boolean",
|
|
61
|
+
"description": "Declares whether this dimension is allowed to hold null/missing values. Descriptive metadata only: it documents intent for consumers, schema generation and AI mappers, and does not change runtime behaviour (nulls are already passed through untouched). Use a consumer field 'required' validation to enforce presence."
|
|
62
|
+
},
|
|
55
63
|
"classification": {
|
|
56
64
|
"type": "array",
|
|
57
65
|
"items": {
|
|
@@ -72,7 +80,7 @@
|
|
|
72
80
|
"mask",
|
|
73
81
|
"crypt",
|
|
74
82
|
"random",
|
|
75
|
-
"seeded-random",
|
|
83
|
+
"seeded-random",
|
|
76
84
|
"none",
|
|
77
85
|
"{REMORA_MASK_IN_DEV}"
|
|
78
86
|
]
|
|
@@ -80,12 +88,69 @@
|
|
|
80
88
|
"sourceFilename": {
|
|
81
89
|
"type": "boolean",
|
|
82
90
|
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
91
|
+
},
|
|
92
|
+
"allowedValues": {
|
|
93
|
+
"type": "array",
|
|
94
|
+
"minItems": 1,
|
|
95
|
+
"items": {
|
|
96
|
+
"type": ["string", "number", "boolean"]
|
|
97
|
+
},
|
|
98
|
+
"description": "Inline closed set of allowed values used only by synthetic-data generation ('synth'): one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
99
|
+
},
|
|
100
|
+
"codeSet": {
|
|
101
|
+
"description": "Reference to a named code set used only by synthetic-data generation ('synth'). A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
102
|
+
"oneOf": [
|
|
103
|
+
{
|
|
104
|
+
"type": "string",
|
|
105
|
+
"minLength": 1
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"type": "object",
|
|
109
|
+
"required": ["name", "field"],
|
|
110
|
+
"additionalProperties": false,
|
|
111
|
+
"properties": {
|
|
112
|
+
"name": {
|
|
113
|
+
"type": "string",
|
|
114
|
+
"minLength": 1,
|
|
115
|
+
"description": "Name of a structured (records) code set."
|
|
116
|
+
},
|
|
117
|
+
"field": {
|
|
118
|
+
"type": "string",
|
|
119
|
+
"minLength": 1,
|
|
120
|
+
"description": "Field of the picked record to emit for this dimension."
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
]
|
|
125
|
+
},
|
|
126
|
+
"references": {
|
|
127
|
+
"type": "object",
|
|
128
|
+
"description": "Declares this dimension as a foreign key referencing another producer's field, used only by synthetic-data generation ('synth'). Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
129
|
+
"required": ["producer", "field"],
|
|
130
|
+
"additionalProperties": false,
|
|
131
|
+
"properties": {
|
|
132
|
+
"producer": {
|
|
133
|
+
"type": "string",
|
|
134
|
+
"minLength": 1,
|
|
135
|
+
"description": "Name of the parent producer whose field this dimension references."
|
|
136
|
+
},
|
|
137
|
+
"field": {
|
|
138
|
+
"type": "string",
|
|
139
|
+
"minLength": 1,
|
|
140
|
+
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
141
|
+
}
|
|
142
|
+
}
|
|
83
143
|
}
|
|
84
144
|
},
|
|
85
145
|
"required": [
|
|
86
146
|
"name",
|
|
87
147
|
"type"
|
|
88
148
|
],
|
|
149
|
+
"allOf": [
|
|
150
|
+
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
151
|
+
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
152
|
+
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
153
|
+
],
|
|
89
154
|
"additionalProperties": false
|
|
90
155
|
},
|
|
91
156
|
"minItems": 1
|
|
@@ -118,15 +183,6 @@
|
|
|
118
183
|
},
|
|
119
184
|
"settings": {
|
|
120
185
|
"type": "object",
|
|
121
|
-
"compressionType": {
|
|
122
|
-
"type": "string",
|
|
123
|
-
"enum": [
|
|
124
|
-
"GZ",
|
|
125
|
-
"TAR",
|
|
126
|
-
"ZIP"
|
|
127
|
-
],
|
|
128
|
-
"description": "The compression type used from the file to read"
|
|
129
|
-
},
|
|
130
186
|
"description": "Settings for the producer",
|
|
131
187
|
"properties": {
|
|
132
188
|
"sqlTable": {
|
|
@@ -150,7 +206,7 @@
|
|
|
150
206
|
"enum": [
|
|
151
207
|
"JSON",
|
|
152
208
|
"JSONL",
|
|
153
|
-
"CSV",
|
|
209
|
+
"CSV",
|
|
154
210
|
"TXT",
|
|
155
211
|
"XLS",
|
|
156
212
|
"XLSX",
|
|
@@ -172,8 +228,36 @@
|
|
|
172
228
|
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
|
|
173
229
|
},
|
|
174
230
|
"sheetName": {
|
|
175
|
-
"type": "string",
|
|
231
|
+
"type": "string",
|
|
176
232
|
"description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
|
|
233
|
+
},
|
|
234
|
+
"compressionType": {
|
|
235
|
+
"type": "string",
|
|
236
|
+
"enum": [
|
|
237
|
+
"GZ",
|
|
238
|
+
"ZIP",
|
|
239
|
+
"TAR"
|
|
240
|
+
],
|
|
241
|
+
"description": "The compression of the source file. The file is decompressed before parsing."
|
|
242
|
+
},
|
|
243
|
+
"startRow": {
|
|
244
|
+
"type": "number",
|
|
245
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed row to start reading from (e.g. 3 skips rows 1-2)."
|
|
246
|
+
},
|
|
247
|
+
"startColumn": {
|
|
248
|
+
"type": "number",
|
|
249
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed column to start reading from (e.g. 2 starts at column B)."
|
|
250
|
+
},
|
|
251
|
+
"httpApi": {
|
|
252
|
+
"type": "object",
|
|
253
|
+
"description": "Additional settings when the source is an HTTP API.",
|
|
254
|
+
"properties": {
|
|
255
|
+
"dataProperty": {
|
|
256
|
+
"type": "string",
|
|
257
|
+
"description": "If the response wraps the records in a property, its name (e.g. 'result' for { success: true, result: [...] })."
|
|
258
|
+
}
|
|
259
|
+
},
|
|
260
|
+
"additionalProperties": false
|
|
177
261
|
}
|
|
178
262
|
},
|
|
179
263
|
"additionalProperties": false
|
|
@@ -213,10 +297,6 @@
|
|
|
213
297
|
{
|
|
214
298
|
"name": "email",
|
|
215
299
|
"type": "string",
|
|
216
|
-
"format": {
|
|
217
|
-
"type": "string",
|
|
218
|
-
"description": "Format for datetype of the source ('YYYYMMDD', 'DD-MM-YYYY')"
|
|
219
|
-
},
|
|
220
300
|
"classification": [
|
|
221
301
|
"PII",
|
|
222
302
|
"GDPR"
|
|
@@ -309,4 +389,4 @@
|
|
|
309
389
|
"_version": 1
|
|
310
390
|
}
|
|
311
391
|
]
|
|
312
|
-
}
|
|
392
|
+
}
|
|
@@ -54,6 +54,14 @@
|
|
|
54
54
|
},
|
|
55
55
|
"description": "Array of schema paths"
|
|
56
56
|
},
|
|
57
|
+
"codeSets": {
|
|
58
|
+
"type": "array",
|
|
59
|
+
"items": {
|
|
60
|
+
"type": "string",
|
|
61
|
+
"pattern": "^/"
|
|
62
|
+
},
|
|
63
|
+
"description": "Optional array of code-set paths used to drive synthetic-data generation"
|
|
64
|
+
},
|
|
57
65
|
"settings": {
|
|
58
66
|
"type": "object",
|
|
59
67
|
"required": ["SQL_MAX_QUERY_ROWS"],
|
|
@@ -20,10 +20,8 @@
|
|
|
20
20
|
"type": "string",
|
|
21
21
|
"enum": [
|
|
22
22
|
"aws-redshift",
|
|
23
|
-
"aws-dynamodb",
|
|
24
23
|
"aws-s3",
|
|
25
24
|
"aws-sqs",
|
|
26
|
-
"postgres",
|
|
27
25
|
"local",
|
|
28
26
|
"delta-share",
|
|
29
27
|
"http-api"
|
|
@@ -163,7 +161,10 @@
|
|
|
163
161
|
},
|
|
164
162
|
"clientAuthMethod": {
|
|
165
163
|
"type": "string",
|
|
166
|
-
"enum": [
|
|
164
|
+
"enum": [
|
|
165
|
+
"basic",
|
|
166
|
+
"post"
|
|
167
|
+
],
|
|
167
168
|
"description": "How client credentials are sent to the token endpoint: HTTP Basic header ('basic', default) or request body ('post')",
|
|
168
169
|
"default": "basic"
|
|
169
170
|
},
|
|
@@ -188,7 +189,13 @@
|
|
|
188
189
|
},
|
|
189
190
|
"httpMethod": {
|
|
190
191
|
"type": "string",
|
|
191
|
-
"enum": [
|
|
192
|
+
"enum": [
|
|
193
|
+
"GET",
|
|
194
|
+
"POST",
|
|
195
|
+
"PUT",
|
|
196
|
+
"PATCH",
|
|
197
|
+
"DELETE"
|
|
198
|
+
],
|
|
192
199
|
"description": "HTTP method to use for API requests",
|
|
193
200
|
"default": "GET"
|
|
194
201
|
},
|
|
@@ -208,7 +215,9 @@
|
|
|
208
215
|
"minimum": 1000
|
|
209
216
|
}
|
|
210
217
|
},
|
|
211
|
-
"required": [
|
|
218
|
+
"required": [
|
|
219
|
+
"method"
|
|
220
|
+
]
|
|
212
221
|
},
|
|
213
222
|
"_version": {
|
|
214
223
|
"type": "number",
|
|
@@ -222,21 +231,6 @@
|
|
|
222
231
|
],
|
|
223
232
|
"additionalProperties": false,
|
|
224
233
|
"examples": [
|
|
225
|
-
{
|
|
226
|
-
"name": "Production PostgreSQL Database",
|
|
227
|
-
"description": "Main production database for customer data",
|
|
228
|
-
"engine": "postgres",
|
|
229
|
-
"authentication": {
|
|
230
|
-
"method": "username-password",
|
|
231
|
-
"host": "prod-db.example.com",
|
|
232
|
-
"user": "app_user",
|
|
233
|
-
"password": "password123",
|
|
234
|
-
"database": "customers",
|
|
235
|
-
"schema": "public",
|
|
236
|
-
"port": "5432"
|
|
237
|
-
},
|
|
238
|
-
"_version": 1
|
|
239
|
-
},
|
|
240
234
|
{
|
|
241
235
|
"name": "Data Lake",
|
|
242
236
|
"description": "AWS S3 bucket containing analytics data",
|
|
@@ -338,4 +332,4 @@
|
|
|
338
332
|
"_version": 1
|
|
339
333
|
}
|
|
340
334
|
]
|
|
341
|
-
}
|
|
335
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"version": "npx tsx ./src/index.ts -v",
|
|
21
21
|
"run": "npm run sync:internal && npx tsx ./src/index.ts run",
|
|
22
22
|
"discover": "npm run sync:internal && npx tsx ./src/index.ts discover",
|
|
23
|
+
"synth": "npm run sync:internal && npx tsx ./src/index.ts synth",
|
|
23
24
|
"sample": "npm run sync:internal && npx tsx ./src/index.ts sample",
|
|
24
25
|
"compile": "npm run sync:internal && npx tsx ./src/index.ts compile",
|
|
25
26
|
"deploy": "npm run sync:internal && npx tsx ./src/index.ts deploy",
|