@forzalabs/remora 1.2.12 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/index.js +8786 -7996
- package/json_schemas/consumer-schema.json +728 -297
- package/json_schemas/producer-schema.json +36 -17
- package/json_schemas/source-schema.json +15 -21
- package/package.json +1 -1
- package/workers/ExecutorWorker.js +8970 -8206
|
@@ -44,6 +44,10 @@
|
|
|
44
44
|
],
|
|
45
45
|
"description": "The data type of the dimension"
|
|
46
46
|
},
|
|
47
|
+
"format": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
50
|
+
},
|
|
47
51
|
"alias": {
|
|
48
52
|
"type": "string",
|
|
49
53
|
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
@@ -72,7 +76,7 @@
|
|
|
72
76
|
"mask",
|
|
73
77
|
"crypt",
|
|
74
78
|
"random",
|
|
75
|
-
"seeded-random",
|
|
79
|
+
"seeded-random",
|
|
76
80
|
"none",
|
|
77
81
|
"{REMORA_MASK_IN_DEV}"
|
|
78
82
|
]
|
|
@@ -118,15 +122,6 @@
|
|
|
118
122
|
},
|
|
119
123
|
"settings": {
|
|
120
124
|
"type": "object",
|
|
121
|
-
"compressionType": {
|
|
122
|
-
"type": "string",
|
|
123
|
-
"enum": [
|
|
124
|
-
"GZ",
|
|
125
|
-
"TAR",
|
|
126
|
-
"ZIP"
|
|
127
|
-
],
|
|
128
|
-
"description": "The compression type used from the file to read"
|
|
129
|
-
},
|
|
130
125
|
"description": "Settings for the producer",
|
|
131
126
|
"properties": {
|
|
132
127
|
"sqlTable": {
|
|
@@ -150,7 +145,7 @@
|
|
|
150
145
|
"enum": [
|
|
151
146
|
"JSON",
|
|
152
147
|
"JSONL",
|
|
153
|
-
"CSV",
|
|
148
|
+
"CSV",
|
|
154
149
|
"TXT",
|
|
155
150
|
"XLS",
|
|
156
151
|
"XLSX",
|
|
@@ -172,8 +167,36 @@
|
|
|
172
167
|
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
|
|
173
168
|
},
|
|
174
169
|
"sheetName": {
|
|
175
|
-
"type": "string",
|
|
170
|
+
"type": "string",
|
|
176
171
|
"description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
|
|
172
|
+
},
|
|
173
|
+
"compressionType": {
|
|
174
|
+
"type": "string",
|
|
175
|
+
"enum": [
|
|
176
|
+
"GZ",
|
|
177
|
+
"ZIP",
|
|
178
|
+
"TAR"
|
|
179
|
+
],
|
|
180
|
+
"description": "The compression of the source file. The file is decompressed before parsing."
|
|
181
|
+
},
|
|
182
|
+
"startRow": {
|
|
183
|
+
"type": "number",
|
|
184
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed row to start reading from (e.g. 3 skips rows 1-2)."
|
|
185
|
+
},
|
|
186
|
+
"startColumn": {
|
|
187
|
+
"type": "number",
|
|
188
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed column to start reading from (e.g. 2 starts at column B)."
|
|
189
|
+
},
|
|
190
|
+
"httpApi": {
|
|
191
|
+
"type": "object",
|
|
192
|
+
"description": "Additional settings when the source is an HTTP API.",
|
|
193
|
+
"properties": {
|
|
194
|
+
"dataProperty": {
|
|
195
|
+
"type": "string",
|
|
196
|
+
"description": "If the response wraps the records in a property, its name (e.g. 'result' for { success: true, result: [...] })."
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
"additionalProperties": false
|
|
177
200
|
}
|
|
178
201
|
},
|
|
179
202
|
"additionalProperties": false
|
|
@@ -213,10 +236,6 @@
|
|
|
213
236
|
{
|
|
214
237
|
"name": "email",
|
|
215
238
|
"type": "string",
|
|
216
|
-
"format": {
|
|
217
|
-
"type": "string",
|
|
218
|
-
"description": "Format for datetype of the source ('YYYYMMDD', 'DD-MM-YYYY')"
|
|
219
|
-
},
|
|
220
239
|
"classification": [
|
|
221
240
|
"PII",
|
|
222
241
|
"GDPR"
|
|
@@ -309,4 +328,4 @@
|
|
|
309
328
|
"_version": 1
|
|
310
329
|
}
|
|
311
330
|
]
|
|
312
|
-
}
|
|
331
|
+
}
|
|
@@ -20,10 +20,8 @@
|
|
|
20
20
|
"type": "string",
|
|
21
21
|
"enum": [
|
|
22
22
|
"aws-redshift",
|
|
23
|
-
"aws-dynamodb",
|
|
24
23
|
"aws-s3",
|
|
25
24
|
"aws-sqs",
|
|
26
|
-
"postgres",
|
|
27
25
|
"local",
|
|
28
26
|
"delta-share",
|
|
29
27
|
"http-api"
|
|
@@ -163,7 +161,10 @@
|
|
|
163
161
|
},
|
|
164
162
|
"clientAuthMethod": {
|
|
165
163
|
"type": "string",
|
|
166
|
-
"enum": [
|
|
164
|
+
"enum": [
|
|
165
|
+
"basic",
|
|
166
|
+
"post"
|
|
167
|
+
],
|
|
167
168
|
"description": "How client credentials are sent to the token endpoint: HTTP Basic header ('basic', default) or request body ('post')",
|
|
168
169
|
"default": "basic"
|
|
169
170
|
},
|
|
@@ -188,7 +189,13 @@
|
|
|
188
189
|
},
|
|
189
190
|
"httpMethod": {
|
|
190
191
|
"type": "string",
|
|
191
|
-
"enum": [
|
|
192
|
+
"enum": [
|
|
193
|
+
"GET",
|
|
194
|
+
"POST",
|
|
195
|
+
"PUT",
|
|
196
|
+
"PATCH",
|
|
197
|
+
"DELETE"
|
|
198
|
+
],
|
|
192
199
|
"description": "HTTP method to use for API requests",
|
|
193
200
|
"default": "GET"
|
|
194
201
|
},
|
|
@@ -208,7 +215,9 @@
|
|
|
208
215
|
"minimum": 1000
|
|
209
216
|
}
|
|
210
217
|
},
|
|
211
|
-
"required": [
|
|
218
|
+
"required": [
|
|
219
|
+
"method"
|
|
220
|
+
]
|
|
212
221
|
},
|
|
213
222
|
"_version": {
|
|
214
223
|
"type": "number",
|
|
@@ -222,21 +231,6 @@
|
|
|
222
231
|
],
|
|
223
232
|
"additionalProperties": false,
|
|
224
233
|
"examples": [
|
|
225
|
-
{
|
|
226
|
-
"name": "Production PostgreSQL Database",
|
|
227
|
-
"description": "Main production database for customer data",
|
|
228
|
-
"engine": "postgres",
|
|
229
|
-
"authentication": {
|
|
230
|
-
"method": "username-password",
|
|
231
|
-
"host": "prod-db.example.com",
|
|
232
|
-
"user": "app_user",
|
|
233
|
-
"password": "password123",
|
|
234
|
-
"database": "customers",
|
|
235
|
-
"schema": "public",
|
|
236
|
-
"port": "5432"
|
|
237
|
-
},
|
|
238
|
-
"_version": 1
|
|
239
|
-
},
|
|
240
234
|
{
|
|
241
235
|
"name": "Data Lake",
|
|
242
236
|
"description": "AWS S3 bucket containing analytics data",
|
|
@@ -338,4 +332,4 @@
|
|
|
338
332
|
"_version": 1
|
|
339
333
|
}
|
|
340
334
|
]
|
|
341
|
-
}
|
|
335
|
+
}
|