@forzalabs/remora 1.2.11 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/index.js +34377 -14350
- package/json_schemas/consumer-schema.json +744 -296
- package/json_schemas/producer-schema.json +36 -17
- package/json_schemas/source-schema.json +83 -20
- package/package.json +1 -1
- package/workers/ExecutorWorker.js +33949 -13948
|
@@ -44,6 +44,10 @@
|
|
|
44
44
|
],
|
|
45
45
|
"description": "The data type of the dimension"
|
|
46
46
|
},
|
|
47
|
+
"format": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"description": "Optional source format used when parsing/casting the value, e.g. a date format like 'YYYYMMDD' or 'DD-MM-YYYY'."
|
|
50
|
+
},
|
|
47
51
|
"alias": {
|
|
48
52
|
"type": "string",
|
|
49
53
|
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
|
|
@@ -72,7 +76,7 @@
|
|
|
72
76
|
"mask",
|
|
73
77
|
"crypt",
|
|
74
78
|
"random",
|
|
75
|
-
"seeded-random",
|
|
79
|
+
"seeded-random",
|
|
76
80
|
"none",
|
|
77
81
|
"{REMORA_MASK_IN_DEV}"
|
|
78
82
|
]
|
|
@@ -118,15 +122,6 @@
|
|
|
118
122
|
},
|
|
119
123
|
"settings": {
|
|
120
124
|
"type": "object",
|
|
121
|
-
"compressionType": {
|
|
122
|
-
"type": "string",
|
|
123
|
-
"enum": [
|
|
124
|
-
"GZ",
|
|
125
|
-
"TAR",
|
|
126
|
-
"ZIP"
|
|
127
|
-
],
|
|
128
|
-
"description": "The compression type used from the file to read"
|
|
129
|
-
},
|
|
130
125
|
"description": "Settings for the producer",
|
|
131
126
|
"properties": {
|
|
132
127
|
"sqlTable": {
|
|
@@ -150,7 +145,7 @@
|
|
|
150
145
|
"enum": [
|
|
151
146
|
"JSON",
|
|
152
147
|
"JSONL",
|
|
153
|
-
"CSV",
|
|
148
|
+
"CSV",
|
|
154
149
|
"TXT",
|
|
155
150
|
"XLS",
|
|
156
151
|
"XLSX",
|
|
@@ -172,8 +167,36 @@
|
|
|
172
167
|
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
|
|
173
168
|
},
|
|
174
169
|
"sheetName": {
|
|
175
|
-
"type": "string",
|
|
170
|
+
"type": "string",
|
|
176
171
|
"description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
|
|
172
|
+
},
|
|
173
|
+
"compressionType": {
|
|
174
|
+
"type": "string",
|
|
175
|
+
"enum": [
|
|
176
|
+
"GZ",
|
|
177
|
+
"ZIP",
|
|
178
|
+
"TAR"
|
|
179
|
+
],
|
|
180
|
+
"description": "The compression of the source file. The file is decompressed before parsing."
|
|
181
|
+
},
|
|
182
|
+
"startRow": {
|
|
183
|
+
"type": "number",
|
|
184
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed row to start reading from (e.g. 3 skips rows 1-2)."
|
|
185
|
+
},
|
|
186
|
+
"startColumn": {
|
|
187
|
+
"type": "number",
|
|
188
|
+
"description": "For Excel files (.xls/.xlsx), the 1-indexed column to start reading from (e.g. 2 starts at column B)."
|
|
189
|
+
},
|
|
190
|
+
"httpApi": {
|
|
191
|
+
"type": "object",
|
|
192
|
+
"description": "Additional settings when the source is an HTTP API.",
|
|
193
|
+
"properties": {
|
|
194
|
+
"dataProperty": {
|
|
195
|
+
"type": "string",
|
|
196
|
+
"description": "If the response wraps the records in a property, its name (e.g. 'result' for { success: true, result: [...] })."
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
"additionalProperties": false
|
|
177
200
|
}
|
|
178
201
|
},
|
|
179
202
|
"additionalProperties": false
|
|
@@ -213,10 +236,6 @@
|
|
|
213
236
|
{
|
|
214
237
|
"name": "email",
|
|
215
238
|
"type": "string",
|
|
216
|
-
"format": {
|
|
217
|
-
"type": "string",
|
|
218
|
-
"description": "Format for datetype of the source ('YYYYMMDD', 'DD-MM-YYYY')"
|
|
219
|
-
},
|
|
220
239
|
"classification": [
|
|
221
240
|
"PII",
|
|
222
241
|
"GDPR"
|
|
@@ -309,4 +328,4 @@
|
|
|
309
328
|
"_version": 1
|
|
310
329
|
}
|
|
311
330
|
]
|
|
312
|
-
}
|
|
331
|
+
}
|
|
@@ -20,10 +20,8 @@
|
|
|
20
20
|
"type": "string",
|
|
21
21
|
"enum": [
|
|
22
22
|
"aws-redshift",
|
|
23
|
-
"aws-dynamodb",
|
|
24
23
|
"aws-s3",
|
|
25
24
|
"aws-sqs",
|
|
26
|
-
"postgres",
|
|
27
25
|
"local",
|
|
28
26
|
"delta-share",
|
|
29
27
|
"http-api"
|
|
@@ -44,6 +42,7 @@
|
|
|
44
42
|
"implicit",
|
|
45
43
|
"bearer-token",
|
|
46
44
|
"api-key",
|
|
45
|
+
"oidc",
|
|
47
46
|
"none"
|
|
48
47
|
],
|
|
49
48
|
"description": "The authentication method to use"
|
|
@@ -112,6 +111,18 @@
|
|
|
112
111
|
"type": "string",
|
|
113
112
|
"description": "IAM role or profile name"
|
|
114
113
|
},
|
|
114
|
+
"roleArn": {
|
|
115
|
+
"type": "string",
|
|
116
|
+
"description": "ARN of the IAM role to assume via STS (method 'arn'), e.g. arn:aws:iam::123456789012:role/remora-reader. The base identity that assumes it comes from the AWS SDK default credential chain, not from this config."
|
|
117
|
+
},
|
|
118
|
+
"roleSessionName": {
|
|
119
|
+
"type": "string",
|
|
120
|
+
"description": "Session name attached to the assumed-role credentials (method 'arn'). Defaults to 'remora'."
|
|
121
|
+
},
|
|
122
|
+
"externalId": {
|
|
123
|
+
"type": "string",
|
|
124
|
+
"description": "Optional STS ExternalId for third-party / cross-account trust policies (method 'arn')."
|
|
125
|
+
},
|
|
115
126
|
"clusterId": {
|
|
116
127
|
"type": "string",
|
|
117
128
|
"description": "Redshift cluster identifier"
|
|
@@ -128,6 +139,35 @@
|
|
|
128
139
|
"type": "string",
|
|
129
140
|
"description": "Bearer token used for authentication (Delta Sharing or HTTP API)"
|
|
130
141
|
},
|
|
142
|
+
"tokenEndpoint": {
|
|
143
|
+
"type": "string",
|
|
144
|
+
"description": "OAuth2 token endpoint used with the 'oidc' method to obtain an access token via the client-credentials grant (e.g. Okta or Cognito)"
|
|
145
|
+
},
|
|
146
|
+
"clientId": {
|
|
147
|
+
"type": "string",
|
|
148
|
+
"description": "OAuth2 client identifier for the 'oidc' method"
|
|
149
|
+
},
|
|
150
|
+
"clientSecret": {
|
|
151
|
+
"type": "string",
|
|
152
|
+
"description": "OAuth2 client secret for the 'oidc' method"
|
|
153
|
+
},
|
|
154
|
+
"scope": {
|
|
155
|
+
"type": "string",
|
|
156
|
+
"description": "Space-delimited OAuth2 scopes to request (required by Okta/Cognito for client-credentials)"
|
|
157
|
+
},
|
|
158
|
+
"audience": {
|
|
159
|
+
"type": "string",
|
|
160
|
+
"description": "Optional OAuth2 audience/resource parameter for IdPs that require it"
|
|
161
|
+
},
|
|
162
|
+
"clientAuthMethod": {
|
|
163
|
+
"type": "string",
|
|
164
|
+
"enum": [
|
|
165
|
+
"basic",
|
|
166
|
+
"post"
|
|
167
|
+
],
|
|
168
|
+
"description": "How client credentials are sent to the token endpoint: HTTP Basic header ('basic', default) or request body ('post')",
|
|
169
|
+
"default": "basic"
|
|
170
|
+
},
|
|
131
171
|
"url": {
|
|
132
172
|
"type": "string",
|
|
133
173
|
"format": "uri",
|
|
@@ -149,7 +189,13 @@
|
|
|
149
189
|
},
|
|
150
190
|
"httpMethod": {
|
|
151
191
|
"type": "string",
|
|
152
|
-
"enum": [
|
|
192
|
+
"enum": [
|
|
193
|
+
"GET",
|
|
194
|
+
"POST",
|
|
195
|
+
"PUT",
|
|
196
|
+
"PATCH",
|
|
197
|
+
"DELETE"
|
|
198
|
+
],
|
|
153
199
|
"description": "HTTP method to use for API requests",
|
|
154
200
|
"default": "GET"
|
|
155
201
|
},
|
|
@@ -169,7 +215,9 @@
|
|
|
169
215
|
"minimum": 1000
|
|
170
216
|
}
|
|
171
217
|
},
|
|
172
|
-
"required": [
|
|
218
|
+
"required": [
|
|
219
|
+
"method"
|
|
220
|
+
]
|
|
173
221
|
},
|
|
174
222
|
"_version": {
|
|
175
223
|
"type": "number",
|
|
@@ -183,21 +231,6 @@
|
|
|
183
231
|
],
|
|
184
232
|
"additionalProperties": false,
|
|
185
233
|
"examples": [
|
|
186
|
-
{
|
|
187
|
-
"name": "Production PostgreSQL Database",
|
|
188
|
-
"description": "Main production database for customer data",
|
|
189
|
-
"engine": "postgres",
|
|
190
|
-
"authentication": {
|
|
191
|
-
"method": "username-password",
|
|
192
|
-
"host": "prod-db.example.com",
|
|
193
|
-
"user": "app_user",
|
|
194
|
-
"password": "password123",
|
|
195
|
-
"database": "customers",
|
|
196
|
-
"schema": "public",
|
|
197
|
-
"port": "5432"
|
|
198
|
-
},
|
|
199
|
-
"_version": 1
|
|
200
|
-
},
|
|
201
234
|
{
|
|
202
235
|
"name": "Data Lake",
|
|
203
236
|
"description": "AWS S3 bucket containing analytics data",
|
|
@@ -239,6 +272,19 @@
|
|
|
239
272
|
},
|
|
240
273
|
"_version": 1
|
|
241
274
|
},
|
|
275
|
+
{
|
|
276
|
+
"name": "S3 via assumed IAM role",
|
|
277
|
+
"description": "S3 source that assumes a cross-account IAM role via STS. Base identity comes from the SDK default chain (instance/task role, SSO, or an assume-only key in env vars) — no data-access keys stored here.",
|
|
278
|
+
"engine": "aws-s3",
|
|
279
|
+
"authentication": {
|
|
280
|
+
"method": "arn",
|
|
281
|
+
"region": "eu-west-1",
|
|
282
|
+
"bucket": "analytics-exports",
|
|
283
|
+
"roleArn": "arn:aws:iam::123456789012:role/remora-s3-reader",
|
|
284
|
+
"externalId": "{REMORA_STS_EXTERNAL_ID}"
|
|
285
|
+
},
|
|
286
|
+
"_version": 1
|
|
287
|
+
},
|
|
242
288
|
{
|
|
243
289
|
"name": "REST API with Bearer Token",
|
|
244
290
|
"description": "HTTP API source with bearer token authentication",
|
|
@@ -267,6 +313,23 @@
|
|
|
267
313
|
"httpMethod": "GET"
|
|
268
314
|
},
|
|
269
315
|
"_version": 1
|
|
316
|
+
},
|
|
317
|
+
{
|
|
318
|
+
"name": "Delta Share with OIDC",
|
|
319
|
+
"description": "Delta Sharing source authenticated via OAuth2 client-credentials (Okta/Cognito)",
|
|
320
|
+
"engine": "delta-share",
|
|
321
|
+
"authentication": {
|
|
322
|
+
"method": "oidc",
|
|
323
|
+
"host": "https://sharing.example.com/delta-sharing",
|
|
324
|
+
"share": "my_share",
|
|
325
|
+
"schema": "my_schema",
|
|
326
|
+
"table": "my_table",
|
|
327
|
+
"tokenEndpoint": "https://example.okta.com/oauth2/default/v1/token",
|
|
328
|
+
"clientId": "{DELTA_SHARE_CLIENT_ID}",
|
|
329
|
+
"clientSecret": "{DELTA_SHARE_CLIENT_SECRET}",
|
|
330
|
+
"scope": "delta_sharing"
|
|
331
|
+
},
|
|
332
|
+
"_version": 1
|
|
270
333
|
}
|
|
271
334
|
]
|
|
272
|
-
}
|
|
335
|
+
}
|