@forzalabs/remora 1.2.5 → 1.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +6989 -1230
- package/json_schemas/project-schema.json +16 -1
- package/json_schemas/source-schema.json +23 -0
- package/package.json +15 -11
- package/workers/ExecutorWorker.js +7151 -1387
|
@@ -63,6 +63,18 @@
|
|
|
63
63
|
"minimum": 1,
|
|
64
64
|
"description": "Maximum number of rows for SQL queries"
|
|
65
65
|
},
|
|
66
|
+
"API_QUEUE": {
|
|
67
|
+
"type": "object",
|
|
68
|
+
"required": ["source"],
|
|
69
|
+
"additionalProperties": false,
|
|
70
|
+
"properties": {
|
|
71
|
+
"source": {
|
|
72
|
+
"type": "string",
|
|
73
|
+
"minLength": 1,
|
|
74
|
+
"description": "Name of an aws-sqs source used as an alternative execute-consumer API"
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
},
|
|
66
78
|
"STRING_MAX_CHARACTERS_LENGTH": {
|
|
67
79
|
"type": "integer",
|
|
68
80
|
"minimum": 1,
|
|
@@ -88,7 +100,10 @@
|
|
|
88
100
|
"sources": ["/sources"],
|
|
89
101
|
"schemas": ["/schemas"],
|
|
90
102
|
"settings": {
|
|
91
|
-
"SQL_MAX_QUERY_ROWS": 10000
|
|
103
|
+
"SQL_MAX_QUERY_ROWS": 10000,
|
|
104
|
+
"API_QUEUE": {
|
|
105
|
+
"source": "remora-api-queue"
|
|
106
|
+
}
|
|
92
107
|
}
|
|
93
108
|
}
|
|
94
109
|
]
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
"aws-redshift",
|
|
23
23
|
"aws-dynamodb",
|
|
24
24
|
"aws-s3",
|
|
25
|
+
"aws-sqs",
|
|
25
26
|
"postgres",
|
|
26
27
|
"local",
|
|
27
28
|
"delta-share",
|
|
@@ -51,6 +52,10 @@
|
|
|
51
52
|
"type": "string",
|
|
52
53
|
"description": "Hostname or endpoint of the data source"
|
|
53
54
|
},
|
|
55
|
+
"accountId": {
|
|
56
|
+
"type": "string",
|
|
57
|
+
"description": "AWS account ID used to construct resource URLs when only a resource name is provided"
|
|
58
|
+
},
|
|
54
59
|
"user": {
|
|
55
60
|
"type": "string",
|
|
56
61
|
"description": "Username for authentication"
|
|
@@ -95,6 +100,10 @@
|
|
|
95
100
|
"type": "string",
|
|
96
101
|
"description": "AWS region"
|
|
97
102
|
},
|
|
103
|
+
"queue": {
|
|
104
|
+
"type": "string",
|
|
105
|
+
"description": "SQS queue URL or queue name"
|
|
106
|
+
},
|
|
98
107
|
"bucket": {
|
|
99
108
|
"type": "string",
|
|
100
109
|
"description": "S3 bucket name"
|
|
@@ -202,6 +211,20 @@
|
|
|
202
211
|
},
|
|
203
212
|
"_version": 2
|
|
204
213
|
},
|
|
214
|
+
{
|
|
215
|
+
"name": "Consumer Execute Queue",
|
|
216
|
+
"description": "SQS queue used as an alternative execute-consumer API",
|
|
217
|
+
"engine": "aws-sqs",
|
|
218
|
+
"authentication": {
|
|
219
|
+
"method": "access-secret-key",
|
|
220
|
+
"queue": "remora-execute-consumer",
|
|
221
|
+
"accountId": "{AWS_ACCOUNT_ID}",
|
|
222
|
+
"region": "us-east-1",
|
|
223
|
+
"accessKey": "{AWS_ACCESS_KEY_ID}",
|
|
224
|
+
"secretKey": "{AWS_SECRET_ACCESS_KEY}"
|
|
225
|
+
},
|
|
226
|
+
"_version": 1
|
|
227
|
+
},
|
|
205
228
|
{
|
|
206
229
|
"name": "Redshift Data Warehouse",
|
|
207
230
|
"engine": "aws-redshift",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.7",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -8,19 +8,23 @@
|
|
|
8
8
|
"remora": "index.js"
|
|
9
9
|
},
|
|
10
10
|
"scripts": {
|
|
11
|
-
"dev": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
11
|
+
"dev": "clear && npm run sync:internal && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
12
12
|
"dev:build": "clear && npm install --workspaces && clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
13
|
-
"dev:w": "clear && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
13
|
+
"dev:w": "clear && npm run sync:internal && npm run fast-build && clear && npx tsx scripts/dev.ts",
|
|
14
|
+
"sync:drivers": "cd ../.. && npm run build --workspace=@remora/app-drivers",
|
|
15
|
+
"watch:drivers": "cd ../.. && npm run dev --workspace=@remora/app-drivers",
|
|
16
|
+
"sync:internal": "cd ../.. && npm run sync:internal",
|
|
17
|
+
"watch:internal": "cd ../.. && npm run watch:internal",
|
|
14
18
|
"ts-check": "npx tsc --noemit",
|
|
15
|
-
"init": "npx tsx ./src/index.ts init",
|
|
19
|
+
"init": "npm run sync:internal && npx tsx ./src/index.ts init",
|
|
16
20
|
"version": "npx tsx ./src/index.ts -v",
|
|
17
|
-
"run": "npx tsx ./src/index.ts run",
|
|
18
|
-
"discover": "npx tsx ./src/index.ts discover",
|
|
19
|
-
"sample": "npx tsx ./src/index.ts sample",
|
|
20
|
-
"compile": "npx tsx ./src/index.ts compile",
|
|
21
|
-
"deploy": "npx tsx ./src/index.ts deploy",
|
|
22
|
-
"
|
|
23
|
-
"create-producer": "npx tsx ./src/index.ts create-producer",
|
|
21
|
+
"run": "npm run sync:internal && npx tsx ./src/index.ts run",
|
|
22
|
+
"discover": "npm run sync:internal && npx tsx ./src/index.ts discover",
|
|
23
|
+
"sample": "npm run sync:internal && npx tsx ./src/index.ts sample",
|
|
24
|
+
"compile": "npm run sync:internal && npx tsx ./src/index.ts compile",
|
|
25
|
+
"deploy": "npm run sync:internal && npx tsx ./src/index.ts deploy",
|
|
26
|
+
"verify": "npm run sync:internal && npx tsx ./src/index.ts verify",
|
|
27
|
+
"create-producer": "npm run sync:internal && npx tsx ./src/index.ts create-producer",
|
|
24
28
|
"copy-static-file": "npx tsx ./scripts/CopyStaticFile.ts",
|
|
25
29
|
"build": "npm i && tsup && npm run copy-static-file",
|
|
26
30
|
"fast-build": "tsup",
|