@forzalabs/remora 1.1.10 → 1.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/documentation/default_resources/consumer.json +16 -52
- package/documentation/default_resources/mock_data.csv +6 -0
- package/documentation/default_resources/producer.json +15 -32
- package/documentation/default_resources/project.json +2 -2
- package/documentation/default_resources/source.json +7 -14
- package/index.js +1320 -2694
- package/package.json +2 -1
- package/workers/ExecutorWorker.js +1797 -3186
|
@@ -1,53 +1,17 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
],
|
|
20
|
-
"fields": [
|
|
21
|
-
{ "key": "<producer field name>", "from": "<producer name>" },
|
|
22
|
-
{ "key": "<original field name>", "from": "<producer name>", "alias": "<new field name>" },
|
|
23
|
-
{ "key": "<secondary producer field name>", "from": "<secondary producer name>" },
|
|
24
|
-
{ "key": "<another field name>", "from": "<producer name>" }
|
|
25
|
-
],
|
|
26
|
-
"filters": [
|
|
27
|
-
{
|
|
28
|
-
"sql": "<filter condition>"
|
|
29
|
-
}
|
|
30
|
-
],
|
|
31
|
-
"outputs": [
|
|
32
|
-
{ "format": "API" },
|
|
33
|
-
{
|
|
34
|
-
"format": "JSON",
|
|
35
|
-
"exportDestination": "<export destination>"
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
"format": "CSV",
|
|
39
|
-
"exportDestination": "<export destination>",
|
|
40
|
-
"trigger": {
|
|
41
|
-
"type": "CRON",
|
|
42
|
-
"value": "0 0 * * *"
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
],
|
|
46
|
-
"metadata": {
|
|
47
|
-
"<metadata tag key>": "<metadata tag value>",
|
|
48
|
-
"owner_email": "<owner email>"
|
|
49
|
-
},
|
|
50
|
-
"project": "<project name>",
|
|
51
|
-
"schema": "<schema name>",
|
|
52
|
-
"_version": 1
|
|
53
|
-
}
|
|
2
|
+
"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
|
|
3
|
+
"name": "c_default",
|
|
4
|
+
"producers": [
|
|
5
|
+
{ "name": "p_default" }
|
|
6
|
+
],
|
|
7
|
+
"fields": [
|
|
8
|
+
{ "key": "id" },
|
|
9
|
+
{ "key": "name" },
|
|
10
|
+
{ "key": "age" },
|
|
11
|
+
{ "key": "country" },
|
|
12
|
+
{ "key": "spend" }
|
|
13
|
+
],
|
|
14
|
+
"outputs": [
|
|
15
|
+
{ "format": "JSON", "exportDestination": "s_default" }
|
|
16
|
+
]
|
|
17
|
+
}
|
|
@@ -1,33 +1,16 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"mask": "<hash | mask | crypt>",
|
|
18
|
-
"description": "<field description>"
|
|
19
|
-
}
|
|
20
|
-
],
|
|
21
|
-
"measures": [
|
|
22
|
-
{
|
|
23
|
-
"name": "<calculated measure name>",
|
|
24
|
-
"description": "<measure description>",
|
|
25
|
-
"sql": "<sql expression>"
|
|
26
|
-
}
|
|
27
|
-
],
|
|
28
|
-
"settings": {
|
|
29
|
-
"sqlTable": "<source table name>",
|
|
30
|
-
"direct": true
|
|
31
|
-
},
|
|
32
|
-
"_version": 1
|
|
33
|
-
}
|
|
2
|
+
"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
|
|
3
|
+
"name": "p_default",
|
|
4
|
+
"source": "s_default",
|
|
5
|
+
"dimensions": [
|
|
6
|
+
{ "name": "id", "type": "string" },
|
|
7
|
+
{ "name": "name", "type": "string" },
|
|
8
|
+
{ "name": "age", "type": "number" },
|
|
9
|
+
{ "name": "country", "type": "string" },
|
|
10
|
+
{ "name": "spend", "type": "number" }
|
|
11
|
+
],
|
|
12
|
+
"settings": {
|
|
13
|
+
"fileType": "CSV",
|
|
14
|
+
"fileKey": "mock_data.csv"
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json",
|
|
3
|
-
"name": "
|
|
3
|
+
"name": "Default Remora Project",
|
|
4
4
|
"version": "1.0.0",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "This is the standard Remora project created when you run init.",
|
|
6
6
|
"consumers": ["/consumers"],
|
|
7
7
|
"producers": ["/producers"],
|
|
8
8
|
"sources": ["/sources"],
|
|
@@ -1,16 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
"accessKey": "<access key var from env>",
|
|
10
|
-
"secretKey": "<secret key var from env>",
|
|
11
|
-
"region": "<aws region>",
|
|
12
|
-
"database": "<db name>",
|
|
13
|
-
"workgroup": "<db workgroup>"
|
|
14
|
-
},
|
|
15
|
-
"engine": "<aws-redshift | aws-dynamodb | aws-s3 | postgres | local>"
|
|
2
|
+
"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
|
|
3
|
+
"name": "s_default",
|
|
4
|
+
"authentication": {
|
|
5
|
+
"method": "implicit",
|
|
6
|
+
"path": "./default_remora_data"
|
|
7
|
+
},
|
|
8
|
+
"engine": "local"
|
|
16
9
|
}
|