@forzalabs/remora 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/documentation/README.md +3 -2
- package/documentation/default_resources/project.json +1 -3
- package/index.js +5098 -2266
- package/json_schemas/code-set-schema.json +60 -0
- package/json_schemas/consumer-schema.json +13 -1
- package/json_schemas/producer-schema.json +61 -0
- package/json_schemas/project-schema.json +8 -0
- package/package.json +2 -1
- package/workers/ExecutorWorker.js +4903 -2213
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
+
"title": "Code Set Schema",
|
|
4
|
+
"description": "A named, reusable set of allowed values used to drive synthetic-data generation (the 'synth' command). Exactly one of 'values' or 'records' must be provided.",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["name"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"$schema": {
|
|
9
|
+
"type": "string",
|
|
10
|
+
"format": "uri"
|
|
11
|
+
},
|
|
12
|
+
"name": {
|
|
13
|
+
"type": "string",
|
|
14
|
+
"minLength": 1,
|
|
15
|
+
"description": "Unique name; referenced from a producer dimension's 'codeSet'."
|
|
16
|
+
},
|
|
17
|
+
"description": {
|
|
18
|
+
"type": "string",
|
|
19
|
+
"description": "Optional description of the code set"
|
|
20
|
+
},
|
|
21
|
+
"values": {
|
|
22
|
+
"type": "array",
|
|
23
|
+
"minItems": 1,
|
|
24
|
+
"items": {
|
|
25
|
+
"type": ["string", "number", "boolean"]
|
|
26
|
+
},
|
|
27
|
+
"description": "Flat list form: one value is picked per row. Mutually exclusive with 'records'."
|
|
28
|
+
},
|
|
29
|
+
"records": {
|
|
30
|
+
"type": "array",
|
|
31
|
+
"minItems": 1,
|
|
32
|
+
"items": {
|
|
33
|
+
"type": "object",
|
|
34
|
+
"minProperties": 1,
|
|
35
|
+
"additionalProperties": {
|
|
36
|
+
"type": ["string", "number", "boolean"]
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"description": "Structured form: correlated rows. Dimensions bind to a field and land on the same row per record. Mutually exclusive with 'values'."
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
"oneOf": [
|
|
43
|
+
{ "required": ["values"], "properties": { "values": { "type": "array" } } },
|
|
44
|
+
{ "required": ["records"], "properties": { "records": { "type": "array" } } }
|
|
45
|
+
],
|
|
46
|
+
"additionalProperties": false,
|
|
47
|
+
"examples": [
|
|
48
|
+
{
|
|
49
|
+
"name": "gap_statuses",
|
|
50
|
+
"values": ["OPEN", "CLOSED", "COMPLIANT", "NON-COMPLIANT", "EXCLUDED"]
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"name": "hedis_measures",
|
|
54
|
+
"records": [
|
|
55
|
+
{ "code": "CBP", "name": "Controlling High Blood Pressure", "system": "LOINC" },
|
|
56
|
+
{ "code": "COL", "name": "Colorectal Cancer Screening", "system": "CPT" }
|
|
57
|
+
]
|
|
58
|
+
}
|
|
59
|
+
]
|
|
60
|
+
}
|
|
@@ -83,6 +83,10 @@
|
|
|
83
83
|
"type": "string",
|
|
84
84
|
"description": "The dimension/measure of the producer OR '*'. If '*' then return all dimensions"
|
|
85
85
|
},
|
|
86
|
+
"description": {
|
|
87
|
+
"type": "string",
|
|
88
|
+
"description": "Optional human-readable description of this field. Descriptive metadata only; it documents intent and does not change runtime behaviour."
|
|
89
|
+
},
|
|
86
90
|
"alias": {
|
|
87
91
|
"type": "string",
|
|
88
92
|
"description": "If set, changes the key returned by the consumer to this alias"
|
|
@@ -1767,13 +1771,21 @@
|
|
|
1767
1771
|
"action": {
|
|
1768
1772
|
"type": "string",
|
|
1769
1773
|
"enum": [
|
|
1770
|
-
"move-file"
|
|
1774
|
+
"move-file",
|
|
1775
|
+
"run-consumer"
|
|
1771
1776
|
],
|
|
1772
1777
|
"description": "The action to perform"
|
|
1773
1778
|
},
|
|
1774
1779
|
"moveToDestination": {
|
|
1775
1780
|
"type": "string",
|
|
1776
1781
|
"description": "If the action is 'move-file', this specifies the source destination in remora where the source file should be moved"
|
|
1782
|
+
},
|
|
1783
|
+
"consumers": {
|
|
1784
|
+
"type": "array",
|
|
1785
|
+
"items": {
|
|
1786
|
+
"type": "string"
|
|
1787
|
+
},
|
|
1788
|
+
"description": "If the action is 'run-consumer', the names of the consumers to run sequentially once this output finishes"
|
|
1777
1789
|
}
|
|
1778
1790
|
},
|
|
1779
1791
|
"required": [
|
|
@@ -56,6 +56,10 @@
|
|
|
56
56
|
"type": "boolean",
|
|
57
57
|
"description": "Whether this is a primary key column or not"
|
|
58
58
|
},
|
|
59
|
+
"nullable": {
|
|
60
|
+
"type": "boolean",
|
|
61
|
+
"description": "Declares whether this dimension is allowed to hold null/missing values. Descriptive metadata only: it documents intent for consumers, schema generation and AI mappers, and does not change runtime behaviour (nulls are already passed through untouched). Use a consumer field 'required' validation to enforce presence."
|
|
62
|
+
},
|
|
59
63
|
"classification": {
|
|
60
64
|
"type": "array",
|
|
61
65
|
"items": {
|
|
@@ -84,12 +88,69 @@
|
|
|
84
88
|
"sourceFilename": {
|
|
85
89
|
"type": "boolean",
|
|
86
90
|
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
|
|
91
|
+
},
|
|
92
|
+
"allowedValues": {
|
|
93
|
+
"type": "array",
|
|
94
|
+
"minItems": 1,
|
|
95
|
+
"items": {
|
|
96
|
+
"type": ["string", "number", "boolean"]
|
|
97
|
+
},
|
|
98
|
+
"description": "Inline closed set of allowed values used only by synthetic-data generation ('synth'): one value is picked per row. Use 'codeSet' for a reusable/shared list. Mutually exclusive with 'codeSet'."
|
|
99
|
+
},
|
|
100
|
+
"codeSet": {
|
|
101
|
+
"description": "Reference to a named code set used only by synthetic-data generation ('synth'). A string names a flat code set (a value picked per row); an object binds this dimension to a field of a structured code set so correlated columns line up. Mutually exclusive with 'allowedValues'.",
|
|
102
|
+
"oneOf": [
|
|
103
|
+
{
|
|
104
|
+
"type": "string",
|
|
105
|
+
"minLength": 1
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"type": "object",
|
|
109
|
+
"required": ["name", "field"],
|
|
110
|
+
"additionalProperties": false,
|
|
111
|
+
"properties": {
|
|
112
|
+
"name": {
|
|
113
|
+
"type": "string",
|
|
114
|
+
"minLength": 1,
|
|
115
|
+
"description": "Name of a structured (records) code set."
|
|
116
|
+
},
|
|
117
|
+
"field": {
|
|
118
|
+
"type": "string",
|
|
119
|
+
"minLength": 1,
|
|
120
|
+
"description": "Field of the picked record to emit for this dimension."
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
]
|
|
125
|
+
},
|
|
126
|
+
"references": {
|
|
127
|
+
"type": "object",
|
|
128
|
+
"description": "Declares this dimension as a foreign key referencing another producer's field, used only by synthetic-data generation ('synth'). Generated values are drawn from the referenced parent field's key domain so joins/unions line up. Single-column relationships only. Mutually exclusive with 'allowedValues' and 'codeSet'.",
|
|
129
|
+
"required": ["producer", "field"],
|
|
130
|
+
"additionalProperties": false,
|
|
131
|
+
"properties": {
|
|
132
|
+
"producer": {
|
|
133
|
+
"type": "string",
|
|
134
|
+
"minLength": 1,
|
|
135
|
+
"description": "Name of the parent producer whose field this dimension references."
|
|
136
|
+
},
|
|
137
|
+
"field": {
|
|
138
|
+
"type": "string",
|
|
139
|
+
"minLength": 1,
|
|
140
|
+
"description": "Name of the parent producer's dimension whose key values are drawn from."
|
|
141
|
+
}
|
|
142
|
+
}
|
|
87
143
|
}
|
|
88
144
|
},
|
|
89
145
|
"required": [
|
|
90
146
|
"name",
|
|
91
147
|
"type"
|
|
92
148
|
],
|
|
149
|
+
"allOf": [
|
|
150
|
+
{ "not": { "required": ["allowedValues", "codeSet"], "properties": { "allowedValues": {}, "codeSet": {} } } },
|
|
151
|
+
{ "not": { "required": ["allowedValues", "references"], "properties": { "allowedValues": {}, "references": {} } } },
|
|
152
|
+
{ "not": { "required": ["codeSet", "references"], "properties": { "codeSet": {}, "references": {} } } }
|
|
153
|
+
],
|
|
93
154
|
"additionalProperties": false
|
|
94
155
|
},
|
|
95
156
|
"minItems": 1
|
|
@@ -54,6 +54,14 @@
|
|
|
54
54
|
},
|
|
55
55
|
"description": "Array of schema paths"
|
|
56
56
|
},
|
|
57
|
+
"codeSets": {
|
|
58
|
+
"type": "array",
|
|
59
|
+
"items": {
|
|
60
|
+
"type": "string",
|
|
61
|
+
"pattern": "^/"
|
|
62
|
+
},
|
|
63
|
+
"description": "Optional array of code-set paths used to drive synthetic-data generation"
|
|
64
|
+
},
|
|
57
65
|
"settings": {
|
|
58
66
|
"type": "object",
|
|
59
67
|
"required": ["SQL_MAX_QUERY_ROWS"],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forzalabs/remora",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
4
|
"description": "A powerful CLI tool for seamless data translation.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"private": false,
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"version": "npx tsx ./src/index.ts -v",
|
|
21
21
|
"run": "npm run sync:internal && npx tsx ./src/index.ts run",
|
|
22
22
|
"discover": "npm run sync:internal && npx tsx ./src/index.ts discover",
|
|
23
|
+
"synth": "npm run sync:internal && npx tsx ./src/index.ts synth",
|
|
23
24
|
"sample": "npm run sync:internal && npx tsx ./src/index.ts sample",
|
|
24
25
|
"compile": "npm run sync:internal && npx tsx ./src/index.ts compile",
|
|
25
26
|
"deploy": "npm run sync:internal && npx tsx ./src/index.ts deploy",
|