@forzalabs/remora 1.1.15 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/CHANGELOG.md +113 -0
  2. package/index.js +381 -124
  3. package/json_schemas/consumer-schema.json +107 -21
  4. package/package.json +1 -1
  5. package/workers/ExecutorWorker.js +371 -114
package/json_schemas/consumer-schema.json CHANGED
@@ -129,31 +129,70 @@
129
129
  ]
130
130
  },
131
131
  "validate": {
132
- "type": "object",
133
- "description": "Rules to check field value compliance and data quality",
134
- "properties": {
135
- "min": {
136
- "type": "number",
137
- "description": "Minimum value for numeric fields"
138
- },
139
- "max": {
140
- "type": "number",
141
- "description": "Maximum value for numeric fields"
142
- },
143
- "regex": {
144
- "type": "string",
145
- "description": "Regular expression pattern to validate string fields"
132
+ "type": "array",
133
+ "description": "Rules to check field value compliance and data quality. Each validation has its own rule and action to take on failure.",
134
+ "items": {
135
+ "type": "object",
136
+ "properties": {
137
+ "rule": {
138
+ "type": "object",
139
+ "description": "The validation rule to check",
140
+ "oneOf": [
141
+ {
142
+ "properties": { "min": { "type": "number", "description": "Minimum value for numeric fields" } },
143
+ "required": ["min"],
144
+ "additionalProperties": false
145
+ },
146
+ {
147
+ "properties": { "max": { "type": "number", "description": "Maximum value for numeric fields" } },
148
+ "required": ["max"],
149
+ "additionalProperties": false
150
+ },
151
+ {
152
+ "properties": { "regex": { "type": "string", "description": "Regular expression pattern to validate string fields" } },
153
+ "required": ["regex"],
154
+ "additionalProperties": false
155
+ },
156
+ {
157
+ "properties": { "required": { "type": "boolean", "const": true, "description": "Whether the field value must be present" } },
158
+ "required": ["required"],
159
+ "additionalProperties": false
160
+ },
161
+ {
162
+ "properties": { "min_length": { "type": "number", "description": "Minimum string length" } },
163
+ "required": ["min_length"],
164
+ "additionalProperties": false
165
+ },
166
+ {
167
+ "properties": { "max_length": { "type": "number", "description": "Maximum string length" } },
168
+ "required": ["max_length"],
169
+ "additionalProperties": false
170
+ },
171
+ {
172
+ "properties": { "in": { "type": "array", "items": { "oneOf": [{ "type": "string" }, { "type": "number" }, { "type": "boolean" }] }, "description": "Allowed values" } },
173
+ "required": ["in"],
174
+ "additionalProperties": false
175
+ },
176
+ {
177
+ "properties": { "not_in": { "type": "array", "items": { "oneOf": [{ "type": "string" }, { "type": "number" }, { "type": "boolean" }] }, "description": "Disallowed values" } },
178
+ "required": ["not_in"],
179
+ "additionalProperties": false
180
+ }
181
+ ]
182
+ },
183
+ "onFail": {
184
+ "type": "string",
185
+ "description": "Action to take when validation fails",
186
+ "enum": ["fail", "skip", "warn", "set_default"]
187
+ }
146
188
  },
147
- "required": {
148
- "type": "boolean",
149
- "description": "Whether the field is required"
150
- }
151
- },
152
- "additionalProperties": false
189
+ "required": ["rule", "onFail"],
190
+ "additionalProperties": false
191
+ }
153
192
  },
154
193
  "onError": {
155
194
  "type": "string",
156
- "description": "Action to take if an error occurs during transformations or validation",
195
+ "description": "Action to take if an error occurs during transformations",
157
196
  "enum": ["set_default", "skip", "fail"]
158
197
  },
159
198
  "default": {
@@ -463,6 +502,53 @@
463
502
  "_version": {
464
503
  "type": "number",
465
504
  "description": "Version number of the consumer configuration"
505
+ },
506
+ "validate": {
507
+ "type": "array",
508
+ "description": "Dataset-level validations applied to the final result set before export",
509
+ "items": {
510
+ "type": "object",
511
+ "properties": {
512
+ "rule": {
513
+ "type": "object",
514
+ "description": "The dataset validation rule to check",
515
+ "oneOf": [
516
+ {
517
+ "properties": { "unique_fields": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "Field(s) that must have unique values across the dataset" } },
518
+ "required": ["unique_fields"],
519
+ "additionalProperties": false
520
+ },
521
+ {
522
+ "properties": { "min_rows": { "type": "number", "description": "Minimum number of rows expected in the dataset" } },
523
+ "required": ["min_rows"],
524
+ "additionalProperties": false
525
+ },
526
+ {
527
+ "properties": { "max_rows": { "type": "number", "description": "Maximum number of rows allowed in the dataset" } },
528
+ "required": ["max_rows"],
529
+ "additionalProperties": false
530
+ },
531
+ {
532
+ "properties": { "no_duplicates": { "type": "boolean", "const": true, "description": "No fully duplicate rows allowed" } },
533
+ "required": ["no_duplicates"],
534
+ "additionalProperties": false
535
+ },
536
+ {
537
+ "properties": { "not_empty": { "type": "boolean", "const": true, "description": "Dataset must contain at least one row" } },
538
+ "required": ["not_empty"],
539
+ "additionalProperties": false
540
+ }
541
+ ]
542
+ },
543
+ "onFail": {
544
+ "type": "string",
545
+ "description": "Action to take when dataset validation fails",
546
+ "enum": ["fail", "warn"]
547
+ }
548
+ },
549
+ "required": ["rule", "onFail"],
550
+ "additionalProperties": false
551
+ }
466
552
  }
467
553
  },
468
554
  "required": [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@forzalabs/remora",
3
- "version": "1.1.15",
3
+ "version": "1.2.2",
4
4
  "description": "A powerful CLI tool for seamless data translation.",
5
5
  "main": "index.js",
6
6
  "private": false,