@forzalabs/remora 2.3.0 → 2.5.0

package/json_schemas/consumer-schema.json

@@ -81,7 +81,7 @@
                 "properties": {
                     "key": {
                         "type": "string",
-                        "description": "The dimension/measure of the producer, OR '*' for every dimension of one producer, OR a pattern holding a '*' (e.g. 'addr*Street') that selects every matching field — each keeping the rest of this field's configuration, so one entry can transform or validate a whole family of columns."
+                        "description": "The dimension of the producer, OR '*' for every dimension of one producer, OR a pattern holding a '*' (e.g. 'addr*Street') that selects every matching field — each keeping the rest of this field's configuration, so one entry can transform or validate a whole family of columns."
                     },
                     "description": {
                         "type": "string",
@@ -290,6 +290,46 @@
                     "copyFrom": {
                         "type": "string",
                         "description": "If set, this field will be added as new to the consumer dataset and will be a copy of the specified field. Use the alias if set, otherwise the key. The source field should come before this field in the fields list."
+                    },
+                    "position": {
+                        "type": "object",
+                        "description": "'FIXED' output only, and required on every visible field of one: where this field sits on the positional line and how its value is filled out to that width. Column order is field order and the offsets are computed from the widths, so no column number is transcribed by hand. A value longer than 'width' fails the run rather than being cut.",
+                        "additionalProperties": false,
+                        "required": ["width"],
+                        "properties": {
+                            "width": {
+                                "type": "integer",
+                                "minimum": 1,
+                                "description": "How many characters the field occupies on the line."
+                            },
+                            "align": {
+                                "type": "string",
+                                "enum": ["left", "right"],
+                                "description": "Which end of the field the value sits at, the other end taking the padding. Defaults by the field's type: 'left' for a string, 'right' for a number."
+                            },
+                            "pad": {
+                                "type": "string",
+                                "minLength": 1,
+                                "maxLength": 1,
+                                "description": "The single character the field is filled out with. Defaults by the field's type: a space for a string, '0' for a number."
+                            },
+                            "nullFill": {
+                                "type": "string",
+                                "minLength": 1,
+                                "maxLength": 1,
+                                "description": "The single character the full width of a NULL field is filled with. Defaults to a space, not to 'pad': zero-filling an empty numeric field writes a real value where there was none. An empty string counts as a null here."
+                            },
+                            "overflow": {
+                                "type": "string",
+                                "enum": ["fail", "truncate"],
+                                "description": "What a value longer than 'width' does. 'fail' (the default) stops the run naming the field, the value and both widths; 'truncate' cuts the value to 'width', for the layouts that genuinely specify truncation."
+                            },
+                            "start": {
+                                "type": "integer",
+                                "minimum": 1,
+                                "description": "Optional 1-based column the field is expected to start at. Not a second source of truth — the offsets always come from the widths — but an assertion that compile recomputes and fails on, so a forgotten filler is caught where it is written."
+                            }
+                        }
                     }
                 },
                 "required": [
@@ -301,7 +341,7 @@
         },
         "outputs": {
             "type": "array",
-            "description": "Output formats and configurations for the consumer",
+            "description": "Where and how the consumer exports its data. Exactly one: a run stages its records in one encoding and moves that file to the destination, so a second output has nothing left to write.",
             "items": {
                 "type": "object",
                 "properties": {
@@ -310,10 +350,12 @@
                         "enum": [
                             "API",
                             "CSV",
+                            "FIXED",
                             "JSON",
+                            "PARQUET",
                             "TXT"
                         ],
-                        "description": "The output format of the consumer. JSON/CSV/TXT export to a file destination; API exposes the result over the API."
+                        "description": "The output format of the consumer. JSON/CSV/TXT/PARQUET/FIXED export to a file destination; API exposes the result over the API. PARQUET writes a typed, SNAPPY-compressed columnar file whose schema comes from the consumer's dimension types. FIXED writes a positional flat file — no delimiter, no quotes, no header — where each visible field declares its own 'position'."
                     },
                     "exportDestination": {
                         "type": "string",
@@ -325,15 +367,90 @@
                     },
                     "delimiter": {
                         "type": "string",
-                        "description": "Only used when the format is 'TXT'. The single-character delimiter used between fields in the exported file. Defaults to a tab if not set."
+                        "description": "Only used when the format is 'TXT'. The single-character delimiter used between fields in the exported file. Defaults to a tab if not set. Ignored by every other format, PARQUET included."
                     },
                     "header": {
                         "type": "boolean",
-                        "description": "'CSV'/'TXT' only. Writes the field names as the first line of each exported file, quoted and delimited exactly as the rows under them. Defaults to true; set it to false for a destination that reads the first line as data."
+                        "description": "'CSV'/'TXT' only. Writes the field names as the first line of each exported file, quoted and delimited exactly as the rows under them. Defaults to true; set it to false for a destination that reads the first line as data. Rejected on PARQUET, which carries its own schema."
                     },
                     "omitNull": {
                         "type": "boolean",
-                        "description": "'JSON'/'API' only. Omits the fields whose value is null from each exported record instead of writing them as \"field\": null — worth it on a wide, sparse record. Records then legitimately differ in shape, so it is rejected on a positional format (CSV/TXT), and on a JSON output whose consumer also writes one, since every output is serialized from the same record."
+                        "description": "'JSON'/'API' only. Omits the fields whose value is null from each exported record instead of writing them as \"field\": null — worth it on a wide, sparse record. Records then legitimately differ in shape, so it is rejected on a positional format (CSV/TXT/PARQUET), and on a JSON output whose consumer also writes one, since every output is serialized from the same record."
+                    },
+                    "recordLength": {
+                        "type": "integer",
+                        "minimum": 1,
+                        "description": "'FIXED' only, and not alongside 'layouts', where each layout declares its own. The declared width of one record, which compile checks the field widths sum to and the export checks every composed line against. It is the length of the record as written, not the length of the copybook it came from: the fields past the cut are simply not declared rather than padded to. Every record is exactly this long, including one whose last field is null — that field is written full width in 'nullFill' and the line is never cut short."
+                    },
+                    "defaults": {
+                        "type": "object",
+                        "description": "'FIXED' only. The fill rules every field of one type takes unless its own 'position' overrides them, keyed by the field's type. A wide layout declares 'numbers are right-aligned and zero-padded' once instead of on each of 90 fields.",
+                        "additionalProperties": false,
+                        "properties": {
+                            "string": { "$ref": "#/definitions/fixedFieldFill" },
+                            "number": { "$ref": "#/definitions/fixedFieldFill" },
+                            "datetime": { "$ref": "#/definitions/fixedFieldFill" },
+                            "boolean": { "$ref": "#/definitions/fixedFieldFill" }
+                        }
+                    },
+                    "lineEnding": {
+                        "type": "string",
+                        "enum": ["lf", "crlf"],
+                        "description": "'FIXED' only. What ends every record, the last one included. Defaults to 'lf'. A real positional feed picks one and holds it — the NASCO POD files are CRLF on every line. A file with no terminator at all is fixed record framing (RECFM=FB) and is not expressible here."
+                    },
+                    "layouts": {
+                        "type": "object",
+                        "description": "'FIXED' only. Several record layouts in one file, picked per record by the value of one field — the write-side mirror of a producer's 'recordSelector'. The records come out grouped, one contiguous block per value in 'order', keeping within each block the order they were read in. Rejected together with a top-level 'recordLength': each layout declares its own. Without it a FIXED output writes every visible field as one layout.",
+                        "additionalProperties": false,
+                        "required": ["discriminator", "order", "values"],
+                        "properties": {
+                            "discriminator": {
+                                "type": "string",
+                                "description": "The consumer field whose value picks the layout, by the name it is exported under (the alias where one is set). It does not have to be written by any of the layouts, though it usually is."
+                            },
+                            "order": {
+                                "type": "array",
+                                "description": "The discriminator values in the order their blocks are written. The complete list: a record carrying a value that is not here has no block to go in and fails the run.",
+                                "items": { "type": "string" },
+                                "minItems": 1,
+                                "uniqueItems": true
+                            },
+                            "values": {
+                                "type": "object",
+                                "description": "The layout each discriminator value is written with. Every value in 'order' needs an entry.",
+                                "minProperties": 1,
+                                "additionalProperties": {
+                                    "type": "object",
+                                    "additionalProperties": false,
+                                    "required": ["fields"],
+                                    "properties": {
+                                        "fields": {
+                                            "type": "array",
+                                            "description": "The fields this layout writes, in column order, by their exported names. A subset of the consumer's visible fields: the consumer declares the union of every layout's fields and each layout names the ones its records carry.",
+                                            "items": { "type": "string" },
+                                            "minItems": 1,
+                                            "uniqueItems": true
+                                        },
+                                        "recordLength": {
+                                            "type": "integer",
+                                            "minimum": 1,
+                                            "description": "As the output's 'recordLength', for this layout alone. A multi-layout output has no single one."
+                                        },
+                                        "defaults": {
+                                            "type": "object",
+                                            "description": "As the output's 'defaults', applied over them for this layout's fields alone.",
+                                            "additionalProperties": false,
+                                            "properties": {
+                                                "string": { "$ref": "#/definitions/fixedFieldFill" },
+                                                "number": { "$ref": "#/definitions/fixedFieldFill" },
+                                                "datetime": { "$ref": "#/definitions/fixedFieldFill" },
+                                                "boolean": { "$ref": "#/definitions/fixedFieldFill" }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
                     },
                     "trigger": {
                         "type": "object",
@@ -443,7 +560,8 @@
                 ],
                 "additionalProperties": false
             },
-            "minItems": 1
+            "minItems": 1,
+            "maxItems": 1
         },
         "filters": {
             "type": "array",
@@ -765,6 +883,35 @@
     ],
     "additionalProperties": false,
     "definitions": {
+        "fixedFieldFill": {
+            "type": "object",
+            "description": "How a value is filled out to the width of its fixed-width field, set once for every field of one type. A field's own 'position' wins where both set the same rule.",
+            "additionalProperties": false,
+            "properties": {
+                "align": {
+                    "type": "string",
+                    "enum": ["left", "right"],
+                    "description": "Which end of the field the value sits at, the other end taking the padding. Defaults by the field's type: 'left' for a string, 'right' for a number."
+                },
+                "pad": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 1,
+                    "description": "The single character the field is filled out with. Defaults by the field's type: a space for a string, '0' for a number."
+                },
+                "nullFill": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 1,
+                    "description": "The single character the full width of a NULL field is filled with. Defaults to a space, not to 'pad': zero-filling an empty numeric field writes a real value where there was none. An empty string counts as a null here."
+                },
+                "overflow": {
+                    "type": "string",
+                    "enum": ["fail", "truncate"],
+                    "description": "What a value longer than 'width' does. 'fail' (the default) stops the run naming the field, the value and both widths; 'truncate' cuts the value to 'width', for the layouts that genuinely specify truncation."
+                }
+            }
+        },
         "singleTransformation": {
             "oneOf": [
                 {
@@ -1875,9 +2022,6 @@
                 }
             ],
             "outputs": [
-                {
-                    "format": "API"
-                },
                 {
                     "format": "JSON",
                     "exportDestination": "Data Lake",

package/json_schemas/producer-schema.json

@@ -26,32 +26,6 @@
             "items": { "$ref": "#/definitions/dimension" },
             "minItems": 1
         },
-        "measures": {
-            "type": "array",
-            "description": "Optional measures for the producer",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string",
-                        "description": "The name of the measure"
-                    },
-                    "description": {
-                        "type": "string",
-                        "description": "Optional description of the measure"
-                    },
-                    "sql": {
-                        "type": "string",
-                        "description": "SQL command used to create a metric"
-                    }
-                },
-                "required": [
-                    "name",
-                    "sql"
-                ],
-                "additionalProperties": false
-            }
-        },
         "settings": {
             "type": "object",
             "description": "Settings for the producer",
@@ -515,18 +489,6 @@
                     "type": "datetime"
                 }
             ],
-            "measures": [
-                {
-                    "name": "total_orders",
-                    "description": "Total number of orders by this customer",
-                    "sql": "COUNT(order_id)"
-                },
-                {
-                    "name": "total_spent",
-                    "description": "Total amount spent by this customer",
-                    "sql": "SUM(order_amount)"
-                }
-            ],
             "settings": {
                 "sqlTable": "customers"
             },

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "2.3.0",
+    "version": "2.5.0",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -14,8 +14,9 @@
         "sync:drivers": "cd ../.. && npm run build --workspace=@remora/app-drivers",
         "watch:drivers": "cd ../.. && npm run dev --workspace=@remora/app-drivers",
         "sync:internal": "cd ../.. && npm run sync:internal",
+        "sync:preview": "cd ../.. && npm run build:preview",
         "watch:internal": "cd ../.. && npm run watch:internal",
-        "ts-check": "npx tsc --noemit",
+        "ts-check": "npm run sync:preview && npx tsc --noemit",
         "perf": "npm run sync:internal && npm run fast-build && npm run copy-static-file && npx tsx ./scripts/perf/RunPerfBaseline.ts",
         "perf:quick": "npx tsx ./scripts/perf/RunPerfBaseline.ts",
         "init": "npm run sync:internal && npx tsx ./src/index.ts init",
@@ -64,6 +65,7 @@
         "fast-xml-parser": "^5.3.5",
         "fs-extra": "^11.1.0",
         "hyparquet": "^1.17.4",
+        "hyparquet-writer": "^0.16.12",
         "inquirer": "^8.2.5",
         "json-schema": "^0.4.0",
         "jsonwebtoken": "^9.0.2",