@forzalabs/remora 2.2.1 → 2.4.0

package/json_schemas/consumer-schema.json

@@ -81,7 +81,7 @@
                 "properties": {
                     "key": {
                         "type": "string",
-                        "description": "The dimension/measure of the producer, OR '*' for every dimension of one producer, OR a pattern holding a '*' (e.g. 'addr*Street') that selects every matching field — each keeping the rest of this field's configuration, so one entry can transform or validate a whole family of columns."
+                        "description": "The dimension of the producer, OR '*' for every dimension of one producer, OR a pattern holding a '*' (e.g. 'addr*Street') that selects every matching field — each keeping the rest of this field's configuration, so one entry can transform or validate a whole family of columns."
                     },
                     "description": {
                         "type": "string",
@@ -301,7 +301,7 @@
         },
         "outputs": {
             "type": "array",
-            "description": "Output formats and configurations for the consumer",
+            "description": "Where and how the consumer exports its data. Exactly one: a run stages its records in one encoding and moves that file to the destination, so a second output has nothing left to write.",
             "items": {
                 "type": "object",
                 "properties": {
@@ -311,9 +311,10 @@
                             "API",
                             "CSV",
                             "JSON",
+                            "PARQUET",
                             "TXT"
                         ],
-                        "description": "The output format of the consumer. JSON/CSV/TXT export to a file destination; API exposes the result over the API."
+                        "description": "The output format of the consumer. JSON/CSV/TXT/PARQUET export to a file destination; API exposes the result over the API. PARQUET writes a typed, SNAPPY-compressed columnar file whose schema comes from the consumer's dimension types."
                     },
                     "exportDestination": {
                         "type": "string",
@@ -325,11 +326,15 @@
                     },
                     "delimiter": {
                         "type": "string",
-                        "description": "Only used when the format is 'TXT'. The single-character delimiter used between fields in the exported file. Defaults to a tab if not set."
+                        "description": "Only used when the format is 'TXT'. The single-character delimiter used between fields in the exported file. Defaults to a tab if not set. Ignored by every other format, PARQUET included."
+                    },
+                    "header": {
+                        "type": "boolean",
+                        "description": "'CSV'/'TXT' only. Writes the field names as the first line of each exported file, quoted and delimited exactly as the rows under them. Defaults to true; set it to false for a destination that reads the first line as data. Rejected on PARQUET, which carries its own schema."
                     },
                     "omitNull": {
                         "type": "boolean",
-                        "description": "'JSON'/'API' only. Omits the fields whose value is null from each exported record instead of writing them as \"field\": null — worth it on a wide, sparse record. Records then legitimately differ in shape, so it is rejected on a positional format (CSV/TXT), and on a JSON output whose consumer also writes one, since every output is serialized from the same record."
+                        "description": "'JSON'/'API' only. Omits the fields whose value is null from each exported record instead of writing them as \"field\": null — worth it on a wide, sparse record. Records then legitimately differ in shape, so it is rejected on a positional format (CSV/TXT/PARQUET), and on a JSON output whose consumer also writes one, since every output is serialized from the same record."
                     },
                     "trigger": {
                         "type": "object",
@@ -439,7 +444,8 @@
                 ],
                 "additionalProperties": false
             },
-            "minItems": 1
+            "minItems": 1,
+            "maxItems": 1
         },
         "filters": {
             "type": "array",
@@ -1871,9 +1877,6 @@
                 }
             ],
             "outputs": [
-                {
-                    "format": "API"
-                },
                 {
                     "format": "JSON",
                     "exportDestination": "Data Lake",

package/json_schemas/producer-schema.json

@@ -26,32 +26,6 @@
             "items": { "$ref": "#/definitions/dimension" },
             "minItems": 1
         },
-        "measures": {
-            "type": "array",
-            "description": "Optional measures for the producer",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string",
-                        "description": "The name of the measure"
-                    },
-                    "description": {
-                        "type": "string",
-                        "description": "Optional description of the measure"
-                    },
-                    "sql": {
-                        "type": "string",
-                        "description": "SQL command used to create a metric"
-                    }
-                },
-                "required": [
-                    "name",
-                    "sql"
-                ],
-                "additionalProperties": false
-            }
-        },
         "settings": {
             "type": "object",
             "description": "Settings for the producer",
@@ -515,18 +489,6 @@
                     "type": "datetime"
                 }
             ],
-            "measures": [
-                {
-                    "name": "total_orders",
-                    "description": "Total number of orders by this customer",
-                    "sql": "COUNT(order_id)"
-                },
-                {
-                    "name": "total_spent",
-                    "description": "Total amount spent by this customer",
-                    "sql": "SUM(order_amount)"
-                }
-            ],
             "settings": {
                 "sqlTable": "customers"
             },

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "2.2.1",
+    "version": "2.4.0",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,
@@ -14,8 +14,9 @@
         "sync:drivers": "cd ../.. && npm run build --workspace=@remora/app-drivers",
         "watch:drivers": "cd ../.. && npm run dev --workspace=@remora/app-drivers",
         "sync:internal": "cd ../.. && npm run sync:internal",
+        "sync:preview": "cd ../.. && npm run build:preview",
         "watch:internal": "cd ../.. && npm run watch:internal",
-        "ts-check": "npx tsc --noemit",
+        "ts-check": "npm run sync:preview && npx tsc --noemit",
         "perf": "npm run sync:internal && npm run fast-build && npm run copy-static-file && npx tsx ./scripts/perf/RunPerfBaseline.ts",
         "perf:quick": "npx tsx ./scripts/perf/RunPerfBaseline.ts",
         "init": "npm run sync:internal && npx tsx ./src/index.ts init",
@@ -64,6 +65,7 @@
         "fast-xml-parser": "^5.3.5",
         "fs-extra": "^11.1.0",
         "hyparquet": "^1.17.4",
+        "hyparquet-writer": "^0.16.12",
         "inquirer": "^8.2.5",
         "json-schema": "^0.4.0",
         "jsonwebtoken": "^9.0.2",