@forzalabs/remora 2.1.1 → 2.2.0

package/json_schemas/consumer-schema.json

@@ -634,7 +634,25 @@
         },
         "maximumFileSize": {
             "type": "number",
-            "description": "The maximum size of each output file, expressed in Gb. When the current output file exceeds this size it is rotated and a new file is started."
+            "description": "Shorthand for split.maxFileSizeGb: the maximum size of each output file, expressed in Gb. When the current output file reaches this size it is rotated and a new numbered file is started. Can't be combined with 'split'."
+        },
+        "split": {
+            "type": "object",
+            "description": "Splits the output into several files: by the value of one or more fields, by a size cap, or both (key first, then size within each key). At least one of the two properties must be set.",
+            "properties": {
+                "by": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "Consumer field key(s) — the alias where one is set. Each distinct combination of values is written to its own file, named after the value(s). A null value buckets as '_null' and an empty one as '_empty'."
+                },
+                "maxFileSizeGb": {
+                    "type": "number",
+                    "description": "The maximum size of ONE output file, expressed in Gb. With 'by' set the cap applies per key, not across the whole output."
+                }
+            },
+            "additionalProperties": false
         },
         "_version": {
             "type": "number",

package/json_schemas/producer-schema.json

@@ -58,16 +58,12 @@
             "properties": {
                 "sqlTable": {
                     "type": "string",
-                    "description": "The SQL table name, which is a concise way of setting the sql property to 'SELECT * FROM <sql table name>'"
+                    "description": "The table a SQL producer reads, as 'table' or 'schema.table'. The statement is built from it and the producer's dimensions"
                 },
                 "direct": {
                     "type": "boolean",
                     "description": "If true, no view is created on the producer side due to permission limitations"
                 },
-                "sql": {
-                    "type": "string",
-                    "description": "The name of the file that has the SQL statement to run to create this producer"
-                },
                 "fileKey": {
                     "type": "string",
                     "description": "For S3/local sources: the file key/path that identifies the file to read. For HTTP API sources: the API endpoint path (e.g., '/api/v1/users')"
@@ -126,7 +122,7 @@
                 },
                 "disablePushdown": {
                     "type": "boolean",
-                    "description": "Stops the run's filters and record limit from being sent to the source as query hints (Delta Share). Both are still applied here, so this only costs bandwidth."
+                    "description": "Stops the run's filters and record limit from being sent to the source — as query hints on a Delta Share, as a WHERE and a LIMIT on a SQL source. Both are still applied here, so this only costs bandwidth."
                 },
                 "hasHeaderRow": {
                     "type": "boolean",
@@ -254,7 +250,7 @@
                 },
                 "nullable": {
                     "type": "boolean",
-                    "description": "Declares whether this dimension is allowed to hold null/missing values. Descriptive metadata only: it documents intent for consumers, schema generation and AI mappers, and does not change runtime behaviour (nulls are already passed through untouched). Use a consumer field 'required' validation to enforce presence."
+                    "description": "Declares whether this dimension is allowed to hold null/missing values. Defaults to true. A missing value (null, or the empty field a NULL is staged as) is kept as null; set this to false to get the type's own empty value instead (string -> \"\", number -> 0, boolean -> false; a datetime stays null, having no empty value). Use a consumer field 'required' validation to reject a missing value rather than fill it."
                 },
                 "classification": {
                     "type": "array",

package/json_schemas/project-schema.json

@@ -3,7 +3,7 @@
     "title": "Project Schema",
     "description": "Schema for defining remora project configuration",
     "type": "object",
-    "required": ["name", "version", "consumers", "producers", "sources", "settings"],
+    "required": ["name", "version", "settings"],
     "properties": {
         "$schema": {
             "type": "string",
@@ -28,7 +28,8 @@
                 "type": "string",
                 "pattern": "^/"
             },
-            "description": "Array of consumer paths"
+            "deprecated": true,
+            "description": "DEPRECATED and ignored. Resource folders are fixed: /sources, /producers, /consumers, /schemas and /code_sets. Remove this property"
         },
         "producers": {
             "type": "array",
@@ -36,7 +37,8 @@
                 "type": "string",
                 "pattern": "^/"
             },
-            "description": "Array of producer paths"
+            "deprecated": true,
+            "description": "DEPRECATED and ignored. Resource folders are fixed: /sources, /producers, /consumers, /schemas and /code_sets. Remove this property"
         },
         "sources": {
             "type": "array",
@@ -44,7 +46,8 @@
                 "type": "string",
                 "pattern": "^/"
             },
-            "description": "Array of source paths"
+            "deprecated": true,
+            "description": "DEPRECATED and ignored. Resource folders are fixed: /sources, /producers, /consumers, /schemas and /code_sets. Remove this property"
         },
         "schemas": {
             "type": "array",
@@ -52,7 +55,8 @@
                 "type": "string",
                 "pattern": "^/"
             },
-            "description": "Array of schema paths"
+            "deprecated": true,
+            "description": "DEPRECATED and ignored. Resource folders are fixed: /sources, /producers, /consumers, /schemas and /code_sets. Remove this property"
         },
         "codeSets": {
             "type": "array",
@@ -60,7 +64,8 @@
                 "type": "string",
                 "pattern": "^/"
             },
-            "description": "Optional array of code-set paths used to drive synthetic-data generation"
+            "deprecated": true,
+            "description": "DEPRECATED and ignored. Resource folders are fixed: /sources, /producers, /consumers, /schemas and /code_sets. Remove this property"
         },
         "settings": {
             "type": "object",
@@ -108,10 +113,6 @@
             "name": "analytics-project",
             "version": "1.0.0",
             "description": "Analytics data processing project",
-            "consumers": ["/consumers"],
-            "producers": ["/producers"],
-            "sources": ["/sources"],
-            "schemas": ["/schemas"],
             "settings": {
                 "SQL_MAX_QUERY_ROWS": 10000,
                 "API_QUEUE": {

package/json_schemas/source-schema.json

@@ -25,7 +25,8 @@
                 "aws-sqs",
                 "local",
                 "delta-share",
-                "http-api"
+                "http-api",
+                "presigned-url"
             ],
             "description": "The type of data engine"
         },
@@ -229,6 +230,29 @@
                     "description": "Request timeout in milliseconds",
                     "default": 30000,
                     "minimum": 1000
+                },
+                "signedUrlField": {
+                    "type": "string",
+                    "description": "presigned-url only. Dot path to the signed URL inside the minting endpoint's JSON response, e.g. 'data.uploadUrl'",
+                    "default": "url"
+                },
+                "signedHeadersField": {
+                    "type": "string",
+                    "description": "presigned-url only. Dot path to an object of headers in the minting response. Set it when the minter signs headers (a Content-Type, an x-amz-*): the transfer must send them back verbatim or S3 answers 403 SignatureDoesNotMatch"
+                },
+                "fileNameParam": {
+                    "type": "string",
+                    "description": "presigned-url only. Name of the field carrying the file name to the minting endpoint - in the JSON body on a POST/PUT/PATCH, as a query parameter otherwise",
+                    "default": "filename"
+                },
+                "signedUrlMethod": {
+                    "type": "string",
+                    "enum": [
+                        "GET",
+                        "PUT",
+                        "POST"
+                    ],
+                    "description": "presigned-url only. HTTP method used against the signed URL itself. Defaults to GET when the source is read from and PUT when it is exported to"
                 }
             },
             "required": [
@@ -377,6 +401,38 @@
                 "scope": "delta_sharing"
             },
             "_version": 1
+        },
+        {
+            "name": "Partner upload",
+            "description": "Exports to a bucket we hold no credentials for: an API Gateway endpoint with M2M (OAuth2 client-credentials) auth signs an upload URL per file, and remora PUTs to it",
+            "engine": "presigned-url",
+            "authentication": {
+                "method": "oidc",
+                "url": "https://api.example.com/v1/uploads/presign",
+                "httpMethod": "POST",
+                "fileNameParam": "filename",
+                "signedUrlField": "uploadUrl",
+                "tokenEndpoint": "https://example.auth.eu-west-1.amazoncognito.com/oauth2/token",
+                "clientId": "{PRESIGN_CLIENT_ID}",
+                "clientSecret": "{PRESIGN_CLIENT_SECRET}",
+                "scope": "uploads/write"
+            },
+            "_version": 1
+        },
+        {
+            "name": "Partner download",
+            "description": "Reads a file signed on demand by an API key protected endpoint. The producer's fileKey is the name sent to it",
+            "engine": "presigned-url",
+            "authentication": {
+                "method": "api-key",
+                "url": "https://api.example.com/v1/downloads/presign",
+                "httpMethod": "GET",
+                "apiKey": "{PRESIGN_API_KEY}",
+                "apiKeyHeader": "x-api-key",
+                "fileNameParam": "key",
+                "signedUrlField": "data.url"
+            },
+            "_version": 1
         }
     ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "2.1.1",
+    "version": "2.2.0",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,