@forzalabs/remora 0.0.12

package/definitions/json_schemas/producer-schema.json

@@ -0,0 +1,236 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Producer Schema",
+    "description": "Schema for defining data producers",
+    "type": "object",
+    "properties": {
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "name": {
+            "type": "string",
+            "description": "The name of the producer"
+        },
+        "description": {
+            "type": "string",
+            "description": "Optional description of the producer"
+        },
+        "source": {
+            "type": "string",
+            "description": "The name of the Source"
+        },
+        "dimensions": {
+            "type": "array",
+            "description": "Dimensions for the producer",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string",
+                        "description": "The name of the dimension"
+                    },
+                    "description": {
+                        "type": "string",
+                        "description": "Optional description of the dimension"
+                    },
+                    "type": {
+                        "type": "string",
+                        "enum": [
+                            "string",
+                            "number",
+                            "datetime"
+                        ],
+                        "description": "The data type of the dimension"
+                    },
+                    "alias": {
+                        "type": "string",
+                        "description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name"
+                    },
+                    "pk": {
+                        "type": "boolean",
+                        "description": "Whether this is a primary key column or not"
+                    },
+                    "classification": {
+                        "type": "array",
+                        "items": {
+                            "type": "string",
+                            "enum": [
+                                "PHI",
+                                "PII",
+                                "GDPR"
+                            ],
+                            "description": "Classification categories for this dimension"
+                        }
+                    },
+                    "mask": {
+                        "type": "string",
+                        "enum": [
+                            "hash",
+                            "mask",
+                            "crypt"
+                        ],
+                        "description": "Masking type to apply to this dimension"
+                    }
+                },
+                "required": [
+                    "name",
+                    "type"
+                ],
+                "additionalProperties": false
+            },
+            "minItems": 1
+        },
+        "measures": {
+            "type": "array",
+            "description": "Optional measures for the producer",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string",
+                        "description": "The name of the measure"
+                    },
+                    "description": {
+                        "type": "string",
+                        "description": "Optional description of the measure"
+                    },
+                    "sql": {
+                        "type": "string",
+                        "description": "SQL command used to create a metric"
+                    }
+                },
+                "required": [
+                    "name",
+                    "sql"
+                ],
+                "additionalProperties": false
+            }
+        },
+        "settings": {
+            "type": "object",
+            "description": "Settings for the producer",
+            "properties": {
+                "sqlTable": {
+                    "type": "string",
+                    "description": "The SQL table name, which is a concise way of setting the sql property to 'SELECT * FROM <sql table name>'"
+                },
+                "direct": {
+                    "type": "boolean",
+                    "description": "If true, no view is created on the producer side due to permission limitations"
+                },
+                "sql": {
+                    "type": "string",
+                    "description": "The name of the file that has the SQL statement to run to create this producer"
+                },
+                "fileKey": {
+                    "type": "string",
+                    "description": "If the source is a bucket, this is the file key that identifies the file to read"
+                },
+                "fileType": {
+                    "type": "string",
+                    "enum": [
+                        "JSON",
+                        "JSONL",
+                        "CSV"
+                    ],
+                    "description": "The type of file to read"
+                }
+            },
+            "additionalProperties": false
+        },
+        "_version": {
+            "type": "number",
+            "description": "Version number of the producer configuration"
+        }
+    },
+    "required": [
+        "name",
+        "source",
+        "dimensions",
+        "settings",
+        "_version"
+    ],
+    "additionalProperties": false,
+    "examples": [
+        {
+            "name": "CustomerData",
+            "description": "Producer for customer data from the main database",
+            "source": "Production PostgreSQL Database",
+            "dimensions": [
+                {
+                    "name": "customer_id",
+                    "type": "string",
+                    "pk": true
+                },
+                {
+                    "name": "full_name",
+                    "type": "string",
+                    "classification": [
+                        "PII",
+                        "GDPR"
+                    ],
+                    "mask": "hash"
+                },
+                {
+                    "name": "email",
+                    "type": "string",
+                    "classification": [
+                        "PII",
+                        "GDPR"
+                    ],
+                    "mask": "mask"
+                },
+                {
+                    "name": "signup_date",
+                    "type": "datetime"
+                }
+            ],
+            "measures": [
+                {
+                    "name": "total_orders",
+                    "description": "Total number of orders by this customer",
+                    "sql": "COUNT(order_id)"
+                },
+                {
+                    "name": "total_spent",
+                    "description": "Total amount spent by this customer",
+                    "sql": "SUM(order_amount)"
+                }
+            ],
+            "settings": {
+                "sqlTable": "customers"
+            },
+            "_version": 1
+        },
+        {
+            "name": "SalesData",
+            "description": "Producer for sales data from S3 bucket",
+            "source": "Data Lake",
+            "dimensions": [
+                {
+                    "name": "transaction_id",
+                    "type": "string",
+                    "pk": true
+                },
+                {
+                    "name": "product_id",
+                    "type": "string"
+                },
+                {
+                    "name": "sale_amount",
+                    "type": "number"
+                },
+                {
+                    "name": "sale_date",
+                    "type": "datetime"
+                }
+            ],
+            "settings": {
+                "fileKey": "sales/daily_sales.csv",
+                "fileType": "CSV"
+            },
+            "_version": 2
+        }
+    ]
+}

package/definitions/json_schemas/project-schema.json

@@ -0,0 +1,59 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "required": ["name", "version", "consumers", "producers", "sources", "schemas", "settings"],
+    "properties": {
+        "name": {
+            "type": "string",
+            "description": "Name of the remora project"
+        },
+        "version": {
+            "type": "string",
+            "pattern": "^\\d+\\.\\d+\\.\\d+$",
+            "description": "Version of the project in semver format"
+        },
+        "consumers": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": "^/"
+            },
+            "description": "Array of consumer paths"
+        },
+        "producers": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": "^/"
+            },
+            "description": "Array of producer paths"
+        },
+        "sources": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": "^/"
+            },
+            "description": "Array of source paths"
+        },
+        "schemas": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                "pattern": "^/"
+            },
+            "description": "Array of schema paths"
+        },
+        "settings": {
+            "type": "object",
+            "required": ["SQL_MAX_QUERY_ROWS"],
+            "properties": {
+                "SQL_MAX_QUERY_ROWS": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "description": "Maximum number of rows for SQL queries"
+                }
+            }
+        }
+    }
+}

package/definitions/json_schemas/source-schema.json

@@ -0,0 +1,109 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Source Schema",
+    "description": "Schema for defining data sources and their authentication methods",
+    "type": "object",
+    "properties": {
+        "$schema": {
+            "type": "string",
+            "format": "uri"
+        },
+        "name": {
+            "type": "string",
+            "description": "The name of the data source"
+        },
+        "description": {
+            "type": "string",
+            "description": "Optional description of the data source"
+        },
+        "engine": {
+            "type": "string",
+            "enum": [
+                "aws-redshift",
+                "aws-dynamodb",
+                "aws-s3",
+                "postgres",
+                "local"
+            ],
+            "description": "The type of data engine"
+        },
+        "authentication": {
+            "type": "object",
+            "description": "Authentication details for connecting to the data source",
+            "properties": {
+                "method": {
+                    "type": "string",
+                    "enum": [
+                        "iam",
+                        "username-password",
+                        "access-secret-key",
+                        "arn",
+                        "implicit"
+                    ],
+                    "description": "The authentication method to use"
+                }
+            },
+            "required": [
+                "method"
+            ],
+            "additionalProperties": {
+                "type": "string"
+            }
+        },
+        "_version": {
+            "type": "number",
+            "description": "Version number of the source configuration"
+        }
+    },
+    "required": [
+        "name",
+        "engine",
+        "authentication",
+        "_version"
+    ],
+    "additionalProperties": false,
+    "examples": [
+        {
+            "name": "Production PostgreSQL Database",
+            "description": "Main production database for customer data",
+            "engine": "postgres",
+            "authentication": {
+                "method": "username-password",
+                "host": "prod-db.example.com",
+                "user": "app_user",
+                "password": "password123",
+                "dbName": "customers",
+                "schema": "public",
+                "port": "5432"
+            },
+            "_version": 1
+        },
+        {
+            "name": "Data Lake",
+            "description": "AWS S3 bucket containing analytics data",
+            "engine": "aws-s3",
+            "authentication": {
+                "method": "access-secret-key",
+                "accessKey": "AKIAIOSFODNN7EXAMPLE",
+                "secretKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+                "region": "us-east-1",
+                "bucket": "analytics-data"
+            },
+            "_version": 2
+        },
+        {
+            "name": "Redshift Data Warehouse",
+            "engine": "aws-redshift",
+            "authentication": {
+                "method": "iam",
+                "host": "redshift-cluster.region.redshift.amazonaws.com",
+                "dbName": "analytics",
+                "port": "5439",
+                "iamProfile": "redshift-access-role",
+                "region": "us-west-2",
+                "clusterId": "analytics-cluster"
+            },
+            "_version": 1
+        }
+    ]
+}

package/definitions/requests/ConsumerRequest.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/requests/Developer.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/requests/Mapping.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/requests/ProducerRequest.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/requests/Request.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Compiled.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Consumer.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Environment.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Library.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Producer.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Project.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Schema.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/definitions/resources/Source.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/documentation/README.md

@@ -0,0 +1,123 @@
+# Configuration Guide
+
+This README explains the available configurations for each resource type in the project. 
+Each section details the expected values for different keys in the JSON files.
+
+## Remora Configuration
+
+The main project configuration file.
+
+| Key | Description | Possible Values |
+|-----|-------------|----------------|
+| `name` | Project identifier | Any string |
+| `version` | Project version number | Semantic version (e.g., `1.0.0`) |
+| `description` | Project purpose | Any descriptive text |
+| `consumers` | Path to consumer definitions | Directory path as string |
+| `producers` | Path to producer definitions | Directory path as string |
+| `sources` | Path to source definitions | Directory path as string |
+| `schemas` | Path to schema definitions | Directory path as string |
+| `settings.SQL_MAX_QUERY_ROWS` | Maximum rows in query results | Positive integer |
+| `settings.DEFAULT_TIMEOUT_SECONDS` | Maximum query execution time | Positive integer (seconds) |
+| `settings.LOG_LEVEL` | Logging detail level | `DEBUG`, `INFO`, `WARNING`, `ERROR` |
+
+## Source Configuration
+
+Sources define the connection to your data storage systems.
+
+| Key | Description | Possible Values |
+|-----|-------------|----------------|
+| `name` | Unique identifier for the source | Any string without spaces |
+| `description` | Explanation of what this source represents | Any descriptive text |
+| `_version` | Version number for tracking changes | Positive integer |
+| `engine` | Type of data storage system | `aws-redshift`, `aws-dynamodb`, `aws-s3`, `postgres`, `local` |
+| `authentication.method` | Authentication method for connecting to the source | `iam`, `username-password`, `access-secret-key`, `arn`, `implicit` |
+| `authentication.schema` | Database schema to use | Schema name as string |
+| `authentication.bucketName` | Bucket s3 name | Bucket name as string |
+| `authentication.accessKey` | AWS access key | String (preferably environment variable) |
+| `authentication.secretKey` | AWS secret key | String (preferably environment variable) |
+| `authentication.region` | AWS region | AWS region code (e.g., `us-east-1`) |
+| `authentication.database` | Database name | String |
+| `authentication.workgroup` | AWS workgroup | String |
+| `authentication.host` | Database host (for username-password method) | Hostname or IP address |
+| `authentication.user` | Database username (for username-password method) | String |
+| `authentication.password` | Database password (for username-password method) | String (preferably environment variable) |
+| `authentication.port` | Database port (for username-password method) | Integer |
+
+## Producer Configuration
+
+Producers extract data from sources and define the structure of the data.
+
+| Key | Description | Possible Values |
+|-----|-------------|----------------|
+| `name` | Unique identifier for the producer | Any string without spaces |
+| `description` | Explanation of what this producer represents | Any descriptive text |
+| `source` | Name of the source this producer uses | Must match a source `name` |
+| `_version` | Version number for tracking changes | Positive integer |
+| `dimensions[].name` | Name of the field/column | Any string without spaces |
+| `dimensions[].type` | Data type of the field | `string`, `number`, `datetime` |
+| `dimensions[].pk` | Whether this is a primary key | `true`, `false` |
+| `dimensions[].description` | Description of the field | Any descriptive text |
+| `dimensions[].alias` | Alternative name for the column in the source | String |
+| `dimensions[].classification` | Data privacy classification | Array containing: `PII`, `PHI`, `GDPR` |
+| `dimensions[].mask` | How to protect sensitive data | `hash`, `mask`, `crypt` |
+| `measures[].name` | Name of calculated measure | Any string without spaces |
+| `measures[].description` | Description of the measure | Any descriptive text |
+| `measures[].sql` | SQL expression to calculate the measure | Valid SQL expression (e.g., `SUM(field_name)`) |
+| `settings.sqlTable` | Source table name | Table name as string |
+| `settings.direct` | Whether to query directly without creating views | `true`, `false` |
+| `settings.sql` | Custom SQL for this producer | SQL query string |
+| `settings.fileKey` | File key for S3 sources | S3 file path |
+| `settings.fileType` | File format for file-based sources | `JSON`, `JSONL`, `CSV` |
+
+## Consumer Configuration
+
+Consumers transform and combine data from producers for specific use cases.
+
+| Key | Description | Possible Values |
+|-----|-------------|----------------|
+| `name` | Unique identifier for the consumer | Any string without spaces |
+| `description` | Explanation of what this consumer represents | Any descriptive text |
+| `_version` | Version number for tracking changes | Positive integer |
+| `producers[].name` | Name of a producer this consumer uses | Must match a producer `name` |
+| `producers[].joins[].otherName` | Name of another producer to join with | Must match a producer `name` |
+| `producers[].joins[].relationship` | Type of relationship for the join | `one-to-one`, `one-to-many`, `many-to-one` |
+| `producers[].joins[].sql` | SQL expression for the join | Valid SQL join condition |
+| `producers[].custom.language` | Language for custom transformations | `js`, `python` |
+| `producers[].custom.code` | Code for custom transformations | Valid code in the specified language |
+| `fields[].key` | Field name from producer or `*` for all fields | Field name or `*` |
+| `fields[].alias` | New name for the field in the output | Any string without spaces |
+| `fields[].from` | Producer to get the field from | Must match a producer `name` |
+| `fields[].grouping.groupingKey` | Field to group by | Any string without spaces |
+| `fields[].grouping.subFields` | Fields to include in each group | Array of field objects |
+| `filters[].sql` | SQL condition for filtering | SQL WHERE condition |
+| `filters[].rule.member` | Field to filter on | Field name |
+| `filters[].rule.operator` | Comparison operator | `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `contains`, `not_contains` |
+| `filters[].rule.values` | Values to compare against | Array of strings |
+| `outputs[].format` | Output format | `SQL`, `API`, `CSV`, `PARQUET`, `JSON` |
+| `outputs[].accellerated` | Whether to materialize for performance | `true`, `false` |
+| `outputs[].direct` | Whether to query directly without creating views | `true`, `false` |
+| `outputs[].exportDestination` | Where to export data | Must match a source `name` |
+| `outputs[].trigger.type` | How to trigger exports | `CRON`, `API` |
+| `outputs[].trigger.value` | Trigger expression | CRON expression (e.g., `0 0 * * *`) or endpoint path |
+| `metadata` | Custom tags | Object with string keys and values |
+| `project` | Project grouping | String |
+| `schema` | JSON schema to validate against | Must match a schema `title` |
+
+## Schema Configuration
+
+Schemas define the structure and validation rules for data.
+
+| Key | Description | Possible Values |
+|-----|-------------|----------------|
+| `$schema` | JSON Schema specification version | URL (e.g., `http://json-schema.org/draft-07/schema#`) |
+| `title` | Name of this schema | Any string |
+| `type` | Data structure type | `object` |
+| `version` | Schema version number | Positive integer |
+| `description` | Explanation of this schema | Any descriptive text |
+| `properties` | Field definitions | Object with field names as keys |
+| `properties.*.type` | Data type of field | `string`, `number`, `integer`, `boolean`, `array`, `object` |
+| `properties.*.format` | Format specification | `date-time`, `email`, `uri`, etc. |
+| `properties.*.description` | Description of field | Any descriptive text |
+| `properties.*.enum` | Allowed values | Array of values |
+| `required` | Fields that must be present | Array of field names |
+| `additionalProperties` | Whether undefined fields are allowed | `true`, `false` |

package/documentation/default_resources/consumer.json

@@ -0,0 +1,52 @@
+{
+  "name": "<consumer name>",
+  "description": "<consumer description>",
+  "producers": [
+    {
+      "name": "<primary producer name>"
+    },
+    {
+      "name": "<secondary producer name>",
+      "joins": [
+        {
+          "otherName": "<primary producer name>",
+          "relationship": "<one-to-one | one-to-many | many-to-one>",
+          "sql": "<join condition>"
+        }
+      ]
+    }
+  ],
+  "fields": [
+    { "key": "<producer field name>", "from": "<producer name>" },
+    { "key": "<original field name>", "from": "<producer name>", "alias": "<new field name>" },
+    { "key": "<secondary producer field name>", "from": "<secondary producer name>" },
+    { "key": "<another field name>", "from": "<producer name>" }
+  ],
+  "filters": [
+    {
+      "sql": "<filter condition>"
+    }
+  ],
+  "outputs": [
+    { "format": "<api | json | csv | parquet | sql>" },
+    {
+      "format": "<json | csv | parquet | sql>",
+      "exportDestination": "<export destination>"
+    },
+    {
+      "format": "<csv>",
+      "exportDestination": "<export destination>",
+      "trigger": {
+        "type": "<cron | api>",
+        "value": "<cron expression or endpoint path>"
+      }
+    }
+  ],
+  "metadata": {
+    "<metadata tag key>": "<metadata tag value>",
+    "owner_email": "<owner email>"
+  },
+  "project": "<project name>",
+  "schema": "<schema name>",
+  "_version": 1
+}

package/documentation/default_resources/producer.json

@@ -0,0 +1,32 @@
+{
+  "name": "<producer name>",
+  "description": "<producer description>",
+  "source": "<source name>",
+  "dimensions": [
+    {
+      "name": "<primary key field name>",
+      "type": "<number | string | datetime>",
+      "pk": true,
+      "description": "<field description>"
+    },
+    {
+      "name": "<sensitive field name>",
+      "type": "<number | string | datetime>",
+      "classification": ["<'PHI' | 'PII' | 'GDPR'>"],
+      "mask": "<hash | mask | crypt>",
+      "description": "<field description>"
+    }
+  ],
+  "measures": [
+    {
+      "name": "<calculated measure name>",
+      "description": "<measure description>",
+      "sql": "<sql expression>"
+    }
+  ],
+  "settings": {
+    "sqlTable": "<source table name>",
+    "direct": true
+  },
+  "_version": 1
+}