@forzalabs/remora 0.0.13 → 0.0.15

package/Constants.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const CONSTANTS = {
-    cliVersion: '0.0.1',
+    cliVersion: '0.0.15',
     lambdaVersion: 1,
     port: 5069
 };

package/actions/init.js

@@ -32,7 +32,7 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
         const defaultSource = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/source.json`), 'utf-8');
         const defaultConsumer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/consumer.json`), 'utf-8');
         const defaultProducer = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/producer.json`), 'utf-8');
-        const defaultSchema = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8');
+        // const defaultSchema = fs.readFileSync(path.join(__dirname, `../documentation/default_resources/schema.json`), 'utf-8')
         const defaultRemoraProject = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/default_resources/project.json`), 'utf-8');
         const readme = fs_extra_1.default.readFileSync(path_1.default.join(__dirname, `../documentation/README.md`), 'utf-8');
         const files = [
@@ -43,7 +43,6 @@ const init = () => __awaiter(void 0, void 0, void 0, function* () {
             { path: 'remora/producers/.gitkeep', content: '' },
             { path: 'remora/producers/default-producer.json', content: defaultProducer },
             { path: 'remora/schemas/.gitkeep', content: '' },
-            { path: 'remora/schemas/default-schemas.json', content: defaultSchema },
             { path: 'remora/project.json', content: defaultRemoraProject },
             { path: 'remora/README.md', content: readme }
         ];

package/definitions/json_schemas/source-schema.json

@@ -41,14 +41,65 @@
                         "implicit"
                     ],
                     "description": "The authentication method to use"
+                },
+                "host": {
+                    "type": "string",
+                    "description": "Hostname or endpoint of the data source"
+                },
+                "user": {
+                    "type": "string",
+                    "description": "Username for authentication"
+                },
+                "password": {
+                    "type": "string",
+                    "description": "Password for authentication"
+                },
+                "database": {
+                    "type": "string",
+                    "description": "Database name"
+                },
+                "workgroup": {
+                    "type": "string",
+                    "description": "Workgroup name"
+                },
+                "schema": {
+                    "type": "string",
+                    "description": "Database schema name"
+                },
+                "port": {
+                    "type": "string",
+                    "description": "Port number for the connection"
+                },
+                "accessKey": {
+                    "type": "string",
+                    "description": "AWS access key ID"
+                },
+                "secretKey": {
+                    "type": "string",
+                    "description": "AWS secret access key"
+                },
+                "sessionToken": {
+                    "type": "string",
+                    "description": "AWS session token (if required)"
+                },
+                "region": {
+                    "type": "string",
+                    "description": "AWS region"
+                },
+                "bucket": {
+                    "type": "string",
+                    "description": "S3 bucket name"
+                },
+                "iamProfile": {
+                    "type": "string",
+                    "description": "IAM role or profile name"
+                },
+                "clusterId": {
+                    "type": "string",
+                    "description": "Redshift cluster identifier"
                 }
             },
-            "required": [
-                "method"
-            ],
-            "additionalProperties": {
-                "type": "string"
-            }
+            "required": ["method"]
         },
         "_version": {
             "type": "number",
@@ -72,7 +123,7 @@
                 "host": "prod-db.example.com",
                 "user": "app_user",
                 "password": "password123",
-                "dbName": "customers",
+                "database": "customers",
                 "schema": "public",
                 "port": "5432"
             },
@@ -97,7 +148,7 @@
             "authentication": {
                 "method": "iam",
                 "host": "redshift-cluster.region.redshift.amazonaws.com",
-                "dbName": "analytics",
+                "database": "analytics",
                 "port": "5439",
                 "iamProfile": "redshift-access-role",
                 "region": "us-west-2",

package/documentation/README.md

@@ -3,6 +3,15 @@
 This README explains the available configurations for each resource type in the project. 
 Each section details the expected values for different keys in the JSON files.
 
+## JSON Schemas
+
+All configuration files follow specific JSON schemas to ensure consistency and validation. You can find the schema definitions at the following URLs:
+
+- **Source**: [source-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json)  
+- **Producer**: [producer-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json)  
+- **Consumer**: [consumer-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json)  
+- **Project**: [project-schema.json](https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json)
+
 ## Remora Configuration
 
 The main project configuration file.

package/documentation/default_resources/consumer.json

@@ -29,13 +29,13 @@
     }
   ],
   "outputs": [
-    { "format": "<api | json | csv | parquet | sql>" },
+    { "format": "<api | sql>" },
     {
-      "format": "<json | csv | parquet | sql>",
+      "format": "<json | csv | parquet>",
       "exportDestination": "<export destination>"
     },
     {
-      "format": "<csv>",
+      "format": "<csv | json>",
       "exportDestination": "<export destination>",
       "trigger": {
         "type": "<cron | api>",

package/drivers/RedshiftDriver.js

@@ -58,7 +58,6 @@ class RedshiftDriver {
             const executeStatementRes = yield this._client.executeStatement(input);
             const queryId = (_a = executeStatementRes.Id) !== null && _a !== void 0 ? _a : '';
             const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
-            console.log(res);
             return {
                 rows: res.result
             };
@@ -77,7 +76,6 @@ class RedshiftDriver {
             const queryId = (_b = executeStatementRes.Id) !== null && _b !== void 0 ? _b : '';
             // TODO: handle errors (permission, wrong query, ...)
             const res = (yield this.executeStatement(queryId, { retry: 5 }))[0];
-            console.log(res);
             return {
                 rows: res.result
             };

package/drivers/S3Driver.js

@@ -18,7 +18,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 class S3Driver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            this._bucketName = source.authentication['bucketName'];
+            this._bucketName = source.authentication['bucket'];
             const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
             const config = {
                 region: source.authentication['region'],

package/drivers/S3SourceDriver.js

@@ -26,7 +26,7 @@ const SecretManager_1 = __importDefault(require("../engines/SecretManager"));
 class S3SourceDriver {
     constructor() {
         this.init = (source) => __awaiter(this, void 0, void 0, function* () {
-            this._bucketName = source.authentication['bucketName'];
+            this._bucketName = source.authentication['bucket'];
             const sessionToken = SecretManager_1.default.replaceSecret(source.authentication['sessionToken']);
             const config = {
                 region: source.authentication['region'],

package/engines/ai/DeveloperEngine.js

@@ -37,6 +37,7 @@ class DeveloperEngineClass {
                 measures: [],
                 _version: (_a = producer._version) !== null && _a !== void 0 ? _a : 1
             };
+            mappedProducer['$schema'] = producer['$schema'];
             // Save the mapped producer to file
             const producerPath = path_1.default.join(process.cwd(), 'remora', 'producers', `${producer.name}.json`);
             yield promises_1.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), 'utf-8');

package/engines/file/FileExporter.js

@@ -68,7 +68,7 @@ class FileExporterClass {
                     for (let i = 0; i < data.length; i++) {
                         const row = data[i];
                         const columns = keys.map(key => row[key]);
-                        const rowValues = columns.map(c => Algo_1.default.replaceAll(c.toString(), '"', '""')).map(c => `"${c}"`).join(',');
+                        const rowValues = columns.map(c => { var _a; return Algo_1.default.replaceAll((_a = c === null || c === void 0 ? void 0 : c.toString()) !== null && _a !== void 0 ? _a : '', '"', '""'); }).map(c => `"${c}"`).join(',');
                         lines.push(rowValues);
                     }
                     const csv = lines.join('\n');
@@ -105,7 +105,8 @@ class FileExporterClass {
                 }
                 case 'aws-s3': {
                     const driver = yield DriverFactory_1.default.instantiateDestination(source);
-                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${DSTE_1.default.now().toJSON()}` });
+                    const date = DSTE_1.default.now().toISOString().split('.')[0];
+                    const res = yield driver.uploadFile({ content: exportData, name: `${consumer.name}_${date}.${extension}` });
                     return res.key;
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "0.0.13",
+    "version": "0.0.15",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,