@forzalabs/remora 1.1.11 → 1.1.12

package/documentation/default_resources/consumer.json

@@ -1,53 +1,17 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
-  "name": "<consumer name>",
-  "description": "<consumer description>",
-  "producers": [
-    {
-      "name": "<primary producer name>"
-    },
-    {
-      "name": "<secondary producer name>",
-      "joins": [
-        {
-          "otherName": "<primary producer name>",
-          "relationship": "one-to-many",
-          "sql": "${P.id} = ${<primary producer name>.fk_id}"
-        }
-      ]
-    }
-  ],
-  "fields": [
-    { "key": "<producer field name>", "from": "<producer name>" },
-    { "key": "<original field name>", "from": "<producer name>", "alias": "<new field name>" },
-    { "key": "<secondary producer field name>", "from": "<secondary producer name>" },
-    { "key": "<another field name>", "from": "<producer name>" }
-  ],
-  "filters": [
-    {
-      "sql": "<filter condition>"
-    }
-  ],
-  "outputs": [
-    { "format": "API" },
-    {
-      "format": "JSON",
-      "exportDestination": "<export destination>"
-    },
-    {
-      "format": "CSV",
-      "exportDestination": "<export destination>",
-      "trigger": {
-        "type": "CRON",
-        "value": "0 0 * * *"
-      }
-    }
-  ],
-  "metadata": {
-    "<metadata tag key>": "<metadata tag value>",
-    "owner_email": "<owner email>"
-  },
-  "project": "<project name>",
-  "schema": "<schema name>",
-  "_version": 1
-}
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
+	"name": "c_default",
+	"producers": [
+		{ "name": "p_default" }
+	],
+	"fields": [
+		{ "key": "id" },
+		{ "key": "name" },
+		{ "key": "age" },
+		{ "key": "country" },
+		{ "key": "spend" }
+	],
+	"outputs": [
+		{ "format": "JSON", "exportDestination": "s_default" }
+	]
+}

package/documentation/default_resources/mock_data.csv

@@ -0,0 +1,6 @@
+id,name,age,country,spend
+USR-00001,Alice,28,Italy,0,48
+USR-00002,Bob,35,Portugal,191.75
+USR-00003,Carla,36,Netherlands,23.5
+USR-00004,Diego,54,France,35.25
+USR-00005,Elisa,62,Germany,47,52

package/documentation/default_resources/producer.json

@@ -1,33 +1,16 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
-  "name": "<producer name>",
-  "description": "<producer description>",
-  "source": "<source name>",
-  "dimensions": [
-    {
-      "name": "<primary key field name>",
-      "type": "<number | string | datetime>",
-      "pk": true,
-      "description": "<field description>"
-    },
-    {
-      "name": "<sensitive field name>",
-      "type": "<number | string | datetime>",
-      "classification": ["<'PHI' | 'PII' | 'GDPR'>"],
-      "mask": "<hash | mask | crypt>",
-      "description": "<field description>"
-    }
-  ],
-  "measures": [
-    {
-      "name": "<calculated measure name>",
-      "description": "<measure description>",
-      "sql": "<sql expression>"
-    }
-  ],
-  "settings": {
-    "sqlTable": "<source table name>",
-    "direct": true
-  },
-  "_version": 1
-}
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
+	"name": "p_default",
+	"source": "s_default",
+	"dimensions": [
+		{ "name": "id", "type": "string" },
+		{ "name": "name", "type": "string" },
+		{ "name": "age", "type": "number" },
+		{ "name": "country", "type": "string" },
+		{ "name": "spend", "type": "number" }
+	],
+	"settings": {
+		"fileType": "CSV",
+		"fileKey": "mock_data.csv"
+	}
+}

package/documentation/default_resources/project.json

@@ -1,8 +1,8 @@
 {
     "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json",
-    "name": "PROJECT_NAME_EXAMPLE",
+    "name": "Default Remora Project",
     "version": "1.0.0",
-    "description": "DESCRIPTION_OF_THIS_PROJECT",
+    "description": "This is the standard Remora project created when you run init.",
     "consumers": ["/consumers"],
     "producers": ["/producers"],
     "sources": ["/sources"],

package/documentation/default_resources/source.json

@@ -1,16 +1,9 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
-  "name": "<source name>",
-  "description": "<source description>",
-  "_version": 1,
-  "authentication": {
-    "method": "<iam | username-password | access-secret-key | arn | implicit>",
-    "schema": "<database_schema_name - db schema>",
-    "accessKey": "<access key var from env>",
-    "secretKey": "<secret key var from env>",
-    "region": "<aws region>",
-    "database": "<db name>",
-    "workgroup": "<db workgroup>"
-  },
-  "engine": "<aws-redshift | aws-dynamodb | aws-s3 | postgres | local>"
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
+	"name": "s_default",
+	"authentication": {
+		"method": "implicit",
+		"path": "./default_remora_data"
+	},
+	"engine": "local"
 }

package/index.js

@@ -13228,6 +13228,8 @@ var FileLogServiceClass = class {
   constructor() {
     this._enabled = false;
     this.enable = (folder = "logs", file = "remora.log") => {
+      this._folder = folder;
+      this._file = file;
       this._logger = import_winston.default.createLogger({
         level: "debug",
         format: import_winston.default.format.combine(
@@ -13260,6 +13262,23 @@ ${stack}` : base;
       });
     };
     this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        let pending = this._logger.transports.length;
+        if (pending === 0) return resolve();
+        for (const transport of this._logger.transports) {
+          if (typeof transport.on === "function") {
+            transport.once("logged", () => {
+              if (--pending === 0) resolve();
+            });
+            this._logger.info("");
+          } else {
+            if (--pending === 0) resolve();
+          }
+        }
+      });
+    };
+    this.close = () => {
       if (!this._enabled || !this._logger) return Promise.resolve();
       return new Promise((resolve) => {
         this._logger.on("finish", resolve);
@@ -13339,6 +13358,7 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.flush = () => FileLogService_default.flush();
+    this.close = () => FileLogService_default.close();
     this.error = (error) => {
       let message;
       let stack;
@@ -13480,7 +13500,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.11",
+  cliVersion: "1.1.12",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -16071,7 +16091,7 @@ var DOCUMENTATION_DIR = resolveAssetDir("documentation");
 
 // src/actions/init.ts
 var init = async () => {
-  console.log(import_chalk5.default.blue.bold("\u{1F4E6} Initializing your application..."));
+  console.log(import_chalk5.default.blue.bold("\u{1F4E6} Initializing your Remora app (make sure you have set your REMORA_LICENCE_KEY in the environment variables)..."));
   try {
     const spinner = (0, import_ora3.default)("Creating configuration files...").start();
     const directories = [
@@ -16079,23 +16099,22 @@ var init = async () => {
       "remora/consumers",
       "remora/producers",
       "remora/schemas",
-      "remora/sources"
+      "remora/sources",
+      "default_remora_data"
     ];
     const defaultSource = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/source.json"), "utf-8");
     const defaultConsumer = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"), "utf-8");
     const defaultProducer = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"), "utf-8");
     const defaultRemoraProject = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/project.json"), "utf-8");
+    const defaultMockData = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/mock_data.csv"), "utf-8");
     const readme = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "README.md"), "utf-8");
     const files = [
-      { path: "remora/sources/.gitkeep", content: "" },
-      { path: "remora/sources/default-source.json", content: defaultSource },
-      { path: "remora/consumers/.gitkeep", content: "" },
-      { path: "remora/consumers/default-consumer.json", content: defaultConsumer },
-      { path: "remora/producers/.gitkeep", content: "" },
-      { path: "remora/producers/default-producer.json", content: defaultProducer },
-      { path: "remora/schemas/.gitkeep", content: "" },
+      { path: "remora/sources/s_default.json", content: defaultSource },
+      { path: "remora/consumers/c_default.json", content: defaultConsumer },
+      { path: "remora/producers/p_default.json", content: defaultProducer },
       { path: "remora/project.json", content: defaultRemoraProject },
-      { path: "remora/README.md", content: readme }
+      { path: "remora/README.md", content: readme },
+      { path: "default_remora_data/mock_data.csv", content: defaultMockData }
     ];
     for (let i = 0; i < directories.length; i++) {
       const dir = directories[i];
@@ -19326,19 +19345,17 @@ var ExecutorWriter_default = ExecutorWriter;
 var import_promises10 = require("stream/promises");
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.init = () => {
-      if (!this._executorPool) {
-        const options = {
-          workerThreadOpts: {
-            resourceLimits: {
-              maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
-            }
+    this.createPool = () => {
+      const options = {
+        workerThreadOpts: {
+          resourceLimits: {
+            maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
-        };
-        const workerPath = this._getWorkerPath();
-        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-        this._executorPool = import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
-      }
+        }
+      };
+      const workerPath = this._getWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      return import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19349,9 +19366,9 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.MaximumFileSize };
+      const pool = this.createPool();
       try {
         const start = performance.now();
-        this.init();
         const executorResults = [];
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
@@ -19397,16 +19414,16 @@ var ExecutorOrchestratorClass = class {
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
               Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(this._executorPool.exec("executor", [workerData], {
+              workerThreads.push(pool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
             Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
             Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
-            await this._executorPool.terminate();
           }
         }
+        await pool.terminate();
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -19465,6 +19482,7 @@ var ExecutorOrchestratorClass = class {
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
+        await pool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
@@ -20070,6 +20088,8 @@ var mock = async (producerName, records) => {
 
 // src/index.ts
 import_dotenv.default.configDotenv();
+if (!process.env.NODE_ENV)
+  process.env.NODE_ENV = "production";
 if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true") {
   Logger_default.enableFileLogging("./remora/logs");
   console.log(`Enabled file logger.`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.11",
+    "version": "1.1.12",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13222,6 +13222,8 @@ var FileLogServiceClass = class {
   constructor() {
     this._enabled = false;
     this.enable = (folder = "logs", file = "remora.log") => {
+      this._folder = folder;
+      this._file = file;
       this._logger = import_winston.default.createLogger({
         level: "debug",
         format: import_winston.default.format.combine(
@@ -13254,6 +13256,23 @@ ${stack}` : base;
       });
     };
     this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        let pending = this._logger.transports.length;
+        if (pending === 0) return resolve();
+        for (const transport of this._logger.transports) {
+          if (typeof transport.on === "function") {
+            transport.once("logged", () => {
+              if (--pending === 0) resolve();
+            });
+            this._logger.info("");
+          } else {
+            if (--pending === 0) resolve();
+          }
+        }
+      });
+    };
+    this.close = () => {
       if (!this._enabled || !this._logger) return Promise.resolve();
       return new Promise((resolve) => {
         this._logger.on("finish", resolve);
@@ -13333,6 +13352,7 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.flush = () => FileLogService_default.flush();
+    this.close = () => FileLogService_default.close();
     this.error = (error) => {
       let message;
       let stack;
@@ -13474,7 +13494,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.11",
+  cliVersion: "1.1.12",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -19084,19 +19104,17 @@ var ExecutorWriter_default = ExecutorWriter;
 var import_promises10 = require("stream/promises");
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.init = () => {
-      if (!this._executorPool) {
-        const options = {
-          workerThreadOpts: {
-            resourceLimits: {
-              maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
-            }
+    this.createPool = () => {
+      const options = {
+        workerThreadOpts: {
+          resourceLimits: {
+            maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
-        };
-        const workerPath = this._getWorkerPath();
-        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-        this._executorPool = import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
-      }
+        }
+      };
+      const workerPath = this._getWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      return import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19107,9 +19125,9 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.MaximumFileSize };
+      const pool = this.createPool();
       try {
         const start = performance.now();
-        this.init();
         const executorResults = [];
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
@@ -19155,16 +19173,16 @@ var ExecutorOrchestratorClass = class {
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
               Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(this._executorPool.exec("executor", [workerData], {
+              workerThreads.push(pool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
             Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
             Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
-            await this._executorPool.terminate();
           }
         }
+        await pool.terminate();
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -19223,6 +19241,7 @@ var ExecutorOrchestratorClass = class {
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
+        await pool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);