@forzalabs/remora 1.1.11 → 1.1.13

package/documentation/default_resources/consumer.json

@@ -1,53 +1,17 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
-  "name": "<consumer name>",
-  "description": "<consumer description>",
-  "producers": [
-    {
-      "name": "<primary producer name>"
-    },
-    {
-      "name": "<secondary producer name>",
-      "joins": [
-        {
-          "otherName": "<primary producer name>",
-          "relationship": "one-to-many",
-          "sql": "${P.id} = ${<primary producer name>.fk_id}"
-        }
-      ]
-    }
-  ],
-  "fields": [
-    { "key": "<producer field name>", "from": "<producer name>" },
-    { "key": "<original field name>", "from": "<producer name>", "alias": "<new field name>" },
-    { "key": "<secondary producer field name>", "from": "<secondary producer name>" },
-    { "key": "<another field name>", "from": "<producer name>" }
-  ],
-  "filters": [
-    {
-      "sql": "<filter condition>"
-    }
-  ],
-  "outputs": [
-    { "format": "API" },
-    {
-      "format": "JSON",
-      "exportDestination": "<export destination>"
-    },
-    {
-      "format": "CSV",
-      "exportDestination": "<export destination>",
-      "trigger": {
-        "type": "CRON",
-        "value": "0 0 * * *"
-      }
-    }
-  ],
-  "metadata": {
-    "<metadata tag key>": "<metadata tag value>",
-    "owner_email": "<owner email>"
-  },
-  "project": "<project name>",
-  "schema": "<schema name>",
-  "_version": 1
-}
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/consumer-schema.json",
+	"name": "c_default",
+	"producers": [
+		{ "name": "p_default" }
+	],
+	"fields": [
+		{ "key": "id" },
+		{ "key": "name" },
+		{ "key": "age" },
+		{ "key": "country" },
+		{ "key": "spend" }
+	],
+	"outputs": [
+		{ "format": "JSON", "exportDestination": "s_default" }
+	]
+}

package/documentation/default_resources/mock_data.csv

@@ -0,0 +1,6 @@
+id,name,age,country,spend
+USR-00001,Alice,28,Italy,0,48
+USR-00002,Bob,35,Portugal,191.75
+USR-00003,Carla,36,Netherlands,23.5
+USR-00004,Diego,54,France,35.25
+USR-00005,Elisa,62,Germany,47,52

package/documentation/default_resources/producer.json

@@ -1,33 +1,16 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
-  "name": "<producer name>",
-  "description": "<producer description>",
-  "source": "<source name>",
-  "dimensions": [
-    {
-      "name": "<primary key field name>",
-      "type": "<number | string | datetime>",
-      "pk": true,
-      "description": "<field description>"
-    },
-    {
-      "name": "<sensitive field name>",
-      "type": "<number | string | datetime>",
-      "classification": ["<'PHI' | 'PII' | 'GDPR'>"],
-      "mask": "<hash | mask | crypt>",
-      "description": "<field description>"
-    }
-  ],
-  "measures": [
-    {
-      "name": "<calculated measure name>",
-      "description": "<measure description>",
-      "sql": "<sql expression>"
-    }
-  ],
-  "settings": {
-    "sqlTable": "<source table name>",
-    "direct": true
-  },
-  "_version": 1
-}
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/producer-schema.json",
+	"name": "p_default",
+	"source": "s_default",
+	"dimensions": [
+		{ "name": "id", "type": "string" },
+		{ "name": "name", "type": "string" },
+		{ "name": "age", "type": "number" },
+		{ "name": "country", "type": "string" },
+		{ "name": "spend", "type": "number" }
+	],
+	"settings": {
+		"fileType": "CSV",
+		"fileKey": "mock_data.csv"
+	}
+}

package/documentation/default_resources/project.json

@@ -1,8 +1,8 @@
 {
     "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/project-schema.json",
-    "name": "PROJECT_NAME_EXAMPLE",
+    "name": "Default Remora Project",
     "version": "1.0.0",
-    "description": "DESCRIPTION_OF_THIS_PROJECT",
+    "description": "This is the standard Remora project created when you run init.",
     "consumers": ["/consumers"],
     "producers": ["/producers"],
     "sources": ["/sources"],

package/documentation/default_resources/source.json

@@ -1,16 +1,9 @@
 {
-  "$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
-  "name": "<source name>",
-  "description": "<source description>",
-  "_version": 1,
-  "authentication": {
-    "method": "<iam | username-password | access-secret-key | arn | implicit>",
-    "schema": "<database_schema_name - db schema>",
-    "accessKey": "<access key var from env>",
-    "secretKey": "<secret key var from env>",
-    "region": "<aws region>",
-    "database": "<db name>",
-    "workgroup": "<db workgroup>"
-  },
-  "engine": "<aws-redshift | aws-dynamodb | aws-s3 | postgres | local>"
+	"$schema": "https://raw.githubusercontent.com/ForzaLabs/remora-public/refs/heads/main/json_schemas/source-schema.json",
+	"name": "s_default",
+	"authentication": {
+		"method": "implicit",
+		"path": "./default_remora_data"
+	},
+	"engine": "local"
 }

package/index.js

@@ -13228,6 +13228,8 @@ var FileLogServiceClass = class {
   constructor() {
     this._enabled = false;
     this.enable = (folder = "logs", file = "remora.log") => {
+      this._folder = folder;
+      this._file = file;
       this._logger = import_winston.default.createLogger({
         level: "debug",
         format: import_winston.default.format.combine(
@@ -13260,6 +13262,23 @@ ${stack}` : base;
       });
     };
     this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        let pending = this._logger.transports.length;
+        if (pending === 0) return resolve();
+        for (const transport of this._logger.transports) {
+          if (typeof transport.on === "function") {
+            transport.once("logged", () => {
+              if (--pending === 0) resolve();
+            });
+            this._logger.info("");
+          } else {
+            if (--pending === 0) resolve();
+          }
+        }
+      });
+    };
+    this.close = () => {
       if (!this._enabled || !this._logger) return Promise.resolve();
       return new Promise((resolve) => {
         this._logger.on("finish", resolve);
@@ -13339,6 +13358,7 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.flush = () => FileLogService_default.flush();
+    this.close = () => FileLogService_default.close();
     this.error = (error) => {
       let message;
       let stack;
@@ -13480,7 +13500,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.11",
+  cliVersion: "1.1.13",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13652,6 +13672,42 @@ var SchemaValidatorClass = class {
 var SchemaValidator = new SchemaValidatorClass();
 var SchemaValidator_default = SchemaValidator;
 
+// ../../packages/common/src/ResourcesUtils.ts
+var ResourcesUtilsClass = class {
+  constructor() {
+    this.getAvailableColumns = (consumer) => {
+      return consumer.producers.flatMap((cProd) => {
+        const producer = Environment_default.getProducer(cProd.name);
+        if (!producer) {
+          const subConsumer = Environment_default.getConsumer(cProd.name);
+          Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
+          return this.getAvailableColumns(subConsumer);
+        } else {
+          const dims = producer.dimensions.map((x) => ({
+            consumerAlias: null,
+            consumerKey: null,
+            nameInProducer: x.name,
+            aliasInProducer: x.alias,
+            dimension: x,
+            owner: cProd.name
+          }));
+          const meas = producer.measures?.map((x) => ({
+            consumerAlias: null,
+            consumerKey: null,
+            nameInProducer: x.name,
+            aliasInProducer: x.name,
+            measure: x,
+            owner: cProd.name
+          })) ?? [];
+          return [...dims, ...meas];
+        }
+      });
+    };
+  }
+};
+var ResourcesUtils = new ResourcesUtilsClass();
+var ResourcesUtils_default = ResourcesUtils;
+
 // ../../packages/common/src/validation/Validator.ts
 var ValidatorClass = class {
   constructor() {
@@ -13762,6 +13818,38 @@ var ValidatorClass = class {
         const uniqNames = Algo_default.uniqBy(sources, "name");
         if (uniqNames.length !== 1)
           errors.push(`Producers with different sources are used in the consumer "${consumer.name}" (${uniqNames.join(", ")})`);
+        const availableColumns = ResourcesUtils_default.getAvailableColumns(consumer);
+        const availableFieldsByProducer = /* @__PURE__ */ new Map();
+        for (const col of availableColumns) {
+          const existing = availableFieldsByProducer.get(col.owner) ?? [];
+          if (!existing.includes(col.nameInProducer))
+            existing.push(col.nameInProducer);
+          availableFieldsByProducer.set(col.owner, existing);
+        }
+        const allAvailableFields = [...availableFieldsByProducer.values()].flat();
+        for (const field of consumer.fields) {
+          if (field.key === "*" || field.fixed || field.copyFrom) continue;
+          if (field.from) {
+            const producerFields = availableFieldsByProducer.get(field.from);
+            if (producerFields && !producerFields.includes(field.key))
+              errors.push(`Field "${field.key}" (from: "${field.from}") is not found in producer "${field.from}" in consumer "${consumer.name}". Available fields: ${producerFields.join(", ")}`);
+          } else if (consumer.producers.length === 1) {
+            const producerFields = availableFieldsByProducer.get(consumer.producers[0].name);
+            if (producerFields && !producerFields.includes(field.key))
+              errors.push(`Field "${field.key}" is not found in producer "${consumer.producers[0].name}" in consumer "${consumer.name}". Available fields: ${producerFields.join(", ")}`);
+          } else {
+            if (allAvailableFields.length > 0 && !allAvailableFields.includes(field.key))
+              errors.push(`Field "${field.key}" is not found in any of the producers of consumer "${consumer.name}".`);
+          }
+        }
+        for (let i = 0; i < consumer.fields.length; i++) {
+          const field = consumer.fields[i];
+          if (!field.copyFrom) continue;
+          const precedingFields = consumer.fields.slice(0, i);
+          const found = precedingFields.find((f) => (f.alias ?? f.key) === field.copyFrom);
+          if (!found)
+            errors.push(`Field "${field.alias ?? field.key}" uses copyFrom "${field.copyFrom}" but no field with that name/alias exists before it in consumer "${consumer.name}".`);
+        }
         if (consumer.filters && consumer.filters.length > 0) {
           if (consumer.filters.some((x) => x.sql && x.rule))
             errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
@@ -13834,6 +13922,20 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.distinctOn)) {
+            const { distinctOn } = consumer.options;
+            const hasWildcard = consumer.fields.some((x) => x.key === "*");
+            const consumerFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            if (!hasWildcard) {
+              const missingKeys = distinctOn.keys.filter((k) => !consumerFieldKeys.includes(k));
+              if (missingKeys.length > 0)
+                errors.push(`distinctOn references key(s) "${missingKeys.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+            }
+            if (distinctOn.resolution?.orderBy && !hasWildcard) {
+              if (!consumerFieldKeys.includes(distinctOn.resolution.orderBy))
+                errors.push(`distinctOn resolution orderBy field "${distinctOn.resolution.orderBy}" is not present in the consumer "${consumer.name}".`);
+            }
+          }
           if (Algo_default.hasVal(consumer.options.pivot)) {
             if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
               errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
@@ -16071,7 +16173,7 @@ var DOCUMENTATION_DIR = resolveAssetDir("documentation");
 
 // src/actions/init.ts
 var init = async () => {
-  console.log(import_chalk5.default.blue.bold("\u{1F4E6} Initializing your application..."));
+  console.log(import_chalk5.default.blue.bold("\u{1F4E6} Initializing your Remora app (make sure you have set your REMORA_LICENCE_KEY in the environment variables)..."));
   try {
     const spinner = (0, import_ora3.default)("Creating configuration files...").start();
     const directories = [
@@ -16079,23 +16181,22 @@ var init = async () => {
       "remora/consumers",
       "remora/producers",
       "remora/schemas",
-      "remora/sources"
+      "remora/sources",
+      "default_remora_data"
     ];
     const defaultSource = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/source.json"), "utf-8");
     const defaultConsumer = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"), "utf-8");
     const defaultProducer = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"), "utf-8");
     const defaultRemoraProject = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/project.json"), "utf-8");
+    const defaultMockData = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "default_resources/mock_data.csv"), "utf-8");
     const readme = import_fs_extra2.default.readFileSync(import_path13.default.join(DOCUMENTATION_DIR, "README.md"), "utf-8");
     const files = [
-      { path: "remora/sources/.gitkeep", content: "" },
-      { path: "remora/sources/default-source.json", content: defaultSource },
-      { path: "remora/consumers/.gitkeep", content: "" },
-      { path: "remora/consumers/default-consumer.json", content: defaultConsumer },
-      { path: "remora/producers/.gitkeep", content: "" },
-      { path: "remora/producers/default-producer.json", content: defaultProducer },
-      { path: "remora/schemas/.gitkeep", content: "" },
+      { path: "remora/sources/s_default.json", content: defaultSource },
+      { path: "remora/consumers/c_default.json", content: defaultConsumer },
+      { path: "remora/producers/p_default.json", content: defaultProducer },
       { path: "remora/project.json", content: defaultRemoraProject },
-      { path: "remora/README.md", content: readme }
+      { path: "remora/README.md", content: readme },
+      { path: "default_remora_data/mock_data.csv", content: defaultMockData }
     ];
     for (let i = 0; i < directories.length; i++) {
       const dir = directories[i];
@@ -16136,8 +16237,15 @@ var DatabaseEngineClass = class {
     this.MAX_TRY_CONNECTION = 3;
     this.db = () => this._db;
     this.connect = async () => {
-      this._uri = ProcessENVManager_default.getEnvVariable("MONGO_URI") ?? Helper_default.isDev() ? "mongodb://mongo:27017/remora" : "mongodb://localhost:27017/remora";
-      this._client = new import_mongodb.MongoClient(this._uri);
+      if (!ProcessENVManager_default.getEnvVariable("MONGO_URI")) {
+        if (Helper_default.isDev()) {
+          this._uri = "mongodb://mongo:27017/remora";
+        } else {
+          this._uri = "mongodb://localhost:27017/remora";
+        }
+      } else {
+        this._uri = ProcessENVManager_default.getEnvVariable("MONGO_URI");
+      }
       const errors = [];
       this._client = new import_mongodb.MongoClient(this._uri);
       for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
@@ -17715,33 +17823,7 @@ var ConsumerManagerClass = class {
       return expandedFields;
     };
     this.getAvailableColumns = (consumer) => {
-      const availableColumns = consumer.producers.flatMap((cProd) => {
-        const producer = Environment_default.getProducer(cProd.name);
-        if (!producer) {
-          const subConsumer = Environment_default.getConsumer(cProd.name);
-          Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
-          return this.getAvailableColumns(subConsumer);
-        } else {
-          const dims = producer.dimensions.map((x) => ({
-            consumerAlias: null,
-            consumerKey: null,
-            nameInProducer: x.name,
-            aliasInProducer: x.alias,
-            dimension: x,
-            owner: cProd.name
-          }));
-          const meas = producer.measures?.map((x) => ({
-            consumerAlias: null,
-            consumerKey: null,
-            nameInProducer: x.name,
-            aliasInProducer: x.name,
-            measure: x,
-            owner: cProd.name
-          })) ?? [];
-          return [...dims, ...meas];
-        }
-      });
-      return availableColumns;
+      return ResourcesUtils_default.getAvailableColumns(consumer);
     };
     this.expandField = (consumer, field, availableColumns) => {
       Affirm_default(consumer, "Invalid consumer");
@@ -19326,19 +19408,17 @@ var ExecutorWriter_default = ExecutorWriter;
 var import_promises10 = require("stream/promises");
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.init = () => {
-      if (!this._executorPool) {
-        const options = {
-          workerThreadOpts: {
-            resourceLimits: {
-              maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
-            }
+    this.createPool = () => {
+      const options = {
+        workerThreadOpts: {
+          resourceLimits: {
+            maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
-        };
-        const workerPath = this._getWorkerPath();
-        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-        this._executorPool = import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
-      }
+        }
+      };
+      const workerPath = this._getWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      return import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19349,9 +19429,9 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.MaximumFileSize };
+      const pool = this.createPool();
       try {
         const start = performance.now();
-        this.init();
         const executorResults = [];
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
@@ -19397,16 +19477,16 @@ var ExecutorOrchestratorClass = class {
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
               Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(this._executorPool.exec("executor", [workerData], {
+              workerThreads.push(pool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
             Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
             Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
-            await this._executorPool.terminate();
           }
         }
+        await pool.terminate();
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -19465,6 +19545,7 @@ var ExecutorOrchestratorClass = class {
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
+        await pool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
@@ -20070,6 +20151,8 @@ var mock = async (producerName, records) => {
 
 // src/index.ts
 import_dotenv.default.configDotenv();
+if (!process.env.NODE_ENV)
+  process.env.NODE_ENV = "production";
 if (process.env.NODE_ENV !== "development" && process.env.REMORA_DEBUG_MODE === "true") {
   Logger_default.enableFileLogging("./remora/logs");
   console.log(`Enabled file logger.`);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.1.11",
+    "version": "1.1.13",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13222,6 +13222,8 @@ var FileLogServiceClass = class {
   constructor() {
     this._enabled = false;
     this.enable = (folder = "logs", file = "remora.log") => {
+      this._folder = folder;
+      this._file = file;
       this._logger = import_winston.default.createLogger({
         level: "debug",
         format: import_winston.default.format.combine(
@@ -13254,6 +13256,23 @@ ${stack}` : base;
       });
     };
     this.flush = () => {
+      if (!this._enabled || !this._logger) return Promise.resolve();
+      return new Promise((resolve) => {
+        let pending = this._logger.transports.length;
+        if (pending === 0) return resolve();
+        for (const transport of this._logger.transports) {
+          if (typeof transport.on === "function") {
+            transport.once("logged", () => {
+              if (--pending === 0) resolve();
+            });
+            this._logger.info("");
+          } else {
+            if (--pending === 0) resolve();
+          }
+        }
+      });
+    };
+    this.close = () => {
       if (!this._enabled || !this._logger) return Promise.resolve();
       return new Promise((resolve) => {
         this._logger.on("finish", resolve);
@@ -13333,6 +13352,7 @@ var Logger = class {
       FileLogService_default.write("INFO", String(message));
     };
     this.flush = () => FileLogService_default.flush();
+    this.close = () => FileLogService_default.close();
     this.error = (error) => {
       let message;
       let stack;
@@ -13474,7 +13494,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.11",
+  cliVersion: "1.1.13",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -13646,6 +13666,42 @@ var SchemaValidatorClass = class {
 var SchemaValidator = new SchemaValidatorClass();
 var SchemaValidator_default = SchemaValidator;
 
+// ../../packages/common/src/ResourcesUtils.ts
+var ResourcesUtilsClass = class {
+  constructor() {
+    this.getAvailableColumns = (consumer) => {
+      return consumer.producers.flatMap((cProd) => {
+        const producer = Environment_default.getProducer(cProd.name);
+        if (!producer) {
+          const subConsumer = Environment_default.getConsumer(cProd.name);
+          Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
+          return this.getAvailableColumns(subConsumer);
+        } else {
+          const dims = producer.dimensions.map((x) => ({
+            consumerAlias: null,
+            consumerKey: null,
+            nameInProducer: x.name,
+            aliasInProducer: x.alias,
+            dimension: x,
+            owner: cProd.name
+          }));
+          const meas = producer.measures?.map((x) => ({
+            consumerAlias: null,
+            consumerKey: null,
+            nameInProducer: x.name,
+            aliasInProducer: x.name,
+            measure: x,
+            owner: cProd.name
+          })) ?? [];
+          return [...dims, ...meas];
+        }
+      });
+    };
+  }
+};
+var ResourcesUtils = new ResourcesUtilsClass();
+var ResourcesUtils_default = ResourcesUtils;
+
 // ../../packages/common/src/validation/Validator.ts
 var ValidatorClass = class {
   constructor() {
@@ -13756,6 +13812,38 @@ var ValidatorClass = class {
         const uniqNames = Algo_default.uniqBy(sources, "name");
         if (uniqNames.length !== 1)
           errors.push(`Producers with different sources are used in the consumer "${consumer.name}" (${uniqNames.join(", ")})`);
+        const availableColumns = ResourcesUtils_default.getAvailableColumns(consumer);
+        const availableFieldsByProducer = /* @__PURE__ */ new Map();
+        for (const col of availableColumns) {
+          const existing = availableFieldsByProducer.get(col.owner) ?? [];
+          if (!existing.includes(col.nameInProducer))
+            existing.push(col.nameInProducer);
+          availableFieldsByProducer.set(col.owner, existing);
+        }
+        const allAvailableFields = [...availableFieldsByProducer.values()].flat();
+        for (const field of consumer.fields) {
+          if (field.key === "*" || field.fixed || field.copyFrom) continue;
+          if (field.from) {
+            const producerFields = availableFieldsByProducer.get(field.from);
+            if (producerFields && !producerFields.includes(field.key))
+              errors.push(`Field "${field.key}" (from: "${field.from}") is not found in producer "${field.from}" in consumer "${consumer.name}". Available fields: ${producerFields.join(", ")}`);
+          } else if (consumer.producers.length === 1) {
+            const producerFields = availableFieldsByProducer.get(consumer.producers[0].name);
+            if (producerFields && !producerFields.includes(field.key))
+              errors.push(`Field "${field.key}" is not found in producer "${consumer.producers[0].name}" in consumer "${consumer.name}". Available fields: ${producerFields.join(", ")}`);
+          } else {
+            if (allAvailableFields.length > 0 && !allAvailableFields.includes(field.key))
+              errors.push(`Field "${field.key}" is not found in any of the producers of consumer "${consumer.name}".`);
+          }
+        }
+        for (let i = 0; i < consumer.fields.length; i++) {
+          const field = consumer.fields[i];
+          if (!field.copyFrom) continue;
+          const precedingFields = consumer.fields.slice(0, i);
+          const found = precedingFields.find((f) => (f.alias ?? f.key) === field.copyFrom);
+          if (!found)
+            errors.push(`Field "${field.alias ?? field.key}" uses copyFrom "${field.copyFrom}" but no field with that name/alias exists before it in consumer "${consumer.name}".`);
+        }
         if (consumer.filters && consumer.filters.length > 0) {
           if (consumer.filters.some((x) => x.sql && x.rule))
             errors.push(`A single consumer can't have both filters based on SQL and filters based on rules.`);
@@ -13828,6 +13916,20 @@ var ValidatorClass = class {
         if (consumer.options) {
           if (Algo_default.hasVal(consumer.options.distinct) && Algo_default.hasVal(consumer.options.distinctOn))
             errors.push(`Can't specify a "distinct" and a "distinctOn" clause on the same consumer (${consumer.name}); use one or the other.`);
+          if (Algo_default.hasVal(consumer.options.distinctOn)) {
+            const { distinctOn } = consumer.options;
+            const hasWildcard = consumer.fields.some((x) => x.key === "*");
+            const consumerFieldKeys = consumer.fields.map((x) => x.alias ?? x.key);
+            if (!hasWildcard) {
+              const missingKeys = distinctOn.keys.filter((k) => !consumerFieldKeys.includes(k));
+              if (missingKeys.length > 0)
+                errors.push(`distinctOn references key(s) "${missingKeys.join(", ")}" that are not present in the consumer "${consumer.name}".`);
+            }
+            if (distinctOn.resolution?.orderBy && !hasWildcard) {
+              if (!consumerFieldKeys.includes(distinctOn.resolution.orderBy))
+                errors.push(`distinctOn resolution orderBy field "${distinctOn.resolution.orderBy}" is not present in the consumer "${consumer.name}".`);
+            }
+          }
           if (Algo_default.hasVal(consumer.options.pivot)) {
             if (Algo_default.hasVal(consumer.options.distinct) || Algo_default.hasVal(consumer.options.distinctOn))
               errors.push(`Can't specify "pivot" together with "distinct" or "distinctOn" on the same consumer (${consumer.name}).`);
@@ -17051,33 +17153,7 @@ var ConsumerManagerClass = class {
       return expandedFields;
     };
     this.getAvailableColumns = (consumer) => {
-      const availableColumns = consumer.producers.flatMap((cProd) => {
-        const producer = Environment_default.getProducer(cProd.name);
-        if (!producer) {
-          const subConsumer = Environment_default.getConsumer(cProd.name);
-          Affirm_default(subConsumer, `No producer found with name "${cProd.name}"`);
-          return this.getAvailableColumns(subConsumer);
-        } else {
-          const dims = producer.dimensions.map((x) => ({
-            consumerAlias: null,
-            consumerKey: null,
-            nameInProducer: x.name,
-            aliasInProducer: x.alias,
-            dimension: x,
-            owner: cProd.name
-          }));
-          const meas = producer.measures?.map((x) => ({
-            consumerAlias: null,
-            consumerKey: null,
-            nameInProducer: x.name,
-            aliasInProducer: x.name,
-            measure: x,
-            owner: cProd.name
-          })) ?? [];
-          return [...dims, ...meas];
-        }
-      });
-      return availableColumns;
+      return ResourcesUtils_default.getAvailableColumns(consumer);
     };
     this.expandField = (consumer, field, availableColumns) => {
       Affirm_default(consumer, "Invalid consumer");
@@ -17789,8 +17865,15 @@ var DatabaseEngineClass = class {
     this.MAX_TRY_CONNECTION = 3;
     this.db = () => this._db;
     this.connect = async () => {
-      this._uri = ProcessENVManager_default.getEnvVariable("MONGO_URI") ?? Helper_default.isDev() ? "mongodb://mongo:27017/remora" : "mongodb://localhost:27017/remora";
-      this._client = new import_mongodb.MongoClient(this._uri);
+      if (!ProcessENVManager_default.getEnvVariable("MONGO_URI")) {
+        if (Helper_default.isDev()) {
+          this._uri = "mongodb://mongo:27017/remora";
+        } else {
+          this._uri = "mongodb://localhost:27017/remora";
+        }
+      } else {
+        this._uri = ProcessENVManager_default.getEnvVariable("MONGO_URI");
+      }
       const errors = [];
       this._client = new import_mongodb.MongoClient(this._uri);
       for (let i = 0; i < this.MAX_TRY_CONNECTION; i++) {
@@ -19084,19 +19167,17 @@ var ExecutorWriter_default = ExecutorWriter;
 var import_promises10 = require("stream/promises");
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.init = () => {
-      if (!this._executorPool) {
-        const options = {
-          workerThreadOpts: {
-            resourceLimits: {
-              maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
-            }
+    this.createPool = () => {
+      const options = {
+        workerThreadOpts: {
+          resourceLimits: {
+            maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
-        };
-        const workerPath = this._getWorkerPath();
-        Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-        this._executorPool = import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
-      }
+        }
+      };
+      const workerPath = this._getWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      return import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19107,9 +19188,9 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.MaximumFileSize };
+      const pool = this.createPool();
       try {
         const start = performance.now();
-        this.init();
         const executorResults = [];
         Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
         let counter = performance.now();
@@ -19155,16 +19236,16 @@ var ExecutorOrchestratorClass = class {
               _progress.register((currentWorkerIndex + 1).toString(), prod.name, fileIndex, totalFiles);
               scope.workersId.push(workerId);
               Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(this._executorPool.exec("executor", [workerData], {
+              workerThreads.push(pool.exec("executor", [workerData], {
                 on: (payload) => this.onWorkAdvanced(payload, currentWorkerIndex, _progress)
               }));
             }
             Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
             executorResults.push(...await Promise.all(workerThreads));
             Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
-            await this._executorPool.terminate();
           }
         }
+        await pool.terminate();
         _progress.complete();
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -19223,6 +19304,7 @@ var ExecutorOrchestratorClass = class {
       } catch (error) {
         Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
         Logger_default.error(Helper_default.asError(error));
+        await pool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);