@forzalabs/remora 1.2.5 → 1.2.6

package/index.js

@@ -13518,12 +13518,10 @@ var CONSTANTS = {
     REMORA_PATH: "./remora",
     PRODUCER_TEMP_FOLDER: ".temp",
     SQL_MAX_QUERY_ROWS: 1e4,
-    STRING_MAX_CHARACTERS_LENGTH: 1e7,
-    MAX_ITEMS_IN_MEMORY: 2e5,
     MIN_RUNTIME_HEAP_MB: 4e3,
     RECOMMENDED_RUNTIME_HEAP_MB: 8e3,
     INDICATIVE_THREAD_LINE_COUNT: 75e4,
-    MAX_THREAD_COUNT: 8,
+    MAX_THREAD_COUNT: 99,
     /**
      * Minimum file size to consider parallel processing (10 MB)
      */
@@ -13605,6 +13603,8 @@ var ExecutorScope_default = ExecutorScope;
 
 // ../../packages/common/src/Environment.ts
 var import_fs4 = __toESM(require("fs"), 1);
+var import_crypto = __toESM(require("crypto"), 1);
+var import_adm_zip = __toESM(require("adm-zip"), 1);
 var import_path5 = __toESM(require("path"), 1);
 
 // ../../packages/common/src/schema/SchemaValidator.ts
@@ -13726,6 +13726,12 @@ var ValidatorClass = class {
           const source = sources[i];
           if (source.engine === "local" && !source.authentication.path)
             errors.push(`For source ${source.name}, the path has not been configured`);
+          if (source.engine === "aws-sqs") {
+            if (!source.authentication.queue)
+              errors.push(`For source ${source.name}, the queue has not been configured`);
+            if (!source.authentication.region && !source.authentication.queue?.startsWith("https://"))
+              errors.push(`For source ${source.name}, the region has not been configured`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -14016,8 +14022,11 @@ var ValidatorClass = class {
         if (!auth) continue;
         const ctx = (field) => `source "${source.name}" authentication.${field}`;
         checkValue(auth.accessKey, ctx("accessKey"));
+        checkValue(auth.accountId, ctx("accountId"));
         checkValue(auth.secretKey, ctx("secretKey"));
         checkValue(auth.sessionToken, ctx("sessionToken"));
+        checkValue(auth.queue, ctx("queue"));
+        checkValue(auth.region, ctx("region"));
         checkValue(auth.bearerToken, ctx("bearerToken"));
         checkValue(auth.url, ctx("url"));
         checkValue(auth.apiKey, ctx("apiKey"));
@@ -14036,6 +14045,8 @@ var ValidatorClass = class {
       switch (engine) {
         case "aws-dynamodb":
           return "no-sql";
+        case "aws-sqs":
+          return "no-sql";
         case "aws-redshift":
         case "postgres":
           return "sql";
@@ -14055,10 +14066,72 @@ var Validator_default = Validator;
 var EnvironmentClass = class {
   constructor() {
     this._env = null;
+    this._configUri = null;
+    this._localPath = null;
+    this._configHash = null;
+    this._lastEtag = null;
+    this._projectSettings = null;
     this.init = (env) => {
       this._env = env;
     };
+    /**
+     * Load environment from a local path or remote URL (e.g. S3 presigned URL).
+     * Remote configs are downloaded as a zip and extracted to the local cache path.
+     * Falls back to `REMORA_CONFIG_URI` env variable if no URI is provided.
+     */
+    this.loadFromUri = async (uri) => {
+      const configUri = uri || process.env.REMORA_CONFIG_URI;
+      Affirm_default(configUri, "No configuration URI provided. Set REMORA_CONFIG_URI environment variable or pass a URI.");
+      this._configUri = configUri;
+      if (this._isRemoteUri(configUri)) {
+        this._localPath = this._resolveLocalCachePath();
+        await this._downloadRemoteConfig(configUri);
+      } else {
+        this._localPath = configUri;
+      }
+      this.load(this._localPath);
+    };
+    /**
+     * Check if the configuration has changed and reload if so.
+     * For remote URIs, re-downloads using ETag caching (HTTP 304 avoids redundant downloads).
+     * For local paths, compares a SHA-256 hash of all config JSON files against the last loaded hash.
+     */
+    this.refreshIfNeeded = async () => {
+      if (!this._localPath && !this._configUri) return;
+      if (this._configUri && this._isRemoteUri(this._configUri)) {
+        const changed = await this._downloadRemoteConfig(this._configUri);
+        if (!changed) return;
+      }
+      const newHash = this._computeConfigHash();
+      if (newHash !== this._configHash) {
+        Logger_default.log("Configuration change detected, reloading environment");
+        this.load(this._localPath);
+      }
+    };
+    /**
+     * Synchronous load for worker threads. Reads `REMORA_CONFIG_URI` to determine
+     * the config source — for remote URIs it loads from the local cache (already downloaded
+     * by the main thread), for local paths it loads directly.
+     */
+    this.loadFromResolvedUri = () => {
+      const configUri = process.env.REMORA_CONFIG_URI;
+      if (!configUri) {
+        this.load("./");
+        return;
+      }
+      if (this._isRemoteUri(configUri)) {
+        this.load(this._resolveLocalCachePath());
+      } else {
+        this.load(configUri);
+      }
+    };
+    /**
+     * Load environment configuration from a local directory.
+     * Reads `{remoraPath}/remora/project.json` and all referenced sources, producers,
+     * consumers and schemas, validates them against JSON schemas, and initializes the singleton.
+     */
     this.load = (remoraPath) => {
+      this._localPath = remoraPath;
       const envPath = import_path5.default.join(remoraPath, "remora");
       const projectPath = import_path5.default.join(envPath, "project.json");
       if (!import_fs4.default.existsSync(projectPath))
@@ -14109,6 +14182,14 @@ var EnvironmentClass = class {
         if (!SchemaValidator_default.validate("source-schema", source))
           throw new Error(`Invalid source configuration: ${source.name}`);
       });
+      const projectApiQueueSource = projectConfig.settings.API_QUEUE?.source;
+      if (projectApiQueueSource) {
+        const source = sources.find((item) => item.name === projectApiQueueSource);
+        if (!source)
+          throw new Error(`Invalid project configuration: API_QUEUE source "${projectApiQueueSource}" was not found`);
+        if (source.engine !== "aws-sqs")
+          throw new Error(`Invalid project configuration: API_QUEUE source "${projectApiQueueSource}" must use engine "aws-sqs"`);
+      }
       const producers = loadConfigurations(envPath, projectConfig.producers);
       producers.forEach((producer) => {
         if (!SchemaValidator_default.validate("producer-schema", producer))
@@ -14122,13 +14203,10 @@ var EnvironmentClass = class {
       const envSettings = new Map(Object.entries({ ...projectConfig.settings }).map(([key, value]) => [key, String(value)]));
       if (!envSettings.has("SQL_MAX_QUERY_ROWS"))
         envSettings.set("SQL_MAX_QUERY_ROWS", Constants_default.defaults.SQL_MAX_QUERY_ROWS.toString());
-      if (!envSettings.has("STRING_MAX_CHARACTERS_LENGTH"))
-        envSettings.set("STRING_MAX_CHARACTERS_LENGTH", Constants_default.defaults.STRING_MAX_CHARACTERS_LENGTH.toString());
-      if (!envSettings.has("MAX_ITEMS_IN_MEMORY"))
-        envSettings.set("MAX_ITEMS_IN_MEMORY", Constants_default.defaults.MAX_ITEMS_IN_MEMORY.toString());
       const debugMode = process.env.REMORA_DEBUG_MODE;
       if (debugMode && debugMode.toLowerCase() === "true")
         Logger_default.setLevel("debug");
+      this._projectSettings = projectConfig.settings;
       this.init({
         settings: envSettings,
         sources,
@@ -14139,10 +14217,14 @@ var EnvironmentClass = class {
         // TODO: Add SQL library loading if needed
       });
       Logger_default.log(`Environment loaded: ${sources.length} source(s), ${producers.length} producer(s), ${consumers.length} consumer(s)`);
+      this._configHash = this._computeConfigHash();
     };
     this.get = (setting) => {
       return this._env.settings.get(setting);
     };
+    this.getProjectSettings = () => {
+      return this._projectSettings;
+    };
     this.getSource = (sourceName) => {
       Affirm_default(sourceName, "Invalid source name");
       return this._env.sources.find((x) => x.name === sourceName);
@@ -14221,6 +14303,58 @@ ${ce.map((x) => `	-${x}
       }
       return errors;
     };
+    this._isRemoteUri = (uri) => {
+      return uri.startsWith("http://") || uri.startsWith("https://");
+    };
+    this._resolveLocalCachePath = () => {
+      return process.env.REMORA_LOCAL_CACHE_PATH || "/app";
+    };
+    this._downloadRemoteConfig = async (url) => {
+      const headers = {};
+      if (this._lastEtag)
+        headers["If-None-Match"] = this._lastEtag;
+      const response = await fetch(url, { headers });
+      if (response.status === 304) return false;
+      if (!response.ok)
+        throw new Error(`Failed to download configuration from ${url}: HTTP ${response.status}`);
+      const etag = response.headers.get("etag");
+      if (etag) this._lastEtag = etag;
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const zip = new import_adm_zip.default(buffer);
+      const extractPath = import_path5.default.join(this._localPath, "remora");
+      if (import_fs4.default.existsSync(extractPath)) {
+        for (const file of import_fs4.default.readdirSync(extractPath)) {
+          if (file === "temp") continue;
+          const filePath = import_path5.default.join(extractPath, file);
+          if (import_fs4.default.statSync(filePath).isDirectory())
+            import_fs4.default.rmSync(filePath, { recursive: true, force: true });
+          else
+            import_fs4.default.unlinkSync(filePath);
+        }
+      }
+      zip.extractAllTo(extractPath, true);
+      Logger_default.log(`Remote configuration downloaded and extracted from ${url}`);
+      return true;
+    };
+    this._computeConfigHash = () => {
+      if (!this._localPath) return null;
+      const envPath = import_path5.default.join(this._localPath, "remora");
+      if (!import_fs4.default.existsSync(envPath)) return null;
+      const hash = import_crypto.default.createHash("sha256");
+      const hashDir = (dirPath) => {
+        if (!import_fs4.default.existsSync(dirPath)) return;
+        for (const entry of import_fs4.default.readdirSync(dirPath, { withFileTypes: true })) {
+          if (entry.name === "temp" || entry.name === ".temp") continue;
+          const fullEntry = import_path5.default.join(dirPath, entry.name);
+          if (entry.isDirectory())
+            hashDir(fullEntry);
+          else if (entry.name.endsWith(".json"))
+            hash.update(import_fs4.default.readFileSync(fullEntry, "utf-8"));
+        }
+      };
+      hashDir(envPath);
+      return hash.digest("hex");
+    };
   }
 };
 var Environment = new EnvironmentClass();
@@ -15213,11 +15347,13 @@ var DriverHelper = {
     const { append, destinationPath, objects, delimiter } = options;
     const writeOptions = append ? { flags: "a" } : {};
     const writeStream = (0, import_fs5.createWriteStream)(destinationPath, writeOptions);
+    const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
     let lineCount = 0;
     const keys = Object.keys(objects[0]);
     for (const obj of objects) {
       const serialized = keys.map((k) => obj[k]).join(delimiter) + "\n";
-      writeStream.write(serialized);
+      if (!writeStream.write(serialized))
+        await waitForDrain();
       lineCount++;
     }
     writeStream.end();
@@ -15564,8 +15700,10 @@ var LocalDestinationDriver = class {
         const reader = fs9.createReadStream(fromPath);
         const lineReader = import_readline3.default.createInterface({ input: reader, crlfDelay: Infinity });
         const writer = fs9.createWriteStream(toFilePath);
+        const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
         for await (const line of lineReader) {
-          writer.write(transform(line) + "\n");
+          if (!writer.write(transform(line) + "\n"))
+            await waitForDrain();
         }
         writer.end();
         await new Promise((resolve, reject) => {
@@ -16102,7 +16240,7 @@ var debug = async (options) => {
 // src/actions/deploy.ts
 var import_chalk4 = __toESM(require("chalk"));
 var import_fs8 = __toESM(require("fs"));
-var import_adm_zip = __toESM(require("adm-zip"));
+var import_adm_zip2 = __toESM(require("adm-zip"));
 var import_path11 = __toESM(require("path"));
 var deploy = async (options) => {
   console.log(import_chalk4.default.blue.bold(`\u{1F680} Deploying to ${options.env}...`));
@@ -16110,7 +16248,7 @@ var deploy = async (options) => {
     const rootDir = "./remora";
     if (!import_fs8.default.existsSync("./remora"))
       throw new Error(import_chalk4.default.red("Missing directory: ") + import_chalk4.default.yellow("./remora"));
-    const zip = new import_adm_zip.default();
+    const zip = new import_adm_zip2.default();
     const addDirectoryToZip = (directoryPath, zipPath = "") => {
       const files = import_fs8.default.readdirSync(directoryPath);
       files.forEach((file) => {
@@ -16616,7 +16754,7 @@ var ApiKeysManagerClass = class {
 var ApiKeysManager = new ApiKeysManagerClass();
 
 // ../../packages/auth/src/LicenceManager.ts
-var import_crypto = __toESM(require("crypto"), 1);
+var import_crypto2 = __toESM(require("crypto"), 1);
 var PUBLICK_KEY = `-----BEGIN PUBLIC KEY-----
 MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA7BWugM83YKGzTyZ6kJyy
 M01JoGYBQYn/9H9utQQyC/uugV4g9d7vv87I2yUfqiHtx7BQj0mOGctqnK7vuRcg
@@ -16638,7 +16776,7 @@ var LicenceManagerClass = class {
         expires: expirationDate.toISOString()
       };
       const licenceString = JSON.stringify(licenceData);
-      const sign = import_crypto.default.createSign("SHA256");
+      const sign = import_crypto2.default.createSign("SHA256");
       sign.update(licenceString);
       sign.end();
       const signature = sign.sign(privateKey, "base64");
@@ -16654,7 +16792,7 @@ var LicenceManagerClass = class {
         const expirationDate = new Date(data.expires);
         if (now > expirationDate)
           return { valid: false, reason: "License expired", expiryDate: expirationDate };
-        const verify = import_crypto.default.createVerify("SHA256");
+        const verify = import_crypto2.default.createVerify("SHA256");
         verify.update(JSON.stringify(data));
         const isSignatureValid = verify.verify(PUBLICK_KEY, signature, "base64");
         if (!isSignatureValid)
@@ -16680,9 +16818,10 @@ var import_path18 = __toESM(require("path"));
 var import_fs11 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
+var import_crypto5 = __toESM(require("crypto"));
 
 // ../../packages/engines/src/CryptoEngine.ts
-var import_crypto2 = __toESM(require("crypto"), 1);
+var import_crypto3 = __toESM(require("crypto"), 1);
 
 // ../../packages/engines/src/RandomEngine.ts
 var import_seedrandom = __toESM(require("seedrandom"), 1);
@@ -16748,7 +16887,7 @@ var CryptoEngineClass = class {
     };
     this.valueToHash = (value) => {
       const textValue = JSON.stringify(value);
-      return import_crypto2.default.createHash("sha256").update(textValue).digest("hex");
+      return import_crypto3.default.createHash("sha256").update(textValue).digest("hex");
     };
     this.hashValue = (maskType, value, valueType) => {
       if (!Algo_default.hasVal(value)) return value;
@@ -17064,7 +17203,7 @@ var AutoMapperEngine_default = AutoMapperEngine;
 
 // ../../packages/engines/src/producer/ProducerEngine.ts
 var import_path14 = __toESM(require("path"), 1);
-var import_crypto3 = require("crypto");
+var import_crypto4 = require("crypto");
 
 // ../../packages/engines/src/transform/TypeCaster.ts
 var import_dayjs = __toESM(require("dayjs"), 1);
@@ -17270,7 +17409,7 @@ var ProducerEngineClass = class {
             let effectiveProducer = producer;
             let effectiveSource = source;
             if (compressionType) {
-              scope = { id: (0, import_crypto3.randomUUID)(), folder: `sample-${(0, import_crypto3.randomUUID)()}`, workersId: [] };
+              scope = { id: (0, import_crypto4.randomUUID)(), folder: `sample-${(0, import_crypto4.randomUUID)()}`, workersId: [] };
               const driver = await DriverFactory_default.instantiateSource(source);
               const readyResult = await driver.ready({ producer, scope });
               const firstFile = readyResult.files[0].fullUri;
@@ -18419,8 +18558,9 @@ var TransformationEngineClass = class {
           throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
         }
         const { fields, separator = "", template } = transformations.combine_fields;
+        const currentFieldKey = field.alias ?? field.key;
         const fieldValues = fields.map((fieldName) => {
-          const fieldValue = record[fieldName];
+          const fieldValue = fieldName === currentFieldKey ? value : record[fieldName];
           return fieldValue !== null && fieldValue !== void 0 ? String(fieldValue) : "";
         });
         if (template) {
@@ -19180,12 +19320,15 @@ var ConsumerExecutorClass = class {
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let newLineCount = 0;
       const seen = /* @__PURE__ */ new Set();
       for await (const line of lineReader) {
-        if (!seen.has(line)) {
-          seen.add(line);
-          writer.write(line + "\n");
+        const hash = import_crypto5.default.createHash("sha1").update(line).digest("base64");
+        if (!seen.has(hash)) {
+          seen.add(hash);
+          if (!writer.write(line + "\n"))
+            await waitForDrain();
           newLineCount++;
         }
       }
@@ -19230,8 +19373,10 @@ var ConsumerExecutorClass = class {
       lineReader.close();
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       for (const { line } of winners.values()) {
-        writer.write(line + "\n");
+        if (!writer.write(line + "\n"))
+          await waitForDrain();
       }
       await new Promise((resolve, reject) => {
         writer.on("close", resolve);
@@ -19305,6 +19450,7 @@ var ConsumerExecutorClass = class {
       ];
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let outputCount = 0;
       for (const { rowRecord, cells } of groups.values()) {
         const outputRecord = { ...rowRecord };
@@ -19334,7 +19480,8 @@ var ConsumerExecutorClass = class {
           }
         }
         const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
-        writer.write(line + "\n");
+        if (!writer.write(line + "\n"))
+          await waitForDrain();
         outputCount++;
       }
       await new Promise((resolve, reject) => {
@@ -19581,6 +19728,7 @@ var ExecutorWriter = class {
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
       let writeStream = fs19.createWriteStream(destPath, { flags: "a" });
+      const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19592,7 +19740,8 @@ var ExecutorWriter = class {
           destPath = this.getCompletedPath(sourcePath, writerIndex);
           writeStream = fs19.createWriteStream(destPath, { flags: "a" });
         }
-        writeStream.write(line + "\n");
+        if (!writeStream.write(line + "\n"))
+          await waitForDrain();
       }
       writeStream.end();
       await new Promise((resolve, reject) => {
@@ -19673,7 +19822,9 @@ var ExecutorOrchestratorClass = class {
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
-      const { consumer, details, logProgress, options } = request;
+      await Environment_default.refreshIfNeeded();
+      const { details, logProgress, options } = request;
+      const consumer = Environment_default.getConsumer(request.consumer.name) ?? request.consumer;
       Affirm_default(consumer, "Invalid consumer");
       Affirm_default(details, "Invalid execution details");
       const tracker = new ExecutorPerformance_default();
@@ -19843,7 +19994,8 @@ var ExecutorOrchestratorClass = class {
       if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
         return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
       }
-      const cpus = numChunks ?? import_os.default.cpus().length - 1;
+      const availableCores = Math.max(1, Math.floor(import_os.default.cpus().length * 0.75));
+      const cpus = numChunks ?? Math.min(availableCores, Constants_default.defaults.MAX_THREAD_COUNT);
       const maxChunksBySize = Math.floor(fileSize / Constants_default.defaults.MIN_CHUNK_SIZE);
       const effectiveChunks = Math.min(cpus, maxChunksBySize);
       if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
@@ -19852,7 +20004,7 @@ var ExecutorOrchestratorClass = class {
       try {
         const offsets = [];
         let currentStart = 0;
-        for (let i = 0; i < cpus - 1; i++) {
+        for (let i = 0; i < effectiveChunks - 1; i++) {
           const targetEnd = currentStart + targetChunkSize;
           if (targetEnd >= fileSize) {
             break;
@@ -19926,6 +20078,7 @@ var ExecutorOrchestratorClass = class {
             import_fs12.default.createReadStream(workerResult.resultUri),
             import_fs12.default.createWriteStream(mainPath, { flags: "a" })
           );
+          await import_promises9.default.unlink(workerResult.resultUri);
         }
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);

package/json_schemas/project-schema.json

@@ -63,6 +63,18 @@
                     "minimum": 1,
                     "description": "Maximum number of rows for SQL queries"
                 },
+                "API_QUEUE": {
+                    "type": "object",
+                    "required": ["source"],
+                    "additionalProperties": false,
+                    "properties": {
+                        "source": {
+                            "type": "string",
+                            "minLength": 1,
+                            "description": "Name of an aws-sqs source used as an alternative execute-consumer API"
+                        }
+                    }
+                },
                 "STRING_MAX_CHARACTERS_LENGTH": {
                     "type": "integer",
                     "minimum": 1,
@@ -88,7 +100,10 @@
             "sources": ["/sources"],
             "schemas": ["/schemas"],
             "settings": {
-                "SQL_MAX_QUERY_ROWS": 10000
+                "SQL_MAX_QUERY_ROWS": 10000,
+                "API_QUEUE": {
+                    "source": "remora-api-queue"
+                }
             }
         }
     ]

package/json_schemas/source-schema.json

@@ -22,6 +22,7 @@
                 "aws-redshift",
                 "aws-dynamodb",
                 "aws-s3",
+                "aws-sqs",
                 "postgres",
                 "local",
                 "delta-share",
@@ -51,6 +52,10 @@
                     "type": "string",
                     "description": "Hostname or endpoint of the data source"
                 },
+                "accountId": {
+                    "type": "string",
+                    "description": "AWS account ID used to construct resource URLs when only a resource name is provided"
+                },
                 "user": {
                     "type": "string",
                     "description": "Username for authentication"
@@ -95,6 +100,10 @@
                     "type": "string",
                     "description": "AWS region"
                 },
+                "queue": {
+                    "type": "string",
+                    "description": "SQS queue URL or queue name"
+                },
                 "bucket": {
                     "type": "string",
                     "description": "S3 bucket name"
@@ -202,6 +211,20 @@
             },
             "_version": 2
         },
+        {
+            "name": "Consumer Execute Queue",
+            "description": "SQS queue used as an alternative execute-consumer API",
+            "engine": "aws-sqs",
+            "authentication": {
+                "method": "access-secret-key",
+                "queue": "remora-execute-consumer",
+                "accountId": "{AWS_ACCOUNT_ID}",
+                "region": "us-east-1",
+                "accessKey": "{AWS_ACCESS_KEY_ID}",
+                "secretKey": "{AWS_SECRET_ACCESS_KEY}"
+            },
+            "_version": 1
+        },
         {
             "name": "Redshift Data Warehouse",
             "engine": "aws-redshift",

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.5",
+    "version": "1.2.6",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -13512,12 +13512,10 @@ var CONSTANTS = {
     REMORA_PATH: "./remora",
     PRODUCER_TEMP_FOLDER: ".temp",
     SQL_MAX_QUERY_ROWS: 1e4,
-    STRING_MAX_CHARACTERS_LENGTH: 1e7,
-    MAX_ITEMS_IN_MEMORY: 2e5,
     MIN_RUNTIME_HEAP_MB: 4e3,
     RECOMMENDED_RUNTIME_HEAP_MB: 8e3,
     INDICATIVE_THREAD_LINE_COUNT: 75e4,
-    MAX_THREAD_COUNT: 8,
+    MAX_THREAD_COUNT: 99,
     /**
      * Minimum file size to consider parallel processing (10 MB)
      */
@@ -13599,6 +13597,8 @@ var ExecutorScope_default = ExecutorScope;
 
 // ../../packages/common/src/Environment.ts
 var import_fs4 = __toESM(require("fs"), 1);
+var import_crypto = __toESM(require("crypto"), 1);
+var import_adm_zip = __toESM(require("adm-zip"), 1);
 var import_path5 = __toESM(require("path"), 1);
 
 // ../../packages/common/src/schema/SchemaValidator.ts
@@ -13720,6 +13720,12 @@ var ValidatorClass = class {
           const source = sources[i];
           if (source.engine === "local" && !source.authentication.path)
             errors.push(`For source ${source.name}, the path has not been configured`);
+          if (source.engine === "aws-sqs") {
+            if (!source.authentication.queue)
+              errors.push(`For source ${source.name}, the queue has not been configured`);
+            if (!source.authentication.region && !source.authentication.queue?.startsWith("https://"))
+              errors.push(`For source ${source.name}, the region has not been configured`);
+          }
         }
       } catch (e) {
         if (errors.length === 0)
@@ -14010,8 +14016,11 @@ var ValidatorClass = class {
         if (!auth) continue;
         const ctx = (field) => `source "${source.name}" authentication.${field}`;
         checkValue(auth.accessKey, ctx("accessKey"));
+        checkValue(auth.accountId, ctx("accountId"));
         checkValue(auth.secretKey, ctx("secretKey"));
         checkValue(auth.sessionToken, ctx("sessionToken"));
+        checkValue(auth.queue, ctx("queue"));
+        checkValue(auth.region, ctx("region"));
         checkValue(auth.bearerToken, ctx("bearerToken"));
         checkValue(auth.url, ctx("url"));
         checkValue(auth.apiKey, ctx("apiKey"));
@@ -14030,6 +14039,8 @@ var ValidatorClass = class {
       switch (engine) {
         case "aws-dynamodb":
           return "no-sql";
+        case "aws-sqs":
+          return "no-sql";
         case "aws-redshift":
         case "postgres":
           return "sql";
@@ -14049,10 +14060,72 @@ var Validator_default = Validator;
 var EnvironmentClass = class {
   constructor() {
     this._env = null;
+    this._configUri = null;
+    this._localPath = null;
+    this._configHash = null;
+    this._lastEtag = null;
+    this._projectSettings = null;
     this.init = (env) => {
       this._env = env;
     };
+    /**
+     * Load environment from a local path or remote URL (e.g. S3 presigned URL).
+     * Remote configs are downloaded as a zip and extracted to the local cache path.
+     * Falls back to `REMORA_CONFIG_URI` env variable if no URI is provided.
+     */
+    this.loadFromUri = async (uri) => {
+      const configUri = uri || process.env.REMORA_CONFIG_URI;
+      Affirm_default(configUri, "No configuration URI provided. Set REMORA_CONFIG_URI environment variable or pass a URI.");
+      this._configUri = configUri;
+      if (this._isRemoteUri(configUri)) {
+        this._localPath = this._resolveLocalCachePath();
+        await this._downloadRemoteConfig(configUri);
+      } else {
+        this._localPath = configUri;
+      }
+      this.load(this._localPath);
+    };
+    /**
+     * Check if the configuration has changed and reload if so.
+     * For remote URIs, re-downloads using ETag caching (HTTP 304 avoids redundant downloads).
+     * For local paths, compares a SHA-256 hash of all config JSON files against the last loaded hash.
+     */
+    this.refreshIfNeeded = async () => {
+      if (!this._localPath && !this._configUri) return;
+      if (this._configUri && this._isRemoteUri(this._configUri)) {
+        const changed = await this._downloadRemoteConfig(this._configUri);
+        if (!changed) return;
+      }
+      const newHash = this._computeConfigHash();
+      if (newHash !== this._configHash) {
+        Logger_default.log("Configuration change detected, reloading environment");
+        this.load(this._localPath);
+      }
+    };
+    /**
+     * Synchronous load for worker threads. Reads `REMORA_CONFIG_URI` to determine
+     * the config source — for remote URIs it loads from the local cache (already downloaded
+     * by the main thread), for local paths it loads directly.
+     */
+    this.loadFromResolvedUri = () => {
+      const configUri = process.env.REMORA_CONFIG_URI;
+      if (!configUri) {
+        this.load("./");
+        return;
+      }
+      if (this._isRemoteUri(configUri)) {
+        this.load(this._resolveLocalCachePath());
+      } else {
+        this.load(configUri);
+      }
+    };
+    /**
+     * Load environment configuration from a local directory.
+     * Reads `{remoraPath}/remora/project.json` and all referenced sources, producers,
+     * consumers and schemas, validates them against JSON schemas, and initializes the singleton.
+     */
     this.load = (remoraPath) => {
+      this._localPath = remoraPath;
       const envPath = import_path5.default.join(remoraPath, "remora");
       const projectPath = import_path5.default.join(envPath, "project.json");
       if (!import_fs4.default.existsSync(projectPath))
@@ -14103,6 +14176,14 @@ var EnvironmentClass = class {
         if (!SchemaValidator_default.validate("source-schema", source))
           throw new Error(`Invalid source configuration: ${source.name}`);
       });
+      const projectApiQueueSource = projectConfig.settings.API_QUEUE?.source;
+      if (projectApiQueueSource) {
+        const source = sources.find((item) => item.name === projectApiQueueSource);
+        if (!source)
+          throw new Error(`Invalid project configuration: API_QUEUE source "${projectApiQueueSource}" was not found`);
+        if (source.engine !== "aws-sqs")
+          throw new Error(`Invalid project configuration: API_QUEUE source "${projectApiQueueSource}" must use engine "aws-sqs"`);
+      }
       const producers = loadConfigurations(envPath, projectConfig.producers);
       producers.forEach((producer) => {
         if (!SchemaValidator_default.validate("producer-schema", producer))
@@ -14116,13 +14197,10 @@ var EnvironmentClass = class {
       const envSettings = new Map(Object.entries({ ...projectConfig.settings }).map(([key, value]) => [key, String(value)]));
       if (!envSettings.has("SQL_MAX_QUERY_ROWS"))
         envSettings.set("SQL_MAX_QUERY_ROWS", Constants_default.defaults.SQL_MAX_QUERY_ROWS.toString());
-      if (!envSettings.has("STRING_MAX_CHARACTERS_LENGTH"))
-        envSettings.set("STRING_MAX_CHARACTERS_LENGTH", Constants_default.defaults.STRING_MAX_CHARACTERS_LENGTH.toString());
-      if (!envSettings.has("MAX_ITEMS_IN_MEMORY"))
-        envSettings.set("MAX_ITEMS_IN_MEMORY", Constants_default.defaults.MAX_ITEMS_IN_MEMORY.toString());
       const debugMode = process.env.REMORA_DEBUG_MODE;
       if (debugMode && debugMode.toLowerCase() === "true")
         Logger_default.setLevel("debug");
+      this._projectSettings = projectConfig.settings;
       this.init({
         settings: envSettings,
         sources,
@@ -14133,10 +14211,14 @@ var EnvironmentClass = class {
         // TODO: Add SQL library loading if needed
       });
       Logger_default.log(`Environment loaded: ${sources.length} source(s), ${producers.length} producer(s), ${consumers.length} consumer(s)`);
+      this._configHash = this._computeConfigHash();
     };
     this.get = (setting) => {
       return this._env.settings.get(setting);
     };
+    this.getProjectSettings = () => {
+      return this._projectSettings;
+    };
     this.getSource = (sourceName) => {
       Affirm_default(sourceName, "Invalid source name");
       return this._env.sources.find((x) => x.name === sourceName);
@@ -14215,6 +14297,58 @@ ${ce.map((x) => `	-${x}
       }
       return errors;
     };
+    this._isRemoteUri = (uri) => {
+      return uri.startsWith("http://") || uri.startsWith("https://");
+    };
+    this._resolveLocalCachePath = () => {
+      return process.env.REMORA_LOCAL_CACHE_PATH || "/app";
+    };
+    this._downloadRemoteConfig = async (url) => {
+      const headers = {};
+      if (this._lastEtag)
+        headers["If-None-Match"] = this._lastEtag;
+      const response = await fetch(url, { headers });
+      if (response.status === 304) return false;
+      if (!response.ok)
+        throw new Error(`Failed to download configuration from ${url}: HTTP ${response.status}`);
+      const etag = response.headers.get("etag");
+      if (etag) this._lastEtag = etag;
+      const buffer = Buffer.from(await response.arrayBuffer());
+      const zip = new import_adm_zip.default(buffer);
+      const extractPath = import_path5.default.join(this._localPath, "remora");
+      if (import_fs4.default.existsSync(extractPath)) {
+        for (const file of import_fs4.default.readdirSync(extractPath)) {
+          if (file === "temp") continue;
+          const filePath = import_path5.default.join(extractPath, file);
+          if (import_fs4.default.statSync(filePath).isDirectory())
+            import_fs4.default.rmSync(filePath, { recursive: true, force: true });
+          else
+            import_fs4.default.unlinkSync(filePath);
+        }
+      }
+      zip.extractAllTo(extractPath, true);
+      Logger_default.log(`Remote configuration downloaded and extracted from ${url}`);
+      return true;
+    };
+    this._computeConfigHash = () => {
+      if (!this._localPath) return null;
+      const envPath = import_path5.default.join(this._localPath, "remora");
+      if (!import_fs4.default.existsSync(envPath)) return null;
+      const hash = import_crypto.default.createHash("sha256");
+      const hashDir = (dirPath) => {
+        if (!import_fs4.default.existsSync(dirPath)) return;
+        for (const entry of import_fs4.default.readdirSync(dirPath, { withFileTypes: true })) {
+          if (entry.name === "temp" || entry.name === ".temp") continue;
+          const fullEntry = import_path5.default.join(dirPath, entry.name);
+          if (entry.isDirectory())
+            hashDir(fullEntry);
+          else if (entry.name.endsWith(".json"))
+            hash.update(import_fs4.default.readFileSync(fullEntry, "utf-8"));
+        }
+      };
+      hashDir(envPath);
+      return hash.digest("hex");
+    };
   }
 };
 var Environment = new EnvironmentClass();
@@ -14225,9 +14359,10 @@ var import_path15 = __toESM(require("path"));
 var import_fs9 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
+var import_crypto4 = __toESM(require("crypto"));
 
 // ../../packages/engines/src/CryptoEngine.ts
-var import_crypto = __toESM(require("crypto"), 1);
+var import_crypto2 = __toESM(require("crypto"), 1);
 
 // ../../packages/engines/src/RandomEngine.ts
 var import_seedrandom = __toESM(require("seedrandom"), 1);
@@ -14293,7 +14428,7 @@ var CryptoEngineClass = class {
     };
     this.valueToHash = (value) => {
       const textValue = JSON.stringify(value);
-      return import_crypto.default.createHash("sha256").update(textValue).digest("hex");
+      return import_crypto2.default.createHash("sha256").update(textValue).digest("hex");
     };
     this.hashValue = (maskType, value, valueType) => {
       if (!Algo_default.hasVal(value)) return value;
@@ -14608,7 +14743,7 @@ var AutoMapperEngine = new AutoMapperEngineClass();
 
 // ../../packages/engines/src/producer/ProducerEngine.ts
 var import_path11 = __toESM(require("path"), 1);
-var import_crypto2 = require("crypto");
+var import_crypto3 = require("crypto");
 
 // ../../packages/drivers/src/DeltaShareDriver.ts
 var DeltaShareSourceDriver = class {
@@ -15555,11 +15690,13 @@ var DriverHelper = {
     const { append, destinationPath, objects, delimiter } = options;
     const writeOptions = append ? { flags: "a" } : {};
     const writeStream = (0, import_fs5.createWriteStream)(destinationPath, writeOptions);
+    const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
     let lineCount = 0;
     const keys = Object.keys(objects[0]);
     for (const obj of objects) {
       const serialized = keys.map((k) => obj[k]).join(delimiter) + "\n";
-      writeStream.write(serialized);
+      if (!writeStream.write(serialized))
+        await waitForDrain();
       lineCount++;
     }
     writeStream.end();
@@ -15906,8 +16043,10 @@ var LocalDestinationDriver = class {
         const reader = fs8.createReadStream(fromPath);
         const lineReader = import_readline3.default.createInterface({ input: reader, crlfDelay: Infinity });
         const writer = fs8.createWriteStream(toFilePath);
+        const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
         for await (const line of lineReader) {
-          writer.write(transform(line) + "\n");
+          if (!writer.write(transform(line) + "\n"))
+            await waitForDrain();
         }
         writer.end();
         await new Promise((resolve, reject) => {
@@ -16600,7 +16739,7 @@ var ProducerEngineClass = class {
             let effectiveProducer = producer;
             let effectiveSource = source;
             if (compressionType) {
-              scope = { id: (0, import_crypto2.randomUUID)(), folder: `sample-${(0, import_crypto2.randomUUID)()}`, workersId: [] };
+              scope = { id: (0, import_crypto3.randomUUID)(), folder: `sample-${(0, import_crypto3.randomUUID)()}`, workersId: [] };
               const driver = await DriverFactory_default.instantiateSource(source);
               const readyResult = await driver.ready({ producer, scope });
               const firstFile = readyResult.files[0].fullUri;
@@ -17749,8 +17888,9 @@ var TransformationEngineClass = class {
           throw new Error(`Cannot apply combine_fields transformation without record context in field '${field.key}'`);
         }
         const { fields, separator = "", template } = transformations.combine_fields;
+        const currentFieldKey = field.alias ?? field.key;
         const fieldValues = fields.map((fieldName) => {
-          const fieldValue = record[fieldName];
+          const fieldValue = fieldName === currentFieldKey ? value : record[fieldName];
           return fieldValue !== null && fieldValue !== void 0 ? String(fieldValue) : "";
         });
         if (template) {
@@ -18779,12 +18919,15 @@ var ConsumerExecutorClass = class {
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs9.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let newLineCount = 0;
       const seen = /* @__PURE__ */ new Set();
       for await (const line of lineReader) {
-        if (!seen.has(line)) {
-          seen.add(line);
-          writer.write(line + "\n");
+        const hash = import_crypto4.default.createHash("sha1").update(line).digest("base64");
+        if (!seen.has(hash)) {
+          seen.add(hash);
+          if (!writer.write(line + "\n"))
+            await waitForDrain();
           newLineCount++;
         }
       }
@@ -18829,8 +18972,10 @@ var ConsumerExecutorClass = class {
       lineReader.close();
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs9.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       for (const { line } of winners.values()) {
-        writer.write(line + "\n");
+        if (!writer.write(line + "\n"))
+          await waitForDrain();
       }
       await new Promise((resolve, reject) => {
         writer.on("close", resolve);
@@ -18904,6 +19049,7 @@ var ConsumerExecutorClass = class {
       ];
       const tempWorkPath = datasetPath + "_tmp";
       const writer = import_fs9.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let outputCount = 0;
       for (const { rowRecord, cells } of groups.values()) {
         const outputRecord = { ...rowRecord };
@@ -18933,7 +19079,8 @@ var ConsumerExecutorClass = class {
           }
         }
         const line = OutputExecutor_default.outputRecord(outputRecord, consumer, pivotedFields);
-        writer.write(line + "\n");
+        if (!writer.write(line + "\n"))
+          await waitForDrain();
         outputCount++;
       }
       await new Promise((resolve, reject) => {
@@ -19136,6 +19283,12 @@ var Executor = class {
       Logger_default.log(`[${workerId}] Opened streams \u2014 chunk ${start}-${end} (${Math.round(totalBytes / 1024)}KB), ${fields.length} field(s)`);
       Logger_default.log(`[${workerId}] Starting line-by-line processing`);
       const lineStream = import_readline7.default.createInterface({ input: readStream, crlfDelay: Infinity });
+      let drainCount = 0;
+      const waitForDrain = () => {
+        drainCount++;
+        return new Promise((resolve) => writeStream.once("drain", resolve));
+      };
+      const isDebug = Logger_default._level === "debug";
       for await (const line of lineStream) {
         if (lineIndex === 0 && isFirstChunk) {
           if (!this.shouldProcessFirstLine(producer)) {
@@ -19154,11 +19307,11 @@ var Executor = class {
         });
         this._performance.measure("process-line", performance.now() - perf);
         if (!record) {
-          Logger_default.log(`[${workerId}] Line ${lineIndex}: skipped by producer (no record produced)`);
+          if (isDebug) Logger_default.log(`[${workerId}] Line ${lineIndex}: skipped by producer (no record produced)`);
           lineIndex++;
           continue;
         }
-        Logger_default.log(`[${workerId}] Line ${lineIndex}: parsed by producer`);
+        if (isDebug) Logger_default.log(`[${workerId}] Line ${lineIndex}: parsed by producer`);
         perf = performance.now();
         record = ConsumerExecutor_default.processRecord({
           record,
@@ -19171,16 +19324,17 @@ var Executor = class {
         });
         this._performance.measure("process-record", performance.now() - perf);
         if (!record) {
-          Logger_default.log(`[${workerId}] Line ${lineIndex}: filtered out by consumer`);
+          if (isDebug) Logger_default.log(`[${workerId}] Line ${lineIndex}: filtered out by consumer`);
           lineIndex++;
           continue;
         }
-        Logger_default.log(`[${workerId}] Line ${lineIndex}: processed by consumer`);
+        if (isDebug) Logger_default.log(`[${workerId}] Line ${lineIndex}: processed by consumer`);
         perf = performance.now();
         const outputLine = OutputExecutor_default.outputRecord(record, consumer, fields);
         this._performance.measure("output-record", performance.now() - perf);
-        Logger_default.log(`[${workerId}] Line ${lineIndex}: output written (record #${totalOutputCount + 1})`);
-        writeStream.write(outputLine + "\n");
+        if (isDebug) Logger_default.log(`[${workerId}] Line ${lineIndex}: output written (record #${totalOutputCount + 1})`);
+        if (!writeStream.write(outputLine + "\n"))
+          await waitForDrain();
         totalOutputCount++;
         lineIndex++;
         if (reportWork && lineIndex % this._REPORT_WORK_AFTER_LINES === 0) {
@@ -19192,7 +19346,8 @@ var Executor = class {
           break;
         }
       }
-      Logger_default.log(`[${workerId}] Line processing complete \u2014 ${lineIndex} lines read, ${totalOutputCount} records written`);
+      Logger_default.log(`[${workerId}] Line processing complete \u2014 ${lineIndex} lines read, ${totalOutputCount} records written, ${drainCount} drain(s)`);
+      this._performance.measure("drain-count", drainCount);
       writeStream.end();
       await new Promise((resolve, reject) => {
         writeStream.on("finish", resolve);
@@ -19340,6 +19495,7 @@ var ExecutorWriter = class {
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
       let writeStream = fs16.createWriteStream(destPath, { flags: "a" });
+      const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19351,7 +19507,8 @@ var ExecutorWriter = class {
           destPath = this.getCompletedPath(sourcePath, writerIndex);
           writeStream = fs16.createWriteStream(destPath, { flags: "a" });
         }
-        writeStream.write(line + "\n");
+        if (!writeStream.write(line + "\n"))
+          await waitForDrain();
       }
       writeStream.end();
       await new Promise((resolve, reject) => {
@@ -19432,7 +19589,9 @@ var ExecutorOrchestratorClass = class {
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
-      const { consumer, details, logProgress, options } = request;
+      await Environment_default.refreshIfNeeded();
+      const { details, logProgress, options } = request;
+      const consumer = Environment_default.getConsumer(request.consumer.name) ?? request.consumer;
       Affirm_default(consumer, "Invalid consumer");
       Affirm_default(details, "Invalid execution details");
       const tracker = new ExecutorPerformance_default();
@@ -19602,7 +19761,8 @@ var ExecutorOrchestratorClass = class {
       if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
         return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
       }
-      const cpus = numChunks ?? import_os.default.cpus().length - 1;
+      const availableCores = Math.max(1, Math.floor(import_os.default.cpus().length * 0.75));
+      const cpus = numChunks ?? Math.min(availableCores, Constants_default.defaults.MAX_THREAD_COUNT);
       const maxChunksBySize = Math.floor(fileSize / Constants_default.defaults.MIN_CHUNK_SIZE);
       const effectiveChunks = Math.min(cpus, maxChunksBySize);
       if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri }];
@@ -19611,7 +19771,7 @@ var ExecutorOrchestratorClass = class {
       try {
         const offsets = [];
         let currentStart = 0;
-        for (let i = 0; i < cpus - 1; i++) {
+        for (let i = 0; i < effectiveChunks - 1; i++) {
           const targetEnd = currentStart + targetChunkSize;
           if (targetEnd >= fileSize) {
             break;
@@ -19685,6 +19845,7 @@ var ExecutorOrchestratorClass = class {
             import_fs11.default.createReadStream(workerResult.resultUri),
             import_fs11.default.createWriteStream(mainPath, { flags: "a" })
           );
+          await import_promises9.default.unlink(workerResult.resultUri);
         }
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
@@ -19751,7 +19912,7 @@ var ExecutorOrchestrator = new ExecutorOrchestratorClass();
 // src/workers/ExecutorWorker.ts
 import_dotenv.default.configDotenv();
 var run = async (workerData) => {
-  Environment_default.load("./");
+  Environment_default.loadFromResolvedUri();
   if (workerData.loggerConfig)
     Logger_default.initFromConfig(workerData.loggerConfig);
   try {