@forzalabs/remora 2.0.0 → 2.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -11057,9 +11057,20 @@ var init_FileLogService = __esm({
     FileLogServiceClass = class {
       constructor() {
         this._enabled = false;
+        /**
+         * Points the sink at a file, once per target.
+         *
+         * Idempotent on purpose: the executor worker re-applies the parent's logger config at the START
+         * OF EVERY TASK, and a thread under the run-long pool handles many tasks. Building a transport
+         * per call left one open append stream per task on the same file.
+         */
         this.enable = (folder = "./remora/.temp/logs", file = "remora.log") => {
+          if (this._enabled && this._folder === folder && this._file === file)
+            return;
           if (!(0, import_fs.existsSync)(folder))
             (0, import_fs.mkdirSync)(folder, { recursive: true });
+          if (this._logger)
+            this._logger.end();
           this._folder = folder;
           this._file = file;
           this._logger = import_winston.default.createLogger({
@@ -11073,6 +11084,9 @@ ${stack}` : base;
               })
             ),
             transports: [
+              // Rotation is the transport's job. A run writes this file from the main process and
+              // from every executor thread, so a rotation renames a file others hold open — the
+              // 'error' handler below is what keeps that a logged line rather than a dead thread.
               new import_winston.default.transports.File({
                 filename: import_path.default.join(folder, file),
                 maxsize: 10 * 1024 * 1024,
@@ -11082,6 +11096,7 @@ ${stack}` : base;
               })
             ]
           });
+          this._logger.on("error", (error) => console.error(`File logger error: ${error?.message ?? String(error)}`));
           this._enabled = true;
         };
         /**
@@ -11123,11 +11138,22 @@ ${stack}` : base;
             }
           });
         };
+        /**
+         * Ends the sink. The state goes with it, so the next write re-enables a fresh one.
+         *
+         * That reset is the point: a closed winston logger still accepts `log()` calls and writes them
+         * into an ended stream, which throws ERR_STREAM_WRITE_AFTER_END from a stream nothing is
+         * listening to. Leaving `_enabled` true after a close is what turned "someone logged after
+         * close" into "the process died". Use `flush()` when the sink is still needed afterwards.
+         */
         this.close = () => {
           if (!this._enabled || !this._logger) return Promise.resolve();
+          const closing = this._logger;
+          this._enabled = false;
+          this._logger = null;
           return new Promise((resolve) => {
-            this._logger.on("finish", resolve);
-            this._logger.end();
+            closing.on("finish", resolve);
+            closing.end();
           });
         };
       }
@@ -11549,7 +11575,7 @@ var CONSTANTS, Constants_default;
 var init_Constants = __esm({
   "../../packages/constants/src/Constants.ts"() {
     CONSTANTS = {
-      cliVersion: "2.0.0",
+      cliVersion: "2.0.1",
       backendVersion: 1,
       backendPort: 5088,
       workerVersion: 2,
@@ -11834,6 +11860,132 @@ var init_SecretManager = __esm({
   }
 });
 
+// ../../packages/common/src/schema/SchemaErrorReporter.ts
+var MAX_REPORTED_ERRORS, MAX_LISTED_VALUES, MAX_QUOTED_LENGTH, VALUE_KEYWORDS, ITEM_NAME_KEYS, SchemaErrorReporterClass, SchemaErrorReporter, SchemaErrorReporter_default;
+var init_SchemaErrorReporter = __esm({
+  "../../packages/common/src/schema/SchemaErrorReporter.ts"() {
+    MAX_REPORTED_ERRORS = 3;
+    MAX_LISTED_VALUES = 30;
+    MAX_QUOTED_LENGTH = 40;
+    VALUE_KEYWORDS = ["type", "pattern", "format", "maxLength", "minLength", "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "multipleOf", "const"];
+    ITEM_NAME_KEYS = ["name", "alias", "key", "title"];
+    SchemaErrorReporterClass = class {
+      constructor() {
+        /**
+         * One human-readable line for a failed validation. `data` is the value that was validated, and is
+         * read to quote the offending value back and to name the resource's own parts.
+         */
+        this.describe = (errors, data) => {
+          const relevant = this._prune(errors ?? []);
+          const shown = relevant.slice(0, MAX_REPORTED_ERRORS).map((x2) => this._explain(x2, data));
+          const hidden = relevant.length - shown.length;
+          return [shown.join("; "), hidden > 0 ? `+${hidden} more` : null].filter(Boolean).join(", ");
+        };
+        /**
+         * Drops the alternatives' complaints from every failed `oneOf`/`anyOf`, because they are one
+         * rejection per alternative rather than one problem. What survives is whatever an alternative
+         * said about a value BELOW the branch point — the alternative that got far enough in to find the
+         * real mistake — and when no alternative got that far, the branch point itself, which then reads
+         * as the list of what was allowed there.
+         */
+        this._prune = (errors) => {
+          const branchPoints = errors.filter((x2) => x2.keyword === "oneOf" || x2.keyword === "anyOf");
+          const kept = branchPoints.reduce((remaining, branch) => this._resolveBranch(remaining, branch), errors);
+          return this._dedupe(kept);
+        };
+        /** Keeps, of everything reported at one branch point, only what says where the mistake actually is. */
+        this._resolveBranch = (kept, branch) => {
+          if (!kept.includes(branch)) return kept;
+          const isAtBranch = (x2) => x2.instancePath === branch.instancePath;
+          const hasDeeper = kept.some((x2) => x2.instancePath.startsWith(branch.instancePath + "/"));
+          return hasDeeper ? kept.filter((x2) => !isAtBranch(x2)) : kept.filter((x2) => !isAtBranch(x2) || x2 === branch);
+        };
+        /** The same complaint from several alternatives is still one complaint. */
+        this._dedupe = (errors) => {
+          const seen = /* @__PURE__ */ new Set();
+          return errors.filter((x2) => {
+            const signature = `${x2.instancePath}|${x2.keyword}|${x2.message}|${JSON.stringify(x2.params ?? {})}`;
+            if (seen.has(signature)) return false;
+            seen.add(signature);
+            return true;
+          });
+        };
+        this._explain = (error, data) => `${this._locate(error.instancePath, data)} ${this._reason(error, data)}`;
+        /**
+         * The JSON pointer to the offending value, followed by the name of the array item it sits in.
+         * "/fields/2" is a position; "(field "created_at")" is what the author called it.
+         */
+        this._locate = (instancePath, data) => {
+          const named = this._nameItem(instancePath, data);
+          return [instancePath || "/", named].filter(Boolean).join(" ");
+        };
+        /** `/fields/2/transform/cast` in a consumer -> `(field "created_at")`, when the item carries a name. */
+        this._nameItem = (instancePath, data) => {
+          const segments = this._segments(instancePath);
+          for (let index = segments.length - 1; index > 0; index--) {
+            if (!/^\d+$/.test(segments[index])) continue;
+            const item = this._valueAt(data, segments.slice(0, index + 1));
+            const nameKey = ITEM_NAME_KEYS.find((key) => typeof item?.[key] === "string");
+            if (!nameKey) continue;
+            return `(${this._singular(segments[index - 1])} "${item[nameKey]}")`;
+          }
+          return null;
+        };
+        /** Reads the way a config author speaks about the collection: `fields` -> `field`. */
+        this._singular = (collection) => collection.endsWith("s") ? collection.slice(0, -1) : collection;
+        /**
+         * What was wrong, in the terms the author wrote. Ajv's own message says what the schema wanted;
+         * these add what was actually there, which is the half that names the typo.
+         */
+        this._reason = (error, data) => {
+          const value = "data" in error ? error.data : this._valueAt(data, this._segments(error.instancePath));
+          if (error.keyword === "enum")
+            return `must be one of: ${this._list(error.params.allowedValues)} (got ${this._quote(value)})`;
+          if (error.keyword === "additionalProperties")
+            return `has unknown property "${error.params.additionalProperty}"`;
+          if (error.keyword === "oneOf" || error.keyword === "anyOf")
+            return this._describeAlternatives(error, value);
+          if (VALUE_KEYWORDS.includes(error.keyword))
+            return `${error.message} (got ${this._quote(value)})`;
+          return error.message;
+        };
+        /**
+         * A branch point no alternative matched, named by what the alternatives ARE — read off the schema
+         * rather than off their rejections, because an alternative says nothing about the property it
+         * requires when that property is present and the rest of it is what failed.
+         */
+        this._describeAlternatives = (error, value) => {
+          const shapes = (error.schema ?? []).map((x2) => this._shape(x2)).filter(Boolean);
+          if (shapes.length === 0) return error.message;
+          if (value && typeof value === "object" && !Array.isArray(value))
+            return `must have exactly one of: ${this._list(shapes)} (has: ${this._list(Object.keys(value))})`;
+          return `must be one of: ${this._list(shapes)} (got ${this._quote(value)})`;
+        };
+        /** How one alternative would be described to whoever has to write it: a property, or a type. */
+        this._shape = (alternative) => {
+          if (alternative?.required?.length > 0) return alternative.required.join(" + ");
+          if (alternative?.pattern) return `${alternative.type ?? "string"} matching "${alternative.pattern}"`;
+          return alternative?.type ?? null;
+        };
+        /** `/fields/2/transform` -> ['fields', '2', 'transform'], undoing the pointer's escapes. */
+        this._segments = (instancePath) => (instancePath || "").split("/").slice(1).map((x2) => x2.replace(/~1/g, "/").replace(/~0/g, "~"));
+        this._valueAt = (data, segments) => segments.reduce((current, segment) => current?.[segment], data);
+        this._list = (values) => {
+          const shown = (values ?? []).slice(0, MAX_LISTED_VALUES).join(", ");
+          const hidden = (values ?? []).length - MAX_LISTED_VALUES;
+          return hidden > 0 ? `${shown}, +${hidden} more` : shown;
+        };
+        this._quote = (value) => {
+          const text = value === void 0 ? "nothing" : JSON.stringify(value);
+          return text.length > MAX_QUOTED_LENGTH ? `${text.slice(0, MAX_QUOTED_LENGTH)}\u2026` : text;
+        };
+      }
+    };
+    SchemaErrorReporter = new SchemaErrorReporterClass();
+    SchemaErrorReporter_default = SchemaErrorReporter;
+  }
+});
+
 // ../../packages/common/src/schema/SchemaValidator.ts
 var import_ajv, import_ajv_formats, import_fs3, import_path3, SchemaValidatorClass, SchemaValidator, SchemaValidator_default;
 var init_SchemaValidator = __esm({
@@ -11843,11 +11995,12 @@ var init_SchemaValidator = __esm({
     import_fs3 = __toESM(require("fs"), 1);
     import_path3 = __toESM(require("path"), 1);
     init_src();
+    init_SchemaErrorReporter();
     SchemaValidatorClass = class {
       constructor() {
         this.addSchema = (schema, schemaKey) => {
           const isValid = this.ajv.validateSchema(schema);
-          Affirm_default(isValid, `Invalid JSON Schema: ${JSON.stringify(this.ajv.errors)}`);
+          Affirm_default(isValid, `Invalid JSON Schema: ${SchemaErrorReporter_default.describe(this.ajv.errors, schema)}`);
           this.ajv.addSchema(schema, schemaKey);
         };
         this.getSchema = (schemaKey) => {
@@ -11901,7 +12054,10 @@ var init_SchemaValidator = __esm({
           strictSchema: false,
           // Code values may be string | number | boolean, so schemas legitimately use union types.
           allowUnionTypes: true,
-          validateFormats: true
+          validateFormats: true,
+          // Carries the failing schema and value on each error, which is what lets
+          // SchemaErrorReporter say what was allowed there and what was found instead.
+          verbose: true
         });
         (0, import_ajv_formats.default)(this.ajv);
         this.loadInternalSchema();
@@ -13231,7 +13387,7 @@ ${errors.map((x2) => `  - ${x2}`).join("\n")}`);
 });
 
 // ../../packages/common/src/Environment.ts
-var import_fs4, import_crypto, import_adm_zip, import_path4, import_client_s3, RESOURCE_SCHEMA_KEYS, INVALID_RESOURCE_MESSAGES, MAX_REPORTED_SCHEMA_ERRORS, EnvironmentClass, Environment, Environment_default;
+var import_fs4, import_crypto, import_adm_zip, import_path4, import_client_s3, RESOURCE_SCHEMA_KEYS, INVALID_RESOURCE_MESSAGES, EnvironmentClass, Environment, Environment_default;
 var init_Environment = __esm({
   "../../packages/common/src/Environment.ts"() {
     import_fs4 = __toESM(require("fs"), 1);
@@ -13240,6 +13396,7 @@ var init_Environment = __esm({
     init_src();
     import_path4 = __toESM(require("path"), 1);
     init_SchemaValidator();
+    init_SchemaErrorReporter();
     init_Validator();
     init_ProducerExpander();
     init_src3();
@@ -13259,7 +13416,6 @@ var init_Environment = __esm({
       codeSet: "Invalid code set configuration",
       schema: "Invalid schema configuration"
     };
-    MAX_REPORTED_SCHEMA_ERRORS = 3;
     EnvironmentClass = class {
       constructor() {
         this._env = null;
@@ -13432,21 +13588,21 @@ var init_Environment = __esm({
           schemas.forEach((schema) => this._registerSchema(schema, problems));
           const sourceFiles = read(projectConfig.sources);
           const sources = sourceFiles.map((x2) => x2.config);
-          sources.forEach((source) => this._recordSchemaProblem(source, "source", problems));
+          sourceFiles.forEach((source) => this._recordSchemaProblem(source, "source", problems));
           this._checkApiQueueSource(projectConfig, sources, problems);
           const producerFiles = read(projectConfig.producers);
           const producers = producerFiles.map((x2) => x2.config);
-          producers.forEach((producer) => {
-            const schemaProblem = this._schemaProblem(producer, "producer");
+          producerFiles.forEach(({ config: producer, file }) => {
+            const schemaProblem = this._schemaProblem(producer, "producer", file);
             this._expandProducer(producer, problems);
             if (schemaProblem) problems.push(schemaProblem);
           });
           const consumerFiles = read(projectConfig.consumers);
           const consumers = consumerFiles.map((x2) => x2.config);
-          consumers.forEach((consumer) => this._recordSchemaProblem(consumer, "consumer", problems));
+          consumerFiles.forEach((consumer) => this._recordSchemaProblem(consumer, "consumer", problems));
           const codeSetFiles = read(projectConfig.codeSets ?? []);
           const codeSets = codeSetFiles.map((x2) => x2.config);
-          codeSets.forEach((codeSet) => this._recordSchemaProblem(codeSet, "codeSet", problems));
+          codeSetFiles.forEach((codeSet) => this._recordSchemaProblem(codeSet, "codeSet", problems));
           this._checkCodeSetNames(codeSets, problems);
           return {
             env: { settings: this._readSettings(projectConfig), sources, producers, consumers, schemas, codeSets },
@@ -13526,7 +13682,7 @@ var init_Environment = __esm({
               severity: "error",
               scope: "project",
               file: projectPath,
-              message: `Invalid project configuration (${this._describeSchemaErrors(validation.errors)})`
+              message: `Invalid project configuration - ${SchemaErrorReporter_default.describe(validation.errors, projectConfig)}`
             });
           }
           return projectConfig;
@@ -13582,8 +13738,8 @@ var init_Environment = __esm({
           }
           return results;
         };
-        this._recordSchemaProblem = (resource, kind, problems) => {
-          const problem = this._schemaProblem(resource, kind);
+        this._recordSchemaProblem = (read, kind, problems) => {
+          const problem = this._schemaProblem(read.config, kind, read.file);
           if (problem) problems.push(problem);
         };
         /**
@@ -13594,15 +13750,17 @@ var init_Environment = __esm({
          * Returns the problem instead of recording it, because a producer's is reported after its
          * expansion has had its say (see `inspect`).
          */
-        this._schemaProblem = (resource, kind) => {
+        this._schemaProblem = (resource, kind, file) => {
           const name = resource?.name;
           const validation = SchemaValidator_default.validate(RESOURCE_SCHEMA_KEYS[kind], resource);
           if (validation.isValid) return null;
+          const where = [name, file ? `(${file})` : null].filter(Boolean).join(" ");
           return {
             severity: "error",
             scope: "resource",
+            file,
             resource: { kind, name },
-            message: `${INVALID_RESOURCE_MESSAGES[kind]}: ${name} (${this._describeSchemaErrors(validation.errors)})`
+            message: `${INVALID_RESOURCE_MESSAGES[kind]}: ${where} - ${SchemaErrorReporter_default.describe(validation.errors, resource)}`
           };
         };
         this._registerSchema = (schema, problems) => {
@@ -13679,12 +13837,6 @@ var init_Environment = __esm({
         /** Kept verbatim from the batch loader this replaced, so `load()` still throws the same string. */
         this._loadFailureMessage = (remoraPath, configPath, error) => `Error loading from ${import_path4.default.resolve(remoraPath)} configuration from ${configPath}: ${this._messageOf(error)}`;
         this._messageOf = (error) => error instanceof Error ? error.message : String(error);
-        /** The first few Ajv complaints, so "Invalid producer configuration: p_x" says what is wrong with it. */
-        this._describeSchemaErrors = (errors) => {
-          const shown = (errors ?? []).slice(0, MAX_REPORTED_SCHEMA_ERRORS).map((item) => `${item.instancePath || "/"} ${item.message}`);
-          const hidden = (errors ?? []).length - shown.length;
-          return [shown.join("; "), hidden > 0 ? `+${hidden} more` : null].filter(Boolean).join(", ");
-        };
         /**
          * The loaded environment, flattened for transport to a worker thread.
          *
@@ -52262,8 +52414,9 @@ var init_ExecutorOrchestrator = __esm({
          *
          * `maxWorkers` must come from getParallelWorkerCount(), never from the amount of work: that count
          * is memory-aware and is what stops a low-end machine oversubscribing itself. Surplus work queues
-         * instead of spawning threads. `census` counts threads as workerpool creates them, which is the
-         * only way to know the peak — they spawn lazily, on first exec.
+         * instead of spawning threads. `census.created` counts every thread workerpool spawns — they
+         * spawn lazily, on first exec, and a thread that dies mid-run is replaced by another, so this is
+         * a count of spawns and NOT a concurrency figure (see IWorkerCensus).
          *
          * `workerData` carries the loaded environment, so it is cloned once per THREAD at spawn rather
          * than read and validated from disk once per chunk. Only run-constant data belongs there; per-task
@@ -52280,12 +52433,8 @@ var init_ExecutorOrchestrator = __esm({
               }
             },
             onCreateWorker: () => {
-              census.alive++;
-              census.peak = Math.max(census.peak, census.alive);
+              census.created++;
               return void 0;
-            },
-            onTerminateWorker: () => {
-              census.alive--;
             }
           };
           const workerPath = OrchestratorHelper_default.getPhysicalWorkerPath();
@@ -52304,7 +52453,7 @@ var init_ExecutorOrchestrator = __esm({
           const { usageId } = UsageManager_default.startUsage(consumer, details);
           const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
           const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
-          const census = { alive: 0, peak: 0 };
+          const census = { created: 0, peak: 0 };
           try {
             const start = performance.now();
             const executorResults = [];
@@ -52543,6 +52692,7 @@ var init_ExecutorOrchestrator = __esm({
           const context = {
             consumer: request.consumer,
             scope,
+            census,
             options: request.options,
             lookups: request.lookups,
             progress: request.progress,
@@ -52557,8 +52707,12 @@ var init_ExecutorOrchestrator = __esm({
             phase = performance.now();
             const resultsByFile = await Promise.all(pendingByFile.map((file) => Promise.all(file)));
             tracker.measure("worker-wait", performance.now() - phase);
+            census.peak = Math.max(census.peak, pool.stats().totalWorkers);
             Affirm_default(census.peak <= poolSize, `The worker pool ran ${census.peak} thread(s) at once but was sized ${poolSize}`);
             const results = resultsByFile.flat();
+            const failures = results.filter((x2) => !Algo_default.hasVal(x2)).length;
+            if (failures > 0 || census.created > poolSize)
+              Logger_default.warn(`[${scope.id}] ${failures} of ${chunkCount} chunk(s) failed and ${census.created} thread(s) were spawned for a pool of ${poolSize} \u2014 a thread that dies is replaced, so the chunk error(s) logged above are the cause`);
             this.recordWorkerBoot(results, context.requestedAtByWorker, tracker);
             for (const [index, fileResults] of resultsByFile.entries())
               this._assignChunkRowMetadata(workFiles[index].chunks, fileResults);
@@ -52589,7 +52743,12 @@ var init_ExecutorOrchestrator = __esm({
           context.requestedAtByWorker.set(workerId, performance.timeOrigin + performance.now());
           Logger_default.log(`[${scope.id}] Queued worker ${workerId} for ${file.label} \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
           return pool.exec("executor", [workerData], {
-            on: (payload) => this.onWorkAdvanced(payload, workerId, context)
+            // Progress packets are the run's only regular tick on the main thread, which makes them
+            // the cheapest place to sample how many threads are actually alive at once.
+            on: (payload) => {
+              context.census.peak = Math.max(context.census.peak, pool.stats().totalWorkers);
+              this.onWorkAdvanced(payload, workerId, context);
+            }
           }).catch((error) => {
             Logger_default.error(error);
             return null;
@@ -53393,11 +53552,11 @@ var runExecutorTask = async (workerData) => {
       rssMB: Math.round(memory.rss / (1024 * 1024)),
       heapUsedMB: Math.round(memory.heapUsed / (1024 * 1024))
     };
-    await Logger_default.close();
+    await Logger_default.flush();
     return res;
   } catch (error) {
     Logger_default.error(error);
-    await Logger_default.close();
+    await Logger_default.flush();
     throw error;
   }
 };