@dudousxd/nestjs-catalog 0.21.0 → 0.23.0

package/dist/transform-runner.js

@@ -12,11 +12,15 @@ var SubprocessTransformRunner_1;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SubprocessTransformRunner = void 0;
 const node_child_process_1 = require("node:child_process");
+const node_crypto_1 = require("node:crypto");
 const node_fs_1 = require("node:fs");
+const promises_1 = require("node:fs/promises");
 const node_os_1 = require("node:os");
 const node_path_1 = require("node:path");
+const node_url_1 = require("node:url");
 const common_1 = require("@nestjs/common");
 const catalog_pipeline_1 = require("./catalog.pipeline");
+const transform_shape_1 = require("./transform-shape");
 const DEFAULT_TIMEOUT_MS = 30_000;
 const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
 /**
@@ -127,7 +131,12 @@ const REPORTED_PACKAGES = ['pandas', 'numpy', 'pyarrow', 'requests'];
  * - it runs in a working directory of this runner's choosing but on the host's
  *   filesystem, so a service account token under
  *   `/var/run/secrets/kubernetes.io/serviceaccount/` is an absolute path away;
- * - it can open sockets, as whatever user the service runs as.
+ * - it can open sockets, as whatever user the service runs as;
+ * - a module-shaped transform is written to a file in that temporary directory
+ *   for the length of the run, so its own source is briefly on disk. That is
+ *   not a new exposure — the code is the thing running, and it can read itself
+ *   from anywhere — but it is a fact worth having written down next to the
+ *   others rather than discovered in a directory listing.
  *
  * So the allowlist is a guard rail against the accident, and the reachability of
  * everything it names is a property of the process boundary, not a leak to be
@@ -223,24 +232,40 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
         if (!interpreter) {
             throw new Error('No python3 on PATH, so python transforms cannot run here. Use javascript or typescript, or install python in the image.');
         }
-        const script = python ? pythonHarness(transform.code) : javascriptHarness(transform.code);
-        // `module-typescript` is Node's own stripping — types are erased, never
-        // checked. A transform with a wrong type still runs; the editor's try pane
-        // is what catches it, not the compiler.
-        const args = python
-            ? ['-c', script]
-            : [
-                '--input-type',
-                transform.language === 'typescript' ? 'module-typescript' : 'module',
-                '-e',
-                script,
-            ];
+        // Python is not asked: its harness writes the `def`, so a Python transform
+        // never states a signature and has nothing to detect. See
+        // {@link pythonHarness}.
+        const shape = python ? 'body' : (0, transform_shape_1.transformShape)(transform.code);
+        // Written to disk only for the module shape, and only for the length of the
+        // run. A module has to be *imported* to be a module — its `export default`
+        // creates no binding this harness could name, and rewriting the keyword
+        // into an assignment would be surgery on somebody's source. A file also
+        // gives Node the extension it needs to strip TypeScript, and gives the
+        // author stack frames with real line numbers instead of `[eval]`.
+        const modulePath = shape === 'module'
+            ? (0, node_path_1.join)((0, node_os_1.tmpdir)(), `catalog-transform-${(0, node_crypto_1.randomUUID)()}.${transform.language === 'typescript' ? 'mts' : 'mjs'}`)
+            : undefined;
+        try {
+            if (modulePath)
+                await (0, promises_1.writeFile)(modulePath, transform.code, 'utf8');
+            const args = interpreterArgs(transform, modulePath);
+            return await this.execute(interpreter, args, records, context, timeoutMs, shape, started);
+        }
+        finally {
+            // Unlinked whether the run returned, threw, or was killed on the timeout —
+            // the parent settles in all three, so nothing is left in `tmpdir` for an
+            // operator to find later and wonder about.
+            if (modulePath)
+                await (0, promises_1.rm)(modulePath, { force: true });
+        }
+    }
+    async execute(interpreter, args, records, context, timeoutMs, shape, started) {
         // An envelope rather than the bare array stdin used to carry. The context
         // travels beside the records rather than in the child's `env`, and that is
         // the deliberate half of it: the child's own environment stays
         // `{PATH, NODE_ENV}`, so nothing about what a transform may read changes by
         // accident when somebody edits the spawn options later.
-        const { stdout, stderr } = await this.spawn(interpreter, args, JSON.stringify({ records, context }), timeoutMs);
+        const { stdout, stderr } = await this.spawn(interpreter, args, JSON.stringify({ records, context }), timeoutMs, shape);
         let parsed;
         try {
             // The harness prints exactly one JSON line last; anything the code wrote
@@ -262,7 +287,7 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
             elapsedMs: Date.now() - started,
         };
     }
-    spawn(command, args, input, timeoutMs) {
+    spawn(command, args, input, timeoutMs, shape = 'body') {
         return new Promise((resolve, reject) => {
             const child = (0, node_child_process_1.spawn)(command, args, {
                 // An empty environment, not the parent's. A transform has no business
@@ -321,7 +346,11 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
                 settled = true;
                 clearTimeout(timer);
                 if (code !== 0 && stdout.trim().length === 0) {
-                    reject(new Error(`The transform exited with code ${code}. ${stderr.slice(0, 500)}`));
+                    // The shape hint rides along here specifically: a body whose author
+                    // meant it as a module fails before the harness's own try/catch is
+                    // ever entered, so this branch is the only place that error can be
+                    // annotated. See {@link transformShapeHint}.
+                    reject(new Error(`The transform exited with code ${code}. ${stderr.slice(0, 500)}${(0, transform_shape_1.transformShapeHint)(shape, stderr)}`));
                     return;
                 }
                 resolve({ stdout, stderr });
@@ -369,6 +398,27 @@ exports.SubprocessTransformRunner = SubprocessTransformRunner = SubprocessTransf
     (0, common_1.Injectable)(),
     __metadata("design:paramtypes", [Object])
 ], SubprocessTransformRunner);
+/**
+ * What the interpreter is invoked with, and which harness it is handed.
+ *
+ * `module-typescript` is Node's own stripping — types are erased, never
+ * checked. A transform with a wrong type still runs; the editor's try pane is
+ * what catches it, not the compiler.
+ *
+ * The module shape needs none of that flag here: the harness itself is plain
+ * JavaScript, and the author's `.mts` file is stripped on import by its
+ * extension. That confines the stripper to the code that asked for it, rather
+ * than running this file's own generated source through it as well.
+ */
+function interpreterArgs(transform, modulePath) {
+    if (transform.language === 'python')
+        return ['-c', pythonHarness(transform.code)];
+    const script = modulePath
+        ? javascriptModuleHarness((0, node_url_1.pathToFileURL)(modulePath).href)
+        : javascriptHarness(transform.code);
+    const inputType = transform.language === 'typescript' && !modulePath ? 'module-typescript' : 'module';
+    return ['--input-type', inputType, '-e', script];
+}
 /**
  * The context for a run that has none: a spec, or a host driving the runner by
  * hand.
@@ -437,7 +487,15 @@ function withFinalLogs(error, logs) {
     return `${error}\n${heading}\n${tail.map((line) => `  ${line}`).join('\n')}`;
 }
 /**
- * The JavaScript and TypeScript harness.
+ * The JavaScript and TypeScript harness for the **bare-body** shape: the
+ * author's code is the inside of a function this string writes.
+ *
+ * Unchanged, and that is the feature. Every transform stored before the module
+ * shape existed is a bare body, and it runs through the identical wrapper with
+ * the identical positional parameters and the identical interpreter flags — the
+ * new shape is a second path beside this one, not a rewrite of it. See
+ * `transform-shape.ts` for the rule that decides which path a given piece of
+ * code takes, and why a stored transform cannot be sent down the wrong one.
  *
  * `console.log` is captured rather than left on stdout so user code cannot
  * corrupt the single JSON line this prints — a transform that logs a `{` would
@@ -462,7 +520,84 @@ function withFinalLogs(error, logs) {
  * gets as far as being serialised.
  */
 function javascriptHarness(code) {
-    return `
+    return `${JAVASCRIPT_PRELUDE}
+try {
+  ${JAVASCRIPT_PAYLOAD}
+  const transform = async (records, context) => { ${code} };
+  const rows = await transform(records, context);
+  process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
+} catch (error) {
+  ${JAVASCRIPT_FAILURE}
+}
+`;
+}
+/**
+ * The harness for the module shape: import the author's module, call what it
+ * exports with one object.
+ *
+ * Everything above the call is the same prelude the bare-body harness uses —
+ * the same six console channels, the same two caps, the same envelope, the same
+ * frozen `context`. A transform's log behaviour changing because of the shape it
+ * happens to be written in would be exactly as surprising as it changing because
+ * of the language, and the constants say why that is not allowed to happen.
+ *
+ * The code arrives as a **file URL**, not as text spliced into this string, and
+ * the difference matters three times over. `export default` binds nothing that
+ * an enclosing scope could name, so the module genuinely has to be imported;
+ * `.mts` is what tells Node to strip the types, so the extension does the job
+ * `--input-type module-typescript` does for a body; and a stack frame reads
+ * `catalog-transform-….mts:3:11` rather than `[eval]`, which is the difference
+ * between a line number and a shrug.
+ *
+ * ## What it accepts, and what it refuses
+ *
+ * `export default`, or a named export called `transform`. Two spellings rather
+ * than one because both are things people write without being told to, and
+ * because both are *real exports* — neither is a guess about a name in scope.
+ *
+ * A module that exports neither is **refused, by name**. The alternative is a
+ * transform that returns no rows and says nothing about why, which downstream
+ * reads as a source that produced nothing — a connector would commit an empty
+ * snapshot over live data on the strength of a missing `default` keyword.
+ *
+ * The import is deliberately not wrapped in a fallback to the body shape. Code
+ * that fails to parse as a module has one honest answer — the parse error, with
+ * the line — and re-running it in the other shape would replace that with a
+ * second, different error about text the author never wrote.
+ */
+function javascriptModuleHarness(moduleUrl) {
+    return `${JAVASCRIPT_PRELUDE}
+try {
+  ${JAVASCRIPT_PAYLOAD}
+  const mod = await import(${JSON.stringify(moduleUrl)});
+  const exported = typeof mod.default === "function"
+    ? mod.default
+    : typeof mod.transform === "function" ? mod.transform : null;
+  if (!exported) {
+    const names = Object.keys(mod).filter((key) => key !== "default");
+    throw new Error(
+      "This transform is a module — it has a top-level \`export\` — so the catalog imported it and " +
+      "looked for a function to call. \`export default\` is " + (("default" in mod) ? typeof mod.default : "missing") +
+      " and there is no exported \`transform\` function." +
+      (names.length > 0 ? " It does export: " + names.join(", ") + "." : "") +
+      " Export the function as \`export default\`, or name it \`transform\`."
+    );
+  }
+  const rows = await exported({ records, context });
+  process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
+} catch (error) {
+  ${JAVASCRIPT_FAILURE}
+}
+`;
+}
+/**
+ * Capture the console, before any of the author's code can reach it.
+ *
+ * Shared verbatim by both JavaScript harnesses rather than copied into each: two
+ * copies of a log cap are two numbers that drift, and the one that drifts is
+ * discovered by a run record nobody can explain.
+ */
+const JAVASCRIPT_PRELUDE = `
 const logs = [];
 let dropped = 0;
 const keep = (line) => {
@@ -483,8 +618,9 @@ const captured = () => dropped === 0
 let input = "";
 process.stdin.setEncoding("utf8");
 for await (const chunk of process.stdin) input += chunk;
-
-try {
+`;
+/** Unpack the envelope. `context` is frozen one level down — see below. */
+const JAVASCRIPT_PAYLOAD = `
   const payload = JSON.parse(input || "{}");
   const records = Array.isArray(payload.records) ? payload.records : [];
   // Frozen, and one level down as well, so that a transform assigning to
@@ -492,20 +628,42 @@ try {
   // then confusing whoever reads the next node's code. Nothing propagates out
   // of this process either way; the freeze buys the honest error, not safety.
   const context = Object.freeze({ ...payload.context, env: Object.freeze({ ...payload.context?.env }) });
-  const transform = async (records, context) => { ${code} };
-  const rows = await transform(records, context);
-  process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
-} catch (error) {
+`;
+/** The one JSON line a failed run prints, logs and all. */
+const JAVASCRIPT_FAILURE = `
   process.stdout.write(JSON.stringify({
     error: error instanceof Error ? \`\${error.name}: \${error.message}\` : String(error),
     logs: captured(),
   }));
-}
 `;
-}
 /**
  * The Python harness. `records` in, a list of dicts out.
  *
+ * ## Why this did not move to the one-object shape
+ *
+ * JavaScript moved because a JavaScript transform *states its own signature* —
+ * the harness wrote `(records, context)` and the author's code depended on both
+ * names being where they were, so a third parameter would have been a change to
+ * text nobody was going to re-read. A Python transform states nothing. This
+ * harness writes `def transform(records, context):` and indents the author's
+ * code into it, so a fourth thing to pass is **one line changed here** and not a
+ * single stored transform touched. Python already has the property the object
+ * shape was introduced to buy.
+ *
+ * Moving it anyway would cost the thing the move was for. `records` and
+ * `context` are names in scope today; a `payload` dict makes them
+ * `payload["records"]` and `payload["context"]`, which is a break in every
+ * Python transform in existence — the exact outcome the JavaScript change was
+ * designed to avoid. Consistency between the two languages is worth something,
+ * but not a migration bought with somebody else's pandas code, and not when the
+ * inconsistency is *because* the two languages start from different places.
+ *
+ * The asymmetry left standing, said out loud: a Python author who writes their
+ * own `def transform(...)` at column 0 gets it indented into a nested
+ * definition, and the outer `transform` returns `None` — no error, no rows. That
+ * is a real footgun and it is older than this change; it is named here so the
+ * next person to open this file knows it is known rather than missed.
+ *
  * A DataFrame is accepted as a return value and converted, because a transform
  * that reaches for pandas will naturally end with one — making it write
  * `.to_dict("records")` would be a papercut on the only path pandas is worth

package/dist/transform-shape.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Which of the two shapes a JavaScript or TypeScript transform is written in,
+ * and the one rule that tells them apart.
+ *
+ * ## The two shapes
+ *
+ * A transform used to be — and, for everything already stored, still is — a
+ * **bare body**: the text between the braces of a function the harness supplies.
+ *
+ * ```js
+ * return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
+ * ```
+ *
+ * That shape works, and its problem is not syntax. The harness supplied
+ * `(records, context)` **positionally**, so the set of things a transform can be
+ * given was fixed by the day the second parameter was added: a third one changes
+ * the meaning of every signature ever written, and there is no version of
+ * "records, context, andNowAlsoThis" that does not make the previous shape a
+ * subset by luck rather than by design. So the supported shape is now a real
+ * function over **one object**:
+ *
+ * ```js
+ * export default function transform({ records, context }) {
+ *   return records.map((r) => ({ mgmtCd: r["Mgmt Cd"] }));
+ * }
+ * ```
+ *
+ * A field can be added to that object without touching a single stored
+ * transform, which is the entire argument for it.
+ *
+ * ## The rule
+ *
+ * **A top-level `export` keyword, and nothing else.** Code that has one is a
+ * module; code that has none is a body.
+ *
+ * That is a discriminator rather than a heuristic, and the reason is worth being
+ * exact about: `export` is a *syntax error* inside a function body. Every
+ * transform stored today runs as a function body today, so no stored transform
+ * can contain a top-level `export` — not "probably does not", cannot. Backward
+ * compatibility here is a property of the language, not of how good the guess is.
+ *
+ * ## What the rule deliberately does not look at
+ *
+ * Not the word `function`. Not a function *declaration* named `transform`
+ * either, which is the tempting second rule and is the one that would break
+ * real code:
+ *
+ * ```js
+ * function transform(r) { return { mgmtCd: r["Mgmt Cd"] }; }
+ * return records.map(transform);
+ * ```
+ *
+ * That is a bare body which declares a local helper it happens to have named
+ * `transform`. A detector that called it the new shape would call the helper
+ * with `{records, context}` — one object where a record was expected — and store
+ * 100,000 rows of `undefined` without erroring once. Silent wrong data is the
+ * worst failure available here, so the detector does not offer an opinion about
+ * names at all.
+ *
+ * ## The scan, and its one known limit
+ *
+ * Strings, template literals (including `${}` nesting), regular-expression
+ * literals and both kinds of comment are skipped, so `// export default` and
+ * `"export"` are not exports. The keyword must then appear at **statement
+ * position** — start of input, or after `;`, `}`, or a newline — at brace,
+ * paren and bracket depth zero.
+ *
+ * Regular-expression literals are found by the usual rule (a `/` is a regex
+ * unless what precedes it could end a value), which is the one place a scanner
+ * without a full parser can be wrong. Its consequence is bounded on purpose: a
+ * misread makes this return `false` for a module, the code is run as a body, and
+ * the author gets `SyntaxError: Unexpected token 'export'` — which
+ * {@link transformShapeHint} turns into a sentence naming this exact rule. A
+ * wrong answer here produces a reported error, never a silently different run.
+ */
+/** What shape a transform's code is in. */
+export type TransformShape = 'module' | 'body';
+/**
+ * Does this code declare an ES module — and so ask to be called as a function
+ * over one object?
+ *
+ * See the module docblock for the rule and why it is the rule. Python is not
+ * asked this question: its harness writes the `def` itself, so a Python
+ * transform never states a signature and never had the problem this detector
+ * exists to solve.
+ */
+export declare function transformDeclaresModule(code: string): boolean;
+/** {@link transformDeclaresModule}, as the shape it names. */
+export declare function transformShape(code: string): TransformShape;
+/**
+ * The sentence to add when a body-shaped run died on the one syntax error that
+ * means the detector and the author disagreed.
+ *
+ * The rule above is a scan, not a parser, and its documented limit is that a
+ * regular-expression literal read as a division can hide a real `export`. The
+ * author then sees `Unexpected token 'export'` from code they believe is a
+ * perfectly good module, and has no way to know that a *rule they have never
+ * read* is what decided otherwise. Naming the rule in the error is the whole
+ * difference between a two-minute fix and an afternoon.
+ *
+ * Deliberately not a fallback re-run in the other shape. Running code twice
+ * because the first attempt failed is guessing with extra steps: the second
+ * attempt would report a different error for the same text, and neither error
+ * would be trustworthy.
+ */
+export declare function transformShapeHint(shape: TransformShape, stderr: string): string;