@dudousxd/nestjs-catalog 0.22.0 → 0.24.0

package/dist/transform-runner.js

@@ -23,6 +23,51 @@ const catalog_pipeline_1 = require("./catalog.pipeline");
 const transform_shape_1 = require("./transform-shape");
 const DEFAULT_TIMEOUT_MS = 30_000;
 const MAX_OUTPUT_BYTES = 32 * 1024 * 1024;
+/**
+ * How many records ride on one line of the wire into the child.
+ *
+ * **Framing only.** The child calls the author's function once per record
+ * whatever this is; nothing about the contract, the memory bound or what a
+ * transform can see changes with the number. What changes is how many times
+ * `JSON.parse` is entered, and that turned out to be worth measuring rather than
+ * assuming: `bench/transform-stream.mjs` runs the identical rename at one record
+ * per line and at five hundred, and over the real 102,520-row `af_fleet.csv` the
+ * batched wire is about 7% faster end to end and holds the same memory.
+ *
+ * Five hundred, not five thousand, because this is the one buffer on the write
+ * side that scales with nothing else: it is the *only* thing standing between a
+ * streamed source and a bounded parent, so it is kept at the same order as the
+ * `BATCH_SIZE` the pipeline's own writers use. At five columns that is roughly
+ * 40 KB in flight, which is not a memory decision anybody has to think about.
+ */
+const RECORDS_PER_LINE = 500;
+/**
+ * When the child stops accumulating rows and puts a line on the wire.
+ *
+ * Two triggers rather than one, and each covers a case the other cannot.
+ *
+ * The **byte** trigger is the memory bound: a transform that fans one record out
+ * into thousands of rows would otherwise hold all of them until the record after
+ * it, and the child's heap would be a property of somebody's data.
+ *
+ * The **record** trigger is the liveness bound, and it is why the flush happens
+ * even when there is nothing to say. A per-record transform that drops most of
+ * what it sees — a normaliser that returns `[]` for a blank row — emits no bytes
+ * for a long time, and a parent watching only the byte stream cannot tell that
+ * from a child hung on record 60,000. The empty line carries `at`, so the stall
+ * clock has something to reset on. See {@link SubprocessTransformRunner.runStream}.
+ *
+ * It is also the **attribution window**, which is what fixes its size at 200
+ * rather than at a round thousand. A child killed mid-record cannot report where
+ * it got to, so the finest a failure can be located is "after the last `at` the
+ * child sent, and before the last record the parent sent" — and this number is
+ * the first half of that. Two hundred costs about five hundred extra lines over a
+ * hundred-thousand-record load, which is nothing measurable, and buys a window
+ * five times narrower for the person reading the failure. {@link stallError}
+ * states the window rather than picking a record inside it.
+ */
+const FLUSH_BYTES = 256 * 1024;
+const FLUSH_RECORDS = 200;
 /**
  * How much of the child's stderr is held, and why it is a different number from
  * {@link MAX_OUTPUT_BYTES} with a different consequence.
@@ -62,6 +107,41 @@ const MAX_CAPTURED_STDERR_BYTES = 64 * 1024;
  * would throw on every timeout.
  */
 const KILL_PROCESS_GROUP = process.platform !== 'win32';
+/**
+ * The whole of the isolation, in one object, spawned identically by every path
+ * in this file.
+ *
+ * One object rather than a literal per call site, and that is the point rather
+ * than tidiness. The class docblock spends four paragraphs being exact about
+ * what a transform can and cannot reach, and every sentence of it is a claim
+ * about *these three fields*. A second literal somewhere else in the file would
+ * be a second answer to "what environment does user code get", and the one that
+ * drifted would be discovered by a transform reading something the docblock says
+ * it cannot. The streaming path added below reuses this untouched, which is why
+ * "a streamed transform is isolated exactly as a batched one is" is a property of
+ * the code rather than a promise in a comment.
+ *
+ * - **`env`** is `{PATH, NODE_ENV}` and not the parent's. A transform has no
+ *   business reading the database password, and inheriting env is how it would.
+ *   Read the class docblock before treating this as containment: the same values
+ *   are a `/proc/<ppid>/environ` read away, and this is a guard rail against the
+ *   accidental read rather than a boundary.
+ * - **`cwd`** is not the parent's, which is a running service's directory and
+ *   holds the `.env` the allowlist exists to withhold — a transform whose first
+ *   line is `readFileSync(".env")` was reading the host application's
+ *   configuration by relative path. A temporary directory keeps the file writes a
+ *   transform may legitimately want working while making the one path it can name
+ *   without knowing anything about the deployment uninteresting. Absolute paths
+ *   are unaffected, and cannot be.
+ * - **`detached`** puts the child in its own process group, so a timeout can
+ *   reach a grandchild. See {@link KILL_PROCESS_GROUP} and {@link stop}.
+ */
+const CHILD_PROCESS_OPTIONS = {
+    env: { PATH: process.env.PATH ?? '', NODE_ENV: 'production' },
+    cwd: (0, node_os_1.tmpdir)(),
+    detached: KILL_PROCESS_GROUP,
+    stdio: ['pipe', 'pipe', 'pipe'],
+};
 /**
  * How much of what a transform logged is carried back, on both axes.
  *
@@ -259,6 +339,116 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
                 await (0, promises_1.rm)(modulePath, { force: true });
         }
     }
+    /**
+     * Run a `'record'`-mode transform over a stream, and hand the rows back as a
+     * stream.
+     *
+     * ## What is and is not different from {@link run}
+     *
+     * **The isolation is not different, at all.** Same interpreter, same
+     * {@link CHILD_PROCESS_OPTIONS} — the same `{PATH, NODE_ENV}`, the same
+     * temporary cwd, the same process group — and the context still travels beside
+     * the records on stdin rather than in the environment. It is worth being blunt
+     * about why that is easy to say: the child here is **not longer-lived than the
+     * one `run` spawns.** Both live for exactly one node run and are gone when it
+     * ends. `run` was never a spawn per batch; it was a spawn per node with the
+     * whole dataset in one blob. So there is no new window in which state could
+     * leak between batches or between runs, because there was never a window to
+     * widen. What a transform may retain *within* one run is stated exactly on
+     * {@link javascriptRecordHarness}.
+     *
+     * **The timeout is different, and it has to be.** See {@link stallError}.
+     *
+     * **Failure names a record.** A batch call can only report that the transform
+     * threw; here the child counts what it has consumed and puts that number on
+     * every line, so a stream that dies names the record it died on. What was
+     * already staged is a matter for the caller — for the connector runner it sits
+     * in an uncommitted snapshot, for a workflow node it is overwritten by the next
+     * attempt — and in neither case does a watermark move, because nothing here
+     * reaches a commit.
+     *
+     * ## The one shape that would deadlock, and how it is avoided
+     *
+     * The writer runs as a floating loop and the reader is driven by the consumer
+     * pulling rows. Awaiting the writer before yielding the first row is the one
+     * arrangement that hangs: the child fills its stdout buffer with rows nobody is
+     * draining, stops reading stdin, and this side waits forever for a `drain` that
+     * requires the reader that has not started. Written the way it is, a slow
+     * consumer simply back-pressures the whole chain, which is the point.
+     */
+    async runStream(transform, records, options = {}) {
+        const mode = (0, catalog_pipeline_1.transformMode)(transform);
+        if (mode === 'batch') {
+            throw new Error('This transform is a function over the whole batch, so it cannot be streamed a record at a time — that would call it once per record and return one partial answer per record, which is exactly the silent wrong result TRANSFORM_MODES exists to prevent. Run it through `run`, or set the transform to per-record mode.');
+        }
+        if (mode !== 'record')
+            return (0, catalog_pipeline_1.unreachableTransformMode)(mode, 'SubprocessTransformRunner.runStream');
+        // Asked here as well as at the controller, because a transform row can reach
+        // a runner without ever passing through this build's controller — promoted
+        // from another environment, restored, written by an older version — and the
+        // failure this must never have is the quiet one.
+        const refusal = (0, catalog_pipeline_1.recordModeRefusal)(transform);
+        if (refusal)
+            throw new Error(refusal);
+        const started = Date.now();
+        const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+        // `rowCount` is 0 rather than a guess: a stream does not know how many
+        // records there are, and `contextlessRun` is only reached by a caller that
+        // gave no context at all. A caller with a real run supplies the count it
+        // knows, exactly as it does for a batch.
+        const context = options.context ?? contextlessRun(0);
+        // A record-mode transform is always a module — `recordModeRefusal` has just
+        // refused anything else — so the file is unconditional here where `run` has
+        // to decide. Removed in the generator's `finally` rather than in one here,
+        // because the run is not over when this method returns: it is over when the
+        // rows have been drained, thrown, or abandoned by a `break`.
+        const modulePath = (0, node_path_1.join)((0, node_os_1.tmpdir)(), `catalog-transform-${(0, node_crypto_1.randomUUID)()}.${transform.language === 'typescript' ? 'mts' : 'mjs'}`);
+        await (0, promises_1.writeFile)(modulePath, transform.code, 'utf8');
+        const child = (0, node_child_process_1.spawn)(process.execPath, ['--input-type', 'module', '-e', javascriptRecordHarness((0, node_url_1.pathToFileURL)(modulePath).href)], CHILD_PROCESS_OPTIONS);
+        const pump = new RecordStreamPump(child, timeoutMs);
+        pump.feed(records, context);
+        let summary;
+        const logger = this.logger;
+        async function* rows() {
+            try {
+                for await (const message of pump.messages()) {
+                    if (message.done) {
+                        summary = { ...message.done, elapsedMs: Date.now() - started };
+                        continue;
+                    }
+                    for (const row of message.rows) {
+                        // The same filter `run` applies before it returns, through the same
+                        // predicate: anything that is not a plain object cannot be written
+                        // as a row, and dropping it silently downstream is how a load comes
+                        // out short with nothing to explain it.
+                        if (isRowObject(row))
+                            yield row;
+                    }
+                }
+            }
+            finally {
+                // Every exit: drained, thrown, or a `break` in the consumer's loop. The
+                // last one is why this cannot be a `finally` around the spawn — a
+                // consumer that stops early leaves a child holding a pipe, and the kill
+                // has to happen where the abandonment is observable.
+                pump.close();
+                await (0, promises_1.rm)(modulePath, { force: true }).catch(() => {
+                    // A temporary file that will not unlink is not worth failing a load
+                    // that produced correct rows. It is logged rather than thrown.
+                    logger.warn(`Could not remove the transform's temporary module at ${modulePath}.`);
+                });
+            }
+        }
+        return {
+            rows: rows(),
+            summary: () => {
+                if (!summary) {
+                    throw new Error('The transform stream has not finished, so there is no summary yet. `summary()` is answered only once `rows` is exhausted — a running total asked for early is exactly the number somebody would go on to record as `fetched`.');
+                }
+                return summary;
+            },
+        };
+    }
     async execute(interpreter, args, records, context, timeoutMs, shape, started) {
         // An envelope rather than the bare array stdin used to carry. The context
         // travels beside the records rather than in the child's `env`, and that is
@@ -282,33 +472,14 @@ let SubprocessTransformRunner = SubprocessTransformRunner_1 = class SubprocessTr
             throw new Error('The transform must return an array of rows. Returning anything else would leave the load ambiguous.');
         }
         return {
-            rows: parsed.rows.filter((row) => typeof row === 'object' && row !== null && !Array.isArray(row)),
+            rows: parsed.rows.filter(isRowObject),
             logs,
             elapsedMs: Date.now() - started,
         };
     }
     spawn(command, args, input, timeoutMs, shape = 'body') {
         return new Promise((resolve, reject) => {
-            const child = (0, node_child_process_1.spawn)(command, args, {
-                // An empty environment, not the parent's. A transform has no business
-                // reading the database password, and inheriting env is how it would.
-                // Read the class docblock before treating this as containment: the same
-                // values are a `/proc/<ppid>/environ` read away, and this is a guard
-                // rail against the accidental read rather than a boundary.
-                env: { PATH: process.env.PATH ?? '', NODE_ENV: 'production' },
-                // Not the parent's, which is a running service's directory and holds
-                // the `.env` the allowlist above exists to withhold — a transform whose
-                // first line is `readFileSync(".env")` was reading the host application's
-                // configuration by relative path. A temporary directory keeps the file
-                // writes a transform may legitimately want working while making the one
-                // path it can name without knowing anything about the deployment
-                // uninteresting. Absolute paths are unaffected, and cannot be.
-                cwd: (0, node_os_1.tmpdir)(),
-                // Its own process group, so the timeout below can reach a grandchild.
-                // See {@link KILL_PROCESS_GROUP}.
-                detached: KILL_PROCESS_GROUP,
-                stdio: ['pipe', 'pipe', 'pipe'],
-            });
+            const child = (0, node_child_process_1.spawn)(command, args, CHILD_PROCESS_OPTIONS);
             let stdout = '';
             let stderr = '';
             let settled = false;
@@ -398,6 +569,313 @@ exports.SubprocessTransformRunner = SubprocessTransformRunner = SubprocessTransf
     (0, common_1.Injectable)(),
     __metadata("design:paramtypes", [Object])
 ], SubprocessTransformRunner);
+/**
+ * Whether this is something that can be stored as a row.
+ *
+ * One predicate for both paths rather than the same three clauses written twice.
+ * "A row is a plain object" is a rule the publish side already depends on, and
+ * two copies of it are two places for an array or a `null` to start counting as
+ * a row in one mode and not the other.
+ */
+function isRowObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/**
+ * Both halves of the pipe to a record-mode child: records in, rows out, and the
+ * clock that decides the child has stopped.
+ *
+ * A class rather than closures inside `runStream` because the two halves have to
+ * see each other. The stall clock cannot be armed by the reader alone — whether
+ * the child *owes* an answer is a fact about what the writer has sent — and the
+ * writer cannot arm it either, because whether anybody is *waiting* is a fact
+ * about the reader. Separating them into two functions would mean passing a
+ * mutable cell between them, which is this object with the name taken off.
+ *
+ * ## The stall clock, and why it is not the batch timeout
+ *
+ * `run` bounds total wall clock: the whole payload is written at once, so
+ * "elapsed" is time the child spent working and nothing else. That bound cannot
+ * be carried over unchanged, and this is the one genuine semantic change in the
+ * streaming path, so it is worth stating rather than discovering. A streamed
+ * transform's clock would include **time waiting for the source** — a SQL cursor
+ * paging over ten million rows, an S3 prefix listing — which the batch path
+ * finished before it spawned anything. Thirty seconds of total wall clock would
+ * therefore fail loads that work today, for reasons that have nothing to do with
+ * the transform.
+ *
+ * So what is bounded is a **stall**: `timeoutMs` with the child owing an answer
+ * and nobody hearing one. The three states are kept apart deliberately —
+ *
+ * - the child is hung on a record → the reader is waiting, records are
+ *   outstanding, the clock runs, and at `timeoutMs` the process group is killed.
+ *   This is the failure the timeout exists for and it is caught *sooner* than the
+ *   old bound caught it on a long load;
+ * - the source is slow → the child has answered everything it was given, nothing
+ *   is outstanding, the clock is not running. A load that spends an hour waiting
+ *   on a database is not a stalled transform;
+ * - the consumer is slow → the reader is not waiting on the child at all, so the
+ *   clock is not running. A sink writing to a busy warehouse back-pressures the
+ *   whole chain, which is the design, and must not read as a hang.
+ *
+ * What is given up is a bound on **total** time, and that is deliberate: a
+ * stream's total time is a property of how much data there is, so a total bound
+ * is a bound on dataset size wearing a clock's clothes. The outer bound has not
+ * disappeared — a node runs inside a durable step, and `abandoned-runs.ts` is
+ * what closes a run whose worker went away.
+ *
+ * On the kill, rows already yielded have already been written by whoever was
+ * consuming them. Nothing promotes a watermark or commits a snapshot on this
+ * path, so "staged but not committed" is where they stay — the same place every
+ * other mid-run failure leaves its work.
+ */
+class RecordStreamPump {
+    child;
+    timeoutMs;
+    chunks = [];
+    carry = '';
+    stderr = '';
+    /** How many records the writer has put on the wire. */
+    sent = 0;
+    /** How many the child has said it consumed, from the last line it wrote. */
+    answered = 0;
+    waiting = false;
+    timer;
+    wake;
+    fail;
+    ended = false;
+    failure;
+    /** Set once, and preferred over any later error: the reason we killed it. */
+    stalled = false;
+    constructor(child, timeoutMs) {
+        this.child = child;
+        this.timeoutMs = timeoutMs;
+        child.stdout.setEncoding('utf8');
+        child.stdout.on('data', (chunk) => {
+            this.chunks.push(chunk);
+            this.notify();
+        });
+        child.stderr.on('data', (chunk) => {
+            // Bounded exactly as the batch path bounds it, and for the reason
+            // {@link MAX_CAPTURED_STDERR_BYTES} gives at length: a transform looping on
+            // writes to fd 2 grows the *parent's* heap, and a stream gives it longer to
+            // do so.
+            if (this.stderr.length >= MAX_CAPTURED_STDERR_BYTES)
+                return;
+            this.stderr += chunk.toString().slice(0, MAX_CAPTURED_STDERR_BYTES - this.stderr.length);
+        });
+        child.on('error', (error) => this.abort(error));
+        child.on('close', (code) => {
+            this.ended = true;
+            if (code !== 0 && !this.failure) {
+                this.failure = new Error(`The transform exited with code ${code} after ${this.answered} record(s), before it finished the stream. ${this.stderr.slice(0, 500)}`);
+            }
+            this.notify();
+        });
+    }
+    /**
+     * Write the context, then the records, in lines of
+     * {@link RECORDS_PER_LINE}.
+     *
+     * Floating on purpose — see {@link SubprocessTransformRunner.runStream} on the
+     * deadlock this shape avoids. Its failure is recorded rather than thrown into
+     * nowhere: a source that dies mid-read must surface on the row stream, not as
+     * an unhandled rejection with the run reporting a short but successful load.
+     */
+    feed(records, context) {
+        void (async () => {
+            const stdin = this.child.stdin;
+            await write(stdin, `${JSON.stringify(context)}\n`);
+            let line = [];
+            for await (const record of records) {
+                line.push(record);
+                if (line.length < RECORDS_PER_LINE)
+                    continue;
+                await write(stdin, `${JSON.stringify(line)}\n`);
+                this.sent += line.length;
+                line = [];
+                // The clock may have been idle while the child had nothing outstanding.
+                // Sending work is one of the two events that can start it running.
+                this.rearm();
+            }
+            if (line.length > 0) {
+                await write(stdin, `${JSON.stringify(line)}\n`);
+                this.sent += line.length;
+                this.rearm();
+            }
+            stdin.end();
+        })().catch((error) => {
+            this.abort(error instanceof Error ? error : new Error(String(error)));
+        });
+    }
+    /** Every line the child wrote, in order, until it says it is done or it fails. */
+    async *messages() {
+        for (;;) {
+            const line = await this.nextLine();
+            if (line === undefined) {
+                if (this.failure)
+                    throw this.failure;
+                return;
+            }
+            // `parse` throws on the child's failure line, so that the record number is
+            // in the sentence a consumer sees rather than in a field nobody reads.
+            const message = this.parse(line);
+            this.answered = message.at;
+            yield message;
+            if (message.done)
+                return;
+        }
+    }
+    /** Kill the child and everything it started. Idempotent. */
+    close() {
+        this.clearTimer();
+        if (!this.ended)
+            stop(this.child);
+    }
+    parse(line) {
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            throw new Error(`The transform wrote a line this runner could not read after ${this.answered} record(s). That is a bug in the catalog's harness rather than in the transform. stderr: ${this.stderr.slice(0, 500)}`);
+        }
+        if (parsed.failed) {
+            const at = typeof parsed.failed.at === 'number' ? parsed.failed.at : this.answered + 1;
+            const logs = Array.isArray(parsed.failed.logs) ? parsed.failed.logs.map(String) : [];
+            throw new Error(withFinalLogs(`The transform failed on record ${at}: ${String(parsed.failed.error)}`, logs));
+        }
+        if (isDoneLine(parsed.done)) {
+            return { at: parsed.done.recordsIn, rows: [], done: parsed.done };
+        }
+        return {
+            at: typeof parsed.at === 'number' ? parsed.at : this.answered,
+            rows: Array.isArray(parsed.rows) ? parsed.rows : [],
+        };
+    }
+    /** One complete line, or `undefined` when the child has closed for good. */
+    async nextLine() {
+        for (;;) {
+            const nl = this.carry.indexOf('\n');
+            if (nl !== -1) {
+                const line = this.carry.slice(0, nl);
+                this.carry = this.carry.slice(nl + 1);
+                if (line.length > 0)
+                    return line;
+                continue;
+            }
+            if (this.chunks.length > 0) {
+                this.carry += this.chunks.shift();
+                if (this.carry.length > MAX_OUTPUT_BYTES) {
+                    this.abort(new Error(`The transform wrote more than ${MAX_OUTPUT_BYTES} bytes without a line break, at record ${this.answered}. One record cannot fan out to more rows than the runner can hold.`));
+                }
+                continue;
+            }
+            if (this.ended)
+                return undefined;
+            if (this.failure)
+                throw this.failure;
+            await this.awaitChunk();
+        }
+    }
+    /**
+     * Wait for the child to say something, with the clock running only if it owes
+     * us an answer.
+     *
+     * This method *is* the arming rule: the clock runs while, and only while,
+     * control is inside it and {@link owes} holds.
+     */
+    awaitChunk() {
+        return new Promise((resolve, reject) => {
+            this.waiting = true;
+            this.wake = () => {
+                this.waiting = false;
+                this.wake = undefined;
+                this.fail = undefined;
+                this.clearTimer();
+                resolve();
+            };
+            this.fail = (error) => {
+                this.waiting = false;
+                this.wake = undefined;
+                this.fail = undefined;
+                this.clearTimer();
+                reject(error);
+            };
+            this.rearm();
+        });
+    }
+    /** Any output at all is progress: the clock stops, and whoever waits wakes. */
+    notify() {
+        this.clearTimer();
+        this.wake?.();
+    }
+    owes() {
+        return this.sent > this.answered && !this.ended;
+    }
+    rearm() {
+        this.clearTimer();
+        if (!this.waiting || !this.owes())
+            return;
+        this.timer = setTimeout(() => {
+            this.stalled = true;
+            this.abort(stallError(this.timeoutMs, this.answered, this.sent));
+        }, this.timeoutMs);
+    }
+    clearTimer() {
+        if (this.timer === undefined)
+            return;
+        clearTimeout(this.timer);
+        this.timer = undefined;
+    }
+    abort(error) {
+        // The first failure wins, except that a stall always does: a killed child
+        // then closes with a signal, and reporting "exited with code null" instead of
+        // "made no progress for 30000ms" would name the symptom rather than the
+        // cause.
+        if (!this.failure || this.stalled)
+            this.failure = error;
+        this.close();
+        this.ended = true;
+        if (this.fail)
+            this.fail(this.failure);
+    }
+}
+/** Whether the child's last line is the summary, with the fields it promises. */
+function isDoneLine(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const done = value;
+    return typeof done.recordsIn === 'number' && typeof done.rowsOut === 'number';
+}
+/**
+ * The sentence a stalled stream fails with.
+ *
+ * It states a **window** rather than a record, and the difference is the whole
+ * honesty of the message. A child killed with SIGKILL cannot report where it
+ * got to, so what the parent knows is the last `at` the child sent and the
+ * number of records it has been given since; the record that hung is somewhere
+ * between them. Naming `answered + 1` would read as a precise answer and would
+ * be wrong by up to {@link FLUSH_RECORDS} — and wrong in the direction that
+ * sends somebody to look at a row that transformed perfectly well.
+ *
+ * The window collapses to one record when it can, because "somewhere in records
+ * 618 to 618" is a sentence nobody should have to read.
+ *
+ * It also says what became of the rows, which is the second question anybody
+ * asks and the one #96 established the rule for: rows already yielded were
+ * already written by whoever consumed them, and they sit in an uncommitted
+ * snapshot because nothing on this path reaches a commit or moves a watermark.
+ */
+function stallError(timeoutMs, answered, sent) {
+    const where = sent <= answered + 1 ? `on record ${sent}` : `somewhere in records ${answered + 1} to ${sent}`;
+    return new Error(`The transform stopped making progress for ${timeoutMs}ms and was stopped. It had finished ${answered} record(s) and been given ${sent}, so it stopped ${where}. Rows it had already produced were passed on and are in an uncommitted snapshot; nothing was committed and no watermark moved.`);
+}
+/** Write, and wait for the pipe to drain if it asked us to. */
+function write(stream, text) {
+    if (stream.write(text))
+        return undefined;
+    return new Promise((resolve) => stream.once('drain', resolve));
+}
 /**
  * What the interpreter is invoked with, and which harness it is handed.
  *
@@ -569,6 +1047,168 @@ function javascriptModuleHarness(moduleUrl) {
     return `${JAVASCRIPT_PRELUDE}
 try {
   ${JAVASCRIPT_PAYLOAD}
+  ${javascriptExportedFunction(moduleUrl)}
+  const rows = await exported({ records, context });
+  process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
+} catch (error) {
+  ${JAVASCRIPT_FAILURE}
+}
+`;
+}
+/**
+ * The harness for `'record'` mode: import the author's module, call it once per
+ * record, and put the rows on the wire as they are produced.
+ *
+ * Everything about the *code* is the same as {@link javascriptModuleHarness} —
+ * the same six console channels, the same two caps, the same frozen `context`,
+ * the same two accepted export spellings, the same refusal by name when there is
+ * neither. What differs is only the argument and the transport, and keeping the
+ * rest identical is what makes "the same transform, called differently" true
+ * rather than approximately true.
+ *
+ * ## What a per-record transform can and cannot retain
+ *
+ * The question is worth answering exactly, because "it cannot see the batch" is
+ * a claim about a contract and people will build on it.
+ *
+ * **It cannot see other records.** The function is handed one `record` and there
+ * is no array anywhere in scope. That is enforced by the shape of the call, not
+ * by convention.
+ *
+ * **It cannot emit at the end.** There is no finish hook, no flush callback, no
+ * second export this harness looks for. A transform that accumulates into a
+ * module-scope `Map` intending to return the totals afterwards has nowhere to
+ * return them *to*, so it emits nothing and the node's row count is zero — which
+ * is loud, immediate and visible on the run, rather than a partial aggregate
+ * committed as though it were the answer. This is the enforcement that matters,
+ * because it is the one that turns "you have used the wrong mode" from silently
+ * wrong data into an obvious failure.
+ *
+ * **It cannot retain anything past the node.** The process is spawned for one
+ * node run and killed at the end of it, so nothing carries to the next run, the
+ * next node, or another connector sharing the same transform. That is enforced
+ * by process lifetime and is the identical guarantee the whole-batch path has
+ * always had — the child was never reused there either.
+ *
+ * **It can retain state in module scope for the length of one run**, and no
+ * honest harness can stop it: a module may close over a `let`, and preventing
+ * that means either re-importing per record (which would cost more than the
+ * subprocess this change exists to make cheaper) or forbidding modules, which is
+ * the only shape the mode accepts. So it is stated rather than pretended
+ * otherwise. What that buys an author is a memo table or a compiled regex, which
+ * is legitimate and useful; what it does not buy is an aggregate, for the reason
+ * directly above. Records arrive in source order, so such a transform is
+ * deterministic — it is simply not what the mode is for.
+ *
+ * ## The wire, and why the child talks in lines
+ *
+ * Records arrive as JSON arrays, one per line, {@link RECORDS_PER_LINE} at a
+ * time; the first line is the context. Rows leave as `{"at":N,"rows":[…]}`
+ * lines, then one `{"done":…}` or `{"failed":…}`. `at` is how many records have
+ * been *consumed*, and it is on every line for two separate consumers: it is
+ * what a failure names so a stack trace can be tied to a row, and it is what the
+ * parent's stall clock resets on. See {@link FLUSH_BYTES} and
+ * {@link FLUSH_RECORDS} for why a line is written even when it carries no rows.
+ *
+ * `process.stdout.write` is awaited through its `drain`, which is the child's
+ * half of the back-pressure: a parent that stops reading stops this loop, which
+ * stops it reading stdin, which stops the source. Without it the child would
+ * happily buffer the whole output in its own heap while congratulating itself on
+ * streaming.
+ */
+function javascriptRecordHarness(moduleUrl) {
+    return `${JAVASCRIPT_CONSOLE}
+let at = 0;
+let rowsOut = 0;
+let pending = [];
+let pendingBytes = 0;
+let sinceFlush = 0;
+const send = async (line) => {
+  if (!process.stdout.write(line)) await new Promise((r) => process.stdout.once("drain", r));
+};
+const flush = async () => {
+  const line = '{"at":' + at + ',"rows":[' + pending.join(",") + ']}\\n';
+  pending = [];
+  pendingBytes = 0;
+  sinceFlush = 0;
+  await send(line);
+};
+// One rule for four return shapes: an object is a row, an array is those rows,
+// and null or undefined is none. See CatalogRecordTransformFunction, which is
+// where the argument for that rule lives.
+const collect = (value) => {
+  if (value === null || value === undefined) return;
+  const rows = Array.isArray(value) ? value : [value];
+  for (const row of rows) {
+    const json = JSON.stringify(row);
+    pending.push(json);
+    pendingBytes += json.length;
+    rowsOut += 1;
+  }
+};
+try {
+  ${javascriptExportedFunction(moduleUrl)}
+  let context = null;
+  let carry = "";
+  process.stdin.setEncoding("utf8");
+  for await (const chunk of process.stdin) {
+    carry += chunk;
+    let nl = carry.indexOf("\\n");
+    while (nl !== -1) {
+      const line = carry.slice(0, nl);
+      carry = carry.slice(nl + 1);
+      nl = carry.indexOf("\\n");
+      if (line.length === 0) continue;
+      if (context === null) {
+        const sent = JSON.parse(line);
+        // Frozen one level down, exactly as the batch harnesses freeze it, so
+        // assigning to context.env.TOKEN fails loudly here rather than appearing
+        // to work. Frozen ONCE, outside the record loop: it is the same object
+        // for every record by construction, so there is nothing a transform
+        // could write on it that would reach the next one.
+        context = Object.freeze({ ...sent, env: Object.freeze({ ...sent.env }) });
+        continue;
+      }
+      for (const record of JSON.parse(line)) {
+        at += 1;
+        sinceFlush += 1;
+        collect(await exported({ record, context }));
+        if (pendingBytes >= ${FLUSH_BYTES} || sinceFlush >= ${FLUSH_RECORDS}) await flush();
+      }
+    }
+  }
+  await flush();
+  await send(JSON.stringify({ done: { recordsIn: at, rowsOut, logs: captured() } }) + "\\n");
+} catch (error) {
+  // Whatever was pending is deliberately NOT flushed. The node has failed, so
+  // nothing downstream will read this stage; sending the rows anyway would put
+  // work on the wire that exists only to be discarded, and would make the last
+  // "at" a reader sees larger than the record that actually threw.
+  await send(JSON.stringify({
+    failed: {
+      at,
+      error: error instanceof Error ? \`\${error.name}: \${error.message}\` : String(error),
+      logs: captured(),
+    },
+  }) + "\\n");
+}
+`;
+}
+/**
+ * Find the function the author exported, or refuse by name.
+ *
+ * Shared verbatim by the module harness and the record harness rather than
+ * copied into each, for the reason {@link JAVASCRIPT_CONSOLE} is shared: two
+ * copies of "which exports are accepted" are two answers, and the one that
+ * drifts is discovered by an author whose perfectly good `export default` works
+ * in one mode and not the other.
+ *
+ * The sentence it throws deliberately does not name a mode. Both modes accept
+ * exactly the same two spellings, and what differs — the argument — is already
+ * on the type the author is writing against.
+ */
+function javascriptExportedFunction(moduleUrl) {
+    return `
   const mod = await import(${JSON.stringify(moduleUrl)});
   const exported = typeof mod.default === "function"
     ? mod.default
@@ -583,11 +1223,6 @@ try {
       " Export the function as \`export default\`, or name it \`transform\`."
     );
   }
-  const rows = await exported({ records, context });
-  process.stdout.write(JSON.stringify({ rows: rows ?? [], logs: captured() }));
-} catch (error) {
-  ${JAVASCRIPT_FAILURE}
-}
 `;
 }
 /**
@@ -597,7 +1232,7 @@ try {
  * copies of a log cap are two numbers that drift, and the one that drifts is
  * discovered by a run record nobody can explain.
  */
-const JAVASCRIPT_PRELUDE = `
+const JAVASCRIPT_CONSOLE = `
 const logs = [];
 let dropped = 0;
 const keep = (line) => {
@@ -614,11 +1249,25 @@ console.debug = write; console.trace = write;
 const captured = () => dropped === 0
   ? logs
   : logs.concat(["… " + dropped + " more line(s) were logged and dropped: a transform keeps its first ${MAX_LOG_LINES}."]);
-
+`;
+/**
+ * The whole of stdin, as one string.
+ *
+ * Split out of the prelude when the streaming harness arrived, because that one
+ * must **not** do this: slurping the input is precisely the thing a per-record
+ * run exists to avoid, and a harness that inherited it by sharing a constant
+ * would have held the dataset in the child while the parent was carefully not
+ * holding it in itself. The console capture above is shared by all three
+ * harnesses and this is shared by the two that read a finished batch, which is
+ * the line the split follows.
+ */
+const JAVASCRIPT_SLURP_STDIN = `
 let input = "";
 process.stdin.setEncoding("utf8");
 for await (const chunk of process.stdin) input += chunk;
 `;
+/** What the two whole-batch harnesses open with: capture the console, read stdin. */
+const JAVASCRIPT_PRELUDE = `${JAVASCRIPT_CONSOLE}${JAVASCRIPT_SLURP_STDIN}`;
 /** Unpack the envelope. `context` is frozen one level down — see below. */
 const JAVASCRIPT_PAYLOAD = `
   const payload = JSON.parse(input || "{}");