@tabnas/transduce 0.1.0

package/dist/limits.js

@@ -0,0 +1,133 @@
+"use strict";
+/* Copyright (c) 2026 tabnas, MIT License */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbortFlag = exports.Metrics = exports.NODE_BYTES = exports.Limits = exports.LIMIT_NAMES = void 0;
+exports.utf8Bytes = utf8Bytes;
+// The names of the `Limits` fields, in declaration order.
+exports.LIMIT_NAMES = Object.freeze([
+    'max_depth',
+    'max_key_bytes',
+    'max_scalar_bytes',
+    'max_metadata_bytes',
+    'max_columns',
+    'max_record_bytes',
+    'max_capture_bytes',
+    'max_output_bytes',
+]);
+exports.Limits = Object.freeze({
+    // The defaults: 256, 64 KiB, 16 MiB, 16 MiB, 10 000, 64 MiB, 64 MiB, none.
+    default() {
+        return {
+            max_depth: 256,
+            max_key_bytes: 64 * 1024,
+            max_scalar_bytes: 16 * 1024 * 1024,
+            max_metadata_bytes: 16 * 1024 * 1024,
+            max_columns: 10_000,
+            max_record_bytes: 64 * 1024 * 1024,
+            max_capture_bytes: 64 * 1024 * 1024,
+            max_output_bytes: null,
+        };
+    },
+    // No limit on anything that can be unlimited, and the largest values
+    // otherwise. For tests and trusted, measured inputs only.
+    unlimited() {
+        const max = Number.MAX_SAFE_INTEGER;
+        return {
+            max_depth: max,
+            max_key_bytes: max,
+            max_scalar_bytes: max,
+            max_metadata_bytes: max,
+            max_columns: max,
+            max_record_bytes: max,
+            max_capture_bytes: max,
+            max_output_bytes: null,
+        };
+    },
+    // The defaults with `fields` applied over them.
+    with(fields) {
+        return { ...exports.Limits.default(), ...fields };
+    },
+});
+// The fixed allowance counted for every retained node (a container, a
+// member, an element) on top of its payload bytes.
+exports.NODE_BYTES = 16;
+// The UTF-8 length of a string: `é` is 2, `日` 3, `😀` 4. A lone
+// surrogate counts 3, as its replacement character would.
+function utf8Bytes(s) {
+    const n = s.length;
+    // ASCII fast path: most keys and scalars never leave it.
+    let i = 0;
+    for (; i < n; i++) {
+        if (s.charCodeAt(i) >= 0x80)
+            break;
+    }
+    if (i === n)
+        return n;
+    return Buffer.byteLength(s, 'utf8');
+}
+// Counters and high-water marks a run reports, shared between stages.
+class Metrics {
+    constructor() {
+        // Source events seen.
+        this.events = 0;
+        // Object keys seen.
+        this.keys = 0;
+        // Scalars seen.
+        this.scalars = 0;
+        // Rows delivered to the table protocol.
+        this.rows = 0;
+        // Bytes currently held by materialized captures.
+        this.captured_bytes = 0;
+        // The most bytes ever held by captures at once.
+        this.captured_bytes_high = 0;
+        // The most bytes ever retained at once across every retaining stage.
+        this.retained_bytes_high = 0;
+        // Bytes written to the output.
+        this.output_bytes = 0;
+    }
+    // Account `bytes` as captured now, and raise the high-water marks.
+    capture(bytes) {
+        this.captured_bytes += bytes;
+        if (this.captured_bytes > this.captured_bytes_high) {
+            this.captured_bytes_high = this.captured_bytes;
+        }
+        if (this.captured_bytes > this.retained_bytes_high) {
+            this.retained_bytes_high = this.captured_bytes;
+        }
+    }
+    // Release `bytes` captured earlier.
+    release(bytes) {
+        this.captured_bytes -= bytes;
+    }
+    // The metrics as a plain object, one field per counter.
+    toJSON() {
+        return {
+            events: this.events,
+            keys: this.keys,
+            scalars: this.scalars,
+            rows: this.rows,
+            captured_bytes: this.captured_bytes,
+            captured_bytes_high: this.captured_bytes_high,
+            retained_bytes_high: this.retained_bytes_high,
+            output_bytes: this.output_bytes,
+        };
+    }
+}
+exports.Metrics = Metrics;
+// A cancellation flag shared by the caller, the source and the stages.
+//
+// The source polls it between parse steps through the engine's parse
+// budget and stops with `ABORTED`; long loops in stages poll it too.
+class AbortFlag {
+    constructor() {
+        this.aborted = false;
+    }
+    abort() {
+        this.aborted = true;
+    }
+    isAborted() {
+        return this.aborted;
+    }
+}
+exports.AbortFlag = AbortFlag;
+//# sourceMappingURL=limits.js.map

package/dist/limits.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"limits.js","sourceRoot":"","sources":["../src/limits.ts"],"names":[],"mappings":";AAAA,4CAA4C;;;;AAkC5C,0DAA0D;AAC7C,QAAA,WAAW,GAA8B,MAAM,CAAC,MAAM,CAAC;IAClE,WAAW;IACX,eAAe;IACf,kBAAkB;IAClB,oBAAoB;IACpB,aAAa;IACb,kBAAkB;IAClB,mBAAmB;IACnB,kBAAkB;CACnB,CAAC,CAAA;AAEW,QAAA,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,2EAA2E;IAC3E,OAAO;QACL,OAAO;YACL,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,EAAE,GAAG,IAAI;YACxB,gBAAgB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YAClC,kBAAkB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YACpC,WAAW,EAAE,MAAM;YACnB,gBAAgB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YAClC,iBAAiB,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YACnC,gBAAgB,EAAE,IAAI;SACvB,CAAA;IACH,CAAC;IAED,qEAAqE;IACrE,0DAA0D;IAC1D,SAAS;QACP,MAAM,GAAG,GAAG,MAAM,CAAC,gBAAgB,CAAA;QACnC,OAAO;YACL,SAAS,EAAE,GAAG;YACd,aAAa,EAAE,GAAG;YAClB,gBAAgB,EAAE,GAAG;YACrB,kBAAkB,EAAE,GAAG;YACvB,WAAW,EAAE,GAAG;YAChB,gBAAgB,EAAE,GAAG;YACrB,iBAAiB,EAAE,GAAG;YACtB,gBAAgB,EAAE,IAAI;SACvB,CAAA;IACH,CAAC;IAED,gDAAgD;IAChD,IAAI,CAAC,MAAuB;QAC1B,OAAO,EAAE,GAAG,QAAA,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,CAAA;IAC3C,CAAC;CACF,CAAC,CAAA;AAEF,sEAAsE;AACtE,mDAAmD;AACtC,QAAA,UAAU,GAAG,EAAE,CAAA;AAE5B,gEAAgE;AAChE,0DAA0D;AAC1D,mBAA0B,CAAS;IACjC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAA;IAClB,yDAAyD;IACzD,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAClB,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI;YAAE,MAAK;IACpC,CAAC;IACD,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IACrB,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;AACrC,CAAC;AAED,sEAAsE;AACtE;IAAA;QACE,sBAAsB;QACtB,WAAM,GAAG,CAAC,CAAA;QACV,oBAAoB;QACpB,SAAI,GAAG,CAAC,CAAA;QACR,gBAAgB;QAChB,YAAO,GAAG,CAAC,CAAA;QACX,wCAAwC;QACxC,SAAI,GAAG,CAAC,CAAA;QACR,iDAAiD;QACjD,mBAAc,GAAG,CAAC,CAAA;QAClB,gDAAgD;QAChD,wBAAmB,GAAG,CAAC,CAAA;QACvB,qEAAqE;QACrE,wBAAmB,GAAG,CAAC,CAAA;QACvB,+BAA+B;QAC/B,iBAAY,GAAG,CAAC,CAAA;IA+BlB,CAAC;IA7BC,mEAAmE;IACnE,OAAO,CAAC,KAAa;QACnB,IAAI,CAAC,cAAc,IAAI,KAAK,CAAA;QAC5B,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,cAAc,CAAA;QAChD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,cAAc,CAAA;QAChD,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,OAAO,CAAC,KAAa;QACnB,IAAI,CAAC,cAAc,IAAI,KAAK,CAAA;IAC9B,CAAC;IAED,wDAAwD;IACxD,MAAM;QACJ,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAA;IACH,CAAC;CACF;;AAED,uEAAuE;AACvE,EAAE;AACF,qEAAqE;AACrE,qEAAqE;AACrE;IAAA;QACU,YAAO,GAAG,KAAK,CAAA;IASzB,CAAC;IAPC,KAAK;QACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;IACrB,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF"}

package/dist/lines.d.ts

@@ -0,0 +1,39 @@
+import { AbortFlag, Limits, Metrics } from './limits';
+import { Flow, Sink } from './sink';
+import { Source } from './source';
+export declare const DEFAULT_CHUNK_BYTES: number;
+export type LineFormat = {
+    type: 'jsonl';
+} | {
+    type: 'csv';
+    header: boolean;
+    options: Record<string, unknown>;
+};
+export declare const LineFormat: Readonly<{
+    jsonl(): LineFormat;
+    csv(header?: boolean, options?: Record<string, unknown>): LineFormat;
+}>;
+export type LinesChunk = string | Uint8Array;
+export type LinesInput = LinesChunk | Iterable<LinesChunk>;
+export interface LinesWriter {
+    write(chunk: LinesChunk): Flow;
+    end(): Flow;
+}
+export declare class LinesSource implements Source {
+    private input;
+    private format;
+    private sourceLimits;
+    private abortFlag;
+    private sourceMetrics;
+    private chunkSize;
+    private consumed;
+    constructor(input: LinesInput | null, format: LineFormat);
+    limits(limits: Limits): this;
+    abort(abort: AbortFlag): this;
+    metrics(metrics: Metrics): this;
+    chunkBytes(bytes: number): this;
+    run(sink: Sink): Flow;
+    runIncremental(sink: Sink): Flow;
+    private drive;
+    writer(sink: Sink, incremental?: boolean): LinesWriter;
+}