@tabnas/transduce 0.1.0

package/dist/lines.js

@@ -0,0 +1,1139 @@
+"use strict";
+/* Copyright (c) 2026 tabnas, MIT License */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LinesSource = exports.LineFormat = exports.DEFAULT_CHUNK_BYTES = void 0;
+// `LinesSource`: JSON Lines and CSV a record (or a chunk of records) at a
+// time, from a synchronous iterable of chunks or pushed chunk by chunk.
+//
+// The engine parses a whole string, so a document is retained at least
+// once however it is consumed. Line-delimited formats do not need to be one
+// document: JSON Lines is one value per line, and CSV is a header plus
+// independent records, so each can be parsed a piece at a time with one
+// reused parser and the memory a run needs stops depending on the input's
+// size. Its events are exactly the whole-input parse's (the array of the
+// per-line values, or of the records).
+//
+// Both formats cut the input where the grammar ends a record, reading the
+// text as the grammar's own lexer does (with the line characters, quotes,
+// separators and comments its parser's resolved options set), and nothing
+// read is left unparsed but a blank JSON Lines record, which the grammar
+// skips too.
+//
+// JSON Lines: each record is parsed with one `@tabnas/json` parser. A
+// record is the text between line characters outside a string, so a lone
+// `\r` ends one as `\n` does. A record of nothing but the grammar's spaces
+// (a space or a tab) is blank and skipped; any other is parsed, so a line
+// holding a form feed or a no-break space fails as the grammar fails it.
+// On the incremental path (`runIncremental`, or a writer asked for it) the
+// record goes through the rule-event adapter, reset per record, so numbers
+// keep their lexemes; on the walking path (`run`) the value is walked and
+// numbers carry none. A record that does not parse is `INPUT_INVALID` with
+// its line's number as the row.
+//
+// CSV: the input is cut into chunks of whole records. A quote opens a
+// quoted field only where the lexer starts a token (a record's start, after
+// the field separator or a space), so a quote inside a field is that
+// field's text; the configured quote reads `""` as one quote; a quoted
+// field, a backtick string and a block comment may span lines; and a line
+// character anywhere else, a lone `\r` as much as `\n`, ends a record. The
+// header record, the first record the grammar reads (a blank or comment
+// line before it is none), is kept and prepended to every chunk after the
+// first when `header` is on, so the reused `@tabnas/csv` parser names each
+// record's fields as the whole input would. A chunk closes at the first
+// record boundary at or past `DEFAULT_CHUNK_BYTES` (or the configured
+// size), so one chunk, and never a fraction of a record, is what a parse
+// holds. An input that ends inside a quoted field fails with the grammar's
+// `unterminated_string`, in the header as anywhere. CSV is walked on both
+// paths.
+//
+// The input is read a piece at a time, and a piece ends just past each of
+// the grammar's line endings: one of its line characters (a lone `\r` or a
+// configured separator as much as `\n`), a `\r\n`, or under `record.empty`
+// the whole line token the lexer reads. A line character inside one of the
+// grammar's fixed tokens (a field separator such as `"\n~"`) is the token's
+// and ends no piece. So a record never waits for a `\n` to end, and a chunk
+// can close after any record, however many of them one `\n`-terminated line
+// holds. Where the input so far ends inside what may be one line ending,
+// the piece waits for the next chunk of it.
+//
+// Every count is in UTF-8 bytes of the input as it arrives. A single
+// record larger than `max_record_bytes` (a record with its line ending; for
+// the CSV header, everything up to its end) fails with that limit's name
+// the moment it passes the limit, whatever its length, rather than growing
+// a chunk without bound.
+const csv_1 = require("@tabnas/csv");
+const json_1 = require("@tabnas/json");
+const error_1 = require("./error");
+const event_1 = require("./event");
+const guard_1 = require("./guard");
+const limits_1 = require("./limits");
+const rule_events_1 = require("./rule-events");
+const source_1 = require("./source");
+// How much of the input one CSV parse holds, at most one record over.
+exports.DEFAULT_CHUNK_BYTES = 256 * 1024;
+exports.LineFormat = Object.freeze({
+    jsonl() {
+        return { type: 'jsonl' };
+    },
+    // CSV with a header line and the grammar's default options, or the given
+    // ones.
+    csv(header = true, options = {}) {
+        return { type: 'csv', header, options };
+    },
+});
+// A line-delimited input as a source.
+class LinesSource {
+    // `input` may be `null` for a source used only through `writer`.
+    constructor(input, format) {
+        this.sourceLimits = limits_1.Limits.default();
+        this.abortFlag = new limits_1.AbortFlag();
+        this.sourceMetrics = new limits_1.Metrics();
+        this.chunkSize = exports.DEFAULT_CHUNK_BYTES;
+        this.consumed = false;
+        this.input = input;
+        this.format = format;
+    }
+    limits(limits) {
+        this.sourceLimits = limits;
+        return this;
+    }
+    abort(abort) {
+        this.abortFlag = abort;
+        return this;
+    }
+    metrics(metrics) {
+        this.sourceMetrics = metrics;
+        return this;
+    }
+    // The CSV chunk size; a chunk closes at the first record boundary at or
+    // past it. Zero closes a chunk at every record.
+    chunkBytes(bytes) {
+        this.chunkSize = bytes;
+        return this;
+    }
+    // Both formats through the walk; JSON Lines numbers carry no lexeme.
+    run(sink) {
+        return this.drive(this.writer(sink, false));
+    }
+    // JSON Lines through the rule-event adapter (lexemes kept), CSV through
+    // the walk.
+    runIncremental(sink) {
+        return this.drive(this.writer(sink, true));
+    }
+    drive(writer) {
+        if (this.consumed) {
+            throw new error_1.Fail('STREAM_REUSED', 'a LinesSource reads its input once');
+        }
+        if (null === this.input) {
+            throw new TypeError('LinesSource: no input; use writer() to push chunks');
+        }
+        const input = this.input;
+        this.input = null;
+        this.consumed = true;
+        const chunks = 'string' === typeof input || input instanceof Uint8Array ? [input] : input;
+        for (const chunk of chunks) {
+            if ('stop' === writer.write(chunk))
+                return 'stop';
+        }
+        return writer.end();
+    }
+    // A writer that pushes the input into `sink`, on the incremental path
+    // when `incremental` is true.
+    writer(sink, incremental = false) {
+        const options = {
+            limits: this.sourceLimits,
+            abort: this.abortFlag,
+            metrics: this.sourceMetrics,
+            chunkBytes: this.chunkSize,
+        };
+        const driver = 'jsonl' === this.format.type
+            ? incremental
+                ? new JsonlIncremental(sink, options)
+                : new JsonlWalk(sink, options)
+            : new CsvWalk(sink, options, this.format.header, this.format.options);
+        return new Writer(driver);
+    }
+}
+exports.LinesSource = LinesSource;
+// The writer's state machine: a stop or a failure ends it for good.
+class Writer {
+    constructor(driver) {
+        this.started = false;
+        this.state = 'open';
+        this.failure = null;
+        this.driver = driver;
+        this.onPiece = (row, text, bytes) => this.driver.piece(row, text, bytes);
+    }
+    guard(step) {
+        if ('stopped' === this.state)
+            return 'stop';
+        if ('failed' === this.state)
+            throw this.failure;
+        if ('ended' === this.state)
+            throw error_1.Fail.protocol('the line source has already ended');
+        try {
+            if (!this.started) {
+                this.started = true;
+                if ('stop' === this.driver.begin())
+                    return this.finish('stopped');
+            }
+            const flow = step();
+            if ('stop' === flow)
+                return this.finish('stopped');
+            return flow;
+        }
+        catch (err) {
+            this.failure = err;
+            this.finish('failed');
+            throw err;
+        }
+    }
+    finish(state) {
+        this.state = state;
+        this.driver.close();
+        return 'stop';
+    }
+    write(chunk) {
+        return this.guard(() => {
+            const bytes = 'string' === typeof chunk ? Buffer.from(chunk, 'utf8') : chunk;
+            return this.driver.splitter.feed(bytes, this.onPiece);
+        });
+    }
+    end() {
+        const flow = this.guard(() => {
+            if ('stop' === this.driver.splitter.end(this.onPiece))
+                return 'stop';
+            return this.driver.finish();
+        });
+        if ('open' === this.state) {
+            this.finish('ended');
+            return flow;
+        }
+        return flow;
+    }
+}
+const EMPTY = new Uint8Array(0);
+const ENCODER = new TextEncoder();
+// The characters of one of the engine's resolved character maps while it
+// lexes their kind, else none.
+function lexed(on, chars) {
+    return on && chars ? Object.keys(chars) : [];
+}
+// The input in pieces, each ending just past a line ending: one of the
+// grammar's line characters, a `\r\n`, or under `line.single`
+// (`record.empty`) the whole line token the lexer reads, every line
+// character up to a repeated one. A line character inside one of the
+// grammar's fixed tokens (a field separator such as `"\n~"`) is the token's
+// and ends no piece, and where a fixed token holds one, a piece takes the
+// whole run of line characters the lexer reads as one token, so that no
+// piece starts inside a line token. A record therefore never ends inside a
+// piece, only at its end. Each piece carries the row the engine gives its
+// first character: one more than the row characters before it. Where the
+// input so far ends inside what may be a line ending, the piece waits for
+// more.
+class PieceSplitter {
+    constructor(line, rows, single, fixed) {
+        // The bytes of the piece being read, and any after it not yet taken.
+        this.pending = EMPTY;
+        // How much of `pending` has been searched for a line ending.
+        this.scanned = 0;
+        // Once a line ending is found: where its token ends so far, and its
+        // characters.
+        this.tokenEnd = -1;
+        this.token = [];
+        // Where the next piece starts: its row, and its byte offset in it.
+        this.row = 1;
+        this.offset = 0;
+        this.ends = new Uint8Array(256);
+        // How many more bytes the piece being read may take, and the failure
+        // when it takes more, given the row it starts on: the bound is the
+        // record's, so the format that reads the records sets both.
+        this.budget = () => Infinity;
+        this.over = (row) => error_1.Fail.input(`line ${row} is too long`).at(row, 1);
+        // The engine keeps a character outside the basic plane as its two
+        // UTF-16 halves, each a line character; the input holds it as one
+        // four-byte form, which ends a piece when both halves are line
+        // characters.
+        this.line = line.filter((c) => !isHigh(c) && !isLow(c));
+        for (const h of line.filter(isHigh)) {
+            for (const l of line.filter(isLow))
+                this.line.push(h + l);
+        }
+        this.forms = this.line.map((c) => ENCODER.encode(c));
+        for (const form of this.forms)
+            this.ends[form[form.length - 1]] = 1;
+        this.single = single;
+        this.spanning = fixed
+            .filter((token) => Array.from({ length: token.length }, (_, i) => token[i]).some((c) => line.includes(c)))
+            .map((token) => ENCODER.encode(token));
+        this.rows = rows;
+    }
+    // The row the next piece starts on.
+    get nextRow() {
+        return this.row;
+    }
+    feed(bytes, piece) {
+        if (0 === bytes.length)
+            return 'continue';
+        this.pending = 0 === this.pending.length ? bytes : Buffer.concat([this.pending, bytes]);
+        return this.drain(piece, false);
+    }
+    end(piece) {
+        if ('stop' === this.drain(piece, true))
+            return 'stop';
+        if (0 === this.pending.length)
+            return 'continue';
+        const raw = this.pending;
+        this.pending = EMPTY;
+        this.scanned = 0;
+        return this.emit(raw, piece);
+    }
+    drain(piece, atEnd) {
+        for (;;) {
+            if (this.tokenEnd < 0 && !this.findEnding(atEnd)) {
+                if (this.pending.length > this.budget())
+                    throw this.over(this.row);
+                return 'continue';
+            }
+            if (!this.followToken(atEnd))
+                return 'continue';
+            const raw = this.pending.subarray(0, this.tokenEnd);
+            this.pending = this.pending.subarray(this.tokenEnd);
+            this.scanned = 0;
+            this.tokenEnd = -1;
+            this.token = [];
+            if ('stop' === this.emit(raw, piece))
+                return 'stop';
+        }
+    }
+    // Finds the first line ending past `scanned`, checked against the line
+    // character's whole UTF-8 form, and starts its token. False while the
+    // input so far holds none, or cannot yet tell where one ends.
+    findEnding(atEnd) {
+        const p = this.pending;
+        for (let i = this.scanned; i < p.length; i++) {
+            if (!this.ends[p[i]])
+                continue;
+            const k = this.formEnding(i + 1);
+            if (k < 0)
+                continue;
+            const at = i + 1 - this.forms[k].length;
+            const inside = this.inFixed(at, k, atEnd);
+            if (true === inside)
+                continue;
+            if (null === inside) {
+                this.scanned = at;
+                return false;
+            }
+            this.tokenEnd = i + 1;
+            this.token = [this.line[k]];
+            if (this.tokenEnd > this.budget())
+                throw this.over(this.row);
+            return true;
+        }
+        this.scanned = p.length;
+        return false;
+    }
+    // Whether the `k`th line character, at `at`, lies inside one of the fixed
+    // tokens that hold a line character; null while the input so far cannot
+    // tell.
+    inFixed(at, k, atEnd) {
+        const p = this.pending;
+        const form = this.forms[k];
+        let unsure = false;
+        for (const token of this.spanning) {
+            for (let j = 0; j + form.length <= token.length; j++) {
+                if (!sameBytes(token, j, form, 0, form.length))
+                    continue;
+                const start = at - j;
+                if (start < 0 || !sameBytes(p, start, token, 0, j))
+                    continue;
+                const after = at + form.length;
+                const rest = token.length - j - form.length;
+                const have = Math.min(rest, p.length - after);
+                if (!sameBytes(p, after, token, j + form.length, have))
+                    continue;
+                if (have === rest)
+                    return true;
+                if (!atEnd)
+                    unsure = true;
+            }
+        }
+        return unsure ? null : false;
+    }
+    // The line character whose UTF-8 form `pending` holds just before `end`,
+    // by its index, or -1.
+    formEnding(end) {
+        const p = this.pending;
+        next: for (let k = 0; k < this.forms.length; k++) {
+            const form = this.forms[k];
+            if (end < form.length)
+                continue;
+            for (let j = 0; j < form.length; j++) {
+                if (p[end - form.length + j] !== form[j])
+                    continue next;
+            }
+            return k;
+        }
+        return -1;
+    }
+    // Takes the rest of the line token the piece ends with: under
+    // `line.single`, every line character not yet in it, as the lexer reads
+    // one; where a fixed token holds a line character, every line character,
+    // as the lexer reads a run; otherwise a `\n` after a `\r`, so that `\r\n`
+    // is one line ending and a record is held to its own line. False while
+    // the input so far cannot tell whether the token goes on.
+    followToken(atEnd) {
+        const p = this.pending;
+        for (;;) {
+            let next = -1;
+            let partial = false;
+            for (let k = 0; k < this.line.length && next < 0; k++) {
+                const c = this.line[k];
+                const goesOn = this.single
+                    ? !this.token.includes(c)
+                    : 0 < this.spanning.length || (1 === this.token.length && '\r' === this.token[0] && '\n' === c);
+                if (!goesOn)
+                    continue;
+                const form = this.forms[k];
+                const have = p.length - this.tokenEnd;
+                let prefix = true;
+                for (let j = 0; j < Math.min(have, form.length); j++) {
+                    if (p[this.tokenEnd + j] !== form[j]) {
+                        prefix = false;
+                        break;
+                    }
+                }
+                if (!prefix)
+                    continue;
+                if (have >= form.length)
+                    next = k;
+                else
+                    partial = true;
+            }
+            if (0 <= next) {
+                this.tokenEnd += this.forms[next].length;
+                this.token.push(this.line[next]);
+                if (this.tokenEnd > this.budget())
+                    throw this.over(this.row);
+                continue;
+            }
+            return !partial || atEnd;
+        }
+    }
+    emit(raw, piece) {
+        const row = this.row;
+        const bad = utf8Invalid(raw);
+        if (0 <= bad) {
+            const valid = DECODER.decode(raw.subarray(0, bad));
+            const [badRow, offset] = advance(this.rows, this.row, this.offset, valid);
+            throw error_1.Fail.input(`line ${badRow} is not UTF-8 from its byte ${offset + 1}`).at(badRow, offset + 1);
+        }
+        const text = DECODER.decode(raw);
+        [this.row, this.offset] = advance(this.rows, this.row, this.offset, text);
+        return piece(row, text, raw.length);
+    }
+}
+// The row, and the byte offset in it, just after `text`, which starts at
+// `row` and `offset`: a row character starts the next row. The engine reads
+// a UTF-16 code unit at a time, so a row character outside the basic plane
+// is its two halves, and counts as both.
+function advance(rows, row, offset, text) {
+    for (let i = 0; i < text.length; i++) {
+        const c = text[i];
+        if (rows.includes(c)) {
+            row++;
+            offset = 0;
+        }
+        else {
+            // A character outside the basic plane is four bytes, counted at its
+            // first half.
+            const u = c.charCodeAt(0);
+            offset += u < 0x80 ? 1 : u < 0x800 ? 2 : isHigh(c) ? 4 : isLow(c) ? 0 : 3;
+        }
+    }
+    return [row, offset];
+}
+// Whether one UTF-16 code unit is the first or the second half of a
+// character outside the basic plane.
+function isHigh(c) {
+    const u = c.charCodeAt(0);
+    return 1 === c.length && 0xd800 <= u && u < 0xdc00;
+}
+function isLow(c) {
+    const u = c.charCodeAt(0);
+    return 1 === c.length && 0xdc00 <= u && u < 0xe000;
+}
+// Whether `n` bytes of `a` from `ai` are those of `b` from `bi`.
+function sameBytes(a, ai, b, bi, n) {
+    if (ai + n > a.length || bi + n > b.length)
+        return false;
+    for (let j = 0; j < n; j++)
+        if (a[ai + j] !== b[bi + j])
+            return false;
+    return true;
+}
+// The sources of a parser's fixed tokens, while it lexes them.
+function fixedTokens(config) {
+    return config.fixed.lex
+        ? Object.values(config.fixed.token)
+            .map((tin) => config.fixed.ref[tin])
+            .filter((src) => 'string' === typeof src && 0 < src.length)
+        : [];
+}
+const DECODER = new TextDecoder('utf-8', { ignoreBOM: true });
+// The index of the first byte of the first invalid UTF-8 sequence, or -1.
+function utf8Invalid(b) {
+    let i = 0;
+    const n = b.length;
+    while (i < n) {
+        const c = b[i];
+        if (c < 0x80) {
+            i++;
+            continue;
+        }
+        let need;
+        let min;
+        if (c >= 0xc2 && c <= 0xdf) {
+            need = 1;
+            min = 0x80;
+        }
+        else if (c >= 0xe0 && c <= 0xef) {
+            need = 2;
+            min = 0x800;
+        }
+        else if (c >= 0xf0 && c <= 0xf4) {
+            need = 3;
+            min = 0x10000;
+        }
+        else {
+            return i;
+        }
+        if (i + need >= n)
+            return i;
+        let cp = c & (0xff >> (need + 2));
+        for (let k = 1; k <= need; k++) {
+            const d = b[i + k];
+            if (undefined === d || (d & 0xc0) !== 0x80)
+                return i;
+            cp = (cp << 6) | (d & 0x3f);
+        }
+        if (cp < min || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
+            return i;
+        i += need + 1;
+    }
+    return -1;
+}
+// An engine error in the record or chunk that starts on row `start`, at
+// row `line` of the input. An abort names `start` and no column: it lands
+// between two of the engine's steps, where the engine often holds no token
+// to place it at and falls back to 1:1 of the text it was parsing, so its
+// own position says nothing.
+function lineFailure(error, line, start, abort) {
+    const fail = (0, source_1.engineFailure)(error, abort);
+    if ('ABORTED' === fail.code) {
+        fail.row = start;
+    }
+    else {
+        fail.row = line;
+        fail.col = (0, error_1.enginePosition)(error)?.col ?? 0;
+    }
+    return fail;
+}
+// A failure while the record or chunk that starts on row `start` was read:
+// an abort that names no row names that one.
+function atRecord(error, start) {
+    if (error instanceof error_1.Fail && 'ABORTED' === error.code && undefined === error.row) {
+        error.row = start;
+    }
+    return error;
+}
+// JSON Lines records from the input's pieces. A record ends at a line
+// character outside a string, as the grammar's lexer ends one: a line
+// character inside a string is the string's, which the grammar refuses
+// there, so the record goes on into the next piece and fails whole as the
+// grammar fails it. A record of nothing but the grammar's spaces is blank
+// and skipped.
+class JsonRecords {
+    constructor(parser, maxBytes) {
+        // The record being read: its text and bytes so far, the row it starts on
+        // (0 before its first piece), and whether a string is open in it.
+        this.text = '';
+        this.bytes = 0;
+        this.start = 0;
+        this.quote = null;
+        this.escaped = false;
+        const config = parser.internal().config;
+        this.line = lexed(config.line.lex, config.line.chars);
+        this.rows = Object.keys(config.line.rowChars ?? {});
+        this.single = !!config.line.single;
+        this.fixed = fixedTokens(config);
+        this.space = lexed(config.space.lex, config.space.chars);
+        this.quotes = lexed(config.string.lex, config.string.quoteMap);
+        this.escape = config.string.escChar ?? '\\';
+        this.maxBytes = maxBytes;
+    }
+    // A splitter for this grammar's line endings, bounded by the record.
+    splitter() {
+        const splitter = new PieceSplitter(this.line, this.rows, this.single, this.fixed);
+        splitter.budget = () => this.maxBytes - this.bytes;
+        splitter.over = (row) => {
+            const first = this.start || row;
+            return error_1.Fail.limit('max_record_bytes', this.maxBytes, `line ${first} is longer than ${this.maxBytes} bytes`).at(first, 1);
+        };
+        return splitter;
+    }
+    // Reads one piece, and hands `record` each record it ends that is not
+    // blank, with the row it starts on.
+    piece(row, text, bytes, record) {
+        if (0 === this.start)
+            this.start = row;
+        let end = -1;
+        for (let i = 0; i < text.length; i++) {
+            const c = text[i];
+            if (null !== this.quote) {
+                if (this.escaped)
+                    this.escaped = false;
+                else if (c === this.escape)
+                    this.escaped = true;
+                else if (c === this.quote)
+                    this.quote = null;
+            }
+            else if (this.quotes.includes(c)) {
+                this.quote = c;
+            }
+            else if (this.line.includes(c)) {
+                end = i;
+                break;
+            }
+        }
+        if (end < 0) {
+            this.text += text;
+            this.bytes += bytes;
+            return 'continue';
+        }
+        return this.take(this.text + text.slice(0, end), record);
+    }
+    // The record the input ends inside, if any.
+    end(record) {
+        if (0 === this.start)
+            return 'continue';
+        return this.take(this.text, record);
+    }
+    take(text, record) {
+        const start = this.start;
+        this.text = '';
+        this.bytes = 0;
+        this.start = 0;
+        this.quote = null;
+        this.escaped = false;
+        for (const c of text) {
+            if (!this.space.includes(c))
+                return record(start, text);
+        }
+        return 'continue';
+    }
+}
+// JSON Lines through the walk.
+class JsonlWalk {
+    constructor(sink, options) {
+        this.guarded = new guard_1.Guarded(sink, options.limits, options.abort, options.metrics);
+        this.abort = options.abort;
+        this.parser = (0, json_1.make)();
+        const abort = options.abort;
+        (0, rule_events_1.prepare)(this.parser, () => !abort.isAborted());
+        this.records = new JsonRecords(this.parser, options.limits.max_record_bytes);
+        this.splitter = this.records.splitter();
+        this.onRecord = (row, text) => this.record(row, text);
+    }
+    begin() {
+        return this.guarded.event(event_1.Ev.arrayStart);
+    }
+    piece(row, text, bytes) {
+        return this.records.piece(row, text, bytes, this.onRecord);
+    }
+    record(number, text) {
+        let value;
+        try {
+            value = this.parser.parse(text);
+        }
+        catch (err) {
+            throw lineFailure(err, number, number, this.abort);
+        }
+        try {
+            return (0, source_1.walkValue)(value, this.guarded);
+        }
+        catch (err) {
+            throw atRecord(err, number);
+        }
+    }
+    finish() {
+        if ('stop' === this.records.end(this.onRecord))
+            return 'stop';
+        if ('stop' === this.guarded.event(event_1.Ev.arrayEnd))
+            return 'stop';
+        return this.guarded.event(event_1.Ev.end);
+    }
+    close() {
+        this.guarded.flush();
+    }
+}
+// JSON Lines through the rule-event adapter: one parser, one subscriber,
+// the adapter reset before each record.
+class JsonlIncremental {
+    constructor(sink, options) {
+        // The record being read, which an abort the adapter raised names.
+        this.reading = 0;
+        const stop = new limits_1.AbortFlag();
+        const abort = options.abort;
+        this.abort = abort;
+        const adapter = new rule_events_1.Adapter(sink, options.limits, abort, options.metrics, source_1.Prune.never(), stop);
+        this.adapter = adapter;
+        this.parser = (0, json_1.make)();
+        (0, rule_events_1.prepare)(this.parser, () => !abort.isAborted() && !stop.isAborted(), (rule, done) => adapter.onDone(rule, done));
+        this.records = new JsonRecords(this.parser, options.limits.max_record_bytes);
+        this.splitter = this.records.splitter();
+        this.onRecord = (row, text) => this.record(row, text);
+    }
+    // The adapter's own outcome, when it stopped or failed inside a parse.
+    status() {
+        const status = this.adapter.status;
+        if ('failed' === status.type) {
+            throw 0 < this.reading ? atRecord(status.error, this.reading) : status.error;
+        }
+        if ('stopped' === status.type)
+            return 'stop';
+        return null;
+    }
+    begin() {
+        return this.adapter.send(event_1.Ev.arrayStart);
+    }
+    piece(row, text, bytes) {
+        return this.records.piece(row, text, bytes, this.onRecord);
+    }
+    record(number, text) {
+        this.reading = number;
+        let parsed;
+        try {
+            parsed = { ok: true, value: this.parser.parse(text) };
+        }
+        catch (error) {
+            parsed = { ok: false, error };
+        }
+        const status = this.status();
+        if (null !== status)
+            return status;
+        const adapter = this.adapter;
+        if (!parsed.ok)
+            throw lineFailure(parsed.error, number, number, this.abort);
+        if (adapter.complete()) {
+            adapter.reset();
+            return 'continue';
+        }
+        if (adapter.idle()) {
+            let flow;
+            try {
+                flow = adapter.walkWhole(parsed.value);
+            }
+            catch (err) {
+                throw atRecord(err, number);
+            }
+            if ('stop' === flow)
+                return 'stop';
+            adapter.reset();
+            return 'continue';
+        }
+        throw (0, rule_events_1.notStreamable)();
+    }
+    finish() {
+        if ('stop' === this.records.end(this.onRecord))
+            return 'stop';
+        const status = this.status();
+        if (null !== status)
+            return status;
+        if ('stop' === this.adapter.send(event_1.Ev.arrayEnd))
+            return 'stop';
+        return this.adapter.send(event_1.Ev.end);
+    }
+    close() {
+        this.adapter.sink.flush();
+    }
+}
+// What decides where the CSV grammar ends a record, read from its parser's
+// resolved options and from the options the plugin builds its quote
+// matcher from, so a separator, quote, record separator or comment setting
+// moves the chunker's cut as it moves the grammar. Each set is empty while
+// the engine does not lex its kind.
+class Lexis {
+    constructor() {
+        // Line characters: outside a token, each ends a record.
+        this.line = [];
+        // Whether a run of line characters stops at a repeated one
+        // (`record.empty`), so that `\n\n` is two line tokens.
+        this.single = false;
+        this.rows = [];
+        this.space = [];
+        // The fixed tokens: the field separator, and outside strict mode the
+        // JSON structure characters.
+        this.fixed = [];
+        // The RFC 4180 quote, while the grammar's own matcher reads it: a
+        // doubled one inside is one quote, and a line character is text.
+        this.quote = null;
+        // The engine's own string quotes, after that one: an `escape` takes the
+        // next character whatever it is, and a line character is text only
+        // inside the `multi` ones (a backtick).
+        this.strings = [];
+        this.multi = [];
+        this.escape = '\\';
+        // The comment markers, longest first, each with its end: `null` for a
+        // line comment, which a line character ends without being part of.
+        this.comments = [];
+        // What ends a run of text, as characters and as token starts.
+        this.stops = [];
+        this.stopPrefixes = [];
+        // Whether the grammar ignores a space (outside strict mode) and a
+        // comment, so that a record of nothing else is blank.
+        this.spaceIgnored = false;
+        this.commentIgnored = false;
+    }
+    static csv(parser) {
+        const config = parser.internal().config;
+        const options = parser.options.plugin?.csv ?? {};
+        const lexis = new Lexis();
+        lexis.line = lexed(config.line.lex, config.line.chars);
+        lexis.single = !!config.line.single;
+        lexis.rows = Object.keys(config.line.rowChars ?? {});
+        lexis.space = lexed(config.space.lex, config.space.chars);
+        lexis.fixed = fixedTokens(config);
+        // The longest marker first, and a tie by name: the engine's order.
+        lexis.comments = config.comment.lex
+            ? Object.entries(config.comment.def)
+                .filter(([, def]) => def.lex && def.start)
+                .sort(([an, a], [bn, b]) => b.start.length - a.start.length || (an < bn ? -1 : an > bn ? 1 : 0))
+                .map(([, def]) => [def.start, def.line ? null : (def.end ?? '')])
+            : [];
+        // The RFC 4180 matcher as the plugin installs it: on in strict mode
+        // unless `string.csv` is false, off otherwise unless it is true, and
+        // inert for a quote that is not one UTF-16 code unit.
+        const matcher = options.strict ? false !== options.string?.csv : true === options.string?.csv;
+        const quote = options.string?.quote;
+        lexis.quote = matcher && 'string' === typeof quote && 1 === quote.length ? quote : null;
+        lexis.strings = lexed(config.string.lex, config.string.quoteMap);
+        lexis.multi = Object.keys(config.string.multiChars ?? {});
+        lexis.escape = config.string.escChar ?? '\\';
+        // What the engine's text matcher stops at: its ender pattern, which is
+        // the space and line characters, the fixed tokens, the comment markers
+        // and the grammar's own enders.
+        lexis.stops = [...lexis.space, ...lexis.line];
+        const enders = parser.options.ender;
+        lexis.stopPrefixes = [
+            ...lexis.fixed,
+            ...lexis.comments.map(([start]) => start),
+            ...('string' === typeof enders ? [...enders] : Array.isArray(enders) ? enders : []).filter((e) => 'string' === typeof e && 0 < e.length),
+        ];
+        const ignored = config.tokenSetTins?.IGNORE ?? {};
+        lexis.spaceIgnored = !!ignored[parser.token('#SP')];
+        lexis.commentIgnored = !!ignored[parser.token('#CM')];
+        return lexis;
+    }
+    // The length of the longest fixed token at `i`, or 0.
+    fixedAt(text, i) {
+        let len = 0;
+        for (const source of this.fixed) {
+            if (source.length > len && text.startsWith(source, i))
+                len = source.length;
+        }
+        return len;
+    }
+    // The comment that starts at `i`, by its index, or -1.
+    commentAt(text, i) {
+        return this.comments.findIndex(([start]) => text.startsWith(start, i));
+    }
+    // The length of the line token at `i`.
+    lineRun(text, i) {
+        let j = i;
+        while (j < text.length) {
+            const c = text[j];
+            if (!this.line.includes(c) || (this.single && text.slice(i, j).includes(c)))
+                break;
+            j++;
+        }
+        return j - i;
+    }
+    // Whether a run of text stops at `i`.
+    endsText(text, i) {
+        return this.stops.includes(text[i]) || this.stopPrefixes.some((p) => text.startsWith(p, i));
+    }
+}
+const START = { type: 'start' };
+const TEXT = { type: 'text' };
+const QUOTED = { type: 'quoted' };
+// Follows CSV text through the grammar's tokens, a piece at a time, to
+// find where its records end.
+class Scanner {
+    constructor(lexis) {
+        this.at = START;
+        // Whether the record so far holds anything the grammar does not ignore,
+        // so that it is not blank.
+        this.content = false;
+        this.lexis = lexis;
+    }
+    // Scan one piece of the input, which ends just past a line ending unless
+    // it is the last, calling `end(offset, content)` where each record in it
+    // ends: `offset` is just past the line token that ends it, and `content`
+    // says whether it held anything the grammar does not ignore. Returns
+    // whether the piece ends a record, so that a chunk may close after it.
+    piece(text, end) {
+        const lexis = this.lexis;
+        let ended = false;
+        let i = 0;
+        while (i < text.length) {
+            const c = text[i];
+            ended = false;
+            const at = this.at;
+            switch (at.type) {
+                // In the lexer's order: the RFC 4180 matcher, fixed tokens, space,
+                // lines, strings, comments, and then a run of text.
+                case 'start': {
+                    let len = 0;
+                    if (lexis.quote === c) {
+                        this.at = QUOTED;
+                        this.content = true;
+                        i++;
+                    }
+                    else if (0 < (len = lexis.fixedAt(text, i))) {
+                        this.content = true;
+                        i += len;
+                    }
+                    else if (lexis.space.includes(c)) {
+                        this.content ||= !lexis.spaceIgnored;
+                        i++;
+                    }
+                    else if (lexis.line.includes(c)) {
+                        i += lexis.lineRun(text, i);
+                        end(i, this.content);
+                        this.content = false;
+                        ended = true;
+                    }
+                    else if (lexis.strings.includes(c)) {
+                        this.at = { type: 'str', quote: c, multi: lexis.multi.includes(c) };
+                        this.content = true;
+                        i++;
+                    }
+                    else {
+                        const comment = lexis.commentAt(text, i);
+                        if (0 <= comment) {
+                            this.at = { type: 'comment', index: comment };
+                            this.content ||= !lexis.commentIgnored;
+                            i += lexis.comments[comment][0].length;
+                        }
+                        else {
+                            // A character no token starts with (an ender) is one the
+                            // grammar refuses, and lexing resumes after it.
+                            if (!lexis.endsText(text, i))
+                                this.at = TEXT;
+                            this.content = true;
+                            i++;
+                        }
+                    }
+                    break;
+                }
+                case 'text':
+                    if (lexis.endsText(text, i))
+                        this.at = START;
+                    else
+                        i++;
+                    break;
+                case 'quoted':
+                    i++;
+                    if (lexis.quote === c) {
+                        if (text[i] === c)
+                            i++;
+                        else
+                            this.at = START;
+                    }
+                    break;
+                case 'str':
+                    if (c === at.quote) {
+                        this.at = START;
+                        i++;
+                    }
+                    else if (c === lexis.escape) {
+                        i += i + 1 < text.length ? 2 : 1;
+                    }
+                    else if (!at.multi && lexis.line.includes(c)) {
+                        // The grammar refuses the string here, so the line character
+                        // is read as a line.
+                        this.at = START;
+                    }
+                    else {
+                        i++;
+                    }
+                    break;
+                case 'comment': {
+                    const close = lexis.comments[at.index][1];
+                    if (null !== close && '' !== close && text.startsWith(close, i)) {
+                        this.at = START;
+                        i += close.length;
+                    }
+                    else if (null === close && lexis.line.includes(c)) {
+                        this.at = START;
+                    }
+                    else {
+                        i++;
+                    }
+                    break;
+                }
+            }
+        }
+        return ended;
+    }
+}
+// The UTF-8 length of a string.
+function utf8Bytes(text) {
+    return Buffer.byteLength(text, 'utf8');
+}
+// CSV through the walk: pieces into whole records, records into chunks,
+// each chunk parsed with one reused parser.
+class CsvWalk {
+    constructor(sink, options, header, grammar) {
+        // The header record as the grammar reads it, once read, and its bytes:
+        // every chunk after the first starts with it.
+        this.header = null;
+        this.headerBytes = 0;
+        this.started = false;
+        // The chunk being filled, while one is: its text and bytes; where it may
+        // first close (past the header in front of it, or in the first chunk past
+        // the header record itself), as an index and in bytes; the file row its
+        // own text starts on, and the rows of the header in front of it; where
+        // the text it cannot close inside starts (one record, or before the
+        // header everything up to its end), in bytes and as a row; and where the
+        // last record ended, which is where the next one starts.
+        this.open = false;
+        this.text = '';
+        this.bytes = 0;
+        this.body = 0;
+        this.bodyBytes = 0;
+        this.firstLine = 0;
+        this.prefixLines = 0;
+        this.recordAt = 0;
+        this.recordRow = 0;
+        this.lastEnd = 0;
+        this.guarded = new guard_1.Guarded(sink, options.limits, options.abort, options.metrics);
+        this.abort = options.abort;
+        this.maxRecordBytes = options.limits.max_record_bytes;
+        this.chunkBytes = options.chunkBytes;
+        this.wantHeader = header;
+        this.parser = (0, csv_1.make)({ ...grammar, header });
+        this.recordEmpty = !!this.parser.options.plugin?.csv?.record?.empty;
+        const abort = options.abort;
+        (0, rule_events_1.prepare)(this.parser, () => !abort.isAborted());
+        const lexis = Lexis.csv(this.parser);
+        this.scanner = new Scanner(lexis);
+        const splitter = new PieceSplitter(lexis.line, lexis.rows, lexis.single, lexis.fixed);
+        splitter.budget = () => this.maxRecordBytes - (this.open ? this.bytes - this.recordAt : 0);
+        splitter.over = (row) => {
+            const first = (this.open && this.recordRow) || row;
+            return error_1.Fail.limit('max_record_bytes', this.maxRecordBytes, `the record starting at line ${first} is longer than ${this.maxRecordBytes} bytes`).at(first, 1);
+        };
+        this.splitter = splitter;
+    }
+    begin() {
+        return this.guarded.event(event_1.Ev.arrayStart);
+    }
+    // Opens a chunk: after the first, the header goes in front of it.
+    openChunk() {
+        this.open = true;
+        this.text = this.started ? (this.header ?? '') : '';
+        this.bytes = this.started ? this.headerBytes : 0;
+        this.started = true;
+        this.prefixLines = advance(this.scanner.lexis.rows, 0, 0, this.text)[0];
+        this.body = this.text.length;
+        this.bodyBytes = this.bytes;
+        this.firstLine = 0;
+        this.recordAt = this.bytes;
+        this.recordRow = 0;
+        this.lastEnd = 0;
+    }
+    piece(row, text, bytes) {
+        if (!this.open)
+            this.openChunk();
+        if (0 === this.firstLine)
+            this.firstLine = row;
+        if (0 === this.recordRow)
+            this.recordRow = row;
+        const at = this.text.length;
+        const atBytes = this.bytes;
+        this.text += text;
+        this.bytes += bytes;
+        const seeking = this.wantHeader && null === this.header;
+        const found = { from: -1, to: -1 };
+        const ended = this.scanner.piece(text, (end, content) => {
+            if (seeking && found.to < 0 && (content || this.recordEmpty)) {
+                found.from = this.lastEnd;
+                found.to = at + end;
+            }
+            this.lastEnd = at + end;
+        });
+        if (0 <= found.to) {
+            const { from, to } = found;
+            this.header = this.text.slice(from, to);
+            this.headerBytes = utf8Bytes(this.header);
+            this.body = to;
+            this.bodyBytes = atBytes + utf8Bytes(text.slice(0, to - at));
+        }
+        // A chunk may close where a piece ends a record, once the header is
+        // behind it.
+        if (ended && !(this.wantHeader && null === this.header)) {
+            this.recordAt = this.bytes;
+            this.recordRow = 0;
+            if (this.bytes > this.bodyBytes && this.bytes >= this.chunkBytes)
+                return this.flush();
+        }
+        return 'continue';
+    }
+    // Parse the chunk and walk its records.
+    flush() {
+        if (!this.open)
+            return 'continue';
+        const text = this.text;
+        const firstLine = this.firstLine;
+        const prefixLines = this.prefixLines;
+        this.open = false;
+        this.text = '';
+        let value;
+        try {
+            value = this.parser.parse(text);
+        }
+        catch (err) {
+            const pos = (0, error_1.enginePosition)(err);
+            const rowInChunk = Math.max(pos?.row ?? 0, 1);
+            // An error in the prepended header itself is the file's first line.
+            const line = rowInChunk > prefixLines ? firstLine + rowInChunk - prefixLines - 1 : 1;
+            throw lineFailure(err, line, firstLine, this.abort);
+        }
+        // The grammar returns an array of records; anything else has none.
+        const records = Array.isArray(value) ? value : [];
+        try {
+            for (const record of records) {
+                if ('stop' === (0, source_1.walkValue)(record, this.guarded))
+                    return 'stop';
+            }
+        }
+        catch (err) {
+            throw atRecord(err, firstLine);
+        }
+        return 'continue';
+    }
+    finish() {
+        if ('stop' === this.flush())
+            return 'stop';
+        if ('stop' === this.guarded.event(event_1.Ev.arrayEnd))
+            return 'stop';
+        return this.guarded.event(event_1.Ev.end);
+    }
+    close() {
+        this.guarded.flush();
+    }
+}
+//# sourceMappingURL=lines.js.map