@graphty/graph-io 0.3.40 → 0.3.42

package/dist/src/common/input.js

@@ -405,31 +405,58 @@ class ByteDecoder {
         const bad = this.encoding === "utf-8" ? invalidUtf8(all) : null;
         const at = base + (bad === null ? 0 : bad.index);
         if (this.mayFallBack && this.asciiSoFar) {
-            // windows-1252 text holds no control bytes but TAB, LF, FF and CR: one anywhere in
-            // undeclared input that is not UTF-8 marks binary data (watch() saw every byte so far)
-            if (this.control !== null) {
-                return this.binary(this.control);
-            }
-            if (this.textFollows && bad?.cut === true) {
-                return this.report.fail(INVALID_UTF8_CODE, `invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`, undefined, { byteOffset: at });
-            }
-            if (!stream && bad?.truncated === true) {
-                return this.report.fail(INVALID_UTF8_CODE, `the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`, undefined, { byteOffset: at });
-            }
-            if (!startsWithUtf8(all)) {
-                const cutNote = !stream && bad?.cut === true
-                    ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated"
-                    : "";
-                this.report.warning("coercion", ENCODING_FALLBACK_CODE, this.declaredUtf8
-                    ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)`
-                    : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`);
-                this.use("windows-1252", false);
-                this.fellBack = true;
-                this.mixedPending = true;
-                this.reportUtf8After(all, base);
-                return this.decoder.decode(all, { stream });
+            const text = this.fallBack(all, bad, at, base, stream);
+            if (text !== null) {
+                return text;
             }
         }
+        return this.failDecode(bytes, at, base, stream);
+    }
+    /**
+     * Undeclared input that was ASCII so far failed to decode: fail on binary data or a cut
+     * sequence, else switch to windows-1252 with a warning unless the bytes start with valid
+     * non-ASCII UTF-8.
+     * @param all - the held-back bytes followed by the bytes that failed
+     * @param bad - the first invalid UTF-8 sequence, or null when the encoding is not UTF-8
+     * @param at - the byte offset of the first invalid byte
+     * @param base - the byte offset of `all`
+     * @param stream - whether more bytes follow
+     * @returns the bytes decoded as windows-1252, or null when they do not fall back
+     */
+    fallBack(all, bad, at, base, stream) {
+        // windows-1252 text holds no control bytes but TAB, LF, FF and CR: one anywhere in
+        // undeclared input that is not UTF-8 marks binary data (watch() saw every byte so far)
+        if (this.control !== null) {
+            return this.binary(this.control);
+        }
+        if (this.textFollows && bad?.cut === true) {
+            return this.report.fail(INVALID_UTF8_CODE, `invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`, undefined, { byteOffset: at });
+        }
+        if (!stream && bad?.truncated === true) {
+            return this.report.fail(INVALID_UTF8_CODE, `the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`, undefined, { byteOffset: at });
+        }
+        if (startsWithUtf8(all)) {
+            return null;
+        }
+        const cutNote = !stream && bad?.cut === true ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated" : "";
+        this.report.warning("coercion", ENCODING_FALLBACK_CODE, this.declaredUtf8
+            ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)`
+            : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`);
+        this.use("windows-1252", false);
+        this.fellBack = true;
+        this.mixedPending = true;
+        this.reportUtf8After(all, base);
+        return this.decoder.decode(all, { stream });
+    }
+    /**
+     * Fail the import on bytes that are not valid in the encoding.
+     * @param bytes - the bytes that failed
+     * @param at - the byte offset of the first invalid UTF-8 byte
+     * @param base - the byte offset of the held-back bytes and these
+     * @param stream - whether more bytes follow
+     * @returns never: it throws
+     */
+    failDecode(bytes, at, base, stream) {
         if (this.encoding === "utf-8") {
             const after = this.mayFallBack ? " after valid non-ASCII UTF-8 text; pass the encoding option" : "";
             return this.report.fail(INVALID_UTF8_CODE, `invalid UTF-8 at byte ${at}${after}`, undefined, {
@@ -1018,132 +1045,210 @@ function reportTextEncoding(report, encoding) {
 export async function* textChunks(rawInput, report, options = {}) {
     const input = normalizeInput(rawInput);
     const signal = options.signal ?? null;
-    const onProgress = options.onProgress ?? null;
     const filter = new TextFilter(report, options);
     throwIfAborted(signal);
     if (typeof input === "string") {
         reportTextEncoding(report, options.encoding);
-        const total = onProgress === null ? 0 : utf8Length(input);
-        let done = 0;
-        let start = 0;
-        while (start < input.length) {
-            throwIfAborted(signal);
-            let end = Math.min(start + DECODE_SLICE, input.length);
-            const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
-            if (end < input.length && highSurrogate) {
-                end++; // never split a surrogate pair
-            }
-            const piece = start === 0 && end === input.length ? input : input.slice(start, end);
-            start = end;
-            const text = filter.push(piece);
-            if (text.length > 0) {
-                yield text;
-            }
-            if (onProgress !== null) {
-                done += utf8Length(piece);
-                onProgress(done, total);
-            }
-        }
-        if (input.length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* stringTextChunks(input, filter, options);
         return;
     }
     const decoder = new ByteDecoder(report, options);
-    const headBytes = Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
     if (input instanceof Uint8Array) {
-        const length = input.byteLength;
-        decoder.start(input.subarray(0, headBytes));
-        for (let offset = 0; offset < length; offset += DECODE_SLICE) {
-            throwIfAborted(signal);
-            if (input.byteLength !== length) {
-                throw new GraphFormatError("E_UNSUPPORTED", "the input's buffer was detached (transferred) or resized during the import", { reason: "detached" });
-            }
-            const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
-            const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
-            if (text.length > 0) {
-                yield text;
-            }
-            onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
-        }
-        if (length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* byteTextChunks(input, filter, decoder, options);
         return;
     }
-    // the first bytes of a stream are held back until the BOM check (and the declaration check,
-    // when the format has one) can see enough of them, or the input ended, or text arrived, so a
-    // BOM or a declaration split across small chunks is still seen
-    const needed = options.declaredEncoding === undefined ? BOM_BYTES : headBytes;
+    // the stream's loop stays here rather than in a delegated generator: a yield* layer costs
+    // time on every chunk, and a stream may arrive in many small ones
+    const stream = new StreamText(filter, decoder, options);
+    for await (const chunk of isReadableStream(input) ? streamChunks(input, signal) : input) {
+        throwIfAborted(signal);
+        const text = stream.push(chunk);
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(stream.done);
+    }
+    const tail = stream.end();
+    if (tail.length > 0) {
+        yield tail;
+    }
+    filter.end();
+    options.onProgress?.(stream.done, stream.done);
+}
+/**
+ * The end of the slice of a string input that starts at start: DECODE_SLICE code units, one more
+ * when the slice would end between the two halves of a surrogate pair.
+ * @param input - the string
+ * @param start - where the slice starts
+ * @returns where it ends
+ */
+function stringSliceEnd(input, start) {
+    const end = Math.min(start + DECODE_SLICE, input.length);
+    const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
+    return end < input.length && highSurrogate ? end + 1 : end;
+}
+/**
+ * textChunks() of a string input: slices of it, through the text filter.
+ * @param input - the string
+ * @param filter - the text checks
+ * @param options - cancellation and progress
+ * @yields the filtered slices
+ * @returns nothing
+ */
+function* stringTextChunks(input, filter, options) {
+    const signal = options.signal ?? null;
+    const onProgress = options.onProgress ?? null;
+    const total = onProgress === null ? 0 : utf8Length(input);
     let done = 0;
-    let head = [];
-    let headLength = 0;
-    const flushHead = (stream) => {
-        const bytes = concatBytes(head);
-        head = [];
-        headLength = 0;
-        decoder.start(bytes.subarray(0, headBytes));
-        return decoder.decode(bytes, stream);
-    };
-    const chunks = isReadableStream(input) ? streamChunks(input, signal) : input;
-    for await (const chunk of chunks) {
+    let start = 0;
+    while (start < input.length) {
         throwIfAborted(signal);
-        let text;
-        const bytes = typeof chunk === "string" ? null : asBytes(chunk);
-        if (typeof chunk === "string") {
-            // finish any byte sequence still pending in the decoder before switching to text; the
-            // BOM and declaration checks apply to the start of the input only, so bytes after text
-            // are never taken for a BOM
-            let pending = "";
-            if (headLength > 0) {
-                pending = decoder.flushBeforeText(() => flushHead(false));
-            }
-            else if (decoder.started) {
-                pending = decoder.flushBeforeText(() => decoder.decode(new Uint8Array(0), false));
-            }
-            else {
-                decoder.start(new Uint8Array(0));
-            }
-            text = filter.push(pending + chunk);
-            done += onProgress === null ? 0 : utf8Length(chunk);
+        const end = stringSliceEnd(input, start);
+        const piece = start === 0 && end === input.length ? input : input.slice(start, end);
+        start = end;
+        const text = filter.push(piece);
+        if (text.length > 0) {
+            yield text;
         }
-        else if (bytes !== null) {
-            done += bytes.byteLength;
-            if (decoder.started) {
-                text = filter.push(decoder.decode(bytes, true));
-            }
-            else {
-                head.push(bytes);
-                headLength += bytes.byteLength;
-                text = headLength >= needed ? filter.push(flushHead(true)) : "";
-            }
+        if (onProgress !== null) {
+            done += utf8Length(piece);
+            onProgress(done, total);
         }
-        else {
+    }
+    if (input.length === 0) {
+        onProgress?.(0, 0);
+    }
+    filter.end();
+}
+/**
+ * textChunks() of a Uint8Array input: slices of it, decoded, through the text filter.
+ * @param input - the bytes
+ * @param filter - the text checks
+ * @param decoder - the byte decoder
+ * @param options - cancellation, progress and the declaration size
+ * @yields the decoded, filtered slices
+ * @returns nothing
+ */
+function* byteTextChunks(input, filter, decoder, options) {
+    const signal = options.signal ?? null;
+    const length = input.byteLength;
+    decoder.start(input.subarray(0, headByteCount(options)));
+    for (let offset = 0; offset < length; offset += DECODE_SLICE) {
+        throwIfAborted(signal);
+        if (input.byteLength !== length) {
+            throw new GraphFormatError("E_UNSUPPORTED", "the input's buffer was detached (transferred) or resized during the import", { reason: "detached" });
+        }
+        const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
+        const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
+    }
+    if (length === 0) {
+        options.onProgress?.(0, 0);
+    }
+    filter.end();
+}
+/**
+ * How many leading bytes the encoding is chosen from: the format's declaration size, at least a BOM.
+ * @param options - the declaration size
+ * @returns the byte count
+ */
+function headByteCount(options) {
+    return Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
+}
+/**
+ * The chunks of a stream input (a ReadableStream or an async iterable of strings and bytes) turned
+ * into filtered text. The first bytes are held back until the BOM check (and the declaration
+ * check, when the format has one) can see enough of them, or the input ended, or text arrived, so
+ * a BOM or a declaration split across small chunks is still seen.
+ */
+class StreamText {
+    /**
+     * Create the reader of one stream input.
+     * @param filter - the text checks
+     * @param decoder - the byte decoder
+     * @param options - progress, the declaration reader and its size
+     */
+    constructor(filter, decoder, options) {
+        this.filter = filter;
+        this.decoder = decoder;
+        /** Bytes read so far (text counted as UTF-8 when progress is reported), for progress. */
+        this.done = 0;
+        this.head = [];
+        this.headLength = 0;
+        this.headBytes = headByteCount(options);
+        this.needed = options.declaredEncoding === undefined ? BOM_BYTES : this.headBytes;
+        this.counting = (options.onProgress ?? null) !== null;
+    }
+    /**
+     * Read the next chunk.
+     * @param chunk - a string or a Uint8Array (anything else is E_UNSUPPORTED)
+     * @returns its filtered text; empty while bytes are held back
+     */
+    push(chunk) {
+        if (typeof chunk === "string") {
+            this.done += this.counting ? utf8Length(chunk) : 0;
+            return this.filter.push(this.beforeText() + chunk);
+        }
+        const bytes = asBytes(chunk);
+        if (bytes === null) {
             throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
                 reason: "chunk type",
                 found: typeof chunk,
             });
         }
-        if (text.length > 0) {
-            yield text;
+        this.done += bytes.byteLength;
+        if (this.decoder.started) {
+            return this.filter.push(this.decoder.decode(bytes, true));
         }
-        onProgress?.(done);
+        this.head.push(bytes);
+        this.headLength += bytes.byteLength;
+        return this.headLength >= this.needed ? this.filter.push(this.flushHead(true)) : "";
     }
-    let tail = "";
-    if (headLength > 0) {
-        tail = flushHead(false);
+    /**
+     * The input ended: the filtered text of whatever is still held or pending.
+     * @returns the text
+     */
+    end() {
+        let tail = "";
+        if (this.headLength > 0) {
+            tail = this.flushHead(false);
+        }
+        else if (this.decoder.started) {
+            tail = this.decoder.decode(new Uint8Array(0), false);
+        }
+        return this.filter.push(tail);
     }
-    else if (decoder.started) {
-        tail = decoder.decode(new Uint8Array(0), false);
+    /**
+     * Start the decoder on the held bytes and decode them.
+     * @param stream - whether more bytes follow
+     * @returns their text
+     */
+    flushHead(stream) {
+        const bytes = concatBytes(this.head);
+        this.head = [];
+        this.headLength = 0;
+        this.decoder.start(bytes.subarray(0, this.headBytes));
+        return this.decoder.decode(bytes, stream);
     }
-    tail = filter.push(tail);
-    if (tail.length > 0) {
-        yield tail;
+    /**
+     * A text chunk follows: finish any byte sequence still pending in the decoder before switching
+     * to text. The BOM and declaration checks apply to the start of the input only, so bytes after
+     * text are never taken for a BOM.
+     * @returns the text of the pending bytes
+     */
+    beforeText() {
+        if (this.headLength > 0) {
+            return this.decoder.flushBeforeText(() => this.flushHead(false));
+        }
+        if (this.decoder.started) {
+            return this.decoder.flushBeforeText(() => this.decoder.decode(new Uint8Array(0), false));
+        }
+        this.decoder.start(new Uint8Array(0));
+        return "";
     }
-    filter.end();
-    onProgress?.(done, done);
 }
 /**
  * Whether an input is a ReadableStream (by duck type, so a stream from another realm qualifies).
@@ -1309,84 +1414,153 @@ export class LineReader {
      * @returns nothing
      */
     async *[Symbol.asyncIterator]() {
-        // The pieces of the line in progress (none of them holds a line break, except that the last
-        // may end with a `\r` whose meaning the next chunk decides); joined once when the line ends,
-        // so a line spanning many chunks costs its length, not its length times the chunk count.
-        const pending = [];
-        let pendingLength = 0;
+        // The pieces of the line in progress, joined once when the line ends, so a line spanning
+        // many chunks costs its length, not its length times the chunk count.
+        const pending = new PendingLine((line) => tooLarge(this.report, this.lineTooLong(), line));
         let trailingCr = false;
         for await (const chunk of textChunks(this.input, this.report, this.options)) {
             let start = 0;
-            const end = chunk.length;
             if (trailingCr) {
                 // the previous chunk ended with `\r`: that ended a line, and a leading `\n` here is
                 // the second half of the same terminator
                 trailingCr = false;
                 this.lineNumber++;
-                const joined = pending.join("");
-                pending.length = 0;
-                pendingLength = 0;
-                yield joined.slice(0, -1);
+                yield pending.take().slice(0, -1);
                 if (chunk.charCodeAt(0) === 10) {
                     start = 1;
                 }
             }
-            // the next `\n` and `\r` at or after `start`; each is searched for once per chunk and
-            // again only after it was consumed, so a chunk is scanned once whatever its line count
-            let nl = chunk.indexOf("\n", start);
-            let cr = chunk.indexOf("\r", start);
-            while (nl >= 0 || cr >= 0) {
-                let cut;
-                let next;
-                if (cr >= 0 && (nl < 0 || cr < nl)) {
-                    if (cr === end - 1) {
-                        // a trailing \r may be the first half of \r\n split across chunks
-                        break;
-                    }
-                    cut = cr;
-                    next = chunk.charCodeAt(cr + 1) === 10 ? cr + 2 : cr + 1;
-                }
-                else {
-                    cut = nl;
-                    next = nl + 1;
-                }
+            const breaks = new LineBreaks(chunk, start);
+            while (breaks.advance()) {
                 this.lineNumber++;
-                const piece = chunk.slice(start, cut);
-                if (pending.length === 0) {
-                    yield piece;
-                }
-                else {
-                    if (pendingLength + piece.length > MAX_TEXT_LENGTH) {
-                        tooLarge(this.report, this.lineTooLong(), this.lineNumber);
-                    }
-                    pending.push(piece);
-                    const joined = pending.join("");
-                    pending.length = 0;
-                    pendingLength = 0;
-                    yield joined;
-                }
-                start = next;
-                if (nl >= 0 && nl < next) {
-                    nl = chunk.indexOf("\n", next);
-                }
-                if (cr >= 0 && cr < next) {
-                    cr = chunk.indexOf("\r", next);
-                }
+                yield pending.finish(chunk.slice(start, breaks.cut), this.lineNumber);
+                start = breaks.next;
             }
-            if (start < end) {
-                pendingLength += end - start;
-                if (pendingLength > MAX_TEXT_LENGTH) {
-                    tooLarge(this.report, this.lineTooLong(), this.lineNumber + 1);
-                }
-                pending.push(start === 0 ? chunk : chunk.slice(start));
-                trailingCr = chunk.charCodeAt(end - 1) === 13;
+            if (start < chunk.length) {
+                pending.appendRest(chunk, start, this.lineNumber + 1);
+                trailingCr = chunk.codePointAt(chunk.length - 1) === 13;
             }
         }
-        if (pending.length > 0) {
+        if (!pending.empty) {
             this.lineNumber++;
-            const joined = pending.join("");
+            const joined = pending.take();
             yield joined.endsWith("\r") ? joined.slice(0, -1) : joined;
         }
     }
 }
+/**
+ * The pieces of a line that spans chunks. None of them holds a line break, except that the last
+ * may end with a `\r` whose meaning the next chunk decides.
+ */
+class PendingLine {
+    /**
+     * Start an empty line.
+     * @param tooLong - fails the import when the line grows longer than one string can hold
+     */
+    constructor(tooLong) {
+        this.pieces = [];
+        this.length = 0;
+        this.tooLong = tooLong;
+    }
+    /**
+     * Whether no piece is held.
+     * @returns true when the line in progress is empty
+     */
+    get empty() {
+        return this.pieces.length === 0;
+    }
+    /**
+     * Hold the rest of a chunk from `start`: the line goes on in the next chunk.
+     * @param chunk - the chunk
+     * @param start - where the unfinished line starts in it
+     * @param line - the 1-based number of the line, for the too-long error
+     */
+    appendRest(chunk, start, line) {
+        this.append(start === 0 ? chunk : chunk.slice(start), line);
+    }
+    /**
+     * End the line with its last piece.
+     * @param piece - the text from the start of the chunk (or the last break) to the line break
+     * @param line - the 1-based number of the line, for the too-long error
+     * @returns the whole line
+     */
+    finish(piece, line) {
+        if (this.pieces.length === 0) {
+            return piece;
+        }
+        this.append(piece, line);
+        return this.take();
+    }
+    /**
+     * Join the held pieces and start a new line.
+     * @returns the joined text
+     */
+    take() {
+        const joined = this.pieces.join("");
+        this.pieces.length = 0;
+        this.length = 0;
+        return joined;
+    }
+    /**
+     * Hold one more piece.
+     * @param piece - the piece
+     * @param line - the 1-based number of the line, for the too-long error
+     */
+    append(piece, line) {
+        this.length += piece.length;
+        if (this.length > MAX_TEXT_LENGTH) {
+            this.tooLong(line);
+        }
+        this.pieces.push(piece);
+    }
+}
+/**
+ * The line breaks of one chunk, in order: `\n`, `\r\n` and a lone `\r`. Each of `\n` and `\r` is
+ * searched for once per chunk and again only after it was consumed, so a chunk is scanned once
+ * whatever its line count.
+ */
+class LineBreaks {
+    /**
+     * Scan a chunk from a position.
+     * @param chunk - the chunk
+     * @param start - where scanning starts
+     */
+    constructor(chunk, start) {
+        /** Where the line just found ends (its terminator starts there). */
+        this.cut = 0;
+        this.chunk = chunk;
+        this.next = start;
+        this.nl = chunk.indexOf("\n", start);
+        this.cr = chunk.indexOf("\r", start);
+    }
+    /**
+     * Find the next line break.
+     * @returns false when none is left, or the only one left is a trailing `\r` (it may be the first
+     * half of a `\r\n` split across chunks)
+     */
+    advance() {
+        const { chunk, next } = this;
+        if (this.nl >= 0 && this.nl < next) {
+            this.nl = chunk.indexOf("\n", next);
+        }
+        if (this.cr >= 0 && this.cr < next) {
+            this.cr = chunk.indexOf("\r", next);
+        }
+        const { nl, cr } = this;
+        if (cr >= 0 && (nl < 0 || cr < nl)) {
+            if (cr === chunk.length - 1) {
+                return false;
+            }
+            this.cut = cr;
+            this.next = chunk.codePointAt(cr + 1) === 10 ? cr + 2 : cr + 1;
+            return true;
+        }
+        if (nl < 0) {
+            return false;
+        }
+        this.cut = nl;
+        this.next = nl + 1;
+        return true;
+    }
+}
 //# sourceMappingURL=input.js.map