@graphty/graph-io 0.3.39 → 0.3.41

package/src/common/input.ts

@@ -1232,130 +1232,244 @@ export async function* textChunks(
 ): AsyncGenerator<string, void, undefined> {
     const input = normalizeInput(rawInput);
     const signal = options.signal ?? null;
-    const onProgress = options.onProgress ?? null;
     const filter = new TextFilter(report, options);
     throwIfAborted(signal);
     if (typeof input === "string") {
         reportTextEncoding(report, options.encoding);
-        const total = onProgress === null ? 0 : utf8Length(input);
-        let done = 0;
-        let start = 0;
-        while (start < input.length) {
-            throwIfAborted(signal);
-            let end = Math.min(start + DECODE_SLICE, input.length);
-            const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
-            if (end < input.length && highSurrogate) {
-                end++; // never split a surrogate pair
-            }
-            const piece = start === 0 && end === input.length ? input : input.slice(start, end);
-            start = end;
-            const text = filter.push(piece);
-            if (text.length > 0) {
-                yield text;
-            }
-            if (onProgress !== null) {
-                done += utf8Length(piece);
-                onProgress(done, total);
-            }
-        }
-        if (input.length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* stringTextChunks(input, filter, options);
         return;
     }
     const decoder = new ByteDecoder(report, options);
-    const headBytes = Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
     if (input instanceof Uint8Array) {
-        const length = input.byteLength;
-        decoder.start(input.subarray(0, headBytes));
-        for (let offset = 0; offset < length; offset += DECODE_SLICE) {
-            throwIfAborted(signal);
-            if (input.byteLength !== length) {
-                throw new GraphFormatError(
-                    "E_UNSUPPORTED",
-                    "the input's buffer was detached (transferred) or resized during the import",
-                    { reason: "detached" },
-                );
-            }
-            const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
-            const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
-            if (text.length > 0) {
-                yield text;
-            }
-            onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
-        }
-        if (length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* byteTextChunks(input, filter, decoder, options);
         return;
     }
-    // the first bytes of a stream are held back until the BOM check (and the declaration check,
-    // when the format has one) can see enough of them, or the input ended, or text arrived, so a
-    // BOM or a declaration split across small chunks is still seen
-    const needed = options.declaredEncoding === undefined ? BOM_BYTES : headBytes;
+    // the stream's loop stays here rather than in a delegated generator: a yield* layer costs
+    // time on every chunk, and a stream may arrive in many small ones
+    const stream = new StreamText(filter, decoder, options);
+    for await (const chunk of isReadableStream(input) ? streamChunks(input, signal) : input) {
+        throwIfAborted(signal);
+        const text = stream.push(chunk);
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(stream.done);
+    }
+    const tail = stream.end();
+    if (tail.length > 0) {
+        yield tail;
+    }
+    filter.end();
+    options.onProgress?.(stream.done, stream.done);
+}
+
+/**
+ * The end of the slice of a string input that starts at start: DECODE_SLICE code units, one more
+ * when the slice would end between the two halves of a surrogate pair.
+ * @param input - the string
+ * @param start - where the slice starts
+ * @returns where it ends
+ */
+function stringSliceEnd(input: string, start: number): number {
+    const end = Math.min(start + DECODE_SLICE, input.length);
+    const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
+    return end < input.length && highSurrogate ? end + 1 : end;
+}
+
+/**
+ * textChunks() of a string input: slices of it, through the text filter.
+ * @param input - the string
+ * @param filter - the text checks
+ * @param options - cancellation and progress
+ * @yields the filtered slices
+ * @returns nothing
+ */
+function* stringTextChunks(
+    input: string,
+    filter: TextFilter,
+    options: ReadOptions,
+): Generator<string, void, undefined> {
+    const signal = options.signal ?? null;
+    const onProgress = options.onProgress ?? null;
+    const total = onProgress === null ? 0 : utf8Length(input);
     let done = 0;
-    let head: Uint8Array[] = [];
-    let headLength = 0;
-    const flushHead = (stream: boolean): string => {
-        const bytes = concatBytes(head);
-        head = [];
-        headLength = 0;
-        decoder.start(bytes.subarray(0, headBytes));
-        return decoder.decode(bytes, stream);
-    };
-    const chunks = isReadableStream(input) ? streamChunks(input, signal) : input;
-    for await (const chunk of chunks) {
+    let start = 0;
+    while (start < input.length) {
         throwIfAborted(signal);
-        let text: string;
-        const bytes = typeof chunk === "string" ? null : asBytes(chunk);
+        const end = stringSliceEnd(input, start);
+        const piece = start === 0 && end === input.length ? input : input.slice(start, end);
+        start = end;
+        const text = filter.push(piece);
+        if (text.length > 0) {
+            yield text;
+        }
+        if (onProgress !== null) {
+            done += utf8Length(piece);
+            onProgress(done, total);
+        }
+    }
+    if (input.length === 0) {
+        onProgress?.(0, 0);
+    }
+    filter.end();
+}
+
+/**
+ * textChunks() of a Uint8Array input: slices of it, decoded, through the text filter.
+ * @param input - the bytes
+ * @param filter - the text checks
+ * @param decoder - the byte decoder
+ * @param options - cancellation, progress and the declaration size
+ * @yields the decoded, filtered slices
+ * @returns nothing
+ */
+function* byteTextChunks(
+    input: Uint8Array,
+    filter: TextFilter,
+    decoder: ByteDecoder,
+    options: ReadOptions,
+): Generator<string, void, undefined> {
+    const signal = options.signal ?? null;
+    const length = input.byteLength;
+    decoder.start(input.subarray(0, headByteCount(options)));
+    for (let offset = 0; offset < length; offset += DECODE_SLICE) {
+        throwIfAborted(signal);
+        if (input.byteLength !== length) {
+            throw new GraphFormatError(
+                "E_UNSUPPORTED",
+                "the input's buffer was detached (transferred) or resized during the import",
+                { reason: "detached" },
+            );
+        }
+        const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
+        const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
+    }
+    if (length === 0) {
+        options.onProgress?.(0, 0);
+    }
+    filter.end();
+}
+
+/**
+ * How many leading bytes the encoding is chosen from: the format's declaration size, at least a BOM.
+ * @param options - the declaration size
+ * @returns the byte count
+ */
+function headByteCount(options: ReadOptions): number {
+    return Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
+}
+
+/**
+ * The chunks of a stream input (a ReadableStream or an async iterable of strings and bytes) turned
+ * into filtered text. The first bytes are held back until the BOM check (and the declaration
+ * check, when the format has one) can see enough of them, or the input ended, or text arrived, so
+ * a BOM or a declaration split across small chunks is still seen.
+ */
+class StreamText {
+    /** Bytes read so far (text counted as UTF-8 when progress is reported), for progress. */
+    done = 0;
+
+    private head: Uint8Array[] = [];
+
+    private headLength = 0;
+
+    private readonly headBytes: number;
+
+    /** How many bytes to hold back before choosing the encoding. */
+    private readonly needed: number;
+
+    /** Whether progress is reported, so text chunks are measured. */
+    private readonly counting: boolean;
+
+    /**
+     * Create the reader of one stream input.
+     * @param filter - the text checks
+     * @param decoder - the byte decoder
+     * @param options - progress, the declaration reader and its size
+     */
+    constructor(
+        private readonly filter: TextFilter,
+        private readonly decoder: ByteDecoder,
+        options: ReadOptions,
+    ) {
+        this.headBytes = headByteCount(options);
+        this.needed = options.declaredEncoding === undefined ? BOM_BYTES : this.headBytes;
+        this.counting = (options.onProgress ?? null) !== null;
+    }
+
+    /**
+     * Read the next chunk.
+     * @param chunk - a string or a Uint8Array (anything else is E_UNSUPPORTED)
+     * @returns its filtered text; empty while bytes are held back
+     */
+    push(chunk: unknown): string {
         if (typeof chunk === "string") {
-            // finish any byte sequence still pending in the decoder before switching to text; the
-            // BOM and declaration checks apply to the start of the input only, so bytes after text
-            // are never taken for a BOM
-            let pending = "";
-            if (headLength > 0) {
-                pending = decoder.flushBeforeText(() => flushHead(false));
-            } else if (decoder.started) {
-                pending = decoder.flushBeforeText(() => decoder.decode(new Uint8Array(0), false));
-            } else {
-                decoder.start(new Uint8Array(0));
-            }
-            text = filter.push(pending + chunk);
-            done += onProgress === null ? 0 : utf8Length(chunk);
-        } else if (bytes !== null) {
-            done += bytes.byteLength;
-            if (decoder.started) {
-                text = filter.push(decoder.decode(bytes, true));
-            } else {
-                head.push(bytes);
-                headLength += bytes.byteLength;
-                text = headLength >= needed ? filter.push(flushHead(true)) : "";
-            }
-        } else {
+            this.done += this.counting ? utf8Length(chunk) : 0;
+            return this.filter.push(this.beforeText() + chunk);
+        }
+        const bytes = asBytes(chunk);
+        if (bytes === null) {
             throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
                 reason: "chunk type",
                 found: typeof chunk,
             });
         }
-        if (text.length > 0) {
-            yield text;
+        this.done += bytes.byteLength;
+        if (this.decoder.started) {
+            return this.filter.push(this.decoder.decode(bytes, true));
         }
-        onProgress?.(done);
+        this.head.push(bytes);
+        this.headLength += bytes.byteLength;
+        return this.headLength >= this.needed ? this.filter.push(this.flushHead(true)) : "";
     }
-    let tail = "";
-    if (headLength > 0) {
-        tail = flushHead(false);
-    } else if (decoder.started) {
-        tail = decoder.decode(new Uint8Array(0), false);
+
+    /**
+     * The input ended: the filtered text of whatever is still held or pending.
+     * @returns the text
+     */
+    end(): string {
+        let tail = "";
+        if (this.headLength > 0) {
+            tail = this.flushHead(false);
+        } else if (this.decoder.started) {
+            tail = this.decoder.decode(new Uint8Array(0), false);
+        }
+        return this.filter.push(tail);
     }
-    tail = filter.push(tail);
-    if (tail.length > 0) {
-        yield tail;
+
+    /**
+     * Start the decoder on the held bytes and decode them.
+     * @param stream - whether more bytes follow
+     * @returns their text
+     */
+    private flushHead(stream: boolean): string {
+        const bytes = concatBytes(this.head);
+        this.head = [];
+        this.headLength = 0;
+        this.decoder.start(bytes.subarray(0, this.headBytes));
+        return this.decoder.decode(bytes, stream);
+    }
+
+    /**
+     * A text chunk follows: finish any byte sequence still pending in the decoder before switching
+     * to text. The BOM and declaration checks apply to the start of the input only, so bytes after
+     * text are never taken for a BOM.
+     * @returns the text of the pending bytes
+     */
+    private beforeText(): string {
+        if (this.headLength > 0) {
+            return this.decoder.flushBeforeText(() => this.flushHead(false));
+        }
+        if (this.decoder.started) {
+            return this.decoder.flushBeforeText(() => this.decoder.decode(new Uint8Array(0), false));
+        }
+        this.decoder.start(new Uint8Array(0));
+        return "";
     }
-    filter.end();
-    onProgress?.(done, done);
 }
 
 /**

package/src/common/json-elements.ts

@@ -1493,10 +1493,11 @@ async function* blockElements(
  * @category Plugin helpers
  */
 export function reportTooDeep(report: ImportReportBuilder, event: Extract<AspectEvent, { kind: "deep" }>): void {
+    const what = event.aspect === null ? "a member of the document" : `an element of "${event.aspect}"`;
     report.error(
         "parse-error",
         BAD_ASPECT_BLOCK_CODE,
-        `${event.aspect === null ? "a member of the document" : `an element of "${event.aspect}"`} is nested ${event.depth} levels deep, more than the ${MAX_ELEMENT_DEPTH} a graph keeps; skipped`,
+        `${what} is nested ${event.depth} levels deep, more than the ${MAX_ELEMENT_DEPTH} a graph keeps; skipped`,
         { line: event.line, element: event.aspect },
     );
 }
@@ -1893,7 +1894,8 @@ export class CxStructure {
         }
         const error = typeof value.error === "string" ? value.error : "";
         if (value.success === false) {
-            const message = `the producer marked the document as failed${error.length > 0 ? `: ${error}` : ""}`;
+            const detail = error.length > 0 ? `: ${error}` : "";
+            const message = `the producer marked the document as failed${detail}`;
             this.report.error("validation-error", STATUS_FAILED_CODE, message, { line, element: "status" });
             throw this.report.abort(message, { code: STATUS_FAILED_CODE });
         }

package/src/common/text.ts

@@ -202,21 +202,7 @@ export class TextCellWriter {
     write(row: number, text: string): void {
         const kind = inferTextDtype(text);
         const value = parseTextCell(text);
-        if (
-            typeof value === "number" &&
-            !Number.isSafeInteger(value) &&
-            I32_TEXT.test(text) &&
-            BigInt(text) !== BigInt(value)
-        ) {
-            // an integer beyond 2^53 is stored as the nearest double; say so once per column
-            this.report.warnOnce(
-                "precision",
-                PRECISION_CODE,
-                `${this.domain} column "${this.name}": integers beyond 2^53 are stored as the nearest double (first: ${text})`,
-                { element: this.name },
-                `${PRECISION_CODE}:${this.domain}:${this.name}`,
-            );
-        }
+        this.warnIfImprecise(text, value);
         const wasString = this.textDtype === "string";
         const textDtype =
             this.textDtype === null || textDtypeRank(kind) > textDtypeRank(this.textDtype) ? kind : this.textDtype;
@@ -250,6 +236,29 @@ export class TextCellWriter {
         }
     }
 
+    /**
+     * Warn, once per column, when the cell is an integer beyond 2^53 that is stored as the nearest
+     * double.
+     * @param text - the cell text
+     * @param value - the parsed value
+     */
+    private warnIfImprecise(text: string, value: unknown): void {
+        if (
+            typeof value === "number" &&
+            !Number.isSafeInteger(value) &&
+            I32_TEXT.test(text) &&
+            BigInt(text) !== BigInt(value)
+        ) {
+            this.report.warnOnce(
+                "precision",
+                PRECISION_CODE,
+                `${this.domain} column "${this.name}": integers beyond 2^53 are stored as the nearest double (first: ${text})`,
+                { element: this.name },
+                `${PRECISION_CODE}:${this.domain}:${this.name}`,
+            );
+        }
+    }
+
     /**
      * Write a value (the parsed scalar, or a text into a widened column).
      * @param row - the row

package/src/common/xml.ts

@@ -416,7 +416,8 @@ export function decodeEntities(raw: string, line: number, repairs?: XmlRepairs):
             const digits = name.slice(hex ? 2 : 1);
             // XML 1.0 puts no limit on leading zeros (A is "A")
             const ok = hex ? /^0*[\da-fA-F]{1,6}$/.test(digits) : /^0*\d{1,7}$/.test(digits);
-            const cp = ok ? Number.parseInt(digits, hex ? 16 : 10) : -1;
+            const radix = hex ? 16 : 10;
+            const cp = ok ? Number.parseInt(digits, radix) : -1;
             const low = cp >= 0xd800 && cp <= 0xdbff ? lowSurrogateAfter(raw, semi + 1, repairs) : null;
             if (low !== null) {
                 repairs?.surrogatePair?.(lineAt(raw, amp, line));

package/src/common/zip.ts

@@ -134,6 +134,52 @@ export function startsLikeZip(bytes: Uint8Array): boolean {
     return bytes.byteLength >= 4 && bytes[0] === 0x50 && bytes[1] === 0x4b && bytes[2] === 3 && bytes[3] === 4;
 }
 
+/**
+ * Read the end of central directory record, following the zip64 locator when there is one.
+ * @param view - the whole archive
+ * @param eocd - the offset of the end record
+ * @returns the disk numbers, the entry count and where the central directory is; ZipError
+ * "corrupt" when the end record needs a zip64 record the archive does not have
+ */
+function readEndRecord(
+    view: DataView,
+    eocd: number,
+): {
+    disk: number;
+    directoryDisk: number;
+    count: number;
+    directorySize: number;
+    directoryOffset: number;
+    directoryEnd: number;
+} {
+    const locator = eocd - ZIP64_LOCATOR_SIZE;
+    if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIGNATURE) {
+        const record = zip64EndRecord(view, locator);
+        return {
+            disk: view.getUint32(record + 16, true),
+            directoryDisk: view.getUint32(record + 20, true),
+            count: readU64(view, record + 32),
+            directorySize: readU64(view, record + 40),
+            directoryOffset: readU64(view, record + 48),
+            directoryEnd: record,
+        };
+    }
+    const count = view.getUint16(eocd + 10, true);
+    const directorySize = view.getUint32(eocd + 12, true);
+    const directoryOffset = view.getUint32(eocd + 16, true);
+    if (count === U16_MAX || directorySize === U32_MAX || directoryOffset === U32_MAX) {
+        throw new ZipError("corrupt", "the end record needs a zip64 record the archive does not have");
+    }
+    return {
+        disk: view.getUint16(eocd + 4, true),
+        directoryDisk: view.getUint16(eocd + 6, true),
+        count,
+        directorySize,
+        directoryOffset,
+        directoryEnd: eocd,
+    };
+}
+
 /**
  * Read the central directory.
  * @param bytes - the whole archive
@@ -150,24 +196,7 @@ export function readZipDirectory(bytes: Uint8Array): ZipEntry[] {
             ? new ZipError("corrupt", "the archive has no end of central directory record (truncated?)")
             : new ZipError("not-zip", "the input is not a zip archive");
     }
-    let disk = view.getUint16(eocd + 4, true);
-    let directoryDisk = view.getUint16(eocd + 6, true);
-    let count = view.getUint16(eocd + 10, true);
-    let directorySize = view.getUint32(eocd + 12, true);
-    let directoryOffset = view.getUint32(eocd + 16, true);
-    let directoryEnd = eocd;
-    const locator = eocd - ZIP64_LOCATOR_SIZE;
-    if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIGNATURE) {
-        const record = zip64EndRecord(view, locator);
-        disk = view.getUint32(record + 16, true);
-        directoryDisk = view.getUint32(record + 20, true);
-        count = readU64(view, record + 32);
-        directorySize = readU64(view, record + 40);
-        directoryOffset = readU64(view, record + 48);
-        directoryEnd = record;
-    } else if (count === U16_MAX || directorySize === U32_MAX || directoryOffset === U32_MAX) {
-        throw new ZipError("corrupt", "the end record needs a zip64 record the archive does not have");
-    }
+    const { disk, directoryDisk, count, directorySize, directoryOffset, directoryEnd } = readEndRecord(view, eocd);
     if (disk !== 0 || directoryDisk !== 0) {
         throw new ZipError("unsupported", "a split (multi-disk) archive cannot be read");
     }