@graphty/graph-io 0.3.40 → 0.3.41

package/src/common/input.ts

@@ -1232,130 +1232,244 @@ export async function* textChunks(
 ): AsyncGenerator<string, void, undefined> {
     const input = normalizeInput(rawInput);
     const signal = options.signal ?? null;
-    const onProgress = options.onProgress ?? null;
     const filter = new TextFilter(report, options);
     throwIfAborted(signal);
     if (typeof input === "string") {
         reportTextEncoding(report, options.encoding);
-        const total = onProgress === null ? 0 : utf8Length(input);
-        let done = 0;
-        let start = 0;
-        while (start < input.length) {
-            throwIfAborted(signal);
-            let end = Math.min(start + DECODE_SLICE, input.length);
-            const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
-            if (end < input.length && highSurrogate) {
-                end++; // never split a surrogate pair
-            }
-            const piece = start === 0 && end === input.length ? input : input.slice(start, end);
-            start = end;
-            const text = filter.push(piece);
-            if (text.length > 0) {
-                yield text;
-            }
-            if (onProgress !== null) {
-                done += utf8Length(piece);
-                onProgress(done, total);
-            }
-        }
-        if (input.length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* stringTextChunks(input, filter, options);
         return;
     }
     const decoder = new ByteDecoder(report, options);
-    const headBytes = Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
     if (input instanceof Uint8Array) {
-        const length = input.byteLength;
-        decoder.start(input.subarray(0, headBytes));
-        for (let offset = 0; offset < length; offset += DECODE_SLICE) {
-            throwIfAborted(signal);
-            if (input.byteLength !== length) {
-                throw new GraphFormatError(
-                    "E_UNSUPPORTED",
-                    "the input's buffer was detached (transferred) or resized during the import",
-                    { reason: "detached" },
-                );
-            }
-            const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
-            const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
-            if (text.length > 0) {
-                yield text;
-            }
-            onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
-        }
-        if (length === 0) {
-            onProgress?.(0, 0);
-        }
-        filter.end();
+        yield* byteTextChunks(input, filter, decoder, options);
         return;
     }
-    // the first bytes of a stream are held back until the BOM check (and the declaration check,
-    // when the format has one) can see enough of them, or the input ended, or text arrived, so a
-    // BOM or a declaration split across small chunks is still seen
-    const needed = options.declaredEncoding === undefined ? BOM_BYTES : headBytes;
+    // the stream's loop stays here rather than in a delegated generator: a yield* layer costs
+    // time on every chunk, and a stream may arrive in many small ones
+    const stream = new StreamText(filter, decoder, options);
+    for await (const chunk of isReadableStream(input) ? streamChunks(input, signal) : input) {
+        throwIfAborted(signal);
+        const text = stream.push(chunk);
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(stream.done);
+    }
+    const tail = stream.end();
+    if (tail.length > 0) {
+        yield tail;
+    }
+    filter.end();
+    options.onProgress?.(stream.done, stream.done);
+}
+
+/**
+ * The end of the slice of a string input that starts at start: DECODE_SLICE code units, one more
+ * when the slice would end between the two halves of a surrogate pair.
+ * @param input - the string
+ * @param start - where the slice starts
+ * @returns where it ends
+ */
+function stringSliceEnd(input: string, start: number): number {
+    const end = Math.min(start + DECODE_SLICE, input.length);
+    const highSurrogate = (input.charCodeAt(end - 1) & 0xfc00) === 0xd800; // NOSONAR(S7758): reads UTF-16 code units on purpose
+    return end < input.length && highSurrogate ? end + 1 : end;
+}
+
+/**
+ * textChunks() of a string input: slices of it, through the text filter.
+ * @param input - the string
+ * @param filter - the text checks
+ * @param options - cancellation and progress
+ * @yields the filtered slices
+ * @returns nothing
+ */
+function* stringTextChunks(
+    input: string,
+    filter: TextFilter,
+    options: ReadOptions,
+): Generator<string, void, undefined> {
+    const signal = options.signal ?? null;
+    const onProgress = options.onProgress ?? null;
+    const total = onProgress === null ? 0 : utf8Length(input);
     let done = 0;
-    let head: Uint8Array[] = [];
-    let headLength = 0;
-    const flushHead = (stream: boolean): string => {
-        const bytes = concatBytes(head);
-        head = [];
-        headLength = 0;
-        decoder.start(bytes.subarray(0, headBytes));
-        return decoder.decode(bytes, stream);
-    };
-    const chunks = isReadableStream(input) ? streamChunks(input, signal) : input;
-    for await (const chunk of chunks) {
+    let start = 0;
+    while (start < input.length) {
         throwIfAborted(signal);
-        let text: string;
-        const bytes = typeof chunk === "string" ? null : asBytes(chunk);
+        const end = stringSliceEnd(input, start);
+        const piece = start === 0 && end === input.length ? input : input.slice(start, end);
+        start = end;
+        const text = filter.push(piece);
+        if (text.length > 0) {
+            yield text;
+        }
+        if (onProgress !== null) {
+            done += utf8Length(piece);
+            onProgress(done, total);
+        }
+    }
+    if (input.length === 0) {
+        onProgress?.(0, 0);
+    }
+    filter.end();
+}
+
+/**
+ * textChunks() of a Uint8Array input: slices of it, decoded, through the text filter.
+ * @param input - the bytes
+ * @param filter - the text checks
+ * @param decoder - the byte decoder
+ * @param options - cancellation, progress and the declaration size
+ * @yields the decoded, filtered slices
+ * @returns nothing
+ */
+function* byteTextChunks(
+    input: Uint8Array,
+    filter: TextFilter,
+    decoder: ByteDecoder,
+    options: ReadOptions,
+): Generator<string, void, undefined> {
+    const signal = options.signal ?? null;
+    const length = input.byteLength;
+    decoder.start(input.subarray(0, headByteCount(options)));
+    for (let offset = 0; offset < length; offset += DECODE_SLICE) {
+        throwIfAborted(signal);
+        if (input.byteLength !== length) {
+            throw new GraphFormatError(
+                "E_UNSUPPORTED",
+                "the input's buffer was detached (transferred) or resized during the import",
+                { reason: "detached" },
+            );
+        }
+        const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, length));
+        const text = filter.push(decoder.decode(slice, offset + DECODE_SLICE < length));
+        if (text.length > 0) {
+            yield text;
+        }
+        options.onProgress?.(Math.min(offset + DECODE_SLICE, length), length);
+    }
+    if (length === 0) {
+        options.onProgress?.(0, 0);
+    }
+    filter.end();
+}
+
+/**
+ * How many leading bytes the encoding is chosen from: the format's declaration size, at least a BOM.
+ * @param options - the declaration size
+ * @returns the byte count
+ */
+function headByteCount(options: ReadOptions): number {
+    return Math.max(options.declarationBytes ?? HEAD_BYTES, BOM_BYTES);
+}
+
+/**
+ * The chunks of a stream input (a ReadableStream or an async iterable of strings and bytes) turned
+ * into filtered text. The first bytes are held back until the BOM check (and the declaration
+ * check, when the format has one) can see enough of them, or the input ended, or text arrived, so
+ * a BOM or a declaration split across small chunks is still seen.
+ */
+class StreamText {
+    /** Bytes read so far (text counted as UTF-8 when progress is reported), for progress. */
+    done = 0;
+
+    private head: Uint8Array[] = [];
+
+    private headLength = 0;
+
+    private readonly headBytes: number;
+
+    /** How many bytes to hold back before choosing the encoding. */
+    private readonly needed: number;
+
+    /** Whether progress is reported, so text chunks are measured. */
+    private readonly counting: boolean;
+
+    /**
+     * Create the reader of one stream input.
+     * @param filter - the text checks
+     * @param decoder - the byte decoder
+     * @param options - progress, the declaration reader and its size
+     */
+    constructor(
+        private readonly filter: TextFilter,
+        private readonly decoder: ByteDecoder,
+        options: ReadOptions,
+    ) {
+        this.headBytes = headByteCount(options);
+        this.needed = options.declaredEncoding === undefined ? BOM_BYTES : this.headBytes;
+        this.counting = (options.onProgress ?? null) !== null;
+    }
+
+    /**
+     * Read the next chunk.
+     * @param chunk - a string or a Uint8Array (anything else is E_UNSUPPORTED)
+     * @returns its filtered text; empty while bytes are held back
+     */
+    push(chunk: unknown): string {
         if (typeof chunk === "string") {
-            // finish any byte sequence still pending in the decoder before switching to text; the
-            // BOM and declaration checks apply to the start of the input only, so bytes after text
-            // are never taken for a BOM
-            let pending = "";
-            if (headLength > 0) {
-                pending = decoder.flushBeforeText(() => flushHead(false));
-            } else if (decoder.started) {
-                pending = decoder.flushBeforeText(() => decoder.decode(new Uint8Array(0), false));
-            } else {
-                decoder.start(new Uint8Array(0));
-            }
-            text = filter.push(pending + chunk);
-            done += onProgress === null ? 0 : utf8Length(chunk);
-        } else if (bytes !== null) {
-            done += bytes.byteLength;
-            if (decoder.started) {
-                text = filter.push(decoder.decode(bytes, true));
-            } else {
-                head.push(bytes);
-                headLength += bytes.byteLength;
-                text = headLength >= needed ? filter.push(flushHead(true)) : "";
-            }
-        } else {
+            this.done += this.counting ? utf8Length(chunk) : 0;
+            return this.filter.push(this.beforeText() + chunk);
+        }
+        const bytes = asBytes(chunk);
+        if (bytes === null) {
             throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
                 reason: "chunk type",
                 found: typeof chunk,
             });
         }
-        if (text.length > 0) {
-            yield text;
+        this.done += bytes.byteLength;
+        if (this.decoder.started) {
+            return this.filter.push(this.decoder.decode(bytes, true));
         }
-        onProgress?.(done);
+        this.head.push(bytes);
+        this.headLength += bytes.byteLength;
+        return this.headLength >= this.needed ? this.filter.push(this.flushHead(true)) : "";
     }
-    let tail = "";
-    if (headLength > 0) {
-        tail = flushHead(false);
-    } else if (decoder.started) {
-        tail = decoder.decode(new Uint8Array(0), false);
+
+    /**
+     * The input ended: the filtered text of whatever is still held or pending.
+     * @returns the text
+     */
+    end(): string {
+        let tail = "";
+        if (this.headLength > 0) {
+            tail = this.flushHead(false);
+        } else if (this.decoder.started) {
+            tail = this.decoder.decode(new Uint8Array(0), false);
+        }
+        return this.filter.push(tail);
     }
-    tail = filter.push(tail);
-    if (tail.length > 0) {
-        yield tail;
+
+    /**
+     * Start the decoder on the held bytes and decode them.
+     * @param stream - whether more bytes follow
+     * @returns their text
+     */
+    private flushHead(stream: boolean): string {
+        const bytes = concatBytes(this.head);
+        this.head = [];
+        this.headLength = 0;
+        this.decoder.start(bytes.subarray(0, this.headBytes));
+        return this.decoder.decode(bytes, stream);
+    }
+
+    /**
+     * A text chunk follows: finish any byte sequence still pending in the decoder before switching
+     * to text. The BOM and declaration checks apply to the start of the input only, so bytes after
+     * text are never taken for a BOM.
+     * @returns the text of the pending bytes
+     */
+    private beforeText(): string {
+        if (this.headLength > 0) {
+            return this.decoder.flushBeforeText(() => this.flushHead(false));
+        }
+        if (this.decoder.started) {
+            return this.decoder.flushBeforeText(() => this.decoder.decode(new Uint8Array(0), false));
+        }
+        this.decoder.start(new Uint8Array(0));
+        return "";
     }
-    filter.end();
-    onProgress?.(done, done);
 }
 
 /**

package/src/common/text.ts

@@ -202,21 +202,7 @@ export class TextCellWriter {
     write(row: number, text: string): void {
         const kind = inferTextDtype(text);
         const value = parseTextCell(text);
-        if (
-            typeof value === "number" &&
-            !Number.isSafeInteger(value) &&
-            I32_TEXT.test(text) &&
-            BigInt(text) !== BigInt(value)
-        ) {
-            // an integer beyond 2^53 is stored as the nearest double; say so once per column
-            this.report.warnOnce(
-                "precision",
-                PRECISION_CODE,
-                `${this.domain} column "${this.name}": integers beyond 2^53 are stored as the nearest double (first: ${text})`,
-                { element: this.name },
-                `${PRECISION_CODE}:${this.domain}:${this.name}`,
-            );
-        }
+        this.warnIfImprecise(text, value);
         const wasString = this.textDtype === "string";
         const textDtype =
             this.textDtype === null || textDtypeRank(kind) > textDtypeRank(this.textDtype) ? kind : this.textDtype;
@@ -250,6 +236,29 @@ export class TextCellWriter {
         }
     }
 
+    /**
+     * Warn, once per column, when the cell is an integer beyond 2^53 that is stored as the nearest
+     * double.
+     * @param text - the cell text
+     * @param value - the parsed value
+     */
+    private warnIfImprecise(text: string, value: unknown): void {
+        if (
+            typeof value === "number" &&
+            !Number.isSafeInteger(value) &&
+            I32_TEXT.test(text) &&
+            BigInt(text) !== BigInt(value)
+        ) {
+            this.report.warnOnce(
+                "precision",
+                PRECISION_CODE,
+                `${this.domain} column "${this.name}": integers beyond 2^53 are stored as the nearest double (first: ${text})`,
+                { element: this.name },
+                `${PRECISION_CODE}:${this.domain}:${this.name}`,
+            );
+        }
+    }
+
     /**
      * Write a value (the parsed scalar, or a text into a widened column).
      * @param row - the row

package/src/common/zip.ts

@@ -134,6 +134,52 @@ export function startsLikeZip(bytes: Uint8Array): boolean {
     return bytes.byteLength >= 4 && bytes[0] === 0x50 && bytes[1] === 0x4b && bytes[2] === 3 && bytes[3] === 4;
 }
 
+/**
+ * Read the end of central directory record, following the zip64 locator when there is one.
+ * @param view - the whole archive
+ * @param eocd - the offset of the end record
+ * @returns the disk numbers, the entry count and where the central directory is; ZipError
+ * "corrupt" when the end record needs a zip64 record the archive does not have
+ */
+function readEndRecord(
+    view: DataView,
+    eocd: number,
+): {
+    disk: number;
+    directoryDisk: number;
+    count: number;
+    directorySize: number;
+    directoryOffset: number;
+    directoryEnd: number;
+} {
+    const locator = eocd - ZIP64_LOCATOR_SIZE;
+    if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIGNATURE) {
+        const record = zip64EndRecord(view, locator);
+        return {
+            disk: view.getUint32(record + 16, true),
+            directoryDisk: view.getUint32(record + 20, true),
+            count: readU64(view, record + 32),
+            directorySize: readU64(view, record + 40),
+            directoryOffset: readU64(view, record + 48),
+            directoryEnd: record,
+        };
+    }
+    const count = view.getUint16(eocd + 10, true);
+    const directorySize = view.getUint32(eocd + 12, true);
+    const directoryOffset = view.getUint32(eocd + 16, true);
+    if (count === U16_MAX || directorySize === U32_MAX || directoryOffset === U32_MAX) {
+        throw new ZipError("corrupt", "the end record needs a zip64 record the archive does not have");
+    }
+    return {
+        disk: view.getUint16(eocd + 4, true),
+        directoryDisk: view.getUint16(eocd + 6, true),
+        count,
+        directorySize,
+        directoryOffset,
+        directoryEnd: eocd,
+    };
+}
+
 /**
  * Read the central directory.
  * @param bytes - the whole archive
@@ -150,24 +196,7 @@ export function readZipDirectory(bytes: Uint8Array): ZipEntry[] {
             ? new ZipError("corrupt", "the archive has no end of central directory record (truncated?)")
             : new ZipError("not-zip", "the input is not a zip archive");
     }
-    let disk = view.getUint16(eocd + 4, true);
-    let directoryDisk = view.getUint16(eocd + 6, true);
-    let count = view.getUint16(eocd + 10, true);
-    let directorySize = view.getUint32(eocd + 12, true);
-    let directoryOffset = view.getUint32(eocd + 16, true);
-    let directoryEnd = eocd;
-    const locator = eocd - ZIP64_LOCATOR_SIZE;
-    if (locator >= 0 && view.getUint32(locator, true) === ZIP64_LOCATOR_SIGNATURE) {
-        const record = zip64EndRecord(view, locator);
-        disk = view.getUint32(record + 16, true);
-        directoryDisk = view.getUint32(record + 20, true);
-        count = readU64(view, record + 32);
-        directorySize = readU64(view, record + 40);
-        directoryOffset = readU64(view, record + 48);
-        directoryEnd = record;
-    } else if (count === U16_MAX || directorySize === U32_MAX || directoryOffset === U32_MAX) {
-        throw new ZipError("corrupt", "the end record needs a zip64 record the archive does not have");
-    }
+    const { disk, directoryDisk, count, directorySize, directoryOffset, directoryEnd } = readEndRecord(view, eocd);
     if (disk !== 0 || directoryDisk !== 0) {
         throw new ZipError("unsupported", "a split (multi-disk) archive cannot be read");
     }

package/src/formats/csv/exporter.ts

@@ -407,6 +407,25 @@ function nodeTableNote(names: readonly string[], table: string, notes: LossNote[
     }
 }
 
+/**
+ * The number of node ids whose text an earlier id already has (a number and a string).
+ * @param ids - the snapshot's ids
+ * @returns the collisions
+ */
+function idTextCollisions(ids: GraphSnapshot["ids"]): number {
+    const seen = new Set<string>();
+    let collisions = 0;
+    for (let i = 0; i < ids.size; i++) {
+        const text = String(ids.idOf(i));
+        if (seen.has(text)) {
+            collisions++;
+        } else {
+            seen.add(text);
+        }
+    }
+    return collisions;
+}
+
 /**
  * The id notes: text collisions are fatal (export() throws E_INVALID_ID); type changes under the
  * canonical re-read are reported.
@@ -421,16 +440,7 @@ function idNotes(
 ): void {
     const { ids } = snapshot;
     if (ids.kind === "mixed") {
-        const seen = new Set<string>();
-        let collisions = 0;
-        for (let i = 0; i < ids.size; i++) {
-            const text = String(ids.idOf(i));
-            if (seen.has(text)) {
-                collisions++;
-            } else {
-                seen.add(text);
-            }
-        }
+        const collisions = idTextCollisions(ids);
         if (collisions > 0) {
             note(
                 CSV_LOSS.ID_TEXT_COLLISION,
@@ -699,40 +709,81 @@ function attributeColumns(
         if (role !== null && SKIPPED_ROLES.has(role)) {
             continue;
         }
-        const low = lower(name);
         const set = column.length - column.nullCount;
-        const isIdName = domain === "edge" ? findColumn([name], EDGE_ID_NAMES) >= 0 : false;
-        if (reserved.has(low) || isIdName || (idColumn !== null && lower(idColumn.meta.name) === low)) {
-            note?.(
-                CSV_LOSS.RESERVED_NAME,
-                `${domain} column "${name}" is not written: the name is reserved for the ${isIdName ? "edge id" : low} column`,
-                name,
-                set,
-            );
+        if (isReservedName(name, domain, reserved, idColumn, set, note)) {
             continue;
         }
-        if (findColumn([name], LABEL_NAMES) >= 0) {
-            if (labelColumn !== null) {
-                note?.(
-                    CSV_LOSS.RESERVED_NAME,
-                    `${domain} column "${name}" is not written: the name is reserved for the label column`,
-                    name,
-                    set,
-                );
-                continue;
-            }
-            note?.(
-                CSV_LOSS.ROLE_ASSUMED,
-                `${domain} column "${name}" reads back with the label role (string)`,
-                name,
-                set,
-            );
+        if (findColumn([name], LABEL_NAMES) >= 0 && !labelNameWritten(name, domain, labelColumn, set, note)) {
+            continue;
         }
         out.push({ column, header: name });
     }
     return out;
 }
 
+/**
+ * Whether an attribute column's name is reserved for a structural column (a dialect header, an
+ * edge id name, the id column's name), with its note.
+ * @param name - the column name
+ * @param domain - node or edge, for messages
+ * @param reserved - lower-case header names the dialect writes for structure
+ * @param idColumn - the table's id role column, or null
+ * @param set - the column's set cells
+ * @param note - the recorder, or null when this table's notes are not wanted
+ * @returns true when the column is not written
+ */
+function isReservedName(
+    name: string,
+    domain: "node" | "edge",
+    reserved: ReadonlySet<string>,
+    idColumn: Column | null,
+    set: number,
+    note: ((code: string, message: string, column?: string | null, count?: number | null) => void) | null,
+): boolean {
+    const low = lower(name);
+    const isIdName = domain === "edge" ? findColumn([name], EDGE_ID_NAMES) >= 0 : false;
+    if (reserved.has(low) || isIdName || (idColumn !== null && lower(idColumn.meta.name) === low)) {
+        note?.(
+            CSV_LOSS.RESERVED_NAME,
+            `${domain} column "${name}" is not written: the name is reserved for the ${isIdName ? "edge id" : low} column`,
+            name,
+            set,
+        );
+        return true;
+    }
+    return false;
+}
+
+/**
+ * Whether an attribute column with a label header name is written, with its note: not when the
+ * table has a label column (the name is reserved for it), else it reads back with the label role.
+ * @param name - the column name
+ * @param domain - node or edge, for messages
+ * @param labelColumn - the table's label role column, or null
+ * @param set - the column's set cells
+ * @param note - the recorder, or null when this table's notes are not wanted
+ * @returns true when the column is written
+ */
+function labelNameWritten(
+    name: string,
+    domain: "node" | "edge",
+    labelColumn: Column | null,
+    set: number,
+    note: ((code: string, message: string, column?: string | null, count?: number | null) => void) | null,
+): boolean {
+    if (labelColumn !== null) {
+        note?.(
+            CSV_LOSS.RESERVED_NAME,
+            `${domain} column "${name}" is not written: the name is reserved for the label column`,
+            name,
+            set,
+        );
+        return false;
+    }
+    note?.(CSV_LOSS.ROLE_ASSUMED, `${domain} column "${name}" reads back with the label role (string)`, name, set);
+    return true;
+}
+
 /**
  * Notes about a role column (id or label): a name the importer does not map to the role, and a
  * dtype other than string / dict (read back as string).
@@ -912,6 +963,22 @@ function neighbourCell(id: string, weight: string | null): string {
     return id.includes(":") ? `${id}:` : id;
 }
 
+/**
+ * The end of the run of consecutive edge rows that share the source of row `r`.
+ * @param src - the edge sources by edge index
+ * @param edgeRows - the edges in write order
+ * @param r - the run's first row
+ * @returns the index after the run's last row
+ */
+function sourceRunEnd(src: ArrayLike<number>, edgeRows: ArrayLike<number>, r: number): number {
+    const source = src[edgeRows[r]];
+    let end = r;
+    while (end < edgeRows.length && src[edgeRows[end]] === source) {
+        end++;
+    }
+    return end;
+}
+
 /**
  * The rows of the adjacency table (design section 8.5: the order survives the importer's first-
  * appearance numbering). Consecutive edges with one source share a row, so the edge order is kept;
@@ -953,10 +1020,7 @@ function* adjacencyLines(snapshot: GraphSnapshot, plan: Plan): Generator<string,
     let r = 0;
     while (r < edgeRows.length) {
         const source = list.src[edgeRows[r]];
-        let end = r;
-        while (end < edgeRows.length && list.src[edgeRows[end]] === source) {
-            end++;
-        }
+        const end = sourceRunEnd(list.src, edgeRows, r);
         // the nodes this row introduces, in the order the importer meets them; in index order from
         // `next` they need nothing more
         const fresh: number[] = [];