@graphty/graph-io 0.3.41 → 0.3.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphty/graph-io",
-  "version": "0.3.41",
+  "version": "0.3.42",
   "description": "Read and write 13 graph file formats (JSON, GraphML, GEXF, CSV, GML, DOT, Pajek, Neo4j CSV, XGMML, CX2, CX, OBO, Cytoscape sessions) as @graphty/graph-format snapshots",
   "author": "Adam Powers <apowers@ato.ms>",
   "type": "module",
@@ -121,7 +121,7 @@
   },
   "homepage": "https://graphty.app/docs/graph-io/",
   "dependencies": {
-    "@graphty/graph-format": "^1.4.2"
+    "@graphty/graph-format": "^1.4.3"
   },
   "peerDependencies": {
     "@graphty/graph-format": "^1.0.0"

package/src/children.ts

@@ -14,7 +14,7 @@
  * every node is written exactly once.
  */
 
-import { type Column, type GraphSnapshot, INVALID_INDEX } from "@graphty/graph-format";
+import { type Column, type GraphSnapshot, INVALID_INDEX, type ListColumn } from "@graphty/graph-format";
 
 /**
  * The role of the column a children CSR is built over.
@@ -282,7 +282,6 @@ export function childrenFromColumn(nodeCount: number, column: Column | null): Ch
     let dropped = 0;
     const references: number[] = [];
     if (column !== null) {
-        const rows = Math.min(column.length, nodeCount);
         const keep = (child: number, parent: number): void => {
             if (!Number.isInteger(parent) || parent === INVALID_INDEX || parent >= nodeCount || parent === child) {
                 dropped++;
@@ -292,31 +291,9 @@ export function childrenFromColumn(nodeCount: number, column: Column | null): Ch
             rowPtr[parent + 1]++;
             parentCount[child]++;
         };
-        if (column.dtype === "u32") {
-            const { data } = column;
-            for (let i = 0; i < rows; i++) {
-                if (column.isSet(i)) {
-                    keep(i, data[i]);
-                }
-            }
-        } else if (column.dtype === "list") {
-            for (let i = 0; i < rows; i++) {
-                if (!column.isSet(i)) {
-                    continue;
-                }
-                for (const item of column.sliceOf(i)) {
-                    if (typeof item === "number") {
-                        keep(i, item);
-                    } else {
-                        dropped++;
-                    }
-                }
-            }
-        } else {
-            throw new TypeError(
-                `a containment column must be u32 or a list of u32, found ${column.dtype} column "${column.meta.name}"`,
-            );
-        }
+        // keep() counts into `dropped` while it runs, so add the non-numeric items after the call
+        const nonNumeric = readContainment(column, Math.min(column.length, nodeCount), keep);
+        dropped += nonNumeric;
     }
     for (let u = 0; u < nodeCount; u++) {
         rowPtr[u + 1] += rowPtr[u];
@@ -330,6 +307,56 @@ export function childrenFromColumn(nodeCount: number, column: Column | null): Ch
     return new ChildrenCsr(nodeCount, rowPtr, children, parentCount, dropped, column);
 }
 
+/**
+ * Hand every (child, parent) reference of the first `rows` rows of a containment column to `keep`,
+ * in ascending child order.
+ * @param column - a u32 `parent`-shaped column or a list-of-u32 `parents`-shaped column
+ * @param rows - how many rows to read
+ * @param keep - receives each reference
+ * @returns how many list items were dropped for not being a number
+ */
+function readContainment(column: Column, rows: number, keep: (child: number, parent: number) => void): number {
+    if (column.dtype === "u32") {
+        const { data } = column;
+        for (let i = 0; i < rows; i++) {
+            if (column.isSet(i)) {
+                keep(i, data[i]);
+            }
+        }
+        return 0;
+    }
+    if (column.dtype === "list") {
+        return readParentLists(column, rows, keep);
+    }
+    throw new TypeError(
+        `a containment column must be u32 or a list of u32, found ${column.dtype} column "${column.meta.name}"`,
+    );
+}
+
+/**
+ * Hand every numeric item of a `parents`-shaped list column to `keep`.
+ * @param column - the list column
+ * @param rows - how many rows to read
+ * @param keep - receives each reference
+ * @returns how many items were not numbers
+ */
+function readParentLists(column: ListColumn, rows: number, keep: (child: number, parent: number) => void): number {
+    let dropped = 0;
+    for (let i = 0; i < rows; i++) {
+        if (!column.isSet(i)) {
+            continue;
+        }
+        for (const item of column.sliceOf(i)) {
+            if (typeof item === "number") {
+                keep(i, item);
+            } else {
+                dropped++;
+            }
+        }
+    }
+    return dropped;
+}
+
 /**
  * The column a ChildrenOptions selects.
  * @param snapshot - the snapshot

package/src/common/attributes.ts

@@ -132,22 +132,7 @@ export const DICT_SAMPLE_ROWS = 1024;
 export function declareAttribute(input: AttributeDeclarationInput): DeclaredAttribute {
     const issues: DeclarationIssue[] = [];
     const listSyntax = input.listSyntax ?? "gexf";
-    let spec: DeclaredTypeSpec;
-    if (input.type === null) {
-        spec = stringSpec(null);
-    } else {
-        const mapped = mapDeclaredType(input.format, input.type, input.long);
-        if (mapped === null) {
-            spec = stringSpec(input.type);
-            issues.push({
-                category: "unsupported",
-                code: UNKNOWN_TYPE_CODE,
-                message: `${input.format} type "${input.type}" is not supported; values kept as text`,
-            });
-        } else {
-            spec = mapped;
-        }
-    }
+    let spec = declaredSpec(input, issues);
     const { name, renamed } = resolveName(input);
     if (renamed) {
         issues.push({
@@ -179,20 +164,7 @@ export function declareAttribute(input: AttributeDeclarationInput): DeclaredAttr
     }
     const optionsText = input.optionsText ?? null;
     if (optionsText !== null) {
-        try {
-            const options = splitListText(optionsText, listSyntax).map((item) => parseItem(item, spec));
-            decl.options = options;
-            if (spec.dtype === "string" && !spec.list && options.every((o) => typeof o === "string")) {
-                decl.dtype = "dict";
-                spec = { ...spec, dtype: "dict" };
-            }
-        } catch (err) {
-            issues.push({
-                category: "validation-error",
-                code: BAD_OPTIONS_CODE,
-                message: `options "${optionsText}" of attribute "${name}" do not parse as ${spec.declared}: ${messageOf(err)}`,
-            });
-        }
+        spec = declareOptions(decl, spec, optionsText, listSyntax, issues);
     }
     const defaultText = input.defaultText ?? null;
     if (defaultText !== null) {
@@ -210,6 +182,63 @@ export function declareAttribute(input: AttributeDeclarationInput): DeclaredAttr
     return { decl, spec, listSyntax, companion, renamed, issues };
 }
 
+/**
+ * The value spec of a declared type: string for no type, string with UNKNOWN_TYPE_CODE for a type
+ * the format does not define.
+ * @param input - what the file declares
+ * @param issues - receives the unknown-type warning
+ * @returns the spec
+ */
+function declaredSpec(input: AttributeDeclarationInput, issues: DeclarationIssue[]): DeclaredTypeSpec {
+    if (input.type === null) {
+        return stringSpec(null);
+    }
+    const mapped = mapDeclaredType(input.format, input.type, input.long);
+    if (mapped !== null) {
+        return mapped;
+    }
+    issues.push({
+        category: "unsupported",
+        code: UNKNOWN_TYPE_CODE,
+        message: `${input.format} type "${input.type}" is not supported; values kept as text`,
+    });
+    return stringSpec(input.type);
+}
+
+/**
+ * Put the parsed options of an attribute on its declaration; a plain string attribute whose
+ * options are all strings becomes a dict column.
+ * @param decl - the declaration to complete
+ * @param spec - the value spec
+ * @param optionsText - the options text
+ * @param listSyntax - the list syntax the options are split with
+ * @param issues - receives BAD_OPTIONS_CODE when the options do not parse
+ * @returns the spec, with dtype dict when the column became a dict
+ */
+function declareOptions(
+    decl: ColumnDecl,
+    spec: DeclaredTypeSpec,
+    optionsText: string,
+    listSyntax: ListSyntax,
+    issues: DeclarationIssue[],
+): DeclaredTypeSpec {
+    try {
+        const options = splitListText(optionsText, listSyntax).map((item) => parseItem(item, spec));
+        decl.options = options;
+        if (spec.dtype === "string" && !spec.list && options.every((o) => typeof o === "string")) {
+            decl.dtype = "dict";
+            return { ...spec, dtype: "dict" };
+        }
+    } catch (err) {
+        issues.push({
+            category: "validation-error",
+            code: BAD_OPTIONS_CODE,
+            message: `options "${optionsText}" of attribute "${decl.name}" do not parse as ${spec.declared}: ${messageOf(err)}`,
+        });
+    }
+    return spec;
+}
+
 /**
  * Parse a value text by a declaration: a scalar, or a list split by the syntax with each item
  * parsed by the item kind.

package/src/common/input.ts

@@ -506,46 +506,81 @@ class ByteDecoder {
         const bad = this.encoding === "utf-8" ? invalidUtf8(all) : null;
         const at = base + (bad === null ? 0 : bad.index);
         if (this.mayFallBack && this.asciiSoFar) {
-            // windows-1252 text holds no control bytes but TAB, LF, FF and CR: one anywhere in
-            // undeclared input that is not UTF-8 marks binary data (watch() saw every byte so far)
-            if (this.control !== null) {
-                return this.binary(this.control);
-            }
-            if (this.textFollows && bad?.cut === true) {
-                return this.report.fail(
-                    INVALID_UTF8_CODE,
-                    `invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`,
-                    undefined,
-                    { byteOffset: at },
-                );
-            }
-            if (!stream && bad?.truncated === true) {
-                return this.report.fail(
-                    INVALID_UTF8_CODE,
-                    `the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`,
-                    undefined,
-                    { byteOffset: at },
-                );
-            }
-            if (!startsWithUtf8(all)) {
-                const cutNote =
-                    !stream && bad?.cut === true
-                        ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated"
-                        : "";
-                this.report.warning(
-                    "coercion",
-                    ENCODING_FALLBACK_CODE,
-                    this.declaredUtf8
-                        ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)`
-                        : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`,
-                );
-                this.use("windows-1252", false);
-                this.fellBack = true;
-                this.mixedPending = true;
-                this.reportUtf8After(all, base);
-                return (this.decoder as TextDecoder).decode(all, { stream });
+            const text = this.fallBack(all, bad, at, base, stream);
+            if (text !== null) {
+                return text;
             }
         }
+        return this.failDecode(bytes, at, base, stream);
+    }
+
+    /**
+     * Undeclared input that was ASCII so far failed to decode: fail on binary data or a cut
+     * sequence, else switch to windows-1252 with a warning unless the bytes start with valid
+     * non-ASCII UTF-8.
+     * @param all - the held-back bytes followed by the bytes that failed
+     * @param bad - the first invalid UTF-8 sequence, or null when the encoding is not UTF-8
+     * @param at - the byte offset of the first invalid byte
+     * @param base - the byte offset of `all`
+     * @param stream - whether more bytes follow
+     * @returns the bytes decoded as windows-1252, or null when they do not fall back
+     */
+    private fallBack(
+        all: Uint8Array,
+        bad: Utf8Problem | null,
+        at: number,
+        base: number,
+        stream: boolean,
+    ): string | null {
+        // windows-1252 text holds no control bytes but TAB, LF, FF and CR: one anywhere in
+        // undeclared input that is not UTF-8 marks binary data (watch() saw every byte so far)
+        if (this.control !== null) {
+            return this.binary(this.control);
+        }
+        if (this.textFollows && bad?.cut === true) {
+            return this.report.fail(
+                INVALID_UTF8_CODE,
+                `invalid UTF-8 at byte ${at}: a byte chunk ends inside a multi-byte sequence and a text chunk follows`,
+                undefined,
+                { byteOffset: at },
+            );
+        }
+        if (!stream && bad?.truncated === true) {
+            return this.report.fail(
+                INVALID_UTF8_CODE,
+                `the input ends in the middle of a UTF-8 sequence at byte ${at} (it may be truncated)`,
+                undefined,
+                { byteOffset: at },
+            );
+        }
+        if (startsWithUtf8(all)) {
+            return null;
+        }
+        const cutNote =
+            !stream && bad?.cut === true ? ": it ends inside a multi-byte UTF-8 sequence, so it may be truncated" : "";
+        this.report.warning(
+            "coercion",
+            ENCODING_FALLBACK_CODE,
+            this.declaredUtf8
+                ? `the input declares UTF-8 but is not valid UTF-8 (at byte ${at}); everything before was ASCII, so it is read as windows-1252 (pass the encoding option to choose another)`
+                : `the input is not valid UTF-8 (at byte ${at}${cutNote}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`,
+        );
+        this.use("windows-1252", false);
+        this.fellBack = true;
+        this.mixedPending = true;
+        this.reportUtf8After(all, base);
+        return (this.decoder as TextDecoder).decode(all, { stream });
+    }
+
+    /**
+     * Fail the import on bytes that are not valid in the encoding.
+     * @param bytes - the bytes that failed
+     * @param at - the byte offset of the first invalid UTF-8 byte
+     * @param base - the byte offset of the held-back bytes and these
+     * @param stream - whether more bytes follow
+     * @returns never: it throws
+     */
+    private failDecode(bytes: Uint8Array, at: number, base: number, stream: boolean): never {
         if (this.encoding === "utf-8") {
             const after = this.mayFallBack ? " after valid non-ASCII UTF-8 text; pass the encoding option" : "";
             return this.report.fail(INVALID_UTF8_CODE, `invalid UTF-8 at byte ${at}${after}`, undefined, {
@@ -1662,81 +1697,174 @@ export class LineReader implements AsyncIterable<string> {
      * @returns nothing
      */
     async *[Symbol.asyncIterator](): AsyncGenerator<string, void, undefined> {
-        // The pieces of the line in progress (none of them holds a line break, except that the last
-        // may end with a `\r` whose meaning the next chunk decides); joined once when the line ends,
-        // so a line spanning many chunks costs its length, not its length times the chunk count.
-        const pending: string[] = [];
-        let pendingLength = 0;
+        // The pieces of the line in progress, joined once when the line ends, so a line spanning
+        // many chunks costs its length, not its length times the chunk count.
+        const pending = new PendingLine((line) => tooLarge(this.report, this.lineTooLong(), line));
         let trailingCr = false;
         for await (const chunk of textChunks(this.input, this.report, this.options)) {
             let start = 0;
-            const end = chunk.length;
             if (trailingCr) {
                 // the previous chunk ended with `\r`: that ended a line, and a leading `\n` here is
                 // the second half of the same terminator
                 trailingCr = false;
                 this.lineNumber++;
-                const joined = pending.join("");
-                pending.length = 0;
-                pendingLength = 0;
-                yield joined.slice(0, -1);
+                yield pending.take().slice(0, -1);
                 if (chunk.charCodeAt(0) === 10) {
                     start = 1;
                 }
             }
-            // the next `\n` and `\r` at or after `start`; each is searched for once per chunk and
-            // again only after it was consumed, so a chunk is scanned once whatever its line count
-            let nl = chunk.indexOf("\n", start);
-            let cr = chunk.indexOf("\r", start);
-            while (nl >= 0 || cr >= 0) {
-                let cut: number;
-                let next: number;
-                if (cr >= 0 && (nl < 0 || cr < nl)) {
-                    if (cr === end - 1) {
-                        // a trailing \r may be the first half of \r\n split across chunks
-                        break;
-                    }
-                    cut = cr;
-                    next = chunk.charCodeAt(cr + 1) === 10 ? cr + 2 : cr + 1;
-                } else {
-                    cut = nl;
-                    next = nl + 1;
-                }
+            const breaks = new LineBreaks(chunk, start);
+            while (breaks.advance()) {
                 this.lineNumber++;
-                const piece = chunk.slice(start, cut);
-                if (pending.length === 0) {
-                    yield piece;
-                } else {
-                    if (pendingLength + piece.length > MAX_TEXT_LENGTH) {
-                        tooLarge(this.report, this.lineTooLong(), this.lineNumber);
-                    }
-                    pending.push(piece);
-                    const joined = pending.join("");
-                    pending.length = 0;
-                    pendingLength = 0;
-                    yield joined;
-                }
-                start = next;
-                if (nl >= 0 && nl < next) {
-                    nl = chunk.indexOf("\n", next);
-                }
-                if (cr >= 0 && cr < next) {
-                    cr = chunk.indexOf("\r", next);
-                }
+                yield pending.finish(chunk.slice(start, breaks.cut), this.lineNumber);
+                start = breaks.next;
             }
-            if (start < end) {
-                pendingLength += end - start;
-                if (pendingLength > MAX_TEXT_LENGTH) {
-                    tooLarge(this.report, this.lineTooLong(), this.lineNumber + 1);
-                }
-                pending.push(start === 0 ? chunk : chunk.slice(start));
-                trailingCr = chunk.charCodeAt(end - 1) === 13;
+            if (start < chunk.length) {
+                pending.appendRest(chunk, start, this.lineNumber + 1);
+                trailingCr = chunk.codePointAt(chunk.length - 1) === 13;
             }
         }
-        if (pending.length > 0) {
+        if (!pending.empty) {
             this.lineNumber++;
-            const joined = pending.join("");
+            const joined = pending.take();
             yield joined.endsWith("\r") ? joined.slice(0, -1) : joined;
         }
     }
 }
+
+/**
+ * The pieces of a line that spans chunks. None of them holds a line break, except that the last
+ * may end with a `\r` whose meaning the next chunk decides.
+ */
+class PendingLine {
+    private readonly pieces: string[] = [];
+
+    private length = 0;
+
+    private readonly tooLong: (line: number) => never;
+
+    /**
+     * Start an empty line.
+     * @param tooLong - fails the import when the line grows longer than one string can hold
+     */
+    constructor(tooLong: (line: number) => never) {
+        this.tooLong = tooLong;
+    }
+
+    /**
+     * Whether no piece is held.
+     * @returns true when the line in progress is empty
+     */
+    get empty(): boolean {
+        return this.pieces.length === 0;
+    }
+
+    /**
+     * Hold the rest of a chunk from `start`: the line goes on in the next chunk.
+     * @param chunk - the chunk
+     * @param start - where the unfinished line starts in it
+     * @param line - the 1-based number of the line, for the too-long error
+     */
+    appendRest(chunk: string, start: number, line: number): void {
+        this.append(start === 0 ? chunk : chunk.slice(start), line);
+    }
+
+    /**
+     * End the line with its last piece.
+     * @param piece - the text from the start of the chunk (or the last break) to the line break
+     * @param line - the 1-based number of the line, for the too-long error
+     * @returns the whole line
+     */
+    finish(piece: string, line: number): string {
+        if (this.pieces.length === 0) {
+            return piece;
+        }
+        this.append(piece, line);
+        return this.take();
+    }
+
+    /**
+     * Join the held pieces and start a new line.
+     * @returns the joined text
+     */
+    take(): string {
+        const joined = this.pieces.join("");
+        this.pieces.length = 0;
+        this.length = 0;
+        return joined;
+    }
+
+    /**
+     * Hold one more piece.
+     * @param piece - the piece
+     * @param line - the 1-based number of the line, for the too-long error
+     */
+    private append(piece: string, line: number): void {
+        this.length += piece.length;
+        if (this.length > MAX_TEXT_LENGTH) {
+            this.tooLong(line);
+        }
+        this.pieces.push(piece);
+    }
+}
+
+/**
+ * The line breaks of one chunk, in order: `\n`, `\r\n` and a lone `\r`. Each of `\n` and `\r` is
+ * searched for once per chunk and again only after it was consumed, so a chunk is scanned once
+ * whatever its line count.
+ */
+class LineBreaks {
+    /** Where the line just found ends (its terminator starts there). */
+    cut = 0;
+
+    /** Where the next line starts. */
+    next: number;
+
+    private readonly chunk: string;
+
+    private nl: number;
+
+    private cr: number;
+
+    /**
+     * Scan a chunk from a position.
+     * @param chunk - the chunk
+     * @param start - where scanning starts
+     */
+    constructor(chunk: string, start: number) {
+        this.chunk = chunk;
+        this.next = start;
+        this.nl = chunk.indexOf("\n", start);
+        this.cr = chunk.indexOf("\r", start);
+    }
+
+    /**
+     * Find the next line break.
+     * @returns false when none is left, or the only one left is a trailing `\r` (it may be the first
+     * half of a `\r\n` split across chunks)
+     */
+    advance(): boolean {
+        const { chunk, next } = this;
+        if (this.nl >= 0 && this.nl < next) {
+            this.nl = chunk.indexOf("\n", next);
+        }
+        if (this.cr >= 0 && this.cr < next) {
+            this.cr = chunk.indexOf("\r", next);
+        }
+        const { nl, cr } = this;
+        if (cr >= 0 && (nl < 0 || cr < nl)) {
+            if (cr === chunk.length - 1) {
+                return false;
+            }
+            this.cut = cr;
+            this.next = chunk.codePointAt(cr + 1) === 10 ? cr + 2 : cr + 1;
+            return true;
+        }
+        if (nl < 0) {
+            return false;
+        }
+        this.cut = nl;
+        this.next = nl + 1;
+        return true;
+    }
+}

package/src/common/lists.ts

@@ -144,21 +144,9 @@ function splitBracketed(text: string): string[] {
         }
         let item = "";
         if (i < n && (inner[i] === '"' || inner[i] === "'")) {
-            const quote = inner[i];
-            i++;
-            while (i < n) {
-                if (inner[i] === quote) {
-                    if (inner[i + 1] === quote) {
-                        item += quote;
-                        i += 2;
-                        continue;
-                    }
-                    i++;
-                    break;
-                }
-                item += inner[i];
-                i++;
-            }
+            const quoted = readQuoted(inner, i);
+            item = quoted.text;
+            i = quoted.end;
         }
         // unquoted text, or text after a closing quote that is not a separator, runs to the next comma
         let end = inner.indexOf(",", i);
@@ -179,6 +167,32 @@ function splitBracketed(text: string): string[] {
     }
 }
 
+/**
+ * Read a quoted item of a bracketed list: a doubled quote inside it is one quote, and a missing
+ * closing quote ends it at the end of the text.
+ * @param inner - the inside of the brackets
+ * @param open - the index of the opening quote
+ * @returns the unquoted text and the index just past the closing quote
+ */
+function readQuoted(inner: string, open: number): { text: string; end: number } {
+    const quote = inner[open];
+    const n = inner.length;
+    let text = "";
+    let i = open + 1;
+    while (i < n) {
+        if (inner[i] !== quote) {
+            text += inner[i];
+            i++;
+        } else if (inner[i + 1] === quote) {
+            text += quote;
+            i += 2;
+        } else {
+            return { text, end: i + 1 };
+        }
+    }
+    return { text, end: i };
+}
+
 /**
  * Whether a char code is ASCII whitespace.
  * @param c - the char code