@graphty/graph-io 0.3.19 → 0.3.21

package/src/common/cell-budget.ts

@@ -0,0 +1,170 @@
+/**
+ * A memory guard for the builder importGraph() fills. graph-format stores every attribute as a dense
+ * column, one slot per node (or edge) whether or not that element has a value, so a file whose nodes
+ * each carry a differently named attribute holds n values but allocates n x n slots: a few hundred
+ * kilobytes of input can need gigabytes. The guard counts the slots that hold no value and stops the
+ * import once they pass a limit. A dense file has almost no empty slots, so it is never stopped
+ * however large it is.
+ */
+
+import {
+    type ColumnDecl,
+    type ColumnHandle,
+    GraphBuilder,
+    type GraphBuilderOptions,
+    GraphFormatError,
+    type NodeId,
+} from "@graphty/graph-format";
+
+import { describe } from "./options.js";
+import { ImportReportBuilder } from "./report.js";
+
+/** The issue code of an import stopped for allocating too many empty attribute slots. */
+export const TOO_MANY_EMPTY_CELLS_CODE = "E_TOO_MANY_EMPTY_CELLS";
+
+/** The default limit: 2^24 empty slots, about 130 MB of f64 columns. */
+const DEFAULT_MAX_EMPTY_CELLS = 2 ** 24;
+
+/**
+ * Resolve the maxEmptyCells option.
+ * @param value - the caller's value
+ * @returns a non-negative integer or Infinity
+ * @throws GraphFormatError E_UNSUPPORTED for anything else
+ */
+export function maxEmptyCellsOption(value: unknown): number {
+    if (value === undefined) {
+        return DEFAULT_MAX_EMPTY_CELLS;
+    }
+    if (typeof value === "number" && value >= 0 && (Number.isInteger(value) || value === Infinity)) {
+        return value;
+    }
+    throw new GraphFormatError(
+        "E_UNSUPPORTED",
+        `option maxEmptyCells: ${describe(value)} is not a non-negative integer or Infinity`,
+        { option: "maxEmptyCells", found: value },
+    );
+}
+
+/** A GraphBuilder that throws an ImportError once its attribute columns hold too many empty slots. */
+export class CellBudgetBuilder extends GraphBuilder {
+    private readonly nodeColumns = new Set<number>();
+    private readonly edgeColumns = new Set<number>();
+    private values = 0;
+
+    /**
+     * Create a builder with a limit.
+     * @param options - the builder options
+     * @param format - the format being read, for the error's report
+     * @param maxEmptyCells - the most empty slots allowed
+     */
+    constructor(
+        options: GraphBuilderOptions,
+        private readonly format: string,
+        private readonly maxEmptyCells: number,
+    ) {
+        super(options);
+    }
+
+    /**
+     * Add a node, then check the limit (its row adds a slot to every node column).
+     * @param id - the node id
+     * @returns the node index
+     */
+    override addNode(id: NodeId): number {
+        const index = super.addNode(id);
+        this.checkBudget();
+        return index;
+    }
+
+    /**
+     * Add an edge, then check the limit (its row adds a slot to every edge column).
+     * @param source - source id
+     * @param target - target id
+     * @param weight - the weight
+     * @returns the edge index
+     */
+    override addEdge(source: NodeId, target: NodeId, weight?: number): number {
+        const edge = super.addEdge(source, target, weight);
+        this.checkBudget();
+        return edge;
+    }
+
+    /**
+     * Declare a node column, then check the limit.
+     * @param decl - the declaration
+     * @returns the column handle
+     */
+    override declareNodeColumn(decl: ColumnDecl): ColumnHandle {
+        const handle = super.declareNodeColumn(decl);
+        this.nodeColumns.add(handle);
+        this.checkBudget();
+        return handle;
+    }
+
+    /**
+     * Declare an edge column, then check the limit.
+     * @param decl - the declaration
+     * @returns the column handle
+     */
+    override declareEdgeColumn(decl: ColumnDecl): ColumnHandle {
+        const handle = super.declareEdgeColumn(decl);
+        this.edgeColumns.add(handle);
+        this.checkBudget();
+        return handle;
+    }
+
+    /**
+     * Set a node cell (a new name declares a column), then check the limit.
+     * @param column - the handle or name
+     * @param index - the node index
+     * @param value - the value
+     */
+    override setNodeValue(column: ColumnHandle | string, index: number, value: unknown): void {
+        super.setNodeValue(column, index, value);
+        this.nodeColumns.add(typeof column === "string" ? this.nodeColumn(column) : column);
+        this.count(value);
+    }
+
+    /**
+     * Set an edge cell (a new name declares a column), then check the limit.
+     * @param column - the handle or name
+     * @param edge - the edge index
+     * @param value - the value
+     */
+    override setEdgeValue(column: ColumnHandle | string, edge: number, value: unknown): void {
+        super.setEdgeValue(column, edge, value);
+        this.edgeColumns.add(typeof column === "string" ? this.edgeColumn(column) : column);
+        this.count(value);
+    }
+
+    /**
+     * Count a written value and check the budget.
+     * @param value - the value; null and undefined unset a slot and do not count
+     */
+    private count(value: unknown): void {
+        if (value !== null && value !== undefined) {
+            this.values++;
+        }
+        this.checkBudget();
+    }
+
+    /**
+     * Throw when the columns' slots minus the values written pass the limit.
+     * @throws ImportError E_TOO_MANY_EMPTY_CELLS
+     */
+    private checkBudget(): void {
+        const slots = this.nodeColumns.size * this.nodeBound + this.edgeColumns.size * this.edgeBound;
+        const empty = slots - this.values;
+        if (empty > this.maxEmptyCells) {
+            new ImportReportBuilder(this.format, Infinity).fail(
+                TOO_MANY_EMPTY_CELLS_CODE,
+                `the attributes are too sparse: ${this.nodeColumns.size} node and ${this.edgeColumns.size} edge ` +
+                    `attribute column(s) over ${this.nodeBound} node(s) and ${this.edgeBound} edge(s) would ` +
+                    `allocate more than ${this.maxEmptyCells} slots that hold no value; pass a larger ` +
+                    `maxEmptyCells (or Infinity) to read the file anyway`,
+                undefined,
+                { emptyCells: empty, maxEmptyCells: this.maxEmptyCells },
+            );
+        }
+    }
+}

package/src/common/input.ts

@@ -529,6 +529,46 @@ async function* streamChunks(
     }
 }
 
+/**
+ * Read the whole input as bytes, for a binary format (a zip): nothing is decoded. The signal is
+ * checked before every chunk and progress reported after it; a stream is cancelled when the read
+ * stops early.
+ * @param input - the input
+ * @param options - cancellation and progress (the encoding is ignored)
+ * @returns the bytes, or null when the input is text (a string, or a chunk that is a string),
+ * which a binary format cannot read
+ */
+export async function readBytes(input: ImportInput, options: ReadOptions = {}): Promise<Uint8Array | null> {
+    const signal = options.signal ?? null;
+    throwIfAborted(signal);
+    if (typeof input === "string") {
+        return null;
+    }
+    if (input instanceof Uint8Array) {
+        options.onProgress?.(input.byteLength, input.byteLength);
+        return input;
+    }
+    const parts: Uint8Array[] = [];
+    let done = 0;
+    for await (const chunk of isReadableStream(input) ? streamChunks(input, signal) : input) {
+        throwIfAborted(signal);
+        if (typeof chunk === "string") {
+            return null;
+        }
+        if (!(chunk instanceof Uint8Array)) {
+            throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
+                reason: "chunk type",
+                found: typeof chunk,
+            });
+        }
+        parts.push(chunk);
+        done += chunk.byteLength;
+        options.onProgress?.(done);
+    }
+    options.onProgress?.(done, done);
+    return parts.length === 0 ? new Uint8Array(0) : concatBytes(parts);
+}
+
 /**
  * Read the whole input as one string (the GML / DOT / JSON path, design section 8.4).
  * @param input - the input

package/src/common/options.ts

@@ -455,7 +455,7 @@ function encodingOption(value: unknown): string | null {
  * @param value - the value
  * @returns the JSON text of a primitive, or the type name otherwise
  */
-function describe(value: unknown): string {
+export function describe(value: unknown): string {
     switch (typeof value) {
         case "string":
             return JSON.stringify(value);

package/src/common/xml.ts

@@ -72,6 +72,38 @@ export class XmlSyntaxError extends Error {
     }
 }
 
+/**
+ * Opt-in repairs of two defects the Cytoscape XGMML writer is known to produce (research note
+ * `research-xgmml.md` 3.8 and 5). Each is off unless its callback is given; GEXF and GraphML never
+ * pass them, so their documents stay strictly well-formed. The callback is told the line of each
+ * repair, so the importer can warn per occurrence.
+ */
+export interface XmlRepairs {
+    /**
+     * Read an `&` that is not followed by a `;` within the next 7 characters as `&` (Cytoscape's
+     * `cytoscape.xgmml.repair.bare.ampersands` lookahead); an `&name;` that does end in time is
+     * still decoded, and still fatal when the entity is unknown. A complete numeric character
+     * reference is decoded whatever its length (`😀`), where Cytoscape's byte lookahead
+     * would turn it into text.
+     */
+    readonly bareAmpersand?: ((line: number) => void) | undefined;
+    /**
+     * Join a high-surrogate character reference immediately followed by a low-surrogate one
+     * (`��`, which XML 1.0 forbids but Cytoscape writes for astral characters) into
+     * the one character they encode; a lone surrogate stays fatal.
+     */
+    readonly surrogatePair?: ((line: number) => void) | undefined;
+}
+
+/** How far Cytoscape's bare-ampersand repair looks for the `;` that ends an entity reference. */
+const BARE_AMPERSAND_LOOKAHEAD = 7;
+
+/** A numeric character reference at the start of a text: `{` or ``. */
+const CHAR_REFERENCE = /^&#(?:[xX]([0-9a-fA-F]{1,6})|([0-9]{1,7}));/;
+
+/** A numeric character reference at the end of a text. */
+const TRAILING_CHAR_REFERENCE = /&#(?:[xX]([0-9a-fA-F]{1,6})|([0-9]{1,7}));$/;
+
 const NAMED_ENTITIES: Readonly<Record<string, string>> = {
     lt: "<",
     gt: ">",
@@ -280,9 +312,10 @@ function isXmlChar(cp: number): boolean {
  * Decode the predefined entities and character references of a text.
  * @param raw - the text as written
  * @param line - the line, for errors
+ * @param repairs - the opt-in repairs (XGMML only); none by default
  * @returns the decoded text
  */
-export function decodeEntities(raw: string, line: number): string {
+export function decodeEntities(raw: string, line: number, repairs?: XmlRepairs): string {
     let amp = raw.indexOf("&");
     if (amp < 0) {
         return raw;
@@ -292,6 +325,18 @@ export function decodeEntities(raw: string, line: number): string {
     while (amp >= 0) {
         out += raw.slice(start, amp);
         const semi = raw.indexOf(";", amp + 1);
+        if (
+            repairs?.bareAmpersand !== undefined &&
+            (semi < 0 || semi - amp > BARE_AMPERSAND_LOOKAHEAD) &&
+            // a well-formed character reference longer than the lookahead (&#128512;) is not bare
+            !CHAR_REFERENCE.test(raw.slice(amp, amp + 12))
+        ) {
+            repairs.bareAmpersand(lineAt(raw, amp, line));
+            out += "&";
+            start = amp + 1;
+            amp = raw.indexOf("&", start);
+            continue;
+        }
         if (semi < 0) {
             throw new XmlSyntaxError("unterminated entity reference", line);
         }
@@ -301,6 +346,14 @@ export function decodeEntities(raw: string, line: number): string {
             const digits = name.slice(hex ? 2 : 1);
             const ok = hex ? /^[0-9a-fA-F]{1,6}$/.test(digits) : /^[0-9]{1,7}$/.test(digits);
             const cp = ok ? Number.parseInt(digits, hex ? 16 : 10) : -1;
+            const low = cp >= 0xd800 && cp <= 0xdbff ? lowSurrogateAfter(raw, semi + 1, repairs) : null;
+            if (low !== null) {
+                repairs?.surrogatePair?.(lineAt(raw, amp, line));
+                out += String.fromCharCode(cp, low.unit);
+                start = low.end;
+                amp = raw.indexOf("&", start);
+                continue;
+            }
             if (cp < 0 || !isXmlChar(cp)) {
                 throw new XmlSyntaxError(`invalid character reference &${name};`, line);
             }
@@ -318,6 +371,45 @@ export function decodeEntities(raw: string, line: number): string {
     return out + raw.slice(start);
 }
 
+/**
+ * The line of a position inside a text that starts on a known line.
+ * @param raw - the text
+ * @param at - the position
+ * @param line - the line the text starts on
+ * @returns the line of the position
+ */
+function lineAt(raw: string, at: number, line: number): number {
+    let n = line;
+    for (let i = raw.indexOf("\n"); i >= 0 && i < at; i = raw.indexOf("\n", i + 1)) {
+        n++;
+    }
+    return n;
+}
+
+/**
+ * The low-surrogate character reference that may follow a high-surrogate one, when the
+ * surrogate-pair repair is on.
+ * @param raw - the text
+ * @param at - where the next reference would start
+ * @param repairs - the repairs in force
+ * @returns the low surrogate code unit and the index after its reference, or null
+ */
+function lowSurrogateAfter(
+    raw: string,
+    at: number,
+    repairs: XmlRepairs | undefined,
+): { readonly unit: number; readonly end: number } | null {
+    if (repairs?.surrogatePair === undefined || raw.charCodeAt(at) !== 38) {
+        return null;
+    }
+    const match = CHAR_REFERENCE.exec(raw.slice(at, at + 12));
+    if (match === null) {
+        return null;
+    }
+    const unit = match[1] === undefined ? Number.parseInt(match[2], 10) : Number.parseInt(match[1], 16);
+    return unit >= 0xdc00 && unit <= 0xdfff ? { unit, end: at + match[0].length } : null;
+}
+
 /**
  * Whether a text is whitespace only.
  * @param text - the text
@@ -347,9 +439,14 @@ export function localName(name: string): string {
  * malformed input; any error the handler throws propagates unchanged.
  * @param chunks - the text (already UTF-8 decoded, BOM removed)
  * @param handler - the event sink
+ * @param repairs - the opt-in repairs (XGMML only); none by default
  */
-export async function tokenizeXml(chunks: AsyncIterable<string>, handler: XmlHandler): Promise<void> {
-    const tokenizer = new XmlTokenizer(handler);
+export async function tokenizeXml(
+    chunks: AsyncIterable<string>,
+    handler: XmlHandler,
+    repairs?: XmlRepairs,
+): Promise<void> {
+    const tokenizer = new XmlTokenizer(handler, repairs);
     for await (const chunk of chunks) {
         tokenizer.push(chunk);
     }
@@ -435,12 +532,16 @@ export class XmlTokenizer {
 
     private rootClosed = false;
 
+    private readonly repairs: XmlRepairs | undefined;
+
     /**
      * Create a tokenizer.
      * @param handler - the event sink
+     * @param repairs - the opt-in repairs (XGMML only); none by default
      */
-    constructor(handler: XmlHandler) {
+    constructor(handler: XmlHandler, repairs?: XmlRepairs) {
         this.handler = handler;
+        this.repairs = repairs;
     }
 
     /**
@@ -677,6 +778,9 @@ export class XmlTokenizer {
                     if (amp >= pos && amp >= length - MAX_ENTITY_LENGTH && buffer.indexOf(";", amp) < 0) {
                         end = amp;
                     }
+                    if (this.repairs?.surrogatePair !== undefined) {
+                        end = this.holdHighSurrogate(pos, end);
+                    }
                 }
                 this.takeText(pos, end);
                 pos = end;
@@ -928,7 +1032,7 @@ export class XmlTokenizer {
                     this.line,
                 );
             }
-            attrs.set(attrName, decodeEntities(normalizeAttributeValue(raw), this.line));
+            attrs.set(attrName, decodeEntities(normalizeAttributeValue(raw), this.line, this.repairs));
             i = close + 1;
         }
     }
@@ -962,6 +1066,23 @@ export class XmlTokenizer {
         return j >= length && !this.final ? -1 : j;
     }
 
+    /**
+     * Under the surrogate-pair repair, hold back a high-surrogate character reference that ends
+     * the text taken so far, so it is decoded together with the low one the next chunk may start with.
+     * @param pos - the start of the text
+     * @param end - the end of the text that would be taken
+     * @returns the end to take up to
+     */
+    private holdHighSurrogate(pos: number, end: number): number {
+        const tail = this.buffer.slice(Math.max(pos, end - 12), end);
+        const match = TRAILING_CHAR_REFERENCE.exec(tail);
+        if (match === null) {
+            return end;
+        }
+        const unit = match[1] === undefined ? Number.parseInt(match[2], 10) : Number.parseInt(match[1], 16);
+        return unit >= 0xd800 && unit <= 0xdbff ? end - match[0].length : end;
+    }
+
     /**
      * Move text from the buffer into the current run.
      * @param start - the start index
@@ -978,7 +1099,7 @@ export class XmlTokenizer {
         if (hasIllegalXmlChar(raw)) {
             throw new XmlSyntaxError("a character XML 1.0 forbids appears in character data", this.line);
         }
-        this.text += decodeEntities(raw, this.line);
+        this.text += decodeEntities(raw, this.line, this.repairs);
         this.advanceLine(start, end);
     }