@graphty/graph-io 0.3.41 → 0.3.42

package/src/formats/csv/importer.ts

@@ -385,6 +385,13 @@ interface AdjacencyPlan {
 }
 
 /** Everything one import call shares between its tables. */
+/** A row read before the plan exists, kept to be pushed once it does. */
+interface PendingRow {
+    readonly row: string[];
+    readonly quoted: readonly boolean[];
+    readonly line: number;
+}
+
 interface ImportState {
     readonly sink: GraphSink;
     readonly report: ImportReportBuilder;
@@ -455,6 +462,43 @@ function isPadded(text: string, quoted: boolean | undefined): boolean {
  */
 function resolveCsvOptions(options: (CsvImportOptions & CommonImportOptions) | undefined): ResolvedCsvOptions {
     const o: CsvImportOptions = options ?? {};
+    checkDialectOptions(o);
+    for (const name of ["sourceColumn", "targetColumn", "idColumn", "labelColumn"] as const) {
+        checkColumnRef(name, o[name]);
+    }
+    if (o.typeColumn !== null) {
+        checkColumnRef("typeColumn", o.typeColumn);
+    }
+    if (o.table === "adjacency") {
+        checkAdjacencyOptions(o);
+    }
+    if (o.rowNumberIds !== undefined && typeof o.rowNumberIds !== "boolean") {
+        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: expected a boolean", {
+            option: "rowNumberIds",
+            found: o.rowNumberIds,
+        });
+    }
+    return {
+        delimiter: o.delimiter ?? null,
+        decimalComma: o.decimal === ",",
+        header: o.header ?? "auto",
+        table: o.table ?? "auto",
+        sourceColumn: o.sourceColumn ?? null,
+        targetColumn: o.targetColumn ?? null,
+        typeColumn: o.typeColumn,
+        idColumn: o.idColumn ?? null,
+        labelColumn: o.labelColumn ?? null,
+        nodes: nodesInput(o.nodes),
+        rowNumberIds: o.rowNumberIds ?? false,
+        explicitDirected: typeof options?.defaultDirected === "boolean" ? options.defaultDirected : null,
+    };
+}
+
+/**
+ * Check the dialect and table options: delimiter, decimal, header and table.
+ * @param o - the caller's CSV options
+ */
+function checkDialectOptions(o: CsvImportOptions): void {
     if (o.delimiter !== undefined && (typeof o.delimiter !== "string" || o.delimiter.length === 0)) {
         throw new GraphFormatError("E_UNSUPPORTED", "option delimiter: expected a non-empty string", {
             option: "delimiter",
@@ -489,48 +533,27 @@ function resolveCsvOptions(options: (CsvImportOptions & CommonImportOptions) | u
             found: o.table,
         });
     }
-    for (const name of ["sourceColumn", "targetColumn", "idColumn", "labelColumn"] as const) {
-        checkColumnRef(name, o[name]);
-    }
-    if (o.typeColumn !== null) {
-        checkColumnRef("typeColumn", o.typeColumn);
-    }
-    if (o.table === "adjacency") {
-        for (const name of ["sourceColumn", "targetColumn", "typeColumn", "idColumn", "labelColumn"] as const) {
-            if (o[name] !== undefined) {
-                throw new GraphFormatError("E_UNSUPPORTED", `option ${name}: an adjacency table has no columns`, {
-                    option: name,
-                    found: o[name],
-                });
-            }
-        }
-        if (o.rowNumberIds === true) {
-            throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: an adjacency table names its nodes", {
-                option: "rowNumberIds",
-                found: o.rowNumberIds,
+}
+
+/**
+ * Check the options an adjacency table has no use for: column options and rowNumberIds.
+ * @param o - the caller's CSV options
+ */
+function checkAdjacencyOptions(o: CsvImportOptions): void {
+    for (const name of ["sourceColumn", "targetColumn", "typeColumn", "idColumn", "labelColumn"] as const) {
+        if (o[name] !== undefined) {
+            throw new GraphFormatError("E_UNSUPPORTED", `option ${name}: an adjacency table has no columns`, {
+                option: name,
+                found: o[name],
             });
         }
     }
-    if (o.rowNumberIds !== undefined && typeof o.rowNumberIds !== "boolean") {
-        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: expected a boolean", {
+    if (o.rowNumberIds === true) {
+        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: an adjacency table names its nodes", {
             option: "rowNumberIds",
             found: o.rowNumberIds,
         });
     }
-    return {
-        delimiter: o.delimiter ?? null,
-        decimalComma: o.decimal === ",",
-        header: o.header ?? "auto",
-        table: o.table ?? "auto",
-        sourceColumn: o.sourceColumn ?? null,
-        targetColumn: o.targetColumn ?? null,
-        typeColumn: o.typeColumn,
-        idColumn: o.idColumn ?? null,
-        labelColumn: o.labelColumn ?? null,
-        nodes: nodesInput(o.nodes),
-        rowNumberIds: o.rowNumberIds ?? false,
-        explicitDirected: typeof options?.defaultDirected === "boolean" ? options.defaultDirected : null,
-    };
 }
 
 /**
@@ -823,43 +846,12 @@ class TableReader {
             : first.value;
         const firstLine = this.reader.line;
         const firstQuoted = this.reader.quoted.slice(0, firstRow.length);
-        const pending: { row: string[]; quoted: readonly boolean[]; line: number }[] = [];
-        let header: boolean;
-        // an adjacency table has no header unless the caller says so: its rows vary in width; a
-        // table numbered by rowNumberIds has one, since a headerless table takes its ids from column 0
-        let mode = this.state.csv.header;
-        if (mode === "auto" && this.kind === "adjacency") {
-            mode = false;
-        } else if (mode === "auto" && this.rowNumbers) {
-            mode = true;
-        }
-        if (mode === "auto") {
-            const second = await iterator.next();
-            const secondRow: string[] | null = second.done ? null : second.value;
-            const secondLine = this.reader.line;
-            const secondQuoted = this.reader.quoted.slice(0, secondRow?.length ?? 0);
-            // a column option naming a cell of the first row makes that row the header
-            header = looksLikeHeader(firstRow, secondRow) || this.namesExplicitColumn(firstRow);
-            if (!header) {
-                pending.push({ row: firstRow, quoted: firstQuoted, line: firstLine });
-            }
-            if (secondRow !== null) {
-                pending.push({ row: secondRow, quoted: secondQuoted, line: secondLine });
-            }
-        } else {
-            header = mode;
-            if (!header) {
-                pending.push({ row: firstRow, quoted: firstQuoted, line: firstLine });
-            }
-        }
-        if (this.rowNumbers && !header) {
-            report.warning(
-                "unsupported",
-                OPTION_IGNORED_CODE,
-                "option rowNumberIds ignored: a headerless node table takes its ids from column 0",
-                { line: firstLine, element: "rowNumberIds" },
-            );
-        }
+        const pending: PendingRow[] = [];
+        const header = await this.decideHeader(
+            iterator,
+            { row: firstRow, quoted: firstQuoted, line: firstLine },
+            pending,
+        );
         const names = header ? uniqueNames(headerNames(firstRow), report, firstLine) : positionalNames(firstRow.length);
         const last = firstRow.length - 1;
         const optionalLast = header && last > 0 && isUnset(firstRow[last], firstQuoted[last]);
@@ -889,6 +881,80 @@ class TableReader {
         for (const { row, quoted, line } of pending) {
             this.processRow(row, quoted, line);
         }
+        await this.readRemainingRows(iterator);
+        if (this.dataRows === 0 && header) {
+            report.warning("missing-value", NO_DATA_ROWS_CODE, "the table has a header and no data rows", {
+                line: firstLine,
+            });
+        }
+        this.reportReaderFindings();
+    }
+
+    /**
+     * Decide whether the first row is the header, reading the second row when the header option is
+     * "auto", and queue the rows read so far that are data.
+     * @param iterator - the record iterator, past the first row
+     * @param first - the first row
+     * @param pending - receives the data rows read so far, in order
+     * @returns whether the first row is the header
+     */
+    private async decideHeader(
+        iterator: AsyncGenerator<string[], void, undefined>,
+        first: PendingRow,
+        pending: PendingRow[],
+    ): Promise<boolean> {
+        const mode = this.headerMode();
+        let second: PendingRow | null = null;
+        let header: boolean;
+        if (mode === "auto") {
+            const next = await iterator.next();
+            second = next.done
+                ? null
+                : { row: next.value, quoted: this.reader.quoted.slice(0, next.value.length), line: this.reader.line };
+            // a column option naming a cell of the first row makes that row the header
+            header = looksLikeHeader(first.row, second?.row ?? null) || this.namesExplicitColumn(first.row);
+        } else {
+            header = mode;
+        }
+        if (!header) {
+            pending.push(first);
+        }
+        if (second !== null) {
+            pending.push(second);
+        }
+        if (this.rowNumbers && !header) {
+            this.state.report.warning(
+                "unsupported",
+                OPTION_IGNORED_CODE,
+                "option rowNumberIds ignored: a headerless node table takes its ids from column 0",
+                { line: first.line, element: "rowNumberIds" },
+            );
+        }
+        return header;
+    }
+
+    /**
+     * The header option as this table reads it: an adjacency table has no header unless the caller
+     * says so (its rows vary in width); a table numbered by rowNumberIds has one, since a headerless
+     * table takes its ids from column 0.
+     * @returns true, false or "auto"
+     */
+    private headerMode(): boolean | "auto" {
+        const mode = this.state.csv.header;
+        if (mode === "auto" && this.kind === "adjacency") {
+            return false;
+        }
+        if (mode === "auto" && this.rowNumbers) {
+            return true;
+        }
+        return mode;
+    }
+
+    /**
+     * Push every row after the header and the rows read to decide it, then finish the column writers.
+     * @param iterator - the record iterator
+     */
+    private async readRemainingRows(iterator: AsyncGenerator<string[], void, undefined>): Promise<void> {
         const { signal } = this.state.common;
         let sinceCheck = 0;
         for (;;) {
@@ -905,12 +971,6 @@ class TableReader {
         for (const writer of this.writers) {
             writer?.finish();
         }
-        if (this.dataRows === 0 && header) {
-            report.warning("missing-value", NO_DATA_ROWS_CODE, "the table has a header and no data rows", {
-                line: firstLine,
-            });
-        }
-        this.reportReaderFindings();
     }
 
     /**

package/src/formats/csv/records.ts

@@ -586,14 +586,8 @@ export class RecordReader implements AsyncIterable<number> {
         }
         const pieces: string[] = [];
         let length = 0;
-        let breaks = 0;
-        let lastWasCr = false;
         let sniffed = false;
-        // the leading comment lines are left out of the sniff, so they do not count toward its rows
-        const commentCodes = (this.syntax.comments ?? []).map((c) => c.codePointAt(0));
-        let leading = commentCodes.length > 0;
-        let lineStart = true;
-        let inComment = false;
+        const preview = new PreviewBreaks((this.syntax.comments ?? []).map((c) => c.codePointAt(0)));
         for await (const chunk of source) {
             if (sniffed) {
                 yield chunk;
@@ -601,24 +595,8 @@ export class RecordReader implements AsyncIterable<number> {
             }
             pieces.push(chunk);
             length += chunk.length;
-            for (let i = 0; i < chunk.length && breaks < PREVIEW_ROWS; i++) {
-                const c = chunk.charCodeAt(i);
-                if (c === LF || c === CR) {
-                    if (!(c === LF && lastWasCr) && !inComment) {
-                        breaks++;
-                    }
-                    inComment = false;
-                    lineStart = true;
-                } else {
-                    if (lineStart && leading) {
-                        inComment = commentCodes.includes(c);
-                        leading = inComment;
-                    }
-                    lineStart = false;
-                }
-                lastWasCr = c === CR;
-            }
-            if (breaks < PREVIEW_ROWS && length < PREVIEW_CHARS) {
+            preview.scan(chunk);
+            if (preview.breaks < PREVIEW_ROWS && length < PREVIEW_CHARS) {
                 continue;
             }
             const text = pieces.join("");
@@ -668,6 +646,74 @@ export class RecordReader implements AsyncIterable<number> {
     }
 }
 
+/**
+ * Counts the line breaks of the text buffered for the delimiter sniff, up to PREVIEW_ROWS. The
+ * leading comment lines are left out of the sniff, so they do not count toward its rows.
+ */
+class PreviewBreaks {
+    /** The line breaks counted so far (at most PREVIEW_ROWS). */
+    breaks = 0;
+
+    private readonly commentCodes: readonly (number | undefined)[];
+
+    private lastWasCr = false;
+
+    private leading: boolean;
+
+    private lineStart = true;
+
+    private inComment = false;
+
+    /**
+     * Start counting at the start of the input.
+     * @param commentCodes - the code points of the comment characters
+     */
+    constructor(commentCodes: readonly (number | undefined)[]) {
+        this.commentCodes = commentCodes;
+        this.leading = commentCodes.length > 0;
+    }
+
+    /**
+     * Count the line breaks of the next chunk, stopping at PREVIEW_ROWS.
+     * @param chunk - the chunk
+     */
+    scan(chunk: string): void {
+        for (let i = 0; i < chunk.length && this.breaks < PREVIEW_ROWS; i++) {
+            const c = chunk.charCodeAt(i);
+            if (c === LF || c === CR) {
+                this.lineBreak(c);
+            } else {
+                this.character(c);
+            }
+            this.lastWasCr = c === CR;
+        }
+    }
+
+    /**
+     * A line break: counted unless it is the LF of a CR LF or ends a leading comment line.
+     * @param c - LF or CR
+     */
+    private lineBreak(c: number): void {
+        if (!(c === LF && this.lastWasCr) && !this.inComment) {
+            this.breaks++;
+        }
+        this.inComment = false;
+        this.lineStart = true;
+    }
+
+    /**
+     * Any other character: the first of a line decides whether a leading comment line starts.
+     * @param c - the character code
+     */
+    private character(c: number): void {
+        if (this.lineStart && this.leading) {
+            this.inComment = this.commentCodes.includes(c);
+            this.leading = this.inComment;
+        }
+        this.lineStart = false;
+    }
+}
+
 /**
  * The record state machine of RecordReader, resumable at every record end so the reader can
  * yield a record before the next one overwrites the shared cell arrays. Every state lives on the

package/src/formats/dot/tokenizer.ts

@@ -87,6 +87,16 @@ function isDigit(c: number): boolean {
     return c >= 48 && c <= 57;
 }
 
+/**
+ * Whether a code unit is a blank between tokens: space, tab, vertical tab, form feed, or a U+FEFF
+ * after the start (concatenated files), which the shared text check reports.
+ * @param c - the code unit
+ * @returns true for a blank
+ */
+function isBlank(c: number): boolean {
+    return c === 0x20 || c === 0x09 || c === 0x0b || c === 0x0c || c === BOM_CODE;
+}
+
 /**
  * A pull lexer over one DOT document with arbitrary lookahead.
  * @category Plugin helpers
@@ -247,40 +257,68 @@ export class DotTokenizer {
                 this.pos = i + 1;
                 return { kind: "id", text: out, quoted: true, html: false, line };
             }
-            if (c === 0x5c) {
-                const d = text.charCodeAt(i + 1);
-                if (d === 0x22) {
-                    out += `${text.slice(segment, i)}"`;
-                    i += 2;
-                    segment = i;
-                    continue;
-                }
-                if (d === 0x5c) {
-                    // Graphviz's scanner consumes a backslash pair as one unit (both kept), so the
-                    // second backslash never escapes a closing quote: "x\\" is the text x\\
-                    i += 2;
-                    continue;
-                }
-                if (d === 0x0a || d === 0x0d) {
-                    out += text.slice(segment, i);
-                    i += d === 0x0d && text.charCodeAt(i + 2) === 0x0a ? 3 : 2;
-                    this.line++;
-                    segment = i;
-                    continue;
-                }
-                i++;
-                continue;
-            }
-            if (c === 0x0a || c === 0x0d) {
-                if (c === 0x0d && text.charCodeAt(i + 1) === 0x0a) {
-                    i++;
+            if (c === 0x5c && this.isQuotedSplice(i)) {
+                out += text.slice(segment, i);
+                if (text.codePointAt(i + 1) === 0x22) {
+                    out += '"';
                 }
-                this.line++;
+                i = this.quotedSpliceEnd(i);
+                segment = i;
+            } else {
+                i = this.quotedStep(i, c);
             }
-            i++;
         }
     }
 
+    /**
+     * Whether the backslash at `i` inside a quoted string starts a sequence that changes the decoded
+     * text: `\"` (an escaped quote) or a backslash before a line break (a line continuation).
+     * @param i - the offset of the backslash
+     * @returns true for an escaped quote or a line continuation
+     */
+    private isQuotedSplice(i: number): boolean {
+        const d = this.text.codePointAt(i + 1);
+        return d === 0x22 || d === 0x0a || d === 0x0d;
+    }
+
+    /**
+     * The offset just past an escaped quote or a line continuation, counting the line a
+     * continuation ends.
+     * @param i - the offset of the backslash
+     * @returns the offset after the sequence
+     */
+    private quotedSpliceEnd(i: number): number {
+        const { text } = this;
+        const d = text.charCodeAt(i + 1);
+        if (d === 0x22) {
+            return i + 2;
+        }
+        this.line++;
+        return d === 0x0d && text.codePointAt(i + 2) === 0x0a ? i + 3 : i + 2;
+    }
+
+    /**
+     * Step over one unit of a quoted string kept as written: a backslash pair (consumed as one unit,
+     * as Graphviz's scanner does, so the second backslash never escapes a closing quote: "x\\" is
+     * the text x\\), a lone backslash, a raw line break (counted) or any other character.
+     * @param i - the offset of the unit
+     * @param c - the code unit at `i`
+     * @returns the offset after the unit
+     */
+    private quotedStep(i: number, c: number): number {
+        const { text } = this;
+        if (c === 0x5c) {
+            return text.codePointAt(i + 1) === 0x5c ? i + 2 : i + 1;
+        }
+        if (c === 0x0a) {
+            this.line++;
+        } else if (c === 0x0d) {
+            this.line++;
+            return text.codePointAt(i + 1) === 0x0a ? i + 2 : i + 1;
+        }
+        return i + 1;
+    }
+
     /**
      * Lex an HTML string starting at the opening angle bracket: the text up to the matching
      * closing bracket, brackets balanced, kept verbatim including the outer pair.
@@ -313,51 +351,63 @@ export class DotTokenizer {
      */
     private skipTrivia(): void {
         const { text } = this;
-        for (;;) {
-            if (this.pos >= text.length) {
-                return;
-            }
+        while (this.pos < text.length) {
             const c = text.charCodeAt(this.pos);
-            if (c === 0x0a) {
-                this.line++;
-                this.pos++;
-                this.skipPreprocessorLine();
-                continue;
-            }
-            if (c === 0x0d) {
-                this.line++;
-                this.pos += text.charCodeAt(this.pos + 1) === 0x0a ? 2 : 1;
-                this.skipPreprocessorLine();
-                continue;
-            }
-            if (c === 0x20 || c === 0x09 || c === 0x0b || c === 0x0c) {
-                this.pos++;
-                continue;
-            }
-            if (c === BOM_CODE) {
-                // a U+FEFF after the start (concatenated files) is whitespace; the shared text check reports it
+            if (isBlank(c)) {
                 this.pos++;
-                continue;
-            }
-            if (c === 0x2f) {
-                const d = text.charCodeAt(this.pos + 1);
-                if (d === 0x2f) {
-                    this.skipToLineEnd();
-                    continue;
-                }
-                if (d === 0x2a) {
-                    this.skipBlockComment();
-                    continue;
-                }
-            }
-            if (c === 0x23 && this.pos === 0) {
-                this.skipToLineEnd();
-                continue;
+            } else if (!this.skipBreakOrComment(c)) {
+                return;
             }
-            return;
         }
     }
 
+    /**
+     * Skip one line break, comment or leading `#` line at the current position.
+     * @param c - the code unit at the current position
+     * @returns false when the current position starts a token
+     */
+    private skipBreakOrComment(c: number): boolean {
+        if (c === 0x0a || c === 0x0d) {
+            this.skipLineBreak(c);
+            return true;
+        }
+        if (c === 0x2f) {
+            return this.skipComment();
+        }
+        if (c === 0x23 && this.pos === 0) {
+            this.skipToLineEnd();
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Consume the line break at the current position (CR LF as one) and a `#` line after it.
+     * @param c - the code unit at the current position, LF or CR
+     */
+    private skipLineBreak(c: number): void {
+        this.line++;
+        this.pos += c === 0x0d && this.text.codePointAt(this.pos + 1) === 0x0a ? 2 : 1;
+        this.skipPreprocessorLine();
+    }
+
+    /**
+     * Skip a `//` or block comment starting at the current `/`.
+     * @returns false when the `/` starts no comment
+     */
+    private skipComment(): boolean {
+        const d = this.text.codePointAt(this.pos + 1);
+        if (d === 0x2f) {
+            this.skipToLineEnd();
+            return true;
+        }
+        if (d === 0x2a) {
+            this.skipBlockComment();
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Discard a `#` line when one starts at the current position (the start of a line).
      */