@graphty/graph-io 0.3.40 → 0.3.42

package/src/formats/csv/importer.ts

@@ -385,6 +385,13 @@ interface AdjacencyPlan {
 }
 
 /** Everything one import call shares between its tables. */
+/** A row read before the plan exists, kept to be pushed once it does. */
+interface PendingRow {
+    readonly row: string[];
+    readonly quoted: readonly boolean[];
+    readonly line: number;
+}
+
 interface ImportState {
     readonly sink: GraphSink;
     readonly report: ImportReportBuilder;
@@ -455,6 +462,43 @@ function isPadded(text: string, quoted: boolean | undefined): boolean {
  */
 function resolveCsvOptions(options: (CsvImportOptions & CommonImportOptions) | undefined): ResolvedCsvOptions {
     const o: CsvImportOptions = options ?? {};
+    checkDialectOptions(o);
+    for (const name of ["sourceColumn", "targetColumn", "idColumn", "labelColumn"] as const) {
+        checkColumnRef(name, o[name]);
+    }
+    if (o.typeColumn !== null) {
+        checkColumnRef("typeColumn", o.typeColumn);
+    }
+    if (o.table === "adjacency") {
+        checkAdjacencyOptions(o);
+    }
+    if (o.rowNumberIds !== undefined && typeof o.rowNumberIds !== "boolean") {
+        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: expected a boolean", {
+            option: "rowNumberIds",
+            found: o.rowNumberIds,
+        });
+    }
+    return {
+        delimiter: o.delimiter ?? null,
+        decimalComma: o.decimal === ",",
+        header: o.header ?? "auto",
+        table: o.table ?? "auto",
+        sourceColumn: o.sourceColumn ?? null,
+        targetColumn: o.targetColumn ?? null,
+        typeColumn: o.typeColumn,
+        idColumn: o.idColumn ?? null,
+        labelColumn: o.labelColumn ?? null,
+        nodes: nodesInput(o.nodes),
+        rowNumberIds: o.rowNumberIds ?? false,
+        explicitDirected: typeof options?.defaultDirected === "boolean" ? options.defaultDirected : null,
+    };
+}
+
+/**
+ * Check the dialect and table options: delimiter, decimal, header and table.
+ * @param o - the caller's CSV options
+ */
+function checkDialectOptions(o: CsvImportOptions): void {
     if (o.delimiter !== undefined && (typeof o.delimiter !== "string" || o.delimiter.length === 0)) {
         throw new GraphFormatError("E_UNSUPPORTED", "option delimiter: expected a non-empty string", {
             option: "delimiter",
@@ -489,48 +533,27 @@ function resolveCsvOptions(options: (CsvImportOptions & CommonImportOptions) | u
             found: o.table,
         });
     }
-    for (const name of ["sourceColumn", "targetColumn", "idColumn", "labelColumn"] as const) {
-        checkColumnRef(name, o[name]);
-    }
-    if (o.typeColumn !== null) {
-        checkColumnRef("typeColumn", o.typeColumn);
-    }
-    if (o.table === "adjacency") {
-        for (const name of ["sourceColumn", "targetColumn", "typeColumn", "idColumn", "labelColumn"] as const) {
-            if (o[name] !== undefined) {
-                throw new GraphFormatError("E_UNSUPPORTED", `option ${name}: an adjacency table has no columns`, {
-                    option: name,
-                    found: o[name],
-                });
-            }
-        }
-        if (o.rowNumberIds === true) {
-            throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: an adjacency table names its nodes", {
-                option: "rowNumberIds",
-                found: o.rowNumberIds,
+}
+
+/**
+ * Check the options an adjacency table has no use for: column options and rowNumberIds.
+ * @param o - the caller's CSV options
+ */
+function checkAdjacencyOptions(o: CsvImportOptions): void {
+    for (const name of ["sourceColumn", "targetColumn", "typeColumn", "idColumn", "labelColumn"] as const) {
+        if (o[name] !== undefined) {
+            throw new GraphFormatError("E_UNSUPPORTED", `option ${name}: an adjacency table has no columns`, {
+                option: name,
+                found: o[name],
             });
         }
     }
-    if (o.rowNumberIds !== undefined && typeof o.rowNumberIds !== "boolean") {
-        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: expected a boolean", {
+    if (o.rowNumberIds === true) {
+        throw new GraphFormatError("E_UNSUPPORTED", "option rowNumberIds: an adjacency table names its nodes", {
             option: "rowNumberIds",
             found: o.rowNumberIds,
         });
     }
-    return {
-        delimiter: o.delimiter ?? null,
-        decimalComma: o.decimal === ",",
-        header: o.header ?? "auto",
-        table: o.table ?? "auto",
-        sourceColumn: o.sourceColumn ?? null,
-        targetColumn: o.targetColumn ?? null,
-        typeColumn: o.typeColumn,
-        idColumn: o.idColumn ?? null,
-        labelColumn: o.labelColumn ?? null,
-        nodes: nodesInput(o.nodes),
-        rowNumberIds: o.rowNumberIds ?? false,
-        explicitDirected: typeof options?.defaultDirected === "boolean" ? options.defaultDirected : null,
-    };
 }
 
 /**
@@ -575,6 +598,22 @@ function isUnset(text: string, quoted: boolean | undefined): boolean {
     return quoted !== true && isBlank(text);
 }
 
+/**
+ * The number of set neighbor cells of an adjacency row (every cell after the first).
+ * @param row - the cells
+ * @param quoted - whether each cell was quoted
+ * @returns the set neighbor cells
+ */
+function setNeighborCells(row: readonly string[], quoted: readonly boolean[]): number {
+    let neighbors = 0;
+    for (let k = 1; k < row.length; k++) {
+        if (!isUnset(row[k], quoted[k])) {
+            neighbors++;
+        }
+    }
+    return neighbors;
+}
+
 /** Rows between two checks of the cancellation signal on an in-memory input. */
 const ABORT_CHECK_INTERVAL = 64;
 
@@ -807,43 +846,12 @@ class TableReader {
             : first.value;
         const firstLine = this.reader.line;
         const firstQuoted = this.reader.quoted.slice(0, firstRow.length);
-        const pending: { row: string[]; quoted: readonly boolean[]; line: number }[] = [];
-        let header: boolean;
-        // an adjacency table has no header unless the caller says so: its rows vary in width; a
-        // table numbered by rowNumberIds has one, since a headerless table takes its ids from column 0
-        let mode = this.state.csv.header;
-        if (mode === "auto" && this.kind === "adjacency") {
-            mode = false;
-        } else if (mode === "auto" && this.rowNumbers) {
-            mode = true;
-        }
-        if (mode === "auto") {
-            const second = await iterator.next();
-            const secondRow: string[] | null = second.done ? null : second.value;
-            const secondLine = this.reader.line;
-            const secondQuoted = this.reader.quoted.slice(0, secondRow?.length ?? 0);
-            // a column option naming a cell of the first row makes that row the header
-            header = looksLikeHeader(firstRow, secondRow) || this.namesExplicitColumn(firstRow);
-            if (!header) {
-                pending.push({ row: firstRow, quoted: firstQuoted, line: firstLine });
-            }
-            if (secondRow !== null) {
-                pending.push({ row: secondRow, quoted: secondQuoted, line: secondLine });
-            }
-        } else {
-            header = mode;
-            if (!header) {
-                pending.push({ row: firstRow, quoted: firstQuoted, line: firstLine });
-            }
-        }
-        if (this.rowNumbers && !header) {
-            report.warning(
-                "unsupported",
-                OPTION_IGNORED_CODE,
-                "option rowNumberIds ignored: a headerless node table takes its ids from column 0",
-                { line: firstLine, element: "rowNumberIds" },
-            );
-        }
+        const pending: PendingRow[] = [];
+        const header = await this.decideHeader(
+            iterator,
+            { row: firstRow, quoted: firstQuoted, line: firstLine },
+            pending,
+        );
         const names = header ? uniqueNames(headerNames(firstRow), report, firstLine) : positionalNames(firstRow.length);
         const last = firstRow.length - 1;
         const optionalLast = header && last > 0 && isUnset(firstRow[last], firstQuoted[last]);
@@ -873,6 +881,80 @@ class TableReader {
         for (const { row, quoted, line } of pending) {
             this.processRow(row, quoted, line);
         }
+        await this.readRemainingRows(iterator);
+        if (this.dataRows === 0 && header) {
+            report.warning("missing-value", NO_DATA_ROWS_CODE, "the table has a header and no data rows", {
+                line: firstLine,
+            });
+        }
+        this.reportReaderFindings();
+    }
+
+    /**
+     * Decide whether the first row is the header, reading the second row when the header option is
+     * "auto", and queue the rows read so far that are data.
+     * @param iterator - the record iterator, past the first row
+     * @param first - the first row
+     * @param pending - receives the data rows read so far, in order
+     * @returns whether the first row is the header
+     */
+    private async decideHeader(
+        iterator: AsyncGenerator<string[], void, undefined>,
+        first: PendingRow,
+        pending: PendingRow[],
+    ): Promise<boolean> {
+        const mode = this.headerMode();
+        let second: PendingRow | null = null;
+        let header: boolean;
+        if (mode === "auto") {
+            const next = await iterator.next();
+            second = next.done
+                ? null
+                : { row: next.value, quoted: this.reader.quoted.slice(0, next.value.length), line: this.reader.line };
+            // a column option naming a cell of the first row makes that row the header
+            header = looksLikeHeader(first.row, second?.row ?? null) || this.namesExplicitColumn(first.row);
+        } else {
+            header = mode;
+        }
+        if (!header) {
+            pending.push(first);
+        }
+        if (second !== null) {
+            pending.push(second);
+        }
+        if (this.rowNumbers && !header) {
+            this.state.report.warning(
+                "unsupported",
+                OPTION_IGNORED_CODE,
+                "option rowNumberIds ignored: a headerless node table takes its ids from column 0",
+                { line: first.line, element: "rowNumberIds" },
+            );
+        }
+        return header;
+    }
+
+    /**
+     * The header option as this table reads it: an adjacency table has no header unless the caller
+     * says so (its rows vary in width); a table numbered by rowNumberIds has one, since a headerless
+     * table takes its ids from column 0.
+     * @returns true, false or "auto"
+     */
+    private headerMode(): boolean | "auto" {
+        const mode = this.state.csv.header;
+        if (mode === "auto" && this.kind === "adjacency") {
+            return false;
+        }
+        if (mode === "auto" && this.rowNumbers) {
+            return true;
+        }
+        return mode;
+    }
+
+    /**
+     * Push every row after the header and the rows read to decide it, then finish the column writers.
+     * @param iterator - the record iterator
+     */
+    private async readRemainingRows(iterator: AsyncGenerator<string[], void, undefined>): Promise<void> {
         const { signal } = this.state.common;
         let sinceCheck = 0;
         for (;;) {
@@ -889,12 +971,6 @@ class TableReader {
         for (const writer of this.writers) {
             writer?.finish();
         }
-        if (this.dataRows === 0 && header) {
-            report.warning("missing-value", NO_DATA_ROWS_CODE, "the table has a header and no data rows", {
-                line: firstLine,
-            });
-        }
-        this.reportReaderFindings();
     }
 
     /**
@@ -1520,12 +1596,7 @@ class TableReader {
         const { where } = this;
         where.line = line;
         where.element = null;
-        let neighbors = 0;
-        for (let k = 1; k < row.length; k++) {
-            if (!isUnset(row[k], quoted[k])) {
-                neighbors++;
-            }
-        }
+        const neighbors = setNeighborCells(row, quoted);
         if (isUnset(row[0], quoted[0])) {
             report.error("missing-value", MISSING_ID_CODE, `blank node cell`, { line });
             counts.skippedNodes++;

package/src/formats/csv/records.ts

@@ -410,16 +410,7 @@ export function sniffDelimiter(
         if (rows.length === 0) {
             continue;
         }
-        let total = 0;
-        let delta = 0;
-        for (let i = 0; i < rows.length; i++) {
-            const count = rows[i].length;
-            total += count;
-            if (i > 0) {
-                delta += Math.abs(count - rows[i - 1].length);
-            }
-        }
-        const average = total / rows.length;
+        const { average, delta } = rowWidths(rows);
         if (average > 1.99 && delta < bestDelta) {
             best = delimiter;
             bestDelta = delta;
@@ -428,6 +419,24 @@ export function sniffDelimiter(
     return best;
 }
 
+/**
+ * The average width of the rows and the sum of the width changes from each row to the next.
+ * @param rows - the rows (at least one)
+ * @returns the average and the delta
+ */
+function rowWidths(rows: readonly (readonly string[])[]): { average: number; delta: number } {
+    let total = 0;
+    let delta = 0;
+    for (let i = 0; i < rows.length; i++) {
+        const count = rows[i].length;
+        total += count;
+        if (i > 0) {
+            delta += Math.abs(count - rows[i - 1].length);
+        }
+    }
+    return { average: total / rows.length, delta };
+}
+
 /**
  * Whether a parsed row is a blank line: one cell holding only whitespace.
  * @param row - the row
@@ -577,14 +586,8 @@ export class RecordReader implements AsyncIterable<number> {
         }
         const pieces: string[] = [];
         let length = 0;
-        let breaks = 0;
-        let lastWasCr = false;
         let sniffed = false;
-        // the leading comment lines are left out of the sniff, so they do not count toward its rows
-        const commentCodes = (this.syntax.comments ?? []).map((c) => c.codePointAt(0));
-        let leading = commentCodes.length > 0;
-        let lineStart = true;
-        let inComment = false;
+        const preview = new PreviewBreaks((this.syntax.comments ?? []).map((c) => c.codePointAt(0)));
         for await (const chunk of source) {
             if (sniffed) {
                 yield chunk;
@@ -592,24 +595,8 @@ export class RecordReader implements AsyncIterable<number> {
             }
             pieces.push(chunk);
             length += chunk.length;
-            for (let i = 0; i < chunk.length && breaks < PREVIEW_ROWS; i++) {
-                const c = chunk.charCodeAt(i);
-                if (c === LF || c === CR) {
-                    if (!(c === LF && lastWasCr) && !inComment) {
-                        breaks++;
-                    }
-                    inComment = false;
-                    lineStart = true;
-                } else {
-                    if (lineStart && leading) {
-                        inComment = commentCodes.includes(c);
-                        leading = inComment;
-                    }
-                    lineStart = false;
-                }
-                lastWasCr = c === CR;
-            }
-            if (breaks < PREVIEW_ROWS && length < PREVIEW_CHARS) {
+            preview.scan(chunk);
+            if (preview.breaks < PREVIEW_ROWS && length < PREVIEW_CHARS) {
                 continue;
             }
             const text = pieces.join("");
@@ -659,6 +646,74 @@ export class RecordReader implements AsyncIterable<number> {
     }
 }
 
+/**
+ * Counts the line breaks of the text buffered for the delimiter sniff, up to PREVIEW_ROWS. The
+ * leading comment lines are left out of the sniff, so they do not count toward its rows.
+ */
+class PreviewBreaks {
+    /** The line breaks counted so far (at most PREVIEW_ROWS). */
+    breaks = 0;
+
+    private readonly commentCodes: readonly (number | undefined)[];
+
+    private lastWasCr = false;
+
+    private leading: boolean;
+
+    private lineStart = true;
+
+    private inComment = false;
+
+    /**
+     * Start counting at the start of the input.
+     * @param commentCodes - the code points of the comment characters
+     */
+    constructor(commentCodes: readonly (number | undefined)[]) {
+        this.commentCodes = commentCodes;
+        this.leading = commentCodes.length > 0;
+    }
+
+    /**
+     * Count the line breaks of the next chunk, stopping at PREVIEW_ROWS.
+     * @param chunk - the chunk
+     */
+    scan(chunk: string): void {
+        for (let i = 0; i < chunk.length && this.breaks < PREVIEW_ROWS; i++) {
+            const c = chunk.charCodeAt(i);
+            if (c === LF || c === CR) {
+                this.lineBreak(c);
+            } else {
+                this.character(c);
+            }
+            this.lastWasCr = c === CR;
+        }
+    }
+
+    /**
+     * A line break: counted unless it is the LF of a CR LF or ends a leading comment line.
+     * @param c - LF or CR
+     */
+    private lineBreak(c: number): void {
+        if (!(c === LF && this.lastWasCr) && !this.inComment) {
+            this.breaks++;
+        }
+        this.inComment = false;
+        this.lineStart = true;
+    }
+
+    /**
+     * Any other character: the first of a line decides whether a leading comment line starts.
+     * @param c - the character code
+     */
+    private character(c: number): void {
+        if (this.lineStart && this.leading) {
+            this.inComment = this.commentCodes.includes(c);
+            this.leading = this.inComment;
+        }
+        this.lineStart = false;
+    }
+}
+
 /**
  * The record state machine of RecordReader, resumable at every record end so the reader can
  * yield a record before the next one overwrites the shared cell arrays. Every state lives on the

package/src/formats/cys/tables.ts

@@ -217,24 +217,25 @@ export function cellAtt(
     namespace: string | null,
     hidden: boolean,
 ): AttRec | null {
+    const cell = { line, namespace, hidden };
     if (text.startsWith("=")) {
-        return att(column.name, column.list ? "List" : column.type, null, text, line, namespace, hidden, true);
+        return att(column.name, column.list ? "List" : column.type, null, text, true, cell);
     }
     if (!column.list) {
         if (text.length === 0 && column.type !== "String") {
             return null;
         }
-        return att(column.name, column.type, null, text, line, namespace, hidden, false);
+        return att(column.name, column.type, null, text, false, cell);
     }
     if (text.length === 0 && column.type !== "String") {
         return null;
     }
-    const list = att(column.name, "List", column.type, null, line, namespace, hidden, false);
+    const list = att(column.name, "List", column.type, null, false, cell);
     for (const item of javaSplitLines(text)) {
         if (item.length === 0 && column.type !== "String") {
             continue;
         }
-        list.children.push(att(column.name, column.type, null, item, line, namespace, hidden, false));
+        list.children.push(att(column.name, column.type, null, item, false, cell));
     }
     return list;
 }
@@ -262,10 +263,11 @@ function javaSplitLines(text: string): string[] {
  * @param cyType - the Cytoscape type
  * @param elementType - a list's item type, or null
  * @param value - the value, or null
- * @param line - the line
- * @param namespace - the table namespace, or null
- * @param hidden - a hidden column
  * @param equation - the value is a formula
+ * @param cell - where the cell sits
+ * @param cell.line - the line
+ * @param cell.namespace - the table namespace, or null
+ * @param cell.hidden - a hidden column
  * @returns the record
  */
 function att(
@@ -273,11 +275,10 @@ function att(
     cyType: string,
     elementType: string | null,
     value: string | null,
-    line: number,
-    namespace: string | null,
-    hidden: boolean,
     equation: boolean,
+    cell: { readonly line: number; readonly namespace: string | null; readonly hidden: boolean },
 ): AttRec {
+    const { line, namespace, hidden } = cell;
     return {
         name,
         type: null,