@graphty/graph-io 0.2.4 → 0.3.0

package/src/formats/pajek/importer.ts

@@ -26,7 +26,16 @@ import {
 } from "@graphty/graph-format";
 
 import { declareResolved, RENAMED_CODE, ROLE_TAKEN_CODE } from "../../common/attributes.js";
-import { DUPLICATE_NODE_CODE, INVALID_UTF8_CODE, OPTION_IGNORED_CODE, SYNTAX_CODE } from "../../common/codes.js";
+import {
+    DUPLICATE_NODE_CODE,
+    ENCODING_FALLBACK_CODE,
+    INVALID_ENCODING_CODE,
+    INVALID_UTF8_CODE,
+    MULTIPLE_GRAPHS_CODE,
+    OPTION_IGNORED_CODE,
+    SYNTAX_CODE,
+    UNKNOWN_ENCODING_CODE,
+} from "../../common/codes.js";
 import { DirectionResolver, type EdgeKind } from "../../common/direction.js";
 import { coerceIdText, ID_MERGED_CODE, IdCoercer } from "../../common/ids.js";
 import { LineReader, throwIfAborted } from "../../common/input.js";
@@ -75,12 +84,18 @@ export interface PajekImportOptions {
 export const PAJEK_ISSUE = Object.freeze({
     /** The input holds invalid UTF-8 (fatal). */
     INVALID_UTF8: INVALID_UTF8_CODE,
+    /** Invalid bytes in the encoding a BOM, a declaration or the encoding option chose (fatal). */
+    INVALID_ENCODING: INVALID_ENCODING_CODE,
+    /** Bytes that are not UTF-8 and declare no encoding were read as windows-1252. */
+    ENCODING_FALLBACK: ENCODING_FALLBACK_CODE,
+    /** A declared encoding the platform cannot decode was ignored. */
+    UNKNOWN_ENCODING: UNKNOWN_ENCODING_CODE,
     /** Fatal: no `*Vertices` section (an empty file, or not a Pajek network). */
     NO_VERTICES: "E_PAJEK_NO_VERTICES",
     /** Fatal: `*Vertices` without a vertex count, or one the sink cannot hold. */
     VERTICES_COUNT: "E_PAJEK_VERTICES_COUNT",
-    /** Aborts: a second `*Vertices` or `*Network` section; only one network per file is read. */
-    MULTIPLE_NETWORKS: "E_PAJEK_MULTIPLE_NETWORKS",
+    /** A project file holds several networks; import() reads the first, importAll() reads every one. */
+    MULTIPLE_GRAPHS: MULTIPLE_GRAPHS_CODE,
     /** A data line before the first section header. */
     OUTSIDE_SECTION: "E_PAJEK_OUTSIDE_SECTION",
     /** A section header the importer cannot parse. */
@@ -329,6 +344,9 @@ class PajekParser {
 
     private readonly nodeParams = new Map<string, TextCellWriter>();
 
+    /** Set when a header line starts the next network of a project file (that line is not consumed). */
+    private ended = false;
+
     private readonly edgeParams = new Map<string, TextCellWriter>();
 
     /**
@@ -356,15 +374,16 @@ class PajekParser {
      * Handle one line; every problem of the line is recorded as an issue and the line skipped.
      * @param text - the line without its terminator
      * @param line - the 1-based line number
+     * @returns true when the line is the header of the next network (not consumed: this network is complete)
      */
-    line(text: string, line: number): void {
+    line(text: string, line: number): boolean {
         if (isCommentOrBlank(text)) {
-            return;
+            return false;
         }
         try {
             if (isSectionLine(text)) {
                 this.header(text, line);
-                return;
+                return this.ended;
             }
             switch (this.section) {
                 case "none":
@@ -375,18 +394,18 @@ class PajekParser {
                     );
                 case "vertices":
                     this.vertexLine(text, line);
-                    return;
+                    return false;
                 case "lines":
                     this.edgeLine(text, line);
-                    return;
+                    return false;
                 case "list":
                     this.listLine(text, line);
-                    return;
+                    return false;
                 case "matrix":
                     this.matrixLine(text, line);
-                    return;
+                    return false;
                 case "skip":
-                    return;
+                    return false;
                 default: {
                     const name: string = this.section;
                     throw new Error(`unknown section state ${name}`);
@@ -395,9 +414,10 @@ class PajekParser {
         } catch (err) {
             if (err instanceof LineError) {
                 this.report.error(err.category, err.code, err.message, { line });
-                return;
+                return false;
             }
             this.report.recordError(err, { line });
+            return false;
         }
     }
 
@@ -460,7 +480,8 @@ class PajekParser {
         switch (h.kind) {
             case "network":
                 if (this.networkName !== null || this.vertexCount >= 0) {
-                    this.refuseSecondNetwork(h.keyword, line);
+                    this.ended = true;
+                    return;
                 }
                 this.networkName = h.name ?? "";
                 this.section = "none";
@@ -471,6 +492,11 @@ class PajekParser {
                     this.section = "skip";
                     return;
                 }
+                if (this.vertexCount >= 0) {
+                    // a second `*Vertices` without a `*Network` header starts the next network
+                    this.ended = true;
+                    return;
+                }
                 this.verticesHeader(h, line);
                 return;
             case "arcs":
@@ -507,9 +533,6 @@ class PajekParser {
      * @param line - the line number
      */
     private verticesHeader(h: SectionHeader, line: number): void {
-        if (this.vertexCount >= 0) {
-            this.refuseSecondNetwork(h.keyword, line);
-        }
         if (h.count === null) {
             this.report.fail(PAJEK_ISSUE.VERTICES_COUNT, "*Vertices needs a vertex count", { line });
         }
@@ -572,17 +595,6 @@ class PajekParser {
         this.relation = h.relation === null ? null : (h.name ?? String(h.relation));
     }
 
-    /**
-     * Refuse a second network: an issue, then ImportError with the report so far.
-     * @param keyword - the header keyword
-     * @param line - the line number
-     */
-    private refuseSecondNetwork(keyword: string, line: number): never {
-        const message = `*${keyword} starts a second network; the importer reads one network per file`;
-        this.report.error("unsupported", PAJEK_ISSUE.MULTIPLE_NETWORKS, message, { line });
-        throw this.report.abort(message, { code: PAJEK_ISSUE.MULTIPLE_NETWORKS, line });
-    }
-
     /**
      * Checks that run when a section ends.
      */
@@ -1296,16 +1308,121 @@ export const pajekImporter: GraphImporter<PajekImportOptions> = Object.freeze({
         reportUnusedOptions(options, report, USED_OPTIONS);
         const parser = new PajekParser(sink, report, resolved, firstVertex);
         const reader = new LineReader(input, report, resolved);
+        // after the first network: only its successors' headers are counted
+        let rest: NetworkCounter | null = null;
+        let restLine = 0;
         let sinceCheck = 0;
         for await (const text of reader) {
-            parser.line(text, reader.line);
+            if (rest !== null) {
+                rest.line(text);
+            } else if (parser.line(text, reader.line)) {
+                rest = new NetworkCounter();
+                rest.line(text);
+                restLine = reader.line;
+            }
             if (++sinceCheck >= ABORT_CHECK_INTERVAL) {
                 sinceCheck = 0;
                 throwIfAborted(resolved.signal);
             }
         }
         parser.finish();
+        if (rest !== null) {
+            report.warning(
+                "unsupported",
+                PAJEK_ISSUE.MULTIPLE_GRAPHS,
+                `the project file holds ${rest.count} more network(s) after the first; import() reads the first, importAll() reads every one`,
+                { line: restLine },
+            );
+        }
         throwIfAborted(resolved.signal);
         return report.finish();
     },
+
+    /**
+     * Read every network of a Pajek project file (`.paj`), each into its own sink. A network starts
+     * at `*Network` or at a `*Vertices` that does not belong to a `*Partition` / `*Vector` section.
+     * @param input - the text, bytes or stream
+     * @param sinkFor - the sink of the network with this index, called before its first push
+     * @param options - common and Pajek options
+     * @returns one report per network
+     */
+    async importAll(
+        input: ImportInput,
+        sinkFor: (index: number) => GraphSink,
+        options?: PajekImportOptions & CommonImportOptions,
+    ): Promise<ImportReport[]> {
+        const resolved = resolveImportOptions(options, DEFAULTS);
+        const firstVertex = firstVertexOption(options?.firstVertex);
+        const reports = [new ImportReportBuilder("pajek", resolved.errorLimit)];
+        reportUnusedOptions(options, reports[0], USED_OPTIONS);
+        const open = (): PajekParser => {
+            const report = reports[reports.length - 1];
+            const sink = sinkFor(reports.length - 1);
+            reportSinkOptions(sink, options, report);
+            return new PajekParser(sink, report, resolved, firstVertex);
+        };
+        let parser = open();
+        // decoding issues belong to the file, so they go to the first network's report
+        const reader = new LineReader(input, reports[0], resolved);
+        let sinceCheck = 0;
+        for await (const text of reader) {
+            if (parser.line(text, reader.line)) {
+                parser.finish();
+                reports.push(new ImportReportBuilder("pajek", resolved.errorLimit));
+                parser = open();
+                parser.line(text, reader.line);
+            }
+            if (++sinceCheck >= ABORT_CHECK_INTERVAL) {
+                sinceCheck = 0;
+                throwIfAborted(resolved.signal);
+            }
+        }
+        parser.finish();
+        throwIfAborted(resolved.signal);
+        return reports.map((r) => r.finish());
+    },
 });
+
+/**
+ * Counts the networks of the rest of a project file by their headers alone, with the parser's
+ * rule: a network starts at `*Network` after a named or populated one, or at a `*Vertices` after a
+ * populated one unless it belongs to a skipped section (`*Partition`, `*Vector`, ...).
+ */
+class NetworkCounter {
+    /** Networks started so far (the first header fed opens the first). */
+    count = 0;
+
+    private named = false;
+
+    private populated = false;
+
+    private skipping = false;
+
+    /**
+     * Feed one line.
+     * @param text - the line
+     */
+    line(text: string): void {
+        if (!isSectionLine(text)) {
+            return;
+        }
+        const h = parseSectionHeader(text);
+        const kind = h === null ? "unsupported" : h.kind;
+        const { skipping } = this;
+        this.skipping = kind === "unsupported" || (kind === "vertices" && skipping && this.populated);
+        if (kind === "network") {
+            if (this.count === 0 || this.named || this.populated) {
+                this.count++;
+                this.populated = false;
+            }
+            this.named = true;
+        } else if (kind === "vertices" && !this.skipping) {
+            if (this.count === 0 || this.populated) {
+                this.count++;
+                this.named = false;
+            }
+            this.populated = true;
+        }
+    }
+}
+

package/src/index.ts

@@ -40,6 +40,7 @@ export {
     type ExportGraphOptions,
     exportGraphToString,
     FormatRegistry,
+    importAllGraphs,
     importGraph,
     type ImportGraphOptions,
     type ImportGraphResult,

package/src/registry.ts

@@ -214,56 +214,84 @@ export class FormatRegistry {
      * own a sink). The format is the one named in the options, else sniffed from the filename,
      * the MIME type and the first bytes of the content; the builder is seeded from the common
      * options with `directed: true` as a placeholder that the importer overrides from the file.
+     * An input holding several graphs yields the first, with a warning naming how many were
+     * skipped; importAllGraphs() returns every one.
      * @param input - the text, bytes, stream or chunks to read
      * @param options - the format, hints, common and format-specific import options
      * @returns the snapshot, the import report and the freeze report
      */
     async importGraph(input: ImportInput, options: ImportGraphOptions = {}): Promise<ImportGraphResult> {
-        const requested = options.format ?? "auto";
-        let importer: GraphImporter;
-        let sniff: SniffResult | null = null;
-        let source = input;
-        let peeked: PeekedInput | null = null;
-        if (requested === "auto") {
-            peeked = await peekHead(input, SNIFF_HEAD_BYTES, options.signal ?? null);
-            source = peeked.input;
-            sniff = this.sniff({ filename: options.filename, mimeType: options.mimeType, head: peeked.head });
-            if (sniff === null) {
-                const report = new ImportReportBuilder("unknown", 0);
-                return report.fail(
-                    UNKNOWN_FORMAT_CODE,
-                    `no registered importer recognises the input${describeHints(options)}; pass the format explicitly`,
-                    undefined,
-                    { formats: this.formats() },
-                );
-            }
-            importer = this.importer(sniff.format);
-        } else {
-            importer = this.importer(requested);
-        }
-        const builder = new GraphBuilder({
-            weightDtype: options.weightDtype ?? "f64",
-            ...options.builder,
-            directed: true,
-            addMissingNodes: options.addMissingNodes ?? true,
-            duplicateEdges: options.duplicateEdges ?? "keep",
-            selfLoops: options.selfLoops ?? "keep",
-        });
+        const chosen = await this.choose(input, options);
+        const builder = seededBuilder(options);
         let report: ImportReport;
         try {
-            report = await importer.import(source, builder, importerOptions(options));
+            report = await chosen.importer.import(chosen.source, builder, importerOptions(options));
         } catch (err) {
-            await peeked?.close();
+            await chosen.peeked?.close();
             throw err;
         }
-        const frozen = builder.freezeWithReport(options.freeze);
-        return Object.freeze({
-            format: importer.format,
-            sniff,
-            snapshot: frozen.snapshot,
-            report,
-            freeze: frozen.report,
-        });
+        return result(chosen, builder, report, options);
+    }
+
+    /**
+     * Read every graph of an input (a DOT file with several graphs, a Pajek project with several
+     * networks, a JGF document with a `graphs` array), each into its own fresh builder, frozen.
+     * A format whose importer has no importAll() holds one graph per input, so the result has one
+     * entry. Options, sniffing and the builder seed are those of importGraph().
+     * @param input - the text, bytes, stream or chunks to read
+     * @param options - the format, hints, common and format-specific import options
+     * @returns one result per graph, in document order
+     */
+    async importAllGraphs(input: ImportInput, options: ImportGraphOptions = {}): Promise<ImportGraphResult[]> {
+        const chosen = await this.choose(input, options);
+        const { importer } = chosen;
+        const builders: GraphBuilder[] = [];
+        let reports: ImportReport[];
+        try {
+            if (importer.importAll === undefined) {
+                builders.push(seededBuilder(options));
+                reports = [await importer.import(chosen.source, builders[0], importerOptions(options))];
+            } else {
+                reports = await importer.importAll(
+                    chosen.source,
+                    () => {
+                        const builder = seededBuilder(options);
+                        builders.push(builder);
+                        return builder;
+                    },
+                    importerOptions(options),
+                );
+            }
+        } catch (err) {
+            await chosen.peeked?.close();
+            throw err;
+        }
+        return reports.map((report, i) => result(chosen, builders[i], report, options));
+    }
+
+    /**
+     * The importer for an input: the named format, or the sniffed one.
+     * @param input - the input
+     * @param options - the importGraph options
+     * @returns the importer, the sniff, and the input to read (replayed for a stream)
+     */
+    private async choose(input: ImportInput, options: ImportGraphOptions): Promise<ChosenImporter> {
+        const requested = options.format ?? "auto";
+        if (requested !== "auto") {
+            return { importer: this.importer(requested), sniff: null, source: input, peeked: null };
+        }
+        const peeked = await peekHead(input, SNIFF_HEAD_BYTES, options.signal ?? null);
+        const sniff = this.sniff({ filename: options.filename, mimeType: options.mimeType, head: peeked.head });
+        if (sniff === null) {
+            const report = new ImportReportBuilder("unknown", 0);
+            return report.fail(
+                UNKNOWN_FORMAT_CODE,
+                `no registered importer recognises the input${describeHints(options)}; pass the format explicitly`,
+                undefined,
+                { formats: this.formats() },
+            );
+        }
+        return { importer: this.importer(sniff.format), sniff, source: peeked.input, peeked };
     }
 
     /**
@@ -338,6 +366,16 @@ export function importGraph(input: ImportInput, options?: ImportGraphOptions): P
     return registry.importGraph(input, options);
 }
 
+/**
+ * Read every graph of an input, each into its own frozen snapshot, through the default registry.
+ * @param input - the text, bytes, stream or chunks to read
+ * @param options - the format, hints, common and format-specific import options
+ * @returns one result per graph, in document order
+ */
+export function importAllGraphs(input: ImportInput, options?: ImportGraphOptions): Promise<ImportGraphResult[]> {
+    return registry.importAllGraphs(input, options);
+}
+
 /**
  * Write a snapshot in a format through the default registry, as UTF-8 chunks.
  * @param snapshot - the snapshot
@@ -407,6 +445,59 @@ function importerOptions(options: ImportGraphOptions): CommonImportOptions {
     return out;
 }
 
+/** The importer chosen for an input, and the input to hand it. */
+interface ChosenImporter {
+    /** The importer. */
+    readonly importer: GraphImporter;
+    /** The sniff that chose it, or null when the caller named the format. */
+    readonly sniff: SniffResult | null;
+    /** The input to read: the original, or a replaying iterable for a peeked stream. */
+    readonly source: ImportInput;
+    /** The peeked head of a sniffed input, or null. */
+    readonly peeked: PeekedInput | null;
+}
+
+/**
+ * A fresh builder seeded from the common options, `directed: true` as a placeholder the importer
+ * overrides from the file.
+ * @param options - the importGraph options
+ * @returns the builder
+ */
+function seededBuilder(options: ImportGraphOptions): GraphBuilder {
+    return new GraphBuilder({
+        weightDtype: options.weightDtype ?? "f64",
+        ...options.builder,
+        directed: true,
+        addMissingNodes: options.addMissingNodes ?? true,
+        duplicateEdges: options.duplicateEdges ?? "keep",
+        selfLoops: options.selfLoops ?? "keep",
+    });
+}
+
+/**
+ * Freeze an imported builder into an importGraph result.
+ * @param chosen - the importer and its sniff
+ * @param builder - the filled builder
+ * @param report - the importer's report
+ * @param options - the importGraph options
+ * @returns the frozen result
+ */
+function result(
+    chosen: ChosenImporter,
+    builder: GraphBuilder,
+    report: ImportReport,
+    options: ImportGraphOptions,
+): ImportGraphResult {
+    const frozen = builder.freezeWithReport(options.freeze);
+    return Object.freeze({
+        format: chosen.importer.format,
+        sniff: chosen.sniff,
+        snapshot: frozen.snapshot,
+        report,
+        freeze: frozen.report,
+    });
+}
+
 /**
  * The E_UNSUPPORTED error of a format name nothing is registered under.
  * @param kind - importer or exporter

package/src/sniff.ts

@@ -98,7 +98,7 @@ export function normalizeMimeType(mimeType: string): string {
 
 /**
  * The head as bytes for the importers' sniff functions: at most SNIFF_HEAD_BYTES, a string
- * encoded as UTF-8.
+ * encoded as UTF-8, a UTF-16 head with a byte order mark transcoded to UTF-8.
  * @param head - the head as given
  * @returns the bytes
  */
@@ -106,6 +106,11 @@ export function headBytes(head: Uint8Array | string): Uint8Array {
     if (typeof head === "string") {
         return new TextEncoder().encode(head.slice(0, SNIFF_HEAD_BYTES)).subarray(0, SNIFF_HEAD_BYTES);
     }
+    if (head.byteLength >= 2 && ((head[0] === 0xff && head[1] === 0xfe) || (head[0] === 0xfe && head[1] === 0xff))) {
+        // UTF-16 with a BOM (Excel's "Unicode text"): the sniffers read UTF-8, so transcode the head
+        const encoding = head[0] === 0xff ? "utf-16le" : "utf-16be";
+        return headBytes(new TextDecoder(encoding).decode(head.subarray(0, SNIFF_HEAD_BYTES)));
+    }
     return head.byteLength > SNIFF_HEAD_BYTES ? head.subarray(0, SNIFF_HEAD_BYTES) : head;
 }
 

package/src/types.ts

@@ -19,9 +19,10 @@ import {
 
 /**
  * What an importer reads (design section 8.4): whole text, whole bytes, a byte stream (a browser
- * `File.stream()`, a fetch body) or an async iterable of text or byte chunks. Bytes are decoded as
- * UTF-8 with `fatal: true`, so an invalid sequence is a parse-error and never a silent U+FFFD that
- * could alias two ids.
+ * `File.stream()`, a fetch body) or an async iterable of text or byte chunks. Bytes are decoded
+ * strictly in the encoding the `encoding` option, a byte order mark or the file's own declaration
+ * names, else as UTF-8 (undeclared bytes that are not UTF-8 are read as windows-1252 with a
+ * warning), so an invalid sequence is a parse-error and never a silent U+FFFD that could alias two ids.
  */
 export type ImportInput = string | Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<string | Uint8Array>;
 
@@ -65,6 +66,13 @@ export interface CommonImportOptions {
     signal?: AbortSignal | undefined;
     /** Progress in bytes; `bytesTotal` is known for in-memory input only. */
     onProgress?: ((bytesDone: number, bytesTotal?: number) => void) | undefined;
+    /**
+     * The character encoding of byte input (a WHATWG label such as "utf-8", "windows-1252",
+     * "iso-8859-1", "utf-16le"); overrides the byte order mark and any encoding the file declares.
+     * Absent: a BOM decides, else the declaration (XML prolog, DOT `charset`), else UTF-8, and
+     * bytes that are not valid UTF-8 are read as windows-1252 with a warning. Ignored for text input.
+     */
+    encoding?: string | undefined;
 }
 
 /**
@@ -94,6 +102,21 @@ export interface GraphImporter<Opts = unknown> {
      * @returns the import report
      */
     import(input: ImportInput, sink: GraphSink, options?: Opts & CommonImportOptions): Promise<ImportReport>;
+    /**
+     * Read every graph of an input that can hold several (a DOT file with several graphs, a
+     * Pajek project with several networks, a JGF `graphs` array), each into its own sink. An
+     * importer without this method reads one graph per input. `import()` reads the first graph
+     * and warns how many it skipped.
+     * @param input - the text, bytes or stream to read
+     * @param sinkFor - called once per graph, in document order, before that graph's first push
+     * @param options - format-specific and common options
+     * @returns one report per graph, in document order
+     */
+    importAll?(
+        input: ImportInput,
+        sinkFor: (index: number) => GraphSink,
+        options?: Opts & CommonImportOptions,
+    ): Promise<ImportReport[]>;
 }
 
 /**