@graphty/graph-io 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -7
- package/dist/chunks/{escape-DyI8JofU.js → escape-CKied3Ri.js} +16 -10
- package/dist/chunks/escape-CKied3Ri.js.map +1 -0
- package/dist/chunks/{importer-DbnGYr3_.js → importer-C6QcIRIb.js} +129 -39
- package/dist/chunks/importer-C6QcIRIb.js.map +1 -0
- package/dist/chunks/{importer-CpCpfbxr.js → importer-D5pAsweZ.js} +136 -31
- package/dist/chunks/importer-D5pAsweZ.js.map +1 -0
- package/dist/chunks/{importer-GozH8DkN.js → importer-DbMrAR_w.js} +10 -4
- package/dist/chunks/importer-DbMrAR_w.js.map +1 -0
- package/dist/chunks/{importer-CQnJuWJw.js → importer-DkzjTvHc.js} +82 -40
- package/dist/chunks/importer-DkzjTvHc.js.map +1 -0
- package/dist/chunks/{records-CGpxszm1.js → records-BKSMowhR.js} +3 -3
- package/dist/chunks/{records-CGpxszm1.js.map → records-BKSMowhR.js.map} +1 -1
- package/dist/chunks/{text-CajMdVFy.js → text-Dr0Ifpag.js} +2 -2
- package/dist/chunks/{text-CajMdVFy.js.map → text-Dr0Ifpag.js.map} +1 -1
- package/dist/chunks/{writer-DxSKC7TL.js → writer-BdMak4_J.js} +341 -112
- package/dist/chunks/writer-BdMak4_J.js.map +1 -0
- package/dist/csv.js +24 -6
- package/dist/csv.js.map +1 -1
- package/dist/dot.js +1 -1
- package/dist/gexf.js +42 -10
- package/dist/gexf.js.map +1 -1
- package/dist/gml.js +69 -25
- package/dist/gml.js.map +1 -1
- package/dist/graph-io.js +186 -129
- package/dist/graph-io.js.map +1 -1
- package/dist/graphml.js +1 -1
- package/dist/json.js +1 -1
- package/dist/neo4j.js +10 -4
- package/dist/neo4j.js.map +1 -1
- package/dist/pajek.js +1 -1
- package/dist/src/common/codes.d.ts +6 -0
- package/dist/src/common/codes.d.ts.map +1 -1
- package/dist/src/common/codes.js +6 -0
- package/dist/src/common/codes.js.map +1 -1
- package/dist/src/common/escape.d.ts +5 -4
- package/dist/src/common/escape.d.ts.map +1 -1
- package/dist/src/common/escape.js +6 -5
- package/dist/src/common/escape.js.map +1 -1
- package/dist/src/common/format.d.ts +3 -3
- package/dist/src/common/format.js +5 -5
- package/dist/src/common/format.js.map +1 -1
- package/dist/src/common/input.d.ts +24 -6
- package/dist/src/common/input.d.ts.map +1 -1
- package/dist/src/common/input.js +259 -23
- package/dist/src/common/input.js.map +1 -1
- package/dist/src/common/options.d.ts +2 -0
- package/dist/src/common/options.d.ts.map +1 -1
- package/dist/src/common/options.js +17 -0
- package/dist/src/common/options.js.map +1 -1
- package/dist/src/common/text.js +4 -4
- package/dist/src/common/text.js.map +1 -1
- package/dist/src/common/weights.d.ts.map +1 -1
- package/dist/src/common/weights.js +8 -0
- package/dist/src/common/weights.js.map +1 -1
- package/dist/src/common/xml.d.ts +7 -0
- package/dist/src/common/xml.d.ts.map +1 -1
- package/dist/src/common/xml.js +12 -0
- package/dist/src/common/xml.js.map +1 -1
- package/dist/src/formats/csv/exporter.d.ts +5 -1
- package/dist/src/formats/csv/exporter.d.ts.map +1 -1
- package/dist/src/formats/csv/exporter.js +8 -1
- package/dist/src/formats/csv/exporter.js.map +1 -1
- package/dist/src/formats/csv/importer.d.ts.map +1 -1
- package/dist/src/formats/csv/importer.js +1 -0
- package/dist/src/formats/csv/importer.js.map +1 -1
- package/dist/src/formats/csv/index.d.ts +6 -0
- package/dist/src/formats/csv/index.d.ts.map +1 -1
- package/dist/src/formats/csv/index.js +7 -1
- package/dist/src/formats/csv/index.js.map +1 -1
- package/dist/src/formats/csv/records.js +1 -1
- package/dist/src/formats/csv/records.js.map +1 -1
- package/dist/src/formats/dot/exporter.d.ts +1 -1
- package/dist/src/formats/dot/exporter.d.ts.map +1 -1
- package/dist/src/formats/dot/exporter.js +3 -3
- package/dist/src/formats/dot/exporter.js.map +1 -1
- package/dist/src/formats/dot/importer.d.ts +6 -0
- package/dist/src/formats/dot/importer.d.ts.map +1 -1
- package/dist/src/formats/dot/importer.js +151 -24
- package/dist/src/formats/dot/importer.js.map +1 -1
- package/dist/src/formats/gexf/exporter.d.ts.map +1 -1
- package/dist/src/formats/gexf/exporter.js +32 -7
- package/dist/src/formats/gexf/exporter.js.map +1 -1
- package/dist/src/formats/gexf/importer.d.ts.map +1 -1
- package/dist/src/formats/gexf/importer.js +2 -2
- package/dist/src/formats/gexf/importer.js.map +1 -1
- package/dist/src/formats/gexf/index.d.ts +6 -0
- package/dist/src/formats/gexf/index.d.ts.map +1 -1
- package/dist/src/formats/gexf/index.js +7 -1
- package/dist/src/formats/gexf/index.js.map +1 -1
- package/dist/src/formats/gml/importer.d.ts +2 -2
- package/dist/src/formats/gml/importer.d.ts.map +1 -1
- package/dist/src/formats/gml/importer.js +66 -25
- package/dist/src/formats/gml/importer.js.map +1 -1
- package/dist/src/formats/gml/index.d.ts +7 -1
- package/dist/src/formats/gml/index.d.ts.map +1 -1
- package/dist/src/formats/gml/index.js +8 -2
- package/dist/src/formats/gml/index.js.map +1 -1
- package/dist/src/formats/graphml/constants.d.ts +6 -0
- package/dist/src/formats/graphml/constants.d.ts.map +1 -1
- package/dist/src/formats/graphml/constants.js +7 -1
- package/dist/src/formats/graphml/constants.js.map +1 -1
- package/dist/src/formats/graphml/importer.d.ts.map +1 -1
- package/dist/src/formats/graphml/importer.js +2 -2
- package/dist/src/formats/graphml/importer.js.map +1 -1
- package/dist/src/formats/json/exporter.d.ts.map +1 -1
- package/dist/src/formats/json/exporter.js +14 -5
- package/dist/src/formats/json/exporter.js.map +1 -1
- package/dist/src/formats/json/importer.d.ts +8 -0
- package/dist/src/formats/json/importer.d.ts.map +1 -1
- package/dist/src/formats/json/importer.js +84 -37
- package/dist/src/formats/json/importer.js.map +1 -1
- package/dist/src/formats/neo4j/importer.js +1 -1
- package/dist/src/formats/neo4j/importer.js.map +1 -1
- package/dist/src/formats/neo4j/index.d.ts +6 -0
- package/dist/src/formats/neo4j/index.d.ts.map +1 -1
- package/dist/src/formats/neo4j/index.js +7 -1
- package/dist/src/formats/neo4j/index.js.map +1 -1
- package/dist/src/formats/pajek/exporter.d.ts +2 -0
- package/dist/src/formats/pajek/exporter.d.ts.map +1 -1
- package/dist/src/formats/pajek/exporter.js +13 -2
- package/dist/src/formats/pajek/exporter.js.map +1 -1
- package/dist/src/formats/pajek/importer.d.ts +8 -2
- package/dist/src/formats/pajek/importer.d.ts.map +1 -1
- package/dist/src/formats/pajek/importer.js +122 -26
- package/dist/src/formats/pajek/importer.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/registry.d.ts +26 -0
- package/dist/src/registry.d.ts.map +1 -1
- package/dist/src/registry.js +101 -35
- package/dist/src/registry.js.map +1 -1
- package/dist/src/sniff.d.ts +1 -1
- package/dist/src/sniff.d.ts.map +1 -1
- package/dist/src/sniff.js +6 -1
- package/dist/src/sniff.js.map +1 -1
- package/dist/src/types.d.ts +22 -3
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js.map +1 -1
- package/package.json +4 -3
- package/src/common/codes.ts +9 -0
- package/src/common/escape.ts +6 -5
- package/src/common/format.ts +5 -5
- package/src/common/input.ts +293 -22
- package/src/common/options.ts +24 -0
- package/src/common/text.ts +4 -4
- package/src/common/weights.ts +9 -0
- package/src/common/xml.ts +14 -0
- package/src/formats/csv/exporter.ts +12 -1
- package/src/formats/csv/importer.ts +1 -0
- package/src/formats/csv/index.ts +9 -0
- package/src/formats/csv/records.ts +1 -1
- package/src/formats/dot/exporter.ts +3 -3
- package/src/formats/dot/importer.ts +172 -32
- package/src/formats/gexf/exporter.ts +38 -7
- package/src/formats/gexf/importer.ts +12 -2
- package/src/formats/gexf/index.ts +9 -0
- package/src/formats/gml/importer.ts +75 -22
- package/src/formats/gml/index.ts +10 -1
- package/src/formats/graphml/constants.ts +9 -0
- package/src/formats/graphml/importer.ts +9 -2
- package/src/formats/json/exporter.ts +14 -5
- package/src/formats/json/importer.ts +104 -36
- package/src/formats/neo4j/importer.ts +1 -1
- package/src/formats/neo4j/index.ts +9 -0
- package/src/formats/pajek/exporter.ts +19 -2
- package/src/formats/pajek/importer.ts +145 -28
- package/src/index.ts +1 -0
- package/src/registry.ts +131 -40
- package/src/sniff.ts +6 -1
- package/src/types.ts +26 -3
- package/dist/chunks/escape-DyI8JofU.js.map +0 -1
- package/dist/chunks/importer-CQnJuWJw.js.map +0 -1
- package/dist/chunks/importer-CpCpfbxr.js.map +0 -1
- package/dist/chunks/importer-DbnGYr3_.js.map +0 -1
- package/dist/chunks/importer-GozH8DkN.js.map +0 -1
- package/dist/chunks/writer-DxSKC7TL.js.map +0 -1
- package/dist/tsconfig.build.tsbuildinfo +0 -1
package/dist/src/types.d.ts
CHANGED
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
import { type Dtype, type DuplicatePolicy, GraphFormatError, type GraphSink, type GraphSnapshot, type IdCoercion } from "@graphty/graph-format";
|
|
11
11
|
/**
|
|
12
12
|
* What an importer reads (design section 8.4): whole text, whole bytes, a byte stream (a browser
|
|
13
|
-
* `File.stream()`, a fetch body) or an async iterable of text or byte chunks. Bytes are decoded
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* `File.stream()`, a fetch body) or an async iterable of text or byte chunks. Bytes are decoded
|
|
14
|
+
* strictly in the encoding the `encoding` option, a byte order mark or the file's own declaration
|
|
15
|
+
* names, else as UTF-8 (undeclared bytes that are not UTF-8 are read as windows-1252 with a
|
|
16
|
+
* warning), so an invalid sequence is a parse-error and never a silent U+FFFD that could alias two ids.
|
|
16
17
|
*/
|
|
17
18
|
export type ImportInput = string | Uint8Array | ReadableStream<Uint8Array> | AsyncIterable<string | Uint8Array>;
|
|
18
19
|
/**
|
|
@@ -55,6 +56,13 @@ export interface CommonImportOptions {
|
|
|
55
56
|
signal?: AbortSignal | undefined;
|
|
56
57
|
/** Progress in bytes; `bytesTotal` is known for in-memory input only. */
|
|
57
58
|
onProgress?: ((bytesDone: number, bytesTotal?: number) => void) | undefined;
|
|
59
|
+
/**
|
|
60
|
+
* The character encoding of byte input (a WHATWG label such as "utf-8", "windows-1252",
|
|
61
|
+
* "iso-8859-1", "utf-16le"); overrides the byte order mark and any encoding the file declares.
|
|
62
|
+
* Absent: a BOM decides, else the declaration (XML prolog, DOT `charset`), else UTF-8, and
|
|
63
|
+
* bytes that are not valid UTF-8 are read as windows-1252 with a warning. Ignored for text input.
|
|
64
|
+
*/
|
|
65
|
+
encoding?: string | undefined;
|
|
58
66
|
}
|
|
59
67
|
/**
|
|
60
68
|
* An importer plugin (design section 8.4): pushes scalars into the caller's sink in one pass and
|
|
@@ -83,6 +91,17 @@ export interface GraphImporter<Opts = unknown> {
|
|
|
83
91
|
* @returns the import report
|
|
84
92
|
*/
|
|
85
93
|
import(input: ImportInput, sink: GraphSink, options?: Opts & CommonImportOptions): Promise<ImportReport>;
|
|
94
|
+
/**
|
|
95
|
+
* Read every graph of an input that can hold several (a DOT file with several graphs, a
|
|
96
|
+
* Pajek project with several networks, a JGF `graphs` array), each into its own sink. An
|
|
97
|
+
* importer without this method reads one graph per input. `import()` reads the first graph
|
|
98
|
+
* and warns how many it skipped.
|
|
99
|
+
* @param input - the text, bytes or stream to read
|
|
100
|
+
* @param sinkFor - called once per graph, in document order, before that graph's first push
|
|
101
|
+
* @param options - format-specific and common options
|
|
102
|
+
* @returns one report per graph, in document order
|
|
103
|
+
*/
|
|
104
|
+
importAll?(input: ImportInput, sinkFor: (index: number) => GraphSink, options?: Opts & CommonImportOptions): Promise<ImportReport[]>;
|
|
86
105
|
}
|
|
87
106
|
/**
|
|
88
107
|
* What a format can express without loss (design section 8.5): the fidelity matrix of research
|
package/dist/src/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACH,KAAK,KAAK,EACV,KAAK,eAAe,EACpB,gBAAgB,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,UAAU,EAClB,MAAM,uBAAuB,CAAC;AAE/B
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACH,KAAK,KAAK,EACV,KAAK,eAAe,EACpB,gBAAgB,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,UAAU,EAClB,MAAM,uBAAuB,CAAC;AAE/B;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AAEhH;;;;;GAKG;AACH,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IAC7B,kHAAkH;IAClH,UAAU,CAAC,EAAE,IAAI,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,mHAAmH;IACnH,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,gEAAgE;IAChE,cAAc,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;IAC7C,2DAA2D;IAC3D,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,uGAAuG;IACvG,gBAAgB,CAAC,EAAE,QAAQ,GAAG,UAAU,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAC;IAC9E,uFAAuF;IACvF,eAAe,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,4GAA4G;IAC5G,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,iGAAiG;IACjG,WAAW,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,SAAS,CAAC;IACxC,gGAAgG;IAChG,IAAI,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,SAAS,CAAC;IACpC,wGAAwG;IACxG,iBAAiB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACxC,0GAA0G;IAC1G,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC9D,+EAA+E;IAC/E,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,4FAA4F;IAC5F,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACjC,yEAAyE;IACzE,UAAU,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC5E;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACjC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,OAAO;IACzC,yFAAyF;IACzF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4CAA4C;IAC5C,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,0CAA0C;IAC1C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;;;;OAIG;IACH,KAAK,CAAC,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAAC;IACjC;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACzG;;;;;;;;;OASG;IACH,SAAS,CAAC,CACN,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,SAAS,EACrC,OAAO,CAAC,EAAE,IAAI,GAAG,mBAAmB,GACrC,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CAC9B;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B,iDAAiD;IACjD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,sBAAsB;IACtB,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,kBAAkB;IAClB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,sFAAsF;IACtF,QAAQ,CAAC,OAAO,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACnD,+CAA+C;IAC/C,QAAQ,CAAC,SAAS,EAAE,KAAK,GAAG,SAAS,GAAG,SAAS,GAAG,eAAe,CAAC;IACpE,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,wCAAwC;IACxC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,oBAAoB;IACpB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,2BAA2B;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,yBAAyB;IACzB,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,4CAA4C;IAC5C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,gBAAgB,CAAC;IACtE,8BAA8B;IAC9B,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAClC,yBAAyB;IACzB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,wDAAwD;IACxD,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;CACzB;AAED,mHAAmH;AACnH,MAAM,WAAW,QAAQ;IACrB,+CAA+C;IAC/C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,gDAAgD;IAChD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,wEAAwE;IACxE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,gGAAgG;AAChG,MAAM,WAAW,mBAAmB;IAChC,kHAAkH;IAClH,WAAW,CAAC,EAAE,OAAO,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC7C,iGAAiG;IACjG,gBAAgB,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,YAAY,GAAG,SAAS,CAAC;CACtE;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,GAAG,OAAO;IACzC,uBAAuB;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,mCAAmC;IACnC,QAAQ,CAAC,YAAY,EAAE,kBAAkB,CAAC;IAC1C;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG,mBAAmB,GAAG,SAAS,QAAQ,EAAE,CAAC;IAC1F;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACjG;;;;;OAKG;IACH,cAAc,CAAC,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,IAAI,GAAG,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAClG;AAED,6DAA6D;AAC7D,MAAM,MAAM,aAAa,GACrB,aAAa,GAAG,eAAe,GAAG,kBAAkB,GAAG,aAAa,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,CAAC;AAE/G,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IACxB,oBAAoB;IACpB,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;IACjC,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,6DAA6D;IAC7D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,4CAA4C;IAC5C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gFAAgF;IAChF,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,gFAAgF;AAChF,MAAM,WAAW,YAAY;IACzB,kCAAkC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sBAAsB;IACtB,QAAQ,CAAC,MAAM,EAAE;QACb,oBAAoB;QACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,oEAAoE;QACpE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,oCAAoC;QACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,oCAAoC;QACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,2EAA2E;QAC3E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;KAClC,CAAC;IACF,sCAAsC;IACtC,QAAQ,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC;IACxC,oCAAoC;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,sCAAsC;IACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,kEAAkE;IAClE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,CAAC;IACpC,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC/B;AAED;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,gBAAgB;IAC7C,sDAAsD;IACtD,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAE9B;;;;;OAKG;gBACS,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAKjG"}
|
package/dist/src/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAGH,gBAAgB,GAInB,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAGH,gBAAgB,GAInB,MAAM,uBAAuB,CAAC;AAuP/B;;;;GAIG;AACH,MAAM,OAAO,WAAY,SAAQ,gBAAgB;IAI7C;;;;;OAKG;IACH,YAAY,OAAe,EAAE,MAAoB,EAAE,OAA2C;QAC1F,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@graphty/graph-io",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Importers and exporters (GEXF, GraphML, GML, DOT, Pajek, CSV, JSON, Neo4j) for the @graphty/graph-format snapshot",
|
|
5
5
|
"author": "Adam Powers <apowers@ato.ms>",
|
|
6
6
|
"type": "module",
|
|
@@ -56,6 +56,7 @@
|
|
|
56
56
|
"sideEffects": false,
|
|
57
57
|
"files": [
|
|
58
58
|
"dist/",
|
|
59
|
+
"!dist/*.tsbuildinfo",
|
|
59
60
|
"src/",
|
|
60
61
|
"README.md",
|
|
61
62
|
"LICENSE"
|
|
@@ -91,7 +92,7 @@
|
|
|
91
92
|
},
|
|
92
93
|
"homepage": "https://github.com/graphty-org/graphty-monorepo/tree/master/graph-io#readme",
|
|
93
94
|
"dependencies": {
|
|
94
|
-
"@graphty/graph-format": "^1.0.
|
|
95
|
+
"@graphty/graph-format": "^1.0.3"
|
|
95
96
|
},
|
|
96
97
|
"peerDependencies": {
|
|
97
98
|
"@graphty/graph-format": "^1.0.0"
|
|
@@ -118,7 +119,7 @@
|
|
|
118
119
|
"test:ui": "vitest --ui",
|
|
119
120
|
"test:run": "vitest run",
|
|
120
121
|
"coverage": "vitest run --coverage",
|
|
121
|
-
"coverage:preview": "npx serve coverage -p
|
|
122
|
+
"coverage:preview": "npx serve coverage -p ${PORT:?start it through servherd, which sets PORT}",
|
|
122
123
|
"benchmark": "tsx benchmarks/run.ts",
|
|
123
124
|
"ready:commit": "npm run build:all && npm run lint && npm run typecheck:strict-consumer && npm run test:run"
|
|
124
125
|
}
|
package/src/common/codes.ts
CHANGED
|
@@ -75,6 +75,15 @@ export const OPTION_IGNORED_CODE = "W_OPTION_IGNORED";
|
|
|
75
75
|
/** An invalid UTF-8 sequence in the input (fatal). */
|
|
76
76
|
export const INVALID_UTF8_CODE = "E_INVALID_UTF8";
|
|
77
77
|
|
|
78
|
+
/** Bytes that are not valid in the encoding a BOM, a declaration or the `encoding` option chose (fatal). */
|
|
79
|
+
export const INVALID_ENCODING_CODE = "E_INVALID_ENCODING";
|
|
80
|
+
|
|
81
|
+
/** Bytes that are not valid UTF-8 (and declare no other encoding) were read as windows-1252. */
|
|
82
|
+
export const ENCODING_FALLBACK_CODE = "W_ENCODING_FALLBACK";
|
|
83
|
+
|
|
84
|
+
/** The file declares an encoding the platform's TextDecoder does not know; the declaration is ignored. */
|
|
85
|
+
export const UNKNOWN_ENCODING_CODE = "W_UNKNOWN_ENCODING";
|
|
86
|
+
|
|
78
87
|
/** The sink refused the file's direction (locked or non-empty); the file is read as the sink's. */
|
|
79
88
|
export const DIRECTION_REFUSED_CODE = "W_DIRECTION_REFUSED";
|
|
80
89
|
|
package/src/common/escape.ts
CHANGED
|
@@ -148,15 +148,16 @@ function isDotIdentifier(text: string): boolean {
|
|
|
148
148
|
|
|
149
149
|
/**
|
|
150
150
|
* Whether a text can be written as a DOT ID at all. Graphviz's scanner consumes a backslash pair
|
|
151
|
-
* `\\` as one unit
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
151
|
+
* `\\` as one unit, `\"` as an escaped quote and a backslash before a line break as a line
|
|
152
|
+
* continuation (both removed), left to right, so a backslash that precedes a double quote or a
|
|
153
|
+
* line break, or ends the text, cannot be written: the written `\\"` reads as a pair and a closing
|
|
154
|
+
* quote, a backslash before a line break vanishes with the break, and a trailing backslash escapes
|
|
155
|
+
* the closing quote. Every other text is writable; quoteDotId() writes it.
|
|
155
156
|
* @param text - the id, name or value text
|
|
156
157
|
* @returns true when quoteDotId(text) reads back as `text`
|
|
157
158
|
*/
|
|
158
159
|
export function isWritableDotText(text: string): boolean {
|
|
159
|
-
return !text.endsWith("\\") && !text.includes('\\"');
|
|
160
|
+
return !text.endsWith("\\") && !text.includes('\\"') && !/\\[\r\n]/.test(text);
|
|
160
161
|
}
|
|
161
162
|
|
|
162
163
|
/**
|
package/src/common/format.ts
CHANGED
|
@@ -9,14 +9,14 @@
|
|
|
9
9
|
/**
|
|
10
10
|
* The shortest decimal text that reads back to the same f32 value through Math.fround.
|
|
11
11
|
* @param value - an f32 value (a JS number holding one)
|
|
12
|
-
* @returns the text; "Infinity" / "-Infinity" / "NaN" for the non-finite values, "0" for
|
|
12
|
+
* @returns the text; "Infinity" / "-Infinity" / "NaN" for the non-finite values, "-0" for negative zero
|
|
13
13
|
*/
|
|
14
14
|
export function formatF32(value: number): string {
|
|
15
15
|
if (!Number.isFinite(value)) {
|
|
16
16
|
return String(value);
|
|
17
17
|
}
|
|
18
18
|
if (value === 0) {
|
|
19
|
-
return "0";
|
|
19
|
+
return Object.is(value, -0) ? "-0" : "0";
|
|
20
20
|
}
|
|
21
21
|
for (let digits = 1; digits <= 9; digits++) {
|
|
22
22
|
const text = value.toPrecision(digits);
|
|
@@ -29,12 +29,12 @@ export function formatF32(value: number): string {
|
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
31
|
* The shortest text of an f64 value: `String(x)`, which is already the shortest round-tripping
|
|
32
|
-
* decimal in JS.
|
|
32
|
+
* decimal in JS, except for negative zero, which `String()` writes as "0".
|
|
33
33
|
* @param value - the value
|
|
34
|
-
* @returns the text; "Infinity" / "-Infinity" / "NaN" for the non-finite values
|
|
34
|
+
* @returns the text; "Infinity" / "-Infinity" / "NaN" for the non-finite values, "-0" for negative zero
|
|
35
35
|
*/
|
|
36
36
|
export function formatF64(value: number): string {
|
|
37
|
-
return String(value);
|
|
37
|
+
return Object.is(value, -0) ? "-0" : String(value);
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
/**
|
package/src/common/input.ts
CHANGED
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Input handling shared by every importer (design section 8.4): the ImportInput union (whole
|
|
3
3
|
* text, whole bytes, a byte stream, an async iterable of text or byte chunks) read as a sequence of
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* text chunks, as lines, or as one string, with BOM handling, cancellation through an AbortSignal
|
|
5
|
+
* and byte progress.
|
|
6
6
|
*
|
|
7
|
-
* Bytes are decoded
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Bytes are decoded ONCE, here, for every importer. The encoding is, in order of precedence: the
|
|
8
|
+
* caller's `encoding` option; a byte order mark (UTF-8, UTF-16LE, UTF-16BE); the encoding the file
|
|
9
|
+
* declares (the importer's `declaredEncoding` reader: the XML prolog for GEXF and GraphML, the
|
|
10
|
+
* `charset` attribute for DOT); else UTF-8. Decoding is strict (`fatal: true`): an invalid sequence
|
|
11
|
+
* is never a silent U+FFFD that could alias two ids. Only in the last case, when bytes that are not
|
|
12
|
+
* valid UTF-8 appear while everything before them was ASCII, is the rest of the input read as
|
|
13
|
+
* windows-1252 (the superset of ISO 8859-1 that Excel, Pajek and older tools write), with the
|
|
14
|
+
* warning W_ENCODING_FALLBACK; invalid UTF-8 after valid non-ASCII UTF-8 is E_INVALID_UTF8. A
|
|
15
|
+
* leading U+FEFF is stripped from text and from decoded bytes alike.
|
|
11
16
|
*/
|
|
12
17
|
|
|
13
18
|
import { GraphFormatError } from "@graphty/graph-format";
|
|
14
19
|
|
|
15
20
|
import { type ImportInput } from "../types.js";
|
|
16
|
-
import { INVALID_UTF8_CODE } from "./codes.js";
|
|
21
|
+
import { ENCODING_FALLBACK_CODE, INVALID_ENCODING_CODE, INVALID_UTF8_CODE, UNKNOWN_ENCODING_CODE } from "./codes.js";
|
|
17
22
|
import { type ImportReportBuilder } from "./report.js";
|
|
18
23
|
|
|
19
24
|
export { INVALID_UTF8_CODE };
|
|
@@ -28,6 +33,251 @@ export interface ReadOptions {
|
|
|
28
33
|
readonly signal?: AbortSignal | null | undefined;
|
|
29
34
|
/** The progress callback, or null / undefined for none. */
|
|
30
35
|
readonly onProgress?: ((bytesDone: number, bytesTotal?: number) => void) | null | undefined;
|
|
36
|
+
/** The caller's `encoding` option (a label TextDecoder knows), or null / undefined for detection. */
|
|
37
|
+
readonly encoding?: string | null | undefined;
|
|
38
|
+
/**
|
|
39
|
+
* The format's reader of a declared encoding: given the head of the input decoded as
|
|
40
|
+
* windows-1252 (ASCII-compatible), the encoding label it declares, or null.
|
|
41
|
+
*/
|
|
42
|
+
readonly declaredEncoding?: ((head: string) => string | null) | undefined;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** How many leading bytes the declaration check sees (an XML prolog, a DOT `charset` near the top). */
|
|
46
|
+
const HEAD_BYTES = 1024;
|
|
47
|
+
|
|
48
|
+
/** How many leading bytes the BOM check needs. */
|
|
49
|
+
const BOM_BYTES = 3;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The canonical name of an encoding label, or null when the platform's TextDecoder does not know it.
|
|
53
|
+
* @param label - a WHATWG encoding label ("latin1", "UTF-16LE", "cp1252", ...)
|
|
54
|
+
* @returns the canonical name ("windows-1252", "utf-16le", ...) or null
|
|
55
|
+
*/
|
|
56
|
+
export function canonicalEncoding(label: string): string | null {
|
|
57
|
+
try {
|
|
58
|
+
return new TextDecoder(label.trim()).encoding;
|
|
59
|
+
} catch {
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The encoding a byte order mark announces.
|
|
66
|
+
* @param head - the first bytes
|
|
67
|
+
* @returns "utf-8", "utf-16le", "utf-16be" or null
|
|
68
|
+
*/
|
|
69
|
+
function bomEncoding(head: Uint8Array): string | null {
|
|
70
|
+
if (head.byteLength >= 3 && head[0] === 0xef && head[1] === 0xbb && head[2] === 0xbf) {
|
|
71
|
+
return "utf-8";
|
|
72
|
+
}
|
|
73
|
+
if (head.byteLength >= 2 && head[0] === 0xff && head[1] === 0xfe) {
|
|
74
|
+
return "utf-16le";
|
|
75
|
+
}
|
|
76
|
+
if (head.byteLength >= 2 && head[0] === 0xfe && head[1] === 0xff) {
|
|
77
|
+
return "utf-16be";
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The one byte decoder of an import: picks the encoding from the head (option, BOM, declaration,
|
|
84
|
+
* UTF-8) and decodes strictly, switching to windows-1252 with a warning when undeclared input
|
|
85
|
+
* turns out not to be UTF-8 while everything decoded so far was ASCII.
|
|
86
|
+
*/
|
|
87
|
+
class ByteDecoder {
|
|
88
|
+
private decoder: TextDecoder | null = null;
|
|
89
|
+
|
|
90
|
+
private encoding = "utf-8";
|
|
91
|
+
|
|
92
|
+
/** Whether a failed UTF-8 decode may switch to windows-1252 (nothing chose UTF-8 explicitly). */
|
|
93
|
+
private mayFallBack = false;
|
|
94
|
+
|
|
95
|
+
/** Whether every character decoded so far was ASCII (so switching encodings changes no earlier text). */
|
|
96
|
+
private asciiSoFar = true;
|
|
97
|
+
|
|
98
|
+
/** While asciiSoFar: the bytes the UTF-8 decoder holds back (a sequence split across chunks). */
|
|
99
|
+
private carry: Uint8Array = new Uint8Array(0);
|
|
100
|
+
|
|
101
|
+
/** Bytes decoded so far, for error positions. */
|
|
102
|
+
private offset = 0;
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Create the decoder of one import.
|
|
106
|
+
* @param report - where warnings and the fatal decode error go
|
|
107
|
+
* @param options - the encoding option and the format's declaration reader
|
|
108
|
+
*/
|
|
109
|
+
constructor(
|
|
110
|
+
private readonly report: ImportReportBuilder,
|
|
111
|
+
private readonly options: ReadOptions,
|
|
112
|
+
) {}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Whether start() ran.
|
|
116
|
+
* @returns true once the encoding is chosen
|
|
117
|
+
*/
|
|
118
|
+
get started(): boolean {
|
|
119
|
+
return this.decoder !== null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Choose the encoding from the first bytes of the input.
|
|
124
|
+
* @param head - the first bytes (up to HEAD_BYTES; fewer when the input is shorter)
|
|
125
|
+
*/
|
|
126
|
+
start(head: Uint8Array): void {
|
|
127
|
+
const explicit = this.options.encoding ?? null;
|
|
128
|
+
if (explicit !== null) {
|
|
129
|
+
this.use(explicit, false);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const bom = bomEncoding(head);
|
|
133
|
+
if (bom !== null) {
|
|
134
|
+
this.use(bom, false);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const declared = this.options.declaredEncoding?.(new TextDecoder("windows-1252").decode(head)) ?? null;
|
|
138
|
+
if (declared !== null) {
|
|
139
|
+
const canonical = canonicalEncoding(declared);
|
|
140
|
+
if (canonical === null) {
|
|
141
|
+
this.report.warning(
|
|
142
|
+
"unsupported",
|
|
143
|
+
UNKNOWN_ENCODING_CODE,
|
|
144
|
+
`the input declares the encoding ${JSON.stringify(declared)}, which this platform cannot decode; reading it as UTF-8`,
|
|
145
|
+
{ element: declared },
|
|
146
|
+
);
|
|
147
|
+
} else if (canonical !== "utf-8" && !canonical.startsWith("utf-16")) {
|
|
148
|
+
// a declared UTF-16 without a BOM is not UTF-16 (its declaration read as ASCII), and
|
|
149
|
+
// a declared UTF-8 gets the same undeclared treatment: tools often write the default
|
|
150
|
+
// prolog over Latin-1 bytes
|
|
151
|
+
this.use(canonical, false);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
this.use("utf-8", true);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Decode the next bytes.
|
|
160
|
+
* @param bytes - the bytes
|
|
161
|
+
* @param stream - whether more bytes follow
|
|
162
|
+
* @returns the text
|
|
163
|
+
*/
|
|
164
|
+
decode(bytes: Uint8Array, stream: boolean): string {
|
|
165
|
+
const decoder = this.decoder as TextDecoder;
|
|
166
|
+
let text: string;
|
|
167
|
+
try {
|
|
168
|
+
text = decoder.decode(bytes, { stream });
|
|
169
|
+
} catch (err) {
|
|
170
|
+
if (!(err instanceof TypeError)) {
|
|
171
|
+
throw err;
|
|
172
|
+
}
|
|
173
|
+
text = this.recover(bytes, stream);
|
|
174
|
+
}
|
|
175
|
+
if (this.asciiSoFar && this.mayFallBack) {
|
|
176
|
+
if (/[\u0080-\uffff]/.test(text)) {
|
|
177
|
+
this.asciiSoFar = false;
|
|
178
|
+
this.carry = new Uint8Array(0);
|
|
179
|
+
} else {
|
|
180
|
+
// every character so far is one ASCII byte, so what was not emitted is held back
|
|
181
|
+
const all = this.carry.byteLength === 0 ? bytes : concatBytes([this.carry, bytes]);
|
|
182
|
+
this.carry = all.slice(text.length);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
this.offset += bytes.byteLength;
|
|
186
|
+
return text;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* A decode failed: switch to windows-1252 when allowed, else fail the import.
|
|
191
|
+
* @param bytes - the bytes that failed
|
|
192
|
+
* @param stream - whether more bytes follow
|
|
193
|
+
* @returns the bytes decoded as windows-1252
|
|
194
|
+
*/
|
|
195
|
+
private recover(bytes: Uint8Array, stream: boolean): string {
|
|
196
|
+
const where = { byteOffset: this.offset };
|
|
197
|
+
const all = this.carry.byteLength === 0 ? bytes : concatBytes([this.carry, bytes]);
|
|
198
|
+
// a NUL byte never occurs in windows-1252 text: it marks binary data (or BOM-less UTF-16)
|
|
199
|
+
if (this.mayFallBack && this.asciiSoFar && !all.includes(0) && !startsWithUtf8(all)) {
|
|
200
|
+
this.report.warning(
|
|
201
|
+
"coercion",
|
|
202
|
+
ENCODING_FALLBACK_CODE,
|
|
203
|
+
`the input is not valid UTF-8 (near byte ${this.offset}) and declares no encoding; read as windows-1252 (pass the encoding option to choose another)`,
|
|
204
|
+
);
|
|
205
|
+
this.use("windows-1252", false);
|
|
206
|
+
return (this.decoder as TextDecoder).decode(all, { stream });
|
|
207
|
+
}
|
|
208
|
+
if (this.encoding === "utf-8") {
|
|
209
|
+
const after = this.mayFallBack ? " after valid non-ASCII UTF-8 text; pass the encoding option" : "";
|
|
210
|
+
return this.report.fail(
|
|
211
|
+
INVALID_UTF8_CODE,
|
|
212
|
+
`invalid UTF-8 near byte ${this.offset}${after}`,
|
|
213
|
+
undefined,
|
|
214
|
+
where,
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
return this.report.fail(
|
|
218
|
+
INVALID_ENCODING_CODE,
|
|
219
|
+
`the bytes near byte ${this.offset} are not valid ${this.encoding}`,
|
|
220
|
+
undefined,
|
|
221
|
+
where,
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Switch to an encoding.
|
|
227
|
+
* @param label - the encoding label
|
|
228
|
+
* @param mayFallBack - whether a UTF-8 failure may switch to windows-1252
|
|
229
|
+
*/
|
|
230
|
+
private use(label: string, mayFallBack: boolean): void {
|
|
231
|
+
this.decoder = new TextDecoder(label, { fatal: true, ignoreBOM: true });
|
|
232
|
+
this.encoding = this.decoder.encoding;
|
|
233
|
+
this.mayFallBack = mayFallBack;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Whether the first non-ASCII byte of a chunk starts a valid UTF-8 sequence: then the chunk holds
|
|
239
|
+
* valid non-ASCII UTF-8 before whatever made it fail, and the input is not windows-1252.
|
|
240
|
+
* @param bytes - a chunk that failed to decode as UTF-8
|
|
241
|
+
* @returns true when its first non-ASCII sequence is valid UTF-8
|
|
242
|
+
*/
|
|
243
|
+
function startsWithUtf8(bytes: Uint8Array): boolean {
|
|
244
|
+
const i = bytes.findIndex((b) => b >= 0x80);
|
|
245
|
+
const lead = bytes[i];
|
|
246
|
+
let length = 0;
|
|
247
|
+
if (lead >= 0xc2 && lead <= 0xdf) {
|
|
248
|
+
length = 2;
|
|
249
|
+
} else if (lead >= 0xe0 && lead <= 0xef) {
|
|
250
|
+
length = 3;
|
|
251
|
+
} else if (lead >= 0xf0 && lead <= 0xf4) {
|
|
252
|
+
length = 4;
|
|
253
|
+
}
|
|
254
|
+
if (i < 0 || length === 0 || i + length > bytes.byteLength) {
|
|
255
|
+
return false;
|
|
256
|
+
}
|
|
257
|
+
try {
|
|
258
|
+
new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(i, i + length));
|
|
259
|
+
return true;
|
|
260
|
+
} catch {
|
|
261
|
+
return false;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Join byte chunks.
|
|
267
|
+
* @param chunks - the chunks
|
|
268
|
+
* @returns one array holding them in order
|
|
269
|
+
*/
|
|
270
|
+
function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {
|
|
271
|
+
if (chunks.length === 1) {
|
|
272
|
+
return chunks[0];
|
|
273
|
+
}
|
|
274
|
+
const out = new Uint8Array(chunks.reduce((sum, c) => sum + c.byteLength, 0));
|
|
275
|
+
let offset = 0;
|
|
276
|
+
for (const chunk of chunks) {
|
|
277
|
+
out.set(chunk, offset);
|
|
278
|
+
offset += chunk.byteLength;
|
|
279
|
+
}
|
|
280
|
+
return out;
|
|
31
281
|
}
|
|
32
282
|
|
|
33
283
|
/**
|
|
@@ -143,22 +393,13 @@ export async function* textChunks(
|
|
|
143
393
|
onProgress?.(done, total ?? undefined);
|
|
144
394
|
return;
|
|
145
395
|
}
|
|
146
|
-
const decoder = new
|
|
147
|
-
const decode = (bytes: Uint8Array, stream: boolean): string => {
|
|
148
|
-
try {
|
|
149
|
-
return decoder.decode(bytes, { stream });
|
|
150
|
-
} catch (err) {
|
|
151
|
-
if (err instanceof TypeError) {
|
|
152
|
-
report.fail(INVALID_UTF8_CODE, `invalid UTF-8 near byte ${done}`, undefined, { byteOffset: done });
|
|
153
|
-
}
|
|
154
|
-
throw err;
|
|
155
|
-
}
|
|
156
|
-
};
|
|
396
|
+
const decoder = new ByteDecoder(report, options);
|
|
157
397
|
if (input instanceof Uint8Array) {
|
|
398
|
+
decoder.start(input.subarray(0, HEAD_BYTES));
|
|
158
399
|
for (let offset = 0; offset < input.byteLength; offset += DECODE_SLICE) {
|
|
159
400
|
throwIfAborted(signal);
|
|
160
401
|
const slice = input.subarray(offset, Math.min(offset + DECODE_SLICE, input.byteLength));
|
|
161
|
-
const text = emit(decode(slice, offset + DECODE_SLICE < input.byteLength));
|
|
402
|
+
const text = emit(decoder.decode(slice, offset + DECODE_SLICE < input.byteLength));
|
|
162
403
|
done = Math.min(offset + DECODE_SLICE, input.byteLength);
|
|
163
404
|
if (text.length > 0) {
|
|
164
405
|
yield text;
|
|
@@ -170,18 +411,42 @@ export async function* textChunks(
|
|
|
170
411
|
}
|
|
171
412
|
return;
|
|
172
413
|
}
|
|
414
|
+
// the first bytes of a stream are held back until the BOM check (and the declaration check,
|
|
415
|
+
// when the format has one) can see enough of them, or the input ended, or text arrived, so a
|
|
416
|
+
// BOM or a declaration split across small chunks is still seen
|
|
417
|
+
const needed = options.declaredEncoding === undefined ? BOM_BYTES : HEAD_BYTES;
|
|
418
|
+
let head: Uint8Array[] = [];
|
|
419
|
+
let headLength = 0;
|
|
420
|
+
const flushHead = (stream: boolean): string => {
|
|
421
|
+
const bytes = concatBytes(head);
|
|
422
|
+
head = [];
|
|
423
|
+
headLength = 0;
|
|
424
|
+
decoder.start(bytes.subarray(0, HEAD_BYTES));
|
|
425
|
+
return decoder.decode(bytes, stream);
|
|
426
|
+
};
|
|
173
427
|
const chunks = isReadableStream(input) ? streamChunks(input, signal) : input;
|
|
174
428
|
for await (const chunk of chunks) {
|
|
175
429
|
throwIfAborted(signal);
|
|
176
430
|
let text: string;
|
|
177
431
|
if (typeof chunk === "string") {
|
|
178
432
|
// finish any byte sequence still pending in the decoder before switching to text
|
|
179
|
-
|
|
433
|
+
let pending = "";
|
|
434
|
+
if (!decoder.started && headLength > 0) {
|
|
435
|
+
pending = flushHead(false);
|
|
436
|
+
} else if (decoder.started) {
|
|
437
|
+
pending = decoder.decode(new Uint8Array(0), false);
|
|
438
|
+
}
|
|
180
439
|
text = emit(pending + chunk);
|
|
181
440
|
done += chunk.length;
|
|
182
441
|
} else if (chunk instanceof Uint8Array) {
|
|
183
|
-
text = emit(decode(chunk, true));
|
|
184
442
|
done += chunk.byteLength;
|
|
443
|
+
if (decoder.started) {
|
|
444
|
+
text = emit(decoder.decode(chunk, true));
|
|
445
|
+
} else {
|
|
446
|
+
head.push(chunk);
|
|
447
|
+
headLength += chunk.byteLength;
|
|
448
|
+
text = headLength >= needed ? emit(flushHead(true)) : "";
|
|
449
|
+
}
|
|
185
450
|
} else {
|
|
186
451
|
throw new GraphFormatError("E_UNSUPPORTED", "an input chunk must be a string or a Uint8Array", {
|
|
187
452
|
reason: "chunk type",
|
|
@@ -193,7 +458,13 @@ export async function* textChunks(
|
|
|
193
458
|
}
|
|
194
459
|
onProgress?.(done);
|
|
195
460
|
}
|
|
196
|
-
|
|
461
|
+
let tail = "";
|
|
462
|
+
if (!decoder.started && headLength > 0) {
|
|
463
|
+
tail = flushHead(false);
|
|
464
|
+
} else if (decoder.started) {
|
|
465
|
+
tail = decoder.decode(new Uint8Array(0), false);
|
|
466
|
+
}
|
|
467
|
+
tail = emit(tail);
|
|
197
468
|
if (tail.length > 0) {
|
|
198
469
|
yield tail;
|
|
199
470
|
}
|
package/src/common/options.ts
CHANGED
|
@@ -9,6 +9,7 @@ import { type DuplicatePolicy, GraphFormatError, type GraphSink, type IdCoercion
|
|
|
9
9
|
|
|
10
10
|
import { type CommonExportOptions, type CommonImportOptions } from "../types.js";
|
|
11
11
|
import { OPTION_IGNORED_CODE, SINK_OPTION_CODE } from "./codes.js";
|
|
12
|
+
import { canonicalEncoding } from "./input.js";
|
|
12
13
|
import { type ImportReportBuilder } from "./report.js";
|
|
13
14
|
|
|
14
15
|
/** The defaults an importer supplies for the options whose default is per format (design section 8.4). */
|
|
@@ -59,6 +60,8 @@ export interface ResolvedImportOptions {
|
|
|
59
60
|
readonly signal: AbortSignal | null;
|
|
60
61
|
/** The progress callback, or null. */
|
|
61
62
|
readonly onProgress: ((bytesDone: number, bytesTotal?: number) => void) | null;
|
|
63
|
+
/** The caller's byte encoding (canonical name), or null to detect it. */
|
|
64
|
+
readonly encoding: string | null;
|
|
62
65
|
}
|
|
63
66
|
|
|
64
67
|
/** CommonExportOptions with every field present (design section 8.5 defaults applied). */
|
|
@@ -213,6 +216,7 @@ export function resolveImportOptions(
|
|
|
213
216
|
errorLimit: errorLimitOption(o.errorLimit),
|
|
214
217
|
signal: signalOption(o.signal),
|
|
215
218
|
onProgress: progressOption(o.onProgress),
|
|
219
|
+
encoding: encodingOption(o.encoding),
|
|
216
220
|
});
|
|
217
221
|
}
|
|
218
222
|
|
|
@@ -350,6 +354,26 @@ function progressOption(value: unknown): ((bytesDone: number, bytesTotal?: numbe
|
|
|
350
354
|
});
|
|
351
355
|
}
|
|
352
356
|
|
|
357
|
+
/**
|
|
358
|
+
* Resolve the encoding option: a label the platform's TextDecoder knows.
|
|
359
|
+
* @param value - the caller's value
|
|
360
|
+
* @returns the canonical encoding name, or null for detection
|
|
361
|
+
*/
|
|
362
|
+
function encodingOption(value: unknown): string | null {
|
|
363
|
+
if (value === undefined || value === null) {
|
|
364
|
+
return null;
|
|
365
|
+
}
|
|
366
|
+
const canonical = typeof value === "string" ? canonicalEncoding(value) : null;
|
|
367
|
+
if (canonical === null) {
|
|
368
|
+
throw new GraphFormatError(
|
|
369
|
+
"E_UNSUPPORTED",
|
|
370
|
+
`option encoding: ${describe(value)} is not an encoding this platform's TextDecoder knows`,
|
|
371
|
+
{ option: "encoding", found: value },
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
return canonical;
|
|
375
|
+
}
|
|
376
|
+
|
|
353
377
|
/**
|
|
354
378
|
* A short description of an option value for an error message.
|
|
355
379
|
* @param value - the value
|
package/src/common/text.ts
CHANGED
|
@@ -277,7 +277,9 @@ export class TextCellWriter {
|
|
|
277
277
|
export const WIDENING_UNSUPPORTED_CODE = "W_WIDENING_UNSUPPORTED";
|
|
278
278
|
|
|
279
279
|
/**
|
|
280
|
-
* The text dtype a parsed cell value implies on its own (what the sink's inference sees).
|
|
280
|
+
* The text dtype a parsed cell value implies on its own (what the sink's inference sees). It must
|
|
281
|
+
* be graph-format's inference rule exactly: -0 is an integer there, so a column holding only
|
|
282
|
+
* `-0.0` is inferred i32 and must be widened to f64, or the sign is lost.
|
|
281
283
|
* @param value - the parsed value
|
|
282
284
|
* @returns the dtype
|
|
283
285
|
*/
|
|
@@ -288,7 +290,5 @@ function kindOfValue(value: boolean | number | string): TextDtype {
|
|
|
288
290
|
if (typeof value === "string") {
|
|
289
291
|
return "string";
|
|
290
292
|
}
|
|
291
|
-
return Number.isInteger(value) && value >= -2147483648 && value <= 2147483647
|
|
292
|
-
? "i32"
|
|
293
|
-
: "f64";
|
|
293
|
+
return Number.isInteger(value) && value >= -2147483648 && value <= 2147483647 ? "i32" : "f64";
|
|
294
294
|
}
|
package/src/common/weights.ts
CHANGED
|
@@ -135,6 +135,9 @@ export interface ExplicitWeights {
|
|
|
135
135
|
text(e: number, integral?: boolean): string | null;
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
+
/** A weight of -0 keeps its sign: every importer's weight grammar reads "-0" back as -0. */
|
|
139
|
+
const NEGATIVE_ZERO_TEXT = "-0";
|
|
140
|
+
|
|
138
141
|
/** The dtypes a weight role column may have (an integer column declared by a caller included). */
|
|
139
142
|
const NUMERIC_DTYPES: ReadonlySet<string> = new Set(["f32", "f64", "i32", "u32", "u8"]);
|
|
140
143
|
|
|
@@ -175,6 +178,9 @@ export function explicitWeights(snapshot: GraphSnapshot): ExplicitWeights {
|
|
|
175
178
|
return null;
|
|
176
179
|
}
|
|
177
180
|
const value = shadow.value(e) as number;
|
|
181
|
+
if (Object.is(value, -0)) {
|
|
182
|
+
return NEGATIVE_ZERO_TEXT;
|
|
183
|
+
}
|
|
178
184
|
return integral && Number.isInteger(value) ? formatInteger(value) : format(value);
|
|
179
185
|
},
|
|
180
186
|
};
|
|
@@ -196,6 +202,9 @@ export function explicitWeights(snapshot: GraphSnapshot): ExplicitWeights {
|
|
|
196
202
|
value: (e: number): number => weights[e],
|
|
197
203
|
text: (e: number, integral = false): string => {
|
|
198
204
|
const value = weights[e];
|
|
205
|
+
if (Object.is(value, -0)) {
|
|
206
|
+
return NEGATIVE_ZERO_TEXT;
|
|
207
|
+
}
|
|
199
208
|
return integral && Number.isInteger(value) ? formatInteger(value) : formatF32(value);
|
|
200
209
|
},
|
|
201
210
|
};
|