@graphty/graph-format 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +216 -43
- package/dist/graph-format.d.ts +1 -0
- package/dist/graph-format.d.ts.map +1 -0
- package/dist/graph-format.js +15798 -0
- package/dist/graph-format.js.map +1 -0
- package/dist/src/builder/arena.d.ts +85 -0
- package/dist/src/builder/arena.d.ts.map +1 -0
- package/dist/src/builder/arena.js +154 -0
- package/dist/src/builder/arena.js.map +1 -0
- package/dist/src/builder/compact.d.ts +515 -0
- package/dist/src/builder/compact.d.ts.map +1 -0
- package/dist/src/builder/compact.js +1553 -0
- package/dist/src/builder/compact.js.map +1 -0
- package/dist/src/builder/counting-sort.d.ts +96 -0
- package/dist/src/builder/counting-sort.d.ts.map +1 -0
- package/dist/src/builder/counting-sort.js +430 -0
- package/dist/src/builder/counting-sort.js.map +1 -0
- package/dist/src/builder/freeze.d.ts +72 -0
- package/dist/src/builder/freeze.d.ts.map +1 -0
- package/dist/src/builder/freeze.js +548 -0
- package/dist/src/builder/freeze.js.map +1 -0
- package/dist/src/builder/graph-builder.d.ts +602 -0
- package/dist/src/builder/graph-builder.d.ts.map +1 -0
- package/dist/src/builder/graph-builder.js +1940 -0
- package/dist/src/builder/graph-builder.js.map +1 -0
- package/dist/src/builder/options.d.ts +16 -0
- package/dist/src/builder/options.d.ts.map +1 -0
- package/dist/src/builder/options.js +21 -0
- package/dist/src/builder/options.js.map +1 -0
- package/dist/src/columns/bitmap.d.ts +96 -0
- package/dist/src/columns/bitmap.d.ts.map +1 -0
- package/dist/src/columns/bitmap.js +184 -0
- package/dist/src/columns/bitmap.js.map +1 -0
- package/dist/src/columns/column.d.ts +220 -0
- package/dist/src/columns/column.d.ts.map +1 -0
- package/dist/src/columns/column.js +2109 -0
- package/dist/src/columns/column.js.map +1 -0
- package/dist/src/columns/dictionary.d.ts +60 -0
- package/dist/src/columns/dictionary.d.ts.map +1 -0
- package/dist/src/columns/dictionary.js +102 -0
- package/dist/src/columns/dictionary.js.map +1 -0
- package/dist/src/columns/growable.d.ts +198 -0
- package/dist/src/columns/growable.d.ts.map +1 -0
- package/dist/src/columns/growable.js +296 -0
- package/dist/src/columns/growable.js.map +1 -0
- package/dist/src/columns/infer.d.ts +121 -0
- package/dist/src/columns/infer.d.ts.map +1 -0
- package/dist/src/columns/infer.js +268 -0
- package/dist/src/columns/infer.js.map +1 -0
- package/dist/src/columns/remap.d.ts +143 -0
- package/dist/src/columns/remap.d.ts.map +1 -0
- package/dist/src/columns/remap.js +588 -0
- package/dist/src/columns/remap.js.map +1 -0
- package/dist/src/columns/table.d.ts +184 -0
- package/dist/src/columns/table.d.ts.map +1 -0
- package/dist/src/columns/table.js +485 -0
- package/dist/src/columns/table.js.map +1 -0
- package/dist/src/constants.d.ts +34 -0
- package/dist/src/constants.d.ts.map +1 -0
- package/dist/src/constants.js +79 -0
- package/dist/src/constants.js.map +1 -0
- package/dist/src/errors.d.ts +76 -0
- package/dist/src/errors.d.ts.map +1 -0
- package/dist/src/errors.js +32 -0
- package/dist/src/errors.js.map +1 -0
- package/dist/src/ids/edge-id-index.d.ts +77 -0
- package/dist/src/ids/edge-id-index.d.ts.map +1 -0
- package/dist/src/ids/edge-id-index.js +155 -0
- package/dist/src/ids/edge-id-index.js.map +1 -0
- package/dist/src/ids/node-id-map.d.ts +280 -0
- package/dist/src/ids/node-id-map.d.ts.map +1 -0
- package/dist/src/ids/node-id-map.js +1139 -0
- package/dist/src/ids/node-id-map.js.map +1 -0
- package/dist/src/ids/string-store.d.ts +219 -0
- package/dist/src/ids/string-store.d.ts.map +1 -0
- package/dist/src/ids/string-store.js +526 -0
- package/dist/src/ids/string-store.js.map +1 -0
- package/dist/src/index.d.ts +30 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +29 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/populate/from-csr.d.ts +61 -0
- package/dist/src/populate/from-csr.d.ts.map +1 -0
- package/dist/src/populate/from-csr.js +706 -0
- package/dist/src/populate/from-csr.js.map +1 -0
- package/dist/src/populate/from-edge-arrays.d.ts +51 -0
- package/dist/src/populate/from-edge-arrays.d.ts.map +1 -0
- package/dist/src/populate/from-edge-arrays.js +217 -0
- package/dist/src/populate/from-edge-arrays.js.map +1 -0
- package/dist/src/populate/from-records.d.ts +51 -0
- package/dist/src/populate/from-records.d.ts.map +1 -0
- package/dist/src/populate/from-records.js +447 -0
- package/dist/src/populate/from-records.js.map +1 -0
- package/dist/src/snapshot/derived.d.ts +211 -0
- package/dist/src/snapshot/derived.d.ts.map +1 -0
- package/dist/src/snapshot/derived.js +1254 -0
- package/dist/src/snapshot/derived.js.map +1 -0
- package/dist/src/snapshot/graph-meta.d.ts +23 -0
- package/dist/src/snapshot/graph-meta.d.ts.map +1 -0
- package/dist/src/snapshot/graph-meta.js +159 -0
- package/dist/src/snapshot/graph-meta.js.map +1 -0
- package/dist/src/snapshot/graph-snapshot.d.ts +509 -0
- package/dist/src/snapshot/graph-snapshot.d.ts.map +1 -0
- package/dist/src/snapshot/graph-snapshot.js +1247 -0
- package/dist/src/snapshot/graph-snapshot.js.map +1 -0
- package/dist/src/snapshot/hash.d.ts +90 -0
- package/dist/src/snapshot/hash.d.ts.map +1 -0
- package/dist/src/snapshot/hash.js +228 -0
- package/dist/src/snapshot/hash.js.map +1 -0
- package/dist/src/snapshot/queries.d.ts +74 -0
- package/dist/src/snapshot/queries.d.ts.map +1 -0
- package/dist/src/snapshot/queries.js +131 -0
- package/dist/src/snapshot/queries.js.map +1 -0
- package/dist/src/snapshot/validate.d.ts +192 -0
- package/dist/src/snapshot/validate.d.ts.map +1 -0
- package/dist/src/snapshot/validate.js +990 -0
- package/dist/src/snapshot/validate.js.map +1 -0
- package/dist/src/snapshot/views.d.ts +274 -0
- package/dist/src/snapshot/views.d.ts.map +1 -0
- package/dist/src/snapshot/views.js +856 -0
- package/dist/src/snapshot/views.js.map +1 -0
- package/dist/src/types/builder.d.ts +456 -0
- package/dist/src/types/builder.d.ts.map +1 -0
- package/dist/src/types/builder.js +11 -0
- package/dist/src/types/builder.js.map +1 -0
- package/dist/src/types/columns.d.ts +615 -0
- package/dist/src/types/columns.d.ts.map +1 -0
- package/dist/src/types/columns.js +14 -0
- package/dist/src/types/columns.js.map +1 -0
- package/dist/src/types/index.d.ts +14 -0
- package/dist/src/types/index.d.ts.map +1 -0
- package/dist/src/types/index.js +11 -0
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/internal.d.ts +172 -0
- package/dist/src/types/internal.d.ts.map +1 -0
- package/dist/src/types/internal.js +13 -0
- package/dist/src/types/internal.js.map +1 -0
- package/dist/src/types/snapshot.d.ts +801 -0
- package/dist/src/types/snapshot.d.ts.map +1 -0
- package/dist/src/types/snapshot.js +14 -0
- package/dist/src/types/snapshot.js.map +1 -0
- package/dist/src/types/wire.d.ts +196 -0
- package/dist/src/types/wire.d.ts.map +1 -0
- package/dist/src/types/wire.js +8 -0
- package/dist/src/types/wire.js.map +1 -0
- package/dist/src/util/mask.d.ts +55 -0
- package/dist/src/util/mask.d.ts.map +1 -0
- package/dist/src/util/mask.js +71 -0
- package/dist/src/util/mask.js.map +1 -0
- package/dist/src/util/options.d.ts +22 -0
- package/dist/src/util/options.d.ts.map +1 -0
- package/dist/src/util/options.js +49 -0
- package/dist/src/util/options.js.map +1 -0
- package/dist/src/util/shared-buffers.d.ts +42 -0
- package/dist/src/util/shared-buffers.d.ts.map +1 -0
- package/dist/src/util/shared-buffers.js +52 -0
- package/dist/src/util/shared-buffers.js.map +1 -0
- package/dist/src/util/typed-array.d.ts +205 -0
- package/dist/src/util/typed-array.d.ts.map +1 -0
- package/dist/src/util/typed-array.js +289 -0
- package/dist/src/util/typed-array.js.map +1 -0
- package/dist/src/wire/bytes.d.ts +69 -0
- package/dist/src/wire/bytes.d.ts.map +1 -0
- package/dist/src/wire/bytes.js +361 -0
- package/dist/src/wire/bytes.js.map +1 -0
- package/dist/src/wire/carried-views.d.ts +111 -0
- package/dist/src/wire/carried-views.d.ts.map +1 -0
- package/dist/src/wire/carried-views.js +286 -0
- package/dist/src/wire/carried-views.js.map +1 -0
- package/dist/src/wire/from-wire.d.ts +144 -0
- package/dist/src/wire/from-wire.d.ts.map +1 -0
- package/dist/src/wire/from-wire.js +1425 -0
- package/dist/src/wire/from-wire.js.map +1 -0
- package/dist/src/wire/to-wire.d.ts +171 -0
- package/dist/src/wire/to-wire.d.ts.map +1 -0
- package/dist/src/wire/to-wire.js +652 -0
- package/dist/src/wire/to-wire.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -0
- package/package.json +72 -7
- package/src/builder/arena.ts +192 -0
- package/src/builder/compact.ts +1811 -0
- package/src/builder/counting-sort.ts +506 -0
- package/src/builder/freeze.ts +689 -0
- package/src/builder/graph-builder.ts +2264 -0
- package/src/builder/options.ts +29 -0
- package/src/columns/bitmap.ts +196 -0
- package/src/columns/column.ts +2574 -0
- package/src/columns/dictionary.ts +114 -0
- package/src/columns/growable.ts +350 -0
- package/src/columns/infer.ts +285 -0
- package/src/columns/remap.ts +663 -0
- package/src/columns/table.ts +568 -0
- package/src/constants.ts +89 -0
- package/src/errors.ts +115 -0
- package/src/ids/edge-id-index.ts +179 -0
- package/src/ids/node-id-map.ts +1340 -0
- package/src/ids/string-store.ts +589 -0
- package/src/index.ts +139 -0
- package/src/lib-resizable-array-buffer.d.ts +34 -0
- package/src/populate/from-csr.ts +802 -0
- package/src/populate/from-edge-arrays.ts +257 -0
- package/src/populate/from-records.ts +536 -0
- package/src/snapshot/derived.ts +1486 -0
- package/src/snapshot/graph-meta.ts +167 -0
- package/src/snapshot/graph-snapshot.ts +1503 -0
- package/src/snapshot/hash.ts +241 -0
- package/src/snapshot/queries.ts +136 -0
- package/src/snapshot/validate.ts +1108 -0
- package/src/snapshot/views.ts +943 -0
- package/src/types/builder.ts +480 -0
- package/src/types/columns.ts +727 -0
- package/src/types/index.ts +14 -0
- package/src/types/internal.ts +195 -0
- package/src/types/snapshot.ts +886 -0
- package/src/types/wire.ts +207 -0
- package/src/util/mask.ts +89 -0
- package/src/util/options.ts +57 -0
- package/src/util/shared-buffers.ts +56 -0
- package/src/util/typed-array.ts +371 -0
- package/src/wire/bytes.ts +440 -0
- package/src/wire/carried-views.ts +327 -0
- package/src/wire/from-wire.ts +1690 -0
- package/src/wire/to-wire.ts +755 -0
|
@@ -0,0 +1,1425 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The wire decoder (design sections 4.5, 5.9, 9.1, 9.5 and 11.3): `fromWire()` rebuilds a snapshot
|
|
3
|
+
* from a WireSnapshot, and `decodeManifest()` is the shared reader the GSNP container functions of
|
|
4
|
+
* bytes.ts drive with their own buffer regions.
|
|
5
|
+
*
|
|
6
|
+
* Rules (design section 9.1): a manifest whose `formatVersion` is not the reader's or whose wire
|
|
7
|
+
* major is unknown is refused with E_UNSUPPORTED_VERSION; a newer wire minor is read by ignoring
|
|
8
|
+
* unknown manifest fields; an unknown column dtype is E_UNSUPPORTED unless `unknownColumns: "skip"`
|
|
9
|
+
* drops the column and names it in `meta.extra["graphty.skippedColumns"]`; an unknown id-map kind
|
|
10
|
+
* is always E_UNSUPPORTED; unknown view entries are ignored and recomputed. Every WireBufferRef is
|
|
11
|
+
* checked against its buffer before a typed array is built over it (E_BAD_SERIALIZATION with
|
|
12
|
+
* details.ref), whatever the validation level, because a RangeError from a typed-array constructor
|
|
13
|
+
* is never an acceptable failure mode; the "structure" and "full" levels add the checks of design
|
|
14
|
+
* section 9.5 and then run the snapshot's own validate().
|
|
15
|
+
*
|
|
16
|
+
* Buffers are adopted by reference by default: the arrays are views into the caller's buffers and
|
|
17
|
+
* the arena descriptor is honoured. A SharedArrayBuffer is copied (decision D-SAB), and `copy: true`
|
|
18
|
+
* copies every buffer before adoption.
|
|
19
|
+
*/
|
|
20
|
+
import { CORE_ORDER } from "../builder/arena.js";
|
|
21
|
+
import { createColumn, emptyParts, isPlainObject, resolveColumnMeta } from "../columns/column.js";
|
|
22
|
+
import { AttributeTable } from "../columns/table.js";
|
|
23
|
+
import { ALIGNMENT, FORMAT_VERSION, MAX_COUNT, WIRE_FORMAT, WIRE_MAJOR } from "../constants.js";
|
|
24
|
+
import { GraphFormatError } from "../errors.js";
|
|
25
|
+
import { nodeIdMapFromTyped } from "../ids/node-id-map.js";
|
|
26
|
+
import { checkUtf8Layout, decodeUtf8Rows, Utf8Store } from "../ids/string-store.js";
|
|
27
|
+
import { createSnapshot, seedView } from "../snapshot/graph-snapshot.js";
|
|
28
|
+
import { cooViewOf, edgeListViewOf, ReverseAdjacency, rowLengths } from "../snapshot/views.js";
|
|
29
|
+
import { canViewAsPaddedU32, copyToPaddedStore, padTo4 } from "../util/typed-array.js";
|
|
30
|
+
import { checkCooSrc, checkDegree, checkDegreeOrder, checkEdgeList, checkF64View, checkInDegree, checkMate, checkOutDegree, checkReverse, checkSelfLoopArcs, checkSelfLoopsPerNode, CoreFacts, } from "./carried-views.js";
|
|
31
|
+
import { defineJsonKey } from "./to-wire.js";
|
|
32
|
+
// ============================================================ vocabularies
|
|
33
|
+
const WIRE_DTYPES = new Set(["u32", "i32", "f32", "f64", "u8", "utf8"]);
|
|
34
|
+
const COLUMN_DTYPES = new Set([
|
|
35
|
+
"f32",
|
|
36
|
+
"f64",
|
|
37
|
+
"i32",
|
|
38
|
+
"u32",
|
|
39
|
+
"u8",
|
|
40
|
+
"bool",
|
|
41
|
+
"dict",
|
|
42
|
+
"string",
|
|
43
|
+
"list",
|
|
44
|
+
"json",
|
|
45
|
+
]);
|
|
46
|
+
const ID_MAP_KINDS = new Set(["identity", "dense", "numeric", "string", "mixed"]);
|
|
47
|
+
const ID_TYPES = new Set(["string", "integer", "mixed"]);
|
|
48
|
+
const TIME_FORMATS = new Set([
|
|
49
|
+
"integer",
|
|
50
|
+
"double",
|
|
51
|
+
"date",
|
|
52
|
+
"dateTime",
|
|
53
|
+
]);
|
|
54
|
+
const TIME_REPRESENTATIONS = new Set([
|
|
55
|
+
"interval",
|
|
56
|
+
"timestamp",
|
|
57
|
+
]);
|
|
58
|
+
const MODES = new Set(["static", "dynamic", "slice"]);
|
|
59
|
+
const FORBIDDEN_KEYS = new Set(["__proto__", "constructor", "prototype"]);
|
|
60
|
+
const NUM_TAG = "$num";
|
|
61
|
+
const ESC_TAG = "$esc";
|
|
62
|
+
/** The deepest JSON nesting a manifest value may have; deeper input is refused instead of overflowing the stack. */
|
|
63
|
+
const MAX_JSON_DEPTH = 256;
|
|
64
|
+
/**
|
|
65
|
+
* A short plain-ASCII rendering of an untrusted manifest value for an error message: strings are
|
|
66
|
+
* quoted, everything else is named by type (a bigint or a null-prototype object cannot be
|
|
67
|
+
* stringified, and a structuredClone-delivered manifest can carry both).
|
|
68
|
+
* @param value - the value
|
|
69
|
+
* @returns the rendering
|
|
70
|
+
*/
|
|
71
|
+
function describe(value) {
|
|
72
|
+
if (typeof value === "string") {
|
|
73
|
+
return JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value);
|
|
74
|
+
}
|
|
75
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) {
|
|
76
|
+
return String(value);
|
|
77
|
+
}
|
|
78
|
+
return value === undefined ? "undefined" : `a ${typeof value}`;
|
|
79
|
+
}
|
|
80
|
+
/** The meta.extra key naming the columns dropped under unknownColumns: "skip" (design section 9.1). */
|
|
81
|
+
export const SKIPPED_COLUMNS_KEY = "graphty.skippedColumns";
|
|
82
|
+
// ============================================================ errors and shape readers
|
|
83
|
+
/**
|
|
84
|
+
* The E_BAD_SERIALIZATION error of a malformed manifest member or buffer reference.
|
|
85
|
+
* @param ref - the manifest path, e.g. "nodeColumns[2].data"
|
|
86
|
+
* @param reason - what is wrong
|
|
87
|
+
* @param details - extra context
|
|
88
|
+
* @returns the error
|
|
89
|
+
*/
|
|
90
|
+
export function badWire(ref, reason, details = {}) {
|
|
91
|
+
return new GraphFormatError("E_BAD_SERIALIZATION", `${ref}: ${reason}`, { ref, reason, ...details });
|
|
92
|
+
}
|
|
93
|
+
function asObject(value, path) {
|
|
94
|
+
if (!isPlainObject(value)) {
|
|
95
|
+
throw badWire(path, "expected an object");
|
|
96
|
+
}
|
|
97
|
+
return value;
|
|
98
|
+
}
|
|
99
|
+
function asArray(value, path) {
|
|
100
|
+
if (!Array.isArray(value)) {
|
|
101
|
+
throw badWire(path, "expected an array");
|
|
102
|
+
}
|
|
103
|
+
return value;
|
|
104
|
+
}
|
|
105
|
+
function asString(value, path) {
|
|
106
|
+
if (typeof value !== "string") {
|
|
107
|
+
throw badWire(path, "expected a string");
|
|
108
|
+
}
|
|
109
|
+
return value;
|
|
110
|
+
}
|
|
111
|
+
function asStringOrNull(value, path) {
|
|
112
|
+
if (value === undefined || value === null) {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
return asString(value, path);
|
|
116
|
+
}
|
|
117
|
+
function asBoolean(value, path) {
|
|
118
|
+
if (typeof value !== "boolean") {
|
|
119
|
+
throw badWire(path, "expected a boolean");
|
|
120
|
+
}
|
|
121
|
+
return value;
|
|
122
|
+
}
|
|
123
|
+
function asBooleanOrNull(value, path) {
|
|
124
|
+
if (value === undefined || value === null) {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
return asBoolean(value, path);
|
|
128
|
+
}
|
|
129
|
+
function asCount(value, path) {
|
|
130
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > MAX_COUNT) {
|
|
131
|
+
throw badWire(path, `expected an integer in [0, ${MAX_COUNT}]`, { found: value });
|
|
132
|
+
}
|
|
133
|
+
// JSON.parse("-0") is -0; a count is stored as 0 (design section 4.1)
|
|
134
|
+
return value === 0 ? 0 : value;
|
|
135
|
+
}
|
|
136
|
+
function asEnum(value, path, allowed) {
|
|
137
|
+
if (value === undefined || value === null) {
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
if (typeof value !== "string" || !allowed.has(value)) {
|
|
141
|
+
throw badWire(path, `unexpected value ${describe(value)}`, { found: value });
|
|
142
|
+
}
|
|
143
|
+
return value;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Whether a string names a column dtype this reader knows.
|
|
147
|
+
* @param value - the dtype text
|
|
148
|
+
* @returns true for one of the ten dtypes
|
|
149
|
+
*/
|
|
150
|
+
function isKnownDtype(value) {
|
|
151
|
+
return typeof value === "string" && COLUMN_DTYPES.has(value);
|
|
152
|
+
}
|
|
153
|
+
// ============================================================ JSON values (5.9)
|
|
154
|
+
/**
|
|
155
|
+
* Decode a manifest JSON value (design section 5.9): `{ "$num": "Infinity" | "-Infinity" | "NaN" |
|
|
156
|
+
* "-0" }` becomes the number, `{ "$esc": { ... } }` unwraps a user object whose only key collided
|
|
157
|
+
* with a tag (the writer's escape), arrays and plain objects are rebuilt recursively (a key named
|
|
158
|
+
* `__proto__`, `constructor` or `prototype` is refused), everything else is returned as is. Nesting
|
|
159
|
+
* deeper than MAX_JSON_DEPTH is E_BAD_SERIALIZATION rather than a stack overflow.
|
|
160
|
+
* @param value - the value from the manifest
|
|
161
|
+
* @param path - the manifest path for error messages
|
|
162
|
+
* @param depth - the current nesting depth
|
|
163
|
+
* @returns the decoded value
|
|
164
|
+
*/
|
|
165
|
+
export function decodeJsonValue(value, path, depth = 0) {
|
|
166
|
+
if (depth > MAX_JSON_DEPTH) {
|
|
167
|
+
throw badWire(path, `JSON nesting deeper than ${MAX_JSON_DEPTH}`, { reason: "nesting" });
|
|
168
|
+
}
|
|
169
|
+
if (Array.isArray(value)) {
|
|
170
|
+
return value.map((item, i) => decodeJsonValue(item, `${path}[${i}]`, depth + 1));
|
|
171
|
+
}
|
|
172
|
+
if (!isPlainObject(value)) {
|
|
173
|
+
return value;
|
|
174
|
+
}
|
|
175
|
+
const keys = Object.keys(value);
|
|
176
|
+
if (keys.length === 1 && keys[0] === NUM_TAG) {
|
|
177
|
+
const tag = value[NUM_TAG];
|
|
178
|
+
switch (tag) {
|
|
179
|
+
case "NaN":
|
|
180
|
+
return Number.NaN;
|
|
181
|
+
case "Infinity":
|
|
182
|
+
return Infinity;
|
|
183
|
+
case "-Infinity":
|
|
184
|
+
return -Infinity;
|
|
185
|
+
case "-0":
|
|
186
|
+
return -0;
|
|
187
|
+
default:
|
|
188
|
+
throw badWire(path, `unknown number tag ${describe(tag)}`, { found: tag });
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
if (keys.length === 1 && keys[0] === ESC_TAG) {
|
|
192
|
+
const inner = value[ESC_TAG];
|
|
193
|
+
if (!isPlainObject(inner)) {
|
|
194
|
+
throw badWire(path, "an escaped object must hold a plain object", { found: typeof inner });
|
|
195
|
+
}
|
|
196
|
+
return decodeJsonObject(inner, path, depth);
|
|
197
|
+
}
|
|
198
|
+
return decodeJsonObject(value, path, depth);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Rebuild a plain object's members (no tag interpretation of the object itself).
|
|
202
|
+
* @param value - the plain object
|
|
203
|
+
* @param path - the manifest path
|
|
204
|
+
* @param depth - the nesting depth of the object
|
|
205
|
+
* @returns the rebuilt object
|
|
206
|
+
*/
|
|
207
|
+
function decodeJsonObject(value, path, depth) {
|
|
208
|
+
const out = {};
|
|
209
|
+
for (const key of Object.keys(value)) {
|
|
210
|
+
if (FORBIDDEN_KEYS.has(key)) {
|
|
211
|
+
throw badWire(path, `forbidden key "${key}"`, { key });
|
|
212
|
+
}
|
|
213
|
+
defineJsonKey(out, key, decodeJsonValue(value[key], `${path}.${key}`, depth + 1));
|
|
214
|
+
}
|
|
215
|
+
return out;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* The JSON.parse reviver of design section 9.5: refuses the prototype-pollution keys.
|
|
219
|
+
* @param key - the property name being revived
|
|
220
|
+
* @param value - the revived value
|
|
221
|
+
* @returns the value unchanged
|
|
222
|
+
*/
|
|
223
|
+
function guardReviver(key, value) {
|
|
224
|
+
if (FORBIDDEN_KEYS.has(key)) {
|
|
225
|
+
throw badWire("manifest", `forbidden key "${key}"`, { key });
|
|
226
|
+
}
|
|
227
|
+
return value;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Parse JSON text from a container (the manifest or one json column row) with the guarding
|
|
231
|
+
* reviver. A syntax error is E_BAD_SERIALIZATION.
|
|
232
|
+
* @param text - the JSON text
|
|
233
|
+
* @param path - the manifest path for error messages
|
|
234
|
+
* @returns the parsed value
|
|
235
|
+
*/
|
|
236
|
+
export function parseGuardedJson(text, path) {
|
|
237
|
+
try {
|
|
238
|
+
return JSON.parse(text, guardReviver);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
if (err instanceof GraphFormatError) {
|
|
242
|
+
throw err;
|
|
243
|
+
}
|
|
244
|
+
throw badWire(path, `not valid JSON (${err instanceof Error ? err.message : String(err)})`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Apply defaults to FromWireOptions and reject values outside their documented sets (E_UNSUPPORTED).
|
|
249
|
+
* @param options - the caller's options
|
|
250
|
+
* @param defaultLevel - "structure" for fromWire, "full" for fromBytes
|
|
251
|
+
* @returns the resolved options
|
|
252
|
+
*/
|
|
253
|
+
export function resolveFromWireOptions(options, defaultLevel) {
|
|
254
|
+
const level = options?.validate ?? defaultLevel;
|
|
255
|
+
if (level !== "none" && level !== "structure" && level !== "full") {
|
|
256
|
+
throw new GraphFormatError("E_UNSUPPORTED", `unknown validation level ${String(level)}`, {
|
|
257
|
+
option: "validate",
|
|
258
|
+
found: level,
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
const unknownColumns = options?.unknownColumns ?? "error";
|
|
262
|
+
if (unknownColumns !== "error" && unknownColumns !== "skip") {
|
|
263
|
+
throw new GraphFormatError("E_UNSUPPORTED", `unknown unknownColumns policy ${String(unknownColumns)}`, {
|
|
264
|
+
option: "unknownColumns",
|
|
265
|
+
found: unknownColumns,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
const copy = options?.copy ?? false;
|
|
269
|
+
if (typeof copy !== "boolean") {
|
|
270
|
+
throw new GraphFormatError("E_UNSUPPORTED", "copy must be a boolean", { option: "copy", found: copy });
|
|
271
|
+
}
|
|
272
|
+
return { level, copy, unknownColumns };
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* A region over one plain ArrayBuffer.
|
|
276
|
+
* @param buffer - the buffer
|
|
277
|
+
* @param base - the absolute byte offset of the region's start inside the buffer
|
|
278
|
+
* @param byteLength - the region's byte length
|
|
279
|
+
* @returns the region
|
|
280
|
+
*/
|
|
281
|
+
export function bufferRegion(buffer, base, byteLength) {
|
|
282
|
+
return {
|
|
283
|
+
byteLength,
|
|
284
|
+
locate(byteOffset) {
|
|
285
|
+
return { buffer, byteOffset: base + byteOffset };
|
|
286
|
+
},
|
|
287
|
+
read(byteOffset, length) {
|
|
288
|
+
const out = new Uint8Array(new ArrayBuffer(padTo4(length)), 0, length);
|
|
289
|
+
out.set(new Uint8Array(buffer, base + byteOffset, length));
|
|
290
|
+
return out;
|
|
291
|
+
},
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Copy an ArrayBuffer or SharedArrayBuffer into a fresh ArrayBuffer.
|
|
296
|
+
* @param buffer - the source
|
|
297
|
+
* @returns the copy
|
|
298
|
+
*/
|
|
299
|
+
function copyBuffer(buffer) {
|
|
300
|
+
const out = new ArrayBuffer(buffer.byteLength);
|
|
301
|
+
new Uint8Array(out).set(new Uint8Array(buffer));
|
|
302
|
+
return out;
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Whether a value is a SharedArrayBuffer (never adopted, decision D-SAB).
|
|
306
|
+
* @param value - the value
|
|
307
|
+
* @returns true for a SharedArrayBuffer
|
|
308
|
+
*/
|
|
309
|
+
export function isSharedArrayBuffer(value) {
|
|
310
|
+
return typeof SharedArrayBuffer !== "undefined" && value instanceof SharedArrayBuffer;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Bytes per element and constructor of a wire dtype.
|
|
314
|
+
* @param dtype - the wire dtype
|
|
315
|
+
* @returns the element size
|
|
316
|
+
*/
|
|
317
|
+
function wireElementSize(dtype) {
|
|
318
|
+
switch (dtype) {
|
|
319
|
+
case "u32":
|
|
320
|
+
case "i32":
|
|
321
|
+
case "f32":
|
|
322
|
+
return 4;
|
|
323
|
+
case "f64":
|
|
324
|
+
return 8;
|
|
325
|
+
case "u8":
|
|
326
|
+
case "utf8":
|
|
327
|
+
return 1;
|
|
328
|
+
default: {
|
|
329
|
+
const name = dtype;
|
|
330
|
+
throw badWire("dtype", `unknown wire dtype ${name}`, { dtype: name });
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Build the typed array of a wire dtype over a buffer.
|
|
336
|
+
* @param dtype - the wire dtype
|
|
337
|
+
* @param buffer - the buffer
|
|
338
|
+
* @param byteOffset - the absolute start
|
|
339
|
+
* @param length - the element count
|
|
340
|
+
* @returns the array
|
|
341
|
+
*/
|
|
342
|
+
function buildArray(dtype, buffer, byteOffset, length) {
|
|
343
|
+
switch (dtype) {
|
|
344
|
+
case "u32":
|
|
345
|
+
return new Uint32Array(buffer, byteOffset, length);
|
|
346
|
+
case "i32":
|
|
347
|
+
return new Int32Array(buffer, byteOffset, length);
|
|
348
|
+
case "f32":
|
|
349
|
+
return new Float32Array(buffer, byteOffset, length);
|
|
350
|
+
case "f64":
|
|
351
|
+
return new Float64Array(buffer, byteOffset, length);
|
|
352
|
+
case "u8":
|
|
353
|
+
case "utf8":
|
|
354
|
+
return new Uint8Array(buffer, byteOffset, length);
|
|
355
|
+
default: {
|
|
356
|
+
const name = dtype;
|
|
357
|
+
throw badWire("dtype", `unknown wire dtype ${name}`, { dtype: name });
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Read a WireBufferRef's shape from an untrusted manifest member.
|
|
363
|
+
* @param value - the member
|
|
364
|
+
* @param path - its manifest path
|
|
365
|
+
* @returns the reference
|
|
366
|
+
*/
|
|
367
|
+
function readRef(value, path) {
|
|
368
|
+
const o = asObject(value, path);
|
|
369
|
+
const { dtype } = o;
|
|
370
|
+
if (typeof dtype !== "string" || !WIRE_DTYPES.has(dtype)) {
|
|
371
|
+
throw badWire(path, `unknown wire dtype ${describe(dtype)}`, { found: dtype });
|
|
372
|
+
}
|
|
373
|
+
const fields = ["buffer", "byteOffset", "byteLength", "length"];
|
|
374
|
+
for (const field of fields) {
|
|
375
|
+
const v = o[field];
|
|
376
|
+
if (typeof v !== "number" || !Number.isInteger(v) || v < 0) {
|
|
377
|
+
throw badWire(`${path}.${field}`, "expected a non-negative integer", { found: v });
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return {
|
|
381
|
+
buffer: o.buffer,
|
|
382
|
+
byteOffset: o.byteOffset,
|
|
383
|
+
byteLength: o.byteLength,
|
|
384
|
+
dtype: dtype,
|
|
385
|
+
length: o.length,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* The error to throw for a failure while building a column or table from untrusted input: a
|
|
390
|
+
* column-construction error (E_COLUMN_*, and E_DUPLICATE_ROLE from the table constructor) becomes
|
|
391
|
+
* E_BAD_SERIALIZATION naming the manifest path, every other Error passes through, and a non-Error
|
|
392
|
+
* value is wrapped.
|
|
393
|
+
* @param err - the caught value
|
|
394
|
+
* @param path - the manifest path of the column or table
|
|
395
|
+
* @returns the error to throw
|
|
396
|
+
*/
|
|
397
|
+
function asWireError(err, path) {
|
|
398
|
+
if (err instanceof GraphFormatError && (err.code.startsWith("E_COLUMN") || err.code === "E_DUPLICATE_ROLE")) {
|
|
399
|
+
return new GraphFormatError("E_BAD_SERIALIZATION", `${path}: ${err.message}`, {
|
|
400
|
+
ref: path,
|
|
401
|
+
reason: err.message,
|
|
402
|
+
cause: err.code,
|
|
403
|
+
...err.details,
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
if (err instanceof Error) {
|
|
407
|
+
return err;
|
|
408
|
+
}
|
|
409
|
+
return badWire(path, String(err));
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* The stateful reader of one manifest: resolves references against the regions, records placed
|
|
413
|
+
* ranges for the overlap rule and the columns skipped under unknownColumns: "skip".
|
|
414
|
+
*/
|
|
415
|
+
class WireDecoder {
|
|
416
|
+
/**
|
|
417
|
+
* Create a decoder.
|
|
418
|
+
* @param regions - one region per buffer index
|
|
419
|
+
* @param options - the resolved options
|
|
420
|
+
*/
|
|
421
|
+
constructor(regions, options) {
|
|
422
|
+
this.ranges = [];
|
|
423
|
+
this.skipped = [];
|
|
424
|
+
this.regions = regions;
|
|
425
|
+
this.options = options;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Whether the level includes the structure checks.
|
|
429
|
+
* @returns true for "structure" and "full"
|
|
430
|
+
*/
|
|
431
|
+
get structure() {
|
|
432
|
+
return this.options.level !== "none";
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Whether the level is "full".
|
|
436
|
+
* @returns true for "full"
|
|
437
|
+
*/
|
|
438
|
+
get full() {
|
|
439
|
+
return this.options.level === "full";
|
|
440
|
+
}
|
|
441
|
+
/**
|
|
442
|
+
* Check a reference against its region (design section 9.5) and build the typed array over the
|
|
443
|
+
* referenced bytes, adopting them when they are contiguous and aligned and copying otherwise.
|
|
444
|
+
* @param value - the manifest member holding the reference
|
|
445
|
+
* @param path - its manifest path
|
|
446
|
+
* @param expected - the wire dtype the slot requires
|
|
447
|
+
* @param kind - who the range belongs to
|
|
448
|
+
* @returns the typed array
|
|
449
|
+
*/
|
|
450
|
+
array(value, path, expected, kind) {
|
|
451
|
+
const ref = readRef(value, path);
|
|
452
|
+
if (ref.dtype !== expected) {
|
|
453
|
+
throw badWire(path, `expected dtype ${expected}, found ${ref.dtype}`, {
|
|
454
|
+
expected,
|
|
455
|
+
found: ref.dtype,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
const region = this.regions[ref.buffer];
|
|
459
|
+
if (region === undefined) {
|
|
460
|
+
throw badWire(path, `buffer index ${ref.buffer} is out of range (${this.regions.length} buffers)`, {
|
|
461
|
+
found: ref.buffer,
|
|
462
|
+
buffers: this.regions.length,
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
const elementSize = wireElementSize(ref.dtype);
|
|
466
|
+
if (ref.byteOffset % elementSize !== 0) {
|
|
467
|
+
throw badWire(path, `byteOffset ${ref.byteOffset} is not a multiple of ${elementSize}`, {
|
|
468
|
+
byteOffset: ref.byteOffset,
|
|
469
|
+
elementSize,
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
if (ref.byteLength !== ref.length * elementSize) {
|
|
473
|
+
throw badWire(path, `byteLength ${ref.byteLength} is not length ${ref.length} * ${elementSize}`, {
|
|
474
|
+
byteLength: ref.byteLength,
|
|
475
|
+
length: ref.length,
|
|
476
|
+
elementSize,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
if (ref.byteOffset + ref.byteLength > region.byteLength) {
|
|
480
|
+
throw badWire(path, `range [${ref.byteOffset}, ${ref.byteOffset + ref.byteLength}) exceeds the buffer`, {
|
|
481
|
+
byteOffset: ref.byteOffset,
|
|
482
|
+
byteLength: ref.byteLength,
|
|
483
|
+
bufferByteLength: region.byteLength,
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
const located = region.locate(ref.byteOffset, ref.byteLength);
|
|
487
|
+
if (located !== null && located.byteOffset % elementSize === 0) {
|
|
488
|
+
// adopted: the overlap rule is checked on the ArrayBuffer itself, so listing one buffer
|
|
489
|
+
// twice in wire.buffers cannot hide an alias
|
|
490
|
+
this.ranges.push({
|
|
491
|
+
buffer: located.buffer,
|
|
492
|
+
start: located.byteOffset,
|
|
493
|
+
end: located.byteOffset + ref.byteLength,
|
|
494
|
+
kind,
|
|
495
|
+
path,
|
|
496
|
+
});
|
|
497
|
+
return buildArray(ref.dtype, located.buffer, located.byteOffset, ref.length);
|
|
498
|
+
}
|
|
499
|
+
// copied: a private buffer, nothing to overlap
|
|
500
|
+
const copy = region.read(ref.byteOffset, ref.byteLength);
|
|
501
|
+
return buildArray(ref.dtype, copy.buffer, 0, ref.length);
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* A u32 array slot.
|
|
505
|
+
* @param value - the manifest member
|
|
506
|
+
* @param path - its path
|
|
507
|
+
* @param kind - the range owner
|
|
508
|
+
* @returns the array
|
|
509
|
+
*/
|
|
510
|
+
u32(value, path, kind) {
|
|
511
|
+
return this.array(value, path, "u32", kind);
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* A Utf8 store slot (offsets + utf8), its layout checked at the structure level and its bytes at
|
|
515
|
+
* the full level.
|
|
516
|
+
* @param value - the manifest member (a WireUtf8)
|
|
517
|
+
* @param path - its path
|
|
518
|
+
* @param rows - the row count the store must have, or null to take it from the offsets (a dictionary)
|
|
519
|
+
* @param kind - the range owner
|
|
520
|
+
* @param always - check the layout at every level (stores decoded eagerly)
|
|
521
|
+
* @returns the offsets and bytes
|
|
522
|
+
*/
|
|
523
|
+
utf8(value, path, rows, kind, always = false) {
|
|
524
|
+
const o = asObject(value, path);
|
|
525
|
+
const offsets = this.u32(o.offsets, `${path}.offsets`, kind);
|
|
526
|
+
const utf8 = this.array(o.utf8, `${path}.utf8`, "utf8", kind);
|
|
527
|
+
if (offsets.length === 0) {
|
|
528
|
+
throw badWire(`${path}.offsets`, "a Utf8 store needs at least one offset");
|
|
529
|
+
}
|
|
530
|
+
if (this.structure || always) {
|
|
531
|
+
checkUtf8Layout(offsets, utf8, rows ?? offsets.length - 1, path);
|
|
532
|
+
}
|
|
533
|
+
if (this.full) {
|
|
534
|
+
const bad = Utf8Store.fromEncoded(offsets, utf8).firstMalformedRow();
|
|
535
|
+
if (bad !== -1) {
|
|
536
|
+
throw badWire(path, `malformed UTF-8 in row ${bad}`, { row: bad });
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
return { offsets, utf8 };
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Enforce the overlap rule of design section 9.5: no byte overlap between a mutable column and
|
|
543
|
+
* any core or immutable segment.
|
|
544
|
+
*/
|
|
545
|
+
checkOverlaps() {
|
|
546
|
+
const fixed = this.ranges.filter((r) => (r.kind === "core" || r.kind === "immutable") && r.end > r.start);
|
|
547
|
+
for (const range of this.ranges) {
|
|
548
|
+
if (range.kind !== "mutable" || range.end === range.start) {
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
for (const other of fixed) {
|
|
552
|
+
if (other.buffer === range.buffer && range.start < other.end && other.start < range.end) {
|
|
553
|
+
throw badWire(range.path, `mutable column overlaps ${other.path}`, { overlaps: other.path });
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
// ---------------------------------------------------------------- columns
|
|
559
|
+
/**
|
|
560
|
+
* Decode one column (design section 9.1), or drop it under unknownColumns: "skip".
|
|
561
|
+
* @param value - the WireColumn
|
|
562
|
+
* @param domain - the table's domain
|
|
563
|
+
* @param table - the extension table name, or null
|
|
564
|
+
* @param rows - the row count
|
|
565
|
+
* @param path - the manifest path
|
|
566
|
+
* @returns the column, or null when skipped
|
|
567
|
+
*/
|
|
568
|
+
column(value, domain, table, rows, path) {
|
|
569
|
+
const w = asObject(value, path);
|
|
570
|
+
const metaRaw = asObject(w.meta, `${path}.meta`);
|
|
571
|
+
const name = asString(metaRaw.name, `${path}.meta.name`);
|
|
572
|
+
const { dtype, itemDtype } = metaRaw;
|
|
573
|
+
const unknown = !isKnownDtype(dtype) || (dtype === "list" && !isKnownDtype(itemDtype));
|
|
574
|
+
if (unknown) {
|
|
575
|
+
const found = dtype === "list" ? itemDtype : dtype;
|
|
576
|
+
if (this.options.unknownColumns === "skip") {
|
|
577
|
+
this.skipped.push({ domain, table, name, dtype: String(found) });
|
|
578
|
+
return null;
|
|
579
|
+
}
|
|
580
|
+
throw new GraphFormatError("E_UNSUPPORTED", `column "${name}" has unknown dtype ${String(found)}`, {
|
|
581
|
+
dtype: found,
|
|
582
|
+
column: name,
|
|
583
|
+
ref: path,
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
const meta = this.columnMeta(metaRaw, name, domain, path);
|
|
587
|
+
const kind = meta.mutable ? "mutable" : "immutable";
|
|
588
|
+
const parts = emptyParts(meta, rows, w.validity === null || w.validity === undefined ? null : this.u32(w.validity, `${path}.validity`, kind));
|
|
589
|
+
const declaredNullCount = w.nullCount;
|
|
590
|
+
switch (meta.dtype) {
|
|
591
|
+
case "f32":
|
|
592
|
+
case "f64":
|
|
593
|
+
case "i32":
|
|
594
|
+
case "u32":
|
|
595
|
+
parts.data = this.array(w.data, `${path}.data`, meta.dtype, kind);
|
|
596
|
+
break;
|
|
597
|
+
case "u8": {
|
|
598
|
+
const data = this.array(w.data, `${path}.data`, "u8", kind);
|
|
599
|
+
parts.data = canViewAsPaddedU32(data) ? data : copyToPaddedStore(data);
|
|
600
|
+
break;
|
|
601
|
+
}
|
|
602
|
+
case "bool":
|
|
603
|
+
parts.data = this.u32(w.data, `${path}.data`, kind);
|
|
604
|
+
break;
|
|
605
|
+
case "dict": {
|
|
606
|
+
parts.data = this.u32(w.data, `${path}.data`, kind);
|
|
607
|
+
const dict = this.utf8(w.dictionary, `${path}.dictionary`, null, kind, true);
|
|
608
|
+
parts.dictionary = decodeUtf8Rows(dict.utf8, dict.offsets, 0, dict.offsets.length - 1);
|
|
609
|
+
break;
|
|
610
|
+
}
|
|
611
|
+
case "string": {
|
|
612
|
+
const store = this.utf8(w.strings, `${path}.strings`, rows, kind);
|
|
613
|
+
parts.offsets = store.offsets;
|
|
614
|
+
parts.utf8 = store.utf8;
|
|
615
|
+
break;
|
|
616
|
+
}
|
|
617
|
+
case "list": {
|
|
618
|
+
const offsets = this.u32(w.offsets, `${path}.offsets`, kind);
|
|
619
|
+
if (offsets.length !== rows + 1) {
|
|
620
|
+
throw badWire(`${path}.offsets`, `expected ${rows + 1} offsets, found ${offsets.length}`, {
|
|
621
|
+
expected: rows + 1,
|
|
622
|
+
found: offsets.length,
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
parts.offsets = offsets;
|
|
626
|
+
const child = this.column(w.child, domain, table, offsets[rows], `${path}.child`);
|
|
627
|
+
if (child === null || child.dtype === "list") {
|
|
628
|
+
throw badWire(`${path}.child`, "a list column needs a non-list child column");
|
|
629
|
+
}
|
|
630
|
+
parts.child = child;
|
|
631
|
+
break;
|
|
632
|
+
}
|
|
633
|
+
case "json":
|
|
634
|
+
parts.values = this.jsonRows(w.jsonText, rows, `${path}.jsonText`, kind);
|
|
635
|
+
break;
|
|
636
|
+
default: {
|
|
637
|
+
const dtypeName = meta.dtype;
|
|
638
|
+
throw badWire(`${path}.meta.dtype`, `unknown dtype ${dtypeName}`, { dtype: dtypeName });
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
let column;
|
|
642
|
+
try {
|
|
643
|
+
column = createColumn(parts);
|
|
644
|
+
}
|
|
645
|
+
catch (err) {
|
|
646
|
+
throw asWireError(err, path);
|
|
647
|
+
}
|
|
648
|
+
if (this.structure && column.nullCount !== declaredNullCount) {
|
|
649
|
+
throw badWire(`${path}.nullCount`, `manifest says ${String(declaredNullCount)}, the bitmap says ${column.nullCount}`, {
|
|
650
|
+
expected: column.nullCount,
|
|
651
|
+
found: declaredNullCount,
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
return column;
|
|
655
|
+
}
|
|
656
|
+
/**
|
|
657
|
+
* Resolve a column's metadata from its manifest form: the tagged JSON fields decoded, then the
|
|
658
|
+
* declaration resolver of the column module applied so every rule of design section 5.5 holds.
|
|
659
|
+
* @param raw - the manifest meta
|
|
660
|
+
* @param name - the column name
|
|
661
|
+
* @param domain - the table's domain
|
|
662
|
+
* @param path - the column's manifest path
|
|
663
|
+
* @returns the resolved metadata
|
|
664
|
+
*/
|
|
665
|
+
columnMeta(raw, name, domain, path) {
|
|
666
|
+
const metaPath = `${path}.meta`;
|
|
667
|
+
const optional = (key, check) => {
|
|
668
|
+
const v = raw[key];
|
|
669
|
+
return v === undefined || v === null ? undefined : check(v, `${metaPath}.${key}`);
|
|
670
|
+
};
|
|
671
|
+
const numberOf = (v, p) => {
|
|
672
|
+
if (typeof v !== "number") {
|
|
673
|
+
throw badWire(p, "expected a number", { found: v });
|
|
674
|
+
}
|
|
675
|
+
return v;
|
|
676
|
+
};
|
|
677
|
+
const decl = {
|
|
678
|
+
name,
|
|
679
|
+
dtype: raw.dtype,
|
|
680
|
+
components: optional("components", numberOf),
|
|
681
|
+
itemDtype: optional("itemDtype", (v) => v),
|
|
682
|
+
itemComponents: optional("itemComponents", numberOf),
|
|
683
|
+
nullable: optional("nullable", asBoolean),
|
|
684
|
+
mutable: optional("mutable", asBoolean),
|
|
685
|
+
role: optional("role", asString),
|
|
686
|
+
refersTo: optional("refersTo", (v) => v),
|
|
687
|
+
unique: optional("unique", asBoolean),
|
|
688
|
+
default: decodeJsonValue(raw.default, `${metaPath}.default`),
|
|
689
|
+
fill: decodeJsonValue(raw.fill, `${metaPath}.fill`),
|
|
690
|
+
options: optional("options", (v, p) => decodeJsonValue(v, p)),
|
|
691
|
+
origin: optional("origin", (v) => v),
|
|
692
|
+
dynamic: optional("dynamic", asBoolean),
|
|
693
|
+
extra: optional("extra", (v, p) => decodeJsonValue(v, p)),
|
|
694
|
+
};
|
|
695
|
+
try {
|
|
696
|
+
return resolveColumnMeta(name, domain, decl);
|
|
697
|
+
}
|
|
698
|
+
catch (err) {
|
|
699
|
+
throw asWireError(err, path);
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* The values of a json column from its JSON text store: empty text is an unset row (undefined),
|
|
704
|
+
* every other row is parsed with the guarding reviver and untagged.
|
|
705
|
+
* @param value - the WireUtf8
|
|
706
|
+
* @param rows - the row count
|
|
707
|
+
* @param path - the manifest path
|
|
708
|
+
* @param kind - the range owner
|
|
709
|
+
* @returns one value per row
|
|
710
|
+
*/
|
|
711
|
+
jsonRows(value, rows, path, kind) {
|
|
712
|
+
const store = this.utf8(value, path, rows, kind, true);
|
|
713
|
+
const texts = decodeUtf8Rows(store.utf8, store.offsets, 0, rows);
|
|
714
|
+
const values = new Array(rows);
|
|
715
|
+
for (let row = 0; row < rows; row++) {
|
|
716
|
+
const text = texts[row];
|
|
717
|
+
values[row] =
|
|
718
|
+
text === "" ? undefined : decodeJsonValue(parseGuardedJson(text, `${path}[${row}]`), `${path}[${row}]`);
|
|
719
|
+
}
|
|
720
|
+
return values;
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* Decode a table's columns in declaration order.
|
|
724
|
+
* @param value - the WireColumn array
|
|
725
|
+
* @param domain - the domain
|
|
726
|
+
* @param table - the extension table name, or null
|
|
727
|
+
* @param rows - the row count
|
|
728
|
+
* @param path - the manifest path
|
|
729
|
+
* @returns the table
|
|
730
|
+
*/
|
|
731
|
+
table(value, domain, table, rows, path) {
|
|
732
|
+
const list = asArray(value, path);
|
|
733
|
+
const columns = [];
|
|
734
|
+
const names = new Set();
|
|
735
|
+
for (let i = 0; i < list.length; i++) {
|
|
736
|
+
const column = this.column(list[i], domain, table, rows, `${path}[${i}]`);
|
|
737
|
+
if (column === null) {
|
|
738
|
+
continue;
|
|
739
|
+
}
|
|
740
|
+
if (names.has(column.meta.name)) {
|
|
741
|
+
throw badWire(`${path}[${i}].meta.name`, `column "${column.meta.name}" is declared twice`, {
|
|
742
|
+
column: column.meta.name,
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
names.add(column.meta.name);
|
|
746
|
+
columns.push(column);
|
|
747
|
+
}
|
|
748
|
+
try {
|
|
749
|
+
return new AttributeTable({ domain, rowCount: rows, columns });
|
|
750
|
+
}
|
|
751
|
+
catch (err) {
|
|
752
|
+
throw asWireError(err, path);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
// ---------------------------------------------------------------- id map
|
|
756
|
+
/**
|
|
757
|
+
* Decode the id map (design section 4.5). An unknown kind is E_UNSUPPORTED with details.kind.
|
|
758
|
+
* @param value - the WireIdMap
|
|
759
|
+
* @returns the map
|
|
760
|
+
*/
|
|
761
|
+
ids(value) {
|
|
762
|
+
const o = asObject(value, "ids");
|
|
763
|
+
const kindRaw = o.kind;
|
|
764
|
+
if (typeof kindRaw !== "string" || !ID_MAP_KINDS.has(kindRaw)) {
|
|
765
|
+
throw new GraphFormatError("E_UNSUPPORTED", `unknown id map kind ${describe(kindRaw)}`, {
|
|
766
|
+
kind: kindRaw,
|
|
767
|
+
ref: "ids.kind",
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
const kind = kindRaw;
|
|
771
|
+
const size = asCount(o.size, "ids.size");
|
|
772
|
+
const offsetRaw = o.offset ?? 0;
|
|
773
|
+
if (typeof offsetRaw !== "number") {
|
|
774
|
+
throw badWire("ids.offset", "expected a number", { found: offsetRaw });
|
|
775
|
+
}
|
|
776
|
+
const parts = {
|
|
777
|
+
kind,
|
|
778
|
+
size,
|
|
779
|
+
offset: kind === "identity" ? offsetRaw : 0,
|
|
780
|
+
values: null,
|
|
781
|
+
tags: null,
|
|
782
|
+
numbers: null,
|
|
783
|
+
offsets: null,
|
|
784
|
+
utf8: null,
|
|
785
|
+
};
|
|
786
|
+
const typed = parts;
|
|
787
|
+
switch (kind) {
|
|
788
|
+
case "identity":
|
|
789
|
+
break;
|
|
790
|
+
case "dense":
|
|
791
|
+
typed.values = this.u32(o.values, "ids.values", "immutable");
|
|
792
|
+
break;
|
|
793
|
+
case "numeric":
|
|
794
|
+
typed.values = this.array(o.values, "ids.values", "f64", "immutable");
|
|
795
|
+
break;
|
|
796
|
+
case "string": {
|
|
797
|
+
typed.offsets = this.u32(o.offsets, "ids.offsets", "immutable");
|
|
798
|
+
typed.utf8 = this.array(o.utf8, "ids.utf8", "utf8", "immutable");
|
|
799
|
+
break;
|
|
800
|
+
}
|
|
801
|
+
case "mixed": {
|
|
802
|
+
typed.tags = this.array(o.tags, "ids.tags", "u8", "immutable");
|
|
803
|
+
typed.numbers = this.array(o.numbers, "ids.numbers", "f64", "immutable");
|
|
804
|
+
typed.offsets = this.u32(o.offsets, "ids.offsets", "immutable");
|
|
805
|
+
typed.utf8 = this.array(o.utf8, "ids.utf8", "utf8", "immutable");
|
|
806
|
+
break;
|
|
807
|
+
}
|
|
808
|
+
default: {
|
|
809
|
+
const name = kind;
|
|
810
|
+
throw new GraphFormatError("E_UNSUPPORTED", `unknown id map kind ${name}`, { kind: name });
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
return nodeIdMapFromTyped(typed, this.options.level);
|
|
814
|
+
}
|
|
815
|
+
// ---------------------------------------------------------------- views
|
|
816
|
+
/**
|
|
817
|
+
* Resolve the member arrays of the carried views (design section 9.1): every reference is
|
|
818
|
+
* checked like any other, and the arrays are installed on the snapshot by `installViews()`.
|
|
819
|
+
* @param value - the manifest views member
|
|
820
|
+
* @returns the member arrays per view name (unknown names included; they are ignored later)
|
|
821
|
+
*/
|
|
822
|
+
views(value) {
|
|
823
|
+
const out = new Map();
|
|
824
|
+
if (value === null || value === undefined) {
|
|
825
|
+
return out;
|
|
826
|
+
}
|
|
827
|
+
const views = asObject(value, "views");
|
|
828
|
+
for (const name of Object.keys(views)) {
|
|
829
|
+
const members = asObject(views[name], `views.${name}`);
|
|
830
|
+
const arrays = new Map();
|
|
831
|
+
for (const member of Object.keys(members)) {
|
|
832
|
+
const path = `views.${name}.${member}`;
|
|
833
|
+
const ref = readRef(members[member], path);
|
|
834
|
+
arrays.set(member, this.array(ref, path, ref.dtype, "view"));
|
|
835
|
+
}
|
|
836
|
+
out.set(name, arrays);
|
|
837
|
+
}
|
|
838
|
+
return out;
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* One member array of a carried view, checked for class and length.
|
|
843
|
+
* @param members - the view's member arrays
|
|
844
|
+
* @param view - the view name (for the error path)
|
|
845
|
+
* @param member - the member name
|
|
846
|
+
* @param ctor - the required typed-array class
|
|
847
|
+
* @param length - the required length
|
|
848
|
+
* @param required - whether an absent member is an error
|
|
849
|
+
* @returns the array, or null when absent and not required
|
|
850
|
+
*/
|
|
851
|
+
function viewMember(members, view, member, ctor, length, required) {
|
|
852
|
+
const path = `views.${view}.${member}`;
|
|
853
|
+
const array = members.get(member);
|
|
854
|
+
if (array === undefined) {
|
|
855
|
+
if (required) {
|
|
856
|
+
throw badWire(path, "the carried view is missing this member");
|
|
857
|
+
}
|
|
858
|
+
return null;
|
|
859
|
+
}
|
|
860
|
+
if (!(array instanceof ctor)) {
|
|
861
|
+
throw badWire(path, `expected ${ctor.name}, found ${array.constructor.name}`, {
|
|
862
|
+
expected: ctor.name,
|
|
863
|
+
found: array.constructor.name,
|
|
864
|
+
});
|
|
865
|
+
}
|
|
866
|
+
if (array.length !== length) {
|
|
867
|
+
throw badWire(path, `expected ${length} entries, found ${array.length}`, {
|
|
868
|
+
expected: length,
|
|
869
|
+
found: array.length,
|
|
870
|
+
});
|
|
871
|
+
}
|
|
872
|
+
return array;
|
|
873
|
+
}
|
|
874
|
+
/**
|
|
875
|
+
* The per-node or per-arc array of a single-member view.
|
|
876
|
+
* @param members - the view's member arrays
|
|
877
|
+
* @param view - the view name
|
|
878
|
+
* @param ctor - the required typed-array class
|
|
879
|
+
* @param length - the required length
|
|
880
|
+
* @returns the array
|
|
881
|
+
*/
|
|
882
|
+
function dataMember(members, view, ctor, length) {
|
|
883
|
+
return viewMember(members, view, "data", ctor, length, true);
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Install the views a manifest carried into the snapshot's cache (design section 9.1), so the
|
|
887
|
+
* receiver does not recompute them. Views that alias another view on an undirected snapshot
|
|
888
|
+
* (`reverse`, `inDegree`, `weightedInDegree`, `reverseDegreeOrder`) are left to the snapshot, which
|
|
889
|
+
* produces the alias itself; `mate` on a directed snapshot and unknown view names are ignored.
|
|
890
|
+
* Every installed member must have the class and length of the view it claims to be
|
|
891
|
+
* (E_BAD_SERIALIZATION otherwise); at the "structure" level (`verify`) every carried view is also
|
|
892
|
+
* checked against the core before it is installed (src/wire/carried-views.ts), so a corrupt or
|
|
893
|
+
* forged view never enters the cache below "full".
|
|
894
|
+
* @param snapshot - the freshly built snapshot
|
|
895
|
+
* @param carried - the carried member arrays
|
|
896
|
+
* @param verify - whether to check the contents (the "structure" level)
|
|
897
|
+
*/
|
|
898
|
+
function installViews(snapshot, carried, verify) {
|
|
899
|
+
const { directed, nodeCount, edgeCount, arcCount, selfLoopCount } = snapshot;
|
|
900
|
+
let facts = null;
|
|
901
|
+
const factsOf = () => {
|
|
902
|
+
facts ?? (facts = new CoreFacts(snapshot));
|
|
903
|
+
return facts;
|
|
904
|
+
};
|
|
905
|
+
/**
|
|
906
|
+
* Install an f64 view; under `verify` it is compared with the snapshot's own computation instead
|
|
907
|
+
* (which then stays cached), so a corrupt value array never enters the cache.
|
|
908
|
+
* @param name - the view
|
|
909
|
+
*/
|
|
910
|
+
const seedF64 = (name) => {
|
|
911
|
+
const data = dataMember(carried.get(name), name, Float64Array, nodeCount);
|
|
912
|
+
if (verify) {
|
|
913
|
+
checkF64View(name, data, snapshot[name]());
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
seedView(snapshot, name, data);
|
|
917
|
+
};
|
|
918
|
+
for (const [name, members] of carried) {
|
|
919
|
+
switch (name) {
|
|
920
|
+
case "reverse": {
|
|
921
|
+
if (!directed) {
|
|
922
|
+
break;
|
|
923
|
+
}
|
|
924
|
+
const rowPtr = viewMember(members, name, "rowPtr", Uint32Array, nodeCount + 1, true);
|
|
925
|
+
const colIdx = viewMember(members, name, "colIdx", Uint32Array, arcCount, true);
|
|
926
|
+
const fwdArc = viewMember(members, name, "fwdArc", Uint32Array, arcCount, true);
|
|
927
|
+
const weights = viewMember(members, name, "weights", Float32Array, arcCount, snapshot.weights !== null);
|
|
928
|
+
if (weights !== null && snapshot.weights === null) {
|
|
929
|
+
throw badWire(`views.${name}.weights`, "weights carried for an unweighted snapshot");
|
|
930
|
+
}
|
|
931
|
+
const reverse = new ReverseAdjacency(snapshot, rowPtr, colIdx, weights, fwdArc);
|
|
932
|
+
if (verify) {
|
|
933
|
+
checkReverse(snapshot, factsOf(), reverse);
|
|
934
|
+
}
|
|
935
|
+
seedView(snapshot, "reverse", reverse);
|
|
936
|
+
break;
|
|
937
|
+
}
|
|
938
|
+
case "coo": {
|
|
939
|
+
const src = viewMember(members, name, "src", Uint32Array, arcCount, true);
|
|
940
|
+
if (verify) {
|
|
941
|
+
checkCooSrc(factsOf(), src);
|
|
942
|
+
}
|
|
943
|
+
seedView(snapshot, "coo", cooViewOf(snapshot, src));
|
|
944
|
+
break;
|
|
945
|
+
}
|
|
946
|
+
case "edgeList": {
|
|
947
|
+
const identity = snapshot.flags.arcToEdgeIsIdentity;
|
|
948
|
+
const src = viewMember(members, name, "src", Uint32Array, edgeCount, true);
|
|
949
|
+
const dst = viewMember(members, name, "dst", Uint32Array, edgeCount, !identity);
|
|
950
|
+
const weights = viewMember(members, name, "weights", Float32Array, edgeCount, snapshot.weights !== null && !identity);
|
|
951
|
+
if (weights !== null && snapshot.weights === null) {
|
|
952
|
+
throw badWire(`views.${name}.weights`, "weights carried for an unweighted snapshot");
|
|
953
|
+
}
|
|
954
|
+
if (verify) {
|
|
955
|
+
checkEdgeList(snapshot, factsOf(), src, dst, weights);
|
|
956
|
+
}
|
|
957
|
+
seedView(snapshot, "edgeList", edgeListViewOf(snapshot, src, dst ?? snapshot.colIdx, weights ?? snapshot.weights));
|
|
958
|
+
break;
|
|
959
|
+
}
|
|
960
|
+
case "outDegree": {
|
|
961
|
+
const data = dataMember(members, name, Uint32Array, nodeCount);
|
|
962
|
+
if (verify) {
|
|
963
|
+
checkOutDegree(snapshot, data);
|
|
964
|
+
}
|
|
965
|
+
seedView(snapshot, name, data);
|
|
966
|
+
break;
|
|
967
|
+
}
|
|
968
|
+
case "degree": {
|
|
969
|
+
const data = dataMember(members, name, Uint32Array, nodeCount);
|
|
970
|
+
if (verify) {
|
|
971
|
+
checkDegree(snapshot, factsOf(), data);
|
|
972
|
+
}
|
|
973
|
+
seedView(snapshot, name, data);
|
|
974
|
+
break;
|
|
975
|
+
}
|
|
976
|
+
case "selfLoopsPerNode": {
|
|
977
|
+
const data = dataMember(members, name, Uint32Array, nodeCount);
|
|
978
|
+
if (verify) {
|
|
979
|
+
checkSelfLoopsPerNode(factsOf(), data);
|
|
980
|
+
}
|
|
981
|
+
seedView(snapshot, name, data);
|
|
982
|
+
break;
|
|
983
|
+
}
|
|
984
|
+
case "inDegree":
|
|
985
|
+
if (directed) {
|
|
986
|
+
const data = dataMember(members, name, Uint32Array, nodeCount);
|
|
987
|
+
if (verify) {
|
|
988
|
+
checkInDegree(factsOf(), data);
|
|
989
|
+
}
|
|
990
|
+
seedView(snapshot, name, data);
|
|
991
|
+
}
|
|
992
|
+
break;
|
|
993
|
+
case "weightedOutDegree":
|
|
994
|
+
case "weightedDegree":
|
|
995
|
+
case "selfLoopWeight":
|
|
996
|
+
seedF64(name);
|
|
997
|
+
break;
|
|
998
|
+
case "weightedInDegree":
|
|
999
|
+
if (directed) {
|
|
1000
|
+
seedF64(name);
|
|
1001
|
+
}
|
|
1002
|
+
break;
|
|
1003
|
+
case "selfLoopArcs": {
|
|
1004
|
+
const data = dataMember(members, name, Uint32Array, selfLoopCount);
|
|
1005
|
+
if (verify) {
|
|
1006
|
+
checkSelfLoopArcs(snapshot, factsOf(), data);
|
|
1007
|
+
}
|
|
1008
|
+
seedView(snapshot, name, data);
|
|
1009
|
+
break;
|
|
1010
|
+
}
|
|
1011
|
+
case "mate":
|
|
1012
|
+
if (!directed) {
|
|
1013
|
+
const data = dataMember(members, name, Uint32Array, arcCount);
|
|
1014
|
+
if (verify) {
|
|
1015
|
+
checkMate(snapshot, factsOf(), data);
|
|
1016
|
+
}
|
|
1017
|
+
seedView(snapshot, name, data);
|
|
1018
|
+
}
|
|
1019
|
+
break;
|
|
1020
|
+
case "degreeOrder":
|
|
1021
|
+
case "reverseDegreeOrder": {
|
|
1022
|
+
if (name === "reverseDegreeOrder" && !directed) {
|
|
1023
|
+
break;
|
|
1024
|
+
}
|
|
1025
|
+
const perm = viewMember(members, name, "perm", Uint32Array, nodeCount, true);
|
|
1026
|
+
const segmentOffsets = viewMember(members, name, "segmentOffsets", Uint32Array, 5, true);
|
|
1027
|
+
if (verify) {
|
|
1028
|
+
const degree = name === "degreeOrder" ? rowLengths(snapshot.rowPtr, nodeCount) : factsOf().inDegree;
|
|
1029
|
+
checkDegreeOrder(name, degree, perm, segmentOffsets);
|
|
1030
|
+
}
|
|
1031
|
+
seedView(snapshot, name, Object.freeze({ perm, segmentOffsets }));
|
|
1032
|
+
break;
|
|
1033
|
+
}
|
|
1034
|
+
default:
|
|
1035
|
+
// An unknown view name (a newer minor) or a scalar view: ignored and recomputed.
|
|
1036
|
+
break;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
/**
|
|
1041
|
+
* Read the core references from the manifest, applying the null rules of design section 9.1.
|
|
1042
|
+
* @param value - manifest.core
|
|
1043
|
+
* @returns the references
|
|
1044
|
+
*/
|
|
1045
|
+
function readCoreRefs(value) {
|
|
1046
|
+
const core = asObject(value, "core");
|
|
1047
|
+
const optionalRef = (name) => {
|
|
1048
|
+
const v = core[name];
|
|
1049
|
+
return v === null || v === undefined ? null : readRef(v, `core.${name}`);
|
|
1050
|
+
};
|
|
1051
|
+
return {
|
|
1052
|
+
rowPtr: readRef(core.rowPtr, "core.rowPtr"),
|
|
1053
|
+
colIdx: optionalRef("colIdx"),
|
|
1054
|
+
weights: optionalRef("weights"),
|
|
1055
|
+
arcToEdge: optionalRef("arcToEdge"),
|
|
1056
|
+
edgeToArc: optionalRef("edgeToArc"),
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
/**
|
|
1060
|
+
* Build the core arrays: a null colIdx is a zero-arc graph, a null weights array an unweighted (or
|
|
1061
|
+
* zero-arc weighted) graph, a null permutation the identity (when the flag says so) or zero length.
|
|
1062
|
+
* @param decoder - the decoder
|
|
1063
|
+
* @param refs - the core references
|
|
1064
|
+
* @param counts - the manifest counts
|
|
1065
|
+
* @param flags - the manifest flags
|
|
1066
|
+
* @returns the arrays
|
|
1067
|
+
*/
|
|
1068
|
+
function decodeCore(decoder, refs, counts, flags) {
|
|
1069
|
+
const rowPtr = decoder.u32(refs.rowPtr, "core.rowPtr", "core");
|
|
1070
|
+
let colIdx;
|
|
1071
|
+
if (refs.colIdx === null) {
|
|
1072
|
+
if (counts.arcs !== 0) {
|
|
1073
|
+
throw badWire("core.colIdx", `null although counts.arcs is ${counts.arcs}`);
|
|
1074
|
+
}
|
|
1075
|
+
colIdx = new Uint32Array(0);
|
|
1076
|
+
}
|
|
1077
|
+
else {
|
|
1078
|
+
colIdx = decoder.u32(refs.colIdx, "core.colIdx", "core");
|
|
1079
|
+
}
|
|
1080
|
+
let weights = null;
|
|
1081
|
+
if (refs.weights !== null) {
|
|
1082
|
+
if (!flags.weighted) {
|
|
1083
|
+
throw badWire("core.weights", "present although flags.weighted is false");
|
|
1084
|
+
}
|
|
1085
|
+
weights = decoder.array(refs.weights, "core.weights", "f32", "core");
|
|
1086
|
+
}
|
|
1087
|
+
else if (flags.weighted) {
|
|
1088
|
+
if (counts.arcs !== 0) {
|
|
1089
|
+
throw badWire("core.weights", `null although flags.weighted is true and counts.arcs is ${counts.arcs}`);
|
|
1090
|
+
}
|
|
1091
|
+
weights = new Float32Array(0);
|
|
1092
|
+
}
|
|
1093
|
+
const identity = flags.arcToEdgeIsIdentity;
|
|
1094
|
+
const permutation = (ref, name, count) => {
|
|
1095
|
+
if (ref !== null) {
|
|
1096
|
+
if (identity) {
|
|
1097
|
+
throw badWire(`core.${name}`, "present although flags.arcToEdgeIsIdentity is true");
|
|
1098
|
+
}
|
|
1099
|
+
return decoder.u32(ref, `core.${name}`, "core");
|
|
1100
|
+
}
|
|
1101
|
+
if (identity) {
|
|
1102
|
+
return null;
|
|
1103
|
+
}
|
|
1104
|
+
if (count !== 0) {
|
|
1105
|
+
throw badWire(`core.${name}`, `null although the permutation is not the identity and the count is ${count}`);
|
|
1106
|
+
}
|
|
1107
|
+
return new Uint32Array(0);
|
|
1108
|
+
};
|
|
1109
|
+
return {
|
|
1110
|
+
rowPtr,
|
|
1111
|
+
colIdx,
|
|
1112
|
+
weights,
|
|
1113
|
+
arcToEdge: permutation(refs.arcToEdge, "arcToEdge", counts.arcs),
|
|
1114
|
+
edgeToArc: permutation(refs.edgeToArc, "edgeToArc", counts.edges),
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* Honour the manifest's arena descriptor (design sections 9.1 and 10.3): the core arrays lying at
|
|
1119
|
+
* 256-aligned offsets inside the described range become its segments; hotByteLength is recomputed
|
|
1120
|
+
* and, at the structure level, compared with the manifest.
|
|
1121
|
+
* @param value - manifest.arena
|
|
1122
|
+
* @param regions - the buffer regions
|
|
1123
|
+
* @param refs - the core references
|
|
1124
|
+
* @param arrays - the core arrays
|
|
1125
|
+
* @param structure - whether the structure checks run
|
|
1126
|
+
* @returns the arena layout, or null when the manifest carries none or the region is chunked
|
|
1127
|
+
*/
|
|
1128
|
+
function decodeArena(value, regions, refs, arrays, structure) {
|
|
1129
|
+
if (value === null || value === undefined) {
|
|
1130
|
+
return null;
|
|
1131
|
+
}
|
|
1132
|
+
const a = asObject(value, "arena");
|
|
1133
|
+
const buffer = asCount(a.buffer, "arena.buffer");
|
|
1134
|
+
const byteOffset = asCount(a.byteOffset, "arena.byteOffset");
|
|
1135
|
+
const byteLength = asCount(a.byteLength, "arena.byteLength");
|
|
1136
|
+
const region = regions[buffer];
|
|
1137
|
+
if (region === undefined) {
|
|
1138
|
+
throw badWire("arena.buffer", `buffer index ${buffer} is out of range`, { found: buffer });
|
|
1139
|
+
}
|
|
1140
|
+
if (byteOffset + byteLength > region.byteLength) {
|
|
1141
|
+
throw badWire("arena", `range [${byteOffset}, ${byteOffset + byteLength}) exceeds the buffer`, {
|
|
1142
|
+
byteOffset,
|
|
1143
|
+
byteLength,
|
|
1144
|
+
bufferByteLength: region.byteLength,
|
|
1145
|
+
});
|
|
1146
|
+
}
|
|
1147
|
+
const located = region.locate(byteOffset, byteLength);
|
|
1148
|
+
if (located === null) {
|
|
1149
|
+
return null;
|
|
1150
|
+
}
|
|
1151
|
+
const segments = {
|
|
1152
|
+
rowPtr: null,
|
|
1153
|
+
colIdx: null,
|
|
1154
|
+
weights: null,
|
|
1155
|
+
arcToEdge: null,
|
|
1156
|
+
edgeToArc: null,
|
|
1157
|
+
};
|
|
1158
|
+
let hotByteLength = 0;
|
|
1159
|
+
const names = CORE_ORDER;
|
|
1160
|
+
for (const name of names) {
|
|
1161
|
+
const ref = refs[name];
|
|
1162
|
+
const array = arrays[name];
|
|
1163
|
+
if (ref === null || array === null || array.length === 0) {
|
|
1164
|
+
continue;
|
|
1165
|
+
}
|
|
1166
|
+
const inside = ref.buffer === buffer &&
|
|
1167
|
+
ref.byteOffset >= byteOffset &&
|
|
1168
|
+
ref.byteOffset + ref.byteLength <= byteOffset + byteLength;
|
|
1169
|
+
if (!inside) {
|
|
1170
|
+
continue;
|
|
1171
|
+
}
|
|
1172
|
+
const relative = ref.byteOffset - byteOffset;
|
|
1173
|
+
if (relative % ALIGNMENT !== 0) {
|
|
1174
|
+
throw badWire(`core.${name}`, `inside the arena at offset ${relative}, not a multiple of ${ALIGNMENT}`, {
|
|
1175
|
+
byteOffset: relative,
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
if (array.buffer !== located.buffer || array.byteOffset !== located.byteOffset + relative) {
|
|
1179
|
+
throw badWire(`core.${name}`, "the arena describes a segment the array is not a view of");
|
|
1180
|
+
}
|
|
1181
|
+
segments[name] = Object.freeze({ byteOffset: located.byteOffset + relative, byteLength: ref.byteLength });
|
|
1182
|
+
if (name === "rowPtr" || name === "colIdx" || name === "weights") {
|
|
1183
|
+
hotByteLength = relative + ref.byteLength;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
if (structure && a.hotByteLength !== hotByteLength) {
|
|
1187
|
+
throw badWire("arena.hotByteLength", `manifest says ${String(a.hotByteLength)}, the segments say ${hotByteLength}`, {
|
|
1188
|
+
expected: hotByteLength,
|
|
1189
|
+
found: a.hotByteLength,
|
|
1190
|
+
});
|
|
1191
|
+
}
|
|
1192
|
+
return Object.freeze({
|
|
1193
|
+
buffer: located.buffer,
|
|
1194
|
+
byteOffset: located.byteOffset,
|
|
1195
|
+
byteLength,
|
|
1196
|
+
alignment: ALIGNMENT,
|
|
1197
|
+
segments: Object.freeze(segments),
|
|
1198
|
+
hotByteLength,
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
/**
|
|
1202
|
+
* Read the flags (every member a boolean).
|
|
1203
|
+
* @param value - manifest.flags
|
|
1204
|
+
* @returns the flags
|
|
1205
|
+
*/
|
|
1206
|
+
function readFlags(value) {
|
|
1207
|
+
const o = asObject(value, "flags");
|
|
1208
|
+
const flag = (name) => asBoolean(o[name], `flags.${name}`);
|
|
1209
|
+
return {
|
|
1210
|
+
multigraph: flag("multigraph"),
|
|
1211
|
+
hasSelfLoops: flag("hasSelfLoops"),
|
|
1212
|
+
arcToEdgeIsIdentity: flag("arcToEdgeIsIdentity"),
|
|
1213
|
+
weighted: flag("weighted"),
|
|
1214
|
+
allWeightsOne: flag("allWeightsOne"),
|
|
1215
|
+
nonNegativeWeights: flag("nonNegativeWeights"),
|
|
1216
|
+
finiteWeights: flag("finiteWeights"),
|
|
1217
|
+
};
|
|
1218
|
+
}
|
|
1219
|
+
/**
|
|
1220
|
+
* Read a column origin (every field a string or null).
|
|
1221
|
+
* @param value - the manifest member
|
|
1222
|
+
* @param path - its path
|
|
1223
|
+
* @returns the origin, or null
|
|
1224
|
+
*/
|
|
1225
|
+
function readOrigin(value, path) {
|
|
1226
|
+
if (value === null || value === undefined) {
|
|
1227
|
+
return null;
|
|
1228
|
+
}
|
|
1229
|
+
const o = asObject(value, path);
|
|
1230
|
+
const field = (key) => asStringOrNull(o[key], `${path}.${key}`);
|
|
1231
|
+
return {
|
|
1232
|
+
format: field("format"),
|
|
1233
|
+
id: field("id"),
|
|
1234
|
+
title: field("title"),
|
|
1235
|
+
type: field("type"),
|
|
1236
|
+
namespace: field("namespace"),
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
/**
|
|
1240
|
+
* Read the graph metadata (design section 5.9); unknown fields are ignored, missing ones are null.
|
|
1241
|
+
* @param value - manifest.meta
|
|
1242
|
+
* @returns the frozen metadata
|
|
1243
|
+
*/
|
|
1244
|
+
function readGraphMeta(value) {
|
|
1245
|
+
const o = asObject(value, "meta");
|
|
1246
|
+
const keywordsRaw = o.keywords ?? [];
|
|
1247
|
+
const keywords = asArray(keywordsRaw, "meta.keywords").map((k, i) => asString(k, `meta.keywords[${i}]`));
|
|
1248
|
+
const extraRaw = decodeJsonValue(o.extra ?? {}, "meta.extra");
|
|
1249
|
+
if (!isPlainObject(extraRaw)) {
|
|
1250
|
+
throw badWire("meta.extra", "expected an object");
|
|
1251
|
+
}
|
|
1252
|
+
return Object.freeze({
|
|
1253
|
+
name: asStringOrNull(o.name, "meta.name"),
|
|
1254
|
+
description: asStringOrNull(o.description, "meta.description"),
|
|
1255
|
+
creator: asStringOrNull(o.creator, "meta.creator"),
|
|
1256
|
+
created: asStringOrNull(o.created, "meta.created"),
|
|
1257
|
+
modified: asStringOrNull(o.modified, "meta.modified"),
|
|
1258
|
+
keywords: Object.freeze(keywords),
|
|
1259
|
+
sourceFormat: asStringOrNull(o.sourceFormat, "meta.sourceFormat"),
|
|
1260
|
+
sourceVersion: asStringOrNull(o.sourceVersion, "meta.sourceVersion"),
|
|
1261
|
+
idType: asEnum(o.idType, "meta.idType", ID_TYPES),
|
|
1262
|
+
timeFormat: asEnum(o.timeFormat, "meta.timeFormat", TIME_FORMATS),
|
|
1263
|
+
timeRepresentation: asEnum(o.timeRepresentation, "meta.timeRepresentation", TIME_REPRESENTATIONS),
|
|
1264
|
+
mode: asEnum(o.mode, "meta.mode", MODES),
|
|
1265
|
+
declaredMultigraph: asBooleanOrNull(o.declaredMultigraph, "meta.declaredMultigraph"),
|
|
1266
|
+
weightOrigin: readOrigin(o.weightOrigin, "meta.weightOrigin"),
|
|
1267
|
+
extra: Object.freeze(extraRaw),
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
/**
|
|
1271
|
+
* Check the manifest's discriminator and versions (design sections 9.1 and 13.5): `format`, the
|
|
1272
|
+
* wire major (E_UNSUPPORTED_VERSION kind "wire") and `formatVersion` (kind "format"). A newer wire
|
|
1273
|
+
* minor is accepted.
|
|
1274
|
+
* @param manifest - the manifest object
|
|
1275
|
+
* @returns the wire version pair
|
|
1276
|
+
*/
|
|
1277
|
+
export function checkManifestVersions(manifest) {
|
|
1278
|
+
if (manifest.format !== WIRE_FORMAT) {
|
|
1279
|
+
throw badWire("format", `expected "${WIRE_FORMAT}", found ${describe(manifest.format)}`, {
|
|
1280
|
+
found: manifest.format,
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
const wire = asArray(manifest.wire, "wire");
|
|
1284
|
+
const major = wire[0];
|
|
1285
|
+
const minor = wire[1];
|
|
1286
|
+
if (typeof major !== "number" ||
|
|
1287
|
+
typeof minor !== "number" ||
|
|
1288
|
+
!Number.isInteger(major) ||
|
|
1289
|
+
!Number.isInteger(minor)) {
|
|
1290
|
+
throw badWire("wire", "expected [major, minor] integers", { found: manifest.wire });
|
|
1291
|
+
}
|
|
1292
|
+
if (major !== WIRE_MAJOR) {
|
|
1293
|
+
throw new GraphFormatError("E_UNSUPPORTED_VERSION", `wire major ${major} is not supported (reader: ${WIRE_MAJOR})`, {
|
|
1294
|
+
kind: "wire",
|
|
1295
|
+
found: major,
|
|
1296
|
+
supported: WIRE_MAJOR,
|
|
1297
|
+
});
|
|
1298
|
+
}
|
|
1299
|
+
if (manifest.formatVersion !== FORMAT_VERSION) {
|
|
1300
|
+
throw new GraphFormatError("E_UNSUPPORTED_VERSION", `formatVersion ${describe(manifest.formatVersion)} is not supported (reader: ${FORMAT_VERSION})`, { kind: "format", found: manifest.formatVersion, supported: FORMAT_VERSION });
|
|
1301
|
+
}
|
|
1302
|
+
return [major, minor];
|
|
1303
|
+
}
|
|
1304
|
+
// ============================================================ the manifest reader
|
|
1305
|
+
/**
|
|
1306
|
+
* Rebuild a snapshot from a manifest and the regions its references index (design sections 9.1 and
|
|
1307
|
+
* 9.5): versions checked, core arrays and arena adopted, id map and tables decoded, then the
|
|
1308
|
+
* snapshot validated at the requested level.
|
|
1309
|
+
* @param manifestRaw - the manifest (an object; already parsed for a container)
|
|
1310
|
+
* @param regions - one region per buffer index
|
|
1311
|
+
* @param options - the resolved options
|
|
1312
|
+
* @param honourArena - whether the manifest's arena descriptor is adopted (false for chunked input)
|
|
1313
|
+
* @returns the snapshot
|
|
1314
|
+
*/
|
|
1315
|
+
export function decodeManifest(manifestRaw, regions, options, honourArena) {
|
|
1316
|
+
const manifest = asObject(manifestRaw, "manifest");
|
|
1317
|
+
checkManifestVersions(manifest);
|
|
1318
|
+
const decoder = new WireDecoder(regions, options);
|
|
1319
|
+
const directed = asBoolean(manifest.directed, "directed");
|
|
1320
|
+
const countsRaw = asObject(manifest.counts, "counts");
|
|
1321
|
+
const counts = {
|
|
1322
|
+
nodes: asCount(countsRaw.nodes, "counts.nodes"),
|
|
1323
|
+
edges: asCount(countsRaw.edges, "counts.edges"),
|
|
1324
|
+
arcs: asCount(countsRaw.arcs, "counts.arcs"),
|
|
1325
|
+
selfLoops: asCount(countsRaw.selfLoops, "counts.selfLoops"),
|
|
1326
|
+
};
|
|
1327
|
+
const flags = readFlags(manifest.flags);
|
|
1328
|
+
if (flags.arcToEdgeIsIdentity && (!directed || counts.arcs !== counts.edges)) {
|
|
1329
|
+
throw badWire("flags.arcToEdgeIsIdentity", "set on an undirected graph or with arcs !== edges");
|
|
1330
|
+
}
|
|
1331
|
+
const coreRefs = readCoreRefs(manifest.core);
|
|
1332
|
+
const core = decodeCore(decoder, coreRefs, counts, flags);
|
|
1333
|
+
const arena = honourArena ? decodeArena(manifest.arena, regions, coreRefs, core, decoder.structure) : null;
|
|
1334
|
+
const ids = decoder.ids(manifest.ids);
|
|
1335
|
+
const nodes = decoder.table(manifest.nodeColumns ?? [], "node", null, counts.nodes, "nodeColumns");
|
|
1336
|
+
const edges = decoder.table(manifest.edgeColumns ?? [], "edge", null, counts.edges, "edgeColumns");
|
|
1337
|
+
const graph = decoder.table(manifest.graphColumns ?? [], "graph", null, 1, "graphColumns");
|
|
1338
|
+
const extensions = new Map();
|
|
1339
|
+
const extensionsRaw = asArray(manifest.extensions ?? [], "extensions");
|
|
1340
|
+
for (let i = 0; i < extensionsRaw.length; i++) {
|
|
1341
|
+
const path = `extensions[${i}]`;
|
|
1342
|
+
const ext = asObject(extensionsRaw[i], path);
|
|
1343
|
+
const name = asString(ext.name, `${path}.name`);
|
|
1344
|
+
if (extensions.has(name)) {
|
|
1345
|
+
throw badWire(`${path}.name`, `extension table "${name}" appears twice`, { table: name });
|
|
1346
|
+
}
|
|
1347
|
+
const rowCount = asCount(ext.rowCount, `${path}.rowCount`);
|
|
1348
|
+
extensions.set(name, decoder.table(ext.columns, "extension", name, rowCount, `${path}.columns`));
|
|
1349
|
+
}
|
|
1350
|
+
let meta = readGraphMeta(manifest.meta ?? {});
|
|
1351
|
+
if (decoder.skipped.length > 0) {
|
|
1352
|
+
const skipped = Object.freeze(decoder.skipped.map((s) => Object.freeze({ ...s })));
|
|
1353
|
+
meta = Object.freeze({ ...meta, extra: Object.freeze({ ...meta.extra, [SKIPPED_COLUMNS_KEY]: skipped }) });
|
|
1354
|
+
}
|
|
1355
|
+
const carried = decoder.views(manifest.views);
|
|
1356
|
+
if (decoder.structure) {
|
|
1357
|
+
decoder.checkOverlaps();
|
|
1358
|
+
}
|
|
1359
|
+
const parts = {
|
|
1360
|
+
label: asStringOrNull(manifest.label, "label"),
|
|
1361
|
+
serial: null,
|
|
1362
|
+
directed,
|
|
1363
|
+
nodeCount: counts.nodes,
|
|
1364
|
+
edgeCount: counts.edges,
|
|
1365
|
+
arcCount: counts.arcs,
|
|
1366
|
+
selfLoopCount: counts.selfLoops,
|
|
1367
|
+
rowPtr: core.rowPtr,
|
|
1368
|
+
colIdx: core.colIdx,
|
|
1369
|
+
weights: core.weights,
|
|
1370
|
+
arcToEdge: core.arcToEdge,
|
|
1371
|
+
edgeToArc: core.edgeToArc,
|
|
1372
|
+
flags,
|
|
1373
|
+
ids,
|
|
1374
|
+
nodes,
|
|
1375
|
+
edges,
|
|
1376
|
+
graph,
|
|
1377
|
+
extensions,
|
|
1378
|
+
meta,
|
|
1379
|
+
arena,
|
|
1380
|
+
checksum: false,
|
|
1381
|
+
};
|
|
1382
|
+
const snapshot = createSnapshot(parts);
|
|
1383
|
+
if (options.level === "full") {
|
|
1384
|
+
snapshot.validate({ level: "full" });
|
|
1385
|
+
}
|
|
1386
|
+
else if (options.level === "structure") {
|
|
1387
|
+
snapshot.validate({ level: "structure" });
|
|
1388
|
+
}
|
|
1389
|
+
if (options.level !== "full") {
|
|
1390
|
+
installViews(snapshot, carried, options.level === "structure");
|
|
1391
|
+
}
|
|
1392
|
+
return snapshot;
|
|
1393
|
+
}
|
|
1394
|
+
/**
|
|
1395
|
+
* Rebuild a snapshot from its wire form (design section 9.1). Defaults: `validate: "structure"`
|
|
1396
|
+
* (the manifest is trusted to have come from this package; the cheap checks still run) and `copy:
|
|
1397
|
+
* false` (the arrays are views into `wire.buffers`; a SharedArrayBuffer is copied). The reverse id
|
|
1398
|
+
* Map and decoded strings are rebuilt lazily; carried views (`includeViews`) are installed on the
|
|
1399
|
+
* receiver below "full" and recomputed under "full", where nothing carried is trusted.
|
|
1400
|
+
* @param wire - the wire snapshot
|
|
1401
|
+
* @param options - validation level, copy and unknown-column policy
|
|
1402
|
+
* @returns the snapshot; E_UNSUPPORTED_VERSION, E_UNSUPPORTED, E_BAD_SERIALIZATION or E_INVALID_SNAPSHOT on bad input
|
|
1403
|
+
*/
|
|
1404
|
+
export function fromWire(wire, options) {
|
|
1405
|
+
const resolved = resolveFromWireOptions(options, "structure");
|
|
1406
|
+
const w = asObject(wire, "wire");
|
|
1407
|
+
const buffersRaw = asArray(w.buffers, "wire.buffers");
|
|
1408
|
+
const regions = [];
|
|
1409
|
+
for (let i = 0; i < buffersRaw.length; i++) {
|
|
1410
|
+
const b = buffersRaw[i];
|
|
1411
|
+
let buffer;
|
|
1412
|
+
if (b instanceof ArrayBuffer) {
|
|
1413
|
+
buffer = resolved.copy ? copyBuffer(b) : b;
|
|
1414
|
+
}
|
|
1415
|
+
else if (isSharedArrayBuffer(b)) {
|
|
1416
|
+
buffer = copyBuffer(b);
|
|
1417
|
+
}
|
|
1418
|
+
else {
|
|
1419
|
+
throw badWire(`wire.buffers[${i}]`, "expected an ArrayBuffer");
|
|
1420
|
+
}
|
|
1421
|
+
regions.push(bufferRegion(buffer, 0, buffer.byteLength));
|
|
1422
|
+
}
|
|
1423
|
+
return decodeManifest(w.manifest, regions, resolved, true);
|
|
1424
|
+
}
|
|
1425
|
+
//# sourceMappingURL=from-wire.js.map
|