@vortex-rdf/vortex-rdf-store 0.5.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +94 -130
- package/entry/browser.js +3 -2
- package/entry/node.js +3 -2
- package/entry/types.d.ts +16 -0
- package/package.json +25 -15
- package/pkg/web/snippets/vortex-rdf-86539c79cb85e509/js-snippets/lazy-rdf.js +230 -0
- package/pkg/web/snippets/vortex-rdf-86539c79cb85e509/js-snippets/pack-quads.js +70 -0
- package/pkg/web/vortex_rdf.d.ts +243 -0
- package/pkg/web/vortex_rdf.js +960 -0
- package/pkg/web/vortex_rdf_bg.wasm +0 -0
- package/pkg/web/vortex_rdf_bg.wasm.d.ts +42 -0
- package/entry/browser.d.ts +0 -15
- package/entry/node.d.ts +0 -16
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Julian Rojas
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,29 +1,33 @@
|
|
|
1
1
|
# Vortex-RDF for JavaScript
|
|
2
2
|
[](https://www.npmjs.com/package/@vortex-rdf/vortex-rdf-store)
|
|
3
3
|
|
|
4
|
-
JavaScript bindings
|
|
4
|
+
JavaScript bindings for [Vortex-RDF](https://github.com/vortex-rdf/vortex-rdf), a columnar RDF store format built on Vortex: [`vortex-rdf-core`](https://crates.io/crates/vortex-rdf-core) compiled to WebAssembly, exposed through an RDF/JS-shaped API that builds stores from RDF text or quads, queries them with lazy zero-copy quads, mutates them in place, and round-trips them through `.vortex` bytes.
|
|
5
5
|
|
|
6
|
-
##
|
|
6
|
+
## Install
|
|
7
7
|
|
|
8
8
|
```bash
|
|
9
9
|
npm install @vortex-rdf/vortex-rdf-store
|
|
10
10
|
```
|
|
11
11
|
|
|
12
|
-
The package is ESM-only (no CommonJS `require`) and works identically in Node.js and browsers
|
|
13
|
-
|
|
14
|
-
```javascript
|
|
15
|
-
import { VortexRdfStore } from '@vortex-rdf/vortex-rdf-store';
|
|
16
|
-
|
|
17
|
-
const store = await VortexRdfStore.fromString(ttlData, 'turtle');
|
|
18
|
-
```
|
|
19
|
-
|
|
20
|
-
Under the hood, a single `wasm-pack --target web` build backs both: the Node entry point reads the `.wasm` file straight off disk (`node:fs`), and the browser entry point uses the standard `fetch`-based loading that bundlers and browsers already understand via `import.meta.url`. Both call the WASM module's async initialization for you, so there's no `init()` to await yourself.
|
|
12
|
+
The package is ESM-only (no CommonJS `require`) and works identically in Node.js and browsers: the same `import` resolves to a Node entry point (reads the `.wasm` file off disk with `node:fs`) or a browser one (`fetch` via `import.meta.url`), and both await the wasm module's initialization.
|
|
21
13
|
|
|
22
14
|
> **Bundler note:** both entry points use top-level `await`. Vite and Rollup
|
|
23
15
|
> support this by default; webpack 5 needs
|
|
24
16
|
> `experiments: { topLevelAwait: true }` enabled in its config.
|
|
25
17
|
|
|
26
|
-
##
|
|
18
|
+
## Quick start
|
|
19
|
+
|
|
20
|
+
```javascript
|
|
21
|
+
import { VortexRdfStore, serializeRdf } from '@vortex-rdf/vortex-rdf-store';
|
|
22
|
+
|
|
23
|
+
const bytes = await serializeRdf('<http://ex/s> <http://ex/p> "o" .', 'turtle');
|
|
24
|
+
const store = await VortexRdfStore.fromBytes(bytes);
|
|
25
|
+
for await (const quad of store.match(null, 'http://ex/p', null, null)) {
|
|
26
|
+
console.log(quad.subject.value, quad.object.value);
|
|
27
|
+
}
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Reading quads
|
|
27
31
|
|
|
28
32
|
### Loading data
|
|
29
33
|
|
|
@@ -32,30 +36,30 @@ import { VortexRdfStore } from '@vortex-rdf/vortex-rdf-store';
|
|
|
32
36
|
import { Readable } from 'node:stream';
|
|
33
37
|
import type { Quad, Stream } from '@rdfjs/types';
|
|
34
38
|
|
|
35
|
-
// From
|
|
36
|
-
// Supported formats: `ntriples`, `nquads`, `turtle`, `trig`, `n3`, `rdfxml`, `jsonld`
|
|
39
|
+
// From RDF text. Formats: `ntriples`, `nquads`, `turtle`, `trig`, `n3`, `rdfxml`, `jsonld`
|
|
37
40
|
// (plus the short aliases `nt`, `nq`, `ttl`, `rdf`, `xml`).
|
|
38
41
|
const store = await VortexRdfStore.fromString(ttlData, 'turtle');
|
|
39
42
|
|
|
40
|
-
// From RDF
|
|
43
|
+
// From RDF/JS quads: an array ...
|
|
41
44
|
const quads: Quad[] = [...];
|
|
42
45
|
const store = await VortexRdfStore.fromQuads(quads);
|
|
43
46
|
|
|
44
|
-
//
|
|
47
|
+
// ... or any RDF/JS Stream<Quad>
|
|
45
48
|
const quadStream: Stream<Quad> = Readable.from(quads, { objectMode: true });
|
|
46
49
|
const store = await VortexRdfStore.fromQuads(quadStream);
|
|
47
50
|
|
|
48
|
-
// From Vortex
|
|
51
|
+
// From Vortex bytes (e.g., fetched from a server)
|
|
49
52
|
const store = await VortexRdfStore.fromBytes(vortexBytes);
|
|
50
53
|
|
|
51
|
-
// Or
|
|
54
|
+
// Or start empty
|
|
52
55
|
const store = VortexRdfStore.empty();
|
|
53
56
|
```
|
|
54
57
|
|
|
58
|
+
`layout()` and `indexes()` report what a store was built with, as the same kebab-case names `BuildOptions` takes (`store.layout()` is `'dictionary'`; `store.indexes()` is an array such as `['secondary-by-reference']`).
|
|
59
|
+
|
|
55
60
|
### Querying
|
|
56
61
|
|
|
57
|
-
`match` implements the RDF/JS [`Source.match`](https://rdf.js.org/stream-spec/#source-interface) contract. It takes a `(subject, predicate, object, graph)` pattern —
|
|
58
|
-
`null`/`undefined` for a variable position — and returns **synchronously** an RDF/JS `Stream<Quad>`:
|
|
62
|
+
`match` implements the RDF/JS [`Source.match`](https://rdf.js.org/stream-spec/#source-interface) contract. It takes a `(subject, predicate, object, graph)` pattern — `null`/`undefined` or an RDF/JS `Variable` for a wildcard position, a bare string for a NamedNode IRI — and returns **synchronously** an RDF/JS `Stream<Quad>`:
|
|
59
63
|
|
|
60
64
|
```javascript
|
|
61
65
|
store.match(null, myPredicate, null, null)
|
|
@@ -63,7 +67,7 @@ store.match(null, myPredicate, null, null)
|
|
|
63
67
|
.on('end', () => console.log('done'));
|
|
64
68
|
```
|
|
65
69
|
|
|
66
|
-
The returned stream also implements `Symbol.asyncIterator
|
|
70
|
+
The returned stream also implements `Symbol.asyncIterator`, so it can be consumed with `for await` (in TypeScript, cast to `AsyncIterable<Quad>` since the declared type is `Stream<Quad>`):
|
|
67
71
|
|
|
68
72
|
```javascript
|
|
69
73
|
for await (const quad of store.match(null, myPredicate, null, null)) {
|
|
@@ -71,26 +75,28 @@ for await (const quad of store.match(null, myPredicate, null, null)) {
|
|
|
71
75
|
}
|
|
72
76
|
```
|
|
73
77
|
|
|
74
|
-
|
|
78
|
+
`getQuads` is the array-returning counterpart (synchronous — no read path performs I/O, so there is nothing to await), and `countQuads` answers from the match's row selection without materializing a quad:
|
|
75
79
|
|
|
76
80
|
```javascript
|
|
77
|
-
const quads =
|
|
78
|
-
|
|
81
|
+
const quads = store.getQuads(null, myPredicate, null, null);
|
|
82
|
+
const n = store.countQuads(null, myPredicate, null, null);
|
|
79
83
|
```
|
|
80
84
|
|
|
81
|
-
**Quads are lazy and zero-copy.** `match`/`getQuads`
|
|
85
|
+
**Quads are lazy and zero-copy.** `match`/`getQuads` hand back quads backed by the store's columnar data. A term's string is decoded only when you read `.value`/`.termType`, and then interned, so iterating, counting, filtering, and `.equals` never materialize strings you don't use. Under the default `dictionary` layout, `.equals` between terms of the same store is an integer code compare (no decoding at all).
|
|
82
86
|
|
|
83
|
-
The quads implement the RDF/JS `Quad`/`Term` interface (`.subject.value`, `.equals`, …) and interoperate with foreign RDF/JS terms via `.equals` in both directions.
|
|
84
|
-
them or rely on enumerating own properties.)
|
|
87
|
+
The quads implement the RDF/JS `Quad`/`Term` interface (`.subject.value`, `.equals`, …) and interoperate with foreign RDF/JS terms via `.equals` in both directions. They are lazy views into the producing store, so — unlike a plain data object — don't `structuredClone` them or rely on enumerating own properties.
|
|
85
88
|
|
|
86
|
-
Test membership of a single quad with `has` (an exact four-component lookup):
|
|
89
|
+
Test membership of a single quad with `has` (an exact four-component lookup) and read the quad count with `size`; both return promises:
|
|
87
90
|
|
|
88
91
|
```javascript
|
|
89
92
|
if (await store.has(myQuad)) {
|
|
90
93
|
console.log('present');
|
|
91
94
|
}
|
|
95
|
+
console.log(await store.size());
|
|
92
96
|
```
|
|
93
97
|
|
|
98
|
+
An invalid pattern term or a malformed quad object throws (or rejects) with an `Error`.
|
|
99
|
+
|
|
94
100
|
### Mutation
|
|
95
101
|
|
|
96
102
|
```javascript
|
|
@@ -99,157 +105,115 @@ await store.addQuads([quadA, quadB]);
|
|
|
99
105
|
await store.deleteQuad(existingQuad);
|
|
100
106
|
```
|
|
101
107
|
|
|
102
|
-
Mutations follow RDF/JS dataset semantics: adding a quad already present is a no-op, and deleting never rewrites the columnar data (rows are tombstoned).
|
|
108
|
+
Mutations follow RDF/JS dataset semantics: adding a quad already present is a no-op, and deleting never rewrites the columnar data (rows are tombstoned).
|
|
103
109
|
|
|
104
|
-
Added quads accumulate in
|
|
110
|
+
Added quads accumulate in an in-memory tail beside the immutable base, so the store's indexes keep working across edits. When the tail reaches a tenth of the base's rows (never fewer than 4,096) or 100,000 rows, the store compacts itself back into one sorted, indexed array.
|
|
105
111
|
|
|
106
112
|
Prefer `addQuads` over a loop of `addQuad` calls; for bulk loading, build once with `fromString`/`fromQuads`.
|
|
107
113
|
|
|
108
|
-
###
|
|
114
|
+
### RDF text out
|
|
109
115
|
|
|
110
116
|
```javascript
|
|
111
|
-
|
|
112
|
-
const turtle = await store.toRdf('turtle');
|
|
113
|
-
|
|
114
|
-
// To Vortex binary data; read back with VortexRdfStore.fromBytes
|
|
115
|
-
const bytes = await store.toBytes();
|
|
117
|
+
const turtle = await store.toRdf('turtle'); // any supported format name
|
|
116
118
|
```
|
|
117
119
|
|
|
118
|
-
|
|
120
|
+
## Term codes (low-level)
|
|
119
121
|
|
|
120
|
-
|
|
122
|
+
Under the default `dictionary` layout, terms are stored as `u32` codes into a sorted term dictionary. `termDict()` is the one door to code↔term translation: it returns an immutable `TermDict` handle, or `undefined` when the store's rows aren't code-addressable (a non-dictionary layout, or added quads pending in the in-memory tail):
|
|
121
123
|
|
|
122
124
|
```javascript
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
}
|
|
125
|
+
const dict = store.termDict(); // TermDict | undefined
|
|
126
|
+
if (dict) {
|
|
127
|
+
const code = dict.encode('<http://schema.org/name>'); // number | undefined
|
|
128
|
+
console.log(dict.decode(code)); // '<http://schema.org/name>'
|
|
129
|
+
}
|
|
128
130
|
```
|
|
129
131
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
| Value | Build cost | Effect on queries |
|
|
133
|
-
| --- | --- | --- |
|
|
134
|
-
| `'Unsorted'` (default) | Cheapest; natural insertion order | Every `match` is a full column scan |
|
|
135
|
-
| `'Sorted'` | Global in-memory sort by subject → predicate → object → graph | Subject lookups use a binary search |
|
|
132
|
+
`decode`/`encode` speak N-Triples term strings — `<iri>`, `_:blank`, `"lit"@lang`, `"lit"^^<dt>`, and `''` for the default graph. The handle is a snapshot: it keeps decoding correctly after the store is mutated, because it retains the dictionary its codes address. It is a wasm-side handle — call `free()` when done (also wired to `Symbol.dispose`, so `using` disposes it automatically).
|
|
136
133
|
|
|
137
|
-
|
|
134
|
+
`matchCodes` is its pattern-matching counterpart: it resolves a pattern to the matched rows' raw term codes — four columnar `Uint32Array`s `{ s, p, o, g }` plus a `length` — without materializing any term strings, and returns `null` under the same conditions `termDict()` returns `undefined`:
|
|
138
135
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
136
|
+
```javascript
|
|
137
|
+
const cols = store.matchCodes(null, myPredicate, null, null);
|
|
138
|
+
if (cols) {
|
|
139
|
+
console.log(cols.length, dict.decode(cols.o[0]));
|
|
140
|
+
}
|
|
141
|
+
```
|
|
144
142
|
|
|
145
|
-
|
|
143
|
+
`match`/`getQuads` are the supported way to read quads; `matchCodes` is the low-level path for callers that join, count and de-duplicate in code space and decode each distinct term once.
|
|
146
144
|
|
|
147
|
-
|
|
148
|
-
- `'SecondaryByCopy'` embeds two complete extra copies of the quad columns — one sorted by `(p, o, s, g)`, one by `(o, s, p, g)` — giving predicate- and object-bound patterns (including combined predicate+object lookups, resolved in one prefix search) the same sorted access path subjects have, at roughly 2× the storage. Both cost extra space and are only effective alongside `builder: 'Sorted'`.
|
|
145
|
+
## Build options
|
|
149
146
|
|
|
150
|
-
|
|
147
|
+
`fromString`, `fromQuads` and `serializeRdf` accept an optional `BuildOptions` object; every field is optional. Quads are always sorted by subject → predicate → object → graph while the columnar array is built — that global order is what gives subject lookups their binary search and what every secondary index routes against. The wasm build sorts in memory (WebAssembly has no filesystem for an out-of-core builder) and takes no builder option.
|
|
151
148
|
|
|
152
149
|
```javascript
|
|
153
|
-
{
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
};
|
|
150
|
+
const store = await VortexRdfStore.fromString(data, 'nquads', {
|
|
151
|
+
layout: 'dictionary', // 'dictionary' (default) | 'default' | 'typed-object'
|
|
152
|
+
indexes: ['secondary-by-reference'], // default: []
|
|
153
|
+
});
|
|
158
154
|
```
|
|
159
155
|
|
|
156
|
+
**`layout`** — how terms are encoded into columns. `'dictionary'` is the default in every vortex-rdf frontend (JS, Python and the CLI):
|
|
160
157
|
|
|
161
|
-
|
|
158
|
+
| Value | Notes |
|
|
159
|
+
| --- | --- |
|
|
160
|
+
| `'dictionary'` (default) | Terms replaced by codes into a sorted term dictionary. Most compact and fastest to query; backs the integer `.equals` fast path on lazy quads; added quads live in an in-memory string tail until serialized or compacted |
|
|
161
|
+
| `'default'` | All four terms as N-Triples strings |
|
|
162
|
+
| `'typed-object'` | Object split into kind/value/datatype/language columns |
|
|
162
163
|
|
|
163
|
-
|
|
164
|
+
**`indexes`** — secondary access paths, each costing extra space:
|
|
164
165
|
|
|
165
|
-
|
|
166
|
+
| Value | Notes |
|
|
167
|
+
| --- | --- |
|
|
168
|
+
| `'secondary-by-reference'` | Sorted predicate/object columns plus row-id back-references, so predicate-only and object-only patterns use a binary search instead of a full scan |
|
|
169
|
+
| `'secondary-by-copy'` | Two complete extra copies of the quad columns — one sorted by `(p, o, s, g)`, one by `(o, s, p, g)` — giving predicate- and object-bound patterns (including predicate+object prefix lookups) the same sorted access path subjects have |
|
|
170
|
+
|
|
171
|
+
## Bytes & files
|
|
166
172
|
|
|
167
173
|
```javascript
|
|
168
|
-
import {
|
|
174
|
+
import { VortexRdfStore, serializeRdf, deserializeRdf } from '@vortex-rdf/vortex-rdf-store';
|
|
169
175
|
|
|
170
|
-
|
|
171
|
-
const
|
|
176
|
+
// Store <-> native-container bytes (the `.vortex` file format)
|
|
177
|
+
const bytes = await store.toBytes();
|
|
178
|
+
const again = await VortexRdfStore.fromBytes(bytes);
|
|
172
179
|
|
|
173
|
-
//
|
|
174
|
-
const
|
|
175
|
-
const
|
|
180
|
+
// One-shot conversions, without holding a store
|
|
181
|
+
const vortex = await serializeRdf(turtleText, 'turtle', { layout: 'dictionary' });
|
|
182
|
+
const nquads = await deserializeRdf(vortex, 'nquads');
|
|
176
183
|
```
|
|
177
184
|
|
|
178
|
-
|
|
185
|
+
`toBytes` writes the same exchange format the CLI and the Python bindings read, so a buffer can be written to disk as a `.vortex` file or handed across bindings.
|
|
179
186
|
|
|
180
|
-
|
|
187
|
+
A `VortexRdfStore` is a wasm-side handle: call `free()` when you are done with it, or declare it with `using` (`free` is wired to `Symbol.dispose`); an unfreed store is reclaimed only when its JS wrapper is garbage-collected.
|
|
188
|
+
|
|
189
|
+
## TypeScript
|
|
190
|
+
|
|
191
|
+
The package ships a hand-written `src/api.d.ts` typed against `@rdfjs/types`, embedded into the wasm-pack output; it is the public surface, checked by `npm run typecheck`.
|
|
181
192
|
|
|
182
193
|
```typescript
|
|
183
194
|
import { VortexRdfStore, type BuildOptions } from '@vortex-rdf/vortex-rdf-store';
|
|
184
195
|
import { DataFactory } from 'rdf-data-factory';
|
|
185
196
|
|
|
186
197
|
const df = new DataFactory();
|
|
187
|
-
|
|
188
|
-
const options: BuildOptions = { builder: 'Sorted', layout: 'Dictionary' };
|
|
198
|
+
const options: BuildOptions = { layout: 'dictionary' };
|
|
189
199
|
const store = await VortexRdfStore.fromString(data, 'nquads', options);
|
|
190
200
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
const quads = await store.getQuads(null, df.namedNode('http://schema.org/name'), null, null);
|
|
194
|
-
for (const quad of quads) {
|
|
201
|
+
for (const quad of store.getQuads(null, df.namedNode('http://schema.org/name'), null, null)) {
|
|
195
202
|
console.log(quad.subject.value);
|
|
196
203
|
}
|
|
197
204
|
```
|
|
198
205
|
|
|
199
|
-
##
|
|
206
|
+
## Development
|
|
200
207
|
|
|
201
|
-
|
|
208
|
+
The package is built with [wasm-pack](https://rustwasm.github.io/wasm-pack/) targeting `web`; one build (`pkg/web/`) backs both environments. `entry/node.js` and `entry/browser.js` are small hand-written wrappers that differ only in how they supply the `.wasm` bytes to the generated `init()`, and `package.json`'s `exports` map picks one per environment.
|
|
202
209
|
|
|
203
210
|
```bash
|
|
204
|
-
|
|
205
|
-
npm
|
|
206
|
-
|
|
207
|
-
# Run the test suite (requires a build first)
|
|
208
|
-
npm test
|
|
211
|
+
npm run build # wasm-pack build into pkg/web/ (build:fast skips wasm-opt)
|
|
212
|
+
npm test # vitest suite (requires a build)
|
|
213
|
+
npm run typecheck # tsc over the tests and benchmarks, against the published API
|
|
209
214
|
```
|
|
210
215
|
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
There's no separate Node-targeted wasm build to maintain.
|
|
214
|
-
|
|
215
|
-
Types are checked separately from the build, since `wasm-pack` doesn't run `tsc`:
|
|
216
|
-
|
|
217
|
-
```bash
|
|
218
|
-
npm run typecheck # tests and benchmarks, both against the published API
|
|
219
|
-
```
|
|
220
|
-
|
|
221
|
-
## Benchmarks
|
|
222
|
-
|
|
223
|
-
Three surfaces, kept separate because they answer different questions and are trustworthy in different ways. All require `npm run build` first.
|
|
224
|
-
|
|
225
|
-
| Command | Question | Output |
|
|
226
|
-
| --- | --- | --- |
|
|
227
|
-
| `npm run bench` | How do we compare to other JS RDF stores? | `bench/results.json`, rendered to the dashboard |
|
|
228
|
-
| `npm run bench:codspeed` | Did this PR regress anything? | uploaded to CodSpeed |
|
|
229
|
-
| `npm run bench:dict-memory` | Where is wasm memory actually going? | printed; `bench/dict-memory.json` |
|
|
230
|
-
|
|
231
|
-
**`npm run bench`** runs each store — the Vortex build variants, [rdf-stores.js](https://github.com/rubensworks/rdf-stores.js), and [oxigraph](https://github.com/oxigraph/oxigraph) — through the same query, mutation, and serialization workload, one adapter per child process so peak RSS is attributable and no store's garbage taints another's timings. It is wall-clock, so it is only as stable as the machine it runs on; it is deliberately **not** uploaded to CodSpeed. `scripts/render_bench_dashboard.py` turns its `results.json` plus a `cargo bench --bench benchmark` run into `public/index.html`.
|
|
232
|
-
|
|
233
|
-
The dataset is generated by `genDataset` in [bench/shared.ts](bench/shared.ts), with term cardinality as an explicit knob — real RDF has terms scaling with rows, and a generator that draws millions of quads from a handful of IRIs makes every store's term handling invisible. Tunable by env var:
|
|
234
|
-
|
|
235
|
-
| Var | Default | Meaning |
|
|
236
|
-
| --- | --- | --- |
|
|
237
|
-
| `BENCH_DIM` | 128 | triples dataset is `D³` rows |
|
|
238
|
-
| `BENCH_DIM_QUADS` | 32 | quads dataset is `Dq⁴` rows |
|
|
239
|
-
| `BENCH_SUBJ_RATIO` | 0.1 | distinct subjects / rows — the reciprocal of triples per subject |
|
|
240
|
-
| `BENCH_OBJ_RATIO` | 0.5 | distinct objects / rows |
|
|
241
|
-
| `BENCH_PREDICATES` | 32 | distinct predicates (a closed vocabulary) |
|
|
242
|
-
| `BENCH_GRAPHS` | 1 | distinct named graphs; 1 means default graph only |
|
|
243
|
-
| `BENCH_LITERAL_FRAC` | 0.4 | fraction of objects that are literals |
|
|
244
|
-
| `MUT_BATCH` | 10000 | quads per add/delete batch |
|
|
245
|
-
|
|
246
|
-
The two ratios decide how much of the dataset is *terms*: the dictionary holds `subjectRatio + objectRatio` of them per quad. At the defaults that is 2,097,152 triples over ~1.26M distinct terms, ten triples describing each subject, and probes spanning five orders of magnitude of selectivity (1 row for a fully-bound pattern, ~10 for a subject, ~3% for a predicate).
|
|
247
|
-
|
|
248
|
-
`BENCH_SUBJ_RATIO=1.0` gives every row its own subject, which puts more distinct terms in the dictionary than there are rows. That is a useful dictionary-stress configuration — it is what `bench:dict-memory` asks for — but a poor default for the comparison: it makes five of the seven probes match two rows or fewer, so the timings become almost entirely query setup with the decode path barely exercised.
|
|
249
|
-
|
|
250
|
-
Each adapter process peaks at 3–6 GB — check available memory before running, because a swapping run produces numbers that look like regressions.
|
|
251
|
-
|
|
252
|
-
**`npm run bench:dict-memory`** is the only instrument that can attribute wasm memory; `peakRssMb` in the comparative run is the right cross-library number but cannot say *what* holds the memory. Wasm linear memory never shrinks, so it works by differentials — several stores held live at once, memory read after each, slope against store count — and every config runs in its own process. It defaults to a single config as a regression check against the figures recorded when FSST landed; passing several `DICT_MEM_RATIOS` sweeps term cardinality instead and fits the per-term cost. The header of [bench/dict-memory.bench.ts](bench/dict-memory.bench.ts) documents the method and the invocations.
|
|
216
|
+
Benchmarks live in [bench/](bench/README.md).
|
|
253
217
|
|
|
254
218
|
## License
|
|
255
219
|
|
package/entry/browser.js
CHANGED
package/entry/node.js
CHANGED
package/entry/types.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// Types shared by the Node and browser entry points: both expose the same
|
|
2
|
+
// public surface as the generated wasm bindings, minus the low-level
|
|
3
|
+
// `init`/`initSync` (each entry calls init for the caller). Kept in sync with
|
|
4
|
+
// the re-exports in entry/node.js and entry/browser.js.
|
|
5
|
+
export {
|
|
6
|
+
TermDict,
|
|
7
|
+
VortexRdfStore,
|
|
8
|
+
serializeRdf,
|
|
9
|
+
deserializeRdf,
|
|
10
|
+
} from '../pkg/web/vortex_rdf.js';
|
|
11
|
+
export type {
|
|
12
|
+
BuildOptions,
|
|
13
|
+
LayoutStrategy,
|
|
14
|
+
IndexType,
|
|
15
|
+
RdfFormatName,
|
|
16
|
+
} from '../pkg/web/vortex_rdf.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vortex-rdf/vortex-rdf-store",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "High-performance, zero-copy columnar RDF serialization format",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
@@ -12,20 +12,27 @@
|
|
|
12
12
|
"access": "public"
|
|
13
13
|
},
|
|
14
14
|
"main": "./entry/browser.js",
|
|
15
|
-
"types": "./
|
|
15
|
+
"types": "./entry/types.d.ts",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": {
|
|
18
|
-
"
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
"node": {
|
|
19
|
+
"types": "./entry/types.d.ts",
|
|
20
|
+
"default": "./entry/node.js"
|
|
21
|
+
},
|
|
22
|
+
"browser": {
|
|
23
|
+
"types": "./entry/types.d.ts",
|
|
24
|
+
"default": "./entry/browser.js"
|
|
25
|
+
},
|
|
26
|
+
"default": {
|
|
27
|
+
"types": "./entry/types.d.ts",
|
|
28
|
+
"default": "./entry/browser.js"
|
|
29
|
+
}
|
|
22
30
|
}
|
|
23
31
|
},
|
|
24
32
|
"files": [
|
|
25
33
|
"entry/node.js",
|
|
26
|
-
"entry/node.d.ts",
|
|
27
34
|
"entry/browser.js",
|
|
28
|
-
"entry/
|
|
35
|
+
"entry/types.d.ts",
|
|
29
36
|
"pkg/web/vortex_rdf.js",
|
|
30
37
|
"pkg/web/vortex_rdf.d.ts",
|
|
31
38
|
"pkg/web/vortex_rdf_bg.wasm",
|
|
@@ -35,15 +42,18 @@
|
|
|
35
42
|
"scripts": {
|
|
36
43
|
"build:wasm": "wasm-pack build --target web --out-dir pkg/web --no-pack",
|
|
37
44
|
"build:wasm:fast": "wasm-pack build --target web --out-dir pkg/web --no-pack --no-opt",
|
|
38
|
-
"build:
|
|
39
|
-
"build": "
|
|
40
|
-
"build:
|
|
45
|
+
"build:hdt-wasm": "RUSTFLAGS='-C target-feature=+simd128' wasm-pack build bench/hdt-wasm --target web --out-dir ../hdt-pkg --no-pack && npm run build:hdt-clean-generated",
|
|
46
|
+
"build:hdt-clean-generated": "node -e \"require('fs').rmSync('bench/hdt-pkg/.gitignore',{force:true})\"",
|
|
47
|
+
"build:reset": "node -e \"require('fs').rmSync('pkg/web',{recursive:true,force:true})\"",
|
|
48
|
+
"build:clean-generated": "node -e \"const fs=require('fs');for (const f of ['pkg/web/package.json','pkg/web/.gitignore']) fs.rmSync(f,{force:true})\"",
|
|
49
|
+
"build": "npm run build:reset && npm run build:wasm && npm run build:clean-generated",
|
|
50
|
+
"build:fast": "npm run build:reset && npm run build:wasm:fast && npm run build:clean-generated",
|
|
41
51
|
"prepublishOnly": "npm run build",
|
|
42
52
|
"test": "vitest run",
|
|
43
|
-
"typecheck": "tsc -p tsconfig.json
|
|
53
|
+
"typecheck": "tsc -p tsconfig.json && tsc -p bench/tsconfig.json",
|
|
44
54
|
"bench": "tsx --expose-gc --max-old-space-size=8192 bench/compare.bench.ts",
|
|
45
55
|
"bench:build": "npm run build && npm run bench",
|
|
46
|
-
"bench:codspeed": "tsx --expose-gc bench/codspeed.bench.ts",
|
|
56
|
+
"bench:codspeed": "node --import tsx --expose-gc --no-liftoff bench/codspeed.bench.ts",
|
|
47
57
|
"bench:dict-memory": "tsx --expose-gc bench/dict-memory.bench.ts"
|
|
48
58
|
},
|
|
49
59
|
"keywords": [
|
|
@@ -58,12 +68,12 @@
|
|
|
58
68
|
"license": "MIT",
|
|
59
69
|
"devDependencies": {
|
|
60
70
|
"@codspeed/tinybench-plugin": "^5.7.1",
|
|
61
|
-
"@types/node": "^26.1.
|
|
71
|
+
"@types/node": "^26.1.2",
|
|
62
72
|
"oxigraph": "^0.5.9",
|
|
63
73
|
"rdf-data-factory": "^2.0.2",
|
|
64
74
|
"rdf-stores": "^2.2.0",
|
|
65
75
|
"tinybench": "^6.1.2",
|
|
66
|
-
"tsx": "^4.23.
|
|
76
|
+
"tsx": "^4.23.4",
|
|
67
77
|
"typescript": "^7.0.2",
|
|
68
78
|
"vitest": "^4.1.10",
|
|
69
79
|
"wasm-pack": "^0.15.0"
|