datalevin-node 0.10.18 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -46,6 +46,314 @@ try {
46
46
  }
47
47
  ```
48
48
 
49
+ ## Data Style
50
+
51
+ Use ordinary JavaScript data as the canonical style:
52
+
53
+ - schemas are objects keyed by colon-prefixed attribute strings
54
+ - transaction entity maps are objects, and transaction forms are arrays
55
+ - query and pull forms may be EDN strings or JavaScript arrays
56
+ - colon-prefixed strings are converted to keywords in schema/query/form
57
+ positions
58
+ - use `keyword()` when the stored value itself must be a keyword
59
+
60
+ ```js
61
+ import { keyword, readEdn, schemaAttr, txAdd, txEntity, writeEdn } from "datalevin-node";
62
+
63
+ const schema = {
64
+ ":name": schemaAttr({ valueType: ":db.type/string", unique: ":db.unique/identity" }),
65
+ ":status": schemaAttr({ valueType: ":db.type/keyword" })
66
+ };
67
+
68
+ const tx = [
69
+ txEntity(-1, { ":name": "Ada", ":status": await keyword(":active") }),
70
+ txAdd(-1, ":nickname", "A")
71
+ ];
72
+
73
+ const form = await readEdn("[:find ?e :where [?e :name _]]");
74
+ const text = await writeEdn([":find", "?e", ":where", ["?e", ":name", "_"]]);
75
+ ```
76
+
77
+ ## Lazy Entity Example
78
+
79
+ `conn.entity()` returns a lazy entity wrapper. Use `get()` for individual
80
+ attributes, and call `touch()` only when you want a fully materialized object.
81
+ `entityMap()` is available for the old eager touched-object shape.
82
+
83
+ ```js
84
+ const entity = await conn.entity([":name", "Ada"]);
85
+
86
+ console.log(await entity.id());
87
+ console.log(await entity.get(":name"));
88
+ console.log(await entity.get(":db/id"));
89
+
90
+ const touched = await entity.touch();
91
+ const eager = await conn.entityMap(1);
92
+ ```
93
+
94
+ ## Search, Vector, and Idoc Builders
95
+
96
+ Use helper builders for search/vector/idoc schema and option maps instead of
97
+ hand-writing every namespaced key:
98
+
99
+ ```js
100
+ import {
101
+ embeddingAttr,
102
+ embeddingOptions,
103
+ fulltextAttr,
104
+ idocAttr,
105
+ searchDomain,
106
+ searchOptions,
107
+ vectorAttr,
108
+ vectorOptions
109
+ } from "datalevin-node";
110
+
111
+ const schema = {
112
+ ":doc/text": fulltextAttr({ domains: ["docs"], autoDomain: true }),
113
+ ":doc/body": embeddingAttr({ domains: ["docs"], autoDomain: true }),
114
+ ":doc/vec": vectorAttr({ domains: ["docs"] }),
115
+ ":doc/json": idocAttr({ format: "json", domain: "profiles" })
116
+ };
117
+
118
+ const opts = {
119
+ ":search-domains": { docs: searchDomain({ indexPosition: true }) },
120
+ ":search-opts": searchOptions({ top: 5, display: "refs+scores" }),
121
+ ":vector-opts": vectorOptions({ dimensions: 384, metricType: "cosine" }),
122
+ ":embedding-opts": embeddingOptions({
123
+ provider: "openai-compatible",
124
+ model: "text-embedding-3-small",
125
+ baseUrl: "https://api.openai.com/v1",
126
+ apiKeyEnv: "OPENAI_API_KEY",
127
+ requestDimensions: 1536,
128
+ metricType: "cosine"
129
+ })
130
+ };
131
+ ```
132
+
133
+ ## Standalone Vector Index Example
134
+
135
+ Use `newVectorIndex()` when you want a KV-backed vector index without a Datalog
136
+ schema attribute.
137
+
138
+ ```js
139
+ import { newVectorIndex, openKv, vectorOptions } from "datalevin-node";
140
+
141
+ const kv = await openKv("/tmp/dtlv-js-vec");
142
+
143
+ try {
144
+ const index = await newVectorIndex(kv, vectorOptions({ dimensions: 2 }));
145
+ await index.addVec("doc-1", [1.0, 0.0]);
146
+ await index.addVec("doc-2", [0.0, 1.0]);
147
+
148
+ console.log(await index.searchVec([1.0, 0.0], { ":top": 1 }));
149
+ await index.forceCheckpoint();
150
+ console.log(await index.info());
151
+ await index.close();
152
+ } finally {
153
+ await kv.close();
154
+ }
155
+ ```
156
+
157
+ ## Local Llama Example
158
+
159
+ Use `newLlamaEmbedder()` and `newLlamaGenerator()` with local GGUF models when
160
+ you want direct llama.cpp handles outside Datalog embedding/search setup.
161
+
162
+ ```js
163
+ import { newLlamaEmbedder, newLlamaGenerator } from "datalevin-node";
164
+
165
+ const embedder = await newLlamaEmbedder("/models/embed.gguf");
166
+ try {
167
+ const vector = await embedder.embed("Datalevin stores facts.");
168
+ console.log(await embedder.dimensions(), vector.length);
169
+ console.log(await embedder.tokenCount("Datalevin stores facts."));
170
+ } finally {
171
+ await embedder.close();
172
+ }
173
+
174
+ const generator = await newLlamaGenerator("/models/generate.gguf");
175
+ try {
176
+ console.log(await generator.generate("Write one database tagline:", 32));
177
+ } finally {
178
+ await generator.close();
179
+ }
180
+ ```
181
+
182
+ ## Async Transaction Example
183
+
184
+ `transact()` uses Datalevin's async transaction batching path and waits until
185
+ the transaction commits before returning the report.
186
+
187
+ Use `transactAsync()` for ingestion and application-server workloads that
188
+ benefit from Datalevin's async transaction batching. It returns a normal
189
+ JavaScript `Promise`.
190
+
191
+ ```js
192
+ const report = await conn.transactAsync([
193
+ { ":db/id": -1, ":name": "Cara" }
194
+ ]);
195
+ ```
196
+
197
+ ## UDF Example
198
+
199
+ Use `createUdfRegistry()` and `udfDescriptor()` for runtime UDFs. Pass the
200
+ registry in connection runtime options, then call the descriptor from query with
201
+ Datalevin's `udf` function. Fulltext analyzers use the same route with
202
+ `registry.analyzerUdf()` or `registry.queryAnalyzerUdf()` and descriptors in
203
+ `searchDomain({ analyzer, queryAnalyzer })`. Analyzer functions return
204
+ `[term, position, offset]` triples.
205
+
206
+ ```js
207
+ import { connect, createUdfRegistry, udfDescriptor } from "datalevin-node";
208
+
209
+ const registry = await createUdfRegistry();
210
+ await registry.queryUdf(":math/inc", (value) => Number(value) + 1);
211
+
212
+ const descriptor = udfDescriptor(":math/inc");
213
+ const conn = await connect("/tmp/dtlv-js-udf", {
214
+ opts: { ":runtime-opts": { ":udf-registry": registry } }
215
+ });
216
+
217
+ try {
218
+ const value = await conn.query(
219
+ "[:find ?v . :in $ ?desc ?n :where [(udf ?desc ?n) ?v]]",
220
+ descriptor,
221
+ 41
222
+ );
223
+ } finally {
224
+ await conn.close();
225
+ }
226
+ ```
227
+
228
+ ## Fulltext Analyzer UDF Example
229
+
230
+ Use analyzer UDFs when a Datalog fulltext domain needs host-language tokenizing.
231
+ The document analyzer runs while transactions and re-indexing update the
232
+ fulltext index; the query analyzer runs during `fulltext` query evaluation.
233
+
234
+ ```js
235
+ import {
236
+ connect,
237
+ createUdfRegistry,
238
+ schemaAttr,
239
+ searchDomain,
240
+ udfDescriptor
241
+ } from "datalevin-node";
242
+
243
+ const registry = await createUdfRegistry();
244
+ const analyzer = udfDescriptor(":text/hashtags", { kind: ":analyzer" });
245
+ const queryAnalyzer = udfDescriptor(":text/plain-query", { kind: ":query-analyzer" });
246
+
247
+ await registry.analyzerUdf(":text/hashtags", (text) => {
248
+ const tokens = [];
249
+ const pattern = /#\w+/g;
250
+ const source = String(text);
251
+ let match;
252
+ while ((match = pattern.exec(source)) !== null) {
253
+ tokens.push([match[0].slice(1), tokens.length, match.index]);
254
+ }
255
+ return tokens;
256
+ });
257
+
258
+ await registry.queryAnalyzerUdf(":text/plain-query", (text) => (
259
+ String(text).trim().split(/\s+/).filter(Boolean)
260
+ .map((token, position) => [token, position, position])
261
+ ));
262
+
263
+ const conn = await connect("/tmp/dtlv-js-fulltext-udf", {
264
+ schema: {
265
+ ":text": schemaAttr({
266
+ valueType: ":db.type/string",
267
+ fulltext: true,
268
+ fulltextAutoDomain: true
269
+ })
270
+ },
271
+ opts: {
272
+ ":runtime-opts": { ":udf-registry": registry },
273
+ ":search-domains": {
274
+ text: searchDomain({
275
+ indexPosition: true,
276
+ analyzer,
277
+ queryAnalyzer
278
+ })
279
+ }
280
+ }
281
+ });
282
+
283
+ try {
284
+ await conn.transact([
285
+ { ":db/id": 1, ":text": "alpha #needle" },
286
+ { ":db/id": 2, ":text": "needle without hash" }
287
+ ]);
288
+
289
+ console.log(await conn.query(
290
+ "[:find [?e ...] :in $ ?q :where [(fulltext $ :text ?q) [[?e ?a ?v]]]]",
291
+ "needle"
292
+ ));
293
+ } finally {
294
+ await conn.close();
295
+ }
296
+ ```
297
+
298
+ ## Datalog-Backed KV Example
299
+
300
+ Use `datalogKv()` when you need ordinary KV tables in the same store as a
301
+ Datalog connection. The returned KV handle is borrowed from the connection; do
302
+ not close it separately.
303
+
304
+ ```js
305
+ import { datalogKv } from "datalevin-node";
306
+
307
+ const kv = await datalogKv(conn);
308
+ await kv.openDbi("app-state");
309
+ await kv.transact([[":put", "k", "v"]], {
310
+ dbiName: "app-state",
311
+ kType: ":string",
312
+ vType: ":string"
313
+ });
314
+ ```
315
+
316
+ ## Datom Inspection Example
317
+
318
+ Connection objects expose index-level reads for debugging, teaching, and
319
+ migration tooling. Datom reads return objects with `:e`, `:a`, `:v`, `:tx`,
320
+ and `:added` keys; `fulltextDatoms()` returns `[e, attr, value]` triples.
321
+
322
+ ```js
323
+ console.log(await conn.datoms(":eav", { c1: 1, c2: ":name", limit: 10 }));
324
+ console.log(await conn.seekDatoms(":ave", { c1: ":name", c2: "Ada", limit: 5 }));
325
+ console.log(await conn.rseekDatoms(":ave", { c1: ":name", c2: "Bob", limit: 5 }));
326
+ console.log(await conn.indexRange(":name", "A", "C"));
327
+ console.log(await conn.countDatoms({ attr: ":name", value: "Ada" }));
328
+ console.log(await conn.fulltextDatoms("database", { opts: searchOptions({ limit: 5, offset: 10 }) }));
329
+ console.log(await conn.datalogIndexCacheLimit());
330
+ await conn.datalogIndexCacheLimit(1024);
331
+ console.log(await conn.txDataToSimulatedReport([{ ":db/id": -1, ":name": "Dry Run" }]));
332
+ ```
333
+
334
+ ## Bulk Load Example
335
+
336
+ Use `initDb()` and `fillDb()` when you already have Datom-shaped data and want
337
+ the fast bulk-load path. Datoms can be compact arrays in
338
+ `[entityId, attr, value]` shape, and `datom()` creates the same shape.
339
+
340
+ ```js
341
+ import { datom, fillDb, initDb } from "datalevin-node";
342
+
343
+ const schema = { ":name": { ":db/valueType": ":db.type/string" } };
344
+ const conn = await initDb([[1, ":name", "Ada"]], {
345
+ dir: "/tmp/dtlv-js-bulk",
346
+ schema
347
+ });
348
+
349
+ try {
350
+ await fillDb(conn, [[2, ":name", "Bob"]]);
351
+ await conn.fillDb([datom(3, ":name", "Cara")]);
352
+ } finally {
353
+ await conn.close();
354
+ }
355
+ ```
356
+
49
357
  ## KV Example
50
358
 
51
359
  ```js
@@ -69,6 +377,89 @@ try {
69
377
  kType: ":long",
70
378
  vType: ":string"
71
379
  }));
380
+ console.log(await kv.getRank("items", 2, { kType: ":long" }));
381
+ console.log(await kv.getEntryByRank("items", 1, {
382
+ kType: ":long",
383
+ vType: ":string"
384
+ }));
385
+ console.log(await kv.getFirstN("items", 2, [":all"], {
386
+ kType: ":long",
387
+ vType: ":string"
388
+ }));
389
+
390
+ await kv.openListDbi("tags");
391
+ await kv.putListItems("tags", "doc-1", ["clj", "db"], {
392
+ kType: ":string",
393
+ vType: ":string"
394
+ });
395
+ console.log(await kv.getList("tags", "doc-1", {
396
+ kType: ":string",
397
+ vType: ":string"
398
+ }));
399
+ console.log(await kv.listRange("tags", [":all"], {
400
+ kType: ":string",
401
+ vRange: [":all"],
402
+ vType: ":string"
403
+ }));
404
+ console.log(await kv.listRangeFirst("tags", [":all"], {
405
+ kType: ":string",
406
+ vRange: [":all"],
407
+ vType: ":string"
408
+ }));
409
+ console.log(await kv.listRangeFirstN("tags", 2, [":all"], {
410
+ kType: ":string",
411
+ vRange: [":all"],
412
+ vType: ":string"
413
+ }));
414
+ console.log(await kv.listRangeCount("tags", [":all"], { kType: ":string" }));
415
+ console.log(await kv.keyRangeListCount("tags", [":all"], {
416
+ kType: ":string"
417
+ }));
418
+ console.log(await kv.listRangeFilter("tags", (key, value) => (
419
+ key === "doc-1" && value.startsWith("c")
420
+ ), [":all"], {
421
+ kType: ":string",
422
+ vRange: [":all"],
423
+ vType: ":string"
424
+ }));
425
+ console.log(await kv.listRangeKeep("tags", (key, value) => (
426
+ value === "db" ? `${key}:${value}` : null
427
+ ), [":all"], {
428
+ kType: ":string",
429
+ vRange: [":all"],
430
+ vType: ":string"
431
+ }));
432
+ } finally {
433
+ await kv.close();
434
+ }
435
+ ```
436
+
437
+ ## Operational Example
438
+
439
+ KV stores expose backup, durability, snapshot, and WAL inspection helpers
440
+ without raw JSON calls.
441
+
442
+ ```js
443
+ import { openKv } from "datalevin-node";
444
+
445
+ const kv = await openKv("/tmp/dtlv-js-ops", { ":wal?": true });
446
+
447
+ try {
448
+ await kv.openDbi("items");
449
+ await kv.transact([[":put", "a", "alpha"]], {
450
+ dbiName: "items",
451
+ kType: ":string",
452
+ vType: ":string"
453
+ });
454
+
455
+ await kv.sync();
456
+ await kv.copy("/tmp/dtlv-js-ops-copy");
457
+
458
+ console.log(await kv.txLogWatermarks());
459
+ console.log(await kv.openTxLog(1, { limit: 10 }));
460
+ console.log(await kv.createSnapshot());
461
+ console.log(await kv.listSnapshots());
462
+ console.log(await kv.gcTxLogSegments());
72
463
  } finally {
73
464
  await kv.close();
74
465
  }
@@ -108,6 +499,10 @@ try {
108
499
 
109
500
  console.log(info);
110
501
  console.log(await client.listDatabases());
502
+ console.log(await client.replicaStatus("demo"));
503
+
504
+ // For consensus HA databases, operator membership changes are available as:
505
+ // await client.haUpdateMembership("demo", { ":ha-members": [...], ... });
111
506
  } finally {
112
507
  if (opened) {
113
508
  await client.closeDatabase("demo");
@@ -121,13 +516,13 @@ try {
121
516
 
122
517
  ## Embedding Search Options
123
518
 
124
- Node bindings pass Datalevin option maps through unchanged, so newer store
125
- features such as `:embedding-opts`, `:embedding-domains`, and remote
126
- `:openai-compatible` embedding providers are available directly from
127
- `connect()`:
519
+ Node bindings include helper builders for newer store features such as
520
+ `:embedding-opts`, `:embedding-domains`, and remote `:openai-compatible`
521
+ embedding providers. Raw Datalevin option maps are still passed through when
522
+ needed:
128
523
 
129
524
  ```js
130
- import { connect } from "datalevin-node";
525
+ import { connect, embeddingOptions } from "datalevin-node";
131
526
 
132
527
  const conn = await connect("/tmp/dtlv-js-embed", {
133
528
  schema: {
@@ -143,14 +538,14 @@ const conn = await connect("/tmp/dtlv-js-embed", {
143
538
  }
144
539
  },
145
540
  opts: {
146
- ":embedding-opts": {
147
- ":provider": ":openai-compatible",
148
- ":model": "text-embedding-3-small",
149
- ":base-url": "https://api.openai.com/v1",
150
- ":api-key-env": "OPENAI_API_KEY",
151
- ":request-dimensions": 1536,
152
- ":metric-type": ":cosine"
153
- }
541
+ ":embedding-opts": embeddingOptions({
542
+ provider: "openai-compatible",
543
+ model: "text-embedding-3-small",
544
+ baseUrl: "https://api.openai.com/v1",
545
+ apiKeyEnv: "OPENAI_API_KEY",
546
+ requestDimensions: 1536,
547
+ metricType: "cosine"
548
+ })
154
549
  }
155
550
  });
156
551
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "datalevin-node",
3
- "version": "0.10.18",
3
+ "version": "1.0.2",
4
4
  "description": "Node.js bindings for Datalevin over the JVM interop bridge",
5
5
  "type": "module",
6
6
  "repository": {
@@ -27,7 +27,7 @@
27
27
  },
28
28
  "scripts": {
29
29
  "prepack": "node scripts/vendor-runtime.mjs",
30
- "test": "node --test",
30
+ "test": "node --test --test-force-exit",
31
31
  "vendor-runtime": "node scripts/vendor-runtime.mjs"
32
32
  },
33
33
  "keywords": [
package/src/client.js CHANGED
@@ -81,6 +81,18 @@ export class Client extends ResourceWrapper {
81
81
  return toJsResult(await _BINDINGS.clientInvoke("list-databases-in-use", [this.rawHandle()]));
82
82
  }
83
83
 
84
+ async replicaStatus(dbName) {
85
+ return toJsResult(
86
+ await _BINDINGS.clientInvoke("replica-status", [this.rawHandle(), dbName])
87
+ );
88
+ }
89
+
90
+ async haUpdateMembership(dbName, spec) {
91
+ return toJsResult(
92
+ await _BINDINGS.clientInvoke("ha-update-membership!", [this.rawHandle(), dbName, spec])
93
+ );
94
+ }
95
+
84
96
  async createUser(username, password) {
85
97
  await _BINDINGS.clientInvoke("create-user", [this.rawHandle(), username, password]);
86
98
  }