@hajewski/latticedb 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +287 -26
- package/dist/database.d.ts +124 -7
- package/dist/database.d.ts.map +1 -1
- package/dist/database.js +117 -5
- package/dist/database.js.map +1 -1
- package/dist/embedding.d.ts +1 -1
- package/dist/embedding.js +1 -1
- package/dist/ffi/bindings.d.ts +11 -1
- package/dist/ffi/bindings.d.ts.map +1 -1
- package/dist/ffi/bindings.js +70 -1
- package/dist/ffi/bindings.js.map +1 -1
- package/dist/ffi/index.d.ts +93 -4
- package/dist/ffi/index.d.ts.map +1 -1
- package/dist/ffi/index.js +368 -43
- package/dist/ffi/index.js.map +1 -1
- package/dist/ffi/library.d.ts +59 -4
- package/dist/ffi/library.d.ts.map +1 -1
- package/dist/ffi/library.js +198 -36
- package/dist/ffi/library.js.map +1 -1
- package/dist/ffi/types.d.ts +52 -4
- package/dist/ffi/types.d.ts.map +1 -1
- package/dist/ffi/types.js +46 -0
- package/dist/ffi/types.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -5
- package/dist/index.js.map +1 -1
- package/dist/transaction.d.ts +18 -3
- package/dist/transaction.d.ts.map +1 -1
- package/dist/transaction.js +36 -4
- package/dist/transaction.js.map +1 -1
- package/dist/types.d.ts +29 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/lib/darwin-arm64/liblattice.dylib +0 -0
- package/lib/darwin-x64/liblattice.dylib +0 -0
- package/lib/linux-arm64/liblattice.so +0 -0
- package/lib/linux-x64/liblattice.so +0 -0
- package/lib/win32-arm64/lattice.dll +0 -0
- package/lib/win32-x64/lattice.dll +0 -0
- package/package.json +1 -1
package/dist/ffi/index.js
CHANGED
|
@@ -8,13 +8,21 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
8
8
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
9
9
|
};
|
|
10
10
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
-
exports.LatticeFFI = exports.LatticeQueryError = exports.LatticeError = exports.getLibraryPath = exports.isLibraryAvailable = void 0;
|
|
11
|
+
exports.LatticeFFI = exports.LatticeQueryError = exports.LatticeChangesTrimmedError = exports.LatticeError = exports.QueryErrorStage = exports.LatticeErrorCode = exports.ELECTRON_ASAR_UNPACK_GLOB = exports.isElectronRuntime = exports.resolveLibraryPath = exports.getLibraryPath = exports.isLibraryAvailable = void 0;
|
|
12
|
+
exports.changefeedRetentionCode = changefeedRetentionCode;
|
|
12
13
|
exports.getFFI = getFFI;
|
|
13
14
|
const koffi_1 = __importDefault(require("koffi"));
|
|
14
15
|
const bindings_1 = require("./bindings");
|
|
16
|
+
const types_1 = require("./types");
|
|
15
17
|
var library_1 = require("./library");
|
|
16
18
|
Object.defineProperty(exports, "isLibraryAvailable", { enumerable: true, get: function () { return library_1.isLibraryAvailable; } });
|
|
17
19
|
Object.defineProperty(exports, "getLibraryPath", { enumerable: true, get: function () { return library_1.getLibraryPath; } });
|
|
20
|
+
Object.defineProperty(exports, "resolveLibraryPath", { enumerable: true, get: function () { return library_1.resolveLibraryPath; } });
|
|
21
|
+
Object.defineProperty(exports, "isElectronRuntime", { enumerable: true, get: function () { return library_1.isElectronRuntime; } });
|
|
22
|
+
Object.defineProperty(exports, "ELECTRON_ASAR_UNPACK_GLOB", { enumerable: true, get: function () { return library_1.ELECTRON_ASAR_UNPACK_GLOB; } });
|
|
23
|
+
var types_2 = require("./types");
|
|
24
|
+
Object.defineProperty(exports, "LatticeErrorCode", { enumerable: true, get: function () { return types_2.LatticeErrorCode; } });
|
|
25
|
+
Object.defineProperty(exports, "QueryErrorStage", { enumerable: true, get: function () { return types_2.QueryErrorStage; } });
|
|
18
26
|
/**
|
|
19
27
|
* Error class for Lattice operations.
|
|
20
28
|
*/
|
|
@@ -27,6 +35,19 @@ class LatticeError extends Error {
|
|
|
27
35
|
}
|
|
28
36
|
}
|
|
29
37
|
exports.LatticeError = LatticeError;
|
|
38
|
+
/**
|
|
39
|
+
* The changefeed no longer holds the events after the position read from:
|
|
40
|
+
* its retention policy, or a trim, dropped them, so the reader has missed
|
|
41
|
+
* changes. Reading from `afterSequence: 0n` starts at the oldest event held.
|
|
42
|
+
* Its code is `LatticeErrorCode.NotFound`.
|
|
43
|
+
*/
|
|
44
|
+
class LatticeChangesTrimmedError extends LatticeError {
|
|
45
|
+
constructor(message) {
|
|
46
|
+
super(message, types_1.LatticeErrorCode.NotFound);
|
|
47
|
+
this.name = 'LatticeChangesTrimmedError';
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
exports.LatticeChangesTrimmedError = LatticeChangesTrimmedError;
|
|
30
51
|
/**
|
|
31
52
|
* Rich query execution error with stage-aware diagnostics.
|
|
32
53
|
*/
|
|
@@ -43,6 +64,137 @@ class LatticeQueryError extends LatticeError {
|
|
|
43
64
|
}
|
|
44
65
|
}
|
|
45
66
|
exports.LatticeQueryError = LatticeQueryError;
|
|
67
|
+
const U64_MAX = (BigInt(1) << BigInt(64)) - BigInt(1);
|
|
68
|
+
// Longest suffix first, so "ms" is not read as "s".
|
|
69
|
+
const DURATION_UNITS = [
|
|
70
|
+
['ms', BigInt(1)],
|
|
71
|
+
['s', BigInt(1000)],
|
|
72
|
+
['m', BigInt(60_000)],
|
|
73
|
+
['h', BigInt(3_600_000)],
|
|
74
|
+
['d', BigInt(86_400_000)],
|
|
75
|
+
];
|
|
76
|
+
/** Turn a policy such as `'last:1000'` into the C API's mode and value. */
|
|
77
|
+
function changefeedRetentionCode(policy) {
|
|
78
|
+
if (policy === undefined) {
|
|
79
|
+
return [0 /* LatticeChangefeedRetention.Default */, BigInt(0)];
|
|
80
|
+
}
|
|
81
|
+
if (typeof policy === 'string') {
|
|
82
|
+
const lowered = policy.toLowerCase();
|
|
83
|
+
if (lowered === 'all') {
|
|
84
|
+
return [1 /* LatticeChangefeedRetention.All */, BigInt(0)];
|
|
85
|
+
}
|
|
86
|
+
if (lowered === 'off') {
|
|
87
|
+
return [4 /* LatticeChangefeedRetention.Off */, BigInt(0)];
|
|
88
|
+
}
|
|
89
|
+
if (policy.startsWith('last:')) {
|
|
90
|
+
const digits = policy.slice(5);
|
|
91
|
+
if (/^[0-9]+$/.test(digits)) {
|
|
92
|
+
const n = BigInt(digits);
|
|
93
|
+
if (n > BigInt(0) && n <= U64_MAX) {
|
|
94
|
+
return [2 /* LatticeChangefeedRetention.Last */, n];
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
else if (policy.startsWith('age:')) {
|
|
99
|
+
const text = policy.slice(4);
|
|
100
|
+
for (const [suffix, ms] of DURATION_UNITS) {
|
|
101
|
+
const number = text.slice(0, text.length - suffix.length);
|
|
102
|
+
if (text.endsWith(suffix) && /^[0-9]+$/.test(number)) {
|
|
103
|
+
const value = BigInt(number) * ms;
|
|
104
|
+
if (value > BigInt(0) && value <= U64_MAX) {
|
|
105
|
+
return [3 /* LatticeChangefeedRetention.Age */, value];
|
|
106
|
+
}
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
throw new TypeError("changefeedRetention must be 'all', 'off', 'last:<events>' or 'age:<duration>' " +
|
|
113
|
+
`(such as 'age:7d', 'age:12h', 'age:500ms'), not ${String(policy)}`);
|
|
114
|
+
}
|
|
115
|
+
function changefeedRetentionText(mode, value) {
|
|
116
|
+
switch (mode) {
|
|
117
|
+
case 4 /* LatticeChangefeedRetention.Off */:
|
|
118
|
+
return 'off';
|
|
119
|
+
case 2 /* LatticeChangefeedRetention.Last */:
|
|
120
|
+
return `last:${value}`;
|
|
121
|
+
case 3 /* LatticeChangefeedRetention.Age */: {
|
|
122
|
+
for (const [suffix, ms] of DURATION_UNITS.slice(1).reverse()) {
|
|
123
|
+
if (value % ms === BigInt(0)) {
|
|
124
|
+
return `age:${value / ms}${suffix}`;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return `age:${value}ms`;
|
|
128
|
+
}
|
|
129
|
+
default:
|
|
130
|
+
return 'all';
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function syncModeCode(mode) {
|
|
134
|
+
switch (mode) {
|
|
135
|
+
case undefined:
|
|
136
|
+
return 0 /* LatticeSyncMode.Default */;
|
|
137
|
+
case 'full':
|
|
138
|
+
return 1 /* LatticeSyncMode.Full */;
|
|
139
|
+
case 'normal':
|
|
140
|
+
return 2 /* LatticeSyncMode.Normal */;
|
|
141
|
+
default:
|
|
142
|
+
throw new TypeError(`syncMode must be 'full' or 'normal', not ${String(mode)}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function fullFsyncCode(fullFsync) {
|
|
146
|
+
if (fullFsync === undefined) {
|
|
147
|
+
return 0 /* LatticeFullFsync.Default */;
|
|
148
|
+
}
|
|
149
|
+
return fullFsync ? 2 /* LatticeFullFsync.On */ : 1 /* LatticeFullFsync.Off */;
|
|
150
|
+
}
|
|
151
|
+
function vectorMetricCode(metric) {
|
|
152
|
+
switch (metric) {
|
|
153
|
+
case undefined:
|
|
154
|
+
return 0 /* LatticeVectorMetric.Default */;
|
|
155
|
+
case 'cosine':
|
|
156
|
+
return 1 /* LatticeVectorMetric.Cosine */;
|
|
157
|
+
case 'euclidean':
|
|
158
|
+
return 2 /* LatticeVectorMetric.Euclidean */;
|
|
159
|
+
case 'dot_product':
|
|
160
|
+
return 3 /* LatticeVectorMetric.DotProduct */;
|
|
161
|
+
default:
|
|
162
|
+
throw new TypeError(`vectorMetric must be 'cosine', 'euclidean' or 'dot_product', not ${String(metric)}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
function checkpointModeCode(mode) {
|
|
166
|
+
switch (mode) {
|
|
167
|
+
case 'passive':
|
|
168
|
+
return 0 /* LatticeCheckpointMode.Passive */;
|
|
169
|
+
case 'full':
|
|
170
|
+
return 1 /* LatticeCheckpointMode.Full */;
|
|
171
|
+
case 'truncate':
|
|
172
|
+
return 2 /* LatticeCheckpointMode.Truncate */;
|
|
173
|
+
default:
|
|
174
|
+
throw new TypeError(`checkpoint mode must be 'passive', 'full' or 'truncate', not ${String(mode)}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/** A count of frames or milliseconds: a non-negative safe integer. */
|
|
178
|
+
function checkpointSetting(name, value) {
|
|
179
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
|
|
180
|
+
throw new RangeError(`${name} must be a non-negative integer, not ${String(value)}`);
|
|
181
|
+
}
|
|
182
|
+
return BigInt(value);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Copy `len` bytes of native memory into a new Uint8Array.
|
|
186
|
+
*
|
|
187
|
+
* koffi.view (koffi 2.11+) copies in one block; decoding to `number[]` first
|
|
188
|
+
* builds one JS number per byte, which for a serialized database is millions
|
|
189
|
+
* of them.
|
|
190
|
+
*/
|
|
191
|
+
function copyNativeBytes(ptr, len) {
|
|
192
|
+
const view = koffi_1.default.view;
|
|
193
|
+
if (typeof view === 'function') {
|
|
194
|
+
return new Uint8Array(view(ptr, len).slice(0));
|
|
195
|
+
}
|
|
196
|
+
return Uint8Array.from(koffi_1.default.decode(ptr, 'uint8_t', len));
|
|
197
|
+
}
|
|
46
198
|
/**
|
|
47
199
|
* High-level FFI wrapper class.
|
|
48
200
|
*/
|
|
@@ -52,6 +204,11 @@ class LatticeFFI {
|
|
|
52
204
|
static VALUE_SIZE = koffi_1.default.sizeof('lattice_value');
|
|
53
205
|
static MAP_ENTRY_SIZE = koffi_1.default.sizeof('lattice_map_entry');
|
|
54
206
|
static MAP_ENTRY_VALUE_OFFSET = koffi_1.default.offsetof('lattice_map_entry', 'value');
|
|
207
|
+
static CHECKPOINT_STATS_SIZE = koffi_1.default.sizeof('lattice_checkpoint_stats');
|
|
208
|
+
static CHECKPOINT_PAGES_OFFSET = koffi_1.default.offsetof('lattice_checkpoint_stats', 'pages_flushed');
|
|
209
|
+
static CHECKPOINT_DURATION_OFFSET = koffi_1.default.offsetof('lattice_checkpoint_stats', 'duration_ns');
|
|
210
|
+
static CHECKPOINT_LSN_OFFSET = koffi_1.default.offsetof('lattice_checkpoint_stats', 'checkpoint_lsn');
|
|
211
|
+
static CHECKPOINT_TRUNCATED_OFFSET = koffi_1.default.offsetof('lattice_checkpoint_stats', 'wal_truncated');
|
|
55
212
|
constructor() {
|
|
56
213
|
this.bindings = (0, bindings_1.getBindings)();
|
|
57
214
|
}
|
|
@@ -59,30 +216,30 @@ class LatticeFFI {
|
|
|
59
216
|
* Check an error code and throw if not OK.
|
|
60
217
|
*/
|
|
61
218
|
checkError(code) {
|
|
62
|
-
if (code !==
|
|
219
|
+
if (code !== types_1.LatticeErrorCode.Ok) {
|
|
63
220
|
const message = this.bindings.lattice_error_message(code);
|
|
64
221
|
throw new LatticeError(message, code);
|
|
65
222
|
}
|
|
66
223
|
}
|
|
67
224
|
normalizeQueryStage(stage) {
|
|
68
225
|
switch (stage) {
|
|
69
|
-
case
|
|
70
|
-
return
|
|
71
|
-
case
|
|
72
|
-
return
|
|
73
|
-
case
|
|
74
|
-
return
|
|
75
|
-
case
|
|
76
|
-
return
|
|
226
|
+
case types_1.QueryErrorStage.Parse:
|
|
227
|
+
return types_1.QueryErrorStage.Parse;
|
|
228
|
+
case types_1.QueryErrorStage.Semantic:
|
|
229
|
+
return types_1.QueryErrorStage.Semantic;
|
|
230
|
+
case types_1.QueryErrorStage.Plan:
|
|
231
|
+
return types_1.QueryErrorStage.Plan;
|
|
232
|
+
case types_1.QueryErrorStage.Execution:
|
|
233
|
+
return types_1.QueryErrorStage.Execution;
|
|
77
234
|
default:
|
|
78
|
-
return
|
|
235
|
+
return types_1.QueryErrorStage.None;
|
|
79
236
|
}
|
|
80
237
|
}
|
|
81
238
|
/**
|
|
82
239
|
* Check an error code for query execution and include detailed diagnostics.
|
|
83
240
|
*/
|
|
84
241
|
checkQueryError(code, query) {
|
|
85
|
-
if (code ===
|
|
242
|
+
if (code === types_1.LatticeErrorCode.Ok) {
|
|
86
243
|
return;
|
|
87
244
|
}
|
|
88
245
|
const stage = this.normalizeQueryStage(this.bindings.lattice_query_last_error_stage(query));
|
|
@@ -96,7 +253,7 @@ class LatticeFFI {
|
|
|
96
253
|
length: this.bindings.lattice_query_last_error_length(query),
|
|
97
254
|
};
|
|
98
255
|
}
|
|
99
|
-
if (stage !==
|
|
256
|
+
if (stage !== types_1.QueryErrorStage.None || detailMessage !== undefined) {
|
|
100
257
|
const fallbackMessage = this.bindings.lattice_error_message(code);
|
|
101
258
|
throw new LatticeQueryError(detailMessage ?? fallbackMessage, code, stage, detailCode, location);
|
|
102
259
|
}
|
|
@@ -118,19 +275,33 @@ class LatticeFFI {
|
|
|
118
275
|
const enableWal = options.enableWal ?? true;
|
|
119
276
|
const enableAdjacencyCache = options.enableAdjacencyCache ?? false;
|
|
120
277
|
const lock = options.lock ?? true;
|
|
278
|
+
const syncMode = syncModeCode(options.syncMode);
|
|
279
|
+
const fullFsync = fullFsyncCode(options.fullFsync);
|
|
280
|
+
const vectorMetric = vectorMetricCode(options.vectorMetric);
|
|
281
|
+
const vectorM = options.vectorM ?? 0;
|
|
282
|
+
const vectorEfConstruction = options.vectorEfConstruction ?? 0;
|
|
283
|
+
const [changefeedRetention, changefeedRetentionValue] = changefeedRetentionCode(options.changefeedRetention);
|
|
284
|
+
if (changefeedRetention && !this.bindings.lattice_changefeed_status) {
|
|
285
|
+
throw new LatticeError('the native library is too old to support changefeedRetention', types_1.LatticeErrorCode.Unsupported);
|
|
286
|
+
}
|
|
287
|
+
if (!this.bindings.lattice_open_v5 &&
|
|
288
|
+
(syncMode || fullFsync || vectorMetric || vectorM || vectorEfConstruction)) {
|
|
289
|
+
throw new LatticeError('lattice_open_v5 is required for syncMode, fullFsync, vectorMetric, vectorM ' +
|
|
290
|
+
'and vectorEfConstruction', types_1.LatticeErrorCode.Unsupported);
|
|
291
|
+
}
|
|
121
292
|
if (!this.bindings.lattice_open_v4 && !lock) {
|
|
122
|
-
throw new LatticeError('lattice_open_v4 is required to disable the file lock',
|
|
293
|
+
throw new LatticeError('lattice_open_v4 is required to disable the file lock', types_1.LatticeErrorCode.Unsupported);
|
|
123
294
|
}
|
|
124
295
|
if (!this.bindings.lattice_open_v2 && !enableWal) {
|
|
125
|
-
throw new LatticeError('lattice_open_v2 is required to disable WAL',
|
|
296
|
+
throw new LatticeError('lattice_open_v2 is required to disable WAL', types_1.LatticeErrorCode.Unsupported);
|
|
126
297
|
}
|
|
127
298
|
if (!this.bindings.lattice_open_v3 && enableAdjacencyCache) {
|
|
128
|
-
throw new LatticeError('lattice_open_v3 is required to enable the adjacency cache',
|
|
299
|
+
throw new LatticeError('lattice_open_v3 is required to enable the adjacency cache', types_1.LatticeErrorCode.Unsupported);
|
|
129
300
|
}
|
|
130
301
|
const baseOpts = {
|
|
131
302
|
create: options.create ?? false,
|
|
132
303
|
read_only: options.readOnly ?? false,
|
|
133
|
-
cache_size_mb: options.cacheSizeMb ??
|
|
304
|
+
cache_size_mb: options.cacheSizeMb ?? 0,
|
|
134
305
|
page_size: options.pageSize ?? 4096,
|
|
135
306
|
enable_vector: options.enableVectors ?? options.enableVector ?? false,
|
|
136
307
|
vector_dimensions: options.vectorDimensions ?? 128,
|
|
@@ -153,14 +324,27 @@ class LatticeFFI {
|
|
|
153
324
|
enable_adjacency_cache: enableAdjacencyCache,
|
|
154
325
|
lock,
|
|
155
326
|
};
|
|
327
|
+
const v5Opts = {
|
|
328
|
+
...v4Opts,
|
|
329
|
+
struct_size: koffi_1.default.sizeof('lattice_open_options_v5'),
|
|
330
|
+
sync_mode: syncMode,
|
|
331
|
+
full_fsync: fullFsync,
|
|
332
|
+
vector_metric: vectorMetric,
|
|
333
|
+
vector_m: vectorM,
|
|
334
|
+
vector_ef_construction: vectorEfConstruction,
|
|
335
|
+
changefeed_retention: changefeedRetention,
|
|
336
|
+
changefeed_retention_value: changefeedRetentionValue,
|
|
337
|
+
};
|
|
156
338
|
const dbOut = [null];
|
|
157
|
-
const err = this.bindings.
|
|
158
|
-
? this.bindings.
|
|
159
|
-
: this.bindings.
|
|
160
|
-
? this.bindings.
|
|
161
|
-
: this.bindings.
|
|
162
|
-
? this.bindings.
|
|
163
|
-
: this.bindings.
|
|
339
|
+
const err = this.bindings.lattice_open_v5
|
|
340
|
+
? this.bindings.lattice_open_v5(path, v5Opts, dbOut)
|
|
341
|
+
: this.bindings.lattice_open_v4
|
|
342
|
+
? this.bindings.lattice_open_v4(path, v4Opts, dbOut)
|
|
343
|
+
: this.bindings.lattice_open_v3
|
|
344
|
+
? this.bindings.lattice_open_v3(path, v3Opts, dbOut)
|
|
345
|
+
: this.bindings.lattice_open_v2
|
|
346
|
+
? this.bindings.lattice_open_v2(path, v2Opts, dbOut)
|
|
347
|
+
: this.bindings.lattice_open(path, baseOpts, dbOut);
|
|
164
348
|
this.checkError(err);
|
|
165
349
|
return dbOut[0];
|
|
166
350
|
}
|
|
@@ -169,7 +353,7 @@ class LatticeFFI {
|
|
|
169
353
|
*/
|
|
170
354
|
serialize(db) {
|
|
171
355
|
if (!this.bindings.lattice_serialize || !this.bindings.lattice_free_bytes) {
|
|
172
|
-
throw new LatticeError('the native library is too old to support serialize',
|
|
356
|
+
throw new LatticeError('the native library is too old to support serialize', types_1.LatticeErrorCode.Unsupported);
|
|
173
357
|
}
|
|
174
358
|
const bytesOut = [null];
|
|
175
359
|
const lenOut = [0];
|
|
@@ -182,8 +366,7 @@ class LatticeFFI {
|
|
|
182
366
|
try {
|
|
183
367
|
// Copied out before the native buffer is released, so what the caller
|
|
184
368
|
// holds does not depend on memory this function is about to free.
|
|
185
|
-
|
|
186
|
-
return Uint8Array.from(decoded);
|
|
369
|
+
return copyNativeBytes(ptr, len);
|
|
187
370
|
}
|
|
188
371
|
finally {
|
|
189
372
|
this.bindings.lattice_free_bytes(ptr, len);
|
|
@@ -191,19 +374,32 @@ class LatticeFFI {
|
|
|
191
374
|
}
|
|
192
375
|
/**
|
|
193
376
|
* Open a database from bytes produced by `serialize`.
|
|
377
|
+
*
|
|
378
|
+
* Uses the v5 entry point when the library has it, so the sync and vector
|
|
379
|
+
* index options reach the database as they do for `open`; otherwise the v4
|
|
380
|
+
* one, which cannot carry them.
|
|
194
381
|
*/
|
|
195
382
|
deserialize(bytes, options = {}, copy = true) {
|
|
196
383
|
const entry = copy
|
|
197
384
|
? this.bindings.lattice_deserialize
|
|
198
385
|
: this.bindings.lattice_deserialize_borrowed;
|
|
199
|
-
|
|
200
|
-
|
|
386
|
+
const entryV5 = copy
|
|
387
|
+
? this.bindings.lattice_deserialize_v5
|
|
388
|
+
: this.bindings.lattice_deserialize_borrowed_v5;
|
|
389
|
+
if (!entry && !entryV5) {
|
|
390
|
+
throw new LatticeError('the native library is too old to support deserialize', types_1.LatticeErrorCode.Unsupported);
|
|
201
391
|
}
|
|
202
|
-
const
|
|
392
|
+
const syncMode = syncModeCode(options.syncMode);
|
|
393
|
+
const fullFsync = fullFsyncCode(options.fullFsync);
|
|
394
|
+
const vectorMetric = vectorMetricCode(options.vectorMetric);
|
|
395
|
+
const vectorM = options.vectorM ?? 0;
|
|
396
|
+
const vectorEfConstruction = options.vectorEfConstruction ?? 0;
|
|
397
|
+
const [changefeedRetention, changefeedRetentionValue] = changefeedRetentionCode(options.changefeedRetention);
|
|
398
|
+
const v4Opts = {
|
|
203
399
|
struct_size: koffi_1.default.sizeof('lattice_open_options_v4'),
|
|
204
400
|
create: false,
|
|
205
401
|
read_only: false,
|
|
206
|
-
cache_size_mb: options.cacheSizeMb ??
|
|
402
|
+
cache_size_mb: options.cacheSizeMb ?? 0,
|
|
207
403
|
page_size: options.pageSize ?? 4096,
|
|
208
404
|
enable_vector: options.enableVectors ?? options.enableVector ?? false,
|
|
209
405
|
vector_dimensions: options.vectorDimensions ?? 128,
|
|
@@ -212,7 +408,32 @@ class LatticeFFI {
|
|
|
212
408
|
lock: options.lock ?? true,
|
|
213
409
|
};
|
|
214
410
|
const dbOut = [null];
|
|
215
|
-
|
|
411
|
+
if (entryV5) {
|
|
412
|
+
const v5Opts = {
|
|
413
|
+
...v4Opts,
|
|
414
|
+
struct_size: koffi_1.default.sizeof('lattice_open_options_v5'),
|
|
415
|
+
sync_mode: syncMode,
|
|
416
|
+
full_fsync: fullFsync,
|
|
417
|
+
vector_metric: vectorMetric,
|
|
418
|
+
vector_m: vectorM,
|
|
419
|
+
vector_ef_construction: vectorEfConstruction,
|
|
420
|
+
changefeed_retention: changefeedRetention,
|
|
421
|
+
changefeed_retention_value: changefeedRetentionValue,
|
|
422
|
+
};
|
|
423
|
+
this.checkError(entryV5(bytes, bytes.length, v5Opts, dbOut));
|
|
424
|
+
return dbOut[0];
|
|
425
|
+
}
|
|
426
|
+
if (syncMode ||
|
|
427
|
+
fullFsync ||
|
|
428
|
+
vectorMetric ||
|
|
429
|
+
vectorM ||
|
|
430
|
+
vectorEfConstruction ||
|
|
431
|
+
changefeedRetention) {
|
|
432
|
+
throw new LatticeError('lattice_deserialize_v5 is required for syncMode, fullFsync, vectorMetric, ' +
|
|
433
|
+
'vectorM, vectorEfConstruction and changefeedRetention', types_1.LatticeErrorCode.Unsupported);
|
|
434
|
+
}
|
|
435
|
+
// entryV5 is absent here, so the check above guarantees entry.
|
|
436
|
+
this.checkError(entry(bytes, bytes.length, v4Opts, dbOut));
|
|
216
437
|
return dbOut[0];
|
|
217
438
|
}
|
|
218
439
|
/**
|
|
@@ -222,6 +443,34 @@ class LatticeFFI {
|
|
|
222
443
|
const err = this.bindings.lattice_close(db);
|
|
223
444
|
this.checkError(err);
|
|
224
445
|
}
|
|
446
|
+
/**
|
|
447
|
+
* Checkpoint now. The mode is checked before the library is called.
|
|
448
|
+
*/
|
|
449
|
+
checkpoint(db, mode) {
|
|
450
|
+
const modeCode = checkpointModeCode(mode);
|
|
451
|
+
if (!this.bindings.lattice_checkpoint) {
|
|
452
|
+
throw new LatticeError('the native library is too old to support checkpoint', types_1.LatticeErrorCode.Unsupported);
|
|
453
|
+
}
|
|
454
|
+
const statsOut = Buffer.alloc(LatticeFFI.CHECKPOINT_STATS_SIZE);
|
|
455
|
+
this.checkError(this.bindings.lattice_checkpoint(db, modeCode, statsOut));
|
|
456
|
+
return {
|
|
457
|
+
pagesFlushed: statsOut.readUInt32LE(LatticeFFI.CHECKPOINT_PAGES_OFFSET),
|
|
458
|
+
durationNs: statsOut.readBigUInt64LE(LatticeFFI.CHECKPOINT_DURATION_OFFSET),
|
|
459
|
+
checkpointLsn: statsOut.readBigUInt64LE(LatticeFFI.CHECKPOINT_LSN_OFFSET),
|
|
460
|
+
walTruncated: statsOut.readUInt8(LatticeFFI.CHECKPOINT_TRUNCATED_OFFSET) !== 0,
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
/**
|
|
464
|
+
* Set when commits checkpoint from now on; 0 frames switches them off.
|
|
465
|
+
*/
|
|
466
|
+
setAutoCheckpoint(db, maxWalFrames, minIntervalMs) {
|
|
467
|
+
const frames = checkpointSetting('maxWalFrames', maxWalFrames);
|
|
468
|
+
const interval = checkpointSetting('minIntervalMs', minIntervalMs);
|
|
469
|
+
if (!this.bindings.lattice_set_auto_checkpoint) {
|
|
470
|
+
throw new LatticeError('the native library is too old to support setAutoCheckpoint', types_1.LatticeErrorCode.Unsupported);
|
|
471
|
+
}
|
|
472
|
+
this.checkError(this.bindings.lattice_set_auto_checkpoint(db, frames, interval));
|
|
473
|
+
}
|
|
225
474
|
// ============================================================
|
|
226
475
|
// Transaction operations
|
|
227
476
|
// ============================================================
|
|
@@ -276,12 +525,22 @@ class LatticeFFI {
|
|
|
276
525
|
this.checkError(err);
|
|
277
526
|
}
|
|
278
527
|
/**
|
|
279
|
-
* Delete a node.
|
|
528
|
+
* Delete a node. Fails with HasRelationships while the node has any.
|
|
280
529
|
*/
|
|
281
530
|
deleteNode(txn, nodeId) {
|
|
282
531
|
const err = this.bindings.lattice_node_delete(txn, nodeId);
|
|
283
532
|
this.checkError(err);
|
|
284
533
|
}
|
|
534
|
+
/**
|
|
535
|
+
* Delete a node and every relationship it has.
|
|
536
|
+
*/
|
|
537
|
+
detachDeleteNode(txn, nodeId) {
|
|
538
|
+
if (!this.bindings.lattice_node_detach_delete) {
|
|
539
|
+
throw new LatticeError('lattice_node_detach_delete is required to delete a node with its relationships', types_1.LatticeErrorCode.Unsupported);
|
|
540
|
+
}
|
|
541
|
+
const err = this.bindings.lattice_node_detach_delete(txn, nodeId);
|
|
542
|
+
this.checkError(err);
|
|
543
|
+
}
|
|
285
544
|
/**
|
|
286
545
|
* Check if a node exists.
|
|
287
546
|
*/
|
|
@@ -432,7 +691,7 @@ class LatticeFFI {
|
|
|
432
691
|
this.checkError(err);
|
|
433
692
|
const created = countOut.readUInt32LE();
|
|
434
693
|
if (created < count) {
|
|
435
|
-
throw new LatticeError(`Batch insert partially failed: ${created}/${count} nodes created. Transaction should be rolled back.`,
|
|
694
|
+
throw new LatticeError(`Batch insert partially failed: ${created}/${count} nodes created. Transaction should be rolled back.`, types_1.LatticeErrorCode.Error);
|
|
436
695
|
}
|
|
437
696
|
const ids = [];
|
|
438
697
|
for (let i = 0; i < created; i++) {
|
|
@@ -459,6 +718,9 @@ class LatticeFFI {
|
|
|
459
718
|
let ownsNativeValue = false;
|
|
460
719
|
try {
|
|
461
720
|
const err = this.bindings.lattice_node_get_property(txn, nodeId, key, valueOut);
|
|
721
|
+
if (err === types_1.LatticeErrorCode.NotFound) {
|
|
722
|
+
return null;
|
|
723
|
+
}
|
|
462
724
|
this.checkError(err);
|
|
463
725
|
ownsNativeValue = true;
|
|
464
726
|
return this.nativeValueToJs(valueOut);
|
|
@@ -486,12 +748,71 @@ class LatticeFFI {
|
|
|
486
748
|
this.checkError(err);
|
|
487
749
|
return sequenceOut.readBigUInt64LE();
|
|
488
750
|
}
|
|
751
|
+
/**
|
|
752
|
+
* The changefeed's retention policy, the oldest event it holds and the
|
|
753
|
+
* last sequence it handed out.
|
|
754
|
+
*/
|
|
755
|
+
changefeedStatus(db) {
|
|
756
|
+
if (!this.bindings.lattice_changefeed_status) {
|
|
757
|
+
throw new LatticeError("the native library is too old to report the changefeed's status", types_1.LatticeErrorCode.Unsupported);
|
|
758
|
+
}
|
|
759
|
+
const mode = Buffer.alloc(1);
|
|
760
|
+
const value = Buffer.alloc(8);
|
|
761
|
+
const oldest = Buffer.alloc(8);
|
|
762
|
+
const last = Buffer.alloc(8);
|
|
763
|
+
this.checkError(this.bindings.lattice_changefeed_status(db, mode, value, oldest, last));
|
|
764
|
+
const oldestSequence = oldest.readBigUInt64LE();
|
|
765
|
+
return {
|
|
766
|
+
retention: changefeedRetentionText(mode.readUInt8(0), value.readBigUInt64LE()),
|
|
767
|
+
oldestSequence: oldestSequence === BigInt(0) ? null : oldestSequence,
|
|
768
|
+
lastSequence: last.readBigUInt64LE(),
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* Apply the changefeed's retention policy in full now. Returns the number
|
|
773
|
+
* of events dropped.
|
|
774
|
+
*/
|
|
775
|
+
trimChangefeed(db) {
|
|
776
|
+
if (!this.bindings.lattice_changefeed_trim) {
|
|
777
|
+
throw new LatticeError('the native library is too old to trim the changefeed', types_1.LatticeErrorCode.Unsupported);
|
|
778
|
+
}
|
|
779
|
+
const removed = Buffer.alloc(8);
|
|
780
|
+
this.checkError(this.bindings.lattice_changefeed_trim(db, removed));
|
|
781
|
+
return removed.readBigUInt64LE();
|
|
782
|
+
}
|
|
489
783
|
readStream(db, stream, afterSequence = BigInt(0), limit = 100, timeoutMs = 0) {
|
|
490
784
|
const streamBuf = Buffer.from(stream, 'utf8');
|
|
491
785
|
const batchOut = [null];
|
|
492
786
|
const err = this.bindings.lattice_stream_read(db, streamBuf, streamBuf.length, afterSequence, limit, timeoutMs, batchOut);
|
|
493
787
|
this.checkError(err);
|
|
494
|
-
|
|
788
|
+
return this.takeStreamBatch(batchOut[0]);
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* Read a stream on a koffi worker thread, so a wait of `timeoutMs` does not
|
|
792
|
+
* block the event loop. Blocking it would also block the commit the reader
|
|
793
|
+
* is waiting for whenever that commit comes from this same thread.
|
|
794
|
+
*/
|
|
795
|
+
readStreamAsync(db, stream, afterSequence = BigInt(0), limit = 100, timeoutMs = 0) {
|
|
796
|
+
const streamBuf = Buffer.from(stream, 'utf8');
|
|
797
|
+
const batchOut = [null];
|
|
798
|
+
return new Promise((resolve, reject) => {
|
|
799
|
+
this.bindings.lattice_stream_read.async(db, streamBuf, streamBuf.length, afterSequence, limit, timeoutMs, batchOut, (callErr, err) => {
|
|
800
|
+
if (callErr) {
|
|
801
|
+
reject(callErr);
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
try {
|
|
805
|
+
this.checkError(err);
|
|
806
|
+
resolve(this.takeStreamBatch(batchOut[0]));
|
|
807
|
+
}
|
|
808
|
+
catch (decodeErr) {
|
|
809
|
+
reject(decodeErr);
|
|
810
|
+
}
|
|
811
|
+
});
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
/** Decode a stream batch into records and free it. */
|
|
815
|
+
takeStreamBatch(batch) {
|
|
495
816
|
if (!batch) {
|
|
496
817
|
return [];
|
|
497
818
|
}
|
|
@@ -717,8 +1038,7 @@ class LatticeFFI {
|
|
|
717
1038
|
const len = this.decodeSize(value, dataOffset + 8);
|
|
718
1039
|
if (!ptr || len === 0)
|
|
719
1040
|
return new Uint8Array(0);
|
|
720
|
-
|
|
721
|
-
return Uint8Array.from(bytes);
|
|
1041
|
+
return copyNativeBytes(ptr, len);
|
|
722
1042
|
}
|
|
723
1043
|
case 6 /* LatticeValueType.Vector */: {
|
|
724
1044
|
const ptr = this.decodePointer(value, dataOffset);
|
|
@@ -763,7 +1083,7 @@ class LatticeFFI {
|
|
|
763
1083
|
return result;
|
|
764
1084
|
}
|
|
765
1085
|
default:
|
|
766
|
-
throw new LatticeError(`Unsupported native value type: ${type}`,
|
|
1086
|
+
throw new LatticeError(`Unsupported native value type: ${type}`, types_1.LatticeErrorCode.Unsupported);
|
|
767
1087
|
}
|
|
768
1088
|
}
|
|
769
1089
|
/**
|
|
@@ -801,8 +1121,7 @@ class LatticeFFI {
|
|
|
801
1121
|
const len = Number(bv.len);
|
|
802
1122
|
if (!bv.ptr || len === 0)
|
|
803
1123
|
return new Uint8Array(0);
|
|
804
|
-
|
|
805
|
-
return Uint8Array.from(bytes);
|
|
1124
|
+
return copyNativeBytes(bv.ptr, len);
|
|
806
1125
|
}
|
|
807
1126
|
case 6 /* LatticeValueType.Vector */: {
|
|
808
1127
|
const vv = data.vector_val;
|
|
@@ -834,7 +1153,7 @@ class LatticeFFI {
|
|
|
834
1153
|
return result;
|
|
835
1154
|
}
|
|
836
1155
|
default:
|
|
837
|
-
throw new LatticeError(`Unsupported native value type: ${type}`,
|
|
1156
|
+
throw new LatticeError(`Unsupported native value type: ${type}`, types_1.LatticeErrorCode.Unsupported);
|
|
838
1157
|
}
|
|
839
1158
|
}
|
|
840
1159
|
// ============================================================
|
|
@@ -1025,7 +1344,7 @@ class LatticeFFI {
|
|
|
1025
1344
|
let ownsNativeValue = false;
|
|
1026
1345
|
try {
|
|
1027
1346
|
const err = this.bindings.lattice_edge_get_property(txn, edgeId, key, valueOut);
|
|
1028
|
-
if (err ===
|
|
1347
|
+
if (err === types_1.LatticeErrorCode.NotFound) {
|
|
1029
1348
|
return null;
|
|
1030
1349
|
}
|
|
1031
1350
|
this.checkError(err);
|
|
@@ -1107,10 +1426,16 @@ class LatticeFFI {
|
|
|
1107
1426
|
const edgeTypeLenOut = Buffer.alloc(4);
|
|
1108
1427
|
const err = this.bindings.lattice_edge_result_get(result, index, sourceOut, targetOut, edgeTypeOut, edgeTypeLenOut);
|
|
1109
1428
|
this.checkError(err);
|
|
1429
|
+
// The type is a (pointer, length) pair. Libraries before 0.17.0 did not
|
|
1430
|
+
// NUL-terminate it, so it is read by its length rather than as a C string.
|
|
1431
|
+
const edgeTypeLen = edgeTypeLenOut.readUInt32LE();
|
|
1432
|
+
const edgeType = edgeTypeOut[0] && edgeTypeLen > 0
|
|
1433
|
+
? Buffer.from(koffi_1.default.decode(edgeTypeOut[0], 'uint8_t', edgeTypeLen)).toString('utf8')
|
|
1434
|
+
: '';
|
|
1110
1435
|
return {
|
|
1111
1436
|
source: sourceOut.readBigUInt64LE(),
|
|
1112
1437
|
target: targetOut.readBigUInt64LE(),
|
|
1113
|
-
edgeType
|
|
1438
|
+
edgeType,
|
|
1114
1439
|
};
|
|
1115
1440
|
}
|
|
1116
1441
|
/**
|