@lancedb/lancedb 0.42.0-beta.2 → 0.42.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RUST_THIRD_PARTY_LICENSES.html +11637 -6176
- package/dist/arrow.js +3 -1
- package/dist/blob.d.ts +11 -0
- package/dist/blob.js +27 -0
- package/dist/cleanup.d.ts +31 -0
- package/dist/cleanup.js +80 -0
- package/dist/compaction.d.ts +23 -0
- package/dist/compaction.js +21 -0
- package/dist/connection.d.ts +2 -2
- package/dist/index.d.ts +13 -8
- package/dist/index.js +3 -2
- package/dist/indices.d.ts +44 -0
- package/dist/indices.js +42 -0
- package/dist/job.d.ts +117 -4
- package/dist/job.js +190 -12
- package/dist/merge.d.ts +17 -0
- package/dist/merge.js +7 -1
- package/dist/native.d.ts +296 -9
- package/dist/native.js +53 -52
- package/dist/prewarm.d.ts +40 -0
- package/dist/prewarm.js +37 -0
- package/dist/profiling.d.ts +44 -0
- package/dist/profiling.js +37 -0
- package/dist/query.d.ts +12 -2
- package/dist/query.js +55 -4
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +61 -0
- package/dist/table.d.ts +439 -12
- package/dist/table.js +461 -18
- package/package.json +9 -9
- package/pnpm-workspace.yaml +0 -7
package/dist/table.js
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Branches = exports.LocalTable = exports.Table = void 0;
|
|
5
|
+
exports.Branches = exports.TableDelta = exports.LocalTable = exports.Table = void 0;
|
|
6
|
+
const cleanup_1 = require("./cleanup");
|
|
7
|
+
const compaction_1 = require("./compaction");
|
|
8
|
+
const prewarm_1 = require("./prewarm");
|
|
6
9
|
const arrow_1 = require("./arrow");
|
|
7
10
|
const blob_1 = require("./blob");
|
|
8
11
|
const registry_1 = require("./embedding/registry");
|
|
@@ -11,6 +14,7 @@ const materialized_view_1 = require("./materialized_view");
|
|
|
11
14
|
const merge_1 = require("./merge");
|
|
12
15
|
const query_1 = require("./query");
|
|
13
16
|
const sanitize_1 = require("./sanitize");
|
|
17
|
+
const sql_1 = require("./sql");
|
|
14
18
|
const util_1 = require("./util");
|
|
15
19
|
/**
|
|
16
20
|
* A Table is a collection of Records in a LanceDB Database.
|
|
@@ -34,6 +38,145 @@ class Table {
|
|
|
34
38
|
[Symbol.for("nodejs.util.inspect.custom")]() {
|
|
35
39
|
return this.display();
|
|
36
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Preview prewarm with the same options as {@link prewarmDataAsync}, without
|
|
43
|
+
* submitting a warming job. Requires LanceDB Enterprise with structured SQL
|
|
44
|
+
* previews, table Select and database/namespace Usage. Reads metadata and
|
|
45
|
+
* returns the source version, readable plan, and conservative pruning summary.
|
|
46
|
+
* A later submission plans its own snapshot. Historical and Embedded handles
|
|
47
|
+
* are unsupported. Unknown measurements remain null.
|
|
48
|
+
* Custom table implementations can override this method; its default rejects
|
|
49
|
+
* with an unsupported-operation error.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* const plan = await table.explainPrewarmData({
|
|
53
|
+
* columns: ["video"], filter: "quality > $minimum", parameters: {minimum: 0.7},
|
|
54
|
+
* });
|
|
55
|
+
* console.log(plan.pruning.selectedFragments);
|
|
56
|
+
*/
|
|
57
|
+
async explainPrewarmData(_options) {
|
|
58
|
+
throw new Error("data prewarm previews are not supported by this table implementation");
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Read the stored overrides for a remote Enterprise table.
|
|
62
|
+
*
|
|
63
|
+
* Overrides configure background behavior such as automatic compaction,
|
|
64
|
+
* cleanup, and reindex execution. Keys use the server's snake_case names.
|
|
65
|
+
* Returns an empty object when no overrides are set. Local tables are unsupported.
|
|
66
|
+
* Custom table implementations inherit a default rejection unless they override
|
|
67
|
+
* this method.
|
|
68
|
+
*/
|
|
69
|
+
async getTableOverrides() {
|
|
70
|
+
throw new Error("getTableOverrides is not supported on this table type");
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Recursively merge overrides into a remote Enterprise table's stored overrides.
|
|
74
|
+
*
|
|
75
|
+
* Fetches the current overrides, merges nested objects locally, then sends the
|
|
76
|
+
* complete replacement using REST PUT. Other values replace existing values;
|
|
77
|
+
* unrelated fields from the read snapshot are preserved. This read-modify-write
|
|
78
|
+
* is not atomic: concurrent edits can be overwritten, even to unrelated fields.
|
|
79
|
+
* Coordinate writers when concurrent edits must be preserved. An empty update
|
|
80
|
+
* only reads and returns the current overrides.
|
|
81
|
+
*
|
|
82
|
+
* Unlike this SDK method, CLI `set` and REST PUT replace the overrides object
|
|
83
|
+
* without merging. Use {@link Table.resetTableOverrides} to clear all overrides.
|
|
84
|
+
* Local tables and custom table implementations without override support reject
|
|
85
|
+
* this operation.
|
|
86
|
+
*
|
|
87
|
+
* @example
|
|
88
|
+
* ```ts
|
|
89
|
+
* await table.updateTableOverrides({
|
|
90
|
+
* job_types: { compaction: { enabled: false } },
|
|
91
|
+
* });
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
async updateTableOverrides(
|
|
95
|
+
// biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
|
|
96
|
+
overrides) {
|
|
97
|
+
throw new Error("updateTableOverrides is not supported on this table type");
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Clear a remote Enterprise table's overrides and return an empty object.
|
|
101
|
+
* Local tables and custom table implementations without override support reject
|
|
102
|
+
* this operation.
|
|
103
|
+
*/
|
|
104
|
+
async resetTableOverrides() {
|
|
105
|
+
throw new Error("resetTableOverrides is not supported on this table type");
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Compare two persisted versions or same-branch tags on an Embedded table.
|
|
109
|
+
*
|
|
110
|
+
* `fromVersion` is exclusive and `toVersion` is inclusive. Omitting
|
|
111
|
+
* `toVersion` uses the version currently visible through this handle,
|
|
112
|
+
* including a checked-out version. The endpoints are fixed when the delta is
|
|
113
|
+
* created.
|
|
114
|
+
*
|
|
115
|
+
* Row readers require stable row IDs at both endpoints. Create the table with
|
|
116
|
+
* `storageOptions: { new_table_enable_stable_row_ids: "true" }` to enable
|
|
117
|
+
* them. `listTransactions()` works without stable row IDs.
|
|
118
|
+
*
|
|
119
|
+
* Row readers reject ranges containing a Restore, or ranges whose transaction
|
|
120
|
+
* history cannot prove they are restore-free. Recompute the checkpoint with a
|
|
121
|
+
* full rescan and reinitialize it at the range end version.
|
|
122
|
+
*
|
|
123
|
+
* LanceDB Enterprise tables reject this operation.
|
|
124
|
+
*
|
|
125
|
+
* @example
|
|
126
|
+
* const delta = await table.delta("baseline");
|
|
127
|
+
* const reader = delta.insertedRows();
|
|
128
|
+
* console.log((await reader.schema()).fields.map((field) => field.name));
|
|
129
|
+
* for await (const batch of reader) {
|
|
130
|
+
* console.log(batch.getChild("id")?.toJSON());
|
|
131
|
+
* }
|
|
132
|
+
* console.log(await delta.listTransactions());
|
|
133
|
+
*/
|
|
134
|
+
async delta(
|
|
135
|
+
// biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
|
|
136
|
+
fromVersion,
|
|
137
|
+
// biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
|
|
138
|
+
toVersion) {
|
|
139
|
+
throw new Error("table.delta is only supported on LanceDB Embedded tables");
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Submit durable remote cleanup through SQL VACUUM and return after acceptance.
|
|
143
|
+
* Requires table Ownership and parent Usage. Retention inherits from the table
|
|
144
|
+
* then deployment when omitted. Local/historical handles are unsupported.
|
|
145
|
+
* Before submission, disable/drain automatic cleanup and prevent other cleanup,
|
|
146
|
+
* tag/branch publication and restore until completion, including direct clients.
|
|
147
|
+
* Coordination is not enforced by the server. Cancellation cannot undo deletion
|
|
148
|
+
* or prove storage I/O stopped. Preview is available without deletion.
|
|
149
|
+
* Wait timeout leaves work running; save the ID and reopen with db.openJob().
|
|
150
|
+
* @example
|
|
151
|
+
* const job = await table.cleanupAsync({ minAgeSeconds: 604800 });
|
|
152
|
+
* const result = await job.wait({ timeoutMs: 600000 });
|
|
153
|
+
*/
|
|
154
|
+
async cleanupAsync(_options) {
|
|
155
|
+
throw new Error("durable cleanup jobs are only supported on remote tables");
|
|
156
|
+
}
|
|
157
|
+
/** Preview version deletion without creating a job. Execution re-evaluates current state. */
|
|
158
|
+
async explainCleanup(_options) {
|
|
159
|
+
throw new Error("cleanup previews require LanceDB Enterprise");
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Submit remote compaction and return its durable job after acceptance.
|
|
163
|
+
*
|
|
164
|
+
* Uses SQL OPTIMIZE, inheriting branch and namespace from this table. Requires
|
|
165
|
+
* a compatible remote server and table Ownership plus parent Usage. Local and
|
|
166
|
+
* historical-version handles are unsupported. Existing optimize() behavior is unchanged.
|
|
167
|
+
* Waiting tracks the rewrite itself and returns actual row/file statistics.
|
|
168
|
+
* Missing result payloads are errors; older workers can omit file counts.
|
|
169
|
+
* Status, progress, events and cancellation reuse the generic HTTP job APIs.
|
|
170
|
+
* Reopen with db.openJob<CompactionResult>(job.id!). A wait timeout does not cancel work.
|
|
171
|
+
*
|
|
172
|
+
* @example
|
|
173
|
+
* const job = await table.compactAsync({ targetRowsPerFragment: 1000 });
|
|
174
|
+
* const result = await job.wait();
|
|
175
|
+
* console.log(result.rowsSucceeded, result.fragmentsRemoved);
|
|
176
|
+
*/
|
|
177
|
+
async compactAsync(_options) {
|
|
178
|
+
throw new Error("compaction jobs are only supported on remote tables");
|
|
179
|
+
}
|
|
37
180
|
}
|
|
38
181
|
exports.Table = Table;
|
|
39
182
|
class LocalTable extends Table {
|
|
@@ -67,6 +210,12 @@ class LocalTable extends Table {
|
|
|
67
210
|
return tbl.schema;
|
|
68
211
|
}
|
|
69
212
|
async add(data, options) {
|
|
213
|
+
if (options?.retries !== undefined &&
|
|
214
|
+
(!Number.isInteger(options.retries) ||
|
|
215
|
+
options.retries < 0 ||
|
|
216
|
+
options.retries > 255)) {
|
|
217
|
+
throw new Error("retries must be an integer between 0 and 255");
|
|
218
|
+
}
|
|
70
219
|
const mode = options?.mode ?? "append";
|
|
71
220
|
const schema = await this.schema();
|
|
72
221
|
const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
|
|
@@ -84,7 +233,7 @@ class LocalTable extends Table {
|
|
|
84
233
|
}
|
|
85
234
|
}
|
|
86
235
|
: undefined;
|
|
87
|
-
return await this.inner.add(buffer, mode, progress, options?.allowExternalBlobOutsideBases);
|
|
236
|
+
return await this.inner.add(buffer, mode, progress, options?.allowExternalBlobOutsideBases, options?.retries);
|
|
88
237
|
}
|
|
89
238
|
async update(optsOrUpdates, options) {
|
|
90
239
|
const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
|
|
@@ -154,8 +303,16 @@ class LocalTable extends Table {
|
|
|
154
303
|
async prewarmIndex(name) {
|
|
155
304
|
await this.inner.prewarmIndex(name);
|
|
156
305
|
}
|
|
157
|
-
async prewarmData(
|
|
158
|
-
await this.
|
|
306
|
+
async prewarmData(columnsOrOptions) {
|
|
307
|
+
await this.prewarmDataAsync(Array.isArray(columnsOrOptions)
|
|
308
|
+
? { columns: columnsOrOptions }
|
|
309
|
+
: columnsOrOptions);
|
|
310
|
+
}
|
|
311
|
+
async prewarmDataAsync(options) {
|
|
312
|
+
return new job_1.Job(await this.inner.prewarmDataAsync(await toNativePrewarmDataOptions(options)), prewarm_1.decodePrewarmDataResult);
|
|
313
|
+
}
|
|
314
|
+
async explainPrewarmData(options) {
|
|
315
|
+
return (0, prewarm_1.decodePrewarmDataPlan)(await this.inner.explainPrewarmData(await toNativePrewarmDataOptions(options)));
|
|
159
316
|
}
|
|
160
317
|
async waitForIndex(indexNames, timeoutSeconds) {
|
|
161
318
|
await this.inner.waitForIndex(indexNames, timeoutSeconds);
|
|
@@ -180,6 +337,16 @@ class LocalTable extends Table {
|
|
|
180
337
|
// N-API Option maps missing values to undefined. Collapse those to null.
|
|
181
338
|
return values.map((value) => value ?? null);
|
|
182
339
|
}
|
|
340
|
+
async fetchBlobRanges(column, requests, options) {
|
|
341
|
+
const rowIds = rowIdsToBigInts(requests.map((request) => request.rowId));
|
|
342
|
+
const ranges = requests.map((request, i) => ({
|
|
343
|
+
rowId: rowIds[i],
|
|
344
|
+
offset: request.offset,
|
|
345
|
+
length: request.length,
|
|
346
|
+
}));
|
|
347
|
+
const values = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobRanges(column, ranges, signal));
|
|
348
|
+
return values.map((value) => value ?? null);
|
|
349
|
+
}
|
|
183
350
|
async fetchBlobFiles(column, rowIds, options) {
|
|
184
351
|
const files = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal));
|
|
185
352
|
// N-API Option maps missing values to undefined. Collapse those to null.
|
|
@@ -276,16 +443,26 @@ class LocalTable extends Table {
|
|
|
276
443
|
}
|
|
277
444
|
throw new Error("Invalid input type for addColumns");
|
|
278
445
|
}
|
|
279
|
-
async refreshColumn(column) {
|
|
280
|
-
return await this.inner.refreshColumn(column);
|
|
446
|
+
async refreshColumn(column, options) {
|
|
447
|
+
return await this.inner.refreshColumn(column, options);
|
|
281
448
|
}
|
|
282
|
-
async refreshColumnAsync(column) {
|
|
283
|
-
return new job_1.Job(await this.inner.refreshColumnAsync(column));
|
|
449
|
+
async refreshColumnAsync(column, options) {
|
|
450
|
+
return new job_1.Job(await this.inner.refreshColumnAsync(column, options));
|
|
284
451
|
}
|
|
285
452
|
async functionErrors(options) {
|
|
286
453
|
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
287
454
|
return await this.inner.functionErrors(options);
|
|
288
455
|
}
|
|
456
|
+
async functionColumnStatuses(column) {
|
|
457
|
+
return (await this.inner.functionColumnStatuses(column));
|
|
458
|
+
}
|
|
459
|
+
async functionColumnStatus(column) {
|
|
460
|
+
return (await this.inner.functionColumnStatus(column));
|
|
461
|
+
}
|
|
462
|
+
async waitFresh(column, timeoutSeconds) {
|
|
463
|
+
(0, materialized_view_1.validateNonNegativeInteger)(timeoutSeconds, "timeoutSeconds");
|
|
464
|
+
return (await this.inner.waitForFunctionColumn(column, timeoutSeconds));
|
|
465
|
+
}
|
|
289
466
|
async refreshMaterializedView(sourceVersion, full) {
|
|
290
467
|
return await this.inner.refreshMaterializedView(sourceVersion, full);
|
|
291
468
|
}
|
|
@@ -338,6 +515,15 @@ class LocalTable extends Table {
|
|
|
338
515
|
// three valid values, so the cast is safe.
|
|
339
516
|
return ((await this.inner.getRTESpec()) ?? undefined);
|
|
340
517
|
}
|
|
518
|
+
async getTableOverrides() {
|
|
519
|
+
return await this.inner.getTableOverrides();
|
|
520
|
+
}
|
|
521
|
+
async updateTableOverrides(overrides) {
|
|
522
|
+
return await this.inner.updateTableOverrides(overrides);
|
|
523
|
+
}
|
|
524
|
+
async resetTableOverrides() {
|
|
525
|
+
return await this.inner.resetTableOverrides();
|
|
526
|
+
}
|
|
341
527
|
async closeRTEWriters() {
|
|
342
528
|
return await this.inner.closeRTEWriters();
|
|
343
529
|
}
|
|
@@ -365,6 +551,9 @@ class LocalTable extends Table {
|
|
|
365
551
|
async checkoutLatest() {
|
|
366
552
|
await this.inner.checkoutLatest();
|
|
367
553
|
}
|
|
554
|
+
async delta(fromVersion, toVersion) {
|
|
555
|
+
return new TableDelta(await this.inner.delta(toNativeDeltaRef(fromVersion), toVersion === undefined ? undefined : toNativeDeltaRef(toVersion)));
|
|
556
|
+
}
|
|
368
557
|
async listVersions() {
|
|
369
558
|
return (await this.inner.listVersions()).map((version) => ({
|
|
370
559
|
version: version.version,
|
|
@@ -384,8 +573,21 @@ class LocalTable extends Table {
|
|
|
384
573
|
currentBranch() {
|
|
385
574
|
return this.inner.currentBranch() ?? null;
|
|
386
575
|
}
|
|
576
|
+
async compactAsync(options) {
|
|
577
|
+
validateCompactionOptions(options);
|
|
578
|
+
return new job_1.Job(await this.inner.compactAsync(options), compaction_1.decodeCompactionResult);
|
|
579
|
+
}
|
|
580
|
+
async cleanupAsync(options) {
|
|
581
|
+
(0, cleanup_1.validateCleanupOptions)(options);
|
|
582
|
+
return new job_1.Job(await this.inner.cleanupAsync(options?.minAgeSeconds), cleanup_1.decodeCleanupResult);
|
|
583
|
+
}
|
|
584
|
+
async explainCleanup(options) {
|
|
585
|
+
(0, cleanup_1.validateCleanupOptions)(options);
|
|
586
|
+
return (0, cleanup_1.decodeCleanupPlan)(await this.inner.explainCleanup(options?.minAgeSeconds));
|
|
587
|
+
}
|
|
387
588
|
async optimize(options) {
|
|
388
|
-
|
|
589
|
+
validateCompactionOptions(options?.compactionOptions);
|
|
590
|
+
return await this.inner.optimize(options?.cleanupOlderThan?.getTime(), options?.deleteUnverified, options?.compactionOptions);
|
|
389
591
|
}
|
|
390
592
|
async listIndices() {
|
|
391
593
|
return await this.inner.listIndices();
|
|
@@ -443,6 +645,211 @@ class LocalTable extends Table {
|
|
|
443
645
|
}
|
|
444
646
|
}
|
|
445
647
|
exports.LocalTable = LocalTable;
|
|
648
|
+
/**
|
|
649
|
+
* A handle for reading changes between two pinned table versions.
|
|
650
|
+
*
|
|
651
|
+
* Created by {@link Table.delta}. Endpoints are fixed at creation; later writes
|
|
652
|
+
* or tag moves do not change this delta.
|
|
653
|
+
*/
|
|
654
|
+
class TableDelta {
|
|
655
|
+
#inner;
|
|
656
|
+
/**
|
|
657
|
+
* Construct a table-delta handle. Internal use only.
|
|
658
|
+
* @hidden
|
|
659
|
+
*/
|
|
660
|
+
constructor(inner) {
|
|
661
|
+
this.#inner = inner;
|
|
662
|
+
}
|
|
663
|
+
/** The exclusive start version, as an exact u64 bigint. */
|
|
664
|
+
get beginVersion() {
|
|
665
|
+
return this.#inner.beginVersion();
|
|
666
|
+
}
|
|
667
|
+
/** The inclusive end version, as an exact u64 bigint. */
|
|
668
|
+
get endVersion() {
|
|
669
|
+
return this.#inner.endVersion();
|
|
670
|
+
}
|
|
671
|
+
/** Stream rows inserted in `(beginVersion, endVersion]`. */
|
|
672
|
+
insertedRows() {
|
|
673
|
+
return tableDeltaReader(() => this.#inner.insertedRows());
|
|
674
|
+
}
|
|
675
|
+
/** Stream rows updated in `(beginVersion, endVersion]`. */
|
|
676
|
+
updatedRows() {
|
|
677
|
+
return tableDeltaReader(() => this.#inner.updatedRows());
|
|
678
|
+
}
|
|
679
|
+
/** Stream inserted and updated rows in `(beginVersion, endVersion]`. */
|
|
680
|
+
upsertedRows() {
|
|
681
|
+
return tableDeltaReader(() => this.#inner.upsertedRows());
|
|
682
|
+
}
|
|
683
|
+
/** Stream deleted row IDs. Each batch has one `_rowid: uint64` column. */
|
|
684
|
+
deletedRowIds() {
|
|
685
|
+
return tableDeltaReader(() => this.#inner.deletedRowIds());
|
|
686
|
+
}
|
|
687
|
+
/** List readable transactions committed in `(beginVersion, endVersion]`. */
|
|
688
|
+
async listTransactions() {
|
|
689
|
+
const transactions = await this.#inner.listTransactions();
|
|
690
|
+
return transactions.map((transaction) => ({
|
|
691
|
+
readVersion: transaction.readVersion,
|
|
692
|
+
uuid: transaction.uuid,
|
|
693
|
+
operation: transaction.operation,
|
|
694
|
+
tag: transaction.tag ?? undefined,
|
|
695
|
+
transactionProperties: transaction.transactionProperties ?? undefined,
|
|
696
|
+
}));
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
exports.TableDelta = TableDelta;
|
|
700
|
+
function toNativeDeltaRef(value) {
|
|
701
|
+
if (typeof value === "string") {
|
|
702
|
+
if (value.length === 0) {
|
|
703
|
+
throw new Error("delta tag must be a non-empty string");
|
|
704
|
+
}
|
|
705
|
+
return { tag: value };
|
|
706
|
+
}
|
|
707
|
+
if (typeof value === "bigint") {
|
|
708
|
+
if (value <= 0n) {
|
|
709
|
+
throw new Error("delta version must be a positive u64 bigint");
|
|
710
|
+
}
|
|
711
|
+
return { version: value };
|
|
712
|
+
}
|
|
713
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
714
|
+
throw new Error("delta version must be a positive safe integer; use bigint for larger versions");
|
|
715
|
+
}
|
|
716
|
+
return { version: BigInt(value) };
|
|
717
|
+
}
|
|
718
|
+
function tableDeltaReader(open) {
|
|
719
|
+
return new TableDeltaReaderImpl(open);
|
|
720
|
+
}
|
|
721
|
+
class TableDeltaReaderImpl {
|
|
722
|
+
#open;
|
|
723
|
+
#setup;
|
|
724
|
+
#schema;
|
|
725
|
+
#schemaSetup;
|
|
726
|
+
#iterator;
|
|
727
|
+
#error;
|
|
728
|
+
#done = false;
|
|
729
|
+
#pullQueue = Promise.resolve();
|
|
730
|
+
constructor(open) {
|
|
731
|
+
this.#open = open;
|
|
732
|
+
}
|
|
733
|
+
[Symbol.asyncIterator]() {
|
|
734
|
+
return this;
|
|
735
|
+
}
|
|
736
|
+
async schema() {
|
|
737
|
+
if (this.#schema !== undefined) {
|
|
738
|
+
return this.#schema;
|
|
739
|
+
}
|
|
740
|
+
if (this.#error !== undefined) {
|
|
741
|
+
throw this.#error;
|
|
742
|
+
}
|
|
743
|
+
if (this.#schemaSetup === undefined) {
|
|
744
|
+
this.#schemaSetup = this.#loadSchema();
|
|
745
|
+
}
|
|
746
|
+
return this.#schemaSetup;
|
|
747
|
+
}
|
|
748
|
+
async #loadSchema() {
|
|
749
|
+
try {
|
|
750
|
+
const native = await this.#getNative();
|
|
751
|
+
const schema = (0, arrow_1.tableFromIPC)(await native.schema()).schema;
|
|
752
|
+
this.#schema = schema;
|
|
753
|
+
return schema;
|
|
754
|
+
}
|
|
755
|
+
catch (error) {
|
|
756
|
+
return this.#fail(error);
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
next() {
|
|
760
|
+
return this.#serialize(() => this.#pull());
|
|
761
|
+
}
|
|
762
|
+
async return() {
|
|
763
|
+
await this.#serialize(async () => {
|
|
764
|
+
this.#done = true;
|
|
765
|
+
await this.#release();
|
|
766
|
+
});
|
|
767
|
+
return { done: true, value: undefined };
|
|
768
|
+
}
|
|
769
|
+
async throw(error) {
|
|
770
|
+
await this.#serialize(async () => {
|
|
771
|
+
this.#done = true;
|
|
772
|
+
await this.#release();
|
|
773
|
+
});
|
|
774
|
+
throw error;
|
|
775
|
+
}
|
|
776
|
+
async #pull() {
|
|
777
|
+
if (this.#error !== undefined) {
|
|
778
|
+
throw this.#error;
|
|
779
|
+
}
|
|
780
|
+
if (this.#done) {
|
|
781
|
+
return { done: true, value: undefined };
|
|
782
|
+
}
|
|
783
|
+
// Capture the schema before polling the reader's shared native iterator.
|
|
784
|
+
await this.schema();
|
|
785
|
+
try {
|
|
786
|
+
const result = await this.#getIterator().next();
|
|
787
|
+
if (result.done) {
|
|
788
|
+
this.#done = true;
|
|
789
|
+
await this.#release();
|
|
790
|
+
}
|
|
791
|
+
return result;
|
|
792
|
+
}
|
|
793
|
+
catch (error) {
|
|
794
|
+
return this.#fail(error);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
#getIterator() {
|
|
798
|
+
if (this.#iterator === undefined) {
|
|
799
|
+
this.#iterator = (0, query_1.RecordBatchIterator)(this.#getNative())[Symbol.asyncIterator]();
|
|
800
|
+
}
|
|
801
|
+
return this.#iterator;
|
|
802
|
+
}
|
|
803
|
+
async #getNative() {
|
|
804
|
+
if (this.#error !== undefined) {
|
|
805
|
+
throw this.#error;
|
|
806
|
+
}
|
|
807
|
+
if (this.#done) {
|
|
808
|
+
throw new Error("table delta reader is already consumed");
|
|
809
|
+
}
|
|
810
|
+
if (this.#setup === undefined) {
|
|
811
|
+
try {
|
|
812
|
+
this.#setup = this.#open();
|
|
813
|
+
}
|
|
814
|
+
catch (error) {
|
|
815
|
+
return this.#fail(error);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
try {
|
|
819
|
+
return await this.#setup;
|
|
820
|
+
}
|
|
821
|
+
catch (error) {
|
|
822
|
+
return this.#fail(error);
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
async #fail(error) {
|
|
826
|
+
if (this.#error === undefined) {
|
|
827
|
+
this.#done = true;
|
|
828
|
+
this.#error = error;
|
|
829
|
+
await this.#release();
|
|
830
|
+
}
|
|
831
|
+
throw this.#error;
|
|
832
|
+
}
|
|
833
|
+
async #release() {
|
|
834
|
+
const iterator = this.#iterator;
|
|
835
|
+
this.#iterator = undefined;
|
|
836
|
+
this.#setup = undefined;
|
|
837
|
+
if (iterator?.return !== undefined) {
|
|
838
|
+
try {
|
|
839
|
+
await iterator.return();
|
|
840
|
+
}
|
|
841
|
+
catch {
|
|
842
|
+
// The native stream already failed or completed; releasing the
|
|
843
|
+
// reference is enough.
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
#serialize(operation) {
|
|
848
|
+
const next = this.#pullQueue.then(operation, operation);
|
|
849
|
+
this.#pullQueue = next.then(() => undefined, () => undefined);
|
|
850
|
+
return next;
|
|
851
|
+
}
|
|
852
|
+
}
|
|
446
853
|
/**
|
|
447
854
|
* Branch manager for a {@link Table}.
|
|
448
855
|
*
|
|
@@ -462,15 +869,18 @@ class Branches {
|
|
|
462
869
|
async list() {
|
|
463
870
|
return await this.#inner.list();
|
|
464
871
|
}
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
872
|
+
async create(name, fromRefOrOptions, fromVersion) {
|
|
873
|
+
if (typeof fromRefOrOptions === "object" && fromRefOrOptions !== null) {
|
|
874
|
+
// The public options shape only adds `fromTag`. Forward any runtime
|
|
875
|
+
// `fromRef`/`fromVersion` fields so the native check remains the single
|
|
876
|
+
// authority for rejecting mixed sources before `main` normalization.
|
|
877
|
+
const options = fromRefOrOptions;
|
|
878
|
+
if (typeof options.fromTag !== "string") {
|
|
879
|
+
throw new Error("branch source options must include fromTag");
|
|
880
|
+
}
|
|
881
|
+
return new LocalTable(await this.#inner.create(name, options.fromRef, fromVersion ?? options.fromVersion, options.fromTag));
|
|
882
|
+
}
|
|
883
|
+
return new LocalTable(await this.#inner.create(name, fromRefOrOptions, fromVersion));
|
|
474
884
|
}
|
|
475
885
|
/**
|
|
476
886
|
* Check out an existing branch and return a handle scoped to it.
|
|
@@ -521,3 +931,36 @@ function rowIdsToBigInts(rowIds) {
|
|
|
521
931
|
return BigInt(id);
|
|
522
932
|
});
|
|
523
933
|
}
|
|
934
|
+
function validateCompactionOptions(options) {
|
|
935
|
+
// JavaScript callers can supply extra keys that TypeScript would reject.
|
|
936
|
+
// Refuse them before the native binding can silently discard a work limit.
|
|
937
|
+
const supported = new Set([
|
|
938
|
+
"targetRowsPerFragment",
|
|
939
|
+
"maxSourceRows",
|
|
940
|
+
"ioBufferSize",
|
|
941
|
+
"batchSize",
|
|
942
|
+
]);
|
|
943
|
+
for (const key of Object.keys(options ?? {})) {
|
|
944
|
+
if (!supported.has(key)) {
|
|
945
|
+
throw new Error(`Unsupported compaction option: ${key}`);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
async function toNativePrewarmDataOptions(options) {
|
|
950
|
+
validatePrewarmDataOptions(options);
|
|
951
|
+
return options === undefined
|
|
952
|
+
? undefined
|
|
953
|
+
: {
|
|
954
|
+
...options,
|
|
955
|
+
parameters: options.parameters === undefined
|
|
956
|
+
? undefined
|
|
957
|
+
: await (0, sql_1.toParameterBuffer)(options.parameters),
|
|
958
|
+
};
|
|
959
|
+
}
|
|
960
|
+
function validatePrewarmDataOptions(options) {
|
|
961
|
+
for (const key of Object.keys(options ?? {})) {
|
|
962
|
+
if (!["columns", "filter", "concurrency", "parameters"].includes(key)) {
|
|
963
|
+
throw new Error(`Unsupported data prewarm option: ${key}`);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"ann"
|
|
12
12
|
],
|
|
13
13
|
"private": false,
|
|
14
|
-
"version": "0.42.0-beta.
|
|
14
|
+
"version": "0.42.0-beta.21",
|
|
15
15
|
"main": "dist/index.js",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js",
|
|
@@ -109,15 +109,15 @@
|
|
|
109
109
|
"reflect-metadata": "^0.2.2"
|
|
110
110
|
},
|
|
111
111
|
"optionalDependencies": {
|
|
112
|
-
"@huggingface/transformers": "3.
|
|
112
|
+
"@huggingface/transformers": "4.3.2",
|
|
113
113
|
"openai": "4.29.2",
|
|
114
|
-
"@lancedb/lancedb-darwin-arm64": "0.42.0-beta.
|
|
115
|
-
"@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.
|
|
116
|
-
"@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.
|
|
117
|
-
"@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.
|
|
118
|
-
"@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.
|
|
119
|
-
"@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.
|
|
120
|
-
"@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.
|
|
114
|
+
"@lancedb/lancedb-darwin-arm64": "0.42.0-beta.21",
|
|
115
|
+
"@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.21",
|
|
116
|
+
"@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.21",
|
|
117
|
+
"@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.21",
|
|
118
|
+
"@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.21",
|
|
119
|
+
"@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.21",
|
|
120
|
+
"@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.21"
|
|
121
121
|
},
|
|
122
122
|
"peerDependencies": {
|
|
123
123
|
"@types/node": ">=22",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -47,10 +47,3 @@ minimumReleaseAgeExclude:
|
|
|
47
47
|
- js-yaml@4.3.1
|
|
48
48
|
- tar@7.5.21
|
|
49
49
|
- '@opentelemetry/core@2.8.0'
|
|
50
|
-
|
|
51
|
-
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
|
|
52
|
-
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
|
|
53
|
-
# Force the patched line. sharp is only reached by transformers' image pipeline,
|
|
54
|
-
# which LanceDB's text embedding function never uses.
|
|
55
|
-
overrides:
|
|
56
|
-
sharp: ^0.35.4
|