@lancedb/lancedb 0.42.0-beta.2 → 0.42.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.js +3 -1
- package/dist/blob.d.ts +11 -0
- package/dist/blob.js +27 -0
- package/dist/compaction.d.ts +23 -0
- package/dist/compaction.js +21 -0
- package/dist/connection.d.ts +2 -2
- package/dist/index.d.ts +6 -4
- package/dist/job.d.ts +38 -3
- package/dist/job.js +55 -11
- package/dist/merge.d.ts +17 -0
- package/dist/merge.js +7 -1
- package/dist/native.d.ts +220 -6
- package/dist/native.js +52 -52
- package/dist/prewarm.d.ts +19 -0
- package/dist/prewarm.js +19 -0
- package/dist/query.d.ts +12 -2
- package/dist/query.js +55 -4
- package/dist/table.d.ts +245 -11
- package/dist/table.js +85 -8
- package/package.json +9 -9
- package/pnpm-workspace.yaml +0 -7
package/dist/table.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
5
|
exports.Branches = exports.LocalTable = exports.Table = void 0;
|
|
6
|
+
const compaction_1 = require("./compaction");
|
|
7
|
+
const prewarm_1 = require("./prewarm");
|
|
6
8
|
const arrow_1 = require("./arrow");
|
|
7
9
|
const blob_1 = require("./blob");
|
|
8
10
|
const registry_1 = require("./embedding/registry");
|
|
@@ -34,6 +36,25 @@ class Table {
|
|
|
34
36
|
[Symbol.for("nodejs.util.inspect.custom")]() {
|
|
35
37
|
return this.display();
|
|
36
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Submit remote compaction and return its durable job after acceptance.
|
|
41
|
+
*
|
|
42
|
+
* Uses SQL OPTIMIZE, inheriting branch and namespace from this table. Requires
|
|
43
|
+
* a compatible remote server and table Ownership plus parent Usage. Local and
|
|
44
|
+
* historical-version handles are unsupported. Existing optimize() behavior is unchanged.
|
|
45
|
+
* Waiting tracks the rewrite itself and returns actual row/file statistics.
|
|
46
|
+
* Missing result payloads are errors; older workers can omit file counts.
|
|
47
|
+
* Status, progress, events and cancellation reuse the generic HTTP job APIs.
|
|
48
|
+
* Reopen with db.openJob<CompactionResult>(job.id!). A wait timeout does not cancel work.
|
|
49
|
+
*
|
|
50
|
+
* @example
|
|
51
|
+
* const job = await table.compactAsync({ targetRowsPerFragment: 1000 });
|
|
52
|
+
* const result = await job.wait();
|
|
53
|
+
* console.log(result.rowsSucceeded, result.fragmentsRemoved);
|
|
54
|
+
*/
|
|
55
|
+
async compactAsync(_options) {
|
|
56
|
+
throw new Error("compaction jobs are only supported on remote tables");
|
|
57
|
+
}
|
|
37
58
|
}
|
|
38
59
|
exports.Table = Table;
|
|
39
60
|
class LocalTable extends Table {
|
|
@@ -67,6 +88,12 @@ class LocalTable extends Table {
|
|
|
67
88
|
return tbl.schema;
|
|
68
89
|
}
|
|
69
90
|
async add(data, options) {
|
|
91
|
+
if (options?.retries !== undefined &&
|
|
92
|
+
(!Number.isInteger(options.retries) ||
|
|
93
|
+
options.retries < 0 ||
|
|
94
|
+
options.retries > 255)) {
|
|
95
|
+
throw new Error("retries must be an integer between 0 and 255");
|
|
96
|
+
}
|
|
70
97
|
const mode = options?.mode ?? "append";
|
|
71
98
|
const schema = await this.schema();
|
|
72
99
|
const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
|
|
@@ -84,7 +111,7 @@ class LocalTable extends Table {
|
|
|
84
111
|
}
|
|
85
112
|
}
|
|
86
113
|
: undefined;
|
|
87
|
-
return await this.inner.add(buffer, mode, progress, options?.allowExternalBlobOutsideBases);
|
|
114
|
+
return await this.inner.add(buffer, mode, progress, options?.allowExternalBlobOutsideBases, options?.retries);
|
|
88
115
|
}
|
|
89
116
|
async update(optsOrUpdates, options) {
|
|
90
117
|
const isValues = "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
|
|
@@ -154,8 +181,18 @@ class LocalTable extends Table {
|
|
|
154
181
|
async prewarmIndex(name) {
|
|
155
182
|
await this.inner.prewarmIndex(name);
|
|
156
183
|
}
|
|
157
|
-
async prewarmData(
|
|
158
|
-
await this.
|
|
184
|
+
async prewarmData(columnsOrOptions) {
|
|
185
|
+
await this.prewarmDataAsync(Array.isArray(columnsOrOptions)
|
|
186
|
+
? { columns: columnsOrOptions }
|
|
187
|
+
: columnsOrOptions);
|
|
188
|
+
}
|
|
189
|
+
async prewarmDataAsync(options) {
|
|
190
|
+
for (const key of Object.keys(options ?? {})) {
|
|
191
|
+
if (!["columns", "filter", "concurrency"].includes(key)) {
|
|
192
|
+
throw new Error(`Unsupported data prewarm option: ${key}`);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return new job_1.Job(await this.inner.prewarmDataAsync(options), prewarm_1.decodePrewarmDataResult);
|
|
159
196
|
}
|
|
160
197
|
async waitForIndex(indexNames, timeoutSeconds) {
|
|
161
198
|
await this.inner.waitForIndex(indexNames, timeoutSeconds);
|
|
@@ -180,6 +217,16 @@ class LocalTable extends Table {
|
|
|
180
217
|
// N-API Option maps missing values to undefined. Collapse those to null.
|
|
181
218
|
return values.map((value) => value ?? null);
|
|
182
219
|
}
|
|
220
|
+
async fetchBlobRanges(column, requests, options) {
|
|
221
|
+
const rowIds = rowIdsToBigInts(requests.map((request) => request.rowId));
|
|
222
|
+
const ranges = requests.map((request, i) => ({
|
|
223
|
+
rowId: rowIds[i],
|
|
224
|
+
offset: request.offset,
|
|
225
|
+
length: request.length,
|
|
226
|
+
}));
|
|
227
|
+
const values = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobRanges(column, ranges, signal));
|
|
228
|
+
return values.map((value) => value ?? null);
|
|
229
|
+
}
|
|
183
230
|
async fetchBlobFiles(column, rowIds, options) {
|
|
184
231
|
const files = await (0, blob_1.runWithSignal)(options?.signal, (signal) => this.inner.fetchBlobFiles(column, rowIdsToBigInts(rowIds), signal));
|
|
185
232
|
// N-API Option maps missing values to undefined. Collapse those to null.
|
|
@@ -276,16 +323,26 @@ class LocalTable extends Table {
|
|
|
276
323
|
}
|
|
277
324
|
throw new Error("Invalid input type for addColumns");
|
|
278
325
|
}
|
|
279
|
-
async refreshColumn(column) {
|
|
280
|
-
return await this.inner.refreshColumn(column);
|
|
326
|
+
async refreshColumn(column, options) {
|
|
327
|
+
return await this.inner.refreshColumn(column, options);
|
|
281
328
|
}
|
|
282
|
-
async refreshColumnAsync(column) {
|
|
283
|
-
return new job_1.Job(await this.inner.refreshColumnAsync(column));
|
|
329
|
+
async refreshColumnAsync(column, options) {
|
|
330
|
+
return new job_1.Job(await this.inner.refreshColumnAsync(column, options));
|
|
284
331
|
}
|
|
285
332
|
async functionErrors(options) {
|
|
286
333
|
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
287
334
|
return await this.inner.functionErrors(options);
|
|
288
335
|
}
|
|
336
|
+
async functionColumnStatuses(column) {
|
|
337
|
+
return (await this.inner.functionColumnStatuses(column));
|
|
338
|
+
}
|
|
339
|
+
async functionColumnStatus(column) {
|
|
340
|
+
return (await this.inner.functionColumnStatus(column));
|
|
341
|
+
}
|
|
342
|
+
async waitFresh(column, timeoutSeconds) {
|
|
343
|
+
(0, materialized_view_1.validateNonNegativeInteger)(timeoutSeconds, "timeoutSeconds");
|
|
344
|
+
return (await this.inner.waitForFunctionColumn(column, timeoutSeconds));
|
|
345
|
+
}
|
|
289
346
|
async refreshMaterializedView(sourceVersion, full) {
|
|
290
347
|
return await this.inner.refreshMaterializedView(sourceVersion, full);
|
|
291
348
|
}
|
|
@@ -384,8 +441,13 @@ class LocalTable extends Table {
|
|
|
384
441
|
currentBranch() {
|
|
385
442
|
return this.inner.currentBranch() ?? null;
|
|
386
443
|
}
|
|
444
|
+
async compactAsync(options) {
|
|
445
|
+
validateCompactionOptions(options);
|
|
446
|
+
return new job_1.Job(await this.inner.compactAsync(options), compaction_1.decodeCompactionResult);
|
|
447
|
+
}
|
|
387
448
|
async optimize(options) {
|
|
388
|
-
|
|
449
|
+
validateCompactionOptions(options?.compactionOptions);
|
|
450
|
+
return await this.inner.optimize(options?.cleanupOlderThan?.getTime(), options?.deleteUnverified, options?.compactionOptions);
|
|
389
451
|
}
|
|
390
452
|
async listIndices() {
|
|
391
453
|
return await this.inner.listIndices();
|
|
@@ -521,3 +583,18 @@ function rowIdsToBigInts(rowIds) {
|
|
|
521
583
|
return BigInt(id);
|
|
522
584
|
});
|
|
523
585
|
}
|
|
586
|
+
function validateCompactionOptions(options) {
|
|
587
|
+
// JavaScript callers can supply extra keys that TypeScript would reject.
|
|
588
|
+
// Refuse them before the native binding can silently discard a work limit.
|
|
589
|
+
const supported = new Set([
|
|
590
|
+
"targetRowsPerFragment",
|
|
591
|
+
"maxSourceRows",
|
|
592
|
+
"ioBufferSize",
|
|
593
|
+
"batchSize",
|
|
594
|
+
]);
|
|
595
|
+
for (const key of Object.keys(options ?? {})) {
|
|
596
|
+
if (!supported.has(key)) {
|
|
597
|
+
throw new Error(`Unsupported compaction option: ${key}`);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
}
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"ann"
|
|
12
12
|
],
|
|
13
13
|
"private": false,
|
|
14
|
-
"version": "0.42.0-beta.
|
|
14
|
+
"version": "0.42.0-beta.20",
|
|
15
15
|
"main": "dist/index.js",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js",
|
|
@@ -109,15 +109,15 @@
|
|
|
109
109
|
"reflect-metadata": "^0.2.2"
|
|
110
110
|
},
|
|
111
111
|
"optionalDependencies": {
|
|
112
|
-
"@huggingface/transformers": "3.
|
|
112
|
+
"@huggingface/transformers": "4.3.2",
|
|
113
113
|
"openai": "4.29.2",
|
|
114
|
-
"@lancedb/lancedb-darwin-arm64": "0.42.0-beta.
|
|
115
|
-
"@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.
|
|
116
|
-
"@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.
|
|
117
|
-
"@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.
|
|
118
|
-
"@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.
|
|
119
|
-
"@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.
|
|
120
|
-
"@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.
|
|
114
|
+
"@lancedb/lancedb-darwin-arm64": "0.42.0-beta.20",
|
|
115
|
+
"@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.20",
|
|
116
|
+
"@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.20",
|
|
117
|
+
"@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.20",
|
|
118
|
+
"@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.20",
|
|
119
|
+
"@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.20",
|
|
120
|
+
"@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.20"
|
|
121
121
|
},
|
|
122
122
|
"peerDependencies": {
|
|
123
123
|
"@types/node": ">=22",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -47,10 +47,3 @@ minimumReleaseAgeExclude:
|
|
|
47
47
|
- js-yaml@4.3.1
|
|
48
48
|
- tar@7.5.21
|
|
49
49
|
- '@opentelemetry/core@2.8.0'
|
|
50
|
-
|
|
51
|
-
# @huggingface/transformers pins sharp ^0.33.5 and no released version has moved
|
|
52
|
-
# past ^0.34.5, all of which inherit the libvips CVEs in GHSA-f88m-g3jw-g9cj.
|
|
53
|
-
# Force the patched line. sharp is only reached by transformers' image pipeline,
|
|
54
|
-
# which LanceDB's text embedding function never uses.
|
|
55
|
-
overrides:
|
|
56
|
-
sharp: ^0.35.4
|