@lancedb/lancedb 0.41.0-beta.7 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +13 -0
- package/dist/arrow.d.ts +1 -0
- package/dist/arrow.js +16 -5
- package/dist/blob.d.ts +24 -1
- package/dist/blob.js +56 -5
- package/dist/catalog.d.ts +6 -1
- package/dist/catalog.js +6 -2
- package/dist/connection.d.ts +26 -79
- package/dist/connection.js +28 -26
- package/dist/embedding/transformers.js +4 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.js +8 -5
- package/dist/indices.d.ts +2 -1
- package/dist/job.d.ts +8 -5
- package/dist/job.js +8 -5
- package/dist/materialized_view.d.ts +7 -3
- package/dist/materialized_view.js +7 -4
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +128 -129
- package/dist/native.js +53 -52
- package/dist/query.d.ts +68 -20
- package/dist/query.js +181 -36
- package/dist/table.d.ts +64 -55
- package/dist/table.js +26 -20
- package/package.json +10 -9
package/dist/query.js
CHANGED
|
@@ -21,6 +21,99 @@ async function* RecordBatchIterator(promisedInner) {
|
|
|
21
21
|
yield batches[0];
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
+
const BLOB_BYTES_ATTEMPTS = 3;
|
|
25
|
+
function blobSources(blobColumns, projection) {
|
|
26
|
+
if (projection === undefined) {
|
|
27
|
+
return blobColumns.map((source) => {
|
|
28
|
+
const [output, ...path] = source.split(".");
|
|
29
|
+
return { output, path, source };
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
const sources = [];
|
|
33
|
+
for (const [output, source] of projection) {
|
|
34
|
+
for (const column of blobColumns) {
|
|
35
|
+
if (column === source || column.startsWith(`${source}.`)) {
|
|
36
|
+
sources.push({
|
|
37
|
+
output,
|
|
38
|
+
path: column === source ? [] : column.slice(source.length + 1).split("."),
|
|
39
|
+
source: column,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return sources;
|
|
45
|
+
}
|
|
46
|
+
async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
47
|
+
const rowIdColumn = table.getChild("_rowid");
|
|
48
|
+
if (table.numRows > 0 && rowIdColumn === null) {
|
|
49
|
+
throw new Error('blobMode "bytes" needs _rowid in the query result');
|
|
50
|
+
}
|
|
51
|
+
const rowIds = rowIdColumn === null
|
|
52
|
+
? []
|
|
53
|
+
: Array.from(rowIdColumn.toArray());
|
|
54
|
+
const outputFields = new Map(table.schema.fields.map((field) => [field.name, field]));
|
|
55
|
+
const replacements = new Map();
|
|
56
|
+
for (const { output, path, source } of sources) {
|
|
57
|
+
let field = outputFields.get(output);
|
|
58
|
+
for (const name of path) {
|
|
59
|
+
field =
|
|
60
|
+
field !== undefined && arrow_1.DataType.isStruct(field.type)
|
|
61
|
+
? field.type.children.find((child) => child.name === name)
|
|
62
|
+
: undefined;
|
|
63
|
+
}
|
|
64
|
+
if (field === undefined) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
|
|
68
|
+
replacements.set(field, values.map((value) => value ?? null));
|
|
69
|
+
}
|
|
70
|
+
const result = withBinaryColumns(table, replacements);
|
|
71
|
+
if (!dropRowId) {
|
|
72
|
+
return result;
|
|
73
|
+
}
|
|
74
|
+
return result.select(result.schema.fields
|
|
75
|
+
.map((field) => field.name)
|
|
76
|
+
.filter((name) => name !== "_rowid"));
|
|
77
|
+
}
|
|
78
|
+
function withBinaryColumns(table, replacements) {
|
|
79
|
+
if (replacements.size === 0) {
|
|
80
|
+
return table;
|
|
81
|
+
}
|
|
82
|
+
const replaceField = (field) => {
|
|
83
|
+
if (replacements.has(field)) {
|
|
84
|
+
// The descriptor's extension and encoding metadata must not follow bytes.
|
|
85
|
+
return new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true);
|
|
86
|
+
}
|
|
87
|
+
if (arrow_1.DataType.isStruct(field.type)) {
|
|
88
|
+
const children = field.type.children.map(replaceField);
|
|
89
|
+
if (children.some((child, i) => child !== field.type.children[i])) {
|
|
90
|
+
return field.clone({ type: new arrow_1.Struct(children) });
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return field;
|
|
94
|
+
};
|
|
95
|
+
const fields = table.schema.fields.map(replaceField);
|
|
96
|
+
const schema = new arrow_1.Schema(fields, table.schema.metadata);
|
|
97
|
+
let offset = 0;
|
|
98
|
+
const batches = table.batches.map((batch) => {
|
|
99
|
+
const replaceData = (data, field, updated) => {
|
|
100
|
+
if (field === updated) {
|
|
101
|
+
return data;
|
|
102
|
+
}
|
|
103
|
+
const values = replacements.get(field);
|
|
104
|
+
if (values !== undefined) {
|
|
105
|
+
return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
|
|
106
|
+
}
|
|
107
|
+
const children = data.children.map((child, index) => replaceData(child, field.type.children[index], updated.type.children[index]));
|
|
108
|
+
// Keep the struct's validity bitmap and offset, including null parents.
|
|
109
|
+
return data.clone(updated.type, undefined, undefined, undefined, undefined, children);
|
|
110
|
+
};
|
|
111
|
+
const children = batch.data.children.map((child, index) => replaceData(child, table.schema.fields[index], fields[index]));
|
|
112
|
+
offset += batch.numRows;
|
|
113
|
+
return new arrow_1.RecordBatch(schema, batch.data.clone(new arrow_1.Struct(fields), undefined, undefined, undefined, undefined, children));
|
|
114
|
+
});
|
|
115
|
+
return new arrow_1.Table(schema, batches);
|
|
116
|
+
}
|
|
24
117
|
function nearestToNative(inner, vector) {
|
|
25
118
|
const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
|
|
26
119
|
if (raw) {
|
|
@@ -49,10 +142,15 @@ class QueryBase {
|
|
|
49
142
|
/**
|
|
50
143
|
* @hidden
|
|
51
144
|
*/
|
|
52
|
-
|
|
145
|
+
blobContext;
|
|
146
|
+
/**
|
|
147
|
+
* @hidden
|
|
148
|
+
*/
|
|
149
|
+
constructor(inner, blobContext) {
|
|
53
150
|
if (inner !== undefined) {
|
|
54
151
|
this.inner = inner;
|
|
55
152
|
}
|
|
153
|
+
this.blobContext = blobContext;
|
|
56
154
|
}
|
|
57
155
|
// call a function on the inner (either a promise or the actual object)
|
|
58
156
|
/**
|
|
@@ -118,17 +216,25 @@ class QueryBase {
|
|
|
118
216
|
inner.select(columnTuples);
|
|
119
217
|
});
|
|
120
218
|
};
|
|
219
|
+
let projection;
|
|
121
220
|
if (typeof columns === "string") {
|
|
122
221
|
selectColumns([columns]);
|
|
222
|
+
projection = [[columns, columns]];
|
|
123
223
|
}
|
|
124
224
|
else if (Array.isArray(columns)) {
|
|
125
225
|
selectColumns(columns);
|
|
226
|
+
projection = columns.map((column) => [column, column]);
|
|
126
227
|
}
|
|
127
228
|
else if (columns instanceof Map) {
|
|
128
|
-
|
|
229
|
+
projection = Array.from(columns.entries());
|
|
230
|
+
selectMapping(projection);
|
|
129
231
|
}
|
|
130
232
|
else {
|
|
131
|
-
|
|
233
|
+
projection = Object.entries(columns);
|
|
234
|
+
selectMapping(projection);
|
|
235
|
+
}
|
|
236
|
+
if (this.blobContext !== undefined) {
|
|
237
|
+
this.blobContext = { ...this.blobContext, projection };
|
|
132
238
|
}
|
|
133
239
|
return this;
|
|
134
240
|
}
|
|
@@ -141,14 +247,17 @@ class QueryBase {
|
|
|
141
247
|
*/
|
|
142
248
|
withRowId() {
|
|
143
249
|
this.doCall((inner) => inner.withRowId());
|
|
250
|
+
if (this.blobContext !== undefined) {
|
|
251
|
+
this.blobContext = { ...this.blobContext, withRowId: true };
|
|
252
|
+
}
|
|
144
253
|
return this;
|
|
145
254
|
}
|
|
146
255
|
/**
|
|
147
256
|
* @hidden
|
|
148
257
|
*/
|
|
149
|
-
async nativeExecute(options) {
|
|
258
|
+
async nativeExecute(options, withRowId = false) {
|
|
150
259
|
const inner = await this.getInner();
|
|
151
|
-
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
|
260
|
+
return inner.execute(options?.maxBatchLength, options?.timeoutMs, withRowId);
|
|
152
261
|
}
|
|
153
262
|
/**
|
|
154
263
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
@@ -177,8 +286,32 @@ class QueryBase {
|
|
|
177
286
|
* Empty results retain the query's output schema.
|
|
178
287
|
*/
|
|
179
288
|
async toArrow(options) {
|
|
289
|
+
const context = this.blobBytesContext(options?.blobMode);
|
|
290
|
+
if (context === undefined) {
|
|
291
|
+
return this.collect(options, false);
|
|
292
|
+
}
|
|
293
|
+
for (let attempt = 1;; attempt++) {
|
|
294
|
+
// The snapshot pins the bytes. The query runs on the live table, so an
|
|
295
|
+
// unchanged version afterwards proves it read the snapshot's version.
|
|
296
|
+
const snapshot = await context.table.querySnapshot();
|
|
297
|
+
const version = await snapshot.version();
|
|
298
|
+
const sources = blobSources(await snapshot.blobColumns(), context.projection);
|
|
299
|
+
if (sources.length === 0) {
|
|
300
|
+
return this.collect(options, false);
|
|
301
|
+
}
|
|
302
|
+
const addRowId = !context.withRowId;
|
|
303
|
+
const table = await this.collect(options, addRowId);
|
|
304
|
+
if ((await context.table.version()) === version) {
|
|
305
|
+
return readBlobBytes(table, sources, snapshot, addRowId);
|
|
306
|
+
}
|
|
307
|
+
if (attempt === BLOB_BYTES_ATTEMPTS) {
|
|
308
|
+
throw new Error(`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
async collect(options, withRowId) {
|
|
180
313
|
const batches = [];
|
|
181
|
-
const iterator = this.nativeExecute(options);
|
|
314
|
+
const iterator = this.nativeExecute(options, withRowId);
|
|
182
315
|
for await (const batch of RecordBatchIterator(iterator)) {
|
|
183
316
|
batches.push(batch);
|
|
184
317
|
}
|
|
@@ -187,6 +320,18 @@ class QueryBase {
|
|
|
187
320
|
}
|
|
188
321
|
return new arrow_1.Table(batches);
|
|
189
322
|
}
|
|
323
|
+
blobBytesContext(mode) {
|
|
324
|
+
if (mode === undefined || mode === "descriptions") {
|
|
325
|
+
return undefined;
|
|
326
|
+
}
|
|
327
|
+
if (mode !== "bytes") {
|
|
328
|
+
throw new Error(`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`);
|
|
329
|
+
}
|
|
330
|
+
if (this.blobContext === undefined) {
|
|
331
|
+
throw new Error('blobMode "bytes" is not supported for this query');
|
|
332
|
+
}
|
|
333
|
+
return this.blobContext;
|
|
334
|
+
}
|
|
190
335
|
/** Collect the results as an array of objects. */
|
|
191
336
|
// biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
|
|
192
337
|
async toArray(options) {
|
|
@@ -264,8 +409,8 @@ class QueryBase {
|
|
|
264
409
|
}
|
|
265
410
|
exports.QueryBase = QueryBase;
|
|
266
411
|
class StandardQueryBase extends QueryBase {
|
|
267
|
-
constructor(inner) {
|
|
268
|
-
super(inner);
|
|
412
|
+
constructor(inner, blobContext) {
|
|
413
|
+
super(inner, blobContext);
|
|
269
414
|
}
|
|
270
415
|
/**
|
|
271
416
|
* A filter statement to be applied to this query.
|
|
@@ -361,26 +506,26 @@ class StandardQueryBase extends QueryBase {
|
|
|
361
506
|
return this;
|
|
362
507
|
}
|
|
363
508
|
/**
|
|
364
|
-
* Control
|
|
509
|
+
* Control RTE read routing for this query.
|
|
365
510
|
*
|
|
366
|
-
* By default (unset), when the table carries
|
|
367
|
-
* {@link Table#
|
|
511
|
+
* By default (unset), when the table carries an RTE write spec (see
|
|
512
|
+
* {@link Table#enableRTE}), reads are routed through the LSM scanner so
|
|
368
513
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
369
514
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
370
|
-
* the
|
|
515
|
+
* the SSTables), deduplicated by primary key; a table without a spec
|
|
371
516
|
* reads the base table.
|
|
372
517
|
*
|
|
373
518
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
374
|
-
*
|
|
519
|
+
* RTE write spec. `false` bypasses the RTE and reads the base table only,
|
|
375
520
|
* even when a spec is present.
|
|
376
521
|
*
|
|
377
522
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
378
|
-
* hybrid search, `orderBy`). On
|
|
379
|
-
* `
|
|
380
|
-
* un-compacted
|
|
523
|
+
* hybrid search, `orderBy`). On an RTE table those shapes error unless
|
|
524
|
+
* `useRTE(false)` is set, because a base-only read would silently exclude
|
|
525
|
+
* un-compacted RTE data.
|
|
381
526
|
*/
|
|
382
|
-
|
|
383
|
-
this.doCall((inner) => inner.
|
|
527
|
+
useRTE(enable) {
|
|
528
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
384
529
|
return this;
|
|
385
530
|
}
|
|
386
531
|
}
|
|
@@ -398,8 +543,8 @@ class VectorQuery extends StandardQueryBase {
|
|
|
398
543
|
/**
|
|
399
544
|
* @hidden
|
|
400
545
|
*/
|
|
401
|
-
constructor(inner) {
|
|
402
|
-
super(inner);
|
|
546
|
+
constructor(inner, blobContext) {
|
|
547
|
+
super(inner, blobContext);
|
|
403
548
|
}
|
|
404
549
|
/**
|
|
405
550
|
* @hidden
|
|
@@ -446,7 +591,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
446
591
|
* probing may continue when too few candidates are found.
|
|
447
592
|
* Any explicit effort, including 0.5, overrides nprobes in either call order.
|
|
448
593
|
* Explicit minimum and maximum bounds still apply. This is not a recall
|
|
449
|
-
* guarantee and does not change refinement or HNSW ef. The
|
|
594
|
+
* guarantee and does not change refinement or HNSW ef. The RTE LSM
|
|
450
595
|
* scanner does not yet support this option.
|
|
451
596
|
*
|
|
452
597
|
* @example
|
|
@@ -634,7 +779,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
634
779
|
addQueryVectorToNative(inner, outcome.value);
|
|
635
780
|
return inner;
|
|
636
781
|
})();
|
|
637
|
-
return new VectorQuery(res);
|
|
782
|
+
return new VectorQuery(res, this.blobContext);
|
|
638
783
|
}
|
|
639
784
|
else {
|
|
640
785
|
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
|
@@ -696,7 +841,7 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
696
841
|
}
|
|
697
842
|
return nearestToNative(route.table.query(), vector);
|
|
698
843
|
};
|
|
699
|
-
return new AutoQuery(createInner);
|
|
844
|
+
return new AutoQuery(createInner, { table, withRowId: false });
|
|
700
845
|
}
|
|
701
846
|
/**
|
|
702
847
|
* A query that returns a subset of the rows in the table.
|
|
@@ -704,20 +849,20 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
704
849
|
* @hideconstructor
|
|
705
850
|
*/
|
|
706
851
|
class TakeQuery extends QueryBase {
|
|
707
|
-
constructor(inner) {
|
|
708
|
-
super(inner);
|
|
852
|
+
constructor(inner, blobContext) {
|
|
853
|
+
super(inner, blobContext);
|
|
709
854
|
}
|
|
710
855
|
/**
|
|
711
|
-
* Control
|
|
856
|
+
* Control RTE read routing for this take query.
|
|
712
857
|
*
|
|
713
|
-
* `false` bypasses the
|
|
858
|
+
* `false` bypasses the RTE and reads the base table only — the escape hatch,
|
|
714
859
|
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
|
|
715
|
-
*
|
|
860
|
+
* RTE table, auto-routes to it and errors otherwise.
|
|
716
861
|
*
|
|
717
862
|
* @param enable - `false` reads the base table only.
|
|
718
863
|
*/
|
|
719
|
-
|
|
720
|
-
this.doCall((inner) => inner.
|
|
864
|
+
useRTE(enable) {
|
|
865
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
721
866
|
return this;
|
|
722
867
|
}
|
|
723
868
|
}
|
|
@@ -735,8 +880,8 @@ class AutoQuery extends StandardQueryBase {
|
|
|
735
880
|
createInner;
|
|
736
881
|
calls = [];
|
|
737
882
|
/** @hidden */
|
|
738
|
-
constructor(createInner) {
|
|
739
|
-
super();
|
|
883
|
+
constructor(createInner, blobContext) {
|
|
884
|
+
super(undefined, blobContext);
|
|
740
885
|
this.createInner = createInner;
|
|
741
886
|
}
|
|
742
887
|
/** @hidden */
|
|
@@ -765,7 +910,7 @@ class Query extends StandardQueryBase {
|
|
|
765
910
|
* @hidden
|
|
766
911
|
*/
|
|
767
912
|
constructor(tbl) {
|
|
768
|
-
super(tbl.query());
|
|
913
|
+
super(tbl.query(), { table: tbl, withRowId: false });
|
|
769
914
|
}
|
|
770
915
|
/**
|
|
771
916
|
* Find the nearest vectors to the given query vector.
|
|
@@ -808,12 +953,12 @@ class Query extends StandardQueryBase {
|
|
|
808
953
|
const inner = this.inner;
|
|
809
954
|
if (inner instanceof Promise) {
|
|
810
955
|
const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
|
|
811
|
-
return new VectorQuery(nativeQuery);
|
|
956
|
+
return new VectorQuery(nativeQuery, this.blobContext);
|
|
812
957
|
}
|
|
813
958
|
if (vector instanceof Promise) {
|
|
814
|
-
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
|
|
959
|
+
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), this.blobContext);
|
|
815
960
|
}
|
|
816
|
-
return new VectorQuery(nearestToNative(inner, vector));
|
|
961
|
+
return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
|
|
817
962
|
}
|
|
818
963
|
nearestToText(query, columns) {
|
|
819
964
|
this.doCall((inner) => {
|
package/dist/table.d.ts
CHANGED
|
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
|
|
|
3
3
|
import { IndexOptions } from "./indices";
|
|
4
4
|
import { Job } from "./job";
|
|
5
5
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics,
|
|
6
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
7
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
8
8
|
import { IntoSql } from "./util";
|
|
9
9
|
export { IndexConfig } from "./native";
|
|
10
|
-
export {
|
|
10
|
+
export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
|
|
11
11
|
/**
|
|
12
12
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
13
13
|
* callback passed to {@link Table.add}.
|
|
@@ -139,8 +139,8 @@ export type TokenizeTableOptions = {
|
|
|
139
139
|
column?: never;
|
|
140
140
|
};
|
|
141
141
|
/**
|
|
142
|
-
* Specification
|
|
143
|
-
* `mergeInsert
|
|
142
|
+
* Specification enabling the realtime engine (RTE), which routes
|
|
143
|
+
* `mergeInsert` through Lance's MemWAL LSM-style write path.
|
|
144
144
|
*
|
|
145
145
|
* `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
|
|
146
146
|
* `column` and `numBuckets` are required; for `"identity"`, `column` is
|
|
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
|
|
|
149
149
|
* `column` value, or upserts of that key can land in different shards and a
|
|
150
150
|
* stale version can win).
|
|
151
151
|
*/
|
|
152
|
-
export interface
|
|
152
|
+
export interface RTEWriteSpec {
|
|
153
153
|
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
154
154
|
specType: "bucket" | "identity" | "unsharded";
|
|
155
155
|
/** Bucket and identity variants: the sharding column. */
|
|
@@ -157,7 +157,7 @@ export interface LsmWriteSpec {
|
|
|
157
157
|
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
158
158
|
numBuckets?: number;
|
|
159
159
|
/**
|
|
160
|
-
* Indexes the
|
|
160
|
+
* Indexes the RTE keeps up to date. Omit for every index the table has,
|
|
161
161
|
* including ones created later. Pass `[]` for none.
|
|
162
162
|
*/
|
|
163
163
|
maintainedIndexes?: string[];
|
|
@@ -477,6 +477,10 @@ export declare abstract class Table {
|
|
|
477
477
|
/**
|
|
478
478
|
* Add new columns with defined values.
|
|
479
479
|
*
|
|
480
|
+
* Local computed-column declaration and refresh require a source build with
|
|
481
|
+
* the `experimental-functions` Cargo feature. Published packages disable it;
|
|
482
|
+
* remote Cloud/Enterprise operations remain available.
|
|
483
|
+
*
|
|
480
484
|
* The `{ computed }` form stores the expression rather than evaluating it
|
|
481
485
|
* now: the column is committed with no values, and rows get them from
|
|
482
486
|
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
|
|
@@ -509,6 +513,7 @@ export declare abstract class Table {
|
|
|
509
513
|
}): Promise<AddColumnsResult>;
|
|
510
514
|
/**
|
|
511
515
|
* Fill the rows of a computed column that hold no value yet.
|
|
516
|
+
* Local refresh requires a source build with `experimental-functions`.
|
|
512
517
|
*
|
|
513
518
|
* Rows appended since the last refresh are filled by the next one, and
|
|
514
519
|
* rows whose inputs changed since they were computed are recomputed;
|
|
@@ -522,6 +527,8 @@ export declare abstract class Table {
|
|
|
522
527
|
/**
|
|
523
528
|
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
524
529
|
* job instead of blocking until it completes.
|
|
530
|
+
* Local refresh requires a source build with `experimental-functions`.
|
|
531
|
+
* Remote Cloud/Enterprise jobs do not require this feature.
|
|
525
532
|
*
|
|
526
533
|
* The job may already be complete when returned; callers must not assume
|
|
527
534
|
* the column is filled until {@link Job.wait} resolves. Invalid input --
|
|
@@ -562,7 +569,7 @@ export declare abstract class Table {
|
|
|
562
569
|
* it: rejects tables that carry no view definition.
|
|
563
570
|
* @ignore
|
|
564
571
|
*/
|
|
565
|
-
abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
572
|
+
abstract refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
566
573
|
/** @ignore */
|
|
567
574
|
abstract materializedViewDefinition(): Promise<string>;
|
|
568
575
|
/**
|
|
@@ -582,8 +589,6 @@ export declare abstract class Table {
|
|
|
582
589
|
* - `lancedb:description`: for a human-readable description of a field.
|
|
583
590
|
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
|
584
591
|
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
|
585
|
-
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
|
586
|
-
* `feature_v2` might be in the same logical column.
|
|
587
592
|
* - `lancedb:status`: for status options (`production`, `candidate`,
|
|
588
593
|
* `deprecated`, `archived`) to designate the current life cycle state of
|
|
589
594
|
* this column.
|
|
@@ -620,10 +625,10 @@ export declare abstract class Table {
|
|
|
620
625
|
*/
|
|
621
626
|
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
622
627
|
/**
|
|
623
|
-
* Install an {@link
|
|
628
|
+
* Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
|
|
624
629
|
* LSM-style write path for future `mergeInsert` calls.
|
|
625
630
|
*
|
|
626
|
-
* `
|
|
631
|
+
* `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
|
|
627
632
|
*
|
|
628
633
|
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
|
|
629
634
|
* key (`column` and `numBuckets` required).
|
|
@@ -635,15 +640,15 @@ export declare abstract class Table {
|
|
|
635
640
|
* requires it to be the single column being bucketed.
|
|
636
641
|
*
|
|
637
642
|
* Omitting `maintainedIndexes` maintains every index the table has,
|
|
638
|
-
* including ones created later, and skips a kind the
|
|
643
|
+
* including ones created later, and skips a kind the RTE cannot maintain.
|
|
639
644
|
* Naming them pins an exact set, and a still-building index is rejected
|
|
640
645
|
* rather than quietly omitted.
|
|
641
|
-
* @param {
|
|
646
|
+
* @param {RTEWriteSpec} spec The sharding spec to install.
|
|
642
647
|
* @returns {Promise<void>}
|
|
643
648
|
* @example
|
|
644
649
|
* ```ts
|
|
645
650
|
* await table.setUnenforcedPrimaryKey("id");
|
|
646
|
-
* await table.
|
|
651
|
+
* await table.enableRTE({
|
|
647
652
|
* specType: "bucket",
|
|
648
653
|
* column: "id",
|
|
649
654
|
* numBuckets: 16,
|
|
@@ -651,59 +656,59 @@ export declare abstract class Table {
|
|
|
651
656
|
* });
|
|
652
657
|
* ```
|
|
653
658
|
*/
|
|
654
|
-
abstract
|
|
659
|
+
abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
655
660
|
/**
|
|
656
|
-
* Remove the {@link
|
|
661
|
+
* Remove the {@link RTEWriteSpec} from this table, reverting to the standard
|
|
657
662
|
* `mergeInsert` write path.
|
|
658
663
|
*
|
|
659
664
|
* Errors if no spec is currently set.
|
|
660
665
|
* @returns {Promise<void>}
|
|
661
666
|
*/
|
|
662
|
-
abstract
|
|
667
|
+
abstract disableRTE(): Promise<void>;
|
|
663
668
|
/**
|
|
664
|
-
* Read the {@link
|
|
669
|
+
* Read the {@link RTEWriteSpec} currently installed on this table.
|
|
665
670
|
*
|
|
666
|
-
* Resolves to `undefined` when the
|
|
667
|
-
* spec has been set, or it was removed with {@link Table#
|
|
671
|
+
* Resolves to `undefined` when the RTE write path is not enabled (no
|
|
672
|
+
* spec has been set, or it was removed with {@link Table#disableRTE}).
|
|
668
673
|
* The spec is the one installed, including its maintained-index selection:
|
|
669
674
|
* an absent `maintainedIndexes` for every index the table has, an empty
|
|
670
675
|
* array for none.
|
|
671
|
-
* @returns {Promise<
|
|
676
|
+
* @returns {Promise<RTEWriteSpec | undefined>}
|
|
672
677
|
*/
|
|
673
|
-
abstract
|
|
678
|
+
abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
674
679
|
/**
|
|
675
|
-
* Drain and close any cached
|
|
680
|
+
* Drain and close any cached RTE shard writers held for this table.
|
|
676
681
|
*
|
|
677
|
-
* When an {@link
|
|
682
|
+
* When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
|
|
678
683
|
* shard writers and caches them for reuse across calls. This closes them,
|
|
679
684
|
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
|
|
680
685
|
* It is a no-op when no writers are cached.
|
|
681
686
|
* @returns {Promise<void>}
|
|
682
687
|
*/
|
|
683
|
-
abstract
|
|
688
|
+
abstract closeRTEWriters(): Promise<void>;
|
|
684
689
|
/**
|
|
685
|
-
*
|
|
690
|
+
* Freeze every table shard's active memtable into a new SSTable.
|
|
686
691
|
*
|
|
687
|
-
* Returns once the
|
|
692
|
+
* Returns once the freeze is committed. Freezing an empty memtable is a no-op,
|
|
688
693
|
* so this is safe to call repeatedly.
|
|
689
694
|
* @returns {Promise<void>}
|
|
690
695
|
*/
|
|
691
|
-
abstract
|
|
696
|
+
abstract flushRTE(): Promise<void>;
|
|
692
697
|
/**
|
|
693
|
-
* Trigger a background
|
|
698
|
+
* Trigger a background SSTable compaction pass per table shard.
|
|
694
699
|
*
|
|
695
700
|
* Returns once the passes are *dispatched*, not once they finish — watch
|
|
696
|
-
* {@link Table#
|
|
697
|
-
* {@link Table#
|
|
701
|
+
* {@link Table#getRTEStats} for progress, or use
|
|
702
|
+
* {@link Table#checkpointRTE} to wait for convergence.
|
|
698
703
|
* @returns {Promise<void>}
|
|
699
704
|
*/
|
|
700
|
-
abstract
|
|
705
|
+
abstract compactRTE(): Promise<void>;
|
|
701
706
|
/**
|
|
702
|
-
* Converge this table's
|
|
707
|
+
* Converge this table's RTE write path into its base table.
|
|
703
708
|
*
|
|
704
|
-
*
|
|
709
|
+
* Freezes once, then triggers compaction and polls until the SSTables that existed
|
|
705
710
|
* at the start is gone. The target set is fixed at the start, so
|
|
706
|
-
*
|
|
711
|
+
* SSTables created *during* the checkpoint are ignored — that is what
|
|
707
712
|
* lets it terminate under write load, and what makes it best-effort: it
|
|
708
713
|
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
|
709
714
|
* any point, and safe to run on a cadence.
|
|
@@ -714,24 +719,24 @@ export declare abstract class Table {
|
|
|
714
719
|
* @returns {Promise<void>}
|
|
715
720
|
* @example
|
|
716
721
|
* ```ts
|
|
717
|
-
* const before = await table.
|
|
718
|
-
* await table.
|
|
719
|
-
* const after = await table.
|
|
722
|
+
* const before = await table.getRTEStats();
|
|
723
|
+
* await table.checkpointRTE();
|
|
724
|
+
* const after = await table.getRTEStats();
|
|
720
725
|
* ```
|
|
721
726
|
*/
|
|
722
|
-
abstract
|
|
727
|
+
abstract checkpointRTE(): Promise<void>;
|
|
723
728
|
/**
|
|
724
|
-
* Read live per-
|
|
729
|
+
* Read live per-table-shard RTE state.
|
|
725
730
|
*
|
|
726
|
-
* Answers "how far behind is my fresh tier", "which
|
|
731
|
+
* Answers "how far behind is my fresh tier", "which table shard is hot", and
|
|
727
732
|
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
|
728
733
|
*
|
|
729
|
-
* Resolves to `undefined` only when the
|
|
730
|
-
* @param {boolean}
|
|
734
|
+
* Resolves to `undefined` only when the RTE write path is not enabled.
|
|
735
|
+
* @param {boolean} includeSstableRows Also count rows per SSTable.
|
|
731
736
|
* Off by default because each count opens an uncached Lance dataset.
|
|
732
|
-
* @returns {Promise<
|
|
737
|
+
* @returns {Promise<RTEStats | undefined>}
|
|
733
738
|
*/
|
|
734
|
-
abstract
|
|
739
|
+
abstract getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
735
740
|
/** Retrieve the version of the table */
|
|
736
741
|
abstract version(): Promise<number>;
|
|
737
742
|
/**
|
|
@@ -842,7 +847,11 @@ export declare abstract class Table {
|
|
|
842
847
|
*
|
|
843
848
|
* Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
|
|
844
849
|
* the client process from index metadata. For remote tables, this means the
|
|
845
|
-
* same tokenizer model files must also exist locally
|
|
850
|
+
* same tokenizer model files must also exist locally (under
|
|
851
|
+
* `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
|
|
852
|
+
* mismatch cannot be detected: the returned tokens may differ from what the
|
|
853
|
+
* server's index produces. If the model files are missing, an error naming
|
|
854
|
+
* the tokenizer is raised.
|
|
846
855
|
*/
|
|
847
856
|
abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
|
|
848
857
|
/** Return the table as an arrow table */
|
|
@@ -934,20 +943,20 @@ export declare class LocalTable extends Table {
|
|
|
934
943
|
refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
935
944
|
refreshColumnAsync(column: string): Promise<Job>;
|
|
936
945
|
functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
937
|
-
refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
946
|
+
refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
938
947
|
materializedViewDefinition(): Promise<string>;
|
|
939
948
|
alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
940
949
|
updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
|
|
941
950
|
dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
942
951
|
setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
952
|
+
enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
953
|
+
disableRTE(): Promise<void>;
|
|
954
|
+
getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
955
|
+
closeRTEWriters(): Promise<void>;
|
|
956
|
+
flushRTE(): Promise<void>;
|
|
957
|
+
compactRTE(): Promise<void>;
|
|
958
|
+
checkpointRTE(): Promise<void>;
|
|
959
|
+
getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
951
960
|
version(): Promise<number>;
|
|
952
961
|
checkout(version: number | string): Promise<void>;
|
|
953
962
|
checkoutLatest(): Promise<void>;
|