@lancedb/lancedb 0.41.0-beta.0 → 0.41.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.js +16 -5
- package/dist/blob.d.ts +26 -1
- package/dist/blob.js +58 -5
- package/dist/connection.d.ts +19 -9
- package/dist/connection.js +10 -6
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +136 -106
- package/dist/native.js +54 -52
- package/dist/query.d.ts +87 -25
- package/dist/query.js +168 -41
- package/dist/table.d.ts +58 -51
- package/dist/table.js +24 -18
- package/package.json +8 -8
package/dist/query.js
CHANGED
|
@@ -21,6 +21,62 @@ async function* RecordBatchIterator(promisedInner) {
|
|
|
21
21
|
yield batches[0];
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
+
const BLOB_BYTES_ATTEMPTS = 3;
|
|
25
|
+
function blobSources(blobColumns, projection) {
|
|
26
|
+
const columns = new Set(blobColumns);
|
|
27
|
+
const pairs = projection ??
|
|
28
|
+
blobColumns.map((column) => [column, column]);
|
|
29
|
+
return new Map(pairs.filter(([, source]) => columns.has(source)));
|
|
30
|
+
}
|
|
31
|
+
async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
32
|
+
const rowIdColumn = table.getChild("_rowid");
|
|
33
|
+
if (table.numRows > 0 && rowIdColumn === null) {
|
|
34
|
+
throw new Error('blobMode "bytes" needs _rowid in the query result');
|
|
35
|
+
}
|
|
36
|
+
const rowIds = rowIdColumn === null
|
|
37
|
+
? []
|
|
38
|
+
: Array.from(rowIdColumn.toArray());
|
|
39
|
+
const replacements = new Map();
|
|
40
|
+
for (const [output, source] of sources) {
|
|
41
|
+
const index = table.schema.fields.findIndex((field) => field.name === output);
|
|
42
|
+
if (index < 0) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
|
|
46
|
+
replacements.set(index, values.map((value) => value ?? null));
|
|
47
|
+
}
|
|
48
|
+
const result = withBinaryColumns(table, replacements);
|
|
49
|
+
if (!dropRowId) {
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
return result.select(result.schema.fields
|
|
53
|
+
.map((field) => field.name)
|
|
54
|
+
.filter((name) => name !== "_rowid"));
|
|
55
|
+
}
|
|
56
|
+
// A fresh field, not a clone: the descriptor field's metadata must not follow
|
|
57
|
+
// the bytes.
|
|
58
|
+
function withBinaryColumns(table, replacements) {
|
|
59
|
+
if (replacements.size === 0) {
|
|
60
|
+
return table;
|
|
61
|
+
}
|
|
62
|
+
const fields = table.schema.fields.map((field, index) => replacements.has(index)
|
|
63
|
+
? new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true)
|
|
64
|
+
: field);
|
|
65
|
+
const schema = new arrow_1.Schema(fields, table.schema.metadata);
|
|
66
|
+
let offset = 0;
|
|
67
|
+
const batches = table.batches.map((batch) => {
|
|
68
|
+
const children = batch.data.children.map((child, index) => {
|
|
69
|
+
const values = replacements.get(index);
|
|
70
|
+
if (values === undefined) {
|
|
71
|
+
return child;
|
|
72
|
+
}
|
|
73
|
+
return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
|
|
74
|
+
});
|
|
75
|
+
offset += batch.numRows;
|
|
76
|
+
return new arrow_1.RecordBatch(schema, (0, arrow_1.makeData)({ type: new arrow_1.Struct(fields), length: batch.numRows, children }));
|
|
77
|
+
});
|
|
78
|
+
return new arrow_1.Table(schema, batches);
|
|
79
|
+
}
|
|
24
80
|
function nearestToNative(inner, vector) {
|
|
25
81
|
const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
|
|
26
82
|
if (raw) {
|
|
@@ -49,10 +105,15 @@ class QueryBase {
|
|
|
49
105
|
/**
|
|
50
106
|
* @hidden
|
|
51
107
|
*/
|
|
52
|
-
|
|
108
|
+
blobContext;
|
|
109
|
+
/**
|
|
110
|
+
* @hidden
|
|
111
|
+
*/
|
|
112
|
+
constructor(inner, blobContext) {
|
|
53
113
|
if (inner !== undefined) {
|
|
54
114
|
this.inner = inner;
|
|
55
115
|
}
|
|
116
|
+
this.blobContext = blobContext;
|
|
56
117
|
}
|
|
57
118
|
// call a function on the inner (either a promise or the actual object)
|
|
58
119
|
/**
|
|
@@ -118,17 +179,25 @@ class QueryBase {
|
|
|
118
179
|
inner.select(columnTuples);
|
|
119
180
|
});
|
|
120
181
|
};
|
|
182
|
+
let projection;
|
|
121
183
|
if (typeof columns === "string") {
|
|
122
184
|
selectColumns([columns]);
|
|
185
|
+
projection = [[columns, columns]];
|
|
123
186
|
}
|
|
124
187
|
else if (Array.isArray(columns)) {
|
|
125
188
|
selectColumns(columns);
|
|
189
|
+
projection = columns.map((column) => [column, column]);
|
|
126
190
|
}
|
|
127
191
|
else if (columns instanceof Map) {
|
|
128
|
-
|
|
192
|
+
projection = Array.from(columns.entries());
|
|
193
|
+
selectMapping(projection);
|
|
129
194
|
}
|
|
130
195
|
else {
|
|
131
|
-
|
|
196
|
+
projection = Object.entries(columns);
|
|
197
|
+
selectMapping(projection);
|
|
198
|
+
}
|
|
199
|
+
if (this.blobContext !== undefined) {
|
|
200
|
+
this.blobContext = { ...this.blobContext, projection };
|
|
132
201
|
}
|
|
133
202
|
return this;
|
|
134
203
|
}
|
|
@@ -141,14 +210,17 @@ class QueryBase {
|
|
|
141
210
|
*/
|
|
142
211
|
withRowId() {
|
|
143
212
|
this.doCall((inner) => inner.withRowId());
|
|
213
|
+
if (this.blobContext !== undefined) {
|
|
214
|
+
this.blobContext = { ...this.blobContext, withRowId: true };
|
|
215
|
+
}
|
|
144
216
|
return this;
|
|
145
217
|
}
|
|
146
218
|
/**
|
|
147
219
|
* @hidden
|
|
148
220
|
*/
|
|
149
|
-
async nativeExecute(options) {
|
|
221
|
+
async nativeExecute(options, withRowId = false) {
|
|
150
222
|
const inner = await this.getInner();
|
|
151
|
-
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
|
223
|
+
return inner.execute(options?.maxBatchLength, options?.timeoutMs, withRowId);
|
|
152
224
|
}
|
|
153
225
|
/**
|
|
154
226
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
@@ -177,8 +249,32 @@ class QueryBase {
|
|
|
177
249
|
* Empty results retain the query's output schema.
|
|
178
250
|
*/
|
|
179
251
|
async toArrow(options) {
|
|
252
|
+
const context = this.blobBytesContext(options?.blobMode);
|
|
253
|
+
if (context === undefined) {
|
|
254
|
+
return this.collect(options, false);
|
|
255
|
+
}
|
|
256
|
+
for (let attempt = 1;; attempt++) {
|
|
257
|
+
// The snapshot pins the bytes. The query runs on the live table, so an
|
|
258
|
+
// unchanged version afterwards proves it read the snapshot's version.
|
|
259
|
+
const snapshot = await context.table.querySnapshot();
|
|
260
|
+
const version = await snapshot.version();
|
|
261
|
+
const sources = blobSources(await snapshot.blobColumns(), context.projection);
|
|
262
|
+
if (sources.size === 0) {
|
|
263
|
+
return this.collect(options, false);
|
|
264
|
+
}
|
|
265
|
+
const addRowId = !context.withRowId;
|
|
266
|
+
const table = await this.collect(options, addRowId);
|
|
267
|
+
if ((await context.table.version()) === version) {
|
|
268
|
+
return readBlobBytes(table, sources, snapshot, addRowId);
|
|
269
|
+
}
|
|
270
|
+
if (attempt === BLOB_BYTES_ATTEMPTS) {
|
|
271
|
+
throw new Error(`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async collect(options, withRowId) {
|
|
180
276
|
const batches = [];
|
|
181
|
-
const iterator = this.nativeExecute(options);
|
|
277
|
+
const iterator = this.nativeExecute(options, withRowId);
|
|
182
278
|
for await (const batch of RecordBatchIterator(iterator)) {
|
|
183
279
|
batches.push(batch);
|
|
184
280
|
}
|
|
@@ -187,6 +283,18 @@ class QueryBase {
|
|
|
187
283
|
}
|
|
188
284
|
return new arrow_1.Table(batches);
|
|
189
285
|
}
|
|
286
|
+
blobBytesContext(mode) {
|
|
287
|
+
if (mode === undefined || mode === "descriptions") {
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
290
|
+
if (mode !== "bytes") {
|
|
291
|
+
throw new Error(`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`);
|
|
292
|
+
}
|
|
293
|
+
if (this.blobContext === undefined) {
|
|
294
|
+
throw new Error('blobMode "bytes" is not supported for this query');
|
|
295
|
+
}
|
|
296
|
+
return this.blobContext;
|
|
297
|
+
}
|
|
190
298
|
/** Collect the results as an array of objects. */
|
|
191
299
|
// biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
|
|
192
300
|
async toArray(options) {
|
|
@@ -264,8 +372,8 @@ class QueryBase {
|
|
|
264
372
|
}
|
|
265
373
|
exports.QueryBase = QueryBase;
|
|
266
374
|
class StandardQueryBase extends QueryBase {
|
|
267
|
-
constructor(inner) {
|
|
268
|
-
super(inner);
|
|
375
|
+
constructor(inner, blobContext) {
|
|
376
|
+
super(inner, blobContext);
|
|
269
377
|
}
|
|
270
378
|
/**
|
|
271
379
|
* A filter statement to be applied to this query.
|
|
@@ -361,26 +469,26 @@ class StandardQueryBase extends QueryBase {
|
|
|
361
469
|
return this;
|
|
362
470
|
}
|
|
363
471
|
/**
|
|
364
|
-
* Control
|
|
472
|
+
* Control RTE read routing for this query.
|
|
365
473
|
*
|
|
366
|
-
* By default (unset), when the table carries
|
|
367
|
-
* {@link Table#
|
|
474
|
+
* By default (unset), when the table carries an RTE write spec (see
|
|
475
|
+
* {@link Table#enableRTE}), reads are routed through the LSM scanner so
|
|
368
476
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
369
477
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
370
|
-
* the
|
|
478
|
+
* the SSTables), deduplicated by primary key; a table without a spec
|
|
371
479
|
* reads the base table.
|
|
372
480
|
*
|
|
373
481
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
374
|
-
*
|
|
482
|
+
* RTE write spec. `false` bypasses the RTE and reads the base table only,
|
|
375
483
|
* even when a spec is present.
|
|
376
484
|
*
|
|
377
485
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
378
|
-
* hybrid search, `orderBy`). On
|
|
379
|
-
* `
|
|
380
|
-
* un-compacted
|
|
486
|
+
* hybrid search, `orderBy`). On an RTE table those shapes error unless
|
|
487
|
+
* `useRTE(false)` is set, because a base-only read would silently exclude
|
|
488
|
+
* un-compacted RTE data.
|
|
381
489
|
*/
|
|
382
|
-
|
|
383
|
-
this.doCall((inner) => inner.
|
|
490
|
+
useRTE(enable) {
|
|
491
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
384
492
|
return this;
|
|
385
493
|
}
|
|
386
494
|
}
|
|
@@ -398,8 +506,8 @@ class VectorQuery extends StandardQueryBase {
|
|
|
398
506
|
/**
|
|
399
507
|
* @hidden
|
|
400
508
|
*/
|
|
401
|
-
constructor(inner) {
|
|
402
|
-
super(inner);
|
|
509
|
+
constructor(inner, blobContext) {
|
|
510
|
+
super(inner, blobContext);
|
|
403
511
|
}
|
|
404
512
|
/**
|
|
405
513
|
* @hidden
|
|
@@ -423,22 +531,41 @@ class VectorQuery extends StandardQueryBase {
|
|
|
423
531
|
* partitions should be searched.
|
|
424
532
|
*
|
|
425
533
|
* Increasing this value will increase the recall of your query but will
|
|
426
|
-
* also increase the latency of your query.
|
|
427
|
-
*
|
|
428
|
-
* your data and the recall that you need to achieve.
|
|
534
|
+
* also increase the latency of your query. If this method is not called,
|
|
535
|
+
* Lance's adaptive probe defaults are used.
|
|
429
536
|
*
|
|
430
537
|
* For best results we recommend tuning this parameter with a benchmark against
|
|
431
538
|
* your actual data to find the smallest possible value that will still give
|
|
432
539
|
* you the desired recall.
|
|
433
540
|
*
|
|
434
|
-
*
|
|
435
|
-
*
|
|
436
|
-
*
|
|
541
|
+
* The value is retained as `nprobes` through client and server request
|
|
542
|
+
* construction. Lance sets both probe bounds to this value. Explicit minimum
|
|
543
|
+
* or maximum settings can override their respective bounds.
|
|
437
544
|
*/
|
|
438
545
|
nprobes(nprobes) {
|
|
439
546
|
this.doVectorCall((inner) => inner.nprobes(nprobes));
|
|
440
547
|
return this;
|
|
441
548
|
}
|
|
549
|
+
/**
|
|
550
|
+
* Set the initial IVF search effort to a finite value in [0, 1].
|
|
551
|
+
*
|
|
552
|
+
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
|
|
553
|
+
* minimum; one searches all partitions within the caller maximum. Later
|
|
554
|
+
* probing may continue when too few candidates are found.
|
|
555
|
+
* Any explicit effort, including 0.5, overrides nprobes in either call order.
|
|
556
|
+
* Explicit minimum and maximum bounds still apply. This is not a recall
|
|
557
|
+
* guarantee and does not change refinement or HNSW ef. The RTE LSM
|
|
558
|
+
* scanner does not yet support this option.
|
|
559
|
+
*
|
|
560
|
+
* @example
|
|
561
|
+
* ```ts
|
|
562
|
+
* table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
|
|
563
|
+
* ```
|
|
564
|
+
*/
|
|
565
|
+
searchEffort(searchEffort) {
|
|
566
|
+
this.doVectorCall((inner) => inner.searchEffort(searchEffort));
|
|
567
|
+
return this;
|
|
568
|
+
}
|
|
442
569
|
/**
|
|
443
570
|
* Set the minimum number of probes used.
|
|
444
571
|
*
|
|
@@ -615,7 +742,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
615
742
|
addQueryVectorToNative(inner, outcome.value);
|
|
616
743
|
return inner;
|
|
617
744
|
})();
|
|
618
|
-
return new VectorQuery(res);
|
|
745
|
+
return new VectorQuery(res, this.blobContext);
|
|
619
746
|
}
|
|
620
747
|
else {
|
|
621
748
|
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
|
@@ -677,7 +804,7 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
677
804
|
}
|
|
678
805
|
return nearestToNative(route.table.query(), vector);
|
|
679
806
|
};
|
|
680
|
-
return new AutoQuery(createInner);
|
|
807
|
+
return new AutoQuery(createInner, { table, withRowId: false });
|
|
681
808
|
}
|
|
682
809
|
/**
|
|
683
810
|
* A query that returns a subset of the rows in the table.
|
|
@@ -685,20 +812,20 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
685
812
|
* @hideconstructor
|
|
686
813
|
*/
|
|
687
814
|
class TakeQuery extends QueryBase {
|
|
688
|
-
constructor(inner) {
|
|
689
|
-
super(inner);
|
|
815
|
+
constructor(inner, blobContext) {
|
|
816
|
+
super(inner, blobContext);
|
|
690
817
|
}
|
|
691
818
|
/**
|
|
692
|
-
* Control
|
|
819
|
+
* Control RTE read routing for this take query.
|
|
693
820
|
*
|
|
694
|
-
* `false` bypasses the
|
|
821
|
+
* `false` bypasses the RTE and reads the base table only — the escape hatch,
|
|
695
822
|
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
|
|
696
|
-
*
|
|
823
|
+
* RTE table, auto-routes to it and errors otherwise.
|
|
697
824
|
*
|
|
698
825
|
* @param enable - `false` reads the base table only.
|
|
699
826
|
*/
|
|
700
|
-
|
|
701
|
-
this.doCall((inner) => inner.
|
|
827
|
+
useRTE(enable) {
|
|
828
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
702
829
|
return this;
|
|
703
830
|
}
|
|
704
831
|
}
|
|
@@ -716,8 +843,8 @@ class AutoQuery extends StandardQueryBase {
|
|
|
716
843
|
createInner;
|
|
717
844
|
calls = [];
|
|
718
845
|
/** @hidden */
|
|
719
|
-
constructor(createInner) {
|
|
720
|
-
super();
|
|
846
|
+
constructor(createInner, blobContext) {
|
|
847
|
+
super(undefined, blobContext);
|
|
721
848
|
this.createInner = createInner;
|
|
722
849
|
}
|
|
723
850
|
/** @hidden */
|
|
@@ -746,7 +873,7 @@ class Query extends StandardQueryBase {
|
|
|
746
873
|
* @hidden
|
|
747
874
|
*/
|
|
748
875
|
constructor(tbl) {
|
|
749
|
-
super(tbl.query());
|
|
876
|
+
super(tbl.query(), { table: tbl, withRowId: false });
|
|
750
877
|
}
|
|
751
878
|
/**
|
|
752
879
|
* Find the nearest vectors to the given query vector.
|
|
@@ -789,12 +916,12 @@ class Query extends StandardQueryBase {
|
|
|
789
916
|
const inner = this.inner;
|
|
790
917
|
if (inner instanceof Promise) {
|
|
791
918
|
const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
|
|
792
|
-
return new VectorQuery(nativeQuery);
|
|
919
|
+
return new VectorQuery(nativeQuery, this.blobContext);
|
|
793
920
|
}
|
|
794
921
|
if (vector instanceof Promise) {
|
|
795
|
-
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
|
|
922
|
+
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), this.blobContext);
|
|
796
923
|
}
|
|
797
|
-
return new VectorQuery(nearestToNative(inner, vector));
|
|
924
|
+
return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
|
|
798
925
|
}
|
|
799
926
|
nearestToText(query, columns) {
|
|
800
927
|
this.doCall((inner) => {
|
package/dist/table.d.ts
CHANGED
|
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
|
|
|
3
3
|
import { IndexOptions } from "./indices";
|
|
4
4
|
import { Job } from "./job";
|
|
5
5
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics,
|
|
6
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
7
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
8
8
|
import { IntoSql } from "./util";
|
|
9
9
|
export { IndexConfig } from "./native";
|
|
10
|
-
export {
|
|
10
|
+
export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
|
|
11
11
|
/**
|
|
12
12
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
13
13
|
* callback passed to {@link Table.add}.
|
|
@@ -139,8 +139,8 @@ export type TokenizeTableOptions = {
|
|
|
139
139
|
column?: never;
|
|
140
140
|
};
|
|
141
141
|
/**
|
|
142
|
-
* Specification
|
|
143
|
-
* `mergeInsert
|
|
142
|
+
* Specification enabling the realtime engine (RTE), which routes
|
|
143
|
+
* `mergeInsert` through Lance's MemWAL LSM-style write path.
|
|
144
144
|
*
|
|
145
145
|
* `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
|
|
146
146
|
* `column` and `numBuckets` are required; for `"identity"`, `column` is
|
|
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
|
|
|
149
149
|
* `column` value, or upserts of that key can land in different shards and a
|
|
150
150
|
* stale version can win).
|
|
151
151
|
*/
|
|
152
|
-
export interface
|
|
152
|
+
export interface RTEWriteSpec {
|
|
153
153
|
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
154
154
|
specType: "bucket" | "identity" | "unsharded";
|
|
155
155
|
/** Bucket and identity variants: the sharding column. */
|
|
@@ -157,7 +157,7 @@ export interface LsmWriteSpec {
|
|
|
157
157
|
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
158
158
|
numBuckets?: number;
|
|
159
159
|
/**
|
|
160
|
-
* Indexes the
|
|
160
|
+
* Indexes the RTE keeps up to date. Omit for every index the table has,
|
|
161
161
|
* including ones created later. Pass `[]` for none.
|
|
162
162
|
*/
|
|
163
163
|
maintainedIndexes?: string[];
|
|
@@ -424,7 +424,8 @@ export declare abstract class Table {
|
|
|
424
424
|
*/
|
|
425
425
|
abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
|
|
426
426
|
/**
|
|
427
|
-
* Blob v2 columns, including nested dotted paths.
|
|
427
|
+
* Blob v2 columns addressable by row ID, including struct-nested dotted paths.
|
|
428
|
+
* Blobs inside lists are omitted because a row ID does not identify an element.
|
|
428
429
|
*/
|
|
429
430
|
abstract blobColumns(): Promise<string[]>;
|
|
430
431
|
/**
|
|
@@ -433,6 +434,10 @@ export declare abstract class Table {
|
|
|
433
434
|
* Reads the table's current checkout. IDs from another version can fail after
|
|
434
435
|
* compaction unless stable row ids are enabled. Results keep input order and
|
|
435
436
|
* duplicates. Null blobs are `null`. Empty blobs are empty buffers.
|
|
437
|
+
* One missing or deleted row ID rejects the entire batch, including on remote
|
|
438
|
+
* tables; no partial results are returned.
|
|
439
|
+
* Blobs inside lists cannot be fetched by row ID.
|
|
440
|
+
*
|
|
436
441
|
* Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
|
|
437
442
|
* The client splits requests automatically and reads an individual larger
|
|
438
443
|
* blob through the Range route. This method still materializes all bytes in
|
|
@@ -445,6 +450,8 @@ export declare abstract class Table {
|
|
|
445
450
|
*
|
|
446
451
|
* Preserves input order, duplicates, and nulls. Use this for large payloads.
|
|
447
452
|
* See {@link Table.fetchBlobs} for row-ID validity across versions.
|
|
453
|
+
* One missing or deleted row ID rejects the entire batch, including on remote
|
|
454
|
+
* tables; no partial handles are returned.
|
|
448
455
|
*/
|
|
449
456
|
abstract fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(BlobFile | null)[]>;
|
|
450
457
|
/**
|
|
@@ -613,10 +620,10 @@ export declare abstract class Table {
|
|
|
613
620
|
*/
|
|
614
621
|
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
615
622
|
/**
|
|
616
|
-
* Install an {@link
|
|
623
|
+
* Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
|
|
617
624
|
* LSM-style write path for future `mergeInsert` calls.
|
|
618
625
|
*
|
|
619
|
-
* `
|
|
626
|
+
* `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
|
|
620
627
|
*
|
|
621
628
|
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
|
|
622
629
|
* key (`column` and `numBuckets` required).
|
|
@@ -628,15 +635,15 @@ export declare abstract class Table {
|
|
|
628
635
|
* requires it to be the single column being bucketed.
|
|
629
636
|
*
|
|
630
637
|
* Omitting `maintainedIndexes` maintains every index the table has,
|
|
631
|
-
* including ones created later, and skips a kind the
|
|
638
|
+
* including ones created later, and skips a kind the RTE cannot maintain.
|
|
632
639
|
* Naming them pins an exact set, and a still-building index is rejected
|
|
633
640
|
* rather than quietly omitted.
|
|
634
|
-
* @param {
|
|
641
|
+
* @param {RTEWriteSpec} spec The sharding spec to install.
|
|
635
642
|
* @returns {Promise<void>}
|
|
636
643
|
* @example
|
|
637
644
|
* ```ts
|
|
638
645
|
* await table.setUnenforcedPrimaryKey("id");
|
|
639
|
-
* await table.
|
|
646
|
+
* await table.enableRTE({
|
|
640
647
|
* specType: "bucket",
|
|
641
648
|
* column: "id",
|
|
642
649
|
* numBuckets: 16,
|
|
@@ -644,59 +651,59 @@ export declare abstract class Table {
|
|
|
644
651
|
* });
|
|
645
652
|
* ```
|
|
646
653
|
*/
|
|
647
|
-
abstract
|
|
654
|
+
abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
648
655
|
/**
|
|
649
|
-
* Remove the {@link
|
|
656
|
+
* Remove the {@link RTEWriteSpec} from this table, reverting to the standard
|
|
650
657
|
* `mergeInsert` write path.
|
|
651
658
|
*
|
|
652
659
|
* Errors if no spec is currently set.
|
|
653
660
|
* @returns {Promise<void>}
|
|
654
661
|
*/
|
|
655
|
-
abstract
|
|
662
|
+
abstract disableRTE(): Promise<void>;
|
|
656
663
|
/**
|
|
657
|
-
* Read the {@link
|
|
664
|
+
* Read the {@link RTEWriteSpec} currently installed on this table.
|
|
658
665
|
*
|
|
659
|
-
* Resolves to `undefined` when the
|
|
660
|
-
* spec has been set, or it was removed with {@link Table#
|
|
666
|
+
* Resolves to `undefined` when the RTE write path is not enabled (no
|
|
667
|
+
* spec has been set, or it was removed with {@link Table#disableRTE}).
|
|
661
668
|
* The spec is the one installed, including its maintained-index selection:
|
|
662
669
|
* an absent `maintainedIndexes` for every index the table has, an empty
|
|
663
670
|
* array for none.
|
|
664
|
-
* @returns {Promise<
|
|
671
|
+
* @returns {Promise<RTEWriteSpec | undefined>}
|
|
665
672
|
*/
|
|
666
|
-
abstract
|
|
673
|
+
abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
667
674
|
/**
|
|
668
|
-
* Drain and close any cached
|
|
675
|
+
* Drain and close any cached RTE shard writers held for this table.
|
|
669
676
|
*
|
|
670
|
-
* When an {@link
|
|
677
|
+
* When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
|
|
671
678
|
* shard writers and caches them for reuse across calls. This closes them,
|
|
672
679
|
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
|
|
673
680
|
* It is a no-op when no writers are cached.
|
|
674
681
|
* @returns {Promise<void>}
|
|
675
682
|
*/
|
|
676
|
-
abstract
|
|
683
|
+
abstract closeRTEWriters(): Promise<void>;
|
|
677
684
|
/**
|
|
678
|
-
*
|
|
685
|
+
* Freeze every table shard's active memtable into a new SSTable.
|
|
679
686
|
*
|
|
680
|
-
* Returns once the
|
|
687
|
+
* Returns once the freeze is committed. Freezing an empty memtable is a no-op,
|
|
681
688
|
* so this is safe to call repeatedly.
|
|
682
689
|
* @returns {Promise<void>}
|
|
683
690
|
*/
|
|
684
|
-
abstract
|
|
691
|
+
abstract flushRTE(): Promise<void>;
|
|
685
692
|
/**
|
|
686
|
-
* Trigger a background
|
|
693
|
+
* Trigger a background SSTable compaction pass per table shard.
|
|
687
694
|
*
|
|
688
695
|
* Returns once the passes are *dispatched*, not once they finish — watch
|
|
689
|
-
* {@link Table#
|
|
690
|
-
* {@link Table#
|
|
696
|
+
* {@link Table#getRTEStats} for progress, or use
|
|
697
|
+
* {@link Table#checkpointRTE} to wait for convergence.
|
|
691
698
|
* @returns {Promise<void>}
|
|
692
699
|
*/
|
|
693
|
-
abstract
|
|
700
|
+
abstract compactRTE(): Promise<void>;
|
|
694
701
|
/**
|
|
695
|
-
* Converge this table's
|
|
702
|
+
* Converge this table's RTE write path into its base table.
|
|
696
703
|
*
|
|
697
|
-
*
|
|
704
|
+
* Freezes once, then triggers compaction and polls until the SSTables that existed
|
|
698
705
|
* at the start is gone. The target set is fixed at the start, so
|
|
699
|
-
*
|
|
706
|
+
* SSTables created *during* the checkpoint are ignored — that is what
|
|
700
707
|
* lets it terminate under write load, and what makes it best-effort: it
|
|
701
708
|
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
|
702
709
|
* any point, and safe to run on a cadence.
|
|
@@ -707,24 +714,24 @@ export declare abstract class Table {
|
|
|
707
714
|
* @returns {Promise<void>}
|
|
708
715
|
* @example
|
|
709
716
|
* ```ts
|
|
710
|
-
* const before = await table.
|
|
711
|
-
* await table.
|
|
712
|
-
* const after = await table.
|
|
717
|
+
* const before = await table.getRTEStats();
|
|
718
|
+
* await table.checkpointRTE();
|
|
719
|
+
* const after = await table.getRTEStats();
|
|
713
720
|
* ```
|
|
714
721
|
*/
|
|
715
|
-
abstract
|
|
722
|
+
abstract checkpointRTE(): Promise<void>;
|
|
716
723
|
/**
|
|
717
|
-
* Read live per-
|
|
724
|
+
* Read live per-table-shard RTE state.
|
|
718
725
|
*
|
|
719
|
-
* Answers "how far behind is my fresh tier", "which
|
|
726
|
+
* Answers "how far behind is my fresh tier", "which table shard is hot", and
|
|
720
727
|
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
|
721
728
|
*
|
|
722
|
-
* Resolves to `undefined` only when the
|
|
723
|
-
* @param {boolean}
|
|
729
|
+
* Resolves to `undefined` only when the RTE write path is not enabled.
|
|
730
|
+
* @param {boolean} includeSstableRows Also count rows per SSTable.
|
|
724
731
|
* Off by default because each count opens an uncached Lance dataset.
|
|
725
|
-
* @returns {Promise<
|
|
732
|
+
* @returns {Promise<RTEStats | undefined>}
|
|
726
733
|
*/
|
|
727
|
-
abstract
|
|
734
|
+
abstract getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
728
735
|
/** Retrieve the version of the table */
|
|
729
736
|
abstract version(): Promise<number>;
|
|
730
737
|
/**
|
|
@@ -933,14 +940,14 @@ export declare class LocalTable extends Table {
|
|
|
933
940
|
updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
|
|
934
941
|
dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
935
942
|
setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
943
|
+
enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
944
|
+
disableRTE(): Promise<void>;
|
|
945
|
+
getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
946
|
+
closeRTEWriters(): Promise<void>;
|
|
947
|
+
flushRTE(): Promise<void>;
|
|
948
|
+
compactRTE(): Promise<void>;
|
|
949
|
+
checkpointRTE(): Promise<void>;
|
|
950
|
+
getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
944
951
|
version(): Promise<number>;
|
|
945
952
|
checkout(version: number | string): Promise<void>;
|
|
946
953
|
checkoutLatest(): Promise<void>;
|