@lancedb/lancedb 0.41.0-beta.7 → 0.41.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/merge.d.ts +5 -5
- package/dist/merge.js +6 -6
- package/dist/native.d.ts +90 -90
- package/dist/native.js +52 -52
- package/dist/query.d.ts +57 -11
- package/dist/query.js +135 -27
- package/dist/table.d.ts +45 -45
- package/dist/table.js +24 -18
- package/package.json +8 -8
package/dist/query.d.ts
CHANGED
|
@@ -3,6 +3,21 @@ import { type IvfPqOptions } from "./indices";
|
|
|
3
3
|
import { JsFullTextQuery, RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, TakeQuery as NativeTakeQuery, VectorQuery as NativeVectorQuery } from "./native";
|
|
4
4
|
import { Reranker } from "./rerankers";
|
|
5
5
|
export declare function RecordBatchIterator(promisedInner: Promise<NativeBatchIterator>): AsyncGenerator<RecordBatch<any>, void, unknown>;
|
|
6
|
+
/**
|
|
7
|
+
* How a query returns blob v2 columns. See {@link QueryExecutionOptions.blobMode}.
|
|
8
|
+
*/
|
|
9
|
+
export type BlobMode = "descriptions" | "bytes";
|
|
10
|
+
/**
|
|
11
|
+
* The table a query reads, and the parts of the query that decide which blob
|
|
12
|
+
* columns `blobMode: "bytes"` fills in.
|
|
13
|
+
*
|
|
14
|
+
* @hidden
|
|
15
|
+
*/
|
|
16
|
+
export interface QueryBlobContext {
|
|
17
|
+
table: NativeTable;
|
|
18
|
+
withRowId: boolean;
|
|
19
|
+
projection?: [string, string][];
|
|
20
|
+
}
|
|
6
21
|
/**
|
|
7
22
|
* Options that control the behavior of a particular query execution
|
|
8
23
|
*/
|
|
@@ -18,6 +33,31 @@ export interface QueryExecutionOptions {
|
|
|
18
33
|
* Timeout for query execution in milliseconds
|
|
19
34
|
*/
|
|
20
35
|
timeoutMs?: number;
|
|
36
|
+
/**
|
|
37
|
+
* How to return blob v2 columns. Applies to `toArrow()` and `toArray()`.
|
|
38
|
+
*
|
|
39
|
+
* - `"descriptions"` (default): each blob is a descriptor. Read the bytes
|
|
40
|
+
* with {@link Table.fetchBlobs} or {@link Table.fetchBlobFiles}.
|
|
41
|
+
* - `"bytes"`: each top-level blob column holds the blob bytes as
|
|
42
|
+
* `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and
|
|
43
|
+
* leaves it out of the result unless {@link QueryBase.withRowId} was
|
|
44
|
+
* called. All the bytes are held in memory.
|
|
45
|
+
*
|
|
46
|
+
* Blobs nested in a struct or a list keep their descriptors.
|
|
47
|
+
*
|
|
48
|
+
* The rows and the bytes come from the same table version. If the table
|
|
49
|
+
* changes while the query runs, the query runs again, up to three times.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```ts
|
|
53
|
+
* const rows = await table
|
|
54
|
+
* .query()
|
|
55
|
+
* .select(["id", "image"])
|
|
56
|
+
* .toArray({ blobMode: "bytes" });
|
|
57
|
+
* const image: Uint8Array | null = rows[0].image;
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
blobMode?: BlobMode;
|
|
21
61
|
}
|
|
22
62
|
export type AnalyzePlanDistributedMetrics = "aggregate" | "per_worker" | "full";
|
|
23
63
|
export interface ColumnOrdering {
|
|
@@ -49,7 +89,11 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
49
89
|
/**
|
|
50
90
|
* @hidden
|
|
51
91
|
*/
|
|
52
|
-
protected
|
|
92
|
+
protected blobContext?: QueryBlobContext;
|
|
93
|
+
/**
|
|
94
|
+
* @hidden
|
|
95
|
+
*/
|
|
96
|
+
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>, blobContext?: QueryBlobContext);
|
|
53
97
|
/**
|
|
54
98
|
* @hidden
|
|
55
99
|
*/
|
|
@@ -102,7 +146,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
102
146
|
/**
|
|
103
147
|
* @hidden
|
|
104
148
|
*/
|
|
105
|
-
protected nativeExecute(options?: Partial<QueryExecutionOptions
|
|
149
|
+
protected nativeExecute(options?: Partial<QueryExecutionOptions>, withRowId?: boolean): Promise<NativeBatchIterator>;
|
|
106
150
|
/**
|
|
107
151
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
108
152
|
* of @see {@link RecordBatch}.
|
|
@@ -125,6 +169,8 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
125
169
|
* Empty results retain the query's output schema.
|
|
126
170
|
*/
|
|
127
171
|
toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable>;
|
|
172
|
+
private collect;
|
|
173
|
+
private blobBytesContext;
|
|
128
174
|
/** Collect the results as an array of objects. */
|
|
129
175
|
toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]>;
|
|
130
176
|
/**
|
|
@@ -185,7 +231,7 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
185
231
|
outputSchema(): Promise<import("./arrow").Schema>;
|
|
186
232
|
}
|
|
187
233
|
export declare class StandardQueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery> extends QueryBase<NativeQueryType> implements ExecutableQuery {
|
|
188
|
-
constructor(inner?: NativeQueryType | Promise<NativeQueryType
|
|
234
|
+
constructor(inner?: NativeQueryType | Promise<NativeQueryType>, blobContext?: QueryBlobContext);
|
|
189
235
|
/**
|
|
190
236
|
* A filter statement to be applied to this query.
|
|
191
237
|
*
|
|
@@ -238,10 +284,10 @@ export declare class StandardQueryBase<NativeQueryType extends NativeQuery | Nat
|
|
|
238
284
|
* Control MemWAL read routing for this query.
|
|
239
285
|
*
|
|
240
286
|
* By default (unset), when the table carries a MemWAL write spec (see
|
|
241
|
-
* {@link Table#
|
|
287
|
+
* {@link Table#enableMemWal}), reads are routed through the LSM scanner so
|
|
242
288
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
243
289
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
244
|
-
* the
|
|
290
|
+
* the SSTables), deduplicated by primary key; a table without a spec
|
|
245
291
|
* reads the base table.
|
|
246
292
|
*
|
|
247
293
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
@@ -250,10 +296,10 @@ export declare class StandardQueryBase<NativeQueryType extends NativeQuery | Nat
|
|
|
250
296
|
*
|
|
251
297
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
252
298
|
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
|
|
253
|
-
* `
|
|
299
|
+
* `useMemWal(false)` is set, because a base-only read would silently exclude
|
|
254
300
|
* un-compacted MemWAL data.
|
|
255
301
|
*/
|
|
256
|
-
|
|
302
|
+
useMemWal(enable: boolean): this;
|
|
257
303
|
}
|
|
258
304
|
/**
|
|
259
305
|
* An interface for a query that can be executed
|
|
@@ -275,7 +321,7 @@ export declare class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
|
|
275
321
|
/**
|
|
276
322
|
* @hidden
|
|
277
323
|
*/
|
|
278
|
-
constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery
|
|
324
|
+
constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>, blobContext?: QueryBlobContext);
|
|
279
325
|
/**
|
|
280
326
|
* @hidden
|
|
281
327
|
*/
|
|
@@ -457,7 +503,7 @@ export declare function createAutoQuery(table: NativeTable, query: string, colum
|
|
|
457
503
|
* @hideconstructor
|
|
458
504
|
*/
|
|
459
505
|
export declare class TakeQuery extends QueryBase<NativeTakeQuery> {
|
|
460
|
-
constructor(inner: NativeTakeQuery);
|
|
506
|
+
constructor(inner: NativeTakeQuery, blobContext?: QueryBlobContext);
|
|
461
507
|
/**
|
|
462
508
|
* Control MemWAL read routing for this take query.
|
|
463
509
|
*
|
|
@@ -467,7 +513,7 @@ export declare class TakeQuery extends QueryBase<NativeTakeQuery> {
|
|
|
467
513
|
*
|
|
468
514
|
* @param enable - `false` reads the base table only.
|
|
469
515
|
*/
|
|
470
|
-
|
|
516
|
+
useMemWal(enable: boolean): this;
|
|
471
517
|
}
|
|
472
518
|
/**
|
|
473
519
|
* A builder for automatic string searches.
|
|
@@ -482,7 +528,7 @@ export declare class AutoQuery extends StandardQueryBase<NativeQuery | NativeVec
|
|
|
482
528
|
private readonly createInner;
|
|
483
529
|
private readonly calls;
|
|
484
530
|
/** @hidden */
|
|
485
|
-
constructor(createInner: () => Promise<NativeQuery | NativeVectorQuery
|
|
531
|
+
constructor(createInner: () => Promise<NativeQuery | NativeVectorQuery>, blobContext?: QueryBlobContext);
|
|
486
532
|
/** @hidden */
|
|
487
533
|
protected doCall(fn: (inner: NativeQuery | NativeVectorQuery) => void): void;
|
|
488
534
|
/** @hidden */
|
package/dist/query.js
CHANGED
|
@@ -21,6 +21,62 @@ async function* RecordBatchIterator(promisedInner) {
|
|
|
21
21
|
yield batches[0];
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
+
const BLOB_BYTES_ATTEMPTS = 3;
|
|
25
|
+
function blobSources(blobColumns, projection) {
|
|
26
|
+
const columns = new Set(blobColumns);
|
|
27
|
+
const pairs = projection ??
|
|
28
|
+
blobColumns.map((column) => [column, column]);
|
|
29
|
+
return new Map(pairs.filter(([, source]) => columns.has(source)));
|
|
30
|
+
}
|
|
31
|
+
async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
32
|
+
const rowIdColumn = table.getChild("_rowid");
|
|
33
|
+
if (table.numRows > 0 && rowIdColumn === null) {
|
|
34
|
+
throw new Error('blobMode "bytes" needs _rowid in the query result');
|
|
35
|
+
}
|
|
36
|
+
const rowIds = rowIdColumn === null
|
|
37
|
+
? []
|
|
38
|
+
: Array.from(rowIdColumn.toArray());
|
|
39
|
+
const replacements = new Map();
|
|
40
|
+
for (const [output, source] of sources) {
|
|
41
|
+
const index = table.schema.fields.findIndex((field) => field.name === output);
|
|
42
|
+
if (index < 0) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
|
|
46
|
+
replacements.set(index, values.map((value) => value ?? null));
|
|
47
|
+
}
|
|
48
|
+
const result = withBinaryColumns(table, replacements);
|
|
49
|
+
if (!dropRowId) {
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
return result.select(result.schema.fields
|
|
53
|
+
.map((field) => field.name)
|
|
54
|
+
.filter((name) => name !== "_rowid"));
|
|
55
|
+
}
|
|
56
|
+
// A fresh field, not a clone: the descriptor field's metadata must not follow
|
|
57
|
+
// the bytes.
|
|
58
|
+
function withBinaryColumns(table, replacements) {
|
|
59
|
+
if (replacements.size === 0) {
|
|
60
|
+
return table;
|
|
61
|
+
}
|
|
62
|
+
const fields = table.schema.fields.map((field, index) => replacements.has(index)
|
|
63
|
+
? new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true)
|
|
64
|
+
: field);
|
|
65
|
+
const schema = new arrow_1.Schema(fields, table.schema.metadata);
|
|
66
|
+
let offset = 0;
|
|
67
|
+
const batches = table.batches.map((batch) => {
|
|
68
|
+
const children = batch.data.children.map((child, index) => {
|
|
69
|
+
const values = replacements.get(index);
|
|
70
|
+
if (values === undefined) {
|
|
71
|
+
return child;
|
|
72
|
+
}
|
|
73
|
+
return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
|
|
74
|
+
});
|
|
75
|
+
offset += batch.numRows;
|
|
76
|
+
return new arrow_1.RecordBatch(schema, (0, arrow_1.makeData)({ type: new arrow_1.Struct(fields), length: batch.numRows, children }));
|
|
77
|
+
});
|
|
78
|
+
return new arrow_1.Table(schema, batches);
|
|
79
|
+
}
|
|
24
80
|
function nearestToNative(inner, vector) {
|
|
25
81
|
const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
|
|
26
82
|
if (raw) {
|
|
@@ -49,10 +105,15 @@ class QueryBase {
|
|
|
49
105
|
/**
|
|
50
106
|
* @hidden
|
|
51
107
|
*/
|
|
52
|
-
|
|
108
|
+
blobContext;
|
|
109
|
+
/**
|
|
110
|
+
* @hidden
|
|
111
|
+
*/
|
|
112
|
+
constructor(inner, blobContext) {
|
|
53
113
|
if (inner !== undefined) {
|
|
54
114
|
this.inner = inner;
|
|
55
115
|
}
|
|
116
|
+
this.blobContext = blobContext;
|
|
56
117
|
}
|
|
57
118
|
// call a function on the inner (either a promise or the actual object)
|
|
58
119
|
/**
|
|
@@ -118,17 +179,25 @@ class QueryBase {
|
|
|
118
179
|
inner.select(columnTuples);
|
|
119
180
|
});
|
|
120
181
|
};
|
|
182
|
+
let projection;
|
|
121
183
|
if (typeof columns === "string") {
|
|
122
184
|
selectColumns([columns]);
|
|
185
|
+
projection = [[columns, columns]];
|
|
123
186
|
}
|
|
124
187
|
else if (Array.isArray(columns)) {
|
|
125
188
|
selectColumns(columns);
|
|
189
|
+
projection = columns.map((column) => [column, column]);
|
|
126
190
|
}
|
|
127
191
|
else if (columns instanceof Map) {
|
|
128
|
-
|
|
192
|
+
projection = Array.from(columns.entries());
|
|
193
|
+
selectMapping(projection);
|
|
129
194
|
}
|
|
130
195
|
else {
|
|
131
|
-
|
|
196
|
+
projection = Object.entries(columns);
|
|
197
|
+
selectMapping(projection);
|
|
198
|
+
}
|
|
199
|
+
if (this.blobContext !== undefined) {
|
|
200
|
+
this.blobContext = { ...this.blobContext, projection };
|
|
132
201
|
}
|
|
133
202
|
return this;
|
|
134
203
|
}
|
|
@@ -141,14 +210,17 @@ class QueryBase {
|
|
|
141
210
|
*/
|
|
142
211
|
withRowId() {
|
|
143
212
|
this.doCall((inner) => inner.withRowId());
|
|
213
|
+
if (this.blobContext !== undefined) {
|
|
214
|
+
this.blobContext = { ...this.blobContext, withRowId: true };
|
|
215
|
+
}
|
|
144
216
|
return this;
|
|
145
217
|
}
|
|
146
218
|
/**
|
|
147
219
|
* @hidden
|
|
148
220
|
*/
|
|
149
|
-
async nativeExecute(options) {
|
|
221
|
+
async nativeExecute(options, withRowId = false) {
|
|
150
222
|
const inner = await this.getInner();
|
|
151
|
-
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
|
|
223
|
+
return inner.execute(options?.maxBatchLength, options?.timeoutMs, withRowId);
|
|
152
224
|
}
|
|
153
225
|
/**
|
|
154
226
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
@@ -177,8 +249,32 @@ class QueryBase {
|
|
|
177
249
|
* Empty results retain the query's output schema.
|
|
178
250
|
*/
|
|
179
251
|
async toArrow(options) {
|
|
252
|
+
const context = this.blobBytesContext(options?.blobMode);
|
|
253
|
+
if (context === undefined) {
|
|
254
|
+
return this.collect(options, false);
|
|
255
|
+
}
|
|
256
|
+
for (let attempt = 1;; attempt++) {
|
|
257
|
+
// The snapshot pins the bytes. The query runs on the live table, so an
|
|
258
|
+
// unchanged version afterwards proves it read the snapshot's version.
|
|
259
|
+
const snapshot = await context.table.querySnapshot();
|
|
260
|
+
const version = await snapshot.version();
|
|
261
|
+
const sources = blobSources(await snapshot.blobColumns(), context.projection);
|
|
262
|
+
if (sources.size === 0) {
|
|
263
|
+
return this.collect(options, false);
|
|
264
|
+
}
|
|
265
|
+
const addRowId = !context.withRowId;
|
|
266
|
+
const table = await this.collect(options, addRowId);
|
|
267
|
+
if ((await context.table.version()) === version) {
|
|
268
|
+
return readBlobBytes(table, sources, snapshot, addRowId);
|
|
269
|
+
}
|
|
270
|
+
if (attempt === BLOB_BYTES_ATTEMPTS) {
|
|
271
|
+
throw new Error(`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async collect(options, withRowId) {
|
|
180
276
|
const batches = [];
|
|
181
|
-
const iterator = this.nativeExecute(options);
|
|
277
|
+
const iterator = this.nativeExecute(options, withRowId);
|
|
182
278
|
for await (const batch of RecordBatchIterator(iterator)) {
|
|
183
279
|
batches.push(batch);
|
|
184
280
|
}
|
|
@@ -187,6 +283,18 @@ class QueryBase {
|
|
|
187
283
|
}
|
|
188
284
|
return new arrow_1.Table(batches);
|
|
189
285
|
}
|
|
286
|
+
blobBytesContext(mode) {
|
|
287
|
+
if (mode === undefined || mode === "descriptions") {
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
290
|
+
if (mode !== "bytes") {
|
|
291
|
+
throw new Error(`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`);
|
|
292
|
+
}
|
|
293
|
+
if (this.blobContext === undefined) {
|
|
294
|
+
throw new Error('blobMode "bytes" is not supported for this query');
|
|
295
|
+
}
|
|
296
|
+
return this.blobContext;
|
|
297
|
+
}
|
|
190
298
|
/** Collect the results as an array of objects. */
|
|
191
299
|
// biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
|
|
192
300
|
async toArray(options) {
|
|
@@ -264,8 +372,8 @@ class QueryBase {
|
|
|
264
372
|
}
|
|
265
373
|
exports.QueryBase = QueryBase;
|
|
266
374
|
class StandardQueryBase extends QueryBase {
|
|
267
|
-
constructor(inner) {
|
|
268
|
-
super(inner);
|
|
375
|
+
constructor(inner, blobContext) {
|
|
376
|
+
super(inner, blobContext);
|
|
269
377
|
}
|
|
270
378
|
/**
|
|
271
379
|
* A filter statement to be applied to this query.
|
|
@@ -364,10 +472,10 @@ class StandardQueryBase extends QueryBase {
|
|
|
364
472
|
* Control MemWAL read routing for this query.
|
|
365
473
|
*
|
|
366
474
|
* By default (unset), when the table carries a MemWAL write spec (see
|
|
367
|
-
* {@link Table#
|
|
475
|
+
* {@link Table#enableMemWal}), reads are routed through the LSM scanner so
|
|
368
476
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
369
477
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
370
|
-
* the
|
|
478
|
+
* the SSTables), deduplicated by primary key; a table without a spec
|
|
371
479
|
* reads the base table.
|
|
372
480
|
*
|
|
373
481
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
@@ -376,11 +484,11 @@ class StandardQueryBase extends QueryBase {
|
|
|
376
484
|
*
|
|
377
485
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
378
486
|
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
|
|
379
|
-
* `
|
|
487
|
+
* `useMemWal(false)` is set, because a base-only read would silently exclude
|
|
380
488
|
* un-compacted MemWAL data.
|
|
381
489
|
*/
|
|
382
|
-
|
|
383
|
-
this.doCall((inner) => inner.
|
|
490
|
+
useMemWal(enable) {
|
|
491
|
+
this.doCall((inner) => inner.useMemWal(enable));
|
|
384
492
|
return this;
|
|
385
493
|
}
|
|
386
494
|
}
|
|
@@ -398,8 +506,8 @@ class VectorQuery extends StandardQueryBase {
|
|
|
398
506
|
/**
|
|
399
507
|
* @hidden
|
|
400
508
|
*/
|
|
401
|
-
constructor(inner) {
|
|
402
|
-
super(inner);
|
|
509
|
+
constructor(inner, blobContext) {
|
|
510
|
+
super(inner, blobContext);
|
|
403
511
|
}
|
|
404
512
|
/**
|
|
405
513
|
* @hidden
|
|
@@ -634,7 +742,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
634
742
|
addQueryVectorToNative(inner, outcome.value);
|
|
635
743
|
return inner;
|
|
636
744
|
})();
|
|
637
|
-
return new VectorQuery(res);
|
|
745
|
+
return new VectorQuery(res, this.blobContext);
|
|
638
746
|
}
|
|
639
747
|
else {
|
|
640
748
|
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
|
|
@@ -696,7 +804,7 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
696
804
|
}
|
|
697
805
|
return nearestToNative(route.table.query(), vector);
|
|
698
806
|
};
|
|
699
|
-
return new AutoQuery(createInner);
|
|
807
|
+
return new AutoQuery(createInner, { table, withRowId: false });
|
|
700
808
|
}
|
|
701
809
|
/**
|
|
702
810
|
* A query that returns a subset of the rows in the table.
|
|
@@ -704,8 +812,8 @@ function createAutoQuery(table, query, columns, getVector) {
|
|
|
704
812
|
* @hideconstructor
|
|
705
813
|
*/
|
|
706
814
|
class TakeQuery extends QueryBase {
|
|
707
|
-
constructor(inner) {
|
|
708
|
-
super(inner);
|
|
815
|
+
constructor(inner, blobContext) {
|
|
816
|
+
super(inner, blobContext);
|
|
709
817
|
}
|
|
710
818
|
/**
|
|
711
819
|
* Control MemWAL read routing for this take query.
|
|
@@ -716,8 +824,8 @@ class TakeQuery extends QueryBase {
|
|
|
716
824
|
*
|
|
717
825
|
* @param enable - `false` reads the base table only.
|
|
718
826
|
*/
|
|
719
|
-
|
|
720
|
-
this.doCall((inner) => inner.
|
|
827
|
+
useMemWal(enable) {
|
|
828
|
+
this.doCall((inner) => inner.useMemWal(enable));
|
|
721
829
|
return this;
|
|
722
830
|
}
|
|
723
831
|
}
|
|
@@ -735,8 +843,8 @@ class AutoQuery extends StandardQueryBase {
|
|
|
735
843
|
createInner;
|
|
736
844
|
calls = [];
|
|
737
845
|
/** @hidden */
|
|
738
|
-
constructor(createInner) {
|
|
739
|
-
super();
|
|
846
|
+
constructor(createInner, blobContext) {
|
|
847
|
+
super(undefined, blobContext);
|
|
740
848
|
this.createInner = createInner;
|
|
741
849
|
}
|
|
742
850
|
/** @hidden */
|
|
@@ -765,7 +873,7 @@ class Query extends StandardQueryBase {
|
|
|
765
873
|
* @hidden
|
|
766
874
|
*/
|
|
767
875
|
constructor(tbl) {
|
|
768
|
-
super(tbl.query());
|
|
876
|
+
super(tbl.query(), { table: tbl, withRowId: false });
|
|
769
877
|
}
|
|
770
878
|
/**
|
|
771
879
|
* Find the nearest vectors to the given query vector.
|
|
@@ -808,12 +916,12 @@ class Query extends StandardQueryBase {
|
|
|
808
916
|
const inner = this.inner;
|
|
809
917
|
if (inner instanceof Promise) {
|
|
810
918
|
const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
|
|
811
|
-
return new VectorQuery(nativeQuery);
|
|
919
|
+
return new VectorQuery(nativeQuery, this.blobContext);
|
|
812
920
|
}
|
|
813
921
|
if (vector instanceof Promise) {
|
|
814
|
-
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
|
|
922
|
+
return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)), this.blobContext);
|
|
815
923
|
}
|
|
816
|
-
return new VectorQuery(nearestToNative(inner, vector));
|
|
924
|
+
return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
|
|
817
925
|
}
|
|
818
926
|
nearestToText(query, columns) {
|
|
819
927
|
this.doCall((inner) => {
|
package/dist/table.d.ts
CHANGED
|
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
|
|
|
3
3
|
import { IndexOptions } from "./indices";
|
|
4
4
|
import { Job } from "./job";
|
|
5
5
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics,
|
|
6
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, MemWalStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
7
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
8
8
|
import { IntoSql } from "./util";
|
|
9
9
|
export { IndexConfig } from "./native";
|
|
10
|
-
export {
|
|
10
|
+
export { TableShardStats, SsTableStats, MemWalStats, MemtableStats, } from "./native";
|
|
11
11
|
/**
|
|
12
12
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
13
13
|
* callback passed to {@link Table.add}.
|
|
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
|
|
|
149
149
|
* `column` value, or upserts of that key can land in different shards and a
|
|
150
150
|
* stale version can win).
|
|
151
151
|
*/
|
|
152
|
-
export interface
|
|
152
|
+
export interface MemWalWriteSpec {
|
|
153
153
|
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
154
154
|
specType: "bucket" | "identity" | "unsharded";
|
|
155
155
|
/** Bucket and identity variants: the sharding column. */
|
|
@@ -620,10 +620,10 @@ export declare abstract class Table {
|
|
|
620
620
|
*/
|
|
621
621
|
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
622
622
|
/**
|
|
623
|
-
* Install an {@link
|
|
623
|
+
* Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
|
|
624
624
|
* LSM-style write path for future `mergeInsert` calls.
|
|
625
625
|
*
|
|
626
|
-
* `
|
|
626
|
+
* `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
|
|
627
627
|
*
|
|
628
628
|
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
|
|
629
629
|
* key (`column` and `numBuckets` required).
|
|
@@ -638,12 +638,12 @@ export declare abstract class Table {
|
|
|
638
638
|
* including ones created later, and skips a kind the MemWAL cannot maintain.
|
|
639
639
|
* Naming them pins an exact set, and a still-building index is rejected
|
|
640
640
|
* rather than quietly omitted.
|
|
641
|
-
* @param {
|
|
641
|
+
* @param {MemWalWriteSpec} spec The sharding spec to install.
|
|
642
642
|
* @returns {Promise<void>}
|
|
643
643
|
* @example
|
|
644
644
|
* ```ts
|
|
645
645
|
* await table.setUnenforcedPrimaryKey("id");
|
|
646
|
-
* await table.
|
|
646
|
+
* await table.enableMemWal({
|
|
647
647
|
* specType: "bucket",
|
|
648
648
|
* column: "id",
|
|
649
649
|
* numBuckets: 16,
|
|
@@ -651,59 +651,59 @@ export declare abstract class Table {
|
|
|
651
651
|
* });
|
|
652
652
|
* ```
|
|
653
653
|
*/
|
|
654
|
-
abstract
|
|
654
|
+
abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
|
|
655
655
|
/**
|
|
656
|
-
* Remove the {@link
|
|
656
|
+
* Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
|
|
657
657
|
* `mergeInsert` write path.
|
|
658
658
|
*
|
|
659
659
|
* Errors if no spec is currently set.
|
|
660
660
|
* @returns {Promise<void>}
|
|
661
661
|
*/
|
|
662
|
-
abstract
|
|
662
|
+
abstract disableMemWal(): Promise<void>;
|
|
663
663
|
/**
|
|
664
|
-
* Read the {@link
|
|
664
|
+
* Read the {@link MemWalWriteSpec} currently installed on this table.
|
|
665
665
|
*
|
|
666
|
-
* Resolves to `undefined` when the MemWAL
|
|
667
|
-
* spec has been set, or it was removed with {@link Table#
|
|
666
|
+
* Resolves to `undefined` when the MemWAL write path is not enabled (no
|
|
667
|
+
* spec has been set, or it was removed with {@link Table#disableMemWal}).
|
|
668
668
|
* The spec is the one installed, including its maintained-index selection:
|
|
669
669
|
* an absent `maintainedIndexes` for every index the table has, an empty
|
|
670
670
|
* array for none.
|
|
671
|
-
* @returns {Promise<
|
|
671
|
+
* @returns {Promise<MemWalWriteSpec | undefined>}
|
|
672
672
|
*/
|
|
673
|
-
abstract
|
|
673
|
+
abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
|
|
674
674
|
/**
|
|
675
675
|
* Drain and close any cached MemWAL shard writers held for this table.
|
|
676
676
|
*
|
|
677
|
-
* When an {@link
|
|
677
|
+
* When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
|
|
678
678
|
* shard writers and caches them for reuse across calls. This closes them,
|
|
679
679
|
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
|
|
680
680
|
* It is a no-op when no writers are cached.
|
|
681
681
|
* @returns {Promise<void>}
|
|
682
682
|
*/
|
|
683
|
-
abstract
|
|
683
|
+
abstract closeMemWalWriters(): Promise<void>;
|
|
684
684
|
/**
|
|
685
|
-
*
|
|
685
|
+
* Freeze every table shard's active memtable into a new SSTable.
|
|
686
686
|
*
|
|
687
|
-
* Returns once the
|
|
687
|
+
* Returns once the freeze is committed. Freezing an empty memtable is a no-op,
|
|
688
688
|
* so this is safe to call repeatedly.
|
|
689
689
|
* @returns {Promise<void>}
|
|
690
690
|
*/
|
|
691
|
-
abstract
|
|
691
|
+
abstract flushMemWal(): Promise<void>;
|
|
692
692
|
/**
|
|
693
|
-
* Trigger a background
|
|
693
|
+
* Trigger a background SSTable compaction pass per table shard.
|
|
694
694
|
*
|
|
695
695
|
* Returns once the passes are *dispatched*, not once they finish — watch
|
|
696
|
-
* {@link Table#
|
|
697
|
-
* {@link Table#
|
|
696
|
+
* {@link Table#getMemWalStats} for progress, or use
|
|
697
|
+
* {@link Table#checkpointMemWal} to wait for convergence.
|
|
698
698
|
* @returns {Promise<void>}
|
|
699
699
|
*/
|
|
700
|
-
abstract
|
|
700
|
+
abstract compactMemWal(): Promise<void>;
|
|
701
701
|
/**
|
|
702
|
-
* Converge this table's
|
|
702
|
+
* Converge this table's MemWAL write path into its base table.
|
|
703
703
|
*
|
|
704
|
-
*
|
|
704
|
+
* Freezes once, then triggers compaction and polls until the SSTables that existed
|
|
705
705
|
* at the start is gone. The target set is fixed at the start, so
|
|
706
|
-
*
|
|
706
|
+
* SSTables created *during* the checkpoint are ignored — that is what
|
|
707
707
|
* lets it terminate under write load, and what makes it best-effort: it
|
|
708
708
|
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
|
709
709
|
* any point, and safe to run on a cadence.
|
|
@@ -714,24 +714,24 @@ export declare abstract class Table {
|
|
|
714
714
|
* @returns {Promise<void>}
|
|
715
715
|
* @example
|
|
716
716
|
* ```ts
|
|
717
|
-
* const before = await table.
|
|
718
|
-
* await table.
|
|
719
|
-
* const after = await table.
|
|
717
|
+
* const before = await table.getMemWalStats();
|
|
718
|
+
* await table.checkpointMemWal();
|
|
719
|
+
* const after = await table.getMemWalStats();
|
|
720
720
|
* ```
|
|
721
721
|
*/
|
|
722
|
-
abstract
|
|
722
|
+
abstract checkpointMemWal(): Promise<void>;
|
|
723
723
|
/**
|
|
724
|
-
* Read live per-
|
|
724
|
+
* Read live per-table-shard MemWAL state.
|
|
725
725
|
*
|
|
726
|
-
* Answers "how far behind is my fresh tier", "which
|
|
726
|
+
* Answers "how far behind is my fresh tier", "which table shard is hot", and
|
|
727
727
|
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
|
728
728
|
*
|
|
729
|
-
* Resolves to `undefined` only when the
|
|
730
|
-
* @param {boolean}
|
|
729
|
+
* Resolves to `undefined` only when the MemWAL write path is not enabled.
|
|
730
|
+
* @param {boolean} includeSstableRows Also count rows per SSTable.
|
|
731
731
|
* Off by default because each count opens an uncached Lance dataset.
|
|
732
|
-
* @returns {Promise<
|
|
732
|
+
* @returns {Promise<MemWalStats | undefined>}
|
|
733
733
|
*/
|
|
734
|
-
abstract
|
|
734
|
+
abstract getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
|
|
735
735
|
/** Retrieve the version of the table */
|
|
736
736
|
abstract version(): Promise<number>;
|
|
737
737
|
/**
|
|
@@ -940,14 +940,14 @@ export declare class LocalTable extends Table {
|
|
|
940
940
|
updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
|
|
941
941
|
dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
942
942
|
setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
943
|
+
enableMemWal(spec: MemWalWriteSpec): Promise<void>;
|
|
944
|
+
disableMemWal(): Promise<void>;
|
|
945
|
+
getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
|
|
946
|
+
closeMemWalWriters(): Promise<void>;
|
|
947
|
+
flushMemWal(): Promise<void>;
|
|
948
|
+
compactMemWal(): Promise<void>;
|
|
949
|
+
checkpointMemWal(): Promise<void>;
|
|
950
|
+
getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
|
|
951
951
|
version(): Promise<number>;
|
|
952
952
|
checkout(version: number | string): Promise<void>;
|
|
953
953
|
checkoutLatest(): Promise<void>;
|