@lancedb/lancedb 0.41.0-beta.8 → 0.42.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +13 -0
- package/dist/arrow.d.ts +1 -0
- package/dist/arrow.js +16 -5
- package/dist/blob.d.ts +25 -2
- package/dist/blob.js +56 -5
- package/dist/catalog.d.ts +6 -1
- package/dist/catalog.js +6 -2
- package/dist/connection.d.ts +36 -89
- package/dist/connection.js +28 -26
- package/dist/embedding/transformers.js +4 -2
- package/dist/expr.d.ts +64 -0
- package/dist/expr.js +148 -0
- package/dist/header.d.ts +1 -1
- package/dist/header.js +1 -1
- package/dist/index.d.ts +10 -8
- package/dist/index.js +17 -6
- package/dist/indices.d.ts +86 -8
- package/dist/indices.js +6 -6
- package/dist/job.d.ts +8 -5
- package/dist/job.js +8 -5
- package/dist/materialized_view.d.ts +7 -3
- package/dist/materialized_view.js +7 -4
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +139 -98
- package/dist/native.js +59 -52
- package/dist/query.d.ts +26 -18
- package/dist/query.js +88 -38
- package/dist/table.d.ts +73 -58
- package/dist/table.js +18 -18
- package/package.json +10 -9
package/dist/query.js
CHANGED
|
@@ -7,6 +7,7 @@ exports.RecordBatchIterator = RecordBatchIterator;
|
|
|
7
7
|
exports.createAutoQuery = createAutoQuery;
|
|
8
8
|
exports.instanceOfFullTextQuery = instanceOfFullTextQuery;
|
|
9
9
|
const arrow_1 = require("./arrow");
|
|
10
|
+
const expr_1 = require("./expr");
|
|
10
11
|
const native_1 = require("./native");
|
|
11
12
|
async function* RecordBatchIterator(promisedInner) {
|
|
12
13
|
const inner = await promisedInner;
|
|
@@ -23,10 +24,25 @@ async function* RecordBatchIterator(promisedInner) {
|
|
|
23
24
|
}
|
|
24
25
|
const BLOB_BYTES_ATTEMPTS = 3;
|
|
25
26
|
function blobSources(blobColumns, projection) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
if (projection === undefined) {
|
|
28
|
+
return blobColumns.map((source) => {
|
|
29
|
+
const [output, ...path] = source.split(".");
|
|
30
|
+
return { output, path, source };
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const sources = [];
|
|
34
|
+
for (const [output, source] of projection) {
|
|
35
|
+
for (const column of blobColumns) {
|
|
36
|
+
if (column === source || column.startsWith(`${source}.`)) {
|
|
37
|
+
sources.push({
|
|
38
|
+
output,
|
|
39
|
+
path: column === source ? [] : column.slice(source.length + 1).split("."),
|
|
40
|
+
source: column,
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return sources;
|
|
30
46
|
}
|
|
31
47
|
async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
32
48
|
const rowIdColumn = table.getChild("_rowid");
|
|
@@ -36,14 +52,21 @@ async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
|
36
52
|
const rowIds = rowIdColumn === null
|
|
37
53
|
? []
|
|
38
54
|
: Array.from(rowIdColumn.toArray());
|
|
55
|
+
const outputFields = new Map(table.schema.fields.map((field) => [field.name, field]));
|
|
39
56
|
const replacements = new Map();
|
|
40
|
-
for (const
|
|
41
|
-
|
|
42
|
-
|
|
57
|
+
for (const { output, path, source } of sources) {
|
|
58
|
+
let field = outputFields.get(output);
|
|
59
|
+
for (const name of path) {
|
|
60
|
+
field =
|
|
61
|
+
field !== undefined && arrow_1.DataType.isStruct(field.type)
|
|
62
|
+
? field.type.children.find((child) => child.name === name)
|
|
63
|
+
: undefined;
|
|
64
|
+
}
|
|
65
|
+
if (field === undefined) {
|
|
43
66
|
continue;
|
|
44
67
|
}
|
|
45
68
|
const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
|
|
46
|
-
replacements.set(
|
|
69
|
+
replacements.set(field, values.map((value) => value ?? null));
|
|
47
70
|
}
|
|
48
71
|
const result = withBinaryColumns(table, replacements);
|
|
49
72
|
if (!dropRowId) {
|
|
@@ -53,27 +76,42 @@ async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
|
53
76
|
.map((field) => field.name)
|
|
54
77
|
.filter((name) => name !== "_rowid"));
|
|
55
78
|
}
|
|
56
|
-
// A fresh field, not a clone: the descriptor field's metadata must not follow
|
|
57
|
-
// the bytes.
|
|
58
79
|
function withBinaryColumns(table, replacements) {
|
|
59
80
|
if (replacements.size === 0) {
|
|
60
81
|
return table;
|
|
61
82
|
}
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
83
|
+
const replaceField = (field) => {
|
|
84
|
+
if (replacements.has(field)) {
|
|
85
|
+
// The descriptor's extension and encoding metadata must not follow bytes.
|
|
86
|
+
return new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true);
|
|
87
|
+
}
|
|
88
|
+
if (arrow_1.DataType.isStruct(field.type)) {
|
|
89
|
+
const children = field.type.children.map(replaceField);
|
|
90
|
+
if (children.some((child, i) => child !== field.type.children[i])) {
|
|
91
|
+
return field.clone({ type: new arrow_1.Struct(children) });
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return field;
|
|
95
|
+
};
|
|
96
|
+
const fields = table.schema.fields.map(replaceField);
|
|
65
97
|
const schema = new arrow_1.Schema(fields, table.schema.metadata);
|
|
66
98
|
let offset = 0;
|
|
67
99
|
const batches = table.batches.map((batch) => {
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
return child;
|
|
100
|
+
const replaceData = (data, field, updated) => {
|
|
101
|
+
if (field === updated) {
|
|
102
|
+
return data;
|
|
72
103
|
}
|
|
73
|
-
|
|
74
|
-
|
|
104
|
+
const values = replacements.get(field);
|
|
105
|
+
if (values !== undefined) {
|
|
106
|
+
return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
|
|
107
|
+
}
|
|
108
|
+
const children = data.children.map((child, index) => replaceData(child, field.type.children[index], updated.type.children[index]));
|
|
109
|
+
// Keep the struct's validity bitmap and offset, including null parents.
|
|
110
|
+
return data.clone(updated.type, undefined, undefined, undefined, undefined, children);
|
|
111
|
+
};
|
|
112
|
+
const children = batch.data.children.map((child, index) => replaceData(child, table.schema.fields[index], fields[index]));
|
|
75
113
|
offset += batch.numRows;
|
|
76
|
-
return new arrow_1.RecordBatch(schema,
|
|
114
|
+
return new arrow_1.RecordBatch(schema, batch.data.clone(new arrow_1.Struct(fields), undefined, undefined, undefined, undefined, children));
|
|
77
115
|
});
|
|
78
116
|
return new arrow_1.Table(schema, batches);
|
|
79
117
|
}
|
|
@@ -259,7 +297,7 @@ class QueryBase {
|
|
|
259
297
|
const snapshot = await context.table.querySnapshot();
|
|
260
298
|
const version = await snapshot.version();
|
|
261
299
|
const sources = blobSources(await snapshot.blobColumns(), context.projection);
|
|
262
|
-
if (sources.
|
|
300
|
+
if (sources.length === 0) {
|
|
263
301
|
return this.collect(options, false);
|
|
264
302
|
}
|
|
265
303
|
const addRowId = !context.withRowId;
|
|
@@ -378,11 +416,16 @@ class StandardQueryBase extends QueryBase {
|
|
|
378
416
|
/**
|
|
379
417
|
* A filter statement to be applied to this query.
|
|
380
418
|
*
|
|
381
|
-
* The filter
|
|
419
|
+
* The filter may be an SQL query string or an expression built with `col`,
|
|
420
|
+
* which preserves case-sensitive column names. For example:
|
|
382
421
|
* @example
|
|
383
422
|
* x > 10
|
|
384
423
|
* y > 0 AND y < 100
|
|
385
424
|
* x > 5 OR y = 'test'
|
|
425
|
+
* @example
|
|
426
|
+
* ```ts
|
|
427
|
+
* table.query().where(col("firstName").eq("Alice"));
|
|
428
|
+
* ```
|
|
386
429
|
*
|
|
387
430
|
* Filtering performance can often be improved by creating a scalar index
|
|
388
431
|
* on the filter column(s).
|
|
@@ -391,7 +434,14 @@ class StandardQueryBase extends QueryBase {
|
|
|
391
434
|
* than replacing the previous filter.
|
|
392
435
|
*/
|
|
393
436
|
where(predicate) {
|
|
394
|
-
this.doCall((inner) =>
|
|
437
|
+
this.doCall((inner) => {
|
|
438
|
+
if (predicate instanceof expr_1.Expr) {
|
|
439
|
+
inner.onlyIf(predicate.toSql());
|
|
440
|
+
}
|
|
441
|
+
else {
|
|
442
|
+
inner.onlyIf(predicate);
|
|
443
|
+
}
|
|
444
|
+
});
|
|
395
445
|
return this;
|
|
396
446
|
}
|
|
397
447
|
/**
|
|
@@ -469,26 +519,26 @@ class StandardQueryBase extends QueryBase {
|
|
|
469
519
|
return this;
|
|
470
520
|
}
|
|
471
521
|
/**
|
|
472
|
-
* Control
|
|
522
|
+
* Control RTE read routing for this query.
|
|
473
523
|
*
|
|
474
|
-
* By default (unset), when the table carries
|
|
475
|
-
* {@link Table#
|
|
524
|
+
* By default (unset), when the table carries an RTE write spec (see
|
|
525
|
+
* {@link Table#enableRTE}), reads are routed through the LSM scanner so
|
|
476
526
|
* they also return data written via the `mergeInsert` LSM path that has not yet
|
|
477
527
|
* been compacted into the base table (the active/frozen in-memory memtables and
|
|
478
528
|
* the SSTables), deduplicated by primary key; a table without a spec
|
|
479
529
|
* reads the base table.
|
|
480
530
|
*
|
|
481
531
|
* @param enable - `true` forces the LSM scanner and errors if the table has no
|
|
482
|
-
*
|
|
532
|
+
* RTE write spec. `false` bypasses the RTE and reads the base table only,
|
|
483
533
|
* even when a spec is present.
|
|
484
534
|
*
|
|
485
535
|
* Note: the LSM scanner does not support every query shape (e.g. reranking,
|
|
486
|
-
* hybrid search, `orderBy`). On
|
|
487
|
-
* `
|
|
488
|
-
* un-compacted
|
|
536
|
+
* hybrid search, `orderBy`). On an RTE table those shapes error unless
|
|
537
|
+
* `useRTE(false)` is set, because a base-only read would silently exclude
|
|
538
|
+
* un-compacted RTE data.
|
|
489
539
|
*/
|
|
490
|
-
|
|
491
|
-
this.doCall((inner) => inner.
|
|
540
|
+
useRTE(enable) {
|
|
541
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
492
542
|
return this;
|
|
493
543
|
}
|
|
494
544
|
}
|
|
@@ -554,7 +604,7 @@ class VectorQuery extends StandardQueryBase {
|
|
|
554
604
|
* probing may continue when too few candidates are found.
|
|
555
605
|
* Any explicit effort, including 0.5, overrides nprobes in either call order.
|
|
556
606
|
* Explicit minimum and maximum bounds still apply. This is not a recall
|
|
557
|
-
* guarantee and does not change refinement or HNSW ef. The
|
|
607
|
+
* guarantee and does not change refinement or HNSW ef. The RTE LSM
|
|
558
608
|
* scanner does not yet support this option.
|
|
559
609
|
*
|
|
560
610
|
* @example
|
|
@@ -816,16 +866,16 @@ class TakeQuery extends QueryBase {
|
|
|
816
866
|
super(inner, blobContext);
|
|
817
867
|
}
|
|
818
868
|
/**
|
|
819
|
-
* Control
|
|
869
|
+
* Control RTE read routing for this take query.
|
|
820
870
|
*
|
|
821
|
-
* `false` bypasses the
|
|
871
|
+
* `false` bypasses the RTE and reads the base table only — the escape hatch,
|
|
822
872
|
* since take-by-row-id/offset is not supported on the LSM scanner and, on a
|
|
823
|
-
*
|
|
873
|
+
* RTE table, auto-routes to it and errors otherwise.
|
|
824
874
|
*
|
|
825
875
|
* @param enable - `false` reads the base table only.
|
|
826
876
|
*/
|
|
827
|
-
|
|
828
|
-
this.doCall((inner) => inner.
|
|
877
|
+
useRTE(enable) {
|
|
878
|
+
this.doCall((inner) => inner.useRTE(enable));
|
|
829
879
|
return this;
|
|
830
880
|
}
|
|
831
881
|
}
|
package/dist/table.d.ts
CHANGED
|
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
|
|
|
3
3
|
import { IndexOptions } from "./indices";
|
|
4
4
|
import { Job } from "./job";
|
|
5
5
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics,
|
|
6
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
7
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
8
8
|
import { IntoSql } from "./util";
|
|
9
9
|
export { IndexConfig } from "./native";
|
|
10
|
-
export { TableShardStats, SsTableStats,
|
|
10
|
+
export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
|
|
11
11
|
/**
|
|
12
12
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
13
13
|
* callback passed to {@link Table.add}.
|
|
@@ -68,8 +68,8 @@ export interface AddDataOptions {
|
|
|
68
68
|
/**
|
|
69
69
|
* Whether blob URIs outside registered external bases may be written.
|
|
70
70
|
*
|
|
71
|
-
* Defaults to `false`. This option is supported only
|
|
72
|
-
*
|
|
71
|
+
* Defaults to `false`. This option is supported only in LanceDB Embedded;
|
|
72
|
+
* LanceDB Distributed returns an error when it is enabled. An enabled write
|
|
73
73
|
* stores the absolute URI reference without registering a base or copying
|
|
74
74
|
* the external object into the database. The object must remain accessible
|
|
75
75
|
* when the blob is read later.
|
|
@@ -139,8 +139,8 @@ export type TokenizeTableOptions = {
|
|
|
139
139
|
column?: never;
|
|
140
140
|
};
|
|
141
141
|
/**
|
|
142
|
-
* Specification
|
|
143
|
-
* `mergeInsert
|
|
142
|
+
* Specification enabling the realtime engine (RTE), which routes
|
|
143
|
+
* `mergeInsert` through Lance's MemWAL LSM-style write path.
|
|
144
144
|
*
|
|
145
145
|
* `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
|
|
146
146
|
* `column` and `numBuckets` are required; for `"identity"`, `column` is
|
|
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
|
|
|
149
149
|
* `column` value, or upserts of that key can land in different shards and a
|
|
150
150
|
* stale version can win).
|
|
151
151
|
*/
|
|
152
|
-
export interface
|
|
152
|
+
export interface RTEWriteSpec {
|
|
153
153
|
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
154
154
|
specType: "bucket" | "identity" | "unsharded";
|
|
155
155
|
/** Bucket and identity variants: the sharding column. */
|
|
@@ -157,7 +157,7 @@ export interface MemWalWriteSpec {
|
|
|
157
157
|
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
158
158
|
numBuckets?: number;
|
|
159
159
|
/**
|
|
160
|
-
* Indexes the
|
|
160
|
+
* Indexes the RTE keeps up to date. Omit for every index the table has,
|
|
161
161
|
* including ones created later. Pass `[]` for none.
|
|
162
162
|
*/
|
|
163
163
|
maintainedIndexes?: string[];
|
|
@@ -297,6 +297,11 @@ export declare abstract class Table {
|
|
|
297
297
|
* }),
|
|
298
298
|
* });
|
|
299
299
|
* @example
|
|
300
|
+
* // Or size the partitions by row count instead of choosing how many.
|
|
301
|
+
* await table.createIndex("vector", {
|
|
302
|
+
* config: lancedb.Index.ivfPq({ targetPartitionSize: 4096 }),
|
|
303
|
+
* });
|
|
304
|
+
* @example
|
|
300
305
|
* // Or create a Scalar index
|
|
301
306
|
* await table.createIndex("my_float_col");
|
|
302
307
|
*/
|
|
@@ -344,7 +349,7 @@ export declare abstract class Table {
|
|
|
344
349
|
* Large columns (e.g. high-dimensional vectors or binary data) may not be
|
|
345
350
|
* practical to prewarm.
|
|
346
351
|
*
|
|
347
|
-
* This feature is currently only supported
|
|
352
|
+
* This feature is currently only supported in LanceDB Distributed.
|
|
348
353
|
*/
|
|
349
354
|
abstract prewarmData(columns?: string[]): Promise<void>;
|
|
350
355
|
/**
|
|
@@ -477,6 +482,10 @@ export declare abstract class Table {
|
|
|
477
482
|
/**
|
|
478
483
|
* Add new columns with defined values.
|
|
479
484
|
*
|
|
485
|
+
* LanceDB Embedded computed-column declaration and refresh require a source
|
|
486
|
+
* build with the `experimental-functions` Cargo feature. Published packages
|
|
487
|
+
* disable it; LanceDB Distributed operations remain available.
|
|
488
|
+
*
|
|
480
489
|
* The `{ computed }` form stores the expression rather than evaluating it
|
|
481
490
|
* now: the column is committed with no values, and rows get them from
|
|
482
491
|
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
|
|
@@ -487,7 +496,7 @@ export declare abstract class Table {
|
|
|
487
496
|
* declaration reads a column, that column cannot be renamed, retyped or
|
|
488
497
|
* dropped.
|
|
489
498
|
*
|
|
490
|
-
*
|
|
499
|
+
* In LanceDB Distributed the expression is planned by the
|
|
491
500
|
* server, and the refresh runs as a server job -- see
|
|
492
501
|
* {@link Table#refreshColumnAsync}.
|
|
493
502
|
* @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
|
|
@@ -509,11 +518,12 @@ export declare abstract class Table {
|
|
|
509
518
|
}): Promise<AddColumnsResult>;
|
|
510
519
|
/**
|
|
511
520
|
* Fill the rows of a computed column that hold no value yet.
|
|
521
|
+
* Local refresh requires a source build with `experimental-functions`.
|
|
512
522
|
*
|
|
513
523
|
* Rows appended since the last refresh are filled by the next one, and
|
|
514
524
|
* rows whose inputs changed since they were computed are recomputed;
|
|
515
|
-
* everything else is left as it is.
|
|
516
|
-
* runs as a server job, through {@link Table#refreshColumnAsync}.
|
|
525
|
+
* everything else is left as it is. LanceDB Embedded only: in LanceDB
|
|
526
|
+
* Distributed a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
|
|
517
527
|
* @param {string} column The name of the computed column to fill.
|
|
518
528
|
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
|
|
519
529
|
* number of rows filled and the new version number of the table.
|
|
@@ -522,12 +532,15 @@ export declare abstract class Table {
|
|
|
522
532
|
/**
|
|
523
533
|
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
524
534
|
* job instead of blocking until it completes.
|
|
535
|
+
* LanceDB Embedded refresh requires a source build with
|
|
536
|
+
* `experimental-functions`. LanceDB Distributed jobs do not require this
|
|
537
|
+
* feature.
|
|
525
538
|
*
|
|
526
539
|
* The job may already be complete when returned; callers must not assume
|
|
527
540
|
* the column is filled until {@link Job.wait} resolves. Invalid input --
|
|
528
541
|
* an unknown column, or one that is not computed -- rejects here rather
|
|
529
|
-
* than failing the job.
|
|
530
|
-
* LanceDB
|
|
542
|
+
* than failing the job. In LanceDB Embedded the job runs in-process; in
|
|
543
|
+
* LanceDB Distributed it is the server's backfill job.
|
|
531
544
|
* @param {string} column The name of the computed column to fill.
|
|
532
545
|
* @example
|
|
533
546
|
* ```ts
|
|
@@ -543,7 +556,7 @@ export declare abstract class Table {
|
|
|
543
556
|
* A refresh running under a skip policy records each row it skipped with
|
|
544
557
|
* the input that failed and the error. This lists those records, newest
|
|
545
558
|
* job first, plus a summary for any fragment whose per-row detail was
|
|
546
|
-
* capped. LanceDB
|
|
559
|
+
* capped. LanceDB Distributed only; reading errors needs read
|
|
547
560
|
* access to the table, since a message carries the value that failed.
|
|
548
561
|
* @param {FunctionErrorsOptions} options Optional filters: `jobId`,
|
|
549
562
|
* `column`, and `limit` (server default 10000, cap 100000).
|
|
@@ -562,7 +575,7 @@ export declare abstract class Table {
|
|
|
562
575
|
* it: rejects tables that carry no view definition.
|
|
563
576
|
* @ignore
|
|
564
577
|
*/
|
|
565
|
-
abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
578
|
+
abstract refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
566
579
|
/** @ignore */
|
|
567
580
|
abstract materializedViewDefinition(): Promise<string>;
|
|
568
581
|
/**
|
|
@@ -582,8 +595,6 @@ export declare abstract class Table {
|
|
|
582
595
|
* - `lancedb:description`: for a human-readable description of a field.
|
|
583
596
|
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
|
584
597
|
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
|
585
|
-
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
|
586
|
-
* `feature_v2` might be in the same logical column.
|
|
587
598
|
* - `lancedb:status`: for status options (`production`, `candidate`,
|
|
588
599
|
* `deprecated`, `archived`) to designate the current life cycle state of
|
|
589
600
|
* this column.
|
|
@@ -620,10 +631,10 @@ export declare abstract class Table {
|
|
|
620
631
|
*/
|
|
621
632
|
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
622
633
|
/**
|
|
623
|
-
* Install an {@link
|
|
634
|
+
* Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
|
|
624
635
|
* LSM-style write path for future `mergeInsert` calls.
|
|
625
636
|
*
|
|
626
|
-
* `
|
|
637
|
+
* `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
|
|
627
638
|
*
|
|
628
639
|
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
|
|
629
640
|
* key (`column` and `numBuckets` required).
|
|
@@ -635,15 +646,15 @@ export declare abstract class Table {
|
|
|
635
646
|
* requires it to be the single column being bucketed.
|
|
636
647
|
*
|
|
637
648
|
* Omitting `maintainedIndexes` maintains every index the table has,
|
|
638
|
-
* including ones created later, and skips a kind the
|
|
649
|
+
* including ones created later, and skips a kind the RTE cannot maintain.
|
|
639
650
|
* Naming them pins an exact set, and a still-building index is rejected
|
|
640
651
|
* rather than quietly omitted.
|
|
641
|
-
* @param {
|
|
652
|
+
* @param {RTEWriteSpec} spec The sharding spec to install.
|
|
642
653
|
* @returns {Promise<void>}
|
|
643
654
|
* @example
|
|
644
655
|
* ```ts
|
|
645
656
|
* await table.setUnenforcedPrimaryKey("id");
|
|
646
|
-
* await table.
|
|
657
|
+
* await table.enableRTE({
|
|
647
658
|
* specType: "bucket",
|
|
648
659
|
* column: "id",
|
|
649
660
|
* numBuckets: 16,
|
|
@@ -651,36 +662,36 @@ export declare abstract class Table {
|
|
|
651
662
|
* });
|
|
652
663
|
* ```
|
|
653
664
|
*/
|
|
654
|
-
abstract
|
|
665
|
+
abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
655
666
|
/**
|
|
656
|
-
* Remove the {@link
|
|
667
|
+
* Remove the {@link RTEWriteSpec} from this table, reverting to the standard
|
|
657
668
|
* `mergeInsert` write path.
|
|
658
669
|
*
|
|
659
670
|
* Errors if no spec is currently set.
|
|
660
671
|
* @returns {Promise<void>}
|
|
661
672
|
*/
|
|
662
|
-
abstract
|
|
673
|
+
abstract disableRTE(): Promise<void>;
|
|
663
674
|
/**
|
|
664
|
-
* Read the {@link
|
|
675
|
+
* Read the {@link RTEWriteSpec} currently installed on this table.
|
|
665
676
|
*
|
|
666
|
-
* Resolves to `undefined` when the
|
|
667
|
-
* spec has been set, or it was removed with {@link Table#
|
|
677
|
+
* Resolves to `undefined` when the RTE write path is not enabled (no
|
|
678
|
+
* spec has been set, or it was removed with {@link Table#disableRTE}).
|
|
668
679
|
* The spec is the one installed, including its maintained-index selection:
|
|
669
680
|
* an absent `maintainedIndexes` for every index the table has, an empty
|
|
670
681
|
* array for none.
|
|
671
|
-
* @returns {Promise<
|
|
682
|
+
* @returns {Promise<RTEWriteSpec | undefined>}
|
|
672
683
|
*/
|
|
673
|
-
abstract
|
|
684
|
+
abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
674
685
|
/**
|
|
675
|
-
* Drain and close any cached
|
|
686
|
+
* Drain and close any cached RTE shard writers held for this table.
|
|
676
687
|
*
|
|
677
|
-
* When an {@link
|
|
688
|
+
* When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
|
|
678
689
|
* shard writers and caches them for reuse across calls. This closes them,
|
|
679
690
|
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
|
|
680
691
|
* It is a no-op when no writers are cached.
|
|
681
692
|
* @returns {Promise<void>}
|
|
682
693
|
*/
|
|
683
|
-
abstract
|
|
694
|
+
abstract closeRTEWriters(): Promise<void>;
|
|
684
695
|
/**
|
|
685
696
|
* Freeze every table shard's active memtable into a new SSTable.
|
|
686
697
|
*
|
|
@@ -688,18 +699,18 @@ export declare abstract class Table {
|
|
|
688
699
|
* so this is safe to call repeatedly.
|
|
689
700
|
* @returns {Promise<void>}
|
|
690
701
|
*/
|
|
691
|
-
abstract
|
|
702
|
+
abstract flushRTE(): Promise<void>;
|
|
692
703
|
/**
|
|
693
704
|
* Trigger a background SSTable compaction pass per table shard.
|
|
694
705
|
*
|
|
695
706
|
* Returns once the passes are *dispatched*, not once they finish — watch
|
|
696
|
-
* {@link Table#
|
|
697
|
-
* {@link Table#
|
|
707
|
+
* {@link Table#getRTEStats} for progress, or use
|
|
708
|
+
* {@link Table#checkpointRTE} to wait for convergence.
|
|
698
709
|
* @returns {Promise<void>}
|
|
699
710
|
*/
|
|
700
|
-
abstract
|
|
711
|
+
abstract compactRTE(): Promise<void>;
|
|
701
712
|
/**
|
|
702
|
-
* Converge this table's
|
|
713
|
+
* Converge this table's RTE write path into its base table.
|
|
703
714
|
*
|
|
704
715
|
* Freezes once, then triggers compaction and polls until the SSTables that existed
|
|
705
716
|
* at the start is gone. The target set is fixed at the start, so
|
|
@@ -714,24 +725,24 @@ export declare abstract class Table {
|
|
|
714
725
|
* @returns {Promise<void>}
|
|
715
726
|
* @example
|
|
716
727
|
* ```ts
|
|
717
|
-
* const before = await table.
|
|
718
|
-
* await table.
|
|
719
|
-
* const after = await table.
|
|
728
|
+
* const before = await table.getRTEStats();
|
|
729
|
+
* await table.checkpointRTE();
|
|
730
|
+
* const after = await table.getRTEStats();
|
|
720
731
|
* ```
|
|
721
732
|
*/
|
|
722
|
-
abstract
|
|
733
|
+
abstract checkpointRTE(): Promise<void>;
|
|
723
734
|
/**
|
|
724
|
-
* Read live per-table-shard
|
|
735
|
+
* Read live per-table-shard RTE state.
|
|
725
736
|
*
|
|
726
737
|
* Answers "how far behind is my fresh tier", "which table shard is hot", and
|
|
727
738
|
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
|
728
739
|
*
|
|
729
|
-
* Resolves to `undefined` only when the
|
|
740
|
+
* Resolves to `undefined` only when the RTE write path is not enabled.
|
|
730
741
|
* @param {boolean} includeSstableRows Also count rows per SSTable.
|
|
731
742
|
* Off by default because each count opens an uncached Lance dataset.
|
|
732
|
-
* @returns {Promise<
|
|
743
|
+
* @returns {Promise<RTEStats | undefined>}
|
|
733
744
|
*/
|
|
734
|
-
abstract
|
|
745
|
+
abstract getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
735
746
|
/** Retrieve the version of the table */
|
|
736
747
|
abstract version(): Promise<number>;
|
|
737
748
|
/**
|
|
@@ -841,8 +852,12 @@ export declare abstract class Table {
|
|
|
841
852
|
* Specify exactly one of `column` or `indexName`.
|
|
842
853
|
*
|
|
843
854
|
* Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
|
|
844
|
-
* the client process from index metadata.
|
|
845
|
-
* same tokenizer model files must also exist locally
|
|
855
|
+
* the client process from index metadata. In LanceDB Distributed, this means the
|
|
856
|
+
* same tokenizer model files must also exist locally (under
|
|
857
|
+
* `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
|
|
858
|
+
* mismatch cannot be detected: the returned tokens may differ from what the
|
|
859
|
+
* server's index produces. If the model files are missing, an error naming
|
|
860
|
+
* the tokenizer is raised.
|
|
846
861
|
*/
|
|
847
862
|
abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
|
|
848
863
|
/** Return the table as an arrow table */
|
|
@@ -934,20 +949,20 @@ export declare class LocalTable extends Table {
|
|
|
934
949
|
refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
935
950
|
refreshColumnAsync(column: string): Promise<Job>;
|
|
936
951
|
functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
937
|
-
refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
952
|
+
refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
938
953
|
materializedViewDefinition(): Promise<string>;
|
|
939
954
|
alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
940
955
|
updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
|
|
941
956
|
dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
942
957
|
setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
958
|
+
enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
959
|
+
disableRTE(): Promise<void>;
|
|
960
|
+
getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
961
|
+
closeRTEWriters(): Promise<void>;
|
|
962
|
+
flushRTE(): Promise<void>;
|
|
963
|
+
compactRTE(): Promise<void>;
|
|
964
|
+
checkpointRTE(): Promise<void>;
|
|
965
|
+
getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
|
|
951
966
|
version(): Promise<number>;
|
|
952
967
|
checkout(version: number | string): Promise<void>;
|
|
953
968
|
checkoutLatest(): Promise<void>;
|
package/dist/table.js
CHANGED
|
@@ -286,8 +286,8 @@ class LocalTable extends Table {
|
|
|
286
286
|
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
287
287
|
return await this.inner.functionErrors(options);
|
|
288
288
|
}
|
|
289
|
-
async refreshMaterializedView(sourceVersion) {
|
|
290
|
-
return await this.inner.refreshMaterializedView(sourceVersion);
|
|
289
|
+
async refreshMaterializedView(sourceVersion, full) {
|
|
290
|
+
return await this.inner.refreshMaterializedView(sourceVersion, full);
|
|
291
291
|
}
|
|
292
292
|
async materializedViewDefinition() {
|
|
293
293
|
return await this.inner.materializedViewDefinition();
|
|
@@ -326,32 +326,32 @@ class LocalTable extends Table {
|
|
|
326
326
|
const cols = typeof columns === "string" ? [columns] : columns;
|
|
327
327
|
return await this.inner.setUnenforcedPrimaryKey(cols);
|
|
328
328
|
}
|
|
329
|
-
async
|
|
330
|
-
return await this.inner.
|
|
329
|
+
async enableRTE(spec) {
|
|
330
|
+
return await this.inner.enableRTE(spec);
|
|
331
331
|
}
|
|
332
|
-
async
|
|
333
|
-
return await this.inner.
|
|
332
|
+
async disableRTE() {
|
|
333
|
+
return await this.inner.disableRTE();
|
|
334
334
|
}
|
|
335
|
-
async
|
|
335
|
+
async getRTESpec() {
|
|
336
336
|
// The native binding types `specType` as a plain `string`; narrow it back
|
|
337
337
|
// to the public union. The Rust `From` impl only ever emits one of the
|
|
338
338
|
// three valid values, so the cast is safe.
|
|
339
|
-
return ((await this.inner.
|
|
339
|
+
return ((await this.inner.getRTESpec()) ?? undefined);
|
|
340
340
|
}
|
|
341
|
-
async
|
|
342
|
-
return await this.inner.
|
|
341
|
+
async closeRTEWriters() {
|
|
342
|
+
return await this.inner.closeRTEWriters();
|
|
343
343
|
}
|
|
344
|
-
async
|
|
345
|
-
return await this.inner.
|
|
344
|
+
async flushRTE() {
|
|
345
|
+
return await this.inner.flushRTE();
|
|
346
346
|
}
|
|
347
|
-
async
|
|
348
|
-
return await this.inner.
|
|
347
|
+
async compactRTE() {
|
|
348
|
+
return await this.inner.compactRTE();
|
|
349
349
|
}
|
|
350
|
-
async
|
|
351
|
-
return await this.inner.
|
|
350
|
+
async checkpointRTE() {
|
|
351
|
+
return await this.inner.checkpointRTE();
|
|
352
352
|
}
|
|
353
|
-
async
|
|
354
|
-
return (await this.inner.
|
|
353
|
+
async getRTEStats(includeSstableRows = false) {
|
|
354
|
+
return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined;
|
|
355
355
|
}
|
|
356
356
|
async version() {
|
|
357
357
|
return await this.inner.version();
|
package/package.json
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"ann"
|
|
12
12
|
],
|
|
13
13
|
"private": false,
|
|
14
|
-
"version": "0.
|
|
14
|
+
"version": "0.42.0-beta.0",
|
|
15
15
|
"main": "dist/index.js",
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js",
|
|
@@ -97,7 +97,8 @@
|
|
|
97
97
|
"postdocs": "node typedoc_post_process.js",
|
|
98
98
|
"lint": "biome check . && biome format .",
|
|
99
99
|
"lint-fix": "biome check --write . && biome format --write .",
|
|
100
|
-
"
|
|
100
|
+
"check:native-symbols": "node scripts/check-native-symbols.js",
|
|
101
|
+
"prepublishOnly": "pnpm check:native-symbols && napi prepublish -t npm",
|
|
101
102
|
"test": "jest --verbose",
|
|
102
103
|
"integration": "S3_TEST=1 pnpm test",
|
|
103
104
|
"universal": "napi universalize",
|
|
@@ -110,13 +111,13 @@
|
|
|
110
111
|
"optionalDependencies": {
|
|
111
112
|
"@huggingface/transformers": "3.0.2",
|
|
112
113
|
"openai": "4.29.2",
|
|
113
|
-
"@lancedb/lancedb-darwin-arm64": "0.
|
|
114
|
-
"@lancedb/lancedb-linux-x64-gnu": "0.
|
|
115
|
-
"@lancedb/lancedb-linux-arm64-gnu": "0.
|
|
116
|
-
"@lancedb/lancedb-linux-x64-musl": "0.
|
|
117
|
-
"@lancedb/lancedb-linux-arm64-musl": "0.
|
|
118
|
-
"@lancedb/lancedb-win32-x64-msvc": "0.
|
|
119
|
-
"@lancedb/lancedb-win32-arm64-msvc": "0.
|
|
114
|
+
"@lancedb/lancedb-darwin-arm64": "0.42.0-beta.0",
|
|
115
|
+
"@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.0",
|
|
116
|
+
"@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.0",
|
|
117
|
+
"@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.0",
|
|
118
|
+
"@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.0",
|
|
119
|
+
"@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.0",
|
|
120
|
+
"@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.0"
|
|
120
121
|
},
|
|
121
122
|
"peerDependencies": {
|
|
122
123
|
"@types/node": ">=22",
|