@lancedb/lancedb 0.41.0-beta.8 → 0.42.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/query.js CHANGED
@@ -7,6 +7,7 @@ exports.RecordBatchIterator = RecordBatchIterator;
7
7
  exports.createAutoQuery = createAutoQuery;
8
8
  exports.instanceOfFullTextQuery = instanceOfFullTextQuery;
9
9
  const arrow_1 = require("./arrow");
10
+ const expr_1 = require("./expr");
10
11
  const native_1 = require("./native");
11
12
  async function* RecordBatchIterator(promisedInner) {
12
13
  const inner = await promisedInner;
@@ -23,10 +24,25 @@ async function* RecordBatchIterator(promisedInner) {
23
24
  }
24
25
  const BLOB_BYTES_ATTEMPTS = 3;
25
26
  function blobSources(blobColumns, projection) {
26
- const columns = new Set(blobColumns);
27
- const pairs = projection ??
28
- blobColumns.map((column) => [column, column]);
29
- return new Map(pairs.filter(([, source]) => columns.has(source)));
27
+ if (projection === undefined) {
28
+ return blobColumns.map((source) => {
29
+ const [output, ...path] = source.split(".");
30
+ return { output, path, source };
31
+ });
32
+ }
33
+ const sources = [];
34
+ for (const [output, source] of projection) {
35
+ for (const column of blobColumns) {
36
+ if (column === source || column.startsWith(`${source}.`)) {
37
+ sources.push({
38
+ output,
39
+ path: column === source ? [] : column.slice(source.length + 1).split("."),
40
+ source: column,
41
+ });
42
+ }
43
+ }
44
+ }
45
+ return sources;
30
46
  }
31
47
  async function readBlobBytes(table, sources, snapshot, dropRowId) {
32
48
  const rowIdColumn = table.getChild("_rowid");
@@ -36,14 +52,21 @@ async function readBlobBytes(table, sources, snapshot, dropRowId) {
36
52
  const rowIds = rowIdColumn === null
37
53
  ? []
38
54
  : Array.from(rowIdColumn.toArray());
55
+ const outputFields = new Map(table.schema.fields.map((field) => [field.name, field]));
39
56
  const replacements = new Map();
40
- for (const [output, source] of sources) {
41
- const index = table.schema.fields.findIndex((field) => field.name === output);
42
- if (index < 0) {
57
+ for (const { output, path, source } of sources) {
58
+ let field = outputFields.get(output);
59
+ for (const name of path) {
60
+ field =
61
+ field !== undefined && arrow_1.DataType.isStruct(field.type)
62
+ ? field.type.children.find((child) => child.name === name)
63
+ : undefined;
64
+ }
65
+ if (field === undefined) {
43
66
  continue;
44
67
  }
45
68
  const values = rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
46
- replacements.set(index, values.map((value) => value ?? null));
69
+ replacements.set(field, values.map((value) => value ?? null));
47
70
  }
48
71
  const result = withBinaryColumns(table, replacements);
49
72
  if (!dropRowId) {
@@ -53,27 +76,42 @@ async function readBlobBytes(table, sources, snapshot, dropRowId) {
53
76
  .map((field) => field.name)
54
77
  .filter((name) => name !== "_rowid"));
55
78
  }
56
- // A fresh field, not a clone: the descriptor field's metadata must not follow
57
- // the bytes.
58
79
  function withBinaryColumns(table, replacements) {
59
80
  if (replacements.size === 0) {
60
81
  return table;
61
82
  }
62
- const fields = table.schema.fields.map((field, index) => replacements.has(index)
63
- ? new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true)
64
- : field);
83
+ const replaceField = (field) => {
84
+ if (replacements.has(field)) {
85
+ // The descriptor's extension and encoding metadata must not follow bytes.
86
+ return new arrow_1.Field(field.name, new arrow_1.LargeBinary(), true);
87
+ }
88
+ if (arrow_1.DataType.isStruct(field.type)) {
89
+ const children = field.type.children.map(replaceField);
90
+ if (children.some((child, i) => child !== field.type.children[i])) {
91
+ return field.clone({ type: new arrow_1.Struct(children) });
92
+ }
93
+ }
94
+ return field;
95
+ };
96
+ const fields = table.schema.fields.map(replaceField);
65
97
  const schema = new arrow_1.Schema(fields, table.schema.metadata);
66
98
  let offset = 0;
67
99
  const batches = table.batches.map((batch) => {
68
- const children = batch.data.children.map((child, index) => {
69
- const values = replacements.get(index);
70
- if (values === undefined) {
71
- return child;
100
+ const replaceData = (data, field, updated) => {
101
+ if (field === updated) {
102
+ return data;
72
103
  }
73
- return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
74
- });
104
+ const values = replacements.get(field);
105
+ if (values !== undefined) {
106
+ return (0, arrow_1.vectorFromArray)(values.slice(offset, offset + batch.numRows), new arrow_1.LargeBinary()).data[0];
107
+ }
108
+ const children = data.children.map((child, index) => replaceData(child, field.type.children[index], updated.type.children[index]));
109
+ // Keep the struct's validity bitmap and offset, including null parents.
110
+ return data.clone(updated.type, undefined, undefined, undefined, undefined, children);
111
+ };
112
+ const children = batch.data.children.map((child, index) => replaceData(child, table.schema.fields[index], fields[index]));
75
113
  offset += batch.numRows;
76
- return new arrow_1.RecordBatch(schema, (0, arrow_1.makeData)({ type: new arrow_1.Struct(fields), length: batch.numRows, children }));
114
+ return new arrow_1.RecordBatch(schema, batch.data.clone(new arrow_1.Struct(fields), undefined, undefined, undefined, undefined, children));
77
115
  });
78
116
  return new arrow_1.Table(schema, batches);
79
117
  }
@@ -259,7 +297,7 @@ class QueryBase {
259
297
  const snapshot = await context.table.querySnapshot();
260
298
  const version = await snapshot.version();
261
299
  const sources = blobSources(await snapshot.blobColumns(), context.projection);
262
- if (sources.size === 0) {
300
+ if (sources.length === 0) {
263
301
  return this.collect(options, false);
264
302
  }
265
303
  const addRowId = !context.withRowId;
@@ -378,11 +416,16 @@ class StandardQueryBase extends QueryBase {
378
416
  /**
379
417
  * A filter statement to be applied to this query.
380
418
  *
381
- * The filter should be supplied as an SQL query string. For example:
419
+ * The filter may be an SQL query string or an expression built with `col`,
420
+ * which preserves case-sensitive column names. For example:
382
421
  * @example
383
422
  * x > 10
384
423
  * y > 0 AND y < 100
385
424
  * x > 5 OR y = 'test'
425
+ * @example
426
+ * ```ts
427
+ * table.query().where(col("firstName").eq("Alice"));
428
+ * ```
386
429
  *
387
430
  * Filtering performance can often be improved by creating a scalar index
388
431
  * on the filter column(s).
@@ -391,7 +434,14 @@ class StandardQueryBase extends QueryBase {
391
434
  * than replacing the previous filter.
392
435
  */
393
436
  where(predicate) {
394
- this.doCall((inner) => inner.onlyIf(predicate));
437
+ this.doCall((inner) => {
438
+ if (predicate instanceof expr_1.Expr) {
439
+ inner.onlyIf(predicate.toSql());
440
+ }
441
+ else {
442
+ inner.onlyIf(predicate);
443
+ }
444
+ });
395
445
  return this;
396
446
  }
397
447
  /**
@@ -469,26 +519,26 @@ class StandardQueryBase extends QueryBase {
469
519
  return this;
470
520
  }
471
521
  /**
472
- * Control MemWAL read routing for this query.
522
+ * Control RTE read routing for this query.
473
523
  *
474
- * By default (unset), when the table carries a MemWAL write spec (see
475
- * {@link Table#enableMemWal}), reads are routed through the LSM scanner so
524
+ * By default (unset), when the table carries an RTE write spec (see
525
+ * {@link Table#enableRTE}), reads are routed through the LSM scanner so
476
526
  * they also return data written via the `mergeInsert` LSM path that has not yet
477
527
  * been compacted into the base table (the active/frozen in-memory memtables and
478
528
  * the SSTables), deduplicated by primary key; a table without a spec
479
529
  * reads the base table.
480
530
  *
481
531
  * @param enable - `true` forces the LSM scanner and errors if the table has no
482
- * MemWAL write spec. `false` bypasses the MemWAL and reads the base table only,
532
+ * RTE write spec. `false` bypasses the RTE and reads the base table only,
483
533
  * even when a spec is present.
484
534
  *
485
535
  * Note: the LSM scanner does not support every query shape (e.g. reranking,
486
- * hybrid search, `orderBy`). On a MemWAL table those shapes error unless
487
- * `useMemWal(false)` is set, because a base-only read would silently exclude
488
- * un-compacted MemWAL data.
536
+ * hybrid search, `orderBy`). On an RTE table those shapes error unless
537
+ * `useRTE(false)` is set, because a base-only read would silently exclude
538
+ * un-compacted RTE data.
489
539
  */
490
- useMemWal(enable) {
491
- this.doCall((inner) => inner.useMemWal(enable));
540
+ useRTE(enable) {
541
+ this.doCall((inner) => inner.useRTE(enable));
492
542
  return this;
493
543
  }
494
544
  }
@@ -554,7 +604,7 @@ class VectorQuery extends StandardQueryBase {
554
604
  * probing may continue when too few candidates are found.
555
605
  * Any explicit effort, including 0.5, overrides nprobes in either call order.
556
606
  * Explicit minimum and maximum bounds still apply. This is not a recall
557
- * guarantee and does not change refinement or HNSW ef. The MemWAL LSM
607
+ * guarantee and does not change refinement or HNSW ef. The RTE LSM
558
608
  * scanner does not yet support this option.
559
609
  *
560
610
  * @example
@@ -816,16 +866,16 @@ class TakeQuery extends QueryBase {
816
866
  super(inner, blobContext);
817
867
  }
818
868
  /**
819
- * Control MemWAL read routing for this take query.
869
+ * Control RTE read routing for this take query.
820
870
  *
821
- * `false` bypasses the MemWAL and reads the base table only — the escape hatch,
871
+ * `false` bypasses the RTE and reads the base table only — the escape hatch,
822
872
  * since take-by-row-id/offset is not supported on the LSM scanner and, on a
823
- * MemWAL table, auto-routes to it and errors otherwise.
873
+ * RTE table, auto-routes to it and errors otherwise.
824
874
  *
825
875
  * @param enable - `false` reads the base table only.
826
876
  */
827
- useMemWal(enable) {
828
- this.doCall((inner) => inner.useMemWal(enable));
877
+ useRTE(enable) {
878
+ this.doCall((inner) => inner.useRTE(enable));
829
879
  return this;
830
880
  }
831
881
  }
package/dist/table.d.ts CHANGED
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
3
3
  import { IndexOptions } from "./indices";
4
4
  import { Job } from "./job";
5
5
  import { MergeInsertBuilder } from "./merge";
6
- import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, MemWalStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
6
+ import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
7
7
  import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
8
8
  import { IntoSql } from "./util";
9
9
  export { IndexConfig } from "./native";
10
- export { TableShardStats, SsTableStats, MemWalStats, MemtableStats, } from "./native";
10
+ export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
11
11
  /**
12
12
  * Progress snapshot for a write operation, delivered to the `progress`
13
13
  * callback passed to {@link Table.add}.
@@ -68,8 +68,8 @@ export interface AddDataOptions {
68
68
  /**
69
69
  * Whether blob URIs outside registered external bases may be written.
70
70
  *
71
- * Defaults to `false`. This option is supported only for local/native
72
- * tables; remote tables return an error when it is enabled. An enabled write
71
+ * Defaults to `false`. This option is supported only in LanceDB Embedded;
72
+ * LanceDB Distributed returns an error when it is enabled. An enabled write
73
73
  * stores the absolute URI reference without registering a base or copying
74
74
  * the external object into the database. The object must remain accessible
75
75
  * when the blob is read later.
@@ -139,8 +139,8 @@ export type TokenizeTableOptions = {
139
139
  column?: never;
140
140
  };
141
141
  /**
142
- * Specification selecting Lance's MemWAL LSM-style write path for
143
- * `mergeInsert`.
142
+ * Specification enabling the realtime engine (RTE), which routes
143
+ * `mergeInsert` through Lance's MemWAL LSM-style write path.
144
144
  *
145
145
  * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
146
146
  * `column` and `numBuckets` are required; for `"identity"`, `column` is
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
149
149
  * `column` value, or upserts of that key can land in different shards and a
150
150
  * stale version can win).
151
151
  */
152
- export interface MemWalWriteSpec {
152
+ export interface RTEWriteSpec {
153
153
  /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
154
154
  specType: "bucket" | "identity" | "unsharded";
155
155
  /** Bucket and identity variants: the sharding column. */
@@ -157,7 +157,7 @@ export interface MemWalWriteSpec {
157
157
  /** Bucket variant: the number of buckets, in `[1, 1024]`. */
158
158
  numBuckets?: number;
159
159
  /**
160
- * Indexes the MemWAL keeps up to date. Omit for every index the table has,
160
+ * Indexes the RTE keeps up to date. Omit for every index the table has,
161
161
  * including ones created later. Pass `[]` for none.
162
162
  */
163
163
  maintainedIndexes?: string[];
@@ -297,6 +297,11 @@ export declare abstract class Table {
297
297
  * }),
298
298
  * });
299
299
  * @example
300
+ * // Or size the partitions by row count instead of choosing how many.
301
+ * await table.createIndex("vector", {
302
+ * config: lancedb.Index.ivfPq({ targetPartitionSize: 4096 }),
303
+ * });
304
+ * @example
300
305
  * // Or create a Scalar index
301
306
  * await table.createIndex("my_float_col");
302
307
  */
@@ -344,7 +349,7 @@ export declare abstract class Table {
344
349
  * Large columns (e.g. high-dimensional vectors or binary data) may not be
345
350
  * practical to prewarm.
346
351
  *
347
- * This feature is currently only supported on remote tables.
352
+ * This feature is currently only supported in LanceDB Distributed.
348
353
  */
349
354
  abstract prewarmData(columns?: string[]): Promise<void>;
350
355
  /**
@@ -477,6 +482,10 @@ export declare abstract class Table {
477
482
  /**
478
483
  * Add new columns with defined values.
479
484
  *
485
+ * LanceDB Embedded computed-column declaration and refresh require a source
486
+ * build with the `experimental-functions` Cargo feature. Published packages
487
+ * disable it; LanceDB Distributed operations remain available.
488
+ *
480
489
  * The `{ computed }` form stores the expression rather than evaluating it
481
490
  * now: the column is committed with no values, and rows get them from
482
491
  * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
@@ -487,7 +496,7 @@ export declare abstract class Table {
487
496
  * declaration reads a column, that column cannot be renamed, retyped or
488
497
  * dropped.
489
498
  *
490
- * On LanceDB Cloud and Enterprise the expression is planned by the
499
+ * In LanceDB Distributed the expression is planned by the
491
500
  * server, and the refresh runs as a server job -- see
492
501
  * {@link Table#refreshColumnAsync}.
493
502
  * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
@@ -509,11 +518,12 @@ export declare abstract class Table {
509
518
  }): Promise<AddColumnsResult>;
510
519
  /**
511
520
  * Fill the rows of a computed column that hold no value yet.
521
+ * Local refresh requires a source build with `experimental-functions`.
512
522
  *
513
523
  * Rows appended since the last refresh are filled by the next one, and
514
524
  * rows whose inputs changed since they were computed are recomputed;
515
- * everything else is left as it is. Local tables only: a remote refresh
516
- * runs as a server job, through {@link Table#refreshColumnAsync}.
525
+ * everything else is left as it is. LanceDB Embedded only: in LanceDB
526
+ * Distributed a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
517
527
  * @param {string} column The name of the computed column to fill.
518
528
  * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
519
529
  * number of rows filled and the new version number of the table.
@@ -522,12 +532,15 @@ export declare abstract class Table {
522
532
  /**
523
533
  * Like {@link Table#refreshColumn}, but returns a handle to the refresh
524
534
  * job instead of blocking until it completes.
535
+ * LanceDB Embedded refresh requires a source build with
536
+ * `experimental-functions`. LanceDB Distributed jobs do not require this
537
+ * feature.
525
538
  *
526
539
  * The job may already be complete when returned; callers must not assume
527
540
  * the column is filled until {@link Job.wait} resolves. Invalid input --
528
541
  * an unknown column, or one that is not computed -- rejects here rather
529
- * than failing the job. On local tables the job runs in-process; on
530
- * LanceDB Cloud and Enterprise it is the server's backfill job.
542
+ * than failing the job. In LanceDB Embedded the job runs in-process; in
543
+ * LanceDB Distributed it is the server's backfill job.
531
544
  * @param {string} column The name of the computed column to fill.
532
545
  * @example
533
546
  * ```ts
@@ -543,7 +556,7 @@ export declare abstract class Table {
543
556
  * A refresh running under a skip policy records each row it skipped with
544
557
  * the input that failed and the error. This lists those records, newest
545
558
  * job first, plus a summary for any fragment whose per-row detail was
546
- * capped. LanceDB Cloud and Enterprise only; reading errors needs read
559
+ * capped. LanceDB Distributed only; reading errors needs read
547
560
  * access to the table, since a message carries the value that failed.
548
561
  * @param {FunctionErrorsOptions} options Optional filters: `jobId`,
549
562
  * `column`, and `limit` (server default 10000, cap 100000).
@@ -562,7 +575,7 @@ export declare abstract class Table {
562
575
  * it: rejects tables that carry no view definition.
563
576
  * @ignore
564
577
  */
565
- abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
578
+ abstract refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
566
579
  /** @ignore */
567
580
  abstract materializedViewDefinition(): Promise<string>;
568
581
  /**
@@ -582,8 +595,6 @@ export declare abstract class Table {
582
595
  * - `lancedb:description`: for a human-readable description of a field.
583
596
  * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
584
597
  * names the tag category; e.g. `lancedb:tag:model: "clip"`.
585
- * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
586
- * `feature_v2` might be in the same logical column.
587
598
  * - `lancedb:status`: for status options (`production`, `candidate`,
588
599
  * `deprecated`, `archived`) to designate the current life cycle state of
589
600
  * this column.
@@ -620,10 +631,10 @@ export declare abstract class Table {
620
631
  */
621
632
  abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
622
633
  /**
623
- * Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
634
+ * Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
624
635
  * LSM-style write path for future `mergeInsert` calls.
625
636
  *
626
- * `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
637
+ * `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
627
638
  *
628
639
  * - `"bucket"` — hash-bucket writes by the single-column unenforced primary
629
640
  * key (`column` and `numBuckets` required).
@@ -635,15 +646,15 @@ export declare abstract class Table {
635
646
  * requires it to be the single column being bucketed.
636
647
  *
637
648
  * Omitting `maintainedIndexes` maintains every index the table has,
638
- * including ones created later, and skips a kind the MemWAL cannot maintain.
649
+ * including ones created later, and skips a kind the RTE cannot maintain.
639
650
  * Naming them pins an exact set, and a still-building index is rejected
640
651
  * rather than quietly omitted.
641
- * @param {MemWalWriteSpec} spec The sharding spec to install.
652
+ * @param {RTEWriteSpec} spec The sharding spec to install.
642
653
  * @returns {Promise<void>}
643
654
  * @example
644
655
  * ```ts
645
656
  * await table.setUnenforcedPrimaryKey("id");
646
- * await table.enableMemWal({
657
+ * await table.enableRTE({
647
658
  * specType: "bucket",
648
659
  * column: "id",
649
660
  * numBuckets: 16,
@@ -651,36 +662,36 @@ export declare abstract class Table {
651
662
  * });
652
663
  * ```
653
664
  */
654
- abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
665
+ abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
655
666
  /**
656
- * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
667
+ * Remove the {@link RTEWriteSpec} from this table, reverting to the standard
657
668
  * `mergeInsert` write path.
658
669
  *
659
670
  * Errors if no spec is currently set.
660
671
  * @returns {Promise<void>}
661
672
  */
662
- abstract disableMemWal(): Promise<void>;
673
+ abstract disableRTE(): Promise<void>;
663
674
  /**
664
- * Read the {@link MemWalWriteSpec} currently installed on this table.
675
+ * Read the {@link RTEWriteSpec} currently installed on this table.
665
676
  *
666
- * Resolves to `undefined` when the MemWAL write path is not enabled (no
667
- * spec has been set, or it was removed with {@link Table#disableMemWal}).
677
+ * Resolves to `undefined` when the RTE write path is not enabled (no
678
+ * spec has been set, or it was removed with {@link Table#disableRTE}).
668
679
  * The spec is the one installed, including its maintained-index selection:
669
680
  * an absent `maintainedIndexes` for every index the table has, an empty
670
681
  * array for none.
671
- * @returns {Promise<MemWalWriteSpec | undefined>}
682
+ * @returns {Promise<RTEWriteSpec | undefined>}
672
683
  */
673
- abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
684
+ abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
674
685
  /**
675
- * Drain and close any cached MemWAL shard writers held for this table.
686
+ * Drain and close any cached RTE shard writers held for this table.
676
687
  *
677
- * When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
688
+ * When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
678
689
  * shard writers and caches them for reuse across calls. This closes them,
679
690
  * flushing pending data; writers reopen lazily on the next `mergeInsert`.
680
691
  * It is a no-op when no writers are cached.
681
692
  * @returns {Promise<void>}
682
693
  */
683
- abstract closeMemWalWriters(): Promise<void>;
694
+ abstract closeRTEWriters(): Promise<void>;
684
695
  /**
685
696
  * Freeze every table shard's active memtable into a new SSTable.
686
697
  *
@@ -688,18 +699,18 @@ export declare abstract class Table {
688
699
  * so this is safe to call repeatedly.
689
700
  * @returns {Promise<void>}
690
701
  */
691
- abstract flushMemWal(): Promise<void>;
702
+ abstract flushRTE(): Promise<void>;
692
703
  /**
693
704
  * Trigger a background SSTable compaction pass per table shard.
694
705
  *
695
706
  * Returns once the passes are *dispatched*, not once they finish — watch
696
- * {@link Table#getMemWalStats} for progress, or use
697
- * {@link Table#checkpointMemWal} to wait for convergence.
707
+ * {@link Table#getRTEStats} for progress, or use
708
+ * {@link Table#checkpointRTE} to wait for convergence.
698
709
  * @returns {Promise<void>}
699
710
  */
700
- abstract compactMemWal(): Promise<void>;
711
+ abstract compactRTE(): Promise<void>;
701
712
  /**
702
- * Converge this table's MemWAL write path into its base table.
713
+ * Converge this table's RTE write path into its base table.
703
714
  *
704
715
  * Freezes once, then triggers compaction and polls until the SSTables that existed
705
716
  * at the start is gone. The target set is fixed at the start, so
@@ -714,24 +725,24 @@ export declare abstract class Table {
714
725
  * @returns {Promise<void>}
715
726
  * @example
716
727
  * ```ts
717
- * const before = await table.getMemWalStats();
718
- * await table.checkpointMemWal();
719
- * const after = await table.getMemWalStats();
728
+ * const before = await table.getRTEStats();
729
+ * await table.checkpointRTE();
730
+ * const after = await table.getRTEStats();
720
731
  * ```
721
732
  */
722
- abstract checkpointMemWal(): Promise<void>;
733
+ abstract checkpointRTE(): Promise<void>;
723
734
  /**
724
- * Read live per-table-shard MemWAL state.
735
+ * Read live per-table-shard RTE state.
725
736
  *
726
737
  * Answers "how far behind is my fresh tier", "which table shard is hot", and
727
738
  * "why is my fresh-tier vector search brute-force". Mutates no table state.
728
739
  *
729
- * Resolves to `undefined` only when the MemWAL write path is not enabled.
740
+ * Resolves to `undefined` only when the RTE write path is not enabled.
730
741
  * @param {boolean} includeSstableRows Also count rows per SSTable.
731
742
  * Off by default because each count opens an uncached Lance dataset.
732
- * @returns {Promise<MemWalStats | undefined>}
743
+ * @returns {Promise<RTEStats | undefined>}
733
744
  */
734
- abstract getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
745
+ abstract getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
735
746
  /** Retrieve the version of the table */
736
747
  abstract version(): Promise<number>;
737
748
  /**
@@ -841,8 +852,12 @@ export declare abstract class Table {
841
852
  * Specify exactly one of `column` or `indexName`.
842
853
  *
843
854
  * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
844
- * the client process from index metadata. For remote tables, this means the
845
- * same tokenizer model files must also exist locally.
855
+ * the client process from index metadata. In LanceDB Distributed, this means the
856
+ * same tokenizer model files must also exist locally (under
857
+ * `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
858
+ * mismatch cannot be detected: the returned tokens may differ from what the
859
+ * server's index produces. If the model files are missing, an error naming
860
+ * the tokenizer is raised.
846
861
  */
847
862
  abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
848
863
  /** Return the table as an arrow table */
@@ -934,20 +949,20 @@ export declare class LocalTable extends Table {
934
949
  refreshColumn(column: string): Promise<RefreshColumnResult>;
935
950
  refreshColumnAsync(column: string): Promise<Job>;
936
951
  functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
937
- refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
952
+ refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
938
953
  materializedViewDefinition(): Promise<string>;
939
954
  alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
940
955
  updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
941
956
  dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
942
957
  setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
943
- enableMemWal(spec: MemWalWriteSpec): Promise<void>;
944
- disableMemWal(): Promise<void>;
945
- getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
946
- closeMemWalWriters(): Promise<void>;
947
- flushMemWal(): Promise<void>;
948
- compactMemWal(): Promise<void>;
949
- checkpointMemWal(): Promise<void>;
950
- getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
958
+ enableRTE(spec: RTEWriteSpec): Promise<void>;
959
+ disableRTE(): Promise<void>;
960
+ getRTESpec(): Promise<RTEWriteSpec | undefined>;
961
+ closeRTEWriters(): Promise<void>;
962
+ flushRTE(): Promise<void>;
963
+ compactRTE(): Promise<void>;
964
+ checkpointRTE(): Promise<void>;
965
+ getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
951
966
  version(): Promise<number>;
952
967
  checkout(version: number | string): Promise<void>;
953
968
  checkoutLatest(): Promise<void>;
package/dist/table.js CHANGED
@@ -286,8 +286,8 @@ class LocalTable extends Table {
286
286
  (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
287
287
  return await this.inner.functionErrors(options);
288
288
  }
289
- async refreshMaterializedView(sourceVersion) {
290
- return await this.inner.refreshMaterializedView(sourceVersion);
289
+ async refreshMaterializedView(sourceVersion, full) {
290
+ return await this.inner.refreshMaterializedView(sourceVersion, full);
291
291
  }
292
292
  async materializedViewDefinition() {
293
293
  return await this.inner.materializedViewDefinition();
@@ -326,32 +326,32 @@ class LocalTable extends Table {
326
326
  const cols = typeof columns === "string" ? [columns] : columns;
327
327
  return await this.inner.setUnenforcedPrimaryKey(cols);
328
328
  }
329
- async enableMemWal(spec) {
330
- return await this.inner.enableMemWal(spec);
329
+ async enableRTE(spec) {
330
+ return await this.inner.enableRTE(spec);
331
331
  }
332
- async disableMemWal() {
333
- return await this.inner.disableMemWal();
332
+ async disableRTE() {
333
+ return await this.inner.disableRTE();
334
334
  }
335
- async getMemWalSpec() {
335
+ async getRTESpec() {
336
336
  // The native binding types `specType` as a plain `string`; narrow it back
337
337
  // to the public union. The Rust `From` impl only ever emits one of the
338
338
  // three valid values, so the cast is safe.
339
- return ((await this.inner.getMemWalSpec()) ?? undefined);
339
+ return ((await this.inner.getRTESpec()) ?? undefined);
340
340
  }
341
- async closeMemWalWriters() {
342
- return await this.inner.closeMemWalWriters();
341
+ async closeRTEWriters() {
342
+ return await this.inner.closeRTEWriters();
343
343
  }
344
- async flushMemWal() {
345
- return await this.inner.flushMemWal();
344
+ async flushRTE() {
345
+ return await this.inner.flushRTE();
346
346
  }
347
- async compactMemWal() {
348
- return await this.inner.compactMemWal();
347
+ async compactRTE() {
348
+ return await this.inner.compactRTE();
349
349
  }
350
- async checkpointMemWal() {
351
- return await this.inner.checkpointMemWal();
350
+ async checkpointRTE() {
351
+ return await this.inner.checkpointRTE();
352
352
  }
353
- async getMemWalStats(includeSstableRows = false) {
354
- return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined;
353
+ async getRTEStats(includeSstableRows = false) {
354
+ return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined;
355
355
  }
356
356
  async version() {
357
357
  return await this.inner.version();
package/package.json CHANGED
@@ -11,7 +11,7 @@
11
11
  "ann"
12
12
  ],
13
13
  "private": false,
14
- "version": "0.41.0-beta.8",
14
+ "version": "0.42.0-beta.0",
15
15
  "main": "dist/index.js",
16
16
  "exports": {
17
17
  ".": "./dist/index.js",
@@ -97,7 +97,8 @@
97
97
  "postdocs": "node typedoc_post_process.js",
98
98
  "lint": "biome check . && biome format .",
99
99
  "lint-fix": "biome check --write . && biome format --write .",
100
- "prepublishOnly": "napi prepublish -t npm",
100
+ "check:native-symbols": "node scripts/check-native-symbols.js",
101
+ "prepublishOnly": "pnpm check:native-symbols && napi prepublish -t npm",
101
102
  "test": "jest --verbose",
102
103
  "integration": "S3_TEST=1 pnpm test",
103
104
  "universal": "napi universalize",
@@ -110,13 +111,13 @@
110
111
  "optionalDependencies": {
111
112
  "@huggingface/transformers": "3.0.2",
112
113
  "openai": "4.29.2",
113
- "@lancedb/lancedb-darwin-arm64": "0.41.0-beta.8",
114
- "@lancedb/lancedb-linux-x64-gnu": "0.41.0-beta.8",
115
- "@lancedb/lancedb-linux-arm64-gnu": "0.41.0-beta.8",
116
- "@lancedb/lancedb-linux-x64-musl": "0.41.0-beta.8",
117
- "@lancedb/lancedb-linux-arm64-musl": "0.41.0-beta.8",
118
- "@lancedb/lancedb-win32-x64-msvc": "0.41.0-beta.8",
119
- "@lancedb/lancedb-win32-arm64-msvc": "0.41.0-beta.8"
114
+ "@lancedb/lancedb-darwin-arm64": "0.42.0-beta.0",
115
+ "@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.0",
116
+ "@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.0",
117
+ "@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.0",
118
+ "@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.0",
119
+ "@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.0",
120
+ "@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.0"
120
121
  },
121
122
  "peerDependencies": {
122
123
  "@types/node": ">=22",