@lancedb/lancedb 0.41.0-beta.8 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/table.d.ts CHANGED
@@ -3,11 +3,11 @@ import { BlobFile, BlobReadOptions } from "./blob";
3
3
  import { IndexOptions } from "./indices";
4
4
  import { Job } from "./job";
5
5
  import { MergeInsertBuilder } from "./merge";
6
- import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, MemWalStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
6
+ import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
7
7
  import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
8
8
  import { IntoSql } from "./util";
9
9
  export { IndexConfig } from "./native";
10
- export { TableShardStats, SsTableStats, MemWalStats, MemtableStats, } from "./native";
10
+ export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
11
11
  /**
12
12
  * Progress snapshot for a write operation, delivered to the `progress`
13
13
  * callback passed to {@link Table.add}.
@@ -139,8 +139,8 @@ export type TokenizeTableOptions = {
139
139
  column?: never;
140
140
  };
141
141
  /**
142
- * Specification selecting Lance's MemWAL LSM-style write path for
143
- * `mergeInsert`.
142
+ * Specification enabling the realtime engine (RTE), which routes
143
+ * `mergeInsert` through Lance's MemWAL LSM-style write path.
144
144
  *
145
145
  * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
146
146
  * `column` and `numBuckets` are required; for `"identity"`, `column` is
@@ -149,7 +149,7 @@ export type TokenizeTableOptions = {
149
149
  * `column` value, or upserts of that key can land in different shards and a
150
150
  * stale version can win).
151
151
  */
152
- export interface MemWalWriteSpec {
152
+ export interface RTEWriteSpec {
153
153
  /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
154
154
  specType: "bucket" | "identity" | "unsharded";
155
155
  /** Bucket and identity variants: the sharding column. */
@@ -157,7 +157,7 @@ export interface MemWalWriteSpec {
157
157
  /** Bucket variant: the number of buckets, in `[1, 1024]`. */
158
158
  numBuckets?: number;
159
159
  /**
160
- * Indexes the MemWAL keeps up to date. Omit for every index the table has,
160
+ * Indexes the RTE keeps up to date. Omit for every index the table has,
161
161
  * including ones created later. Pass `[]` for none.
162
162
  */
163
163
  maintainedIndexes?: string[];
@@ -477,6 +477,10 @@ export declare abstract class Table {
477
477
  /**
478
478
  * Add new columns with defined values.
479
479
  *
480
+ * Local computed-column declaration and refresh require a source build with
481
+ * the `experimental-functions` Cargo feature. Published packages disable it;
482
+ * remote Cloud/Enterprise operations remain available.
483
+ *
480
484
  * The `{ computed }` form stores the expression rather than evaluating it
481
485
  * now: the column is committed with no values, and rows get them from
482
486
  * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
@@ -509,6 +513,7 @@ export declare abstract class Table {
509
513
  }): Promise<AddColumnsResult>;
510
514
  /**
511
515
  * Fill the rows of a computed column that hold no value yet.
516
+ * Local refresh requires a source build with `experimental-functions`.
512
517
  *
513
518
  * Rows appended since the last refresh are filled by the next one, and
514
519
  * rows whose inputs changed since they were computed are recomputed;
@@ -522,6 +527,8 @@ export declare abstract class Table {
522
527
  /**
523
528
  * Like {@link Table#refreshColumn}, but returns a handle to the refresh
524
529
  * job instead of blocking until it completes.
530
+ * Local refresh requires a source build with `experimental-functions`.
531
+ * Remote Cloud/Enterprise jobs do not require this feature.
525
532
  *
526
533
  * The job may already be complete when returned; callers must not assume
527
534
  * the column is filled until {@link Job.wait} resolves. Invalid input --
@@ -562,7 +569,7 @@ export declare abstract class Table {
562
569
  * it: rejects tables that carry no view definition.
563
570
  * @ignore
564
571
  */
565
- abstract refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
572
+ abstract refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
566
573
  /** @ignore */
567
574
  abstract materializedViewDefinition(): Promise<string>;
568
575
  /**
@@ -582,8 +589,6 @@ export declare abstract class Table {
582
589
  * - `lancedb:description`: for a human-readable description of a field.
583
590
  * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
584
591
  * names the tag category; e.g. `lancedb:tag:model: "clip"`.
585
- * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
586
- * `feature_v2` might be in the same logical column.
587
592
  * - `lancedb:status`: for status options (`production`, `candidate`,
588
593
  * `deprecated`, `archived`) to designate the current life cycle state of
589
594
  * this column.
@@ -620,10 +625,10 @@ export declare abstract class Table {
620
625
  */
621
626
  abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
622
627
  /**
623
- * Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
628
+ * Install an {@link RTEWriteSpec} on this table, selecting Lance's MemWAL
624
629
  * LSM-style write path for future `mergeInsert` calls.
625
630
  *
626
- * `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
631
+ * `RTEWriteSpec` chooses one of three sharding strategies via `specType`:
627
632
  *
628
633
  * - `"bucket"` — hash-bucket writes by the single-column unenforced primary
629
634
  * key (`column` and `numBuckets` required).
@@ -635,15 +640,15 @@ export declare abstract class Table {
635
640
  * requires it to be the single column being bucketed.
636
641
  *
637
642
  * Omitting `maintainedIndexes` maintains every index the table has,
638
- * including ones created later, and skips a kind the MemWAL cannot maintain.
643
+ * including ones created later, and skips a kind the RTE cannot maintain.
639
644
  * Naming them pins an exact set, and a still-building index is rejected
640
645
  * rather than quietly omitted.
641
- * @param {MemWalWriteSpec} spec The sharding spec to install.
646
+ * @param {RTEWriteSpec} spec The sharding spec to install.
642
647
  * @returns {Promise<void>}
643
648
  * @example
644
649
  * ```ts
645
650
  * await table.setUnenforcedPrimaryKey("id");
646
- * await table.enableMemWal({
651
+ * await table.enableRTE({
647
652
  * specType: "bucket",
648
653
  * column: "id",
649
654
  * numBuckets: 16,
@@ -651,36 +656,36 @@ export declare abstract class Table {
651
656
  * });
652
657
  * ```
653
658
  */
654
- abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
659
+ abstract enableRTE(spec: RTEWriteSpec): Promise<void>;
655
660
  /**
656
- * Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
661
+ * Remove the {@link RTEWriteSpec} from this table, reverting to the standard
657
662
  * `mergeInsert` write path.
658
663
  *
659
664
  * Errors if no spec is currently set.
660
665
  * @returns {Promise<void>}
661
666
  */
662
- abstract disableMemWal(): Promise<void>;
667
+ abstract disableRTE(): Promise<void>;
663
668
  /**
664
- * Read the {@link MemWalWriteSpec} currently installed on this table.
669
+ * Read the {@link RTEWriteSpec} currently installed on this table.
665
670
  *
666
- * Resolves to `undefined` when the MemWAL write path is not enabled (no
667
- * spec has been set, or it was removed with {@link Table#disableMemWal}).
671
+ * Resolves to `undefined` when the RTE write path is not enabled (no
672
+ * spec has been set, or it was removed with {@link Table#disableRTE}).
668
673
  * The spec is the one installed, including its maintained-index selection:
669
674
  * an absent `maintainedIndexes` for every index the table has, an empty
670
675
  * array for none.
671
- * @returns {Promise<MemWalWriteSpec | undefined>}
676
+ * @returns {Promise<RTEWriteSpec | undefined>}
672
677
  */
673
- abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
678
+ abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
674
679
  /**
675
- * Drain and close any cached MemWAL shard writers held for this table.
680
+ * Drain and close any cached RTE shard writers held for this table.
676
681
  *
677
- * When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
682
+ * When an {@link RTEWriteSpec} is installed, `mergeInsert` opens RTE
678
683
  * shard writers and caches them for reuse across calls. This closes them,
679
684
  * flushing pending data; writers reopen lazily on the next `mergeInsert`.
680
685
  * It is a no-op when no writers are cached.
681
686
  * @returns {Promise<void>}
682
687
  */
683
- abstract closeMemWalWriters(): Promise<void>;
688
+ abstract closeRTEWriters(): Promise<void>;
684
689
  /**
685
690
  * Freeze every table shard's active memtable into a new SSTable.
686
691
  *
@@ -688,18 +693,18 @@ export declare abstract class Table {
688
693
  * so this is safe to call repeatedly.
689
694
  * @returns {Promise<void>}
690
695
  */
691
- abstract flushMemWal(): Promise<void>;
696
+ abstract flushRTE(): Promise<void>;
692
697
  /**
693
698
  * Trigger a background SSTable compaction pass per table shard.
694
699
  *
695
700
  * Returns once the passes are *dispatched*, not once they finish — watch
696
- * {@link Table#getMemWalStats} for progress, or use
697
- * {@link Table#checkpointMemWal} to wait for convergence.
701
+ * {@link Table#getRTEStats} for progress, or use
702
+ * {@link Table#checkpointRTE} to wait for convergence.
698
703
  * @returns {Promise<void>}
699
704
  */
700
- abstract compactMemWal(): Promise<void>;
705
+ abstract compactRTE(): Promise<void>;
701
706
  /**
702
- * Converge this table's MemWAL write path into its base table.
707
+ * Converge this table's RTE write path into its base table.
703
708
  *
704
709
  * Freezes once, then triggers compaction and polls until the SSTables that existed
705
710
  * at the start is gone. The target set is fixed at the start, so
@@ -714,24 +719,24 @@ export declare abstract class Table {
714
719
  * @returns {Promise<void>}
715
720
  * @example
716
721
  * ```ts
717
- * const before = await table.getMemWalStats();
718
- * await table.checkpointMemWal();
719
- * const after = await table.getMemWalStats();
722
+ * const before = await table.getRTEStats();
723
+ * await table.checkpointRTE();
724
+ * const after = await table.getRTEStats();
720
725
  * ```
721
726
  */
722
- abstract checkpointMemWal(): Promise<void>;
727
+ abstract checkpointRTE(): Promise<void>;
723
728
  /**
724
- * Read live per-table-shard MemWAL state.
729
+ * Read live per-table-shard RTE state.
725
730
  *
726
731
  * Answers "how far behind is my fresh tier", "which table shard is hot", and
727
732
  * "why is my fresh-tier vector search brute-force". Mutates no table state.
728
733
  *
729
- * Resolves to `undefined` only when the MemWAL write path is not enabled.
734
+ * Resolves to `undefined` only when the RTE write path is not enabled.
730
735
  * @param {boolean} includeSstableRows Also count rows per SSTable.
731
736
  * Off by default because each count opens an uncached Lance dataset.
732
- * @returns {Promise<MemWalStats | undefined>}
737
+ * @returns {Promise<RTEStats | undefined>}
733
738
  */
734
- abstract getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
739
+ abstract getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
735
740
  /** Retrieve the version of the table */
736
741
  abstract version(): Promise<number>;
737
742
  /**
@@ -842,7 +847,11 @@ export declare abstract class Table {
842
847
  *
843
848
  * Model-backed tokenizers such as `jieba/*` and `lindera/*` are rebuilt in
844
849
  * the client process from index metadata. For remote tables, this means the
845
- * same tokenizer model files must also exist locally.
850
+ * same tokenizer model files must also exist locally (under
851
+ * `LANCE_LANGUAGE_MODEL_HOME`) and must match the files the server uses. A
852
+ * mismatch cannot be detected: the returned tokens may differ from what the
853
+ * server's index produces. If the model files are missing, an error naming
854
+ * the tokenizer is raised.
846
855
  */
847
856
  abstract tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
848
857
  /** Return the table as an arrow table */
@@ -934,20 +943,20 @@ export declare class LocalTable extends Table {
934
943
  refreshColumn(column: string): Promise<RefreshColumnResult>;
935
944
  refreshColumnAsync(column: string): Promise<Job>;
936
945
  functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
937
- refreshMaterializedView(sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
946
+ refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
938
947
  materializedViewDefinition(): Promise<string>;
939
948
  alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
940
949
  updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
941
950
  dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
942
951
  setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
943
- enableMemWal(spec: MemWalWriteSpec): Promise<void>;
944
- disableMemWal(): Promise<void>;
945
- getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
946
- closeMemWalWriters(): Promise<void>;
947
- flushMemWal(): Promise<void>;
948
- compactMemWal(): Promise<void>;
949
- checkpointMemWal(): Promise<void>;
950
- getMemWalStats(includeSstableRows?: boolean): Promise<MemWalStats | undefined>;
952
+ enableRTE(spec: RTEWriteSpec): Promise<void>;
953
+ disableRTE(): Promise<void>;
954
+ getRTESpec(): Promise<RTEWriteSpec | undefined>;
955
+ closeRTEWriters(): Promise<void>;
956
+ flushRTE(): Promise<void>;
957
+ compactRTE(): Promise<void>;
958
+ checkpointRTE(): Promise<void>;
959
+ getRTEStats(includeSstableRows?: boolean): Promise<RTEStats | undefined>;
951
960
  version(): Promise<number>;
952
961
  checkout(version: number | string): Promise<void>;
953
962
  checkoutLatest(): Promise<void>;
package/dist/table.js CHANGED
@@ -286,8 +286,8 @@ class LocalTable extends Table {
286
286
  (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
287
287
  return await this.inner.functionErrors(options);
288
288
  }
289
- async refreshMaterializedView(sourceVersion) {
290
- return await this.inner.refreshMaterializedView(sourceVersion);
289
+ async refreshMaterializedView(sourceVersion, full) {
290
+ return await this.inner.refreshMaterializedView(sourceVersion, full);
291
291
  }
292
292
  async materializedViewDefinition() {
293
293
  return await this.inner.materializedViewDefinition();
@@ -326,32 +326,32 @@ class LocalTable extends Table {
326
326
  const cols = typeof columns === "string" ? [columns] : columns;
327
327
  return await this.inner.setUnenforcedPrimaryKey(cols);
328
328
  }
329
- async enableMemWal(spec) {
330
- return await this.inner.enableMemWal(spec);
329
+ async enableRTE(spec) {
330
+ return await this.inner.enableRTE(spec);
331
331
  }
332
- async disableMemWal() {
333
- return await this.inner.disableMemWal();
332
+ async disableRTE() {
333
+ return await this.inner.disableRTE();
334
334
  }
335
- async getMemWalSpec() {
335
+ async getRTESpec() {
336
336
  // The native binding types `specType` as a plain `string`; narrow it back
337
337
  // to the public union. The Rust `From` impl only ever emits one of the
338
338
  // three valid values, so the cast is safe.
339
- return ((await this.inner.getMemWalSpec()) ?? undefined);
339
+ return ((await this.inner.getRTESpec()) ?? undefined);
340
340
  }
341
- async closeMemWalWriters() {
342
- return await this.inner.closeMemWalWriters();
341
+ async closeRTEWriters() {
342
+ return await this.inner.closeRTEWriters();
343
343
  }
344
- async flushMemWal() {
345
- return await this.inner.flushMemWal();
344
+ async flushRTE() {
345
+ return await this.inner.flushRTE();
346
346
  }
347
- async compactMemWal() {
348
- return await this.inner.compactMemWal();
347
+ async compactRTE() {
348
+ return await this.inner.compactRTE();
349
349
  }
350
- async checkpointMemWal() {
351
- return await this.inner.checkpointMemWal();
350
+ async checkpointRTE() {
351
+ return await this.inner.checkpointRTE();
352
352
  }
353
- async getMemWalStats(includeSstableRows = false) {
354
- return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined;
353
+ async getRTEStats(includeSstableRows = false) {
354
+ return (await this.inner.getRTEStats(includeSstableRows)) ?? undefined;
355
355
  }
356
356
  async version() {
357
357
  return await this.inner.version();
package/package.json CHANGED
@@ -11,7 +11,7 @@
11
11
  "ann"
12
12
  ],
13
13
  "private": false,
14
- "version": "0.41.0-beta.8",
14
+ "version": "0.41.0",
15
15
  "main": "dist/index.js",
16
16
  "exports": {
17
17
  ".": "./dist/index.js",
@@ -97,7 +97,8 @@
97
97
  "postdocs": "node typedoc_post_process.js",
98
98
  "lint": "biome check . && biome format .",
99
99
  "lint-fix": "biome check --write . && biome format --write .",
100
- "prepublishOnly": "napi prepublish -t npm",
100
+ "check:native-symbols": "node scripts/check-native-symbols.js",
101
+ "prepublishOnly": "pnpm check:native-symbols && napi prepublish -t npm",
101
102
  "test": "jest --verbose",
102
103
  "integration": "S3_TEST=1 pnpm test",
103
104
  "universal": "napi universalize",
@@ -110,13 +111,13 @@
110
111
  "optionalDependencies": {
111
112
  "@huggingface/transformers": "3.0.2",
112
113
  "openai": "4.29.2",
113
- "@lancedb/lancedb-darwin-arm64": "0.41.0-beta.8",
114
- "@lancedb/lancedb-linux-x64-gnu": "0.41.0-beta.8",
115
- "@lancedb/lancedb-linux-arm64-gnu": "0.41.0-beta.8",
116
- "@lancedb/lancedb-linux-x64-musl": "0.41.0-beta.8",
117
- "@lancedb/lancedb-linux-arm64-musl": "0.41.0-beta.8",
118
- "@lancedb/lancedb-win32-x64-msvc": "0.41.0-beta.8",
119
- "@lancedb/lancedb-win32-arm64-msvc": "0.41.0-beta.8"
114
+ "@lancedb/lancedb-darwin-arm64": "0.41.0",
115
+ "@lancedb/lancedb-linux-x64-gnu": "0.41.0",
116
+ "@lancedb/lancedb-linux-arm64-gnu": "0.41.0",
117
+ "@lancedb/lancedb-linux-x64-musl": "0.41.0",
118
+ "@lancedb/lancedb-linux-arm64-musl": "0.41.0",
119
+ "@lancedb/lancedb-win32-x64-msvc": "0.41.0",
120
+ "@lancedb/lancedb-win32-arm64-msvc": "0.41.0"
120
121
  },
121
122
  "peerDependencies": {
122
123
  "@types/node": ">=22",