@lancedb/lancedb 0.42.0-beta.20 → 0.42.0-beta.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/table.js CHANGED
@@ -2,7 +2,8 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Branches = exports.LocalTable = exports.Table = void 0;
5
+ exports.Branches = exports.TableDelta = exports.LocalTable = exports.Table = void 0;
6
+ const cleanup_1 = require("./cleanup");
6
7
  const compaction_1 = require("./compaction");
7
8
  const prewarm_1 = require("./prewarm");
8
9
  const arrow_1 = require("./arrow");
@@ -13,6 +14,7 @@ const materialized_view_1 = require("./materialized_view");
13
14
  const merge_1 = require("./merge");
14
15
  const query_1 = require("./query");
15
16
  const sanitize_1 = require("./sanitize");
17
+ const sql_1 = require("./sql");
16
18
  const util_1 = require("./util");
17
19
  /**
18
20
  * A Table is a collection of Records in a LanceDB Database.
@@ -36,6 +38,126 @@ class Table {
36
38
  [Symbol.for("nodejs.util.inspect.custom")]() {
37
39
  return this.display();
38
40
  }
41
+ /**
42
+ * Preview prewarm with the same options as {@link prewarmDataAsync}, without
43
+ * submitting a warming job. Requires LanceDB Enterprise with structured SQL
44
+ * previews, table Select and database/namespace Usage. Reads metadata and
45
+ * returns the source version, readable plan, and conservative pruning summary.
46
+ * A later submission plans its own snapshot. Historical and Embedded handles
47
+ * are unsupported. Unknown measurements remain null.
48
+ * Custom table implementations can override this method; its default rejects
49
+ * with an unsupported-operation error.
50
+ *
51
+ * @example
52
+ * const plan = await table.explainPrewarmData({
53
+ * columns: ["video"], filter: "quality > $minimum", parameters: {minimum: 0.7},
54
+ * });
55
+ * console.log(plan.pruning.selectedFragments);
56
+ */
57
+ async explainPrewarmData(_options) {
58
+ throw new Error("data prewarm previews are not supported by this table implementation");
59
+ }
60
+ /**
61
+ * Read the stored overrides for a remote Enterprise table.
62
+ *
63
+ * Overrides configure background behavior such as automatic compaction,
64
+ * cleanup, and reindex execution. Keys use the server's snake_case names.
65
+ * Returns an empty object when no overrides are set. Local tables are unsupported.
66
+ * Custom table implementations inherit a default rejection unless they override
67
+ * this method.
68
+ */
69
+ async getTableOverrides() {
70
+ throw new Error("getTableOverrides is not supported on this table type");
71
+ }
72
+ /**
73
+ * Recursively merge overrides into a remote Enterprise table's stored overrides.
74
+ *
75
+ * Fetches the current overrides, merges nested objects locally, then sends the
76
+ * complete replacement using REST PUT. Other values replace existing values;
77
+ * unrelated fields from the read snapshot are preserved. This read-modify-write
78
+ * is not atomic: concurrent edits can be overwritten, even to unrelated fields.
79
+ * Coordinate writers when concurrent edits must be preserved. An empty update
80
+ * only reads and returns the current overrides.
81
+ *
82
+ * Unlike this SDK method, CLI `set` and REST PUT replace the overrides object
83
+ * without merging. Use {@link Table.resetTableOverrides} to clear all overrides.
84
+ * Local tables and custom table implementations without override support reject
85
+ * this operation.
86
+ *
87
+ * @example
88
+ * ```ts
89
+ * await table.updateTableOverrides({
90
+ * job_types: { compaction: { enabled: false } },
91
+ * });
92
+ * ```
93
+ */
94
+ async updateTableOverrides(
95
+ // biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
96
+ overrides) {
97
+ throw new Error("updateTableOverrides is not supported on this table type");
98
+ }
99
+ /**
100
+ * Clear a remote Enterprise table's overrides and return an empty object.
101
+ * Local tables and custom table implementations without override support reject
102
+ * this operation.
103
+ */
104
+ async resetTableOverrides() {
105
+ throw new Error("resetTableOverrides is not supported on this table type");
106
+ }
107
+ /**
108
+ * Compare two persisted versions or same-branch tags on an Embedded table.
109
+ *
110
+ * `fromVersion` is exclusive and `toVersion` is inclusive. Omitting
111
+ * `toVersion` uses the version currently visible through this handle,
112
+ * including a checked-out version. The endpoints are fixed when the delta is
113
+ * created.
114
+ *
115
+ * Row readers require stable row IDs at both endpoints. Create the table with
116
+ * `storageOptions: { new_table_enable_stable_row_ids: "true" }` to enable
117
+ * them. `listTransactions()` works without stable row IDs.
118
+ *
119
+ * Row readers reject ranges containing a Restore, or ranges whose transaction
120
+ * history cannot prove they are restore-free. Recompute the checkpoint with a
121
+ * full rescan and reinitialize it at the range end version.
122
+ *
123
+ * LanceDB Enterprise tables reject this operation.
124
+ *
125
+ * @example
126
+ * const delta = await table.delta("baseline");
127
+ * const reader = delta.insertedRows();
128
+ * console.log((await reader.schema()).fields.map((field) => field.name));
129
+ * for await (const batch of reader) {
130
+ * console.log(batch.getChild("id")?.toJSON());
131
+ * }
132
+ * console.log(await delta.listTransactions());
133
+ */
134
+ async delta(
135
+ // biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
136
+ fromVersion,
137
+ // biome-ignore lint/correctness/noUnusedVariables: retain the public parameter name in generated docs
138
+ toVersion) {
139
+ throw new Error("table.delta is only supported on LanceDB Embedded tables");
140
+ }
141
+ /**
142
+ * Submit durable remote cleanup through SQL VACUUM and return after acceptance.
143
+ * Requires table Ownership and parent Usage. Retention inherits from the table
144
+ * then deployment when omitted. Local/historical handles are unsupported.
145
+ * Before submission, disable/drain automatic cleanup and prevent other cleanup,
146
+ * tag/branch publication and restore until completion, including direct clients.
147
+ * Coordination is not enforced by the server. Cancellation cannot undo deletion
148
+ * or prove storage I/O stopped. Preview is available without deletion.
149
+ * Wait timeout leaves work running; save the ID and reopen with db.openJob().
150
+ * @example
151
+ * const job = await table.cleanupAsync({ minAgeSeconds: 604800 });
152
+ * const result = await job.wait({ timeoutMs: 600000 });
153
+ */
154
+ async cleanupAsync(_options) {
155
+ throw new Error("durable cleanup jobs are only supported on remote tables");
156
+ }
157
+ /** Preview version deletion without creating a job. Execution re-evaluates current state. */
158
+ async explainCleanup(_options) {
159
+ throw new Error("cleanup previews require LanceDB Enterprise");
160
+ }
39
161
  /**
40
162
  * Submit remote compaction and return its durable job after acceptance.
41
163
  *
@@ -187,12 +309,10 @@ class LocalTable extends Table {
187
309
  : columnsOrOptions);
188
310
  }
189
311
  async prewarmDataAsync(options) {
190
- for (const key of Object.keys(options ?? {})) {
191
- if (!["columns", "filter", "concurrency"].includes(key)) {
192
- throw new Error(`Unsupported data prewarm option: ${key}`);
193
- }
194
- }
195
- return new job_1.Job(await this.inner.prewarmDataAsync(options), prewarm_1.decodePrewarmDataResult);
312
+ return new job_1.Job(await this.inner.prewarmDataAsync(await toNativePrewarmDataOptions(options)), prewarm_1.decodePrewarmDataResult);
313
+ }
314
+ async explainPrewarmData(options) {
315
+ return (0, prewarm_1.decodePrewarmDataPlan)(await this.inner.explainPrewarmData(await toNativePrewarmDataOptions(options)));
196
316
  }
197
317
  async waitForIndex(indexNames, timeoutSeconds) {
198
318
  await this.inner.waitForIndex(indexNames, timeoutSeconds);
@@ -395,6 +515,15 @@ class LocalTable extends Table {
395
515
  // three valid values, so the cast is safe.
396
516
  return ((await this.inner.getRTESpec()) ?? undefined);
397
517
  }
518
+ async getTableOverrides() {
519
+ return await this.inner.getTableOverrides();
520
+ }
521
+ async updateTableOverrides(overrides) {
522
+ return await this.inner.updateTableOverrides(overrides);
523
+ }
524
+ async resetTableOverrides() {
525
+ return await this.inner.resetTableOverrides();
526
+ }
398
527
  async closeRTEWriters() {
399
528
  return await this.inner.closeRTEWriters();
400
529
  }
@@ -422,6 +551,9 @@ class LocalTable extends Table {
422
551
  async checkoutLatest() {
423
552
  await this.inner.checkoutLatest();
424
553
  }
554
+ async delta(fromVersion, toVersion) {
555
+ return new TableDelta(await this.inner.delta(toNativeDeltaRef(fromVersion), toVersion === undefined ? undefined : toNativeDeltaRef(toVersion)));
556
+ }
425
557
  async listVersions() {
426
558
  return (await this.inner.listVersions()).map((version) => ({
427
559
  version: version.version,
@@ -445,6 +577,14 @@ class LocalTable extends Table {
445
577
  validateCompactionOptions(options);
446
578
  return new job_1.Job(await this.inner.compactAsync(options), compaction_1.decodeCompactionResult);
447
579
  }
580
+ async cleanupAsync(options) {
581
+ (0, cleanup_1.validateCleanupOptions)(options);
582
+ return new job_1.Job(await this.inner.cleanupAsync(options?.minAgeSeconds), cleanup_1.decodeCleanupResult);
583
+ }
584
+ async explainCleanup(options) {
585
+ (0, cleanup_1.validateCleanupOptions)(options);
586
+ return (0, cleanup_1.decodeCleanupPlan)(await this.inner.explainCleanup(options?.minAgeSeconds));
587
+ }
448
588
  async optimize(options) {
449
589
  validateCompactionOptions(options?.compactionOptions);
450
590
  return await this.inner.optimize(options?.cleanupOlderThan?.getTime(), options?.deleteUnverified, options?.compactionOptions);
@@ -505,6 +645,211 @@ class LocalTable extends Table {
505
645
  }
506
646
  }
507
647
  exports.LocalTable = LocalTable;
648
+ /**
649
+ * A handle for reading changes between two pinned table versions.
650
+ *
651
+ * Created by {@link Table.delta}. Endpoints are fixed at creation; later writes
652
+ * or tag moves do not change this delta.
653
+ */
654
+ class TableDelta {
655
+ #inner;
656
+ /**
657
+ * Construct a table-delta handle. Internal use only.
658
+ * @hidden
659
+ */
660
+ constructor(inner) {
661
+ this.#inner = inner;
662
+ }
663
+ /** The exclusive start version, as an exact u64 bigint. */
664
+ get beginVersion() {
665
+ return this.#inner.beginVersion();
666
+ }
667
+ /** The inclusive end version, as an exact u64 bigint. */
668
+ get endVersion() {
669
+ return this.#inner.endVersion();
670
+ }
671
+ /** Stream rows inserted in `(beginVersion, endVersion]`. */
672
+ insertedRows() {
673
+ return tableDeltaReader(() => this.#inner.insertedRows());
674
+ }
675
+ /** Stream rows updated in `(beginVersion, endVersion]`. */
676
+ updatedRows() {
677
+ return tableDeltaReader(() => this.#inner.updatedRows());
678
+ }
679
+ /** Stream inserted and updated rows in `(beginVersion, endVersion]`. */
680
+ upsertedRows() {
681
+ return tableDeltaReader(() => this.#inner.upsertedRows());
682
+ }
683
+ /** Stream deleted row IDs. Each batch has one `_rowid: uint64` column. */
684
+ deletedRowIds() {
685
+ return tableDeltaReader(() => this.#inner.deletedRowIds());
686
+ }
687
+ /** List readable transactions committed in `(beginVersion, endVersion]`. */
688
+ async listTransactions() {
689
+ const transactions = await this.#inner.listTransactions();
690
+ return transactions.map((transaction) => ({
691
+ readVersion: transaction.readVersion,
692
+ uuid: transaction.uuid,
693
+ operation: transaction.operation,
694
+ tag: transaction.tag ?? undefined,
695
+ transactionProperties: transaction.transactionProperties ?? undefined,
696
+ }));
697
+ }
698
+ }
699
+ exports.TableDelta = TableDelta;
700
+ function toNativeDeltaRef(value) {
701
+ if (typeof value === "string") {
702
+ if (value.length === 0) {
703
+ throw new Error("delta tag must be a non-empty string");
704
+ }
705
+ return { tag: value };
706
+ }
707
+ if (typeof value === "bigint") {
708
+ if (value <= 0n) {
709
+ throw new Error("delta version must be a positive u64 bigint");
710
+ }
711
+ return { version: value };
712
+ }
713
+ if (!Number.isSafeInteger(value) || value <= 0) {
714
+ throw new Error("delta version must be a positive safe integer; use bigint for larger versions");
715
+ }
716
+ return { version: BigInt(value) };
717
+ }
718
+ function tableDeltaReader(open) {
719
+ return new TableDeltaReaderImpl(open);
720
+ }
721
+ class TableDeltaReaderImpl {
722
+ #open;
723
+ #setup;
724
+ #schema;
725
+ #schemaSetup;
726
+ #iterator;
727
+ #error;
728
+ #done = false;
729
+ #pullQueue = Promise.resolve();
730
+ constructor(open) {
731
+ this.#open = open;
732
+ }
733
+ [Symbol.asyncIterator]() {
734
+ return this;
735
+ }
736
+ async schema() {
737
+ if (this.#schema !== undefined) {
738
+ return this.#schema;
739
+ }
740
+ if (this.#error !== undefined) {
741
+ throw this.#error;
742
+ }
743
+ if (this.#schemaSetup === undefined) {
744
+ this.#schemaSetup = this.#loadSchema();
745
+ }
746
+ return this.#schemaSetup;
747
+ }
748
+ async #loadSchema() {
749
+ try {
750
+ const native = await this.#getNative();
751
+ const schema = (0, arrow_1.tableFromIPC)(await native.schema()).schema;
752
+ this.#schema = schema;
753
+ return schema;
754
+ }
755
+ catch (error) {
756
+ return this.#fail(error);
757
+ }
758
+ }
759
+ next() {
760
+ return this.#serialize(() => this.#pull());
761
+ }
762
+ async return() {
763
+ await this.#serialize(async () => {
764
+ this.#done = true;
765
+ await this.#release();
766
+ });
767
+ return { done: true, value: undefined };
768
+ }
769
+ async throw(error) {
770
+ await this.#serialize(async () => {
771
+ this.#done = true;
772
+ await this.#release();
773
+ });
774
+ throw error;
775
+ }
776
+ async #pull() {
777
+ if (this.#error !== undefined) {
778
+ throw this.#error;
779
+ }
780
+ if (this.#done) {
781
+ return { done: true, value: undefined };
782
+ }
783
+ // Capture the schema before polling the reader's shared native iterator.
784
+ await this.schema();
785
+ try {
786
+ const result = await this.#getIterator().next();
787
+ if (result.done) {
788
+ this.#done = true;
789
+ await this.#release();
790
+ }
791
+ return result;
792
+ }
793
+ catch (error) {
794
+ return this.#fail(error);
795
+ }
796
+ }
797
+ #getIterator() {
798
+ if (this.#iterator === undefined) {
799
+ this.#iterator = (0, query_1.RecordBatchIterator)(this.#getNative())[Symbol.asyncIterator]();
800
+ }
801
+ return this.#iterator;
802
+ }
803
+ async #getNative() {
804
+ if (this.#error !== undefined) {
805
+ throw this.#error;
806
+ }
807
+ if (this.#done) {
808
+ throw new Error("table delta reader is already consumed");
809
+ }
810
+ if (this.#setup === undefined) {
811
+ try {
812
+ this.#setup = this.#open();
813
+ }
814
+ catch (error) {
815
+ return this.#fail(error);
816
+ }
817
+ }
818
+ try {
819
+ return await this.#setup;
820
+ }
821
+ catch (error) {
822
+ return this.#fail(error);
823
+ }
824
+ }
825
+ async #fail(error) {
826
+ if (this.#error === undefined) {
827
+ this.#done = true;
828
+ this.#error = error;
829
+ await this.#release();
830
+ }
831
+ throw this.#error;
832
+ }
833
+ async #release() {
834
+ const iterator = this.#iterator;
835
+ this.#iterator = undefined;
836
+ this.#setup = undefined;
837
+ if (iterator?.return !== undefined) {
838
+ try {
839
+ await iterator.return();
840
+ }
841
+ catch {
842
+ // The native stream already failed or completed; releasing the
843
+ // reference is enough.
844
+ }
845
+ }
846
+ }
847
+ #serialize(operation) {
848
+ const next = this.#pullQueue.then(operation, operation);
849
+ this.#pullQueue = next.then(() => undefined, () => undefined);
850
+ return next;
851
+ }
852
+ }
508
853
  /**
509
854
  * Branch manager for a {@link Table}.
510
855
  *
@@ -524,15 +869,18 @@ class Branches {
524
869
  async list() {
525
870
  return await this.#inner.list();
526
871
  }
527
- /**
528
- * Create a branch and return a handle scoped to it.
529
- *
530
- * @param name Name of the new branch.
531
- * @param fromRef Source branch to fork from. Defaults to `main`.
532
- * @param fromVersion A specific version on `fromRef`. Defaults to latest.
533
- */
534
- async create(name, fromRef, fromVersion) {
535
- return new LocalTable(await this.#inner.create(name, fromRef, fromVersion));
872
+ async create(name, fromRefOrOptions, fromVersion) {
873
+ if (typeof fromRefOrOptions === "object" && fromRefOrOptions !== null) {
874
+ // The public options shape only adds `fromTag`. Forward any runtime
875
+ // `fromRef`/`fromVersion` fields so the native check remains the single
876
+ // authority for rejecting mixed sources before `main` normalization.
877
+ const options = fromRefOrOptions;
878
+ if (typeof options.fromTag !== "string") {
879
+ throw new Error("branch source options must include fromTag");
880
+ }
881
+ return new LocalTable(await this.#inner.create(name, options.fromRef, fromVersion ?? options.fromVersion, options.fromTag));
882
+ }
883
+ return new LocalTable(await this.#inner.create(name, fromRefOrOptions, fromVersion));
536
884
  }
537
885
  /**
538
886
  * Check out an existing branch and return a handle scoped to it.
@@ -598,3 +946,21 @@ function validateCompactionOptions(options) {
598
946
  }
599
947
  }
600
948
  }
949
+ async function toNativePrewarmDataOptions(options) {
950
+ validatePrewarmDataOptions(options);
951
+ return options === undefined
952
+ ? undefined
953
+ : {
954
+ ...options,
955
+ parameters: options.parameters === undefined
956
+ ? undefined
957
+ : await (0, sql_1.toParameterBuffer)(options.parameters),
958
+ };
959
+ }
960
+ function validatePrewarmDataOptions(options) {
961
+ for (const key of Object.keys(options ?? {})) {
962
+ if (!["columns", "filter", "concurrency", "parameters"].includes(key)) {
963
+ throw new Error(`Unsupported data prewarm option: ${key}`);
964
+ }
965
+ }
966
+ }
package/package.json CHANGED
@@ -11,7 +11,7 @@
11
11
  "ann"
12
12
  ],
13
13
  "private": false,
14
- "version": "0.42.0-beta.20",
14
+ "version": "0.42.0-beta.21",
15
15
  "main": "dist/index.js",
16
16
  "exports": {
17
17
  ".": "./dist/index.js",
@@ -111,13 +111,13 @@
111
111
  "optionalDependencies": {
112
112
  "@huggingface/transformers": "4.3.2",
113
113
  "openai": "4.29.2",
114
- "@lancedb/lancedb-darwin-arm64": "0.42.0-beta.20",
115
- "@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.20",
116
- "@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.20",
117
- "@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.20",
118
- "@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.20",
119
- "@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.20",
120
- "@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.20"
114
+ "@lancedb/lancedb-darwin-arm64": "0.42.0-beta.21",
115
+ "@lancedb/lancedb-linux-x64-gnu": "0.42.0-beta.21",
116
+ "@lancedb/lancedb-linux-arm64-gnu": "0.42.0-beta.21",
117
+ "@lancedb/lancedb-linux-x64-musl": "0.42.0-beta.21",
118
+ "@lancedb/lancedb-linux-arm64-musl": "0.42.0-beta.21",
119
+ "@lancedb/lancedb-win32-x64-msvc": "0.42.0-beta.21",
120
+ "@lancedb/lancedb-win32-arm64-msvc": "0.42.0-beta.21"
121
121
  },
122
122
  "peerDependencies": {
123
123
  "@types/node": ">=22",