@lancedb/lancedb 0.42.0-beta.2 → 0.42.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RUST_THIRD_PARTY_LICENSES.html +11637 -6176
- package/dist/arrow.js +3 -1
- package/dist/blob.d.ts +11 -0
- package/dist/blob.js +27 -0
- package/dist/cleanup.d.ts +31 -0
- package/dist/cleanup.js +80 -0
- package/dist/compaction.d.ts +23 -0
- package/dist/compaction.js +21 -0
- package/dist/connection.d.ts +2 -2
- package/dist/index.d.ts +13 -8
- package/dist/index.js +3 -2
- package/dist/indices.d.ts +44 -0
- package/dist/indices.js +42 -0
- package/dist/job.d.ts +117 -4
- package/dist/job.js +190 -12
- package/dist/merge.d.ts +17 -0
- package/dist/merge.js +7 -1
- package/dist/native.d.ts +296 -9
- package/dist/native.js +53 -52
- package/dist/prewarm.d.ts +40 -0
- package/dist/prewarm.js +37 -0
- package/dist/profiling.d.ts +44 -0
- package/dist/profiling.js +37 -0
- package/dist/query.d.ts +12 -2
- package/dist/query.js +55 -4
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +61 -0
- package/dist/table.d.ts +439 -12
- package/dist/table.js +461 -18
- package/package.json +9 -9
- package/pnpm-workspace.yaml +0 -7
package/dist/sql.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.toParameterBuffer = toParameterBuffer;
|
|
6
|
+
const arrow_1 = require("./arrow");
|
|
7
|
+
const sanitize_1 = require("./sanitize");
|
|
8
|
+
/** @internal */
|
|
9
|
+
async function toParameterBuffer(parameters) {
|
|
10
|
+
let table;
|
|
11
|
+
if ("schema" in parameters &&
|
|
12
|
+
parameters.schema !== null &&
|
|
13
|
+
typeof parameters.schema === "object" &&
|
|
14
|
+
"fields" in parameters.schema) {
|
|
15
|
+
const arrow = parameters;
|
|
16
|
+
table = (0, sanitize_1.sanitizeTable)("batches" in arrow ? arrow : { schema: arrow.schema, batches: [arrow] });
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
const entries = Array.isArray(parameters)
|
|
20
|
+
? parameters.map((value, index) => [String(index + 1), value])
|
|
21
|
+
: Object.entries(parameters);
|
|
22
|
+
const columns = Object.fromEntries(entries.map(([name, input]) => {
|
|
23
|
+
let value = input;
|
|
24
|
+
let type;
|
|
25
|
+
if (value === null)
|
|
26
|
+
type = new arrow_1.Null();
|
|
27
|
+
else if (typeof value === "string")
|
|
28
|
+
type = new arrow_1.Utf8();
|
|
29
|
+
else if (typeof value === "boolean")
|
|
30
|
+
type = new arrow_1.Bool();
|
|
31
|
+
else if (typeof value === "number") {
|
|
32
|
+
if (!Number.isFinite(value) ||
|
|
33
|
+
(Number.isInteger(value) && !Number.isSafeInteger(value))) {
|
|
34
|
+
throw new Error("Numeric parameters must be finite and integers must be safe; use bigint for Int64");
|
|
35
|
+
}
|
|
36
|
+
type = Number.isInteger(value) ? new arrow_1.Int64() : new arrow_1.Float64();
|
|
37
|
+
if (Number.isInteger(value))
|
|
38
|
+
value = BigInt(value);
|
|
39
|
+
}
|
|
40
|
+
else if (typeof value === "bigint") {
|
|
41
|
+
if (value < -(1n << 63n) || value >= 1n << 63n) {
|
|
42
|
+
throw new Error("bigint parameters must fit Int64");
|
|
43
|
+
}
|
|
44
|
+
type = new arrow_1.Int64();
|
|
45
|
+
}
|
|
46
|
+
else if (value instanceof Date && Number.isFinite(value.getTime())) {
|
|
47
|
+
type = new arrow_1.TimestampMillisecond();
|
|
48
|
+
}
|
|
49
|
+
else if (value instanceof Uint8Array)
|
|
50
|
+
type = new arrow_1.Binary();
|
|
51
|
+
else
|
|
52
|
+
throw new Error("Query parameters must be scalar values or a one-row Arrow table");
|
|
53
|
+
return [name, (0, arrow_1.vectorFromArray)([value], type)];
|
|
54
|
+
}));
|
|
55
|
+
table = new arrow_1.Table(columns);
|
|
56
|
+
}
|
|
57
|
+
if (table.numRows !== 1) {
|
|
58
|
+
throw new Error(`Query parameters must contain exactly one row, got ${table.numRows}`);
|
|
59
|
+
}
|
|
60
|
+
return Buffer.from(await arrow_1.RecordBatchFileWriter.writeAll(table).toUint8Array());
|
|
61
|
+
}
|
package/dist/table.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { CleanupOptions, CleanupPlan, CleanupResult } from "./cleanup";
|
|
2
|
+
import { CompactionResult } from "./compaction";
|
|
3
|
+
import { PrewarmDataPlan, PrewarmDataResult } from "./prewarm";
|
|
4
|
+
import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, RecordBatch, Schema } from "./arrow";
|
|
5
|
+
import { BlobFile, BlobRangeRequest, BlobReadOptions } from "./blob";
|
|
3
6
|
import { IndexOptions } from "./indices";
|
|
4
7
|
import { Job } from "./job";
|
|
5
8
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
9
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, JobFailureInfo, Branches as NativeBranches, TableDelta as NativeTableDelta, OptimizeStats, RTEStats, RefreshColumnOptions, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
10
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
11
|
+
import { QueryParameters } from "./sql";
|
|
8
12
|
import { IntoSql } from "./util";
|
|
9
13
|
export { IndexConfig } from "./native";
|
|
10
14
|
export { TableShardStats, SsTableStats, RTEStats, MemtableStats, } from "./native";
|
|
@@ -75,6 +79,12 @@ export interface AddDataOptions {
|
|
|
75
79
|
* when the blob is read later.
|
|
76
80
|
*/
|
|
77
81
|
allowExternalBlobOutsideBases?: boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Override the remote retry budget for this call (0 to 255). Zero disables
|
|
84
|
+
* retries, including multipart parts. Ambiguous commits are never replayed.
|
|
85
|
+
* Ignored by LanceDB Embedded.
|
|
86
|
+
*/
|
|
87
|
+
retries?: number;
|
|
78
88
|
}
|
|
79
89
|
export interface UpdateOptions {
|
|
80
90
|
/**
|
|
@@ -89,7 +99,41 @@ export interface UpdateOptions {
|
|
|
89
99
|
*/
|
|
90
100
|
where: string;
|
|
91
101
|
}
|
|
102
|
+
/** Settings for data-cache prewarm submissions and metadata previews. */
|
|
103
|
+
export interface PrewarmDataOptions {
|
|
104
|
+
/** Lance field paths to warm; omitted or empty means every column. */
|
|
105
|
+
columns?: string[];
|
|
106
|
+
/** A single SQL predicate for conservative metadata pruning. */
|
|
107
|
+
filter?: string;
|
|
108
|
+
/** Typed positional or named filter values, sent separately from SQL. */
|
|
109
|
+
parameters?: QueryParameters;
|
|
110
|
+
/** Positive per-executor concurrency, at most 4294967295. */
|
|
111
|
+
concurrency?: number;
|
|
112
|
+
}
|
|
113
|
+
/** Per-invocation compaction settings. Values must be safe integers. */
|
|
114
|
+
export interface CompactionOptions {
|
|
115
|
+
/** Positive target rows per output fragment; also affects candidate selection. */
|
|
116
|
+
targetRowsPerFragment?: number;
|
|
117
|
+
/** Positive live-source-row budget across whole tasks; an oversized first task can produce no work. */
|
|
118
|
+
maxSourceRows?: number;
|
|
119
|
+
/** Scanner I/O read-ahead budget in bytes; zero disables byte-based I/O backpressure. */
|
|
120
|
+
ioBufferSize?: number;
|
|
121
|
+
/** Positive scanner batch size in rows. */
|
|
122
|
+
batchSize?: number;
|
|
123
|
+
}
|
|
92
124
|
export interface OptimizeOptions {
|
|
125
|
+
/**
|
|
126
|
+
* Compaction settings for this invocation. Omitted remote values inherit table
|
|
127
|
+
* tuning, then deployment/built-in defaults.
|
|
128
|
+
* Remote tables require a server supporting these SQL options; settings do
|
|
129
|
+
* not change table policy. This is not a runtime or total-memory limit.
|
|
130
|
+
* @example
|
|
131
|
+
* await tbl.optimize({ compactionOptions: {
|
|
132
|
+
* targetRowsPerFragment: 1000, maxSourceRows: 5000,
|
|
133
|
+
* ioBufferSize: 64 * 1024 * 1024, batchSize: 128,
|
|
134
|
+
* } });
|
|
135
|
+
*/
|
|
136
|
+
compactionOptions?: CompactionOptions;
|
|
93
137
|
/**
|
|
94
138
|
* If set then all versions older than the given date
|
|
95
139
|
* be removed. The current version will never be removed.
|
|
@@ -122,6 +166,37 @@ export interface Version {
|
|
|
122
166
|
timestamp: Date;
|
|
123
167
|
metadata: Record<string, string>;
|
|
124
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* One endpoint of a table delta: a positive version number/bigint or a tag on
|
|
171
|
+
* the table handle's current branch.
|
|
172
|
+
*
|
|
173
|
+
* Use a `bigint` when the version may exceed `Number.MAX_SAFE_INTEGER`.
|
|
174
|
+
* Strings are always tags.
|
|
175
|
+
*/
|
|
176
|
+
export type DeltaVersion = number | bigint | string;
|
|
177
|
+
/** A committed transaction in a table delta range. */
|
|
178
|
+
export interface TableDeltaTransaction {
|
|
179
|
+
/** The version the transaction was based on. Exact u64 value. */
|
|
180
|
+
readVersion: bigint;
|
|
181
|
+
uuid: string;
|
|
182
|
+
operation: string;
|
|
183
|
+
tag?: string;
|
|
184
|
+
transactionProperties?: Record<string, string>;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* A streaming table-delta reader.
|
|
188
|
+
*
|
|
189
|
+
* `schema()` is available before consuming the stream, including when the
|
|
190
|
+
* range is empty. The stream yields Arrow record batches without
|
|
191
|
+
* materializing the full result.
|
|
192
|
+
*
|
|
193
|
+
* A reader owns one native cursor. It is single-use:
|
|
194
|
+
* `[Symbol.asyncIterator]()` returns the same iterator, concurrent pulls are
|
|
195
|
+
* serialized, and `return()` releases the native stream on early drop.
|
|
196
|
+
*/
|
|
197
|
+
export interface TableDeltaReader extends AsyncIterable<RecordBatch>, AsyncIterator<RecordBatch> {
|
|
198
|
+
schema(): Promise<Schema>;
|
|
199
|
+
}
|
|
125
200
|
/** Token produced by the tokenizer configured on a full-text search index. */
|
|
126
201
|
export interface FtsToken {
|
|
127
202
|
/** Token text after tokenizer filters have been applied. */
|
|
@@ -164,6 +239,55 @@ export interface RTEWriteSpec {
|
|
|
164
239
|
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
|
|
165
240
|
writerConfigDefaults?: Record<string, string>;
|
|
166
241
|
}
|
|
242
|
+
/** Exact immutable Function revision attached to a table output. */
|
|
243
|
+
export interface FunctionColumnVersionStatus {
|
|
244
|
+
name: string;
|
|
245
|
+
namespacePath: string[];
|
|
246
|
+
objectId: string;
|
|
247
|
+
location: string;
|
|
248
|
+
version: string;
|
|
249
|
+
manifestDigest: string;
|
|
250
|
+
}
|
|
251
|
+
/** Latest matching refresh Job projected for a Function output. */
|
|
252
|
+
export interface FunctionColumnLastRefreshJob {
|
|
253
|
+
jobId: string;
|
|
254
|
+
state: string;
|
|
255
|
+
automatic?: boolean;
|
|
256
|
+
completedAtMillis?: bigint;
|
|
257
|
+
/** Structured failure details for a failed Job. */
|
|
258
|
+
failure?: JobFailureInfo;
|
|
259
|
+
}
|
|
260
|
+
/** Automatic-refresh projection for a Function output. */
|
|
261
|
+
export interface FunctionColumnAutoRefreshStatus {
|
|
262
|
+
enabled: boolean;
|
|
263
|
+
intervalSeconds?: bigint;
|
|
264
|
+
activeJobId?: string;
|
|
265
|
+
state: "disabled" | "running" | "queued" | "ready" | "up_to_date" | "waiting_interval" | "waiting_backoff" | "blocked" | "unobservable";
|
|
266
|
+
failureStreak: number;
|
|
267
|
+
effectiveCooldownMillis?: bigint;
|
|
268
|
+
blockedReason?: "table_has_mem_wal" | "inputs_not_settled" | "dependency_in_flight" | "table_capacity" | "cancelled" | "preparation_backoff" | "failure_threshold_reached" | "binding_revision_missing";
|
|
269
|
+
nextRunAtMillis: bigint | null;
|
|
270
|
+
nextRunAtReason?: string;
|
|
271
|
+
}
|
|
272
|
+
/** Exact four-way status of one Function output at one table version. */
|
|
273
|
+
export interface FunctionColumnStatus {
|
|
274
|
+
column: string;
|
|
275
|
+
bindingId: string;
|
|
276
|
+
function: FunctionColumnVersionStatus;
|
|
277
|
+
tableVersion: bigint;
|
|
278
|
+
rows: bigint;
|
|
279
|
+
fresh: bigint;
|
|
280
|
+
stale: bigint;
|
|
281
|
+
neverComputed: bigint;
|
|
282
|
+
failed: bigint;
|
|
283
|
+
lastRefreshJob?: FunctionColumnLastRefreshJob;
|
|
284
|
+
/**
|
|
285
|
+
* Newest failed refresh Job. This remains present when `lastRefreshJob`
|
|
286
|
+
* is a later cancellation.
|
|
287
|
+
*/
|
|
288
|
+
lastFailureJob?: FunctionColumnLastRefreshJob;
|
|
289
|
+
autoRefresh: FunctionColumnAutoRefreshStatus;
|
|
290
|
+
}
|
|
167
291
|
/**
|
|
168
292
|
* A Table is a collection of Records in a LanceDB Database.
|
|
169
293
|
*
|
|
@@ -360,21 +484,71 @@ export declare abstract class Table {
|
|
|
360
484
|
/**
|
|
361
485
|
* Prewarm one or more columns of data in the table.
|
|
362
486
|
*
|
|
363
|
-
* @param columns The columns to prewarm. If undefined, all columns are prewarmed.
|
|
364
|
-
*
|
|
365
487
|
* This will load the column data into the page cache so that future queries that
|
|
366
488
|
* read those columns avoid the initial cold-start latency. This call initiates
|
|
367
489
|
* prewarming and returns once the request is accepted; the warming itself may
|
|
368
|
-
* continue in the background.
|
|
369
|
-
*
|
|
490
|
+
* continue in the background. Submission uses SQL PREWARM and needs Flight
|
|
491
|
+
* SQL connectivity. Use {@link prewarmDataAsync} to retain the job handle.
|
|
370
492
|
*
|
|
371
493
|
* Prewarming is generally useful for columns used in filters or projections.
|
|
372
494
|
* Large columns (e.g. high-dimensional vectors or binary data) may not be
|
|
373
495
|
* practical to prewarm.
|
|
374
496
|
*
|
|
375
497
|
* This feature is currently only supported in LanceDB Enterprise.
|
|
498
|
+
*
|
|
499
|
+
* @param options Columns, a metadata-pruning filter, and per-executor concurrency.
|
|
500
|
+
* Omitted or empty columns warm every column.
|
|
501
|
+
*
|
|
502
|
+
* @example
|
|
503
|
+
* await table.prewarmData({
|
|
504
|
+
* columns: ["payload"], filter: "quality > 0.7", concurrency: 4,
|
|
505
|
+
* });
|
|
506
|
+
*/
|
|
507
|
+
abstract prewarmData(options?: PrewarmDataOptions): Promise<void>;
|
|
508
|
+
/**
|
|
509
|
+
* Submit data prewarm using the columns-array form and return after acceptance.
|
|
510
|
+
* @param columns Columns to warm; omitted or empty means every column.
|
|
376
511
|
*/
|
|
377
512
|
abstract prewarmData(columns?: string[]): Promise<void>;
|
|
513
|
+
/**
|
|
514
|
+
* Submit data prewarm through SQL and return its durable warming-job handle.
|
|
515
|
+
*
|
|
516
|
+
* The table handle supplies the branch and namespace. Requires a compatible
|
|
517
|
+
* SQL PREWARM server and Flight SQL connectivity. Standard authorization
|
|
518
|
+
* requires table Select and database/namespace Usage. Status/wait use generic
|
|
519
|
+
* HTTP job APIs; events and cancellation retain their own privileges.
|
|
520
|
+
* No REST fallback or option-dropping retry is performed. Historical-version
|
|
521
|
+
* handles are unsupported; call `checkoutLatest()` first.
|
|
522
|
+
*
|
|
523
|
+
* Submission returns after acceptance. Use `job.wait()` for completion, or
|
|
524
|
+
* save `job.id` and reopen it with `db.openJob(id)`. A timeout does not cancel
|
|
525
|
+
* the job. After `job.refresh()`, `job.spec.table_version` is the pinned version.
|
|
526
|
+
*
|
|
527
|
+
* @example
|
|
528
|
+
* const job = await table.prewarmDataAsync({
|
|
529
|
+
* columns: ["video"], filter: "quality > $minimum",
|
|
530
|
+
* parameters: { minimum: 0.7 }, concurrency: 4,
|
|
531
|
+
* });
|
|
532
|
+
* await job.wait();
|
|
533
|
+
*/
|
|
534
|
+
abstract prewarmDataAsync(options?: PrewarmDataOptions): Promise<Job<PrewarmDataResult>>;
|
|
535
|
+
/**
|
|
536
|
+
* Preview prewarm with the same options as {@link prewarmDataAsync}, without
|
|
537
|
+
* submitting a warming job. Requires LanceDB Enterprise with structured SQL
|
|
538
|
+
* previews, table Select and database/namespace Usage. Reads metadata and
|
|
539
|
+
* returns the source version, readable plan, and conservative pruning summary.
|
|
540
|
+
* A later submission plans its own snapshot. Historical and Embedded handles
|
|
541
|
+
* are unsupported. Unknown measurements remain null.
|
|
542
|
+
* Custom table implementations can override this method; its default rejects
|
|
543
|
+
* with an unsupported-operation error.
|
|
544
|
+
*
|
|
545
|
+
* @example
|
|
546
|
+
* const plan = await table.explainPrewarmData({
|
|
547
|
+
* columns: ["video"], filter: "quality > $minimum", parameters: {minimum: 0.7},
|
|
548
|
+
* });
|
|
549
|
+
* console.log(plan.pruning.selectedFragments);
|
|
550
|
+
*/
|
|
551
|
+
explainPrewarmData(_options?: PrewarmDataOptions): Promise<PrewarmDataPlan>;
|
|
378
552
|
/**
|
|
379
553
|
* Waits for asynchronous indexing to complete on the table.
|
|
380
554
|
*
|
|
@@ -472,6 +646,26 @@ export declare abstract class Table {
|
|
|
472
646
|
* memory; use {@link Table.fetchBlobFiles} for large values.
|
|
473
647
|
*/
|
|
474
648
|
abstract fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
649
|
+
/**
|
|
650
|
+
* Materializes row-specific byte ranges at the table's current checkout.
|
|
651
|
+
* Preserves request order, duplicates, and nulls. Zero length returns an empty
|
|
652
|
+
* buffer for a non-null blob. Invalid ranges or missing rows reject the entire
|
|
653
|
+
* call. See {@link Table.fetchBlobs} for row-ID validity across versions.
|
|
654
|
+
*
|
|
655
|
+
* Remote tables use the batched endpoint when the server advertises support,
|
|
656
|
+
* splitting at 1024 ranges and 64 MiB, including individual larger ranges.
|
|
657
|
+
* Older servers use descriptor queries and BlobFile range reads. Both protocols
|
|
658
|
+
* use one exact branch and version. Errors from an advertised endpoint propagate
|
|
659
|
+
* without switching protocols. All output bytes occupy memory.
|
|
660
|
+
*
|
|
661
|
+
* @example
|
|
662
|
+
* ```ts
|
|
663
|
+
* const bytes = await table.fetchBlobRanges("image", [
|
|
664
|
+
* { rowId: 10n, offset: 0n, length: 1024n },
|
|
665
|
+
* ]);
|
|
666
|
+
* ```
|
|
667
|
+
*/
|
|
668
|
+
abstract fetchBlobRanges(column: string, requests: readonly BlobRangeRequest[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
475
669
|
/**
|
|
476
670
|
* Opens lazy blob handles for `column` at the given row IDs using the
|
|
477
671
|
* table's current checkout.
|
|
@@ -548,10 +742,12 @@ export declare abstract class Table {
|
|
|
548
742
|
* everything else is left as it is. LanceDB Embedded only: in LanceDB
|
|
549
743
|
* Enterprise a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
|
|
550
744
|
* @param {string} column The name of the computed column to fill.
|
|
745
|
+
* @param {RefreshColumnOptions} options Refresh options, all LanceDB
|
|
746
|
+
* Enterprise only; see {@link Table#refreshColumnAsync}.
|
|
551
747
|
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
|
|
552
748
|
* number of rows filled and the new version number of the table.
|
|
553
749
|
*/
|
|
554
|
-
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
750
|
+
abstract refreshColumn(column: string, options?: RefreshColumnOptions): Promise<RefreshColumnResult>;
|
|
555
751
|
/**
|
|
556
752
|
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
557
753
|
* job instead of blocking until it completes.
|
|
@@ -565,14 +761,54 @@ export declare abstract class Table {
|
|
|
565
761
|
* than failing the job. In LanceDB Embedded the job runs in-process; in
|
|
566
762
|
* LanceDB Enterprise it is the server's backfill job.
|
|
567
763
|
* @param {string} column The name of the computed column to fill.
|
|
764
|
+
* @param {RefreshColumnOptions} options Refresh options, mirroring the SQL
|
|
765
|
+
* `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
|
|
766
|
+
* `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
|
|
767
|
+
* and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
|
|
768
|
+
* `maxConcurrency`, `resourceBudget`, `supervisorResources`,
|
|
769
|
+
* `planningWorkers` and `maxExecutorProcessesPerWorker` size registered
|
|
770
|
+
* Function refreshes. `retryPolicy` overrides user-code retries only when
|
|
771
|
+
* repeating partial work is safe; set `maxAttempts: 1` to disable retries.
|
|
772
|
+
* `maxConcurrency` overrides the registered default for this logical
|
|
773
|
+
* execution only and is a positive integer; it is not an account-wide
|
|
774
|
+
* quota. A `resourceBudget` requires `cpuMillis` and
|
|
775
|
+
* `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
|
|
776
|
+
* `supervisorResources` and each `planningWorkers.resources` entry require
|
|
777
|
+
* positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`.
|
|
778
|
+
* `planningWorkers` also requires positive `maxWorkers`,
|
|
779
|
+
* `rowsPerWorker` and `fragmentsPerWorker`.
|
|
780
|
+
* The budget applies only to this explicit refresh and does not configure
|
|
781
|
+
* a later automatic refresh.
|
|
782
|
+
* `gpuCount` at the top level is the shorthand that preserves the
|
|
783
|
+
* deployment defaults for CPU and memory, and cannot be combined with
|
|
784
|
+
* `resourceBudget`. `numWorkers` applies only to expression columns; use
|
|
785
|
+
* `resourceBudget` for registered Functions. All are LanceDB Enterprise
|
|
786
|
+
* only; a LanceDB Embedded refresh rejects every option other than
|
|
787
|
+
* `batchSize`, `gpuClass` and `gpuCount`.
|
|
788
|
+
* `publicationIntervalRows` opts into publishing completed Function split
|
|
789
|
+
* ranges once their accumulated unpublished rows reach the threshold.
|
|
790
|
+
* It must be a positive safe integer and requires non-Blob Function outputs.
|
|
791
|
+
* Omit it for whole-fragment publication. Publications can exceed the
|
|
792
|
+
* threshold, and remaining rows publish at completion.
|
|
568
793
|
* @example
|
|
569
794
|
* ```ts
|
|
570
|
-
* const job = await table.refreshColumnAsync("
|
|
795
|
+
* const job = await table.refreshColumnAsync("embedding", {
|
|
796
|
+
* where: "published_at > '2026-01-01'",
|
|
797
|
+
* resourceBudget: {
|
|
798
|
+
* cpuMillis: 8000,
|
|
799
|
+
* memoryBytes: 16 * 1024 ** 3,
|
|
800
|
+
* },
|
|
801
|
+
* maxConcurrency: 4,
|
|
802
|
+
* publicationIntervalRows: 10000,
|
|
803
|
+
* errorPolicy: "skip",
|
|
804
|
+
* maxFailedFraction: 0.01,
|
|
805
|
+
* rowTimeoutSeconds: 30,
|
|
806
|
+
* });
|
|
571
807
|
* await job.wait();
|
|
572
808
|
* console.log(await job.status()); // "finished"
|
|
573
809
|
* ```
|
|
574
810
|
*/
|
|
575
|
-
abstract refreshColumnAsync(column: string): Promise<Job>;
|
|
811
|
+
abstract refreshColumnAsync(column: string, options?: RefreshColumnOptions): Promise<Job>;
|
|
576
812
|
/**
|
|
577
813
|
* The per-row errors Function refreshes recorded on this table.
|
|
578
814
|
*
|
|
@@ -591,6 +827,24 @@ export declare abstract class Table {
|
|
|
591
827
|
* ```
|
|
592
828
|
*/
|
|
593
829
|
abstract functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
830
|
+
/**
|
|
831
|
+
* Return exact freshness status for Function-backed output columns.
|
|
832
|
+
*
|
|
833
|
+
* All results describe one server table version. Omit `column` for every
|
|
834
|
+
* Function output. Rows already failed remain visible and do not prevent
|
|
835
|
+
* `waitFresh` from completing. A handle pinned to an exact version requests
|
|
836
|
+
* that version; a latest handle retains normal read-freshness behavior.
|
|
837
|
+
*/
|
|
838
|
+
abstract functionColumnStatuses(column?: string): Promise<FunctionColumnStatus[]>;
|
|
839
|
+
/** Return exact freshness status for one Function-backed output column. */
|
|
840
|
+
abstract functionColumnStatus(column: string): Promise<FunctionColumnStatus>;
|
|
841
|
+
/**
|
|
842
|
+
* Poll until no stale or never-computed rows remain.
|
|
843
|
+
*
|
|
844
|
+
* Only HTTP 503 with the exact finalization-pending error code is retried;
|
|
845
|
+
* every other error propagates. Failed rows do not block completion.
|
|
846
|
+
*/
|
|
847
|
+
abstract waitFresh(column: string, timeoutSeconds?: number): Promise<FunctionColumnStatus>;
|
|
594
848
|
/**
|
|
595
849
|
* Recompute this table's contents from its materialized-view definition.
|
|
596
850
|
*
|
|
@@ -711,6 +965,45 @@ export declare abstract class Table {
|
|
|
711
965
|
* @returns {Promise<RTEWriteSpec | undefined>}
|
|
712
966
|
*/
|
|
713
967
|
abstract getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
968
|
+
/**
|
|
969
|
+
* Read the stored overrides for a remote Enterprise table.
|
|
970
|
+
*
|
|
971
|
+
* Overrides configure background behavior such as automatic compaction,
|
|
972
|
+
* cleanup, and reindex execution. Keys use the server's snake_case names.
|
|
973
|
+
* Returns an empty object when no overrides are set. Local tables are unsupported.
|
|
974
|
+
* Custom table implementations inherit a default rejection unless they override
|
|
975
|
+
* this method.
|
|
976
|
+
*/
|
|
977
|
+
getTableOverrides(): Promise<Record<string, unknown>>;
|
|
978
|
+
/**
|
|
979
|
+
* Recursively merge overrides into a remote Enterprise table's stored overrides.
|
|
980
|
+
*
|
|
981
|
+
* Fetches the current overrides, merges nested objects locally, then sends the
|
|
982
|
+
* complete replacement using REST PUT. Other values replace existing values;
|
|
983
|
+
* unrelated fields from the read snapshot are preserved. This read-modify-write
|
|
984
|
+
* is not atomic: concurrent edits can be overwritten, even to unrelated fields.
|
|
985
|
+
* Coordinate writers when concurrent edits must be preserved. An empty update
|
|
986
|
+
* only reads and returns the current overrides.
|
|
987
|
+
*
|
|
988
|
+
* Unlike this SDK method, CLI `set` and REST PUT replace the overrides object
|
|
989
|
+
* without merging. Use {@link Table.resetTableOverrides} to clear all overrides.
|
|
990
|
+
* Local tables and custom table implementations without override support reject
|
|
991
|
+
* this operation.
|
|
992
|
+
*
|
|
993
|
+
* @example
|
|
994
|
+
* ```ts
|
|
995
|
+
* await table.updateTableOverrides({
|
|
996
|
+
* job_types: { compaction: { enabled: false } },
|
|
997
|
+
* });
|
|
998
|
+
* ```
|
|
999
|
+
*/
|
|
1000
|
+
updateTableOverrides(overrides: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
1001
|
+
/**
|
|
1002
|
+
* Clear a remote Enterprise table's overrides and return an empty object.
|
|
1003
|
+
* Local tables and custom table implementations without override support reject
|
|
1004
|
+
* this operation.
|
|
1005
|
+
*/
|
|
1006
|
+
resetTableOverrides(): Promise<Record<string, unknown>>;
|
|
714
1007
|
/**
|
|
715
1008
|
* Drain and close any cached RTE shard writers held for this table.
|
|
716
1009
|
*
|
|
@@ -854,6 +1147,67 @@ export declare abstract class Table {
|
|
|
854
1147
|
* out state and the read_consistency_interval, if any, will apply.
|
|
855
1148
|
*/
|
|
856
1149
|
abstract restore(): Promise<void>;
|
|
1150
|
+
/**
|
|
1151
|
+
* Compare two persisted versions or same-branch tags on an Embedded table.
|
|
1152
|
+
*
|
|
1153
|
+
* `fromVersion` is exclusive and `toVersion` is inclusive. Omitting
|
|
1154
|
+
* `toVersion` uses the version currently visible through this handle,
|
|
1155
|
+
* including a checked-out version. The endpoints are fixed when the delta is
|
|
1156
|
+
* created.
|
|
1157
|
+
*
|
|
1158
|
+
* Row readers require stable row IDs at both endpoints. Create the table with
|
|
1159
|
+
* `storageOptions: { new_table_enable_stable_row_ids: "true" }` to enable
|
|
1160
|
+
* them. `listTransactions()` works without stable row IDs.
|
|
1161
|
+
*
|
|
1162
|
+
* Row readers reject ranges containing a Restore, or ranges whose transaction
|
|
1163
|
+
* history cannot prove they are restore-free. Recompute the checkpoint with a
|
|
1164
|
+
* full rescan and reinitialize it at the range end version.
|
|
1165
|
+
*
|
|
1166
|
+
* LanceDB Enterprise tables reject this operation.
|
|
1167
|
+
*
|
|
1168
|
+
* @example
|
|
1169
|
+
* const delta = await table.delta("baseline");
|
|
1170
|
+
* const reader = delta.insertedRows();
|
|
1171
|
+
* console.log((await reader.schema()).fields.map((field) => field.name));
|
|
1172
|
+
* for await (const batch of reader) {
|
|
1173
|
+
* console.log(batch.getChild("id")?.toJSON());
|
|
1174
|
+
* }
|
|
1175
|
+
* console.log(await delta.listTransactions());
|
|
1176
|
+
*/
|
|
1177
|
+
delta(fromVersion: DeltaVersion, toVersion?: DeltaVersion): Promise<TableDelta>;
|
|
1178
|
+
/**
|
|
1179
|
+
* Submit durable remote cleanup through SQL VACUUM and return after acceptance.
|
|
1180
|
+
* Requires table Ownership and parent Usage. Retention inherits from the table
|
|
1181
|
+
* then deployment when omitted. Local/historical handles are unsupported.
|
|
1182
|
+
* Before submission, disable/drain automatic cleanup and prevent other cleanup,
|
|
1183
|
+
* tag/branch publication and restore until completion, including direct clients.
|
|
1184
|
+
* Coordination is not enforced by the server. Cancellation cannot undo deletion
|
|
1185
|
+
* or prove storage I/O stopped. Preview is available without deletion.
|
|
1186
|
+
* Wait timeout leaves work running; save the ID and reopen with db.openJob().
|
|
1187
|
+
* @example
|
|
1188
|
+
* const job = await table.cleanupAsync({ minAgeSeconds: 604800 });
|
|
1189
|
+
* const result = await job.wait({ timeoutMs: 600000 });
|
|
1190
|
+
*/
|
|
1191
|
+
cleanupAsync(_options?: CleanupOptions): Promise<Job<CleanupResult>>;
|
|
1192
|
+
/** Preview version deletion without creating a job. Execution re-evaluates current state. */
|
|
1193
|
+
explainCleanup(_options?: CleanupOptions): Promise<CleanupPlan>;
|
|
1194
|
+
/**
|
|
1195
|
+
* Submit remote compaction and return its durable job after acceptance.
|
|
1196
|
+
*
|
|
1197
|
+
* Uses SQL OPTIMIZE, inheriting branch and namespace from this table. Requires
|
|
1198
|
+
* a compatible remote server and table Ownership plus parent Usage. Local and
|
|
1199
|
+
* historical-version handles are unsupported. Existing optimize() behavior is unchanged.
|
|
1200
|
+
* Waiting tracks the rewrite itself and returns actual row/file statistics.
|
|
1201
|
+
* Missing result payloads are errors; older workers can omit file counts.
|
|
1202
|
+
* Status, progress, events and cancellation reuse the generic HTTP job APIs.
|
|
1203
|
+
* Reopen with db.openJob<CompactionResult>(job.id!). A wait timeout does not cancel work.
|
|
1204
|
+
*
|
|
1205
|
+
* @example
|
|
1206
|
+
* const job = await table.compactAsync({ targetRowsPerFragment: 1000 });
|
|
1207
|
+
* const result = await job.wait();
|
|
1208
|
+
* console.log(result.rowsSucceeded, result.fragmentsRemoved);
|
|
1209
|
+
*/
|
|
1210
|
+
compactAsync(_options?: CompactionOptions): Promise<Job<CompactionResult>>;
|
|
857
1211
|
/**
|
|
858
1212
|
* Optimize the on-disk data and indices for better performance.
|
|
859
1213
|
*
|
|
@@ -871,6 +1225,15 @@ export declare abstract class Table {
|
|
|
871
1225
|
* optimize should be run frequently. A good rule of thumb is to run optimize if
|
|
872
1226
|
* you have added or modified 100,000 or more records or run more than 20 data
|
|
873
1227
|
* modification operations.
|
|
1228
|
+
*
|
|
1229
|
+
* On a LanceDB Enterprise (remote) table, only compaction is currently
|
|
1230
|
+
* supported. Pruning old versions and index optimization are silently
|
|
1231
|
+
* skipped there instead of being attempted, so this call resolves
|
|
1232
|
+
* successfully, with zeroed pruning statistics, once the compaction job
|
|
1233
|
+
* has been submitted; it does not wait for that job to finish.
|
|
1234
|
+
* `compactionOptions` can tune the four per-job settings documented in
|
|
1235
|
+
* {@link CompactionOptions}. Older-server errors are propagated without
|
|
1236
|
+
* retrying with the settings removed.
|
|
874
1237
|
*/
|
|
875
1238
|
abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
|
876
1239
|
/**
|
|
@@ -969,12 +1332,16 @@ export declare class LocalTable extends Table {
|
|
|
969
1332
|
createIndexAsync(column: string, options?: Partial<IndexOptions>): Promise<Job>;
|
|
970
1333
|
dropIndex(name: string): Promise<void>;
|
|
971
1334
|
prewarmIndex(name: string): Promise<void>;
|
|
1335
|
+
prewarmData(options?: PrewarmDataOptions): Promise<void>;
|
|
972
1336
|
prewarmData(columns?: string[]): Promise<void>;
|
|
1337
|
+
prewarmDataAsync(options?: PrewarmDataOptions): Promise<Job<PrewarmDataResult>>;
|
|
1338
|
+
explainPrewarmData(options?: PrewarmDataOptions): Promise<PrewarmDataPlan>;
|
|
973
1339
|
waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
|
|
974
1340
|
takeOffsets(offsets: number[]): TakeQuery;
|
|
975
1341
|
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
|
|
976
1342
|
blobColumns(): Promise<string[]>;
|
|
977
1343
|
fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
1344
|
+
fetchBlobRanges(column: string, requests: readonly BlobRangeRequest[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
978
1345
|
fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(BlobFile | null)[]>;
|
|
979
1346
|
query(): Query;
|
|
980
1347
|
search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
|
|
@@ -982,9 +1349,12 @@ export declare class LocalTable extends Table {
|
|
|
982
1349
|
addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
|
|
983
1350
|
computed: AddColumnsSql[];
|
|
984
1351
|
}): Promise<AddColumnsResult>;
|
|
985
|
-
refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
986
|
-
refreshColumnAsync(column: string): Promise<Job>;
|
|
1352
|
+
refreshColumn(column: string, options?: RefreshColumnOptions): Promise<RefreshColumnResult>;
|
|
1353
|
+
refreshColumnAsync(column: string, options?: RefreshColumnOptions): Promise<Job>;
|
|
987
1354
|
functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
1355
|
+
functionColumnStatuses(column?: string): Promise<FunctionColumnStatus[]>;
|
|
1356
|
+
functionColumnStatus(column: string): Promise<FunctionColumnStatus>;
|
|
1357
|
+
waitFresh(column: string, timeoutSeconds?: number): Promise<FunctionColumnStatus>;
|
|
988
1358
|
refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
989
1359
|
materializedViewDefinition(): Promise<string>;
|
|
990
1360
|
alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
@@ -994,6 +1364,9 @@ export declare class LocalTable extends Table {
|
|
|
994
1364
|
enableRTE(spec: RTEWriteSpec): Promise<void>;
|
|
995
1365
|
disableRTE(): Promise<void>;
|
|
996
1366
|
getRTESpec(): Promise<RTEWriteSpec | undefined>;
|
|
1367
|
+
getTableOverrides(): Promise<Record<string, unknown>>;
|
|
1368
|
+
updateTableOverrides(overrides: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
1369
|
+
resetTableOverrides(): Promise<Record<string, unknown>>;
|
|
997
1370
|
closeRTEWriters(): Promise<void>;
|
|
998
1371
|
flushRTE(): Promise<void>;
|
|
999
1372
|
compactRTE(): Promise<void>;
|
|
@@ -1002,11 +1375,15 @@ export declare class LocalTable extends Table {
|
|
|
1002
1375
|
version(): Promise<number>;
|
|
1003
1376
|
checkout(version: number | string): Promise<void>;
|
|
1004
1377
|
checkoutLatest(): Promise<void>;
|
|
1378
|
+
delta(fromVersion: DeltaVersion, toVersion?: DeltaVersion): Promise<TableDelta>;
|
|
1005
1379
|
listVersions(): Promise<Version[]>;
|
|
1006
1380
|
restore(): Promise<void>;
|
|
1007
1381
|
tags(): Promise<Tags>;
|
|
1008
1382
|
branches(): Promise<Branches>;
|
|
1009
1383
|
currentBranch(): string | null;
|
|
1384
|
+
compactAsync(options?: CompactionOptions): Promise<Job<CompactionResult>>;
|
|
1385
|
+
cleanupAsync(options?: CleanupOptions): Promise<Job<CleanupResult>>;
|
|
1386
|
+
explainCleanup(options?: CleanupOptions): Promise<CleanupPlan>;
|
|
1010
1387
|
optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
|
1011
1388
|
listIndices(): Promise<IndexConfig[]>;
|
|
1012
1389
|
tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
|
|
@@ -1149,6 +1526,44 @@ export interface CherryPickResult {
|
|
|
1149
1526
|
preview: CherryPickPreview;
|
|
1150
1527
|
mainVersionAfter?: number;
|
|
1151
1528
|
}
|
|
1529
|
+
/**
|
|
1530
|
+
* Options for creating a branch from a named source.
|
|
1531
|
+
*/
|
|
1532
|
+
export interface CreateBranchOptions {
|
|
1533
|
+
/**
|
|
1534
|
+
* Tag to fork from. The tag keeps the branch and version it points to.
|
|
1535
|
+
* Cannot be combined with `fromRef` or `fromVersion`.
|
|
1536
|
+
*/
|
|
1537
|
+
fromTag: string;
|
|
1538
|
+
}
|
|
1539
|
+
/**
|
|
1540
|
+
* A handle for reading changes between two pinned table versions.
|
|
1541
|
+
*
|
|
1542
|
+
* Created by {@link Table.delta}. Endpoints are fixed at creation; later writes
|
|
1543
|
+
* or tag moves do not change this delta.
|
|
1544
|
+
*/
|
|
1545
|
+
export declare class TableDelta {
|
|
1546
|
+
#private;
|
|
1547
|
+
/**
|
|
1548
|
+
* Construct a table-delta handle. Internal use only.
|
|
1549
|
+
* @hidden
|
|
1550
|
+
*/
|
|
1551
|
+
constructor(inner: NativeTableDelta);
|
|
1552
|
+
/** The exclusive start version, as an exact u64 bigint. */
|
|
1553
|
+
get beginVersion(): bigint;
|
|
1554
|
+
/** The inclusive end version, as an exact u64 bigint. */
|
|
1555
|
+
get endVersion(): bigint;
|
|
1556
|
+
/** Stream rows inserted in `(beginVersion, endVersion]`. */
|
|
1557
|
+
insertedRows(): TableDeltaReader;
|
|
1558
|
+
/** Stream rows updated in `(beginVersion, endVersion]`. */
|
|
1559
|
+
updatedRows(): TableDeltaReader;
|
|
1560
|
+
/** Stream inserted and updated rows in `(beginVersion, endVersion]`. */
|
|
1561
|
+
upsertedRows(): TableDeltaReader;
|
|
1562
|
+
/** Stream deleted row IDs. Each batch has one `_rowid: uint64` column. */
|
|
1563
|
+
deletedRowIds(): TableDeltaReader;
|
|
1564
|
+
/** List readable transactions committed in `(beginVersion, endVersion]`. */
|
|
1565
|
+
listTransactions(): Promise<TableDeltaTransaction[]>;
|
|
1566
|
+
}
|
|
1152
1567
|
/**
|
|
1153
1568
|
* Branch manager for a {@link Table}.
|
|
1154
1569
|
*
|
|
@@ -1164,6 +1579,18 @@ export declare class Branches {
|
|
|
1164
1579
|
constructor(inner: NativeBranches);
|
|
1165
1580
|
/** List all branches, mapping name to branch metadata. */
|
|
1166
1581
|
list(): Promise<Record<string, BranchContents>>;
|
|
1582
|
+
/**
|
|
1583
|
+
* Create a branch from a tag and return a handle scoped to it.
|
|
1584
|
+
*
|
|
1585
|
+
* @param name Name of the new branch.
|
|
1586
|
+
* @param options Named source. `fromTag` keeps the tag's branch and version.
|
|
1587
|
+
* @example
|
|
1588
|
+
* ```typescript
|
|
1589
|
+
* const branches = await table.branches();
|
|
1590
|
+
* const branch = await branches.create("exp", { fromTag: "baseline" });
|
|
1591
|
+
* ```
|
|
1592
|
+
*/
|
|
1593
|
+
create(name: string, options: CreateBranchOptions): Promise<Table>;
|
|
1167
1594
|
/**
|
|
1168
1595
|
* Create a branch and return a handle scoped to it.
|
|
1169
1596
|
*
|