@lancedb/lancedb 0.42.0-beta.2 → 0.42.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.js +3 -1
- package/dist/blob.d.ts +11 -0
- package/dist/blob.js +27 -0
- package/dist/compaction.d.ts +23 -0
- package/dist/compaction.js +21 -0
- package/dist/connection.d.ts +2 -2
- package/dist/index.d.ts +6 -4
- package/dist/job.d.ts +38 -3
- package/dist/job.js +55 -11
- package/dist/merge.d.ts +17 -0
- package/dist/merge.js +7 -1
- package/dist/native.d.ts +220 -6
- package/dist/native.js +52 -52
- package/dist/prewarm.d.ts +19 -0
- package/dist/prewarm.js +19 -0
- package/dist/query.d.ts +12 -2
- package/dist/query.js +55 -4
- package/dist/table.d.ts +245 -11
- package/dist/table.js +85 -8
- package/package.json +9 -9
- package/pnpm-workspace.yaml +0 -7
package/dist/prewarm.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.decodePrewarmDataResult = decodePrewarmDataResult;
|
|
6
|
+
/** @internal */
|
|
7
|
+
function decodePrewarmDataResult(value) {
|
|
8
|
+
// Rust validates the typed wire result before handing it to JavaScript.
|
|
9
|
+
const result = value;
|
|
10
|
+
return {
|
|
11
|
+
tableVersion: result.table_version,
|
|
12
|
+
executorsTargeted: result.executors_targeted ?? null,
|
|
13
|
+
executorsCompleted: result.executors_completed ?? null,
|
|
14
|
+
executorsPartial: result.executors_partial ?? null,
|
|
15
|
+
fragmentsSelected: result.fragments_selected ?? null,
|
|
16
|
+
rangesCompleted: result.ranges_completed ?? null,
|
|
17
|
+
bytesRead: result.bytes_read ?? null,
|
|
18
|
+
};
|
|
19
|
+
}
|
package/dist/query.d.ts
CHANGED
|
@@ -32,6 +32,9 @@ export interface QueryExecutionOptions {
|
|
|
32
32
|
maxBatchLength?: number;
|
|
33
33
|
/**
|
|
34
34
|
* Timeout for query execution in milliseconds
|
|
35
|
+
*
|
|
36
|
+
* The timeout also covers reading the results, so consuming them slowly can
|
|
37
|
+
* exceed it.
|
|
35
38
|
*/
|
|
36
39
|
timeoutMs?: number;
|
|
37
40
|
/**
|
|
@@ -48,8 +51,10 @@ export interface QueryExecutionOptions {
|
|
|
48
51
|
* bytes. Blobs inside lists keep their descriptors. Arbitrary projection
|
|
49
52
|
* expressions are not materialized as bytes.
|
|
50
53
|
*
|
|
51
|
-
* The rows and
|
|
52
|
-
*
|
|
54
|
+
* The rows and bytes come from the same snapshot. Remote queries require
|
|
55
|
+
* authoritative query version and snapshot identity headers, verified again
|
|
56
|
+
* by the blob response. Older servers can still return `"descriptions"`.
|
|
57
|
+
* Local queries retry up to three times if the table changes while they run.
|
|
53
58
|
*
|
|
54
59
|
* @example
|
|
55
60
|
* ```ts
|
|
@@ -365,6 +370,11 @@ export declare class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
|
|
|
365
370
|
/**
|
|
366
371
|
* Set the initial IVF search effort to a finite value in [0, 1].
|
|
367
372
|
*
|
|
373
|
+
* Efforts 0.1 and 0.25 target empirical mean partition-routing recall of
|
|
374
|
+
* 80% and 90% for L2, cosine and dot through k=100000. The probe budget
|
|
375
|
+
* adapts to the metric and k. Hamming and larger k remain uncalibrated;
|
|
376
|
+
* 100000 is not an API limit. Remote queries also require server support.
|
|
377
|
+
*
|
|
368
378
|
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
|
|
369
379
|
* minimum; one searches all partitions within the caller maximum. Later
|
|
370
380
|
* probing may continue when too few candidates are found.
|
package/dist/query.js
CHANGED
|
@@ -7,6 +7,7 @@ exports.RecordBatchIterator = RecordBatchIterator;
|
|
|
7
7
|
exports.createAutoQuery = createAutoQuery;
|
|
8
8
|
exports.instanceOfFullTextQuery = instanceOfFullTextQuery;
|
|
9
9
|
const arrow_1 = require("./arrow");
|
|
10
|
+
const blob_1 = require("./blob");
|
|
10
11
|
const expr_1 = require("./expr");
|
|
11
12
|
const native_1 = require("./native");
|
|
12
13
|
async function* RecordBatchIterator(promisedInner) {
|
|
@@ -76,6 +77,30 @@ async function readBlobBytes(table, sources, snapshot, dropRowId) {
|
|
|
76
77
|
.map((field) => field.name)
|
|
77
78
|
.filter((name) => name !== "_rowid"));
|
|
78
79
|
}
|
|
80
|
+
// Discover descriptors from the executed query, not the live table's cached
|
|
81
|
+
// schema. Projection names are literal aliases; only source paths use dots.
|
|
82
|
+
function projectedBlobSources(schema, projection) {
|
|
83
|
+
const sources = [];
|
|
84
|
+
for (const field of schema.fields) {
|
|
85
|
+
const source = projection?.find(([output]) => output === field.name)?.[1] ?? field.name;
|
|
86
|
+
const visit = (child, path) => {
|
|
87
|
+
if ((0, blob_1.isBlobField)(child) || (0, blob_1.isBlobDescriptorField)(child)) {
|
|
88
|
+
sources.push({
|
|
89
|
+
output: field.name,
|
|
90
|
+
path,
|
|
91
|
+
source: [source, ...path].join("."),
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
else if (arrow_1.DataType.isStruct(child.type)) {
|
|
95
|
+
for (const nested of child.type.children) {
|
|
96
|
+
visit(nested, [...path, nested.name]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
visit(field, []);
|
|
101
|
+
}
|
|
102
|
+
return sources;
|
|
103
|
+
}
|
|
79
104
|
function withBinaryColumns(table, replacements) {
|
|
80
105
|
if (replacements.size === 0) {
|
|
81
106
|
return table;
|
|
@@ -291,6 +316,27 @@ class QueryBase {
|
|
|
291
316
|
if (context === undefined) {
|
|
292
317
|
return this.collect(options, false);
|
|
293
318
|
}
|
|
319
|
+
if (!context.table.isNative()) {
|
|
320
|
+
// Request row IDs in the same query so newly added Blob columns can be
|
|
321
|
+
// fetched without rerunning the query against a different snapshot.
|
|
322
|
+
const addRowId = !context.withRowId;
|
|
323
|
+
const table = await this.collect(options, addRowId);
|
|
324
|
+
const sources = projectedBlobSources(table.schema, context.projection);
|
|
325
|
+
if (sources.length === 0) {
|
|
326
|
+
return addRowId
|
|
327
|
+
? table.select(table.schema.fields
|
|
328
|
+
.map((field) => field.name)
|
|
329
|
+
.filter((name) => name !== "_rowid"))
|
|
330
|
+
: table;
|
|
331
|
+
}
|
|
332
|
+
const version = table.schema.metadata.get("lancedb:query_version");
|
|
333
|
+
const identity = table.schema.metadata.get("lancedb:query_snapshot");
|
|
334
|
+
if (version === undefined || identity === undefined) {
|
|
335
|
+
throw new Error('blobMode "bytes" requires a trustworthy query snapshot: missing, invalid, or inconsistent x-lancedb-query-version or x-lancedb-query-snapshot. Upgrade every serving node, or use blobMode "descriptions".');
|
|
336
|
+
}
|
|
337
|
+
const snapshot = await context.table.querySnapshotWithIdentity(version, identity);
|
|
338
|
+
return readBlobBytes(table, sources, snapshot, addRowId);
|
|
339
|
+
}
|
|
294
340
|
for (let attempt = 1;; attempt++) {
|
|
295
341
|
// The snapshot pins the bytes. The query runs on the live table, so an
|
|
296
342
|
// unchanged version afterwards proves it read the snapshot's version.
|
|
@@ -316,10 +362,10 @@ class QueryBase {
|
|
|
316
362
|
for await (const batch of RecordBatchIterator(iterator)) {
|
|
317
363
|
batches.push(batch);
|
|
318
364
|
}
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
return new arrow_1.Table(batches);
|
|
365
|
+
// Read metadata from the executed stream, including empty results and
|
|
366
|
+
// query types whose individual batches do not carry the stream metadata.
|
|
367
|
+
const schema = (0, arrow_1.tableFromIPC)((await iterator).schema()).schema;
|
|
368
|
+
return new arrow_1.Table(schema, batches);
|
|
323
369
|
}
|
|
324
370
|
blobBytesContext(mode) {
|
|
325
371
|
if (mode === undefined || mode === "descriptions") {
|
|
@@ -599,6 +645,11 @@ class VectorQuery extends StandardQueryBase {
|
|
|
599
645
|
/**
|
|
600
646
|
* Set the initial IVF search effort to a finite value in [0, 1].
|
|
601
647
|
*
|
|
648
|
+
* Efforts 0.1 and 0.25 target empirical mean partition-routing recall of
|
|
649
|
+
* 80% and 90% for L2, cosine and dot through k=100000. The probe budget
|
|
650
|
+
* adapts to the metric and k. Hamming and larger k remain uncalibrated;
|
|
651
|
+
* 100000 is not an API limit. Remote queries also require server support.
|
|
652
|
+
*
|
|
602
653
|
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
|
|
603
654
|
* minimum; one searches all partitions within the caller maximum. Later
|
|
604
655
|
* probing may continue when too few candidates are found.
|
package/dist/table.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { CompactionResult } from "./compaction";
|
|
2
|
+
import { PrewarmDataResult } from "./prewarm";
|
|
1
3
|
import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
|
|
2
|
-
import { BlobFile, BlobReadOptions } from "./blob";
|
|
4
|
+
import { BlobFile, BlobRangeRequest, BlobReadOptions } from "./blob";
|
|
3
5
|
import { IndexOptions } from "./indices";
|
|
4
6
|
import { Job } from "./job";
|
|
5
7
|
import { MergeInsertBuilder } from "./merge";
|
|
6
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
8
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, FunctionErrors, FunctionErrorsOptions, IndexConfig, IndexStatistics, JobFailureInfo, Branches as NativeBranches, OptimizeStats, RTEStats, RefreshColumnOptions, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
7
9
|
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
8
10
|
import { IntoSql } from "./util";
|
|
9
11
|
export { IndexConfig } from "./native";
|
|
@@ -75,6 +77,12 @@ export interface AddDataOptions {
|
|
|
75
77
|
* when the blob is read later.
|
|
76
78
|
*/
|
|
77
79
|
allowExternalBlobOutsideBases?: boolean;
|
|
80
|
+
/**
|
|
81
|
+
* Override the remote retry budget for this call (0 to 255). Zero disables
|
|
82
|
+
* retries, including multipart parts. Ambiguous commits are never replayed.
|
|
83
|
+
* Ignored by LanceDB Embedded.
|
|
84
|
+
*/
|
|
85
|
+
retries?: number;
|
|
78
86
|
}
|
|
79
87
|
export interface UpdateOptions {
|
|
80
88
|
/**
|
|
@@ -89,7 +97,39 @@ export interface UpdateOptions {
|
|
|
89
97
|
*/
|
|
90
98
|
where: string;
|
|
91
99
|
}
|
|
100
|
+
/** Settings for submitting a durable remote data-cache prewarm job. */
|
|
101
|
+
export interface PrewarmDataOptions {
|
|
102
|
+
/** Lance field paths to warm; omitted or empty means every column. */
|
|
103
|
+
columns?: string[];
|
|
104
|
+
/** A single SQL predicate for conservative metadata pruning. */
|
|
105
|
+
filter?: string;
|
|
106
|
+
/** Positive per-executor concurrency, at most 4294967295. */
|
|
107
|
+
concurrency?: number;
|
|
108
|
+
}
|
|
109
|
+
/** Per-invocation compaction settings. Values must be safe integers. */
|
|
110
|
+
export interface CompactionOptions {
|
|
111
|
+
/** Positive target rows per output fragment; also affects candidate selection. */
|
|
112
|
+
targetRowsPerFragment?: number;
|
|
113
|
+
/** Positive live-source-row budget across whole tasks; an oversized first task can produce no work. */
|
|
114
|
+
maxSourceRows?: number;
|
|
115
|
+
/** Scanner I/O read-ahead budget in bytes; zero disables byte-based I/O backpressure. */
|
|
116
|
+
ioBufferSize?: number;
|
|
117
|
+
/** Positive scanner batch size in rows. */
|
|
118
|
+
batchSize?: number;
|
|
119
|
+
}
|
|
92
120
|
export interface OptimizeOptions {
|
|
121
|
+
/**
|
|
122
|
+
* Compaction settings for this invocation. Omitted remote values inherit table
|
|
123
|
+
* tuning, then deployment/built-in defaults.
|
|
124
|
+
* Remote tables require a server supporting these SQL options; settings do
|
|
125
|
+
* not change table policy. This is not a runtime or total-memory limit.
|
|
126
|
+
* @example
|
|
127
|
+
* await tbl.optimize({ compactionOptions: {
|
|
128
|
+
* targetRowsPerFragment: 1000, maxSourceRows: 5000,
|
|
129
|
+
* ioBufferSize: 64 * 1024 * 1024, batchSize: 128,
|
|
130
|
+
* } });
|
|
131
|
+
*/
|
|
132
|
+
compactionOptions?: CompactionOptions;
|
|
93
133
|
/**
|
|
94
134
|
* If set then all versions older than the given date
|
|
95
135
|
* be removed. The current version will never be removed.
|
|
@@ -164,6 +204,55 @@ export interface RTEWriteSpec {
|
|
|
164
204
|
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
|
|
165
205
|
writerConfigDefaults?: Record<string, string>;
|
|
166
206
|
}
|
|
207
|
+
/** Exact immutable Function revision attached to a table output. */
|
|
208
|
+
export interface FunctionColumnVersionStatus {
|
|
209
|
+
name: string;
|
|
210
|
+
namespacePath: string[];
|
|
211
|
+
objectId: string;
|
|
212
|
+
location: string;
|
|
213
|
+
version: string;
|
|
214
|
+
manifestDigest: string;
|
|
215
|
+
}
|
|
216
|
+
/** Latest matching refresh Job projected for a Function output. */
|
|
217
|
+
export interface FunctionColumnLastRefreshJob {
|
|
218
|
+
jobId: string;
|
|
219
|
+
state: string;
|
|
220
|
+
automatic?: boolean;
|
|
221
|
+
completedAtMillis?: bigint;
|
|
222
|
+
/** Structured failure details for a failed Job. */
|
|
223
|
+
failure?: JobFailureInfo;
|
|
224
|
+
}
|
|
225
|
+
/** Automatic-refresh projection for a Function output. */
|
|
226
|
+
export interface FunctionColumnAutoRefreshStatus {
|
|
227
|
+
enabled: boolean;
|
|
228
|
+
intervalSeconds?: bigint;
|
|
229
|
+
activeJobId?: string;
|
|
230
|
+
state: "disabled" | "running" | "queued" | "ready" | "up_to_date" | "waiting_interval" | "waiting_backoff" | "blocked" | "unobservable";
|
|
231
|
+
failureStreak: number;
|
|
232
|
+
effectiveCooldownMillis?: bigint;
|
|
233
|
+
blockedReason?: "table_has_mem_wal" | "inputs_not_settled" | "dependency_in_flight" | "table_capacity" | "cancelled" | "preparation_backoff" | "failure_threshold_reached" | "binding_revision_missing";
|
|
234
|
+
nextRunAtMillis: bigint | null;
|
|
235
|
+
nextRunAtReason?: string;
|
|
236
|
+
}
|
|
237
|
+
/** Exact four-way status of one Function output at one table version. */
|
|
238
|
+
export interface FunctionColumnStatus {
|
|
239
|
+
column: string;
|
|
240
|
+
bindingId: string;
|
|
241
|
+
function: FunctionColumnVersionStatus;
|
|
242
|
+
tableVersion: bigint;
|
|
243
|
+
rows: bigint;
|
|
244
|
+
fresh: bigint;
|
|
245
|
+
stale: bigint;
|
|
246
|
+
neverComputed: bigint;
|
|
247
|
+
failed: bigint;
|
|
248
|
+
lastRefreshJob?: FunctionColumnLastRefreshJob;
|
|
249
|
+
/**
|
|
250
|
+
* Newest failed refresh Job. This remains present when `lastRefreshJob`
|
|
251
|
+
* is a later cancellation.
|
|
252
|
+
*/
|
|
253
|
+
lastFailureJob?: FunctionColumnLastRefreshJob;
|
|
254
|
+
autoRefresh: FunctionColumnAutoRefreshStatus;
|
|
255
|
+
}
|
|
167
256
|
/**
|
|
168
257
|
* A Table is a collection of Records in a LanceDB Database.
|
|
169
258
|
*
|
|
@@ -360,21 +449,53 @@ export declare abstract class Table {
|
|
|
360
449
|
/**
|
|
361
450
|
* Prewarm one or more columns of data in the table.
|
|
362
451
|
*
|
|
363
|
-
* @param columns The columns to prewarm. If undefined, all columns are prewarmed.
|
|
364
|
-
*
|
|
365
452
|
* This will load the column data into the page cache so that future queries that
|
|
366
453
|
* read those columns avoid the initial cold-start latency. This call initiates
|
|
367
454
|
* prewarming and returns once the request is accepted; the warming itself may
|
|
368
|
-
* continue in the background.
|
|
369
|
-
*
|
|
455
|
+
* continue in the background. Submission uses SQL PREWARM and needs Flight
|
|
456
|
+
* SQL connectivity. Use {@link prewarmDataAsync} to retain the job handle.
|
|
370
457
|
*
|
|
371
458
|
* Prewarming is generally useful for columns used in filters or projections.
|
|
372
459
|
* Large columns (e.g. high-dimensional vectors or binary data) may not be
|
|
373
460
|
* practical to prewarm.
|
|
374
461
|
*
|
|
375
462
|
* This feature is currently only supported in LanceDB Enterprise.
|
|
463
|
+
*
|
|
464
|
+
* @param options Columns, a metadata-pruning filter, and per-executor concurrency.
|
|
465
|
+
* Omitted or empty columns warm every column.
|
|
466
|
+
*
|
|
467
|
+
* @example
|
|
468
|
+
* await table.prewarmData({
|
|
469
|
+
* columns: ["payload"], filter: "quality > 0.7", concurrency: 4,
|
|
470
|
+
* });
|
|
471
|
+
*/
|
|
472
|
+
abstract prewarmData(options?: PrewarmDataOptions): Promise<void>;
|
|
473
|
+
/**
|
|
474
|
+
* Submit data prewarm using the columns-array form and return after acceptance.
|
|
475
|
+
* @param columns Columns to warm; omitted or empty means every column.
|
|
376
476
|
*/
|
|
377
477
|
abstract prewarmData(columns?: string[]): Promise<void>;
|
|
478
|
+
/**
|
|
479
|
+
* Submit data prewarm through SQL and return its durable warming-job handle.
|
|
480
|
+
*
|
|
481
|
+
* The table handle supplies the branch and namespace. Requires a compatible
|
|
482
|
+
* SQL PREWARM server and Flight SQL connectivity. Standard authorization
|
|
483
|
+
* requires table Select and database/namespace Usage. Status/wait use generic
|
|
484
|
+
* HTTP job APIs; events and cancellation retain their own privileges.
|
|
485
|
+
* No REST fallback or option-dropping retry is performed. Historical-version
|
|
486
|
+
* handles are unsupported; call `checkoutLatest()` first.
|
|
487
|
+
*
|
|
488
|
+
* Submission returns after acceptance. Use `job.wait()` for completion, or
|
|
489
|
+
* save `job.id` and reopen it with `db.openJob(id)`. A timeout does not cancel
|
|
490
|
+
* the job. After `job.refresh()`, `job.spec.table_version` is the pinned version.
|
|
491
|
+
*
|
|
492
|
+
* @example
|
|
493
|
+
* const job = await table.prewarmDataAsync({
|
|
494
|
+
* columns: ["video"], filter: "quality > 0.7", concurrency: 4,
|
|
495
|
+
* });
|
|
496
|
+
* await job.wait();
|
|
497
|
+
*/
|
|
498
|
+
abstract prewarmDataAsync(options?: PrewarmDataOptions): Promise<Job<PrewarmDataResult>>;
|
|
378
499
|
/**
|
|
379
500
|
* Waits for asynchronous indexing to complete on the table.
|
|
380
501
|
*
|
|
@@ -472,6 +593,26 @@ export declare abstract class Table {
|
|
|
472
593
|
* memory; use {@link Table.fetchBlobFiles} for large values.
|
|
473
594
|
*/
|
|
474
595
|
abstract fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
596
|
+
/**
|
|
597
|
+
* Materializes row-specific byte ranges at the table's current checkout.
|
|
598
|
+
* Preserves request order, duplicates, and nulls. Zero length returns an empty
|
|
599
|
+
* buffer for a non-null blob. Invalid ranges or missing rows reject the entire
|
|
600
|
+
* call. See {@link Table.fetchBlobs} for row-ID validity across versions.
|
|
601
|
+
*
|
|
602
|
+
* Remote tables use the batched endpoint when the server advertises support,
|
|
603
|
+
* splitting at 1024 ranges and 64 MiB, including individual larger ranges.
|
|
604
|
+
* Older servers use descriptor queries and BlobFile range reads. Both protocols
|
|
605
|
+
* use one exact branch and version. Errors from an advertised endpoint propagate
|
|
606
|
+
* without switching protocols. All output bytes occupy memory.
|
|
607
|
+
*
|
|
608
|
+
* @example
|
|
609
|
+
* ```ts
|
|
610
|
+
* const bytes = await table.fetchBlobRanges("image", [
|
|
611
|
+
* { rowId: 10n, offset: 0n, length: 1024n },
|
|
612
|
+
* ]);
|
|
613
|
+
* ```
|
|
614
|
+
*/
|
|
615
|
+
abstract fetchBlobRanges(column: string, requests: readonly BlobRangeRequest[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
475
616
|
/**
|
|
476
617
|
* Opens lazy blob handles for `column` at the given row IDs using the
|
|
477
618
|
* table's current checkout.
|
|
@@ -548,10 +689,12 @@ export declare abstract class Table {
|
|
|
548
689
|
* everything else is left as it is. LanceDB Embedded only: in LanceDB
|
|
549
690
|
* Enterprise a refresh runs as a server job, through {@link Table#refreshColumnAsync}.
|
|
550
691
|
* @param {string} column The name of the computed column to fill.
|
|
692
|
+
* @param {RefreshColumnOptions} options Refresh options, all LanceDB
|
|
693
|
+
* Enterprise only; see {@link Table#refreshColumnAsync}.
|
|
551
694
|
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
|
|
552
695
|
* number of rows filled and the new version number of the table.
|
|
553
696
|
*/
|
|
554
|
-
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
697
|
+
abstract refreshColumn(column: string, options?: RefreshColumnOptions): Promise<RefreshColumnResult>;
|
|
555
698
|
/**
|
|
556
699
|
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
557
700
|
* job instead of blocking until it completes.
|
|
@@ -565,14 +708,54 @@ export declare abstract class Table {
|
|
|
565
708
|
* than failing the job. In LanceDB Embedded the job runs in-process; in
|
|
566
709
|
* LanceDB Enterprise it is the server's backfill job.
|
|
567
710
|
* @param {string} column The name of the computed column to fill.
|
|
711
|
+
* @param {RefreshColumnOptions} options Refresh options, mirroring the SQL
|
|
712
|
+
* `REFRESH COLUMN ... WHERE ... WITH (...)` form: `where` narrows the rows,
|
|
713
|
+
* `errorPolicy`, `maxFailedFraction`, `maxFailedRows`, `rowTimeoutSeconds`
|
|
714
|
+
* and `initTimeoutSeconds` govern failing rows. `batchSize`, `gpuClass`,
|
|
715
|
+
* `maxConcurrency`, `resourceBudget`, `supervisorResources`,
|
|
716
|
+
* `planningWorkers` and `maxExecutorProcessesPerWorker` size registered
|
|
717
|
+
* Function refreshes. `retryPolicy` overrides user-code retries only when
|
|
718
|
+
* repeating partial work is safe; set `maxAttempts: 1` to disable retries.
|
|
719
|
+
* `maxConcurrency` overrides the registered default for this logical
|
|
720
|
+
* execution only and is a positive integer; it is not an account-wide
|
|
721
|
+
* quota. A `resourceBudget` requires `cpuMillis` and
|
|
722
|
+
* `memoryBytes`; its `gpuCount` defaults to zero for a CPU-only Function.
|
|
723
|
+
* `supervisorResources` and each `planningWorkers.resources` entry require
|
|
724
|
+
* positive `cpuMillis`, `memoryBytes` and `ephemeralStorageBytes`.
|
|
725
|
+
* `planningWorkers` also requires positive `maxWorkers`,
|
|
726
|
+
* `rowsPerWorker` and `fragmentsPerWorker`.
|
|
727
|
+
* The budget applies only to this explicit refresh and does not configure
|
|
728
|
+
* a later automatic refresh.
|
|
729
|
+
* `gpuCount` at the top level is the shorthand that preserves the
|
|
730
|
+
* deployment defaults for CPU and memory, and cannot be combined with
|
|
731
|
+
* `resourceBudget`. `numWorkers` applies only to expression columns; use
|
|
732
|
+
* `resourceBudget` for registered Functions. All are LanceDB Enterprise
|
|
733
|
+
* only; a LanceDB Embedded refresh rejects every option other than
|
|
734
|
+
* `batchSize`, `gpuClass` and `gpuCount`.
|
|
735
|
+
* `publicationIntervalRows` opts into publishing completed Function split
|
|
736
|
+
* ranges once their accumulated unpublished rows reach the threshold.
|
|
737
|
+
* It must be a positive safe integer and requires non-Blob Function outputs.
|
|
738
|
+
* Omit it for whole-fragment publication. Publications can exceed the
|
|
739
|
+
* threshold, and remaining rows publish at completion.
|
|
568
740
|
* @example
|
|
569
741
|
* ```ts
|
|
570
|
-
* const job = await table.refreshColumnAsync("
|
|
742
|
+
* const job = await table.refreshColumnAsync("embedding", {
|
|
743
|
+
* where: "published_at > '2026-01-01'",
|
|
744
|
+
* resourceBudget: {
|
|
745
|
+
* cpuMillis: 8000,
|
|
746
|
+
* memoryBytes: 16 * 1024 ** 3,
|
|
747
|
+
* },
|
|
748
|
+
* maxConcurrency: 4,
|
|
749
|
+
* publicationIntervalRows: 10000,
|
|
750
|
+
* errorPolicy: "skip",
|
|
751
|
+
* maxFailedFraction: 0.01,
|
|
752
|
+
* rowTimeoutSeconds: 30,
|
|
753
|
+
* });
|
|
571
754
|
* await job.wait();
|
|
572
755
|
* console.log(await job.status()); // "finished"
|
|
573
756
|
* ```
|
|
574
757
|
*/
|
|
575
|
-
abstract refreshColumnAsync(column: string): Promise<Job>;
|
|
758
|
+
abstract refreshColumnAsync(column: string, options?: RefreshColumnOptions): Promise<Job>;
|
|
576
759
|
/**
|
|
577
760
|
* The per-row errors Function refreshes recorded on this table.
|
|
578
761
|
*
|
|
@@ -591,6 +774,24 @@ export declare abstract class Table {
|
|
|
591
774
|
* ```
|
|
592
775
|
*/
|
|
593
776
|
abstract functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
777
|
+
/**
|
|
778
|
+
* Return exact freshness status for Function-backed output columns.
|
|
779
|
+
*
|
|
780
|
+
* All results describe one server table version. Omit `column` for every
|
|
781
|
+
* Function output. Rows already failed remain visible and do not prevent
|
|
782
|
+
* `waitFresh` from completing. A handle pinned to an exact version requests
|
|
783
|
+
* that version; a latest handle retains normal read-freshness behavior.
|
|
784
|
+
*/
|
|
785
|
+
abstract functionColumnStatuses(column?: string): Promise<FunctionColumnStatus[]>;
|
|
786
|
+
/** Return exact freshness status for one Function-backed output column. */
|
|
787
|
+
abstract functionColumnStatus(column: string): Promise<FunctionColumnStatus>;
|
|
788
|
+
/**
|
|
789
|
+
* Poll until no stale or never-computed rows remain.
|
|
790
|
+
*
|
|
791
|
+
* Only HTTP 503 with the exact finalization-pending error code is retried;
|
|
792
|
+
* every other error propagates. Failed rows do not block completion.
|
|
793
|
+
*/
|
|
794
|
+
abstract waitFresh(column: string, timeoutSeconds?: number): Promise<FunctionColumnStatus>;
|
|
594
795
|
/**
|
|
595
796
|
* Recompute this table's contents from its materialized-view definition.
|
|
596
797
|
*
|
|
@@ -854,6 +1055,23 @@ export declare abstract class Table {
|
|
|
854
1055
|
* out state and the read_consistency_interval, if any, will apply.
|
|
855
1056
|
*/
|
|
856
1057
|
abstract restore(): Promise<void>;
|
|
1058
|
+
/**
|
|
1059
|
+
* Submit remote compaction and return its durable job after acceptance.
|
|
1060
|
+
*
|
|
1061
|
+
* Uses SQL OPTIMIZE, inheriting branch and namespace from this table. Requires
|
|
1062
|
+
* a compatible remote server and table Ownership plus parent Usage. Local and
|
|
1063
|
+
* historical-version handles are unsupported. Existing optimize() behavior is unchanged.
|
|
1064
|
+
* Waiting tracks the rewrite itself and returns actual row/file statistics.
|
|
1065
|
+
* Missing result payloads are errors; older workers can omit file counts.
|
|
1066
|
+
* Status, progress, events and cancellation reuse the generic HTTP job APIs.
|
|
1067
|
+
* Reopen with db.openJob<CompactionResult>(job.id!). A wait timeout does not cancel work.
|
|
1068
|
+
*
|
|
1069
|
+
* @example
|
|
1070
|
+
* const job = await table.compactAsync({ targetRowsPerFragment: 1000 });
|
|
1071
|
+
* const result = await job.wait();
|
|
1072
|
+
* console.log(result.rowsSucceeded, result.fragmentsRemoved);
|
|
1073
|
+
*/
|
|
1074
|
+
compactAsync(_options?: CompactionOptions): Promise<Job<CompactionResult>>;
|
|
857
1075
|
/**
|
|
858
1076
|
* Optimize the on-disk data and indices for better performance.
|
|
859
1077
|
*
|
|
@@ -871,6 +1089,15 @@ export declare abstract class Table {
|
|
|
871
1089
|
* optimize should be run frequently. A good rule of thumb is to run optimize if
|
|
872
1090
|
* you have added or modified 100,000 or more records or run more than 20 data
|
|
873
1091
|
* modification operations.
|
|
1092
|
+
*
|
|
1093
|
+
* On a LanceDB Enterprise (remote) table, only compaction is currently
|
|
1094
|
+
* supported. Pruning old versions and index optimization are silently
|
|
1095
|
+
* skipped there instead of being attempted, so this call resolves
|
|
1096
|
+
* successfully, with zeroed pruning statistics, once the compaction job
|
|
1097
|
+
* has been submitted; it does not wait for that job to finish.
|
|
1098
|
+
* `compactionOptions` can tune the four per-job settings documented in
|
|
1099
|
+
* {@link CompactionOptions}. Older-server errors are propagated without
|
|
1100
|
+
* retrying with the settings removed.
|
|
874
1101
|
*/
|
|
875
1102
|
abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
|
876
1103
|
/**
|
|
@@ -969,12 +1196,15 @@ export declare class LocalTable extends Table {
|
|
|
969
1196
|
createIndexAsync(column: string, options?: Partial<IndexOptions>): Promise<Job>;
|
|
970
1197
|
dropIndex(name: string): Promise<void>;
|
|
971
1198
|
prewarmIndex(name: string): Promise<void>;
|
|
1199
|
+
prewarmData(options?: PrewarmDataOptions): Promise<void>;
|
|
972
1200
|
prewarmData(columns?: string[]): Promise<void>;
|
|
1201
|
+
prewarmDataAsync(options?: PrewarmDataOptions): Promise<Job<PrewarmDataResult>>;
|
|
973
1202
|
waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
|
|
974
1203
|
takeOffsets(offsets: number[]): TakeQuery;
|
|
975
1204
|
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
|
|
976
1205
|
blobColumns(): Promise<string[]>;
|
|
977
1206
|
fetchBlobs(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
1207
|
+
fetchBlobRanges(column: string, requests: readonly BlobRangeRequest[], options?: BlobReadOptions): Promise<(Buffer | null)[]>;
|
|
978
1208
|
fetchBlobFiles(column: string, rowIds: readonly (bigint | number)[], options?: BlobReadOptions): Promise<(BlobFile | null)[]>;
|
|
979
1209
|
query(): Query;
|
|
980
1210
|
search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
|
|
@@ -982,9 +1212,12 @@ export declare class LocalTable extends Table {
|
|
|
982
1212
|
addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
|
|
983
1213
|
computed: AddColumnsSql[];
|
|
984
1214
|
}): Promise<AddColumnsResult>;
|
|
985
|
-
refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
986
|
-
refreshColumnAsync(column: string): Promise<Job>;
|
|
1215
|
+
refreshColumn(column: string, options?: RefreshColumnOptions): Promise<RefreshColumnResult>;
|
|
1216
|
+
refreshColumnAsync(column: string, options?: RefreshColumnOptions): Promise<Job>;
|
|
987
1217
|
functionErrors(options?: FunctionErrorsOptions): Promise<FunctionErrors>;
|
|
1218
|
+
functionColumnStatuses(column?: string): Promise<FunctionColumnStatus[]>;
|
|
1219
|
+
functionColumnStatus(column: string): Promise<FunctionColumnStatus>;
|
|
1220
|
+
waitFresh(column: string, timeoutSeconds?: number): Promise<FunctionColumnStatus>;
|
|
988
1221
|
refreshMaterializedView(sourceVersion?: number, full?: boolean): Promise<RefreshMaterializedViewResult>;
|
|
989
1222
|
materializedViewDefinition(): Promise<string>;
|
|
990
1223
|
alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
@@ -1007,6 +1240,7 @@ export declare class LocalTable extends Table {
|
|
|
1007
1240
|
tags(): Promise<Tags>;
|
|
1008
1241
|
branches(): Promise<Branches>;
|
|
1009
1242
|
currentBranch(): string | null;
|
|
1243
|
+
compactAsync(options?: CompactionOptions): Promise<Job<CompactionResult>>;
|
|
1010
1244
|
optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
|
|
1011
1245
|
listIndices(): Promise<IndexConfig[]>;
|
|
1012
1246
|
tokenize(query: string, options: TokenizeTableOptions): Promise<FtsToken[]>;
|