@lancedb/lancedb 0.40.0 → 0.41.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NODEJS_THIRD_PARTY_LICENSES.md +8 -0
- package/dist/arrow.d.ts +18 -1
- package/dist/arrow.js +194 -14
- package/dist/blob.d.ts +81 -2
- package/dist/blob.js +159 -6
- package/dist/catalog.d.ts +40 -6
- package/dist/catalog.js +58 -9
- package/dist/connection.d.ts +19 -13
- package/dist/connection.js +12 -7
- package/dist/index.d.ts +6 -5
- package/dist/index.js +4 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/materialized_view.d.ts +5 -12
- package/dist/materialized_view.js +6 -12
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +151 -114
- package/dist/native.js +55 -52
- package/dist/query.d.ts +94 -26
- package/dist/query.js +176 -52
- package/dist/sanitize.d.ts +19 -18
- package/dist/sanitize.js +74 -11
- package/dist/schema.js +31 -3
- package/dist/table.d.ts +65 -60
- package/dist/table.js +26 -20
- package/package.json +13 -9
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/** Pagination controls for lazy resource listings. */
|
|
2
|
+
export interface ListingOptions {
|
|
3
|
+
/** Starting continuation token; omitted starts at the beginning. */
|
|
4
|
+
pageToken?: string;
|
|
5
|
+
/** Maximum results per request, not a limit on the whole listing. */
|
|
6
|
+
pageLimit?: number;
|
|
7
|
+
}
|
|
8
|
+
/** @internal */
|
|
9
|
+
export declare function validateListingOptions(options: ListingOptions): void;
|
|
10
|
+
/**
|
|
11
|
+
* A lazy async iterator over database resources with pagination state.
|
|
12
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
17
|
+
* console.log(name);
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare class Listing<T> implements AsyncIterableIterator<T> {
|
|
22
|
+
private readonly inner;
|
|
23
|
+
/** @hidden */
|
|
24
|
+
constructor(inner: {
|
|
25
|
+
next(): Promise<T | null | undefined>;
|
|
26
|
+
numPageResults(): number;
|
|
27
|
+
pageToken(): string | null | undefined;
|
|
28
|
+
});
|
|
29
|
+
[Symbol.asyncIterator](): AsyncIterableIterator<T>;
|
|
30
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
31
|
+
next(): Promise<IteratorResult<T>>;
|
|
32
|
+
/** Number of cached items available without another request. */
|
|
33
|
+
numPageResults(): number;
|
|
34
|
+
/**
|
|
35
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
36
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
37
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
38
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
39
|
+
*/
|
|
40
|
+
pageToken(): string | undefined;
|
|
41
|
+
}
|
package/dist/listing.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.Listing = void 0;
|
|
6
|
+
exports.validateListingOptions = validateListingOptions;
|
|
7
|
+
/** @internal */
|
|
8
|
+
function validateListingOptions(options) {
|
|
9
|
+
if (options.pageLimit !== undefined &&
|
|
10
|
+
(!Number.isInteger(options.pageLimit) ||
|
|
11
|
+
options.pageLimit <= 0 ||
|
|
12
|
+
options.pageLimit > 2147483647)) {
|
|
13
|
+
throw new Error("Listing page limit must be an integer between 1 and 2147483647");
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A lazy async iterator over database resources with pagination state.
|
|
18
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```ts
|
|
22
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
23
|
+
* console.log(name);
|
|
24
|
+
* }
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
class Listing {
|
|
28
|
+
inner;
|
|
29
|
+
/** @hidden */
|
|
30
|
+
constructor(inner) {
|
|
31
|
+
this.inner = inner;
|
|
32
|
+
}
|
|
33
|
+
[Symbol.asyncIterator]() {
|
|
34
|
+
return this;
|
|
35
|
+
}
|
|
36
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
37
|
+
async next() {
|
|
38
|
+
const value = await this.inner.next();
|
|
39
|
+
return value == null
|
|
40
|
+
? { done: true, value: undefined }
|
|
41
|
+
: { done: false, value };
|
|
42
|
+
}
|
|
43
|
+
/** Number of cached items available without another request. */
|
|
44
|
+
numPageResults() {
|
|
45
|
+
return this.inner.numPageResults();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
49
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
50
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
51
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
52
|
+
*/
|
|
53
|
+
pageToken() {
|
|
54
|
+
return this.inner.pageToken() ?? undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.Listing = Listing;
|
|
@@ -4,10 +4,9 @@ import { Table } from "./table";
|
|
|
4
4
|
export declare const DEFINITION_META_KEY = "mv.definition";
|
|
5
5
|
/**
|
|
6
6
|
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
|
|
7
|
-
* format 2 being a query with `GROUP BY
|
|
8
|
-
* native dedup results.
|
|
7
|
+
* format 2 being a query with `GROUP BY`.
|
|
9
8
|
*/
|
|
10
|
-
export declare const DEFINITION_FORMAT =
|
|
9
|
+
export declare const DEFINITION_FORMAT = 2;
|
|
11
10
|
/**
|
|
12
11
|
* The query that defines a materialized view, as stored:
|
|
13
12
|
* `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
|
|
@@ -58,17 +57,11 @@ export declare class MaterializedView {
|
|
|
58
57
|
/**
|
|
59
58
|
* Recompute the view from its source.
|
|
60
59
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* refreshes to that source version instead of the latest.
|
|
65
|
-
*
|
|
66
|
-
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
|
67
|
-
* plan the same source rows conflict on commit, and the loser throws
|
|
68
|
-
* rather than writing them a second time.
|
|
60
|
+
* Every refresh rebuilds the complete view. For local views,
|
|
61
|
+
* `sourceVersion` refreshes to that source version instead of the latest;
|
|
62
|
+
* remote SQL refreshes do not support source-version pinning.
|
|
69
63
|
*/
|
|
70
64
|
refresh(options?: {
|
|
71
|
-
full?: boolean;
|
|
72
65
|
sourceVersion?: number;
|
|
73
66
|
}): Promise<RefreshMaterializedViewResult>;
|
|
74
67
|
}
|
|
@@ -11,10 +11,9 @@ exports.definitionFromJson = definitionFromJson;
|
|
|
11
11
|
exports.DEFINITION_META_KEY = "mv.definition";
|
|
12
12
|
/**
|
|
13
13
|
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
|
|
14
|
-
* format 2 being a query with `GROUP BY
|
|
15
|
-
* native dedup results.
|
|
14
|
+
* format 2 being a query with `GROUP BY`.
|
|
16
15
|
*/
|
|
17
|
-
exports.DEFINITION_FORMAT =
|
|
16
|
+
exports.DEFINITION_FORMAT = 2;
|
|
18
17
|
/**
|
|
19
18
|
* @internal Reject a numeric option N-API would otherwise silently coerce:
|
|
20
19
|
* `Infinity` reaches Rust as 0, `1.5` as 1.
|
|
@@ -133,18 +132,13 @@ class MaterializedView {
|
|
|
133
132
|
/**
|
|
134
133
|
* Recompute the view from its source.
|
|
135
134
|
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
* refreshes to that source version instead of the latest.
|
|
140
|
-
*
|
|
141
|
-
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
|
142
|
-
* plan the same source rows conflict on commit, and the loser throws
|
|
143
|
-
* rather than writing them a second time.
|
|
135
|
+
* Every refresh rebuilds the complete view. For local views,
|
|
136
|
+
* `sourceVersion` refreshes to that source version instead of the latest;
|
|
137
|
+
* remote SQL refreshes do not support source-version pinning.
|
|
144
138
|
*/
|
|
145
139
|
async refresh(options) {
|
|
146
140
|
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
|
|
147
|
-
return await this.inner.refreshMaterializedView(options?.
|
|
141
|
+
return await this.inner.refreshMaterializedView(options?.sourceVersion);
|
|
148
142
|
}
|
|
149
143
|
}
|
|
150
144
|
exports.MaterializedView = MaterializedView;
|
package/dist/merge.d.ts
CHANGED
|
@@ -57,25 +57,25 @@ export declare class MergeInsertBuilder {
|
|
|
57
57
|
*/
|
|
58
58
|
useIndex(useIndex: boolean): MergeInsertBuilder;
|
|
59
59
|
/**
|
|
60
|
-
* Control
|
|
60
|
+
* Control RTE routing for this merge.
|
|
61
61
|
*
|
|
62
|
-
* By default (unset), a `mergeInsert` on a table with an
|
|
62
|
+
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
|
|
63
63
|
* routed through Lance's MemWAL shard writer, and a table without one uses the
|
|
64
64
|
* standard path.
|
|
65
65
|
*
|
|
66
|
-
* @param enable - `true` forces
|
|
67
|
-
*
|
|
66
|
+
* @param enable - `true` forces RTE routing and errors if the table has no
|
|
67
|
+
* RTE write spec. `false` forces the standard write path even when a spec is set.
|
|
68
68
|
*/
|
|
69
|
-
|
|
69
|
+
useRTE(enable: boolean): MergeInsertBuilder;
|
|
70
70
|
/**
|
|
71
71
|
* Controls how an LSM merge checks that its input targets a single shard.
|
|
72
72
|
*
|
|
73
|
-
* When a table has an
|
|
73
|
+
* When a table has an RTE write spec, every row in a `mergeInsert` call must
|
|
74
74
|
* route to the same shard. When `true` (the default), every row is inspected
|
|
75
75
|
* to verify this. When `false`, only the first row is inspected and the
|
|
76
76
|
* shard it routes to is used for the whole input — a faster path for callers
|
|
77
77
|
* that have already pre-sharded their input. Has no effect on tables without
|
|
78
|
-
* an
|
|
78
|
+
* an RTE write spec.
|
|
79
79
|
*
|
|
80
80
|
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
|
|
81
81
|
*/
|
package/dist/merge.js
CHANGED
|
@@ -69,27 +69,27 @@ class MergeInsertBuilder {
|
|
|
69
69
|
return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
|
-
* Control
|
|
72
|
+
* Control RTE routing for this merge.
|
|
73
73
|
*
|
|
74
|
-
* By default (unset), a `mergeInsert` on a table with an
|
|
74
|
+
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
|
|
75
75
|
* routed through Lance's MemWAL shard writer, and a table without one uses the
|
|
76
76
|
* standard path.
|
|
77
77
|
*
|
|
78
|
-
* @param enable - `true` forces
|
|
79
|
-
*
|
|
78
|
+
* @param enable - `true` forces RTE routing and errors if the table has no
|
|
79
|
+
* RTE write spec. `false` forces the standard write path even when a spec is set.
|
|
80
80
|
*/
|
|
81
|
-
|
|
82
|
-
return new MergeInsertBuilder(this.#native.
|
|
81
|
+
useRTE(enable) {
|
|
82
|
+
return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
85
|
* Controls how an LSM merge checks that its input targets a single shard.
|
|
86
86
|
*
|
|
87
|
-
* When a table has an
|
|
87
|
+
* When a table has an RTE write spec, every row in a `mergeInsert` call must
|
|
88
88
|
* route to the same shard. When `true` (the default), every row is inspected
|
|
89
89
|
* to verify this. When `false`, only the first row is inspected and the
|
|
90
90
|
* shard it routes to is used for the whole input — a faster path for callers
|
|
91
91
|
* that have already pre-sharded their input. Has no effect on tables without
|
|
92
|
-
* an
|
|
92
|
+
* an RTE write spec.
|
|
93
93
|
*
|
|
94
94
|
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
|
|
95
95
|
*/
|
package/dist/native.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export declare class Catalog {
|
|
|
32
32
|
createDatabase(name: string, existOk?: boolean | undefined | null): Promise<Connection>
|
|
33
33
|
connectDatabase(name: string): Promise<Connection>
|
|
34
34
|
dropDatabase(name: string, ignoreMissing?: boolean | undefined | null): Promise<void>
|
|
35
|
-
listDatabases(
|
|
35
|
+
listDatabases(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): DatabaseNames
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export declare class Connection {
|
|
@@ -58,7 +58,7 @@ export declare class Connection {
|
|
|
58
58
|
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
59
59
|
createMaterializedView(name: string, source: string, projections: Array<Array<string>> | undefined | null, filter: string | undefined | null, limit: number | undefined | null, withNoData: boolean): Promise<Table>
|
|
60
60
|
openMaterializedView(name: string): Promise<Table>
|
|
61
|
-
listMaterializedViews():
|
|
61
|
+
listMaterializedViews(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
|
|
62
62
|
/** Drop a materialized view. */
|
|
63
63
|
dropMaterializedView(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
|
|
64
64
|
/** Create a view: a named query planned on every read. */
|
|
@@ -73,7 +73,7 @@ export declare class Connection {
|
|
|
73
73
|
/** Start dropping a view and return the job deleting its definition. */
|
|
74
74
|
dropViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
|
|
75
75
|
/** The names of the views in one namespace. */
|
|
76
|
-
listViews(namespacePath?: Array<string> | undefined | null):
|
|
76
|
+
listViews(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
|
|
77
77
|
/** Start dropping a materialized view and return its cleanup job. */
|
|
78
78
|
dropMaterializedViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
|
|
79
79
|
openTable(name: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
|
|
@@ -93,7 +93,7 @@ export declare class Connection {
|
|
|
93
93
|
*/
|
|
94
94
|
openJob(jobId: string): Promise<Job>
|
|
95
95
|
/** List server-side jobs across the database's tables. */
|
|
96
|
-
listJobs():
|
|
96
|
+
listJobs(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): JobListing
|
|
97
97
|
/**
|
|
98
98
|
* Request cancellation of a server-side job by id. Returns true if the
|
|
99
99
|
* server accepted the cancellation, false if no such job exists.
|
|
@@ -125,6 +125,16 @@ export declare class Connection {
|
|
|
125
125
|
renameTable(currentName: string, newName: string, currentNamespacePath?: Array<string> | undefined | null, newNamespacePath?: Array<string> | undefined | null): Promise<void>
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
/** A lazy iterator over database names with inspectable pagination state. */
|
|
129
|
+
export declare class DatabaseNames {
|
|
130
|
+
/** Number of names cached without another REST request. */
|
|
131
|
+
numPageResults(): number
|
|
132
|
+
/** Continuation token for the next REST request. */
|
|
133
|
+
pageToken(): string | null
|
|
134
|
+
/** Fetch the next name, returning None when exhausted. */
|
|
135
|
+
next(): Promise<string | null>
|
|
136
|
+
}
|
|
137
|
+
|
|
128
138
|
export declare class Index {
|
|
129
139
|
static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
|
|
130
140
|
static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
|
|
@@ -193,6 +203,12 @@ export declare class Job {
|
|
|
193
203
|
events(limit?: number | undefined | null, filter?: string | undefined | null): Promise<Buffer>
|
|
194
204
|
}
|
|
195
205
|
|
|
206
|
+
export declare class JobListing {
|
|
207
|
+
numPageResults(): number
|
|
208
|
+
pageToken(): string | null
|
|
209
|
+
next(): Promise<JobInfo | null>
|
|
210
|
+
}
|
|
211
|
+
|
|
196
212
|
export declare class JsFullTextQuery {
|
|
197
213
|
static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
|
|
198
214
|
static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
|
|
@@ -212,6 +228,12 @@ export declare class JsHeaderProvider {
|
|
|
212
228
|
constructor(getHeadersCallback: () => Promise<Record<string, string>>)
|
|
213
229
|
}
|
|
214
230
|
|
|
231
|
+
export declare class NameListing {
|
|
232
|
+
numPageResults(): number
|
|
233
|
+
pageToken(): string | null
|
|
234
|
+
next(): Promise<string | null>
|
|
235
|
+
}
|
|
236
|
+
|
|
215
237
|
/**
|
|
216
238
|
* A Rust-side view of a JS-constructed `Scannable`.
|
|
217
239
|
*
|
|
@@ -243,7 +265,7 @@ export declare class NativeMergeInsertBuilder {
|
|
|
243
265
|
whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
|
|
244
266
|
setTimeout(timeout: number): void
|
|
245
267
|
useIndex(useIndex: boolean): NativeMergeInsertBuilder
|
|
246
|
-
|
|
268
|
+
useRTE(enable: boolean): NativeMergeInsertBuilder
|
|
247
269
|
validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
|
|
248
270
|
execute(buf: Buffer): Promise<MergeResult>
|
|
249
271
|
}
|
|
@@ -321,16 +343,18 @@ export declare class Query {
|
|
|
321
343
|
nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
|
|
322
344
|
fastSearch(): void
|
|
323
345
|
withRowId(): void
|
|
324
|
-
|
|
346
|
+
useRTE(enable: boolean): void
|
|
325
347
|
orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
|
|
326
348
|
outputSchema(): Promise<Buffer>
|
|
327
|
-
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
|
|
349
|
+
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
|
|
328
350
|
explainPlan(verbose: boolean): Promise<string>
|
|
329
351
|
analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
|
|
330
352
|
}
|
|
331
353
|
|
|
332
354
|
/** Typescript-style Async Iterator over RecordBatches */
|
|
333
355
|
export declare class RecordBatchIterator {
|
|
356
|
+
/** Return the executed stream's output schema as an Arrow IPC file. */
|
|
357
|
+
schema(): Buffer
|
|
334
358
|
next(): Promise<Buffer | null>
|
|
335
359
|
}
|
|
336
360
|
|
|
@@ -410,21 +434,21 @@ export declare class Table {
|
|
|
410
434
|
refreshColumn(column: string): Promise<RefreshColumnResult>
|
|
411
435
|
refreshColumnAsync(column: string): Promise<Job>
|
|
412
436
|
functionErrors(options?: FunctionErrorsOptions | undefined | null): Promise<FunctionErrors>
|
|
413
|
-
refreshMaterializedView(
|
|
437
|
+
refreshMaterializedView(sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
|
|
414
438
|
materializedViewDefinition(): Promise<string>
|
|
415
439
|
addColumnsWithSchema(schemaBuf: Buffer): Promise<AddColumnsResult>
|
|
416
440
|
alterColumns(alterations: Array<ColumnAlteration>): Promise<AlterColumnsResult>
|
|
417
441
|
updateFieldMetadata(updates: Array<FieldMetadataUpdate>): Promise<UpdateFieldMetadataResult>
|
|
418
442
|
dropColumns(columns: Array<string>): Promise<DropColumnsResult>
|
|
419
443
|
setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
444
|
+
enableRTE(spec: RTEWriteSpec): Promise<void>
|
|
445
|
+
disableRTE(): Promise<void>
|
|
446
|
+
getRTESpec(): Promise<RTEWriteSpec | null>
|
|
447
|
+
closeRTEWriters(): Promise<void>
|
|
448
|
+
flushRTE(): Promise<void>
|
|
449
|
+
compactRTE(): Promise<void>
|
|
450
|
+
checkpointRTE(): Promise<void>
|
|
451
|
+
getRTEStats(includeSstableRows: boolean): Promise<RTEStats | null>
|
|
428
452
|
version(): Promise<number>
|
|
429
453
|
checkoutCurrent(): Promise<Table>
|
|
430
454
|
checkout(version: number): Promise<void>
|
|
@@ -462,9 +486,9 @@ export declare class TakeQuery {
|
|
|
462
486
|
select(columns: Array<[string, string]>): void
|
|
463
487
|
selectColumns(columns: Array<string>): void
|
|
464
488
|
withRowId(): void
|
|
465
|
-
|
|
489
|
+
useRTE(enable: boolean): void
|
|
466
490
|
outputSchema(): Promise<Buffer>
|
|
467
|
-
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
|
|
491
|
+
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
|
|
468
492
|
explainPlan(verbose: boolean): Promise<string>
|
|
469
493
|
analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
|
|
470
494
|
}
|
|
@@ -477,6 +501,7 @@ export declare class VectorQuery {
|
|
|
477
501
|
postfilter(): void
|
|
478
502
|
refineFactor(refineFactor: number): void
|
|
479
503
|
nprobes(nprobe: number): void
|
|
504
|
+
searchEffort(searchEffort?: number | undefined | null): void
|
|
480
505
|
minimumNprobes(minimumNprobe: number): void
|
|
481
506
|
maximumNprobes(maximumNprobes: number): void
|
|
482
507
|
distanceRange(lowerBound?: number | undefined | null, upperBound?: number | undefined | null): void
|
|
@@ -490,11 +515,11 @@ export declare class VectorQuery {
|
|
|
490
515
|
offset(offset: number): void
|
|
491
516
|
fastSearch(): void
|
|
492
517
|
withRowId(): void
|
|
493
|
-
|
|
518
|
+
useRTE(enable: boolean): void
|
|
494
519
|
rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
|
|
495
520
|
orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
|
|
496
521
|
outputSchema(): Promise<Buffer>
|
|
497
|
-
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
|
|
522
|
+
execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
|
|
498
523
|
explainPlan(verbose: boolean): Promise<string>
|
|
499
524
|
analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
|
|
500
525
|
}
|
|
@@ -527,45 +552,6 @@ export interface BlobRange {
|
|
|
527
552
|
end: bigint
|
|
528
553
|
}
|
|
529
554
|
|
|
530
|
-
/**
|
|
531
|
-
* Live state of one bucket. A table is N buckets on one node; flattening to a
|
|
532
|
-
* single number hides the one hot bucket that is usually why someone opened
|
|
533
|
-
* this endpoint.
|
|
534
|
-
*/
|
|
535
|
-
export interface BucketStats {
|
|
536
|
-
/** The shard this bucket writes. */
|
|
537
|
-
shardId: string
|
|
538
|
-
/** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
|
|
539
|
-
status: string
|
|
540
|
-
/** Epoch of the writer that currently owns the shard. */
|
|
541
|
-
writerEpoch: number
|
|
542
|
-
/** Version of the shard manifest these numbers were read from. */
|
|
543
|
-
manifestVersion: number
|
|
544
|
-
/** The generation the active memtable will become. */
|
|
545
|
-
currentGeneration: number
|
|
546
|
-
/** WAL position replay resumes from. */
|
|
547
|
-
replayAfterWalEntryPosition: number
|
|
548
|
-
/**
|
|
549
|
-
* Highest WAL position the writer has seen. The difference against
|
|
550
|
-
* `replayAfterWalEntryPosition` is the WAL lag.
|
|
551
|
-
*/
|
|
552
|
-
walEntryPositionLastSeen: number
|
|
553
|
-
/** Flushed L0 generations not yet merged into the base table. */
|
|
554
|
-
generations: Array<GenerationStats>
|
|
555
|
-
/**
|
|
556
|
-
* Whether a pass owns this bucket's compaction latch right now. Says *a*
|
|
557
|
-
* driver is running, not *whose*, and the latch is held from dispatch —
|
|
558
|
-
* including while the pass queues for a pod-wide compactor permit. Read it
|
|
559
|
-
* as "do not pile on", never as "mine is progressing".
|
|
560
|
-
*/
|
|
561
|
-
compacting: boolean
|
|
562
|
-
/**
|
|
563
|
-
* Oldest first, active last. Absent for a `"Sealed"` bucket, whose
|
|
564
|
-
* in-memory state is torn down.
|
|
565
|
-
*/
|
|
566
|
-
memtables?: Array<MemtableStats>
|
|
567
|
-
}
|
|
568
|
-
|
|
569
555
|
export interface CatalogOptions {
|
|
570
556
|
apiKey?: string
|
|
571
557
|
clientConfig?: ClientConfig
|
|
@@ -579,6 +565,12 @@ export interface ClientConfig {
|
|
|
579
565
|
userAgent?: string
|
|
580
566
|
retryConfig?: RetryConfig
|
|
581
567
|
timeoutConfig?: TimeoutConfig
|
|
568
|
+
/**
|
|
569
|
+
* Maximum number of concurrent blob HTTP requests across all tables and
|
|
570
|
+
* blob handles on this connection. Defaults to 8. Can also be set with
|
|
571
|
+
* `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
|
|
572
|
+
*/
|
|
573
|
+
blobRequestConcurrency?: number
|
|
582
574
|
extraHeaders?: Record<string, string>
|
|
583
575
|
/**
|
|
584
576
|
* The delimiter joining a namespace path and a name into one object
|
|
@@ -861,19 +853,6 @@ export interface FunctionErrorsOptions {
|
|
|
861
853
|
limit?: number
|
|
862
854
|
}
|
|
863
855
|
|
|
864
|
-
/** One flushed L0 generation. */
|
|
865
|
-
export interface GenerationStats {
|
|
866
|
-
/** The generation number. Increases as memtables are sealed into L0. */
|
|
867
|
-
generation: number
|
|
868
|
-
/** On-disk size of the generation. */
|
|
869
|
-
bytes: number
|
|
870
|
-
/**
|
|
871
|
-
* Present only when `includeGenerationRows` was requested. Off by default
|
|
872
|
-
* because each count opens an uncached Lance dataset.
|
|
873
|
-
*/
|
|
874
|
-
rows?: number
|
|
875
|
-
}
|
|
876
|
-
|
|
877
856
|
/** A description of an index currently configured on a column */
|
|
878
857
|
export interface IndexConfig {
|
|
879
858
|
/** The name of the index */
|
|
@@ -987,11 +966,6 @@ export interface JobInfo {
|
|
|
987
966
|
/** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
|
|
988
967
|
export declare function lancedbMetricsCatalog(): Array<MetricDescription>
|
|
989
968
|
|
|
990
|
-
export interface ListDatabasesResponse {
|
|
991
|
-
databases: Array<string>
|
|
992
|
-
pageToken?: string
|
|
993
|
-
}
|
|
994
|
-
|
|
995
969
|
export interface ListNamespacesResponse {
|
|
996
970
|
namespaces: Array<string>
|
|
997
971
|
pageToken?: string
|
|
@@ -1002,44 +976,9 @@ export interface ListTablesResponse {
|
|
|
1002
976
|
pageToken?: string
|
|
1003
977
|
}
|
|
1004
978
|
|
|
1005
|
-
/**
|
|
1006
|
-
* Live per-bucket LSM state, as returned by `Table#getLsmStats`.
|
|
1007
|
-
*
|
|
1008
|
-
* Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
|
|
1009
|
-
* the caller's to compute.
|
|
1010
|
-
*/
|
|
1011
|
-
export interface LsmStats {
|
|
1012
|
-
/** One entry per bucket backing this table. */
|
|
1013
|
-
buckets: Array<BucketStats>
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
/**
|
|
1017
|
-
* Specification selecting Lance's MemWAL LSM-style write path for
|
|
1018
|
-
* `mergeInsert`.
|
|
1019
|
-
*
|
|
1020
|
-
* `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
|
|
1021
|
-
* `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
|
|
1022
|
-
* `column` is required.
|
|
1023
|
-
*/
|
|
1024
|
-
export interface LsmWriteSpec {
|
|
1025
|
-
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
1026
|
-
specType: string
|
|
1027
|
-
/** Bucket and identity variants: the sharding column. */
|
|
1028
|
-
column?: string
|
|
1029
|
-
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
1030
|
-
numBuckets?: number
|
|
1031
|
-
/**
|
|
1032
|
-
* Indexes the MemWAL keeps up to date. Omitted resolves every
|
|
1033
|
-
* maintainable index on install; an empty array means none.
|
|
1034
|
-
*/
|
|
1035
|
-
maintainedIndexes?: Array<string>
|
|
1036
|
-
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
|
|
1037
|
-
writerConfigDefaults?: Record<string, string>
|
|
1038
|
-
}
|
|
1039
|
-
|
|
1040
979
|
/** One in-memory memtable. */
|
|
1041
980
|
export interface MemtableStats {
|
|
1042
|
-
/** The generation this memtable will become once
|
|
981
|
+
/** The generation this memtable will become once frozen. */
|
|
1043
982
|
generation: number
|
|
1044
983
|
/** Rows currently buffered. */
|
|
1045
984
|
rows: number
|
|
@@ -1175,7 +1114,7 @@ export interface RefreshColumnResult {
|
|
|
1175
1114
|
}
|
|
1176
1115
|
|
|
1177
1116
|
export interface RefreshMaterializedViewResult {
|
|
1178
|
-
/** How the view was brought up to date: "rebuild"
|
|
1117
|
+
/** How the view was brought up to date: "rebuild". */
|
|
1179
1118
|
mode: string
|
|
1180
1119
|
rowsWritten: number
|
|
1181
1120
|
sourceVersion: number
|
|
@@ -1254,6 +1193,41 @@ export interface RetryConfig {
|
|
|
1254
1193
|
statuses?: Array<number>
|
|
1255
1194
|
}
|
|
1256
1195
|
|
|
1196
|
+
/**
|
|
1197
|
+
* Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
|
|
1198
|
+
*
|
|
1199
|
+
* Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
|
|
1200
|
+
* the caller's to compute.
|
|
1201
|
+
*/
|
|
1202
|
+
export interface RTEStats {
|
|
1203
|
+
/** One entry per table shard backing this table. */
|
|
1204
|
+
tableShards: Array<TableShardStats>
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/**
|
|
1208
|
+
* Specification enabling the realtime engine (RTE), which routes
|
|
1209
|
+
* `mergeInsert` through Lance's MemWAL LSM-style write path.
|
|
1210
|
+
*
|
|
1211
|
+
* `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
|
|
1212
|
+
* `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
|
|
1213
|
+
* `column` is required.
|
|
1214
|
+
*/
|
|
1215
|
+
export interface RTEWriteSpec {
|
|
1216
|
+
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
|
|
1217
|
+
specType: string
|
|
1218
|
+
/** Bucket and identity variants: the sharding column. */
|
|
1219
|
+
column?: string
|
|
1220
|
+
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
|
|
1221
|
+
numBuckets?: number
|
|
1222
|
+
/**
|
|
1223
|
+
* Indexes the RTE keeps up to date. Omitted resolves every
|
|
1224
|
+
* maintainable index on install; an empty array means none.
|
|
1225
|
+
*/
|
|
1226
|
+
maintainedIndexes?: Array<string>
|
|
1227
|
+
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
|
|
1228
|
+
writerConfigDefaults?: Record<string, string>
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1257
1231
|
/** Result of `OAuthSession.logout()`. */
|
|
1258
1232
|
export interface SessionLogout {
|
|
1259
1233
|
/**
|
|
@@ -1328,6 +1302,58 @@ export interface SplitSequentialOptions {
|
|
|
1328
1302
|
splitNames?: Array<string>
|
|
1329
1303
|
}
|
|
1330
1304
|
|
|
1305
|
+
/** One SSTable. */
|
|
1306
|
+
export interface SsTableStats {
|
|
1307
|
+
/** The generation number. Increases as memtables are frozen into SSTables. */
|
|
1308
|
+
generation: number
|
|
1309
|
+
/** On-disk size of the SSTable. */
|
|
1310
|
+
bytes: number
|
|
1311
|
+
/**
|
|
1312
|
+
* Present only when `includeSstableRows` was requested. Off by default
|
|
1313
|
+
* because each count opens an uncached Lance dataset.
|
|
1314
|
+
*/
|
|
1315
|
+
rows?: number
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
/**
|
|
1319
|
+
* Live state of one table shard. A table is N table shards on one node; flattening to a
|
|
1320
|
+
* single number hides the one hot table shard that is usually why someone opened
|
|
1321
|
+
* this endpoint.
|
|
1322
|
+
*/
|
|
1323
|
+
export interface TableShardStats {
|
|
1324
|
+
/** The id of this table shard. */
|
|
1325
|
+
shardId: string
|
|
1326
|
+
/** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
|
|
1327
|
+
status: string
|
|
1328
|
+
/** Epoch of the writer that currently owns the shard. */
|
|
1329
|
+
writerEpoch: number
|
|
1330
|
+
/** Version of the shard manifest these numbers were read from. */
|
|
1331
|
+
manifestVersion: number
|
|
1332
|
+
/** The generation the active memtable will become. */
|
|
1333
|
+
currentGeneration: number
|
|
1334
|
+
/** WAL position replay resumes from. */
|
|
1335
|
+
replayAfterWalEntryPosition: number
|
|
1336
|
+
/**
|
|
1337
|
+
* Highest WAL position the writer has seen. The difference against
|
|
1338
|
+
* `replayAfterWalEntryPosition` is the WAL lag.
|
|
1339
|
+
*/
|
|
1340
|
+
walEntryPositionLastSeen: number
|
|
1341
|
+
/** SSTables not yet merged into the base table. */
|
|
1342
|
+
sstables: Array<SsTableStats>
|
|
1343
|
+
/**
|
|
1344
|
+
* Whether a pass owns this table shard's compaction latch right now. Says *a*
|
|
1345
|
+
* driver is running, not *whose*, and the latch is held from dispatch —
|
|
1346
|
+
* including while the pass queues for a pod-wide compactor permit. Read it
|
|
1347
|
+
* as "do not pile on", never as "mine is progressing".
|
|
1348
|
+
*/
|
|
1349
|
+
compacting: boolean
|
|
1350
|
+
/**
|
|
1351
|
+
* Oldest first, active last. Absent for a `"Sealed"` table shard, whose
|
|
1352
|
+
* in-memory state is torn down.
|
|
1353
|
+
*/
|
|
1354
|
+
memtables?: Array<MemtableStats>
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1331
1357
|
export interface TableStatistics {
|
|
1332
1358
|
/**
|
|
1333
1359
|
* The total size, in bytes, of the table's data files, index files, and
|
|
@@ -1338,6 +1364,17 @@ export interface TableStatistics {
|
|
|
1338
1364
|
totalBytes: number
|
|
1339
1365
|
/** The number of rows in the table */
|
|
1340
1366
|
numRows: number
|
|
1367
|
+
/**
|
|
1368
|
+
* The number of rows marked as deleted across all fragments of the table
|
|
1369
|
+
*
|
|
1370
|
+
* These rows are not included in `numRows`, but still occupy space on disk
|
|
1371
|
+
* until the table is compacted, so a large value here indicates that the
|
|
1372
|
+
* table should be optimized. Fragments in which every row was deleted are
|
|
1373
|
+
* dropped outright, so their rows are not counted here.
|
|
1374
|
+
*
|
|
1375
|
+
* Absent (`undefined`) when the backend does not report deletion counts.
|
|
1376
|
+
*/
|
|
1377
|
+
numDeletedRows?: number
|
|
1341
1378
|
/** The number of indices in the table */
|
|
1342
1379
|
numIndices: number
|
|
1343
1380
|
/** Statistics on table fragments */
|