@lancedb/lancedb 0.41.0-beta.0 → 0.41.0-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.js +16 -5
- package/dist/blob.d.ts +26 -1
- package/dist/blob.js +58 -5
- package/dist/connection.d.ts +19 -9
- package/dist/connection.js +10 -6
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +136 -106
- package/dist/native.js +54 -52
- package/dist/query.d.ts +87 -25
- package/dist/query.js +168 -41
- package/dist/table.d.ts +58 -51
- package/dist/table.js +24 -18
- package/package.json +8 -8
package/dist/arrow.js
CHANGED
|
@@ -357,14 +357,25 @@ function makeArrowTable(data, options, metadata) {
|
|
|
357
357
|
}
|
|
358
358
|
if (schema !== undefined) {
|
|
359
359
|
// Validate blob values up front and give every one the full
|
|
360
|
-
//
|
|
360
|
+
// storage shape, so inference sees the same struct whether a row
|
|
361
361
|
// passed bytes, a URI, or a partial struct.
|
|
362
|
-
data = mapBlobInputs(data, schema, (value, field, row) => {
|
|
362
|
+
data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
|
|
363
363
|
if (value === undefined) {
|
|
364
364
|
return value;
|
|
365
365
|
}
|
|
366
366
|
try {
|
|
367
|
-
|
|
367
|
+
const normalized = (0, blob_1.coerceBlobValue)(value, storageField);
|
|
368
|
+
if (normalized === null ||
|
|
369
|
+
!(storageField.type instanceof apache_arrow_1.Struct) ||
|
|
370
|
+
storageField.type.children.length === 2) {
|
|
371
|
+
return normalized;
|
|
372
|
+
}
|
|
373
|
+
// Keep even unused range children during schema inference. Otherwise
|
|
374
|
+
// creating a range-capable table with inline data would lose them.
|
|
375
|
+
return Object.fromEntries(storageField.type.children.map((child) => [
|
|
376
|
+
child.name,
|
|
377
|
+
normalized[child.name] ?? null,
|
|
378
|
+
]));
|
|
368
379
|
}
|
|
369
380
|
catch (e) {
|
|
370
381
|
throw new Error(`Invalid value for blob field ${field} at row ${row}: ${e.message}`);
|
|
@@ -449,7 +460,7 @@ function mapBlobFields(record, fields, prefix, row, visit) {
|
|
|
449
460
|
}
|
|
450
461
|
function mapBlobField(value, field, label, row, visit) {
|
|
451
462
|
if ((0, blob_1.isBlobField)(field)) {
|
|
452
|
-
return visit(value, label, row);
|
|
463
|
+
return visit(value, label, row, field);
|
|
453
464
|
}
|
|
454
465
|
if (field.type instanceof apache_arrow_1.Struct && isObject(value)) {
|
|
455
466
|
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
|
|
@@ -528,7 +539,7 @@ function valueAtPath(datum, path) {
|
|
|
528
539
|
function transposeData(data, field, path = []) {
|
|
529
540
|
const valuesPath = [...path, field.name];
|
|
530
541
|
if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
|
|
531
|
-
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
|
|
542
|
+
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath), field));
|
|
532
543
|
const childVectors = field.type.children.map((child) => {
|
|
533
544
|
const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
|
|
534
545
|
return makeVector(values, child.type, undefined, child.nullable);
|
package/dist/blob.d.ts
CHANGED
|
@@ -14,6 +14,9 @@ export type BlobUri = string | URL;
|
|
|
14
14
|
*
|
|
15
15
|
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
|
|
16
16
|
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
|
|
17
|
+
* External byte ranges require both
|
|
18
|
+
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
|
|
19
|
+
* `bigint` values for these unsigned 64-bit byte counts.
|
|
17
20
|
*
|
|
18
21
|
* @example
|
|
19
22
|
* ```ts
|
|
@@ -31,14 +34,25 @@ export type BlobUri = string | URL;
|
|
|
31
34
|
export type BlobInput = BlobData | BlobUri | {
|
|
32
35
|
data: BlobData;
|
|
33
36
|
uri?: null;
|
|
37
|
+
position?: null;
|
|
38
|
+
size?: null;
|
|
34
39
|
} | {
|
|
35
40
|
data?: null;
|
|
36
41
|
uri: BlobUri;
|
|
42
|
+
position?: null;
|
|
43
|
+
size?: null;
|
|
44
|
+
} | {
|
|
45
|
+
data?: null;
|
|
46
|
+
uri: BlobUri;
|
|
47
|
+
position: number | bigint;
|
|
48
|
+
size: number | bigint;
|
|
37
49
|
} | null | undefined;
|
|
38
50
|
/** A blob value normalized to the columns of the blob struct. */
|
|
39
51
|
export type BlobValue = {
|
|
40
52
|
data: Buffer | Uint8Array | null;
|
|
41
53
|
uri: string | null;
|
|
54
|
+
position?: bigint;
|
|
55
|
+
size?: bigint;
|
|
42
56
|
};
|
|
43
57
|
export type BlobOptions = {
|
|
44
58
|
/** Defaults to true. */
|
|
@@ -61,9 +75,20 @@ export type BlobOptions = {
|
|
|
61
75
|
};
|
|
62
76
|
/**
|
|
63
77
|
* Declares a `lance.blob.v2` column.
|
|
78
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
79
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
80
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
81
|
+
*
|
|
82
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
83
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
84
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
85
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
86
|
+
* are converted using the current table schema.
|
|
64
87
|
*
|
|
65
88
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
66
89
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
90
|
+
* Blobs inside lists can be written and queried as descriptors, but cannot
|
|
91
|
+
* currently be fetched because a row ID does not identify a list element.
|
|
67
92
|
*
|
|
68
93
|
* @example
|
|
69
94
|
* ```ts
|
|
@@ -177,4 +202,4 @@ export declare function blobToRead(value: unknown): Blob | undefined;
|
|
|
177
202
|
* keeping the value's shape.
|
|
178
203
|
*/
|
|
179
204
|
export declare function withBlobBytes(value: unknown, bytes: Uint8Array): unknown;
|
|
180
|
-
export declare function coerceBlobValue(input: unknown): BlobValue | null;
|
|
205
|
+
export declare function coerceBlobValue(input: unknown, field?: Field): BlobValue | null;
|
package/dist/blob.js
CHANGED
|
@@ -18,9 +18,20 @@ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-thresho
|
|
|
18
18
|
const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
|
|
19
19
|
/**
|
|
20
20
|
* Declares a `lance.blob.v2` column.
|
|
21
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
22
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
23
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
24
|
+
*
|
|
25
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
26
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
27
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
28
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
29
|
+
* are converted using the current table schema.
|
|
21
30
|
*
|
|
22
31
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
23
32
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
33
|
+
* Blobs inside lists can be written and queried as descriptors, but cannot
|
|
34
|
+
* currently be fetched because a row ID does not identify a list element.
|
|
24
35
|
*
|
|
25
36
|
* @example
|
|
26
37
|
* ```ts
|
|
@@ -60,6 +71,8 @@ function blob(name, options = {}) {
|
|
|
60
71
|
return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
|
|
61
72
|
new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
|
|
62
73
|
new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
|
|
74
|
+
new apache_arrow_1.Field("position", new apache_arrow_1.Uint64(), true),
|
|
75
|
+
new apache_arrow_1.Field("size", new apache_arrow_1.Uint64(), true),
|
|
63
76
|
]), options.nullable ?? true, metadata);
|
|
64
77
|
}
|
|
65
78
|
/**
|
|
@@ -188,6 +201,8 @@ function normalizeBlobInput(value) {
|
|
|
188
201
|
...value,
|
|
189
202
|
data: value.data,
|
|
190
203
|
uri: value.uri,
|
|
204
|
+
position: value.position,
|
|
205
|
+
size: value.size,
|
|
191
206
|
};
|
|
192
207
|
let changed = false;
|
|
193
208
|
if (value.data instanceof ArrayBuffer) {
|
|
@@ -224,9 +239,15 @@ function withBlobBytes(value, bytes) {
|
|
|
224
239
|
return bytes;
|
|
225
240
|
}
|
|
226
241
|
const record = value;
|
|
227
|
-
return {
|
|
242
|
+
return {
|
|
243
|
+
...record,
|
|
244
|
+
uri: record.uri,
|
|
245
|
+
position: record.position,
|
|
246
|
+
size: record.size,
|
|
247
|
+
data: bytes,
|
|
248
|
+
};
|
|
228
249
|
}
|
|
229
|
-
function coerceBlobValue(input) {
|
|
250
|
+
function coerceBlobValue(input, field) {
|
|
230
251
|
const value = normalizeBlobInput(input);
|
|
231
252
|
if (value == null) {
|
|
232
253
|
return null;
|
|
@@ -248,9 +269,9 @@ function coerceBlobValue(input) {
|
|
|
248
269
|
if (!("data" in record) && !("uri" in record)) {
|
|
249
270
|
throw new Error("Blob struct values must include a 'data' or 'uri' field");
|
|
250
271
|
}
|
|
251
|
-
const unknownKeys = Object.keys(record).filter((key) =>
|
|
272
|
+
const unknownKeys = Object.keys(record).filter((key) => !["data", "uri", "position", "size"].includes(key));
|
|
252
273
|
if (unknownKeys.length > 0) {
|
|
253
|
-
throw new Error(`Blob struct values only support 'data' and '
|
|
274
|
+
throw new Error(`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
|
|
254
275
|
.map((key) => `'${key}'`)
|
|
255
276
|
.join(", ")}`);
|
|
256
277
|
}
|
|
@@ -270,9 +291,41 @@ function coerceBlobValue(input) {
|
|
|
270
291
|
if ((bytes == null) === (uriValue == null)) {
|
|
271
292
|
throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
|
|
272
293
|
}
|
|
294
|
+
const hasPosition = record.position != null;
|
|
295
|
+
const hasSize = record.size != null;
|
|
296
|
+
if (hasPosition !== hasSize) {
|
|
297
|
+
throw new Error("External blob must set both position and size, or neither");
|
|
298
|
+
}
|
|
299
|
+
if (hasPosition) {
|
|
300
|
+
if (uriValue == null) {
|
|
301
|
+
throw new Error("External blob byte ranges require a uri");
|
|
302
|
+
}
|
|
303
|
+
const position = blobRangeInteger(record.position, "position", 0n);
|
|
304
|
+
const size = blobRangeInteger(record.size, "size", 1n);
|
|
305
|
+
if (position + size > 0xffffffffffffffffn) {
|
|
306
|
+
throw new Error("External blob range overflows uint64");
|
|
307
|
+
}
|
|
308
|
+
if (field !== undefined &&
|
|
309
|
+
(!(field.type instanceof apache_arrow_1.Struct) ||
|
|
310
|
+
["position", "size"].some((name) => !field.type.children.some((child) => child.name === name)))) {
|
|
311
|
+
throw new Error("Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size");
|
|
312
|
+
}
|
|
313
|
+
return { data: null, uri: uriValue, position, size };
|
|
314
|
+
}
|
|
273
315
|
return { data: bytes, uri: uriValue };
|
|
274
316
|
}
|
|
275
|
-
throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }");
|
|
317
|
+
throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }");
|
|
318
|
+
}
|
|
319
|
+
function blobRangeInteger(value, name, minimum) {
|
|
320
|
+
if (typeof value !== "bigint" &&
|
|
321
|
+
(typeof value !== "number" || !Number.isSafeInteger(value))) {
|
|
322
|
+
throw new Error(`Blob ${name} must be a safe integer or bigint`);
|
|
323
|
+
}
|
|
324
|
+
const integer = BigInt(value);
|
|
325
|
+
if (integer < minimum || integer > 0xffffffffffffffffn) {
|
|
326
|
+
throw new Error(`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`);
|
|
327
|
+
}
|
|
328
|
+
return integer;
|
|
276
329
|
}
|
|
277
330
|
function isBlobLike(value) {
|
|
278
331
|
return typeof Blob !== "undefined" && value instanceof Blob;
|
package/dist/connection.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
2
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
3
3
|
import { Job } from "./job";
|
|
4
|
+
import { Listing, ListingOptions } from "./listing";
|
|
4
5
|
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
5
6
|
import { Connection as LanceDbConnection } from "./native";
|
|
6
7
|
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
|
|
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
|
|
|
107
108
|
* the last table name from the previous page.
|
|
108
109
|
*/
|
|
109
110
|
startAfter?: string;
|
|
110
|
-
/**
|
|
111
|
+
/** The maximum number of names to return. Omitted returns all names; zero returns none. */
|
|
111
112
|
limit?: number;
|
|
112
113
|
}
|
|
113
114
|
export interface ListTablesOptions {
|
|
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
|
|
|
120
121
|
pageToken?: string;
|
|
121
122
|
/**
|
|
122
123
|
* An upper bound on how many tables to return.
|
|
124
|
+
* Defaults to 100. Zero returns an empty page without a continuation token.
|
|
123
125
|
*
|
|
124
126
|
* A page may hold fewer than this and still not be the last one, so keep
|
|
125
127
|
* going while the response carries a page token rather than while pages are
|
|
@@ -276,9 +278,11 @@ export declare abstract class Connection {
|
|
|
276
278
|
/**
|
|
277
279
|
* The names of the materialized views in this database.
|
|
278
280
|
*
|
|
279
|
-
*
|
|
281
|
+
* Iterate lazily with `for await...of`. Local listings inspect table schemas
|
|
282
|
+
* one page at a time. Options control page size and the starting token.
|
|
283
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
280
284
|
*/
|
|
281
|
-
abstract listMaterializedViews():
|
|
285
|
+
abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
282
286
|
/**
|
|
283
287
|
* Drop the materialized view named `name`.
|
|
284
288
|
*
|
|
@@ -330,8 +334,10 @@ export declare abstract class Connection {
|
|
|
330
334
|
* The names of the views in one namespace.
|
|
331
335
|
*
|
|
332
336
|
* Names only; a definition comes from {@link describeView}.
|
|
337
|
+
* Iterate lazily with `for await...of`. See {@link Listing} for pagination
|
|
338
|
+
* state and error behavior; options control page size and the starting token.
|
|
333
339
|
*/
|
|
334
|
-
abstract listViews(namespacePath?: string[]):
|
|
340
|
+
abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
335
341
|
abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
336
342
|
/**
|
|
337
343
|
* Creates a new Table and initialize it with new data.
|
|
@@ -488,8 +494,12 @@ export declare abstract class Connection {
|
|
|
488
494
|
* connection-level call for any of them.
|
|
489
495
|
*/
|
|
490
496
|
abstract openJob(jobId: string): Promise<Job>;
|
|
491
|
-
/**
|
|
492
|
-
|
|
497
|
+
/**
|
|
498
|
+
* Iterate lazily over server-side jobs across the database's tables.
|
|
499
|
+
* Use `for await...of`. Options control page size and the starting token.
|
|
500
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
501
|
+
*/
|
|
502
|
+
abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
493
503
|
/**
|
|
494
504
|
* Request cancellation of a server-side job by id.
|
|
495
505
|
*
|
|
@@ -530,14 +540,14 @@ export declare class LocalConnection extends Connection {
|
|
|
530
540
|
withNoData?: boolean;
|
|
531
541
|
}): Promise<MaterializedView>;
|
|
532
542
|
openMaterializedView(name: string): Promise<MaterializedView>;
|
|
533
|
-
listMaterializedViews():
|
|
543
|
+
listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
534
544
|
dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
|
|
535
545
|
dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
536
546
|
createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
537
547
|
describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
538
548
|
dropView(name: string, namespacePath?: string[]): Promise<void>;
|
|
539
549
|
dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
540
|
-
listViews(namespacePath?: string[]):
|
|
550
|
+
listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
541
551
|
listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
542
552
|
openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
543
553
|
cloneTable(targetTableName: string, sourceUri: string, options?: {
|
|
@@ -562,7 +572,7 @@ export declare class LocalConnection extends Connection {
|
|
|
562
572
|
dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
|
|
563
573
|
renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
|
564
574
|
openJob(jobId: string): Promise<Job>;
|
|
565
|
-
listJobs():
|
|
575
|
+
listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
566
576
|
cancelJob(jobId: string): Promise<boolean>;
|
|
567
577
|
pauseJob(jobId: string): Promise<string>;
|
|
568
578
|
resumeJob(jobId: string): Promise<string>;
|
package/dist/connection.js
CHANGED
|
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
|
|
|
8
8
|
const arrow_2 = require("./arrow");
|
|
9
9
|
const registry_1 = require("./embedding/registry");
|
|
10
10
|
const job_1 = require("./job");
|
|
11
|
+
const listing_1 = require("./listing");
|
|
11
12
|
const materialized_view_1 = require("./materialized_view");
|
|
12
13
|
const view_1 = require("./view");
|
|
13
14
|
const sanitize_1 = require("./sanitize");
|
|
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
|
|
|
79
80
|
const innerTable = await this.inner.openMaterializedView(name);
|
|
80
81
|
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
81
82
|
}
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
listMaterializedViews(options = {}) {
|
|
84
|
+
(0, listing_1.validateListingOptions)(options);
|
|
85
|
+
return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
|
|
84
86
|
}
|
|
85
87
|
async dropMaterializedView(name, namespacePath) {
|
|
86
88
|
return this.inner.dropMaterializedView(name, namespacePath ?? []);
|
|
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
|
|
|
100
102
|
async dropViewAsync(name, namespacePath) {
|
|
101
103
|
return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
|
|
102
104
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
listViews(namespacePath, options = {}) {
|
|
106
|
+
(0, listing_1.validateListingOptions)(options);
|
|
107
|
+
return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
105
108
|
}
|
|
106
109
|
async listTables(namespacePathOrOptions, options) {
|
|
107
110
|
// Detect if first argument is namespacePath array or options object
|
|
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
|
|
|
243
246
|
async openJob(jobId) {
|
|
244
247
|
return new job_1.Job(await this.inner.openJob(jobId));
|
|
245
248
|
}
|
|
246
|
-
|
|
247
|
-
|
|
249
|
+
listJobs(options = {}) {
|
|
250
|
+
(0, listing_1.validateListingOptions)(options);
|
|
251
|
+
return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
|
|
248
252
|
}
|
|
249
253
|
async cancelJob(jobId) {
|
|
250
254
|
return this.inner.cancelJob(jobId);
|
package/dist/index.d.ts
CHANGED
|
@@ -14,9 +14,9 @@ export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, Blob
|
|
|
14
14
|
export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
15
15
|
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
16
16
|
export { Job, JobEventsOptions } from "./job";
|
|
17
|
-
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
17
|
+
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
18
18
|
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
|
|
19
|
-
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions,
|
|
19
|
+
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
20
20
|
export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
|
|
21
21
|
export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
|
|
22
22
|
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
|
|
@@ -249,3 +249,4 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
|
|
|
249
249
|
*/
|
|
250
250
|
export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
|
|
251
251
|
export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
|
|
252
|
+
export { Listing, ListingOptions } from "./listing";
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
5
|
+
exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
6
6
|
exports.tokenize = tokenize;
|
|
7
7
|
exports.connect = connect;
|
|
8
8
|
exports.connectNamespace = connectNamespace;
|
|
@@ -186,3 +186,5 @@ var catalog_1 = require("./catalog");
|
|
|
186
186
|
Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
|
|
187
187
|
Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
|
|
188
188
|
Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
|
|
189
|
+
var listing_1 = require("./listing");
|
|
190
|
+
Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
|
package/dist/indices.d.ts
CHANGED
|
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
|
|
|
453
453
|
*/
|
|
454
454
|
sampleRate?: number;
|
|
455
455
|
}
|
|
456
|
-
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
456
|
+
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
457
457
|
/**
|
|
458
458
|
* Options to create a full text search index
|
|
459
459
|
*/
|
|
@@ -477,6 +477,8 @@ export interface FtsOptions {
|
|
|
477
477
|
*
|
|
478
478
|
* "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
|
|
479
479
|
*
|
|
480
|
+
* "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
|
|
481
|
+
*
|
|
480
482
|
* "icu" - ICU dictionary-based word segmentation.
|
|
481
483
|
*
|
|
482
484
|
* "icu/split" - ICU segmentation with simple-style delimiter splitting.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/** Pagination controls for lazy resource listings. */
|
|
2
|
+
export interface ListingOptions {
|
|
3
|
+
/** Starting continuation token; omitted starts at the beginning. */
|
|
4
|
+
pageToken?: string;
|
|
5
|
+
/** Maximum results per request, not a limit on the whole listing. */
|
|
6
|
+
pageLimit?: number;
|
|
7
|
+
}
|
|
8
|
+
/** @internal */
|
|
9
|
+
export declare function validateListingOptions(options: ListingOptions): void;
|
|
10
|
+
/**
|
|
11
|
+
* A lazy async iterator over database resources with pagination state.
|
|
12
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
17
|
+
* console.log(name);
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare class Listing<T> implements AsyncIterableIterator<T> {
|
|
22
|
+
private readonly inner;
|
|
23
|
+
/** @hidden */
|
|
24
|
+
constructor(inner: {
|
|
25
|
+
next(): Promise<T | null | undefined>;
|
|
26
|
+
numPageResults(): number;
|
|
27
|
+
pageToken(): string | null | undefined;
|
|
28
|
+
});
|
|
29
|
+
[Symbol.asyncIterator](): AsyncIterableIterator<T>;
|
|
30
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
31
|
+
next(): Promise<IteratorResult<T>>;
|
|
32
|
+
/** Number of cached items available without another request. */
|
|
33
|
+
numPageResults(): number;
|
|
34
|
+
/**
|
|
35
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
36
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
37
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
38
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
39
|
+
*/
|
|
40
|
+
pageToken(): string | undefined;
|
|
41
|
+
}
|
package/dist/listing.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.Listing = void 0;
|
|
6
|
+
exports.validateListingOptions = validateListingOptions;
|
|
7
|
+
/** @internal */
|
|
8
|
+
function validateListingOptions(options) {
|
|
9
|
+
if (options.pageLimit !== undefined &&
|
|
10
|
+
(!Number.isInteger(options.pageLimit) ||
|
|
11
|
+
options.pageLimit <= 0 ||
|
|
12
|
+
options.pageLimit > 2147483647)) {
|
|
13
|
+
throw new Error("Listing page limit must be an integer between 1 and 2147483647");
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A lazy async iterator over database resources with pagination state.
|
|
18
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```ts
|
|
22
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
23
|
+
* console.log(name);
|
|
24
|
+
* }
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
class Listing {
|
|
28
|
+
inner;
|
|
29
|
+
/** @hidden */
|
|
30
|
+
constructor(inner) {
|
|
31
|
+
this.inner = inner;
|
|
32
|
+
}
|
|
33
|
+
[Symbol.asyncIterator]() {
|
|
34
|
+
return this;
|
|
35
|
+
}
|
|
36
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
37
|
+
async next() {
|
|
38
|
+
const value = await this.inner.next();
|
|
39
|
+
return value == null
|
|
40
|
+
? { done: true, value: undefined }
|
|
41
|
+
: { done: false, value };
|
|
42
|
+
}
|
|
43
|
+
/** Number of cached items available without another request. */
|
|
44
|
+
numPageResults() {
|
|
45
|
+
return this.inner.numPageResults();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
49
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
50
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
51
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
52
|
+
*/
|
|
53
|
+
pageToken() {
|
|
54
|
+
return this.inner.pageToken() ?? undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.Listing = Listing;
|
package/dist/merge.d.ts
CHANGED
|
@@ -57,25 +57,25 @@ export declare class MergeInsertBuilder {
|
|
|
57
57
|
*/
|
|
58
58
|
useIndex(useIndex: boolean): MergeInsertBuilder;
|
|
59
59
|
/**
|
|
60
|
-
* Control
|
|
60
|
+
* Control RTE routing for this merge.
|
|
61
61
|
*
|
|
62
|
-
* By default (unset), a `mergeInsert` on a table with an
|
|
62
|
+
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
|
|
63
63
|
* routed through Lance's MemWAL shard writer, and a table without one uses the
|
|
64
64
|
* standard path.
|
|
65
65
|
*
|
|
66
|
-
* @param enable - `true` forces
|
|
67
|
-
*
|
|
66
|
+
* @param enable - `true` forces RTE routing and errors if the table has no
|
|
67
|
+
* RTE write spec. `false` forces the standard write path even when a spec is set.
|
|
68
68
|
*/
|
|
69
|
-
|
|
69
|
+
useRTE(enable: boolean): MergeInsertBuilder;
|
|
70
70
|
/**
|
|
71
71
|
* Controls how an LSM merge checks that its input targets a single shard.
|
|
72
72
|
*
|
|
73
|
-
* When a table has an
|
|
73
|
+
* When a table has an RTE write spec, every row in a `mergeInsert` call must
|
|
74
74
|
* route to the same shard. When `true` (the default), every row is inspected
|
|
75
75
|
* to verify this. When `false`, only the first row is inspected and the
|
|
76
76
|
* shard it routes to is used for the whole input — a faster path for callers
|
|
77
77
|
* that have already pre-sharded their input. Has no effect on tables without
|
|
78
|
-
* an
|
|
78
|
+
* an RTE write spec.
|
|
79
79
|
*
|
|
80
80
|
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
|
|
81
81
|
*/
|
package/dist/merge.js
CHANGED
|
@@ -69,27 +69,27 @@ class MergeInsertBuilder {
|
|
|
69
69
|
return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
|
-
* Control
|
|
72
|
+
* Control RTE routing for this merge.
|
|
73
73
|
*
|
|
74
|
-
* By default (unset), a `mergeInsert` on a table with an
|
|
74
|
+
* By default (unset), a `mergeInsert` on a table with an RTE write spec is
|
|
75
75
|
* routed through Lance's MemWAL shard writer, and a table without one uses the
|
|
76
76
|
* standard path.
|
|
77
77
|
*
|
|
78
|
-
* @param enable - `true` forces
|
|
79
|
-
*
|
|
78
|
+
* @param enable - `true` forces RTE routing and errors if the table has no
|
|
79
|
+
* RTE write spec. `false` forces the standard write path even when a spec is set.
|
|
80
80
|
*/
|
|
81
|
-
|
|
82
|
-
return new MergeInsertBuilder(this.#native.
|
|
81
|
+
useRTE(enable) {
|
|
82
|
+
return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
85
|
* Controls how an LSM merge checks that its input targets a single shard.
|
|
86
86
|
*
|
|
87
|
-
* When a table has an
|
|
87
|
+
* When a table has an RTE write spec, every row in a `mergeInsert` call must
|
|
88
88
|
* route to the same shard. When `true` (the default), every row is inspected
|
|
89
89
|
* to verify this. When `false`, only the first row is inspected and the
|
|
90
90
|
* shard it routes to is used for the whole input — a faster path for callers
|
|
91
91
|
* that have already pre-sharded their input. Has no effect on tables without
|
|
92
|
-
* an
|
|
92
|
+
* an RTE write spec.
|
|
93
93
|
*
|
|
94
94
|
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
|
|
95
95
|
*/
|