@lancedb/lancedb 0.41.0-beta.8 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +13 -0
- package/dist/arrow.d.ts +1 -0
- package/dist/arrow.js +16 -5
- package/dist/blob.d.ts +24 -1
- package/dist/blob.js +56 -5
- package/dist/catalog.d.ts +6 -1
- package/dist/catalog.js +6 -2
- package/dist/connection.d.ts +26 -79
- package/dist/connection.js +28 -26
- package/dist/embedding/transformers.js +4 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +8 -5
- package/dist/indices.d.ts +2 -1
- package/dist/job.d.ts +8 -5
- package/dist/job.js +8 -5
- package/dist/materialized_view.d.ts +7 -3
- package/dist/materialized_view.js +7 -4
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +72 -73
- package/dist/native.js +53 -52
- package/dist/query.d.ts +18 -16
- package/dist/query.js +73 -36
- package/dist/table.d.ts +57 -48
- package/dist/table.js +18 -18
- package/package.json +10 -9
package/CONTRIBUTING.md
CHANGED
|
@@ -50,6 +50,19 @@ pnpm install
|
|
|
50
50
|
pnpm build
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
+
Local SQL computed columns and materialized-view creation/refresh are experimental
|
|
54
|
+
and disabled in published packages. To opt in when building from this directory:
|
|
55
|
+
|
|
56
|
+
```shell
|
|
57
|
+
pnpm build:debug --features experimental-functions
|
|
58
|
+
pnpm tsc
|
|
59
|
+
pnpm test --runInBand table.test.ts materialized_view.test.ts
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Rebuild without `--features experimental-functions` to test the default package.
|
|
63
|
+
The tests detect the compiled feature; remote operations, metadata protections,
|
|
64
|
+
and ordinary eager `addColumns` transforms remain available in both builds.
|
|
65
|
+
|
|
53
66
|
Lint:
|
|
54
67
|
|
|
55
68
|
```shell
|
package/dist/arrow.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { Buffers } from "apache-arrow/data";
|
|
|
3
3
|
import { type EmbeddingFunction } from "./embedding/embedding_function";
|
|
4
4
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
5
5
|
export * from "apache-arrow";
|
|
6
|
+
export type { Data as ArrowData } from "apache-arrow";
|
|
6
7
|
export type SchemaLike = Schema | {
|
|
7
8
|
fields: FieldLike[];
|
|
8
9
|
metadata: Map<string, string>;
|
package/dist/arrow.js
CHANGED
|
@@ -357,14 +357,25 @@ function makeArrowTable(data, options, metadata) {
|
|
|
357
357
|
}
|
|
358
358
|
if (schema !== undefined) {
|
|
359
359
|
// Validate blob values up front and give every one the full
|
|
360
|
-
//
|
|
360
|
+
// storage shape, so inference sees the same struct whether a row
|
|
361
361
|
// passed bytes, a URI, or a partial struct.
|
|
362
|
-
data = mapBlobInputs(data, schema, (value, field, row) => {
|
|
362
|
+
data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
|
|
363
363
|
if (value === undefined) {
|
|
364
364
|
return value;
|
|
365
365
|
}
|
|
366
366
|
try {
|
|
367
|
-
|
|
367
|
+
const normalized = (0, blob_1.coerceBlobValue)(value, storageField);
|
|
368
|
+
if (normalized === null ||
|
|
369
|
+
!(storageField.type instanceof apache_arrow_1.Struct) ||
|
|
370
|
+
storageField.type.children.length === 2) {
|
|
371
|
+
return normalized;
|
|
372
|
+
}
|
|
373
|
+
// Keep even unused range children during schema inference. Otherwise
|
|
374
|
+
// creating a range-capable table with inline data would lose them.
|
|
375
|
+
return Object.fromEntries(storageField.type.children.map((child) => [
|
|
376
|
+
child.name,
|
|
377
|
+
normalized[child.name] ?? null,
|
|
378
|
+
]));
|
|
368
379
|
}
|
|
369
380
|
catch (e) {
|
|
370
381
|
throw new Error(`Invalid value for blob field ${field} at row ${row}: ${e.message}`);
|
|
@@ -449,7 +460,7 @@ function mapBlobFields(record, fields, prefix, row, visit) {
|
|
|
449
460
|
}
|
|
450
461
|
function mapBlobField(value, field, label, row, visit) {
|
|
451
462
|
if ((0, blob_1.isBlobField)(field)) {
|
|
452
|
-
return visit(value, label, row);
|
|
463
|
+
return visit(value, label, row, field);
|
|
453
464
|
}
|
|
454
465
|
if (field.type instanceof apache_arrow_1.Struct && isObject(value)) {
|
|
455
466
|
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
|
|
@@ -528,7 +539,7 @@ function valueAtPath(datum, path) {
|
|
|
528
539
|
function transposeData(data, field, path = []) {
|
|
529
540
|
const valuesPath = [...path, field.name];
|
|
530
541
|
if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
|
|
531
|
-
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
|
|
542
|
+
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath), field));
|
|
532
543
|
const childVectors = field.type.children.map((child) => {
|
|
533
544
|
const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
|
|
534
545
|
return makeVector(values, child.type, undefined, child.nullable);
|
package/dist/blob.d.ts
CHANGED
|
@@ -14,6 +14,9 @@ export type BlobUri = string | URL;
|
|
|
14
14
|
*
|
|
15
15
|
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
|
|
16
16
|
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
|
|
17
|
+
* External byte ranges require both
|
|
18
|
+
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
|
|
19
|
+
* `bigint` values for these unsigned 64-bit byte counts.
|
|
17
20
|
*
|
|
18
21
|
* @example
|
|
19
22
|
* ```ts
|
|
@@ -31,14 +34,25 @@ export type BlobUri = string | URL;
|
|
|
31
34
|
export type BlobInput = BlobData | BlobUri | {
|
|
32
35
|
data: BlobData;
|
|
33
36
|
uri?: null;
|
|
37
|
+
position?: null;
|
|
38
|
+
size?: null;
|
|
34
39
|
} | {
|
|
35
40
|
data?: null;
|
|
36
41
|
uri: BlobUri;
|
|
42
|
+
position?: null;
|
|
43
|
+
size?: null;
|
|
44
|
+
} | {
|
|
45
|
+
data?: null;
|
|
46
|
+
uri: BlobUri;
|
|
47
|
+
position: number | bigint;
|
|
48
|
+
size: number | bigint;
|
|
37
49
|
} | null | undefined;
|
|
38
50
|
/** A blob value normalized to the columns of the blob struct. */
|
|
39
51
|
export type BlobValue = {
|
|
40
52
|
data: Buffer | Uint8Array | null;
|
|
41
53
|
uri: string | null;
|
|
54
|
+
position?: bigint;
|
|
55
|
+
size?: bigint;
|
|
42
56
|
};
|
|
43
57
|
export type BlobOptions = {
|
|
44
58
|
/** Defaults to true. */
|
|
@@ -61,6 +75,15 @@ export type BlobOptions = {
|
|
|
61
75
|
};
|
|
62
76
|
/**
|
|
63
77
|
* Declares a `lance.blob.v2` column.
|
|
78
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
79
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
80
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
81
|
+
*
|
|
82
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
83
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
84
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
85
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
86
|
+
* are converted using the current table schema.
|
|
64
87
|
*
|
|
65
88
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
66
89
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
@@ -179,4 +202,4 @@ export declare function blobToRead(value: unknown): Blob | undefined;
|
|
|
179
202
|
* keeping the value's shape.
|
|
180
203
|
*/
|
|
181
204
|
export declare function withBlobBytes(value: unknown, bytes: Uint8Array): unknown;
|
|
182
|
-
export declare function coerceBlobValue(input: unknown): BlobValue | null;
|
|
205
|
+
export declare function coerceBlobValue(input: unknown, field?: Field): BlobValue | null;
|
package/dist/blob.js
CHANGED
|
@@ -18,6 +18,15 @@ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-thresho
|
|
|
18
18
|
const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
|
|
19
19
|
/**
|
|
20
20
|
* Declares a `lance.blob.v2` column.
|
|
21
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
22
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
23
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
24
|
+
*
|
|
25
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
26
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
27
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
28
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
29
|
+
* are converted using the current table schema.
|
|
21
30
|
*
|
|
22
31
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
23
32
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
@@ -62,6 +71,8 @@ function blob(name, options = {}) {
|
|
|
62
71
|
return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
|
|
63
72
|
new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
|
|
64
73
|
new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
|
|
74
|
+
new apache_arrow_1.Field("position", new apache_arrow_1.Uint64(), true),
|
|
75
|
+
new apache_arrow_1.Field("size", new apache_arrow_1.Uint64(), true),
|
|
65
76
|
]), options.nullable ?? true, metadata);
|
|
66
77
|
}
|
|
67
78
|
/**
|
|
@@ -190,6 +201,8 @@ function normalizeBlobInput(value) {
|
|
|
190
201
|
...value,
|
|
191
202
|
data: value.data,
|
|
192
203
|
uri: value.uri,
|
|
204
|
+
position: value.position,
|
|
205
|
+
size: value.size,
|
|
193
206
|
};
|
|
194
207
|
let changed = false;
|
|
195
208
|
if (value.data instanceof ArrayBuffer) {
|
|
@@ -226,9 +239,15 @@ function withBlobBytes(value, bytes) {
|
|
|
226
239
|
return bytes;
|
|
227
240
|
}
|
|
228
241
|
const record = value;
|
|
229
|
-
return {
|
|
242
|
+
return {
|
|
243
|
+
...record,
|
|
244
|
+
uri: record.uri,
|
|
245
|
+
position: record.position,
|
|
246
|
+
size: record.size,
|
|
247
|
+
data: bytes,
|
|
248
|
+
};
|
|
230
249
|
}
|
|
231
|
-
function coerceBlobValue(input) {
|
|
250
|
+
function coerceBlobValue(input, field) {
|
|
232
251
|
const value = normalizeBlobInput(input);
|
|
233
252
|
if (value == null) {
|
|
234
253
|
return null;
|
|
@@ -250,9 +269,9 @@ function coerceBlobValue(input) {
|
|
|
250
269
|
if (!("data" in record) && !("uri" in record)) {
|
|
251
270
|
throw new Error("Blob struct values must include a 'data' or 'uri' field");
|
|
252
271
|
}
|
|
253
|
-
const unknownKeys = Object.keys(record).filter((key) =>
|
|
272
|
+
const unknownKeys = Object.keys(record).filter((key) => !["data", "uri", "position", "size"].includes(key));
|
|
254
273
|
if (unknownKeys.length > 0) {
|
|
255
|
-
throw new Error(`Blob struct values only support 'data' and '
|
|
274
|
+
throw new Error(`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
|
|
256
275
|
.map((key) => `'${key}'`)
|
|
257
276
|
.join(", ")}`);
|
|
258
277
|
}
|
|
@@ -272,9 +291,41 @@ function coerceBlobValue(input) {
|
|
|
272
291
|
if ((bytes == null) === (uriValue == null)) {
|
|
273
292
|
throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
|
|
274
293
|
}
|
|
294
|
+
const hasPosition = record.position != null;
|
|
295
|
+
const hasSize = record.size != null;
|
|
296
|
+
if (hasPosition !== hasSize) {
|
|
297
|
+
throw new Error("External blob must set both position and size, or neither");
|
|
298
|
+
}
|
|
299
|
+
if (hasPosition) {
|
|
300
|
+
if (uriValue == null) {
|
|
301
|
+
throw new Error("External blob byte ranges require a uri");
|
|
302
|
+
}
|
|
303
|
+
const position = blobRangeInteger(record.position, "position", 0n);
|
|
304
|
+
const size = blobRangeInteger(record.size, "size", 1n);
|
|
305
|
+
if (position + size > 0xffffffffffffffffn) {
|
|
306
|
+
throw new Error("External blob range overflows uint64");
|
|
307
|
+
}
|
|
308
|
+
if (field !== undefined &&
|
|
309
|
+
(!(field.type instanceof apache_arrow_1.Struct) ||
|
|
310
|
+
["position", "size"].some((name) => !field.type.children.some((child) => child.name === name)))) {
|
|
311
|
+
throw new Error("Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size");
|
|
312
|
+
}
|
|
313
|
+
return { data: null, uri: uriValue, position, size };
|
|
314
|
+
}
|
|
275
315
|
return { data: bytes, uri: uriValue };
|
|
276
316
|
}
|
|
277
|
-
throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }");
|
|
317
|
+
throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }");
|
|
318
|
+
}
|
|
319
|
+
function blobRangeInteger(value, name, minimum) {
|
|
320
|
+
if (typeof value !== "bigint" &&
|
|
321
|
+
(typeof value !== "number" || !Number.isSafeInteger(value))) {
|
|
322
|
+
throw new Error(`Blob ${name} must be a safe integer or bigint`);
|
|
323
|
+
}
|
|
324
|
+
const integer = BigInt(value);
|
|
325
|
+
if (integer < minimum || integer > 0xffffffffffffffffn) {
|
|
326
|
+
throw new Error(`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`);
|
|
327
|
+
}
|
|
328
|
+
return integer;
|
|
278
329
|
}
|
|
279
330
|
function isBlobLike(value) {
|
|
280
331
|
return typeof Blob !== "undefined" && value instanceof Blob;
|
package/dist/catalog.d.ts
CHANGED
|
@@ -42,9 +42,14 @@ export declare class Catalog {
|
|
|
42
42
|
}): Promise<Connection>;
|
|
43
43
|
/** Connect to an existing database by its logical name. */
|
|
44
44
|
connectDatabase(name: string): Promise<Connection>;
|
|
45
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Drop a database. A nonempty database is an error unless `cascade` is set,
|
|
47
|
+
* which drops everything in it first; the name stays taken until the server
|
|
48
|
+
* finishes cleaning up their storage.
|
|
49
|
+
*/
|
|
46
50
|
dropDatabase(name: string, options?: {
|
|
47
51
|
ignoreMissing?: boolean;
|
|
52
|
+
cascade?: boolean;
|
|
48
53
|
}): Promise<void>;
|
|
49
54
|
/**
|
|
50
55
|
* Iterate lazily over all database names using `for await...of`.
|
package/dist/catalog.js
CHANGED
|
@@ -60,9 +60,13 @@ class Catalog {
|
|
|
60
60
|
async connectDatabase(name) {
|
|
61
61
|
return new connection_1.LocalConnection(await this.inner.connectDatabase(name));
|
|
62
62
|
}
|
|
63
|
-
/**
|
|
63
|
+
/**
|
|
64
|
+
* Drop a database. A nonempty database is an error unless `cascade` is set,
|
|
65
|
+
* which drops everything in it first; the name stays taken until the server
|
|
66
|
+
* finishes cleaning up their storage.
|
|
67
|
+
*/
|
|
64
68
|
async dropDatabase(name, options = {}) {
|
|
65
|
-
await this.inner.dropDatabase(name, options.ignoreMissing);
|
|
69
|
+
await this.inner.dropDatabase(name, options.ignoreMissing, options.cascade);
|
|
66
70
|
}
|
|
67
71
|
/**
|
|
68
72
|
* Iterate lazily over all database names using `for await...of`.
|
package/dist/connection.d.ts
CHANGED
|
@@ -4,9 +4,9 @@ import { Job } from "./job";
|
|
|
4
4
|
import { Listing, ListingOptions } from "./listing";
|
|
5
5
|
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
6
6
|
import { Connection as LanceDbConnection } from "./native";
|
|
7
|
-
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo
|
|
7
|
+
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo } from "./native";
|
|
8
8
|
import { ViewDescription } from "./view";
|
|
9
|
-
export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse,
|
|
9
|
+
export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, };
|
|
10
10
|
import { Table } from "./table";
|
|
11
11
|
export interface CreateTableOptions {
|
|
12
12
|
/**
|
|
@@ -17,8 +17,11 @@ export interface CreateTableOptions {
|
|
|
17
17
|
* happen. Any provided data will be ignored.
|
|
18
18
|
*
|
|
19
19
|
* If this is set to "overwrite" then any existing table will be replaced.
|
|
20
|
+
*
|
|
21
|
+
* If this is set to "exist_ok" then an existing table is opened and any
|
|
22
|
+
* provided data is ignored.
|
|
20
23
|
*/
|
|
21
|
-
mode: "create" | "overwrite";
|
|
24
|
+
mode: "create" | "overwrite" | "exist_ok";
|
|
22
25
|
/**
|
|
23
26
|
* If this is true and the table already exists and the mode is "create"
|
|
24
27
|
* then no error will be raised.
|
|
@@ -96,7 +99,7 @@ export interface OpenTableOptions {
|
|
|
96
99
|
indexCacheSize?: number;
|
|
97
100
|
}
|
|
98
101
|
/**
|
|
99
|
-
* @deprecated Use {@link
|
|
102
|
+
* @deprecated Use {@link ListingOptions} with {@link Connection.listTables}
|
|
100
103
|
* instead.
|
|
101
104
|
*/
|
|
102
105
|
export interface TableNamesOptions {
|
|
@@ -111,30 +114,6 @@ export interface TableNamesOptions {
|
|
|
111
114
|
/** The maximum number of names to return. Omitted returns all names; zero returns none. */
|
|
112
115
|
limit?: number;
|
|
113
116
|
}
|
|
114
|
-
export interface ListTablesOptions {
|
|
115
|
-
/**
|
|
116
|
-
* Token from a previous response, to resume listing where it left off.
|
|
117
|
-
*
|
|
118
|
-
* The token is opaque: it carries whatever the database needs to resume, and
|
|
119
|
-
* callers should not construct or interpret one.
|
|
120
|
-
*/
|
|
121
|
-
pageToken?: string;
|
|
122
|
-
/**
|
|
123
|
-
* An upper bound on how many tables to return.
|
|
124
|
-
* Defaults to 100. Zero returns an empty page without a continuation token.
|
|
125
|
-
*
|
|
126
|
-
* A page may hold fewer than this and still not be the last one, so keep
|
|
127
|
-
* going while the response carries a page token rather than while pages are
|
|
128
|
-
* full.
|
|
129
|
-
*/
|
|
130
|
-
limit?: number;
|
|
131
|
-
}
|
|
132
|
-
export interface ListNamespacesOptions {
|
|
133
|
-
/** Token from a previous response for pagination. */
|
|
134
|
-
pageToken?: string;
|
|
135
|
-
/** An optional limit to the number of results to return. */
|
|
136
|
-
limit?: number;
|
|
137
|
-
}
|
|
138
117
|
export interface CreateNamespaceOptions {
|
|
139
118
|
/** Creation mode. */
|
|
140
119
|
mode?: "create" | "exist_ok" | "overwrite";
|
|
@@ -216,40 +195,15 @@ export declare abstract class Connection {
|
|
|
216
195
|
* @deprecated Use {@link Connection.listTables} instead.
|
|
217
196
|
*/
|
|
218
197
|
abstract tableNames(namespacePath?: string[], options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
219
|
-
/**
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
* To retrieve the tables after the page, pass the `pageToken` the response
|
|
223
|
-
* carries back in. A page can be shorter than `limit` without being the last
|
|
224
|
-
* one, so walk until a response carries no page token:
|
|
225
|
-
*
|
|
198
|
+
/** Lazily list tables in a namespace, requesting 100 per page by default.
|
|
199
|
+
* @example
|
|
226
200
|
* ```ts
|
|
227
|
-
* const
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* const page = await conn.listTables({ pageToken, limit: 100 });
|
|
231
|
-
* names.push(...page.tables);
|
|
232
|
-
* pageToken = page.pageToken;
|
|
233
|
-
* } while (pageToken);
|
|
201
|
+
* for await (const name of conn.listTables([], { pageLimit: 20 })) {
|
|
202
|
+
* console.log(name);
|
|
203
|
+
* }
|
|
234
204
|
* ```
|
|
235
|
-
*
|
|
236
|
-
* @param {Partial<ListTablesOptions>} options - Pagination options
|
|
237
|
-
* (`pageToken`, `limit`).
|
|
238
|
-
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
|
239
|
-
* optional token for the tables after it.
|
|
240
|
-
*/
|
|
241
|
-
abstract listTables(options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
242
|
-
/**
|
|
243
|
-
* List a page of the tables in this database.
|
|
244
|
-
*
|
|
245
|
-
* @param {string[]} namespacePath - The namespace path to list tables from
|
|
246
|
-
* (defaults to root namespace)
|
|
247
|
-
* @param {Partial<ListTablesOptions>} options - Pagination options
|
|
248
|
-
* (`pageToken`, `limit`).
|
|
249
|
-
* @returns {Promise<ListTablesResponse>} A page of table names and an
|
|
250
|
-
* optional token for the tables after it.
|
|
251
205
|
*/
|
|
252
|
-
abstract listTables(namespacePath?: string[], options?:
|
|
206
|
+
abstract listTables(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
253
207
|
/**
|
|
254
208
|
* Open a table in the database.
|
|
255
209
|
* @param {string} name - The name of the table
|
|
@@ -259,6 +213,10 @@ export declare abstract class Connection {
|
|
|
259
213
|
/**
|
|
260
214
|
* Define a materialized view named `name` over the table `source`.
|
|
261
215
|
*
|
|
216
|
+
* Local creation and refresh require a source build with the
|
|
217
|
+
* `experimental-functions` Cargo feature, disabled in published packages.
|
|
218
|
+
* Remote Cloud/Enterprise operations do not require this feature.
|
|
219
|
+
*
|
|
262
220
|
* The view is populated before creation returns. Set `withNoData` to create
|
|
263
221
|
* only its definition and empty backing table. The view is a normal table:
|
|
264
222
|
* it can be queried, indexed and searched, and it appears in `tableNames`.
|
|
@@ -286,15 +244,16 @@ export declare abstract class Connection {
|
|
|
286
244
|
/**
|
|
287
245
|
* Drop the materialized view named `name`.
|
|
288
246
|
*
|
|
289
|
-
*
|
|
290
|
-
*
|
|
247
|
+
* Remote drops wait for the SQL command to remove the catalog binding.
|
|
248
|
+
* Physical cleanup continues in the background.
|
|
291
249
|
*
|
|
292
250
|
* Rejects a table that exists but is not a materialized view.
|
|
293
251
|
*/
|
|
294
252
|
abstract dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
|
|
295
253
|
/**
|
|
296
|
-
* Start dropping the materialized view named `name` and return its
|
|
297
|
-
*
|
|
254
|
+
* Start dropping the materialized view named `name` and return its job.
|
|
255
|
+
* Remote jobs have type `query` and use the SQL query ID as `id`. Waiting
|
|
256
|
+
* confirms catalog removal; physical cleanup continues in the background.
|
|
298
257
|
*
|
|
299
258
|
* Rejects a table that exists but is not a materialized view.
|
|
300
259
|
*/
|
|
@@ -410,20 +369,8 @@ export declare abstract class Connection {
|
|
|
410
369
|
* (may be undefined if the namespace has none).
|
|
411
370
|
*/
|
|
412
371
|
abstract describeNamespace(namespacePath: string[]): Promise<DescribeNamespaceResponse>;
|
|
413
|
-
/**
|
|
414
|
-
|
|
415
|
-
*
|
|
416
|
-
* Results may be paginated. To retrieve subsequent pages, pass the
|
|
417
|
-
* `pageToken` returned by a previous call.
|
|
418
|
-
*
|
|
419
|
-
* @param {string[]} namespacePath - The parent namespace path. Defaults
|
|
420
|
-
* to the root namespace if omitted.
|
|
421
|
-
* @param {Partial<ListNamespacesOptions>} options - Pagination options
|
|
422
|
-
* (`pageToken`, `limit`).
|
|
423
|
-
* @returns {Promise<ListNamespacesResponse>} Child namespace names and
|
|
424
|
-
* an optional token for fetching the next page.
|
|
425
|
-
*/
|
|
426
|
-
abstract listNamespaces(namespacePath?: string[], options?: Partial<ListNamespacesOptions>): Promise<ListNamespacesResponse>;
|
|
372
|
+
/** Lazily list immediate child namespaces, 100 per page by default. */
|
|
373
|
+
abstract listNamespaces(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
427
374
|
/**
|
|
428
375
|
* Create a new namespace at the given path.
|
|
429
376
|
*
|
|
@@ -548,7 +495,7 @@ export declare class LocalConnection extends Connection {
|
|
|
548
495
|
dropView(name: string, namespacePath?: string[]): Promise<void>;
|
|
549
496
|
dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
550
497
|
listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
551
|
-
listTables(
|
|
498
|
+
listTables(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
552
499
|
openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
553
500
|
cloneTable(targetTableName: string, sourceUri: string, options?: {
|
|
554
501
|
targetNamespacePath?: string[];
|
|
@@ -567,7 +514,7 @@ export declare class LocalConnection extends Connection {
|
|
|
567
514
|
dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
568
515
|
dropAllTables(namespacePath?: string[]): Promise<void>;
|
|
569
516
|
describeNamespace(namespacePath: string[]): Promise<DescribeNamespaceResponse>;
|
|
570
|
-
listNamespaces(namespacePath?: string[], options?:
|
|
517
|
+
listNamespaces(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
571
518
|
createNamespace(namespacePath: string[], options?: Partial<CreateNamespaceOptions>): Promise<CreateNamespaceResponse>;
|
|
572
519
|
dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
|
|
573
520
|
renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
package/dist/connection.js
CHANGED
|
@@ -67,9 +67,9 @@ class LocalConnection extends Connection {
|
|
|
67
67
|
else {
|
|
68
68
|
// First argument is options object (backwards compatibility)
|
|
69
69
|
namespacePath = undefined;
|
|
70
|
-
tableNamesOptions = namespacePathOrOptions;
|
|
70
|
+
tableNamesOptions = namespacePathOrOptions ?? options;
|
|
71
71
|
}
|
|
72
|
-
return this.inner.tableNames(namespacePath ?? [], tableNamesOptions?.startAfter, tableNamesOptions?.limit);
|
|
72
|
+
return this.inner.tableNames(namespacePath ?? [], tableNamesOptions?.startAfter, normalizeTableListingLimit(tableNamesOptions?.limit));
|
|
73
73
|
}
|
|
74
74
|
async createMaterializedView(name, source, options) {
|
|
75
75
|
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
@@ -106,15 +106,9 @@ class LocalConnection extends Connection {
|
|
|
106
106
|
(0, listing_1.validateListingOptions)(options);
|
|
107
107
|
return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
108
108
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
? namespacePathOrOptions
|
|
113
|
-
: undefined;
|
|
114
|
-
const listTablesOptions = Array.isArray(namespacePathOrOptions)
|
|
115
|
-
? options
|
|
116
|
-
: namespacePathOrOptions;
|
|
117
|
-
return this.inner.listTables(namespacePath ?? [], listTablesOptions?.pageToken, listTablesOptions?.limit);
|
|
109
|
+
listTables(namespacePath, options = {}) {
|
|
110
|
+
(0, listing_1.validateListingOptions)(options);
|
|
111
|
+
return new listing_1.Listing(this.inner.listTables(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
118
112
|
}
|
|
119
113
|
async openTable(name, namespacePath, options) {
|
|
120
114
|
const innerTable = await this.inner.openTable(name, namespacePath ?? [], cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
|
|
@@ -138,21 +132,22 @@ class LocalConnection extends Connection {
|
|
|
138
132
|
return new table_1.LocalTable(innerTable);
|
|
139
133
|
}
|
|
140
134
|
getStorageOptions(options) {
|
|
135
|
+
let storageOptions = options?.storageOptions;
|
|
141
136
|
if (options?.dataStorageVersion !== undefined) {
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
options.dataStorageVersion;
|
|
137
|
+
storageOptions = {
|
|
138
|
+
...storageOptions,
|
|
139
|
+
newTableDataStorageVersion: options.dataStorageVersion,
|
|
140
|
+
};
|
|
147
141
|
}
|
|
148
142
|
if (options?.enableV2ManifestPaths !== undefined) {
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
143
|
+
storageOptions = {
|
|
144
|
+
...storageOptions,
|
|
145
|
+
newTableEnableV2ManifestPaths: options.enableV2ManifestPaths
|
|
146
|
+
? "true"
|
|
147
|
+
: "false",
|
|
148
|
+
};
|
|
154
149
|
}
|
|
155
|
-
return cleanseStorageOptions(
|
|
150
|
+
return cleanseStorageOptions(storageOptions);
|
|
156
151
|
}
|
|
157
152
|
async createTable(nameOrOptions, dataOrNamespacePath, namespacePathOrOptions, options) {
|
|
158
153
|
if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
|
|
@@ -175,7 +170,7 @@ class LocalConnection extends Connection {
|
|
|
175
170
|
else {
|
|
176
171
|
// Third argument is options object (backwards compatibility)
|
|
177
172
|
namespacePath = undefined;
|
|
178
|
-
createOptions = namespacePathOrOptions;
|
|
173
|
+
createOptions = namespacePathOrOptions ?? options;
|
|
179
174
|
}
|
|
180
175
|
return this._createTableImpl(name, data, namespacePath, createOptions);
|
|
181
176
|
}
|
|
@@ -200,7 +195,7 @@ class LocalConnection extends Connection {
|
|
|
200
195
|
else {
|
|
201
196
|
// Third argument is options object (backwards compatibility)
|
|
202
197
|
namespacePath = undefined;
|
|
203
|
-
createOptions = namespacePathOrOptions;
|
|
198
|
+
createOptions = namespacePathOrOptions ?? options;
|
|
204
199
|
}
|
|
205
200
|
let mode = createOptions?.mode ?? "create";
|
|
206
201
|
const existOk = createOptions?.existOk ?? false;
|
|
@@ -231,8 +226,9 @@ class LocalConnection extends Connection {
|
|
|
231
226
|
describeNamespace(namespacePath) {
|
|
232
227
|
return this.inner.describeNamespace(namespacePath);
|
|
233
228
|
}
|
|
234
|
-
listNamespaces(namespacePath, options) {
|
|
235
|
-
|
|
229
|
+
listNamespaces(namespacePath, options = {}) {
|
|
230
|
+
(0, listing_1.validateListingOptions)(options);
|
|
231
|
+
return new listing_1.Listing(this.inner.listNamespaces(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
236
232
|
}
|
|
237
233
|
createNamespace(namespacePath, options) {
|
|
238
234
|
return this.inner.createNamespace(namespacePath, options?.mode, options?.properties);
|
|
@@ -261,6 +257,12 @@ class LocalConnection extends Connection {
|
|
|
261
257
|
}
|
|
262
258
|
}
|
|
263
259
|
exports.LocalConnection = LocalConnection;
|
|
260
|
+
function normalizeTableListingLimit(limit) {
|
|
261
|
+
(0, materialized_view_1.validateNonNegativeInteger)(limit, "limit");
|
|
262
|
+
// The native binding takes u32: larger JS numbers otherwise wrap to zero or
|
|
263
|
+
// a smaller page size. A listing limit is an upper bound, so saturate it.
|
|
264
|
+
return limit === undefined ? undefined : Math.min(limit, 0xffffffff);
|
|
265
|
+
}
|
|
264
266
|
/**
|
|
265
267
|
* Takes storage options and makes all the keys snake case.
|
|
266
268
|
*/
|
|
@@ -10,6 +10,7 @@ class TransformersEmbeddingFunction extends embedding_function_1.EmbeddingFuncti
|
|
|
10
10
|
#model;
|
|
11
11
|
#tokenizer;
|
|
12
12
|
#modelName;
|
|
13
|
+
#tokenizerName;
|
|
13
14
|
#initialized = false;
|
|
14
15
|
#tokenizerOptions;
|
|
15
16
|
#ndims;
|
|
@@ -25,6 +26,7 @@ class TransformersEmbeddingFunction extends embedding_function_1.EmbeddingFuncti
|
|
|
25
26
|
};
|
|
26
27
|
this.#ndims = options.ndims;
|
|
27
28
|
this.#modelName = modelName;
|
|
29
|
+
this.#tokenizerName = options.tokenizer ?? modelName;
|
|
28
30
|
}
|
|
29
31
|
async init() {
|
|
30
32
|
let transformers;
|
|
@@ -47,10 +49,10 @@ class TransformersEmbeddingFunction extends embedding_function_1.EmbeddingFuncti
|
|
|
47
49
|
throw new Error(`error loading model ${this.#modelName}. Make sure you are using a wasm compatible model.\nReason: ${e}`);
|
|
48
50
|
}
|
|
49
51
|
try {
|
|
50
|
-
this.#tokenizer = await transformers.AutoTokenizer.from_pretrained(this.#
|
|
52
|
+
this.#tokenizer = await transformers.AutoTokenizer.from_pretrained(this.#tokenizerName);
|
|
51
53
|
}
|
|
52
54
|
catch (e) {
|
|
53
|
-
throw new Error(`error loading tokenizer
|
|
55
|
+
throw new Error(`error loading tokenizer ${this.#tokenizerName}. Make sure you are using a wasm compatible model:\nReason: ${e}`);
|
|
54
56
|
}
|
|
55
57
|
this.#initialized = true;
|
|
56
58
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -11,12 +11,12 @@ export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatist
|
|
|
11
11
|
export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
12
12
|
export { blob, isBlobField, BlobFile } from "./blob";
|
|
13
13
|
export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, BlobUri, } from "./blob";
|
|
14
|
-
export { Connection, CreateTableOptions, TableNamesOptions,
|
|
14
|
+
export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, CreateNamespaceOptions, DropNamespaceOptions, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
15
15
|
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
16
16
|
export { Job, JobEventsOptions } from "./job";
|
|
17
17
|
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
18
18
|
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
|
|
19
|
-
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions,
|
|
19
|
+
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
20
20
|
export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
|
|
21
21
|
export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
|
|
22
22
|
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
|
package/dist/index.js
CHANGED
|
@@ -122,8 +122,10 @@ async function connect(uriOrOptions, optionsOrSession, sessionOrHeaderProvider,
|
|
|
122
122
|
if (!uri) {
|
|
123
123
|
throw new Error("uri is required");
|
|
124
124
|
}
|
|
125
|
-
finalOptions =
|
|
126
|
-
|
|
125
|
+
finalOptions = {
|
|
126
|
+
...finalOptions,
|
|
127
|
+
storageOptions: (0, connection_1.cleanseStorageOptions)(finalOptions.storageOptions),
|
|
128
|
+
};
|
|
127
129
|
// Create native header provider if one was provided
|
|
128
130
|
let nativeProvider;
|
|
129
131
|
if (finalHeaderProvider) {
|
|
@@ -176,9 +178,10 @@ async function connectNamespace(implName, configOrProperties, options) {
|
|
|
176
178
|
else {
|
|
177
179
|
properties = configOrProperties;
|
|
178
180
|
}
|
|
179
|
-
const finalOptions =
|
|
180
|
-
|
|
181
|
-
|
|
181
|
+
const finalOptions = {
|
|
182
|
+
...options,
|
|
183
|
+
storageOptions: (0, connection_1.cleanseStorageOptions)(options?.storageOptions),
|
|
184
|
+
};
|
|
182
185
|
const nativeConn = await native_js_1.Connection.newWithNamespace(implName, properties, finalOptions);
|
|
183
186
|
return new connection_1.LocalConnection(nativeConn);
|
|
184
187
|
}
|
package/dist/indices.d.ts
CHANGED
|
@@ -119,7 +119,8 @@ export interface IvfRqOptions {
|
|
|
119
119
|
*
|
|
120
120
|
* This value controls how much each residual component is compressed. The more
|
|
121
121
|
* bits, the more accurate the index will be but the slower search. Typical values
|
|
122
|
-
* are small integers
|
|
122
|
+
* are small integers. If unset, uses Lance's default (currently 5 bits per
|
|
123
|
+
* dimension). Set to 1 for maximum compression.
|
|
123
124
|
*/
|
|
124
125
|
numBits?: number;
|
|
125
126
|
/**
|