@lancedb/lancedb 0.40.0 → 0.41.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NODEJS_THIRD_PARTY_LICENSES.md +8 -0
- package/dist/arrow.d.ts +18 -1
- package/dist/arrow.js +194 -14
- package/dist/blob.d.ts +81 -2
- package/dist/blob.js +159 -6
- package/dist/catalog.d.ts +40 -6
- package/dist/catalog.js +58 -9
- package/dist/connection.d.ts +19 -13
- package/dist/connection.js +12 -7
- package/dist/index.d.ts +6 -5
- package/dist/index.js +4 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/materialized_view.d.ts +5 -12
- package/dist/materialized_view.js +6 -12
- package/dist/merge.d.ts +7 -7
- package/dist/merge.js +8 -8
- package/dist/native.d.ts +151 -114
- package/dist/native.js +55 -52
- package/dist/query.d.ts +94 -26
- package/dist/query.js +176 -52
- package/dist/sanitize.d.ts +19 -18
- package/dist/sanitize.js +74 -11
- package/dist/schema.js +31 -3
- package/dist/table.d.ts +65 -60
- package/dist/table.js +26 -20
- package/package.json +13 -9
package/dist/blob.js
CHANGED
|
@@ -6,6 +6,9 @@ exports.BlobFile = void 0;
|
|
|
6
6
|
exports.blob = blob;
|
|
7
7
|
exports.isBlobField = isBlobField;
|
|
8
8
|
exports.runWithSignal = runWithSignal;
|
|
9
|
+
exports.normalizeBlobInput = normalizeBlobInput;
|
|
10
|
+
exports.blobToRead = blobToRead;
|
|
11
|
+
exports.withBlobBytes = withBlobBytes;
|
|
9
12
|
exports.coerceBlobValue = coerceBlobValue;
|
|
10
13
|
const apache_arrow_1 = require("apache-arrow");
|
|
11
14
|
const native_1 = require("./native");
|
|
@@ -15,9 +18,20 @@ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-thresho
|
|
|
15
18
|
const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
|
|
16
19
|
/**
|
|
17
20
|
* Declares a `lance.blob.v2` column.
|
|
21
|
+
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
|
|
22
|
+
* whole external files, and external byte ranges. Set both `position` and
|
|
23
|
+
* positive `size` for a range, or omit both for a whole file.
|
|
24
|
+
*
|
|
25
|
+
* Existing two-field columns remain writable. Appends and merge inserts fill
|
|
26
|
+
* missing range fields with nulls for four-field targets and drop all-null
|
|
27
|
+
* range fields for two-field targets. Non-null ranges require a four-field
|
|
28
|
+
* target. Overwrite with Arrow input replaces the table's schema; row objects
|
|
29
|
+
* are converted using the current table schema.
|
|
18
30
|
*
|
|
19
31
|
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
20
32
|
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
33
|
+
* Blobs inside lists can be written and queried as descriptors, but cannot
|
|
34
|
+
* currently be fetched because a row ID does not identify a list element.
|
|
21
35
|
*
|
|
22
36
|
* @example
|
|
23
37
|
* ```ts
|
|
@@ -57,6 +71,8 @@ function blob(name, options = {}) {
|
|
|
57
71
|
return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
|
|
58
72
|
new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
|
|
59
73
|
new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
|
|
74
|
+
new apache_arrow_1.Field("position", new apache_arrow_1.Uint64(), true),
|
|
75
|
+
new apache_arrow_1.Field("size", new apache_arrow_1.Uint64(), true),
|
|
60
76
|
]), options.nullable ?? true, metadata);
|
|
61
77
|
}
|
|
62
78
|
/**
|
|
@@ -159,7 +175,80 @@ async function runWithSignal(signal, run) {
|
|
|
159
175
|
signal.removeEventListener("abort", onAbort);
|
|
160
176
|
}
|
|
161
177
|
}
|
|
162
|
-
|
|
178
|
+
/**
|
|
179
|
+
* Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
|
|
180
|
+
* `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
|
|
181
|
+
* values are returned unchanged and validated later by {@link coerceBlobValue}.
|
|
182
|
+
*
|
|
183
|
+
* Throws on `Blob` / `File`, which must be read into bytes first.
|
|
184
|
+
*/
|
|
185
|
+
function normalizeBlobInput(value) {
|
|
186
|
+
if (blobToRead(value) !== undefined) {
|
|
187
|
+
throw new Error("Blob and File values must be read asynchronously. Pass them to " +
|
|
188
|
+
"createTable, add, or mergeInsert, or convert them with " +
|
|
189
|
+
"`new Uint8Array(await value.arrayBuffer())`");
|
|
190
|
+
}
|
|
191
|
+
if (value instanceof ArrayBuffer) {
|
|
192
|
+
return new Uint8Array(value);
|
|
193
|
+
}
|
|
194
|
+
if (value instanceof URL) {
|
|
195
|
+
return value.href;
|
|
196
|
+
}
|
|
197
|
+
if (isBlobStruct(value)) {
|
|
198
|
+
// Read `data` and `uri` explicitly: spreading a class instance would drop
|
|
199
|
+
// fields defined as getters on its prototype.
|
|
200
|
+
const out = {
|
|
201
|
+
...value,
|
|
202
|
+
data: value.data,
|
|
203
|
+
uri: value.uri,
|
|
204
|
+
position: value.position,
|
|
205
|
+
size: value.size,
|
|
206
|
+
};
|
|
207
|
+
let changed = false;
|
|
208
|
+
if (value.data instanceof ArrayBuffer) {
|
|
209
|
+
out.data = new Uint8Array(value.data);
|
|
210
|
+
changed = true;
|
|
211
|
+
}
|
|
212
|
+
if (value.uri instanceof URL) {
|
|
213
|
+
out.uri = value.uri.href;
|
|
214
|
+
changed = true;
|
|
215
|
+
}
|
|
216
|
+
return changed ? out : value;
|
|
217
|
+
}
|
|
218
|
+
return value;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Returns the `Blob` / `File` a blob value needs read, either the value itself
|
|
222
|
+
* or its `data` field, or `undefined` when there is nothing to read.
|
|
223
|
+
*/
|
|
224
|
+
function blobToRead(value) {
|
|
225
|
+
if (isBlobLike(value)) {
|
|
226
|
+
return value;
|
|
227
|
+
}
|
|
228
|
+
if (isBlobStruct(value) && isBlobLike(value.data)) {
|
|
229
|
+
return value.data;
|
|
230
|
+
}
|
|
231
|
+
return undefined;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
|
|
235
|
+
* keeping the value's shape.
|
|
236
|
+
*/
|
|
237
|
+
function withBlobBytes(value, bytes) {
|
|
238
|
+
if (isBlobLike(value)) {
|
|
239
|
+
return bytes;
|
|
240
|
+
}
|
|
241
|
+
const record = value;
|
|
242
|
+
return {
|
|
243
|
+
...record,
|
|
244
|
+
uri: record.uri,
|
|
245
|
+
position: record.position,
|
|
246
|
+
size: record.size,
|
|
247
|
+
data: bytes,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
function coerceBlobValue(input, field) {
|
|
251
|
+
const value = normalizeBlobInput(input);
|
|
163
252
|
if (value == null) {
|
|
164
253
|
return null;
|
|
165
254
|
}
|
|
@@ -167,7 +256,7 @@ function coerceBlobValue(value) {
|
|
|
167
256
|
return { data: value, uri: null };
|
|
168
257
|
}
|
|
169
258
|
if (ArrayBuffer.isView(value)) {
|
|
170
|
-
throw new Error("Blob data must be Buffer or
|
|
259
|
+
throw new Error("Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob");
|
|
171
260
|
}
|
|
172
261
|
if (typeof value === "string") {
|
|
173
262
|
if (value === "") {
|
|
@@ -175,30 +264,94 @@ function coerceBlobValue(value) {
|
|
|
175
264
|
}
|
|
176
265
|
return { data: null, uri: value };
|
|
177
266
|
}
|
|
178
|
-
if (
|
|
267
|
+
if (isBlobStruct(value)) {
|
|
179
268
|
const record = value;
|
|
180
269
|
if (!("data" in record) && !("uri" in record)) {
|
|
181
270
|
throw new Error("Blob struct values must include a 'data' or 'uri' field");
|
|
182
271
|
}
|
|
272
|
+
const unknownKeys = Object.keys(record).filter((key) => !["data", "uri", "position", "size"].includes(key));
|
|
273
|
+
if (unknownKeys.length > 0) {
|
|
274
|
+
throw new Error(`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
|
|
275
|
+
.map((key) => `'${key}'`)
|
|
276
|
+
.join(", ")}`);
|
|
277
|
+
}
|
|
183
278
|
const uri = record.uri;
|
|
184
279
|
if (uri === "") {
|
|
185
280
|
throw new Error("Blob uri cannot be empty");
|
|
186
281
|
}
|
|
187
282
|
if (uri != null && typeof uri !== "string") {
|
|
188
|
-
throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
|
|
283
|
+
throw new Error(`Blob uri must be a string, URL, or null, got ${typeof uri}`);
|
|
189
284
|
}
|
|
190
285
|
const data = record.data;
|
|
191
286
|
if (data != null && !isBlobBytes(data)) {
|
|
192
|
-
throw new Error("Blob data must be Buffer, Uint8Array, or null");
|
|
287
|
+
throw new Error("Blob data must be Buffer, Uint8Array, ArrayBuffer, Blob, or null");
|
|
193
288
|
}
|
|
194
289
|
const bytes = data ?? null;
|
|
195
290
|
const uriValue = uri ?? null;
|
|
196
291
|
if ((bytes == null) === (uriValue == null)) {
|
|
197
292
|
throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
|
|
198
293
|
}
|
|
294
|
+
const hasPosition = record.position != null;
|
|
295
|
+
const hasSize = record.size != null;
|
|
296
|
+
if (hasPosition !== hasSize) {
|
|
297
|
+
throw new Error("External blob must set both position and size, or neither");
|
|
298
|
+
}
|
|
299
|
+
if (hasPosition) {
|
|
300
|
+
if (uriValue == null) {
|
|
301
|
+
throw new Error("External blob byte ranges require a uri");
|
|
302
|
+
}
|
|
303
|
+
const position = blobRangeInteger(record.position, "position", 0n);
|
|
304
|
+
const size = blobRangeInteger(record.size, "size", 1n);
|
|
305
|
+
if (position + size > 0xffffffffffffffffn) {
|
|
306
|
+
throw new Error("External blob range overflows uint64");
|
|
307
|
+
}
|
|
308
|
+
if (field !== undefined &&
|
|
309
|
+
(!(field.type instanceof apache_arrow_1.Struct) ||
|
|
310
|
+
["position", "size"].some((name) => !field.type.children.some((child) => child.name === name)))) {
|
|
311
|
+
throw new Error("Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size");
|
|
312
|
+
}
|
|
313
|
+
return { data: null, uri: uriValue, position, size };
|
|
314
|
+
}
|
|
199
315
|
return { data: bytes, uri: uriValue };
|
|
200
316
|
}
|
|
201
|
-
throw new Error("Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }");
|
|
317
|
+
throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }");
|
|
318
|
+
}
|
|
319
|
+
function blobRangeInteger(value, name, minimum) {
|
|
320
|
+
if (typeof value !== "bigint" &&
|
|
321
|
+
(typeof value !== "number" || !Number.isSafeInteger(value))) {
|
|
322
|
+
throw new Error(`Blob ${name} must be a safe integer or bigint`);
|
|
323
|
+
}
|
|
324
|
+
const integer = BigInt(value);
|
|
325
|
+
if (integer < minimum || integer > 0xffffffffffffffffn) {
|
|
326
|
+
throw new Error(`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`);
|
|
327
|
+
}
|
|
328
|
+
return integer;
|
|
329
|
+
}
|
|
330
|
+
function isBlobLike(value) {
|
|
331
|
+
return typeof Blob !== "undefined" && value instanceof Blob;
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Whether `value` is a `{ data?, uri? }` struct: any plain object, or any
|
|
335
|
+
* other object with a `data` or `uri` field (such as a class instance), apart
|
|
336
|
+
* from the binary, `Blob`, and `URL` values that are blob inputs themselves.
|
|
337
|
+
*/
|
|
338
|
+
function isBlobStruct(value) {
|
|
339
|
+
if (typeof value !== "object" ||
|
|
340
|
+
value === null ||
|
|
341
|
+
Array.isArray(value) ||
|
|
342
|
+
ArrayBuffer.isView(value) ||
|
|
343
|
+
value instanceof ArrayBuffer ||
|
|
344
|
+
(typeof SharedArrayBuffer !== "undefined" &&
|
|
345
|
+
value instanceof SharedArrayBuffer) ||
|
|
346
|
+
isBlobLike(value) ||
|
|
347
|
+
value instanceof URL) {
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
const prototype = Object.getPrototypeOf(value);
|
|
351
|
+
return (prototype === Object.prototype ||
|
|
352
|
+
prototype === null ||
|
|
353
|
+
"data" in value ||
|
|
354
|
+
"uri" in value);
|
|
202
355
|
}
|
|
203
356
|
function isBlobBytes(value) {
|
|
204
357
|
return Buffer.isBuffer(value) || value instanceof Uint8Array;
|
package/dist/catalog.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Connection } from "./connection";
|
|
2
2
|
import { HeaderProvider } from "./header";
|
|
3
|
-
import {
|
|
3
|
+
import { Catalog as NativeCatalog, CatalogOptions as NativeCatalogOptions, DatabaseNames as NativeDatabaseNames } from "./native.js";
|
|
4
4
|
import { OAuthConfig } from "./oauth";
|
|
5
5
|
/** Options shared by a catalog and the database connections it returns. */
|
|
6
6
|
export interface CatalogOptions extends Omit<NativeCatalogOptions, "oauthConfig"> {
|
|
@@ -8,7 +8,27 @@ export interface CatalogOptions extends Omit<NativeCatalogOptions, "oauthConfig"
|
|
|
8
8
|
/** Called for each request to supply authentication headers. */
|
|
9
9
|
headerProvider?: HeaderProvider | (() => Record<string, string> | Promise<Record<string, string>>);
|
|
10
10
|
}
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* A lazy async iterator of database names with pagination state.
|
|
13
|
+
* Returned by {@link Catalog.listDatabases}.
|
|
14
|
+
*/
|
|
15
|
+
export declare class DatabaseNames implements AsyncIterableIterator<string> {
|
|
16
|
+
private readonly inner;
|
|
17
|
+
/** @hidden */
|
|
18
|
+
constructor(inner: NativeDatabaseNames);
|
|
19
|
+
[Symbol.asyncIterator](): AsyncIterableIterator<string>;
|
|
20
|
+
/** Fetch the next name, requesting another page only when needed. */
|
|
21
|
+
next(): Promise<IteratorResult<string>>;
|
|
22
|
+
/** Number of names available without another REST request. */
|
|
23
|
+
numPageResults(): number;
|
|
24
|
+
/**
|
|
25
|
+
* Token for the next REST request. Initially this is the supplied starting token.
|
|
26
|
+
* Returns undefined after the final page is fetched, even if names remain cached.
|
|
27
|
+
* Drain the cache before saving a token to avoid skipping names when resuming.
|
|
28
|
+
* A failed request terminates iteration and retains its token for a new iterator.
|
|
29
|
+
*/
|
|
30
|
+
pageToken(): string | undefined;
|
|
31
|
+
}
|
|
12
32
|
/** A remote catalog manages databases through the server's root namespace. */
|
|
13
33
|
export declare class Catalog {
|
|
14
34
|
private readonly inner;
|
|
@@ -26,11 +46,23 @@ export declare class Catalog {
|
|
|
26
46
|
dropDatabase(name: string, options?: {
|
|
27
47
|
ignoreMissing?: boolean;
|
|
28
48
|
}): Promise<void>;
|
|
29
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Iterate lazily over all database names using `for await...of`.
|
|
51
|
+
* pageToken resumes from a saved token; omitted starts at the beginning.
|
|
52
|
+
* pageLimit limits each REST response, not the total; omitted uses the server default.
|
|
53
|
+
* Request errors are raised during iteration and terminate the iterator.
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```ts
|
|
57
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
58
|
+
* console.log(name);
|
|
59
|
+
* }
|
|
60
|
+
* ```
|
|
61
|
+
*/
|
|
30
62
|
listDatabases(options?: {
|
|
31
|
-
|
|
63
|
+
pageLimit?: number;
|
|
32
64
|
pageToken?: string;
|
|
33
|
-
}):
|
|
65
|
+
}): DatabaseNames;
|
|
34
66
|
}
|
|
35
67
|
/**
|
|
36
68
|
* Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
|
|
@@ -40,7 +72,9 @@ export declare class Catalog {
|
|
|
40
72
|
* ```ts
|
|
41
73
|
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
|
|
42
74
|
* const db = await catalog.createDatabase("analytics", { existOk: true });
|
|
43
|
-
* const
|
|
75
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
76
|
+
* console.log(name);
|
|
77
|
+
* }
|
|
44
78
|
* ```
|
|
45
79
|
*/
|
|
46
80
|
export declare function connectCatalog(endpoint: string, options?: CatalogOptions): Promise<Catalog>;
|
package/dist/catalog.js
CHANGED
|
@@ -2,10 +2,45 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Catalog = void 0;
|
|
5
|
+
exports.Catalog = exports.DatabaseNames = void 0;
|
|
6
6
|
exports.connectCatalog = connectCatalog;
|
|
7
7
|
const connection_1 = require("./connection");
|
|
8
8
|
const native_js_1 = require("./native.js");
|
|
9
|
+
/**
|
|
10
|
+
* A lazy async iterator of database names with pagination state.
|
|
11
|
+
* Returned by {@link Catalog.listDatabases}.
|
|
12
|
+
*/
|
|
13
|
+
class DatabaseNames {
|
|
14
|
+
inner;
|
|
15
|
+
/** @hidden */
|
|
16
|
+
constructor(inner) {
|
|
17
|
+
this.inner = inner;
|
|
18
|
+
}
|
|
19
|
+
[Symbol.asyncIterator]() {
|
|
20
|
+
return this;
|
|
21
|
+
}
|
|
22
|
+
/** Fetch the next name, requesting another page only when needed. */
|
|
23
|
+
async next() {
|
|
24
|
+
const value = await this.inner.next();
|
|
25
|
+
return value == null
|
|
26
|
+
? { done: true, value: undefined }
|
|
27
|
+
: { done: false, value };
|
|
28
|
+
}
|
|
29
|
+
/** Number of names available without another REST request. */
|
|
30
|
+
numPageResults() {
|
|
31
|
+
return this.inner.numPageResults();
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Token for the next REST request. Initially this is the supplied starting token.
|
|
35
|
+
* Returns undefined after the final page is fetched, even if names remain cached.
|
|
36
|
+
* Drain the cache before saving a token to avoid skipping names when resuming.
|
|
37
|
+
* A failed request terminates iteration and retains its token for a new iterator.
|
|
38
|
+
*/
|
|
39
|
+
pageToken() {
|
|
40
|
+
return this.inner.pageToken() ?? undefined;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.DatabaseNames = DatabaseNames;
|
|
9
44
|
/** A remote catalog manages databases through the server's root namespace. */
|
|
10
45
|
class Catalog {
|
|
11
46
|
inner;
|
|
@@ -29,15 +64,27 @@ class Catalog {
|
|
|
29
64
|
async dropDatabase(name, options = {}) {
|
|
30
65
|
await this.inner.dropDatabase(name, options.ignoreMissing);
|
|
31
66
|
}
|
|
32
|
-
/**
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
67
|
+
/**
|
|
68
|
+
* Iterate lazily over all database names using `for await...of`.
|
|
69
|
+
* pageToken resumes from a saved token; omitted starts at the beginning.
|
|
70
|
+
* pageLimit limits each REST response, not the total; omitted uses the server default.
|
|
71
|
+
* Request errors are raised during iteration and terminate the iterator.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
76
|
+
* console.log(name);
|
|
77
|
+
* }
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
listDatabases(options = {}) {
|
|
81
|
+
if (options.pageLimit !== undefined &&
|
|
82
|
+
(!Number.isInteger(options.pageLimit) ||
|
|
83
|
+
options.pageLimit <= 0 ||
|
|
84
|
+
options.pageLimit > 2147483647)) {
|
|
38
85
|
throw new Error("Database list limit must be an integer between 1 and 2147483647");
|
|
39
86
|
}
|
|
40
|
-
return this.inner.listDatabases(options.
|
|
87
|
+
return new DatabaseNames(this.inner.listDatabases(options.pageToken, options.pageLimit));
|
|
41
88
|
}
|
|
42
89
|
}
|
|
43
90
|
exports.Catalog = Catalog;
|
|
@@ -49,7 +96,9 @@ exports.Catalog = Catalog;
|
|
|
49
96
|
* ```ts
|
|
50
97
|
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
|
|
51
98
|
* const db = await catalog.createDatabase("analytics", { existOk: true });
|
|
52
|
-
* const
|
|
99
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
100
|
+
* console.log(name);
|
|
101
|
+
* }
|
|
53
102
|
* ```
|
|
54
103
|
*/
|
|
55
104
|
async function connectCatalog(endpoint, options = {}) {
|
package/dist/connection.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
2
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
3
3
|
import { Job } from "./job";
|
|
4
|
+
import { Listing, ListingOptions } from "./listing";
|
|
4
5
|
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
5
6
|
import { Connection as LanceDbConnection } from "./native";
|
|
6
7
|
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
|
|
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
|
|
|
107
108
|
* the last table name from the previous page.
|
|
108
109
|
*/
|
|
109
110
|
startAfter?: string;
|
|
110
|
-
/**
|
|
111
|
+
/** The maximum number of names to return. Omitted returns all names; zero returns none. */
|
|
111
112
|
limit?: number;
|
|
112
113
|
}
|
|
113
114
|
export interface ListTablesOptions {
|
|
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
|
|
|
120
121
|
pageToken?: string;
|
|
121
122
|
/**
|
|
122
123
|
* An upper bound on how many tables to return.
|
|
124
|
+
* Defaults to 100. Zero returns an empty page without a continuation token.
|
|
123
125
|
*
|
|
124
126
|
* A page may hold fewer than this and still not be the last one, so keep
|
|
125
127
|
* going while the response carries a page token rather than while pages are
|
|
@@ -260,10 +262,6 @@ export declare abstract class Connection {
|
|
|
260
262
|
* The view is populated before creation returns. Set `withNoData` to create
|
|
261
263
|
* only its definition and empty backing table. The view is a normal table:
|
|
262
264
|
* it can be queried, indexed and searched, and it appears in `tableNames`.
|
|
263
|
-
* The source table must have stable row ids (create it with
|
|
264
|
-
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
|
265
|
-
* provenance valid across source compactions and cannot be enabled after
|
|
266
|
-
* a table exists.
|
|
267
265
|
*/
|
|
268
266
|
abstract createMaterializedView(name: string, source: string, options?: {
|
|
269
267
|
select?: MaterializedViewSelect;
|
|
@@ -280,9 +278,11 @@ export declare abstract class Connection {
|
|
|
280
278
|
/**
|
|
281
279
|
* The names of the materialized views in this database.
|
|
282
280
|
*
|
|
283
|
-
*
|
|
281
|
+
* Iterate lazily with `for await...of`. Local listings inspect table schemas
|
|
282
|
+
* one page at a time. Options control page size and the starting token.
|
|
283
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
284
284
|
*/
|
|
285
|
-
abstract listMaterializedViews():
|
|
285
|
+
abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
286
286
|
/**
|
|
287
287
|
* Drop the materialized view named `name`.
|
|
288
288
|
*
|
|
@@ -334,8 +334,10 @@ export declare abstract class Connection {
|
|
|
334
334
|
* The names of the views in one namespace.
|
|
335
335
|
*
|
|
336
336
|
* Names only; a definition comes from {@link describeView}.
|
|
337
|
+
* Iterate lazily with `for await...of`. See {@link Listing} for pagination
|
|
338
|
+
* state and error behavior; options control page size and the starting token.
|
|
337
339
|
*/
|
|
338
|
-
abstract listViews(namespacePath?: string[]):
|
|
340
|
+
abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
339
341
|
abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
340
342
|
/**
|
|
341
343
|
* Creates a new Table and initialize it with new data.
|
|
@@ -492,8 +494,12 @@ export declare abstract class Connection {
|
|
|
492
494
|
* connection-level call for any of them.
|
|
493
495
|
*/
|
|
494
496
|
abstract openJob(jobId: string): Promise<Job>;
|
|
495
|
-
/**
|
|
496
|
-
|
|
497
|
+
/**
|
|
498
|
+
* Iterate lazily over server-side jobs across the database's tables.
|
|
499
|
+
* Use `for await...of`. Options control page size and the starting token.
|
|
500
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
501
|
+
*/
|
|
502
|
+
abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
497
503
|
/**
|
|
498
504
|
* Request cancellation of a server-side job by id.
|
|
499
505
|
*
|
|
@@ -534,14 +540,14 @@ export declare class LocalConnection extends Connection {
|
|
|
534
540
|
withNoData?: boolean;
|
|
535
541
|
}): Promise<MaterializedView>;
|
|
536
542
|
openMaterializedView(name: string): Promise<MaterializedView>;
|
|
537
|
-
listMaterializedViews():
|
|
543
|
+
listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
538
544
|
dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
|
|
539
545
|
dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
540
546
|
createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
541
547
|
describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
542
548
|
dropView(name: string, namespacePath?: string[]): Promise<void>;
|
|
543
549
|
dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
544
|
-
listViews(namespacePath?: string[]):
|
|
550
|
+
listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
545
551
|
listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
546
552
|
openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
547
553
|
cloneTable(targetTableName: string, sourceUri: string, options?: {
|
|
@@ -566,7 +572,7 @@ export declare class LocalConnection extends Connection {
|
|
|
566
572
|
dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
|
|
567
573
|
renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
|
568
574
|
openJob(jobId: string): Promise<Job>;
|
|
569
|
-
listJobs():
|
|
575
|
+
listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
570
576
|
cancelJob(jobId: string): Promise<boolean>;
|
|
571
577
|
pauseJob(jobId: string): Promise<string>;
|
|
572
578
|
resumeJob(jobId: string): Promise<string>;
|
package/dist/connection.js
CHANGED
|
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
|
|
|
8
8
|
const arrow_2 = require("./arrow");
|
|
9
9
|
const registry_1 = require("./embedding/registry");
|
|
10
10
|
const job_1 = require("./job");
|
|
11
|
+
const listing_1 = require("./listing");
|
|
11
12
|
const materialized_view_1 = require("./materialized_view");
|
|
12
13
|
const view_1 = require("./view");
|
|
13
14
|
const sanitize_1 = require("./sanitize");
|
|
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
|
|
|
79
80
|
const innerTable = await this.inner.openMaterializedView(name);
|
|
80
81
|
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
81
82
|
}
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
listMaterializedViews(options = {}) {
|
|
84
|
+
(0, listing_1.validateListingOptions)(options);
|
|
85
|
+
return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
|
|
84
86
|
}
|
|
85
87
|
async dropMaterializedView(name, namespacePath) {
|
|
86
88
|
return this.inner.dropMaterializedView(name, namespacePath ?? []);
|
|
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
|
|
|
100
102
|
async dropViewAsync(name, namespacePath) {
|
|
101
103
|
return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
|
|
102
104
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
listViews(namespacePath, options = {}) {
|
|
106
|
+
(0, listing_1.validateListingOptions)(options);
|
|
107
|
+
return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
105
108
|
}
|
|
106
109
|
async listTables(namespacePathOrOptions, options) {
|
|
107
110
|
// Detect if first argument is namespacePath array or options object
|
|
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
|
|
|
243
246
|
async openJob(jobId) {
|
|
244
247
|
return new job_1.Job(await this.inner.openJob(jobId));
|
|
245
248
|
}
|
|
246
|
-
|
|
247
|
-
|
|
249
|
+
listJobs(options = {}) {
|
|
250
|
+
(0, listing_1.validateListingOptions)(options);
|
|
251
|
+
return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
|
|
248
252
|
}
|
|
249
253
|
async cancelJob(jobId) {
|
|
250
254
|
return this.inner.cancelJob(jobId);
|
|
@@ -303,7 +307,8 @@ async function parseTableData(data, options, streaming = false) {
|
|
|
303
307
|
table = (0, sanitize_1.sanitizeTable)(data);
|
|
304
308
|
}
|
|
305
309
|
else {
|
|
306
|
-
|
|
310
|
+
const records = await (0, arrow_1.resolveBlobInputs)(data, options?.schema);
|
|
311
|
+
table = (0, arrow_1.makeArrowTable)(records, options);
|
|
307
312
|
}
|
|
308
313
|
if (streaming) {
|
|
309
314
|
const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
|
package/dist/index.d.ts
CHANGED
|
@@ -10,13 +10,13 @@ export { instrumentLanceDbMetrics } from "./otel";
|
|
|
10
10
|
export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
|
|
11
11
|
export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
12
12
|
export { blob, isBlobField, BlobFile } from "./blob";
|
|
13
|
-
export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
|
|
13
|
+
export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, BlobUri, } from "./blob";
|
|
14
14
|
export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
15
15
|
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
16
16
|
export { Job, JobEventsOptions } from "./job";
|
|
17
|
-
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
17
|
+
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
18
18
|
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
|
|
19
|
-
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions,
|
|
19
|
+
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
20
20
|
export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
|
|
21
21
|
export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
|
|
22
22
|
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
|
|
@@ -24,7 +24,7 @@ export * as embedding from "./embedding";
|
|
|
24
24
|
export { permutationBuilder, PermutationBuilder } from "./permutation";
|
|
25
25
|
export { Scannable, ScannableOptions } from "./scannable";
|
|
26
26
|
export * as rerankers from "./rerankers";
|
|
27
|
-
export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
|
|
27
|
+
export { SchemaLike, TableLike, DataTypeLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
|
|
28
28
|
export { IntoSql, packBits } from "./util";
|
|
29
29
|
/**
|
|
30
30
|
* Options for tokenizing a full-text search query without a table index.
|
|
@@ -248,4 +248,5 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
|
|
|
248
248
|
* ```
|
|
249
249
|
*/
|
|
250
250
|
export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
|
|
251
|
-
export { Catalog, CatalogOptions,
|
|
251
|
+
export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
|
|
252
|
+
export { Listing, ListingOptions } from "./listing";
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.connectCatalog = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
5
|
+
exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
6
6
|
exports.tokenize = tokenize;
|
|
7
7
|
exports.connect = connect;
|
|
8
8
|
exports.connectNamespace = connectNamespace;
|
|
@@ -184,4 +184,7 @@ async function connectNamespace(implName, configOrProperties, options) {
|
|
|
184
184
|
}
|
|
185
185
|
var catalog_1 = require("./catalog");
|
|
186
186
|
Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
|
|
187
|
+
Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
|
|
187
188
|
Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
|
|
189
|
+
var listing_1 = require("./listing");
|
|
190
|
+
Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
|
package/dist/indices.d.ts
CHANGED
|
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
|
|
|
453
453
|
*/
|
|
454
454
|
sampleRate?: number;
|
|
455
455
|
}
|
|
456
|
-
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
456
|
+
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
457
457
|
/**
|
|
458
458
|
* Options to create a full text search index
|
|
459
459
|
*/
|
|
@@ -477,6 +477,8 @@ export interface FtsOptions {
|
|
|
477
477
|
*
|
|
478
478
|
* "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
|
|
479
479
|
*
|
|
480
|
+
* "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
|
|
481
|
+
*
|
|
480
482
|
* "icu" - ICU dictionary-based word segmentation.
|
|
481
483
|
*
|
|
482
484
|
* "icu/split" - ICU segmentation with simple-style delimiter splitting.
|