@lancedb/lancedb 0.40.0 → 0.41.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/blob.js CHANGED
@@ -6,6 +6,9 @@ exports.BlobFile = void 0;
6
6
  exports.blob = blob;
7
7
  exports.isBlobField = isBlobField;
8
8
  exports.runWithSignal = runWithSignal;
9
+ exports.normalizeBlobInput = normalizeBlobInput;
10
+ exports.blobToRead = blobToRead;
11
+ exports.withBlobBytes = withBlobBytes;
9
12
  exports.coerceBlobValue = coerceBlobValue;
10
13
  const apache_arrow_1 = require("apache-arrow");
11
14
  const native_1 = require("./native");
@@ -15,9 +18,20 @@ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-thresho
15
18
  const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
16
19
  /**
17
20
  * Declares a `lance.blob.v2` column.
21
+ * Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
22
+ * whole external files, and external byte ranges. Set both `position` and
23
+ * positive `size` for a range, or omit both for a whole file.
24
+ *
25
+ * Existing two-field columns remain writable. Appends and merge inserts fill
26
+ * missing range fields with nulls for four-field targets and drop all-null
27
+ * range fields for two-field targets. Non-null ranges require a four-field
28
+ * target. Overwrite with Arrow input replaces the table's schema; row objects
29
+ * are converted using the current table schema.
18
30
  *
19
31
  * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
20
32
  * or {@link Table.fetchBlobFiles} to read bytes.
33
+ * Blobs inside lists can be written and queried as descriptors, but cannot
34
+ * currently be fetched because a row ID does not identify a list element.
21
35
  *
22
36
  * @example
23
37
  * ```ts
@@ -57,6 +71,8 @@ function blob(name, options = {}) {
57
71
  return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
58
72
  new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
59
73
  new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
74
+ new apache_arrow_1.Field("position", new apache_arrow_1.Uint64(), true),
75
+ new apache_arrow_1.Field("size", new apache_arrow_1.Uint64(), true),
60
76
  ]), options.nullable ?? true, metadata);
61
77
  }
62
78
  /**
@@ -159,7 +175,80 @@ async function runWithSignal(signal, run) {
159
175
  signal.removeEventListener("abort", onAbort);
160
176
  }
161
177
  }
162
- function coerceBlobValue(value) {
178
+ /**
179
+ * Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
180
+ * `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
181
+ * values are returned unchanged and validated later by {@link coerceBlobValue}.
182
+ *
183
+ * Throws on `Blob` / `File`, which must be read into bytes first.
184
+ */
185
+ function normalizeBlobInput(value) {
186
+ if (blobToRead(value) !== undefined) {
187
+ throw new Error("Blob and File values must be read asynchronously. Pass them to " +
188
+ "createTable, add, or mergeInsert, or convert them with " +
189
+ "`new Uint8Array(await value.arrayBuffer())`");
190
+ }
191
+ if (value instanceof ArrayBuffer) {
192
+ return new Uint8Array(value);
193
+ }
194
+ if (value instanceof URL) {
195
+ return value.href;
196
+ }
197
+ if (isBlobStruct(value)) {
198
+ // Read `data` and `uri` explicitly: spreading a class instance would drop
199
+ // fields defined as getters on its prototype.
200
+ const out = {
201
+ ...value,
202
+ data: value.data,
203
+ uri: value.uri,
204
+ position: value.position,
205
+ size: value.size,
206
+ };
207
+ let changed = false;
208
+ if (value.data instanceof ArrayBuffer) {
209
+ out.data = new Uint8Array(value.data);
210
+ changed = true;
211
+ }
212
+ if (value.uri instanceof URL) {
213
+ out.uri = value.uri.href;
214
+ changed = true;
215
+ }
216
+ return changed ? out : value;
217
+ }
218
+ return value;
219
+ }
220
+ /**
221
+ * Returns the `Blob` / `File` a blob value needs read, either the value itself
222
+ * or its `data` field, or `undefined` when there is nothing to read.
223
+ */
224
+ function blobToRead(value) {
225
+ if (isBlobLike(value)) {
226
+ return value;
227
+ }
228
+ if (isBlobStruct(value) && isBlobLike(value.data)) {
229
+ return value.data;
230
+ }
231
+ return undefined;
232
+ }
233
+ /**
234
+ * Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
235
+ * keeping the value's shape.
236
+ */
237
+ function withBlobBytes(value, bytes) {
238
+ if (isBlobLike(value)) {
239
+ return bytes;
240
+ }
241
+ const record = value;
242
+ return {
243
+ ...record,
244
+ uri: record.uri,
245
+ position: record.position,
246
+ size: record.size,
247
+ data: bytes,
248
+ };
249
+ }
250
+ function coerceBlobValue(input, field) {
251
+ const value = normalizeBlobInput(input);
163
252
  if (value == null) {
164
253
  return null;
165
254
  }
@@ -167,7 +256,7 @@ function coerceBlobValue(value) {
167
256
  return { data: value, uri: null };
168
257
  }
169
258
  if (ArrayBuffer.isView(value)) {
170
- throw new Error("Blob data must be Buffer or Uint8Array");
259
+ throw new Error("Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob");
171
260
  }
172
261
  if (typeof value === "string") {
173
262
  if (value === "") {
@@ -175,30 +264,94 @@ function coerceBlobValue(value) {
175
264
  }
176
265
  return { data: null, uri: value };
177
266
  }
178
- if (typeof value === "object") {
267
+ if (isBlobStruct(value)) {
179
268
  const record = value;
180
269
  if (!("data" in record) && !("uri" in record)) {
181
270
  throw new Error("Blob struct values must include a 'data' or 'uri' field");
182
271
  }
272
+ const unknownKeys = Object.keys(record).filter((key) => !["data", "uri", "position", "size"].includes(key));
273
+ if (unknownKeys.length > 0) {
274
+ throw new Error(`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
275
+ .map((key) => `'${key}'`)
276
+ .join(", ")}`);
277
+ }
183
278
  const uri = record.uri;
184
279
  if (uri === "") {
185
280
  throw new Error("Blob uri cannot be empty");
186
281
  }
187
282
  if (uri != null && typeof uri !== "string") {
188
- throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
283
+ throw new Error(`Blob uri must be a string, URL, or null, got ${typeof uri}`);
189
284
  }
190
285
  const data = record.data;
191
286
  if (data != null && !isBlobBytes(data)) {
192
- throw new Error("Blob data must be Buffer, Uint8Array, or null");
287
+ throw new Error("Blob data must be Buffer, Uint8Array, ArrayBuffer, Blob, or null");
193
288
  }
194
289
  const bytes = data ?? null;
195
290
  const uriValue = uri ?? null;
196
291
  if ((bytes == null) === (uriValue == null)) {
197
292
  throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
198
293
  }
294
+ const hasPosition = record.position != null;
295
+ const hasSize = record.size != null;
296
+ if (hasPosition !== hasSize) {
297
+ throw new Error("External blob must set both position and size, or neither");
298
+ }
299
+ if (hasPosition) {
300
+ if (uriValue == null) {
301
+ throw new Error("External blob byte ranges require a uri");
302
+ }
303
+ const position = blobRangeInteger(record.position, "position", 0n);
304
+ const size = blobRangeInteger(record.size, "size", 1n);
305
+ if (position + size > 0xffffffffffffffffn) {
306
+ throw new Error("External blob range overflows uint64");
307
+ }
308
+ if (field !== undefined &&
309
+ (!(field.type instanceof apache_arrow_1.Struct) ||
310
+ ["position", "size"].some((name) => !field.type.children.some((child) => child.name === name)))) {
311
+ throw new Error("Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size");
312
+ }
313
+ return { data: null, uri: uriValue, position, size };
314
+ }
199
315
  return { data: bytes, uri: uriValue };
200
316
  }
201
- throw new Error("Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }");
317
+ throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }");
318
+ }
319
+ function blobRangeInteger(value, name, minimum) {
320
+ if (typeof value !== "bigint" &&
321
+ (typeof value !== "number" || !Number.isSafeInteger(value))) {
322
+ throw new Error(`Blob ${name} must be a safe integer or bigint`);
323
+ }
324
+ const integer = BigInt(value);
325
+ if (integer < minimum || integer > 0xffffffffffffffffn) {
326
+ throw new Error(`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`);
327
+ }
328
+ return integer;
329
+ }
330
+ function isBlobLike(value) {
331
+ return typeof Blob !== "undefined" && value instanceof Blob;
332
+ }
333
+ /**
334
+ * Whether `value` is a `{ data?, uri? }` struct: any plain object, or any
335
+ * other object with a `data` or `uri` field (such as a class instance), apart
336
+ * from the binary, `Blob`, and `URL` values that are blob inputs themselves.
337
+ */
338
+ function isBlobStruct(value) {
339
+ if (typeof value !== "object" ||
340
+ value === null ||
341
+ Array.isArray(value) ||
342
+ ArrayBuffer.isView(value) ||
343
+ value instanceof ArrayBuffer ||
344
+ (typeof SharedArrayBuffer !== "undefined" &&
345
+ value instanceof SharedArrayBuffer) ||
346
+ isBlobLike(value) ||
347
+ value instanceof URL) {
348
+ return false;
349
+ }
350
+ const prototype = Object.getPrototypeOf(value);
351
+ return (prototype === Object.prototype ||
352
+ prototype === null ||
353
+ "data" in value ||
354
+ "uri" in value);
202
355
  }
203
356
  function isBlobBytes(value) {
204
357
  return Buffer.isBuffer(value) || value instanceof Uint8Array;
package/dist/catalog.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { Connection } from "./connection";
2
2
  import { HeaderProvider } from "./header";
3
- import { ListDatabasesResponse, Catalog as NativeCatalog, CatalogOptions as NativeCatalogOptions } from "./native.js";
3
+ import { Catalog as NativeCatalog, CatalogOptions as NativeCatalogOptions, DatabaseNames as NativeDatabaseNames } from "./native.js";
4
4
  import { OAuthConfig } from "./oauth";
5
5
  /** Options shared by a catalog and the database connections it returns. */
6
6
  export interface CatalogOptions extends Omit<NativeCatalogOptions, "oauthConfig"> {
@@ -8,7 +8,27 @@ export interface CatalogOptions extends Omit<NativeCatalogOptions, "oauthConfig"
8
8
  /** Called for each request to supply authentication headers. */
9
9
  headerProvider?: HeaderProvider | (() => Record<string, string> | Promise<Record<string, string>>);
10
10
  }
11
- export type { ListDatabasesResponse } from "./native.js";
11
+ /**
12
+ * A lazy async iterator of database names with pagination state.
13
+ * Returned by {@link Catalog.listDatabases}.
14
+ */
15
+ export declare class DatabaseNames implements AsyncIterableIterator<string> {
16
+ private readonly inner;
17
+ /** @hidden */
18
+ constructor(inner: NativeDatabaseNames);
19
+ [Symbol.asyncIterator](): AsyncIterableIterator<string>;
20
+ /** Fetch the next name, requesting another page only when needed. */
21
+ next(): Promise<IteratorResult<string>>;
22
+ /** Number of names available without another REST request. */
23
+ numPageResults(): number;
24
+ /**
25
+ * Token for the next REST request. Initially this is the supplied starting token.
26
+ * Returns undefined after the final page is fetched, even if names remain cached.
27
+ * Drain the cache before saving a token to avoid skipping names when resuming.
28
+ * A failed request terminates iteration and retains its token for a new iterator.
29
+ */
30
+ pageToken(): string | undefined;
31
+ }
12
32
  /** A remote catalog manages databases through the server's root namespace. */
13
33
  export declare class Catalog {
14
34
  private readonly inner;
@@ -26,11 +46,23 @@ export declare class Catalog {
26
46
  dropDatabase(name: string, options?: {
27
47
  ignoreMissing?: boolean;
28
48
  }): Promise<void>;
29
- /** List one page of databases; pass pageToken from a response for the next page. */
49
+ /**
50
+ * Iterate lazily over all database names using `for await...of`.
51
+ * pageToken resumes from a saved token; omitted starts at the beginning.
52
+ * pageLimit limits each REST response, not the total; omitted uses the server default.
53
+ * Request errors are raised during iteration and terminate the iterator.
54
+ *
55
+ * @example
56
+ * ```ts
57
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
58
+ * console.log(name);
59
+ * }
60
+ * ```
61
+ */
30
62
  listDatabases(options?: {
31
- limit?: number;
63
+ pageLimit?: number;
32
64
  pageToken?: string;
33
- }): Promise<ListDatabasesResponse>;
65
+ }): DatabaseNames;
34
66
  }
35
67
  /**
36
68
  * Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection
@@ -40,7 +72,9 @@ export declare class Catalog {
40
72
  * ```ts
41
73
  * const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
42
74
  * const db = await catalog.createDatabase("analytics", { existOk: true });
43
- * const page = await catalog.listDatabases({ limit: 20 });
75
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
76
+ * console.log(name);
77
+ * }
44
78
  * ```
45
79
  */
46
80
  export declare function connectCatalog(endpoint: string, options?: CatalogOptions): Promise<Catalog>;
package/dist/catalog.js CHANGED
@@ -2,10 +2,45 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Catalog = void 0;
5
+ exports.Catalog = exports.DatabaseNames = void 0;
6
6
  exports.connectCatalog = connectCatalog;
7
7
  const connection_1 = require("./connection");
8
8
  const native_js_1 = require("./native.js");
9
+ /**
10
+ * A lazy async iterator of database names with pagination state.
11
+ * Returned by {@link Catalog.listDatabases}.
12
+ */
13
+ class DatabaseNames {
14
+ inner;
15
+ /** @hidden */
16
+ constructor(inner) {
17
+ this.inner = inner;
18
+ }
19
+ [Symbol.asyncIterator]() {
20
+ return this;
21
+ }
22
+ /** Fetch the next name, requesting another page only when needed. */
23
+ async next() {
24
+ const value = await this.inner.next();
25
+ return value == null
26
+ ? { done: true, value: undefined }
27
+ : { done: false, value };
28
+ }
29
+ /** Number of names available without another REST request. */
30
+ numPageResults() {
31
+ return this.inner.numPageResults();
32
+ }
33
+ /**
34
+ * Token for the next REST request. Initially this is the supplied starting token.
35
+ * Returns undefined after the final page is fetched, even if names remain cached.
36
+ * Drain the cache before saving a token to avoid skipping names when resuming.
37
+ * A failed request terminates iteration and retains its token for a new iterator.
38
+ */
39
+ pageToken() {
40
+ return this.inner.pageToken() ?? undefined;
41
+ }
42
+ }
43
+ exports.DatabaseNames = DatabaseNames;
9
44
  /** A remote catalog manages databases through the server's root namespace. */
10
45
  class Catalog {
11
46
  inner;
@@ -29,15 +64,27 @@ class Catalog {
29
64
  async dropDatabase(name, options = {}) {
30
65
  await this.inner.dropDatabase(name, options.ignoreMissing);
31
66
  }
32
- /** List one page of databases; pass pageToken from a response for the next page. */
33
- async listDatabases(options = {}) {
34
- if (options.limit !== undefined &&
35
- (!Number.isInteger(options.limit) ||
36
- options.limit <= 0 ||
37
- options.limit > 2147483647)) {
67
+ /**
68
+ * Iterate lazily over all database names using `for await...of`.
69
+ * pageToken resumes from a saved token; omitted starts at the beginning.
70
+ * pageLimit limits each REST response, not the total; omitted uses the server default.
71
+ * Request errors are raised during iteration and terminate the iterator.
72
+ *
73
+ * @example
74
+ * ```ts
75
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
76
+ * console.log(name);
77
+ * }
78
+ * ```
79
+ */
80
+ listDatabases(options = {}) {
81
+ if (options.pageLimit !== undefined &&
82
+ (!Number.isInteger(options.pageLimit) ||
83
+ options.pageLimit <= 0 ||
84
+ options.pageLimit > 2147483647)) {
38
85
  throw new Error("Database list limit must be an integer between 1 and 2147483647");
39
86
  }
40
- return this.inner.listDatabases(options.limit, options.pageToken);
87
+ return new DatabaseNames(this.inner.listDatabases(options.pageToken, options.pageLimit));
41
88
  }
42
89
  }
43
90
  exports.Catalog = Catalog;
@@ -49,7 +96,9 @@ exports.Catalog = Catalog;
49
96
  * ```ts
50
97
  * const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
51
98
  * const db = await catalog.createDatabase("analytics", { existOk: true });
52
- * const page = await catalog.listDatabases({ limit: 20 });
99
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
100
+ * console.log(name);
101
+ * }
53
102
  * ```
54
103
  */
55
104
  async function connectCatalog(endpoint, options = {}) {
@@ -1,6 +1,7 @@
1
1
  import { Data, SchemaLike, TableLike } from "./arrow";
2
2
  import { EmbeddingFunctionConfig } from "./embedding/registry";
3
3
  import { Job } from "./job";
4
+ import { Listing, ListingOptions } from "./listing";
4
5
  import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
5
6
  import { Connection as LanceDbConnection } from "./native";
6
7
  import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
107
108
  * the last table name from the previous page.
108
109
  */
109
110
  startAfter?: string;
110
- /** An optional limit to the number of results to return. */
111
+ /** The maximum number of names to return. Omitted returns all names; zero returns none. */
111
112
  limit?: number;
112
113
  }
113
114
  export interface ListTablesOptions {
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
120
121
  pageToken?: string;
121
122
  /**
122
123
  * An upper bound on how many tables to return.
124
+ * Defaults to 100. Zero returns an empty page without a continuation token.
123
125
  *
124
126
  * A page may hold fewer than this and still not be the last one, so keep
125
127
  * going while the response carries a page token rather than while pages are
@@ -260,10 +262,6 @@ export declare abstract class Connection {
260
262
  * The view is populated before creation returns. Set `withNoData` to create
261
263
  * only its definition and empty backing table. The view is a normal table:
262
264
  * it can be queried, indexed and searched, and it appears in `tableNames`.
263
- * The source table must have stable row ids (create it with
264
- * the `newTableEnableStableRowIds` storage option); they keep the view's
265
- * provenance valid across source compactions and cannot be enabled after
266
- * a table exists.
267
265
  */
268
266
  abstract createMaterializedView(name: string, source: string, options?: {
269
267
  select?: MaterializedViewSelect;
@@ -280,9 +278,11 @@ export declare abstract class Connection {
280
278
  /**
281
279
  * The names of the materialized views in this database.
282
280
  *
283
- * Found by reading every table's schema, so this costs an open per table.
281
+ * Iterate lazily with `for await...of`. Local listings inspect table schemas
282
+ * one page at a time. Options control page size and the starting token.
283
+ * See {@link Listing} for pagination state and error behavior.
284
284
  */
285
- abstract listMaterializedViews(): Promise<string[]>;
285
+ abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
286
286
  /**
287
287
  * Drop the materialized view named `name`.
288
288
  *
@@ -334,8 +334,10 @@ export declare abstract class Connection {
334
334
  * The names of the views in one namespace.
335
335
  *
336
336
  * Names only; a definition comes from {@link describeView}.
337
+ * Iterate lazily with `for await...of`. See {@link Listing} for pagination
338
+ * state and error behavior; options control page size and the starting token.
337
339
  */
338
- abstract listViews(namespacePath?: string[]): Promise<string[]>;
340
+ abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
339
341
  abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
340
342
  /**
341
343
  * Creates a new Table and initialize it with new data.
@@ -492,8 +494,12 @@ export declare abstract class Connection {
492
494
  * connection-level call for any of them.
493
495
  */
494
496
  abstract openJob(jobId: string): Promise<Job>;
495
- /** List server-side jobs across the database's tables. */
496
- abstract listJobs(): Promise<JobInfo[]>;
497
+ /**
498
+ * Iterate lazily over server-side jobs across the database's tables.
499
+ * Use `for await...of`. Options control page size and the starting token.
500
+ * See {@link Listing} for pagination state and error behavior.
501
+ */
502
+ abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
497
503
  /**
498
504
  * Request cancellation of a server-side job by id.
499
505
  *
@@ -534,14 +540,14 @@ export declare class LocalConnection extends Connection {
534
540
  withNoData?: boolean;
535
541
  }): Promise<MaterializedView>;
536
542
  openMaterializedView(name: string): Promise<MaterializedView>;
537
- listMaterializedViews(): Promise<string[]>;
543
+ listMaterializedViews(options?: ListingOptions): Listing<string>;
538
544
  dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
539
545
  dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
540
546
  createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
541
547
  describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
542
548
  dropView(name: string, namespacePath?: string[]): Promise<void>;
543
549
  dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
544
- listViews(namespacePath?: string[]): Promise<string[]>;
550
+ listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
545
551
  listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
546
552
  openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
547
553
  cloneTable(targetTableName: string, sourceUri: string, options?: {
@@ -566,7 +572,7 @@ export declare class LocalConnection extends Connection {
566
572
  dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
567
573
  renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
568
574
  openJob(jobId: string): Promise<Job>;
569
- listJobs(): Promise<JobInfo[]>;
575
+ listJobs(options?: ListingOptions): Listing<JobInfo>;
570
576
  cancelJob(jobId: string): Promise<boolean>;
571
577
  pauseJob(jobId: string): Promise<string>;
572
578
  resumeJob(jobId: string): Promise<string>;
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
8
8
  const arrow_2 = require("./arrow");
9
9
  const registry_1 = require("./embedding/registry");
10
10
  const job_1 = require("./job");
11
+ const listing_1 = require("./listing");
11
12
  const materialized_view_1 = require("./materialized_view");
12
13
  const view_1 = require("./view");
13
14
  const sanitize_1 = require("./sanitize");
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
79
80
  const innerTable = await this.inner.openMaterializedView(name);
80
81
  return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
81
82
  }
82
- async listMaterializedViews() {
83
- return await this.inner.listMaterializedViews();
83
+ listMaterializedViews(options = {}) {
84
+ (0, listing_1.validateListingOptions)(options);
85
+ return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
84
86
  }
85
87
  async dropMaterializedView(name, namespacePath) {
86
88
  return this.inner.dropMaterializedView(name, namespacePath ?? []);
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
100
102
  async dropViewAsync(name, namespacePath) {
101
103
  return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
102
104
  }
103
- async listViews(namespacePath) {
104
- return this.inner.listViews(namespacePath ?? []);
105
+ listViews(namespacePath, options = {}) {
106
+ (0, listing_1.validateListingOptions)(options);
107
+ return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
105
108
  }
106
109
  async listTables(namespacePathOrOptions, options) {
107
110
  // Detect if first argument is namespacePath array or options object
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
243
246
  async openJob(jobId) {
244
247
  return new job_1.Job(await this.inner.openJob(jobId));
245
248
  }
246
- async listJobs() {
247
- return this.inner.listJobs();
249
+ listJobs(options = {}) {
250
+ (0, listing_1.validateListingOptions)(options);
251
+ return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
248
252
  }
249
253
  async cancelJob(jobId) {
250
254
  return this.inner.cancelJob(jobId);
@@ -303,7 +307,8 @@ async function parseTableData(data, options, streaming = false) {
303
307
  table = (0, sanitize_1.sanitizeTable)(data);
304
308
  }
305
309
  else {
306
- table = (0, arrow_1.makeArrowTable)(data, options);
310
+ const records = await (0, arrow_1.resolveBlobInputs)(data, options?.schema);
311
+ table = (0, arrow_1.makeArrowTable)(records, options);
307
312
  }
308
313
  if (streaming) {
309
314
  const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
package/dist/index.d.ts CHANGED
@@ -10,13 +10,13 @@ export { instrumentLanceDbMetrics } from "./otel";
10
10
  export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
11
11
  export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
12
12
  export { blob, isBlobField, BlobFile } from "./blob";
13
- export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
13
+ export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, BlobUri, } from "./blob";
14
14
  export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
15
15
  export { JobFailureInfo, JobInfo, Session } from "./native.js";
16
16
  export { Job, JobEventsOptions } from "./job";
17
- export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
17
+ export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
18
18
  export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
19
- export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, LsmStats, BucketStats, GenerationStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
19
+ export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
20
20
  export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
21
21
  export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
22
22
  export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -24,7 +24,7 @@ export * as embedding from "./embedding";
24
24
  export { permutationBuilder, PermutationBuilder } from "./permutation";
25
25
  export { Scannable, ScannableOptions } from "./scannable";
26
26
  export * as rerankers from "./rerankers";
27
- export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
27
+ export { SchemaLike, TableLike, DataTypeLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
28
28
  export { IntoSql, packBits } from "./util";
29
29
  /**
30
30
  * Options for tokenizing a full-text search query without a table index.
@@ -248,4 +248,5 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
248
248
  * ```
249
249
  */
250
250
  export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
251
- export { Catalog, CatalogOptions, ListDatabasesResponse, connectCatalog, } from "./catalog";
251
+ export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
252
+ export { Listing, ListingOptions } from "./listing";
package/dist/index.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.connectCatalog = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
5
+ exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
6
6
  exports.tokenize = tokenize;
7
7
  exports.connect = connect;
8
8
  exports.connectNamespace = connectNamespace;
@@ -184,4 +184,7 @@ async function connectNamespace(implName, configOrProperties, options) {
184
184
  }
185
185
  var catalog_1 = require("./catalog");
186
186
  Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
187
+ Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
187
188
  Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
189
+ var listing_1 = require("./listing");
190
+ Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
package/dist/indices.d.ts CHANGED
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
453
453
  */
454
454
  sampleRate?: number;
455
455
  }
456
- export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
456
+ export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
457
457
  /**
458
458
  * Options to create a full text search index
459
459
  */
@@ -477,6 +477,8 @@ export interface FtsOptions {
477
477
  *
478
478
  * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
479
479
  *
480
+ * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
481
+ *
480
482
  * "icu" - ICU dictionary-based word segmentation.
481
483
  *
482
484
  * "icu/split" - ICU segmentation with simple-style delimiter splitting.