@lancedb/lancedb 0.41.0-beta.0 → 0.41.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/arrow.js CHANGED
@@ -357,14 +357,25 @@ function makeArrowTable(data, options, metadata) {
357
357
  }
358
358
  if (schema !== undefined) {
359
359
  // Validate blob values up front and give every one the full
360
- // `{ data, uri }` shape, so inference sees the same struct whether a row
360
+ // storage shape, so inference sees the same struct whether a row
361
361
  // passed bytes, a URI, or a partial struct.
362
- data = mapBlobInputs(data, schema, (value, field, row) => {
362
+ data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
363
363
  if (value === undefined) {
364
364
  return value;
365
365
  }
366
366
  try {
367
- return (0, blob_1.coerceBlobValue)(value);
367
+ const normalized = (0, blob_1.coerceBlobValue)(value, storageField);
368
+ if (normalized === null ||
369
+ !(storageField.type instanceof apache_arrow_1.Struct) ||
370
+ storageField.type.children.length === 2) {
371
+ return normalized;
372
+ }
373
+ // Keep even unused range children during schema inference. Otherwise
374
+ // creating a range-capable table with inline data would lose them.
375
+ return Object.fromEntries(storageField.type.children.map((child) => [
376
+ child.name,
377
+ normalized[child.name] ?? null,
378
+ ]));
368
379
  }
369
380
  catch (e) {
370
381
  throw new Error(`Invalid value for blob field ${field} at row ${row}: ${e.message}`);
@@ -449,7 +460,7 @@ function mapBlobFields(record, fields, prefix, row, visit) {
449
460
  }
450
461
  function mapBlobField(value, field, label, row, visit) {
451
462
  if ((0, blob_1.isBlobField)(field)) {
452
- return visit(value, label, row);
463
+ return visit(value, label, row, field);
453
464
  }
454
465
  if (field.type instanceof apache_arrow_1.Struct && isObject(value)) {
455
466
  return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
@@ -528,7 +539,7 @@ function valueAtPath(datum, path) {
528
539
  function transposeData(data, field, path = []) {
529
540
  const valuesPath = [...path, field.name];
530
541
  if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
531
- const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
542
+ const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath), field));
532
543
  const childVectors = field.type.children.map((child) => {
533
544
  const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
534
545
  return makeVector(values, child.type, undefined, child.nullable);
package/dist/blob.d.ts CHANGED
@@ -14,6 +14,9 @@ export type BlobUri = string | URL;
14
14
  *
15
15
  * Either inline bytes, a URI pointing at external bytes, or a struct that sets
16
16
  * exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
17
+ * External byte ranges require both
18
+ * `position` (non-negative) and `size` (positive). Use safe integer numbers or
19
+ * `bigint` values for these unsigned 64-bit byte counts.
17
20
  *
18
21
  * @example
19
22
  * ```ts
@@ -31,14 +34,25 @@ export type BlobUri = string | URL;
31
34
  export type BlobInput = BlobData | BlobUri | {
32
35
  data: BlobData;
33
36
  uri?: null;
37
+ position?: null;
38
+ size?: null;
34
39
  } | {
35
40
  data?: null;
36
41
  uri: BlobUri;
42
+ position?: null;
43
+ size?: null;
44
+ } | {
45
+ data?: null;
46
+ uri: BlobUri;
47
+ position: number | bigint;
48
+ size: number | bigint;
37
49
  } | null | undefined;
38
50
  /** A blob value normalized to the columns of the blob struct. */
39
51
  export type BlobValue = {
40
52
  data: Buffer | Uint8Array | null;
41
53
  uri: string | null;
54
+ position?: bigint;
55
+ size?: bigint;
42
56
  };
43
57
  export type BlobOptions = {
44
58
  /** Defaults to true. */
@@ -61,9 +75,20 @@ export type BlobOptions = {
61
75
  };
62
76
  /**
63
77
  * Declares a `lance.blob.v2` column.
78
+ * Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
79
+ * whole external files, and external byte ranges. Set both `position` and
80
+ * positive `size` for a range, or omit both for a whole file.
81
+ *
82
+ * Existing two-field columns remain writable. Appends and merge inserts fill
83
+ * missing range fields with nulls for four-field targets and drop all-null
84
+ * range fields for two-field targets. Non-null ranges require a four-field
85
+ * target. Overwrite with Arrow input replaces the table's schema; row objects
86
+ * are converted using the current table schema.
64
87
  *
65
88
  * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
66
89
  * or {@link Table.fetchBlobFiles} to read bytes.
90
+ * Blobs inside lists can be written and queried as descriptors, but cannot
91
+ * currently be fetched because a row ID does not identify a list element.
67
92
  *
68
93
  * @example
69
94
  * ```ts
@@ -177,4 +202,4 @@ export declare function blobToRead(value: unknown): Blob | undefined;
177
202
  * keeping the value's shape.
178
203
  */
179
204
  export declare function withBlobBytes(value: unknown, bytes: Uint8Array): unknown;
180
- export declare function coerceBlobValue(input: unknown): BlobValue | null;
205
+ export declare function coerceBlobValue(input: unknown, field?: Field): BlobValue | null;
package/dist/blob.js CHANGED
@@ -18,9 +18,20 @@ const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-thresho
18
18
  const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
19
19
  /**
20
20
  * Declares a `lance.blob.v2` column.
21
+ * Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
22
+ * whole external files, and external byte ranges. Set both `position` and
23
+ * positive `size` for a range, or omit both for a whole file.
24
+ *
25
+ * Existing two-field columns remain writable. Appends and merge inserts fill
26
+ * missing range fields with nulls for four-field targets and drop all-null
27
+ * range fields for two-field targets. Non-null ranges require a four-field
28
+ * target. Overwrite with Arrow input replaces the table's schema; row objects
29
+ * are converted using the current table schema.
21
30
  *
22
31
  * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
23
32
  * or {@link Table.fetchBlobFiles} to read bytes.
33
+ * Blobs inside lists can be written and queried as descriptors, but cannot
34
+ * currently be fetched because a row ID does not identify a list element.
24
35
  *
25
36
  * @example
26
37
  * ```ts
@@ -60,6 +71,8 @@ function blob(name, options = {}) {
60
71
  return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
61
72
  new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
62
73
  new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
74
+ new apache_arrow_1.Field("position", new apache_arrow_1.Uint64(), true),
75
+ new apache_arrow_1.Field("size", new apache_arrow_1.Uint64(), true),
63
76
  ]), options.nullable ?? true, metadata);
64
77
  }
65
78
  /**
@@ -188,6 +201,8 @@ function normalizeBlobInput(value) {
188
201
  ...value,
189
202
  data: value.data,
190
203
  uri: value.uri,
204
+ position: value.position,
205
+ size: value.size,
191
206
  };
192
207
  let changed = false;
193
208
  if (value.data instanceof ArrayBuffer) {
@@ -224,9 +239,15 @@ function withBlobBytes(value, bytes) {
224
239
  return bytes;
225
240
  }
226
241
  const record = value;
227
- return { ...record, uri: record.uri, data: bytes };
242
+ return {
243
+ ...record,
244
+ uri: record.uri,
245
+ position: record.position,
246
+ size: record.size,
247
+ data: bytes,
248
+ };
228
249
  }
229
- function coerceBlobValue(input) {
250
+ function coerceBlobValue(input, field) {
230
251
  const value = normalizeBlobInput(input);
231
252
  if (value == null) {
232
253
  return null;
@@ -248,9 +269,9 @@ function coerceBlobValue(input) {
248
269
  if (!("data" in record) && !("uri" in record)) {
249
270
  throw new Error("Blob struct values must include a 'data' or 'uri' field");
250
271
  }
251
- const unknownKeys = Object.keys(record).filter((key) => key !== "data" && key !== "uri");
272
+ const unknownKeys = Object.keys(record).filter((key) => !["data", "uri", "position", "size"].includes(key));
252
273
  if (unknownKeys.length > 0) {
253
- throw new Error(`Blob struct values only support 'data' and 'uri', got ${unknownKeys
274
+ throw new Error(`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
254
275
  .map((key) => `'${key}'`)
255
276
  .join(", ")}`);
256
277
  }
@@ -270,9 +291,41 @@ function coerceBlobValue(input) {
270
291
  if ((bytes == null) === (uriValue == null)) {
271
292
  throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
272
293
  }
294
+ const hasPosition = record.position != null;
295
+ const hasSize = record.size != null;
296
+ if (hasPosition !== hasSize) {
297
+ throw new Error("External blob must set both position and size, or neither");
298
+ }
299
+ if (hasPosition) {
300
+ if (uriValue == null) {
301
+ throw new Error("External blob byte ranges require a uri");
302
+ }
303
+ const position = blobRangeInteger(record.position, "position", 0n);
304
+ const size = blobRangeInteger(record.size, "size", 1n);
305
+ if (position + size > 0xffffffffffffffffn) {
306
+ throw new Error("External blob range overflows uint64");
307
+ }
308
+ if (field !== undefined &&
309
+ (!(field.type instanceof apache_arrow_1.Struct) ||
310
+ ["position", "size"].some((name) => !field.type.children.some((child) => child.name === name)))) {
311
+ throw new Error("Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size");
312
+ }
313
+ return { data: null, uri: uriValue, position, size };
314
+ }
273
315
  return { data: bytes, uri: uriValue };
274
316
  }
275
- throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }");
317
+ throw new Error("Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }");
318
+ }
319
+ function blobRangeInteger(value, name, minimum) {
320
+ if (typeof value !== "bigint" &&
321
+ (typeof value !== "number" || !Number.isSafeInteger(value))) {
322
+ throw new Error(`Blob ${name} must be a safe integer or bigint`);
323
+ }
324
+ const integer = BigInt(value);
325
+ if (integer < minimum || integer > 0xffffffffffffffffn) {
326
+ throw new Error(`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`);
327
+ }
328
+ return integer;
276
329
  }
277
330
  function isBlobLike(value) {
278
331
  return typeof Blob !== "undefined" && value instanceof Blob;
@@ -1,6 +1,7 @@
1
1
  import { Data, SchemaLike, TableLike } from "./arrow";
2
2
  import { EmbeddingFunctionConfig } from "./embedding/registry";
3
3
  import { Job } from "./job";
4
+ import { Listing, ListingOptions } from "./listing";
4
5
  import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
5
6
  import { Connection as LanceDbConnection } from "./native";
6
7
  import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
107
108
  * the last table name from the previous page.
108
109
  */
109
110
  startAfter?: string;
110
- /** An optional limit to the number of results to return. */
111
+ /** The maximum number of names to return. Omitted returns all names; zero returns none. */
111
112
  limit?: number;
112
113
  }
113
114
  export interface ListTablesOptions {
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
120
121
  pageToken?: string;
121
122
  /**
122
123
  * An upper bound on how many tables to return.
124
+ * Defaults to 100. Zero returns an empty page without a continuation token.
123
125
  *
124
126
  * A page may hold fewer than this and still not be the last one, so keep
125
127
  * going while the response carries a page token rather than while pages are
@@ -276,9 +278,11 @@ export declare abstract class Connection {
276
278
  /**
277
279
  * The names of the materialized views in this database.
278
280
  *
279
- * Found by reading every table's schema, so this costs an open per table.
281
+ * Iterate lazily with `for await...of`. Local listings inspect table schemas
282
+ * one page at a time. Options control page size and the starting token.
283
+ * See {@link Listing} for pagination state and error behavior.
280
284
  */
281
- abstract listMaterializedViews(): Promise<string[]>;
285
+ abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
282
286
  /**
283
287
  * Drop the materialized view named `name`.
284
288
  *
@@ -330,8 +334,10 @@ export declare abstract class Connection {
330
334
  * The names of the views in one namespace.
331
335
  *
332
336
  * Names only; a definition comes from {@link describeView}.
337
+ * Iterate lazily with `for await...of`. See {@link Listing} for pagination
338
+ * state and error behavior; options control page size and the starting token.
333
339
  */
334
- abstract listViews(namespacePath?: string[]): Promise<string[]>;
340
+ abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
335
341
  abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
336
342
  /**
337
343
  * Creates a new Table and initialize it with new data.
@@ -488,8 +494,12 @@ export declare abstract class Connection {
488
494
  * connection-level call for any of them.
489
495
  */
490
496
  abstract openJob(jobId: string): Promise<Job>;
491
- /** List server-side jobs across the database's tables. */
492
- abstract listJobs(): Promise<JobInfo[]>;
497
+ /**
498
+ * Iterate lazily over server-side jobs across the database's tables.
499
+ * Use `for await...of`. Options control page size and the starting token.
500
+ * See {@link Listing} for pagination state and error behavior.
501
+ */
502
+ abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
493
503
  /**
494
504
  * Request cancellation of a server-side job by id.
495
505
  *
@@ -530,14 +540,14 @@ export declare class LocalConnection extends Connection {
530
540
  withNoData?: boolean;
531
541
  }): Promise<MaterializedView>;
532
542
  openMaterializedView(name: string): Promise<MaterializedView>;
533
- listMaterializedViews(): Promise<string[]>;
543
+ listMaterializedViews(options?: ListingOptions): Listing<string>;
534
544
  dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
535
545
  dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
536
546
  createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
537
547
  describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
538
548
  dropView(name: string, namespacePath?: string[]): Promise<void>;
539
549
  dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
540
- listViews(namespacePath?: string[]): Promise<string[]>;
550
+ listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
541
551
  listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
542
552
  openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
543
553
  cloneTable(targetTableName: string, sourceUri: string, options?: {
@@ -562,7 +572,7 @@ export declare class LocalConnection extends Connection {
562
572
  dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
563
573
  renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
564
574
  openJob(jobId: string): Promise<Job>;
565
- listJobs(): Promise<JobInfo[]>;
575
+ listJobs(options?: ListingOptions): Listing<JobInfo>;
566
576
  cancelJob(jobId: string): Promise<boolean>;
567
577
  pauseJob(jobId: string): Promise<string>;
568
578
  resumeJob(jobId: string): Promise<string>;
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
8
8
  const arrow_2 = require("./arrow");
9
9
  const registry_1 = require("./embedding/registry");
10
10
  const job_1 = require("./job");
11
+ const listing_1 = require("./listing");
11
12
  const materialized_view_1 = require("./materialized_view");
12
13
  const view_1 = require("./view");
13
14
  const sanitize_1 = require("./sanitize");
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
79
80
  const innerTable = await this.inner.openMaterializedView(name);
80
81
  return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
81
82
  }
82
- async listMaterializedViews() {
83
- return await this.inner.listMaterializedViews();
83
+ listMaterializedViews(options = {}) {
84
+ (0, listing_1.validateListingOptions)(options);
85
+ return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
84
86
  }
85
87
  async dropMaterializedView(name, namespacePath) {
86
88
  return this.inner.dropMaterializedView(name, namespacePath ?? []);
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
100
102
  async dropViewAsync(name, namespacePath) {
101
103
  return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
102
104
  }
103
- async listViews(namespacePath) {
104
- return this.inner.listViews(namespacePath ?? []);
105
+ listViews(namespacePath, options = {}) {
106
+ (0, listing_1.validateListingOptions)(options);
107
+ return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
105
108
  }
106
109
  async listTables(namespacePathOrOptions, options) {
107
110
  // Detect if first argument is namespacePath array or options object
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
243
246
  async openJob(jobId) {
244
247
  return new job_1.Job(await this.inner.openJob(jobId));
245
248
  }
246
- async listJobs() {
247
- return this.inner.listJobs();
249
+ listJobs(options = {}) {
250
+ (0, listing_1.validateListingOptions)(options);
251
+ return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
248
252
  }
249
253
  async cancelJob(jobId) {
250
254
  return this.inner.cancelJob(jobId);
package/dist/index.d.ts CHANGED
@@ -14,9 +14,9 @@ export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, Blob
14
14
  export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
15
15
  export { JobFailureInfo, JobInfo, Session } from "./native.js";
16
16
  export { Job, JobEventsOptions } from "./job";
17
- export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
17
+ export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
18
18
  export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
19
- export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, LsmStats, BucketStats, GenerationStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
19
+ export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
20
20
  export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
21
21
  export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
22
22
  export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -249,3 +249,4 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
249
249
  */
250
250
  export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
251
251
  export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
252
+ export { Listing, ListingOptions } from "./listing";
package/dist/index.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
5
+ exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
6
6
  exports.tokenize = tokenize;
7
7
  exports.connect = connect;
8
8
  exports.connectNamespace = connectNamespace;
@@ -186,3 +186,5 @@ var catalog_1 = require("./catalog");
186
186
  Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
187
187
  Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
188
188
  Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
189
+ var listing_1 = require("./listing");
190
+ Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
package/dist/indices.d.ts CHANGED
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
453
453
  */
454
454
  sampleRate?: number;
455
455
  }
456
- export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
456
+ export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
457
457
  /**
458
458
  * Options to create a full text search index
459
459
  */
@@ -477,6 +477,8 @@ export interface FtsOptions {
477
477
  *
478
478
  * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
479
479
  *
480
+ * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
481
+ *
480
482
  * "icu" - ICU dictionary-based word segmentation.
481
483
  *
482
484
  * "icu/split" - ICU segmentation with simple-style delimiter splitting.
@@ -0,0 +1,41 @@
1
+ /** Pagination controls for lazy resource listings. */
2
+ export interface ListingOptions {
3
+ /** Starting continuation token; omitted starts at the beginning. */
4
+ pageToken?: string;
5
+ /** Maximum results per request, not a limit on the whole listing. */
6
+ pageLimit?: number;
7
+ }
8
+ /** @internal */
9
+ export declare function validateListingOptions(options: ListingOptions): void;
10
+ /**
11
+ * A lazy async iterator over database resources with pagination state.
12
+ * Requests occur during iteration; request errors terminate the iterator.
13
+ *
14
+ * @example
15
+ * ```ts
16
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
17
+ * console.log(name);
18
+ * }
19
+ * ```
20
+ */
21
+ export declare class Listing<T> implements AsyncIterableIterator<T> {
22
+ private readonly inner;
23
+ /** @hidden */
24
+ constructor(inner: {
25
+ next(): Promise<T | null | undefined>;
26
+ numPageResults(): number;
27
+ pageToken(): string | null | undefined;
28
+ });
29
+ [Symbol.asyncIterator](): AsyncIterableIterator<T>;
30
+ /** Fetch the next item, requesting another page only when needed. */
31
+ next(): Promise<IteratorResult<T>>;
32
+ /** Number of cached items available without another request. */
33
+ numPageResults(): number;
34
+ /**
35
+ * Token for the next request. Initially this is the supplied starting token.
36
+ * Returns undefined after fetching the final page, even with cached items.
37
+ * Drain the cache before saving a token to avoid skipping items on resumption.
38
+ * Failed requests terminate iteration and retain their token for a new iterator.
39
+ */
40
+ pageToken(): string | undefined;
41
+ }
@@ -0,0 +1,57 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.Listing = void 0;
6
+ exports.validateListingOptions = validateListingOptions;
7
+ /** @internal */
8
+ function validateListingOptions(options) {
9
+ if (options.pageLimit !== undefined &&
10
+ (!Number.isInteger(options.pageLimit) ||
11
+ options.pageLimit <= 0 ||
12
+ options.pageLimit > 2147483647)) {
13
+ throw new Error("Listing page limit must be an integer between 1 and 2147483647");
14
+ }
15
+ }
16
+ /**
17
+ * A lazy async iterator over database resources with pagination state.
18
+ * Requests occur during iteration; request errors terminate the iterator.
19
+ *
20
+ * @example
21
+ * ```ts
22
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
23
+ * console.log(name);
24
+ * }
25
+ * ```
26
+ */
27
+ class Listing {
28
+ inner;
29
+ /** @hidden */
30
+ constructor(inner) {
31
+ this.inner = inner;
32
+ }
33
+ [Symbol.asyncIterator]() {
34
+ return this;
35
+ }
36
+ /** Fetch the next item, requesting another page only when needed. */
37
+ async next() {
38
+ const value = await this.inner.next();
39
+ return value == null
40
+ ? { done: true, value: undefined }
41
+ : { done: false, value };
42
+ }
43
+ /** Number of cached items available without another request. */
44
+ numPageResults() {
45
+ return this.inner.numPageResults();
46
+ }
47
+ /**
48
+ * Token for the next request. Initially this is the supplied starting token.
49
+ * Returns undefined after fetching the final page, even with cached items.
50
+ * Drain the cache before saving a token to avoid skipping items on resumption.
51
+ * Failed requests terminate iteration and retain their token for a new iterator.
52
+ */
53
+ pageToken() {
54
+ return this.inner.pageToken() ?? undefined;
55
+ }
56
+ }
57
+ exports.Listing = Listing;
package/dist/merge.d.ts CHANGED
@@ -57,25 +57,25 @@ export declare class MergeInsertBuilder {
57
57
  */
58
58
  useIndex(useIndex: boolean): MergeInsertBuilder;
59
59
  /**
60
- * Control MemWAL routing for this merge.
60
+ * Control RTE routing for this merge.
61
61
  *
62
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
62
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
63
63
  * routed through Lance's MemWAL shard writer, and a table without one uses the
64
64
  * standard path.
65
65
  *
66
- * @param enable - `true` forces MemWAL routing and errors if the table has no
67
- * LSM write spec. `false` forces the standard write path even when a spec is set.
66
+ * @param enable - `true` forces RTE routing and errors if the table has no
67
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
68
68
  */
69
- useLsm(enable: boolean): MergeInsertBuilder;
69
+ useRTE(enable: boolean): MergeInsertBuilder;
70
70
  /**
71
71
  * Controls how an LSM merge checks that its input targets a single shard.
72
72
  *
73
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
73
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
74
74
  * route to the same shard. When `true` (the default), every row is inspected
75
75
  * to verify this. When `false`, only the first row is inspected and the
76
76
  * shard it routes to is used for the whole input — a faster path for callers
77
77
  * that have already pre-sharded their input. Has no effect on tables without
78
- * an LSM write spec.
78
+ * an RTE write spec.
79
79
  *
80
80
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
81
81
  */
package/dist/merge.js CHANGED
@@ -69,27 +69,27 @@ class MergeInsertBuilder {
69
69
  return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
70
70
  }
71
71
  /**
72
- * Control MemWAL routing for this merge.
72
+ * Control RTE routing for this merge.
73
73
  *
74
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
74
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
75
75
  * routed through Lance's MemWAL shard writer, and a table without one uses the
76
76
  * standard path.
77
77
  *
78
- * @param enable - `true` forces MemWAL routing and errors if the table has no
79
- * LSM write spec. `false` forces the standard write path even when a spec is set.
78
+ * @param enable - `true` forces RTE routing and errors if the table has no
79
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
80
80
  */
81
- useLsm(enable) {
82
- return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
81
+ useRTE(enable) {
82
+ return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
83
83
  }
84
84
  /**
85
85
  * Controls how an LSM merge checks that its input targets a single shard.
86
86
  *
87
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
87
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
88
88
  * route to the same shard. When `true` (the default), every row is inspected
89
89
  * to verify this. When `false`, only the first row is inspected and the
90
90
  * shard it routes to is used for the whole input — a faster path for callers
91
91
  * that have already pre-sharded their input. Has no effect on tables without
92
- * an LSM write spec.
92
+ * an RTE write spec.
93
93
  *
94
94
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
95
95
  */