@lancedb/lancedb 0.40.0 → 0.41.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/catalog.js CHANGED
@@ -2,10 +2,45 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.Catalog = void 0;
5
+ exports.Catalog = exports.DatabaseNames = void 0;
6
6
  exports.connectCatalog = connectCatalog;
7
7
  const connection_1 = require("./connection");
8
8
  const native_js_1 = require("./native.js");
9
+ /**
10
+ * A lazy async iterator of database names with pagination state.
11
+ * Returned by {@link Catalog.listDatabases}.
12
+ */
13
+ class DatabaseNames {
14
+ inner;
15
+ /** @hidden */
16
+ constructor(inner) {
17
+ this.inner = inner;
18
+ }
19
+ [Symbol.asyncIterator]() {
20
+ return this;
21
+ }
22
+ /** Fetch the next name, requesting another page only when needed. */
23
+ async next() {
24
+ const value = await this.inner.next();
25
+ return value == null
26
+ ? { done: true, value: undefined }
27
+ : { done: false, value };
28
+ }
29
+ /** Number of names available without another REST request. */
30
+ numPageResults() {
31
+ return this.inner.numPageResults();
32
+ }
33
+ /**
34
+ * Token for the next REST request. Initially this is the supplied starting token.
35
+ * Returns undefined after the final page is fetched, even if names remain cached.
36
+ * Drain the cache before saving a token to avoid skipping names when resuming.
37
+ * A failed request terminates iteration and retains its token for a new iterator.
38
+ */
39
+ pageToken() {
40
+ return this.inner.pageToken() ?? undefined;
41
+ }
42
+ }
43
+ exports.DatabaseNames = DatabaseNames;
9
44
  /** A remote catalog manages databases through the server's root namespace. */
10
45
  class Catalog {
11
46
  inner;
@@ -29,15 +64,27 @@ class Catalog {
29
64
  async dropDatabase(name, options = {}) {
30
65
  await this.inner.dropDatabase(name, options.ignoreMissing);
31
66
  }
32
- /** List one page of databases; pass pageToken from a response for the next page. */
33
- async listDatabases(options = {}) {
34
- if (options.limit !== undefined &&
35
- (!Number.isInteger(options.limit) ||
36
- options.limit <= 0 ||
37
- options.limit > 2147483647)) {
67
+ /**
68
+ * Iterate lazily over all database names using `for await...of`.
69
+ * pageToken resumes from a saved token; omitted starts at the beginning.
70
+ * pageLimit limits each REST response, not the total; omitted uses the server default.
71
+ * Request errors are raised during iteration and terminate the iterator.
72
+ *
73
+ * @example
74
+ * ```ts
75
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
76
+ * console.log(name);
77
+ * }
78
+ * ```
79
+ */
80
+ listDatabases(options = {}) {
81
+ if (options.pageLimit !== undefined &&
82
+ (!Number.isInteger(options.pageLimit) ||
83
+ options.pageLimit <= 0 ||
84
+ options.pageLimit > 2147483647)) {
38
85
  throw new Error("Database list limit must be an integer between 1 and 2147483647");
39
86
  }
40
- return this.inner.listDatabases(options.limit, options.pageToken);
87
+ return new DatabaseNames(this.inner.listDatabases(options.pageToken, options.pageLimit));
41
88
  }
42
89
  }
43
90
  exports.Catalog = Catalog;
@@ -49,7 +96,9 @@ exports.Catalog = Catalog;
49
96
  * ```ts
50
97
  * const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
51
98
  * const db = await catalog.createDatabase("analytics", { existOk: true });
52
- * const page = await catalog.listDatabases({ limit: 20 });
99
+ * for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
100
+ * console.log(name);
101
+ * }
53
102
  * ```
54
103
  */
55
104
  async function connectCatalog(endpoint, options = {}) {
@@ -1,6 +1,7 @@
1
1
  import { Data, SchemaLike, TableLike } from "./arrow";
2
2
  import { EmbeddingFunctionConfig } from "./embedding/registry";
3
3
  import { Job } from "./job";
4
+ import { Listing, ListingOptions } from "./listing";
4
5
  import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
5
6
  import { Connection as LanceDbConnection } from "./native";
6
7
  import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
107
108
  * the last table name from the previous page.
108
109
  */
109
110
  startAfter?: string;
110
- /** An optional limit to the number of results to return. */
111
+ /** The maximum number of names to return. Omitted returns all names; zero returns none. */
111
112
  limit?: number;
112
113
  }
113
114
  export interface ListTablesOptions {
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
120
121
  pageToken?: string;
121
122
  /**
122
123
  * An upper bound on how many tables to return.
124
+ * Defaults to 100. Zero returns an empty page without a continuation token.
123
125
  *
124
126
  * A page may hold fewer than this and still not be the last one, so keep
125
127
  * going while the response carries a page token rather than while pages are
@@ -260,10 +262,6 @@ export declare abstract class Connection {
260
262
  * The view is populated before creation returns. Set `withNoData` to create
261
263
  * only its definition and empty backing table. The view is a normal table:
262
264
  * it can be queried, indexed and searched, and it appears in `tableNames`.
263
- * The source table must have stable row ids (create it with
264
- * the `newTableEnableStableRowIds` storage option); they keep the view's
265
- * provenance valid across source compactions and cannot be enabled after
266
- * a table exists.
267
265
  */
268
266
  abstract createMaterializedView(name: string, source: string, options?: {
269
267
  select?: MaterializedViewSelect;
@@ -280,9 +278,11 @@ export declare abstract class Connection {
280
278
  /**
281
279
  * The names of the materialized views in this database.
282
280
  *
283
- * Found by reading every table's schema, so this costs an open per table.
281
+ * Iterate lazily with `for await...of`. Local listings inspect table schemas
282
+ * one page at a time. Options control page size and the starting token.
283
+ * See {@link Listing} for pagination state and error behavior.
284
284
  */
285
- abstract listMaterializedViews(): Promise<string[]>;
285
+ abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
286
286
  /**
287
287
  * Drop the materialized view named `name`.
288
288
  *
@@ -334,8 +334,10 @@ export declare abstract class Connection {
334
334
  * The names of the views in one namespace.
335
335
  *
336
336
  * Names only; a definition comes from {@link describeView}.
337
+ * Iterate lazily with `for await...of`. See {@link Listing} for pagination
338
+ * state and error behavior; options control page size and the starting token.
337
339
  */
338
- abstract listViews(namespacePath?: string[]): Promise<string[]>;
340
+ abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
339
341
  abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
340
342
  /**
341
343
  * Creates a new Table and initialize it with new data.
@@ -492,8 +494,12 @@ export declare abstract class Connection {
492
494
  * connection-level call for any of them.
493
495
  */
494
496
  abstract openJob(jobId: string): Promise<Job>;
495
- /** List server-side jobs across the database's tables. */
496
- abstract listJobs(): Promise<JobInfo[]>;
497
+ /**
498
+ * Iterate lazily over server-side jobs across the database's tables.
499
+ * Use `for await...of`. Options control page size and the starting token.
500
+ * See {@link Listing} for pagination state and error behavior.
501
+ */
502
+ abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
497
503
  /**
498
504
  * Request cancellation of a server-side job by id.
499
505
  *
@@ -534,14 +540,14 @@ export declare class LocalConnection extends Connection {
534
540
  withNoData?: boolean;
535
541
  }): Promise<MaterializedView>;
536
542
  openMaterializedView(name: string): Promise<MaterializedView>;
537
- listMaterializedViews(): Promise<string[]>;
543
+ listMaterializedViews(options?: ListingOptions): Listing<string>;
538
544
  dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
539
545
  dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
540
546
  createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
541
547
  describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
542
548
  dropView(name: string, namespacePath?: string[]): Promise<void>;
543
549
  dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
544
- listViews(namespacePath?: string[]): Promise<string[]>;
550
+ listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
545
551
  listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
546
552
  openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
547
553
  cloneTable(targetTableName: string, sourceUri: string, options?: {
@@ -566,7 +572,7 @@ export declare class LocalConnection extends Connection {
566
572
  dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
567
573
  renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
568
574
  openJob(jobId: string): Promise<Job>;
569
- listJobs(): Promise<JobInfo[]>;
575
+ listJobs(options?: ListingOptions): Listing<JobInfo>;
570
576
  cancelJob(jobId: string): Promise<boolean>;
571
577
  pauseJob(jobId: string): Promise<string>;
572
578
  resumeJob(jobId: string): Promise<string>;
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
8
8
  const arrow_2 = require("./arrow");
9
9
  const registry_1 = require("./embedding/registry");
10
10
  const job_1 = require("./job");
11
+ const listing_1 = require("./listing");
11
12
  const materialized_view_1 = require("./materialized_view");
12
13
  const view_1 = require("./view");
13
14
  const sanitize_1 = require("./sanitize");
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
79
80
  const innerTable = await this.inner.openMaterializedView(name);
80
81
  return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
81
82
  }
82
- async listMaterializedViews() {
83
- return await this.inner.listMaterializedViews();
83
+ listMaterializedViews(options = {}) {
84
+ (0, listing_1.validateListingOptions)(options);
85
+ return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
84
86
  }
85
87
  async dropMaterializedView(name, namespacePath) {
86
88
  return this.inner.dropMaterializedView(name, namespacePath ?? []);
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
100
102
  async dropViewAsync(name, namespacePath) {
101
103
  return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
102
104
  }
103
- async listViews(namespacePath) {
104
- return this.inner.listViews(namespacePath ?? []);
105
+ listViews(namespacePath, options = {}) {
106
+ (0, listing_1.validateListingOptions)(options);
107
+ return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
105
108
  }
106
109
  async listTables(namespacePathOrOptions, options) {
107
110
  // Detect if first argument is namespacePath array or options object
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
243
246
  async openJob(jobId) {
244
247
  return new job_1.Job(await this.inner.openJob(jobId));
245
248
  }
246
- async listJobs() {
247
- return this.inner.listJobs();
249
+ listJobs(options = {}) {
250
+ (0, listing_1.validateListingOptions)(options);
251
+ return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
248
252
  }
249
253
  async cancelJob(jobId) {
250
254
  return this.inner.cancelJob(jobId);
@@ -303,7 +307,8 @@ async function parseTableData(data, options, streaming = false) {
303
307
  table = (0, sanitize_1.sanitizeTable)(data);
304
308
  }
305
309
  else {
306
- table = (0, arrow_1.makeArrowTable)(data, options);
310
+ const records = await (0, arrow_1.resolveBlobInputs)(data, options?.schema);
311
+ table = (0, arrow_1.makeArrowTable)(records, options);
307
312
  }
308
313
  if (streaming) {
309
314
  const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
package/dist/index.d.ts CHANGED
@@ -10,7 +10,7 @@ export { instrumentLanceDbMetrics } from "./otel";
10
10
  export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
11
11
  export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
12
12
  export { blob, isBlobField, BlobFile } from "./blob";
13
- export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
13
+ export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, BlobUri, } from "./blob";
14
14
  export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
15
15
  export { JobFailureInfo, JobInfo, Session } from "./native.js";
16
16
  export { Job, JobEventsOptions } from "./job";
@@ -24,7 +24,7 @@ export * as embedding from "./embedding";
24
24
  export { permutationBuilder, PermutationBuilder } from "./permutation";
25
25
  export { Scannable, ScannableOptions } from "./scannable";
26
26
  export * as rerankers from "./rerankers";
27
- export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
27
+ export { SchemaLike, TableLike, DataTypeLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
28
28
  export { IntoSql, packBits } from "./util";
29
29
  /**
30
30
  * Options for tokenizing a full-text search query without a table index.
@@ -248,4 +248,5 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
248
248
  * ```
249
249
  */
250
250
  export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
251
- export { Catalog, CatalogOptions, ListDatabasesResponse, connectCatalog, } from "./catalog";
251
+ export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
252
+ export { Listing, ListingOptions } from "./listing";
package/dist/index.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.connectCatalog = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
5
+ exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
6
6
  exports.tokenize = tokenize;
7
7
  exports.connect = connect;
8
8
  exports.connectNamespace = connectNamespace;
@@ -184,4 +184,7 @@ async function connectNamespace(implName, configOrProperties, options) {
184
184
  }
185
185
  var catalog_1 = require("./catalog");
186
186
  Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
187
+ Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
187
188
  Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
189
+ var listing_1 = require("./listing");
190
+ Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
package/dist/indices.d.ts CHANGED
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
453
453
  */
454
454
  sampleRate?: number;
455
455
  }
456
- export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
456
+ export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
457
457
  /**
458
458
  * Options to create a full text search index
459
459
  */
@@ -477,6 +477,8 @@ export interface FtsOptions {
477
477
  *
478
478
  * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
479
479
  *
480
+ * "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
481
+ *
480
482
  * "icu" - ICU dictionary-based word segmentation.
481
483
  *
482
484
  * "icu/split" - ICU segmentation with simple-style delimiter splitting.
@@ -0,0 +1,41 @@
1
+ /** Pagination controls for lazy resource listings. */
2
+ export interface ListingOptions {
3
+ /** Starting continuation token; omitted starts at the beginning. */
4
+ pageToken?: string;
5
+ /** Maximum results per request, not a limit on the whole listing. */
6
+ pageLimit?: number;
7
+ }
8
+ /** @internal */
9
+ export declare function validateListingOptions(options: ListingOptions): void;
10
+ /**
11
+ * A lazy async iterator over database resources with pagination state.
12
+ * Requests occur during iteration; request errors terminate the iterator.
13
+ *
14
+ * @example
15
+ * ```ts
16
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
17
+ * console.log(name);
18
+ * }
19
+ * ```
20
+ */
21
+ export declare class Listing<T> implements AsyncIterableIterator<T> {
22
+ private readonly inner;
23
+ /** @hidden */
24
+ constructor(inner: {
25
+ next(): Promise<T | null | undefined>;
26
+ numPageResults(): number;
27
+ pageToken(): string | null | undefined;
28
+ });
29
+ [Symbol.asyncIterator](): AsyncIterableIterator<T>;
30
+ /** Fetch the next item, requesting another page only when needed. */
31
+ next(): Promise<IteratorResult<T>>;
32
+ /** Number of cached items available without another request. */
33
+ numPageResults(): number;
34
+ /**
35
+ * Token for the next request. Initially this is the supplied starting token.
36
+ * Returns undefined after fetching the final page, even with cached items.
37
+ * Drain the cache before saving a token to avoid skipping items on resumption.
38
+ * Failed requests terminate iteration and retain their token for a new iterator.
39
+ */
40
+ pageToken(): string | undefined;
41
+ }
@@ -0,0 +1,57 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.Listing = void 0;
6
+ exports.validateListingOptions = validateListingOptions;
7
+ /** @internal */
8
+ function validateListingOptions(options) {
9
+ if (options.pageLimit !== undefined &&
10
+ (!Number.isInteger(options.pageLimit) ||
11
+ options.pageLimit <= 0 ||
12
+ options.pageLimit > 2147483647)) {
13
+ throw new Error("Listing page limit must be an integer between 1 and 2147483647");
14
+ }
15
+ }
16
+ /**
17
+ * A lazy async iterator over database resources with pagination state.
18
+ * Requests occur during iteration; request errors terminate the iterator.
19
+ *
20
+ * @example
21
+ * ```ts
22
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
23
+ * console.log(name);
24
+ * }
25
+ * ```
26
+ */
27
+ class Listing {
28
+ inner;
29
+ /** @hidden */
30
+ constructor(inner) {
31
+ this.inner = inner;
32
+ }
33
+ [Symbol.asyncIterator]() {
34
+ return this;
35
+ }
36
+ /** Fetch the next item, requesting another page only when needed. */
37
+ async next() {
38
+ const value = await this.inner.next();
39
+ return value == null
40
+ ? { done: true, value: undefined }
41
+ : { done: false, value };
42
+ }
43
+ /** Number of cached items available without another request. */
44
+ numPageResults() {
45
+ return this.inner.numPageResults();
46
+ }
47
+ /**
48
+ * Token for the next request. Initially this is the supplied starting token.
49
+ * Returns undefined after fetching the final page, even with cached items.
50
+ * Drain the cache before saving a token to avoid skipping items on resumption.
51
+ * Failed requests terminate iteration and retain their token for a new iterator.
52
+ */
53
+ pageToken() {
54
+ return this.inner.pageToken() ?? undefined;
55
+ }
56
+ }
57
+ exports.Listing = Listing;
@@ -4,10 +4,9 @@ import { Table } from "./table";
4
4
  export declare const DEFINITION_META_KEY = "mv.definition";
5
5
  /**
6
6
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
7
- * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
8
- * native dedup results.
7
+ * format 2 being a query with `GROUP BY`.
9
8
  */
10
- export declare const DEFINITION_FORMAT = 4;
9
+ export declare const DEFINITION_FORMAT = 2;
11
10
  /**
12
11
  * The query that defines a materialized view, as stored:
13
12
  * `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
@@ -58,17 +57,11 @@ export declare class MaterializedView {
58
57
  /**
59
58
  * Recompute the view from its source.
60
59
  *
61
- * The refresh is incremental when the source's changes can be reconciled
62
- * into the view -- rows added, changed or removed since the last one --
63
- * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
64
- * refreshes to that source version instead of the latest.
65
- *
66
- * Concurrent refreshes of one view do not duplicate its rows. Two that
67
- * plan the same source rows conflict on commit, and the loser throws
68
- * rather than writing them a second time.
60
+ * Every refresh rebuilds the complete view. For local views,
61
+ * `sourceVersion` refreshes to that source version instead of the latest;
62
+ * remote SQL refreshes do not support source-version pinning.
69
63
  */
70
64
  refresh(options?: {
71
- full?: boolean;
72
65
  sourceVersion?: number;
73
66
  }): Promise<RefreshMaterializedViewResult>;
74
67
  }
@@ -11,10 +11,9 @@ exports.definitionFromJson = definitionFromJson;
11
11
  exports.DEFINITION_META_KEY = "mv.definition";
12
12
  /**
13
13
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
14
- * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
15
- * native dedup results.
14
+ * format 2 being a query with `GROUP BY`.
16
15
  */
17
- exports.DEFINITION_FORMAT = 4;
16
+ exports.DEFINITION_FORMAT = 2;
18
17
  /**
19
18
  * @internal Reject a numeric option N-API would otherwise silently coerce:
20
19
  * `Infinity` reaches Rust as 0, `1.5` as 1.
@@ -133,18 +132,13 @@ class MaterializedView {
133
132
  /**
134
133
  * Recompute the view from its source.
135
134
  *
136
- * The refresh is incremental when the source's changes can be reconciled
137
- * into the view -- rows added, changed or removed since the last one --
138
- * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
139
- * refreshes to that source version instead of the latest.
140
- *
141
- * Concurrent refreshes of one view do not duplicate its rows. Two that
142
- * plan the same source rows conflict on commit, and the loser throws
143
- * rather than writing them a second time.
135
+ * Every refresh rebuilds the complete view. For local views,
136
+ * `sourceVersion` refreshes to that source version instead of the latest;
137
+ * remote SQL refreshes do not support source-version pinning.
144
138
  */
145
139
  async refresh(options) {
146
140
  validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
147
- return await this.inner.refreshMaterializedView(options?.full, options?.sourceVersion);
141
+ return await this.inner.refreshMaterializedView(options?.sourceVersion);
148
142
  }
149
143
  }
150
144
  exports.MaterializedView = MaterializedView;
package/dist/native.d.ts CHANGED
@@ -32,7 +32,7 @@ export declare class Catalog {
32
32
  createDatabase(name: string, existOk?: boolean | undefined | null): Promise<Connection>
33
33
  connectDatabase(name: string): Promise<Connection>
34
34
  dropDatabase(name: string, ignoreMissing?: boolean | undefined | null): Promise<void>
35
- listDatabases(limit?: number | undefined | null, pageToken?: string | undefined | null): Promise<ListDatabasesResponse>
35
+ listDatabases(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): DatabaseNames
36
36
  }
37
37
 
38
38
  export declare class Connection {
@@ -58,7 +58,7 @@ export declare class Connection {
58
58
  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
59
59
  createMaterializedView(name: string, source: string, projections: Array<Array<string>> | undefined | null, filter: string | undefined | null, limit: number | undefined | null, withNoData: boolean): Promise<Table>
60
60
  openMaterializedView(name: string): Promise<Table>
61
- listMaterializedViews(): Promise<Array<string>>
61
+ listMaterializedViews(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
62
62
  /** Drop a materialized view. */
63
63
  dropMaterializedView(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
64
64
  /** Create a view: a named query planned on every read. */
@@ -73,7 +73,7 @@ export declare class Connection {
73
73
  /** Start dropping a view and return the job deleting its definition. */
74
74
  dropViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
75
75
  /** The names of the views in one namespace. */
76
- listViews(namespacePath?: Array<string> | undefined | null): Promise<Array<string>>
76
+ listViews(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
77
77
  /** Start dropping a materialized view and return its cleanup job. */
78
78
  dropMaterializedViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
79
79
  openTable(name: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
@@ -93,7 +93,7 @@ export declare class Connection {
93
93
  */
94
94
  openJob(jobId: string): Promise<Job>
95
95
  /** List server-side jobs across the database's tables. */
96
- listJobs(): Promise<Array<JobInfo>>
96
+ listJobs(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): JobListing
97
97
  /**
98
98
  * Request cancellation of a server-side job by id. Returns true if the
99
99
  * server accepted the cancellation, false if no such job exists.
@@ -125,6 +125,16 @@ export declare class Connection {
125
125
  renameTable(currentName: string, newName: string, currentNamespacePath?: Array<string> | undefined | null, newNamespacePath?: Array<string> | undefined | null): Promise<void>
126
126
  }
127
127
 
128
+ /** A lazy iterator over database names with inspectable pagination state. */
129
+ export declare class DatabaseNames {
130
+ /** Number of names cached without another REST request. */
131
+ numPageResults(): number
132
+ /** Continuation token for the next REST request. */
133
+ pageToken(): string | null
134
+ /** Fetch the next name, returning None when exhausted. */
135
+ next(): Promise<string | null>
136
+ }
137
+
128
138
  export declare class Index {
129
139
  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
130
140
  static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
@@ -193,6 +203,12 @@ export declare class Job {
193
203
  events(limit?: number | undefined | null, filter?: string | undefined | null): Promise<Buffer>
194
204
  }
195
205
 
206
+ export declare class JobListing {
207
+ numPageResults(): number
208
+ pageToken(): string | null
209
+ next(): Promise<JobInfo | null>
210
+ }
211
+
196
212
  export declare class JsFullTextQuery {
197
213
  static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
198
214
  static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
@@ -212,6 +228,12 @@ export declare class JsHeaderProvider {
212
228
  constructor(getHeadersCallback: () => Promise<Record<string, string>>)
213
229
  }
214
230
 
231
+ export declare class NameListing {
232
+ numPageResults(): number
233
+ pageToken(): string | null
234
+ next(): Promise<string | null>
235
+ }
236
+
215
237
  /**
216
238
  * A Rust-side view of a JS-constructed `Scannable`.
217
239
  *
@@ -331,6 +353,8 @@ export declare class Query {
331
353
 
332
354
  /** Typescript-style Async Iterator over RecordBatches */
333
355
  export declare class RecordBatchIterator {
356
+ /** Return the executed stream's output schema as an Arrow IPC file. */
357
+ schema(): Buffer
334
358
  next(): Promise<Buffer | null>
335
359
  }
336
360
 
@@ -410,7 +434,7 @@ export declare class Table {
410
434
  refreshColumn(column: string): Promise<RefreshColumnResult>
411
435
  refreshColumnAsync(column: string): Promise<Job>
412
436
  functionErrors(options?: FunctionErrorsOptions | undefined | null): Promise<FunctionErrors>
413
- refreshMaterializedView(full?: boolean | undefined | null, sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
437
+ refreshMaterializedView(sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
414
438
  materializedViewDefinition(): Promise<string>
415
439
  addColumnsWithSchema(schemaBuf: Buffer): Promise<AddColumnsResult>
416
440
  alterColumns(alterations: Array<ColumnAlteration>): Promise<AlterColumnsResult>
@@ -987,11 +1011,6 @@ export interface JobInfo {
987
1011
  /** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
988
1012
  export declare function lancedbMetricsCatalog(): Array<MetricDescription>
989
1013
 
990
- export interface ListDatabasesResponse {
991
- databases: Array<string>
992
- pageToken?: string
993
- }
994
-
995
1014
  export interface ListNamespacesResponse {
996
1015
  namespaces: Array<string>
997
1016
  pageToken?: string
@@ -1175,7 +1194,7 @@ export interface RefreshColumnResult {
1175
1194
  }
1176
1195
 
1177
1196
  export interface RefreshMaterializedViewResult {
1178
- /** How the view was brought up to date: "rebuild", "incremental" or "no_op". */
1197
+ /** How the view was brought up to date: "rebuild". */
1179
1198
  mode: string
1180
1199
  rowsWritten: number
1181
1200
  sourceVersion: number
@@ -1338,6 +1357,17 @@ export interface TableStatistics {
1338
1357
  totalBytes: number
1339
1358
  /** The number of rows in the table */
1340
1359
  numRows: number
1360
+ /**
1361
+ * The number of rows marked as deleted across all fragments of the table
1362
+ *
1363
+ * These rows are not included in `numRows`, but still occupy space on disk
1364
+ * until the table is compacted, so a large value here indicates that the
1365
+ * table should be optimized. Fragments in which every row was deleted are
1366
+ * dropped outright, so their rows are not counted here.
1367
+ *
1368
+ * Absent (`undefined`) when the backend does not report deletion counts.
1369
+ */
1370
+ numDeletedRows?: number
1341
1371
  /** The number of indices in the table */
1342
1372
  numIndices: number
1343
1373
  /** Statistics on table fragments */