@lancedb/lancedb 0.40.0 → 0.41.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NODEJS_THIRD_PARTY_LICENSES.md +8 -0
- package/dist/arrow.d.ts +18 -1
- package/dist/arrow.js +182 -13
- package/dist/blob.d.ts +56 -2
- package/dist/blob.js +106 -6
- package/dist/catalog.d.ts +40 -6
- package/dist/catalog.js +58 -9
- package/dist/connection.d.ts +19 -13
- package/dist/connection.js +12 -7
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -1
- package/dist/indices.d.ts +3 -1
- package/dist/listing.d.ts +41 -0
- package/dist/listing.js +57 -0
- package/dist/materialized_view.d.ts +5 -12
- package/dist/materialized_view.js +6 -12
- package/dist/native.d.ts +41 -11
- package/dist/native.js +55 -52
- package/dist/query.d.ts +7 -1
- package/dist/query.js +12 -15
- package/dist/sanitize.d.ts +19 -18
- package/dist/sanitize.js +74 -11
- package/dist/schema.js +31 -3
- package/dist/table.d.ts +9 -11
- package/dist/table.js +2 -2
- package/package.json +13 -9
package/dist/catalog.js
CHANGED
|
@@ -2,10 +2,45 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.Catalog = void 0;
|
|
5
|
+
exports.Catalog = exports.DatabaseNames = void 0;
|
|
6
6
|
exports.connectCatalog = connectCatalog;
|
|
7
7
|
const connection_1 = require("./connection");
|
|
8
8
|
const native_js_1 = require("./native.js");
|
|
9
|
+
/**
|
|
10
|
+
* A lazy async iterator of database names with pagination state.
|
|
11
|
+
* Returned by {@link Catalog.listDatabases}.
|
|
12
|
+
*/
|
|
13
|
+
class DatabaseNames {
|
|
14
|
+
inner;
|
|
15
|
+
/** @hidden */
|
|
16
|
+
constructor(inner) {
|
|
17
|
+
this.inner = inner;
|
|
18
|
+
}
|
|
19
|
+
[Symbol.asyncIterator]() {
|
|
20
|
+
return this;
|
|
21
|
+
}
|
|
22
|
+
/** Fetch the next name, requesting another page only when needed. */
|
|
23
|
+
async next() {
|
|
24
|
+
const value = await this.inner.next();
|
|
25
|
+
return value == null
|
|
26
|
+
? { done: true, value: undefined }
|
|
27
|
+
: { done: false, value };
|
|
28
|
+
}
|
|
29
|
+
/** Number of names available without another REST request. */
|
|
30
|
+
numPageResults() {
|
|
31
|
+
return this.inner.numPageResults();
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Token for the next REST request. Initially this is the supplied starting token.
|
|
35
|
+
* Returns undefined after the final page is fetched, even if names remain cached.
|
|
36
|
+
* Drain the cache before saving a token to avoid skipping names when resuming.
|
|
37
|
+
* A failed request terminates iteration and retains its token for a new iterator.
|
|
38
|
+
*/
|
|
39
|
+
pageToken() {
|
|
40
|
+
return this.inner.pageToken() ?? undefined;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.DatabaseNames = DatabaseNames;
|
|
9
44
|
/** A remote catalog manages databases through the server's root namespace. */
|
|
10
45
|
class Catalog {
|
|
11
46
|
inner;
|
|
@@ -29,15 +64,27 @@ class Catalog {
|
|
|
29
64
|
async dropDatabase(name, options = {}) {
|
|
30
65
|
await this.inner.dropDatabase(name, options.ignoreMissing);
|
|
31
66
|
}
|
|
32
|
-
/**
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
67
|
+
/**
|
|
68
|
+
* Iterate lazily over all database names using `for await...of`.
|
|
69
|
+
* pageToken resumes from a saved token; omitted starts at the beginning.
|
|
70
|
+
* pageLimit limits each REST response, not the total; omitted uses the server default.
|
|
71
|
+
* Request errors are raised during iteration and terminate the iterator.
|
|
72
|
+
*
|
|
73
|
+
* @example
|
|
74
|
+
* ```ts
|
|
75
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
76
|
+
* console.log(name);
|
|
77
|
+
* }
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
listDatabases(options = {}) {
|
|
81
|
+
if (options.pageLimit !== undefined &&
|
|
82
|
+
(!Number.isInteger(options.pageLimit) ||
|
|
83
|
+
options.pageLimit <= 0 ||
|
|
84
|
+
options.pageLimit > 2147483647)) {
|
|
38
85
|
throw new Error("Database list limit must be an integer between 1 and 2147483647");
|
|
39
86
|
}
|
|
40
|
-
return this.inner.listDatabases(options.
|
|
87
|
+
return new DatabaseNames(this.inner.listDatabases(options.pageToken, options.pageLimit));
|
|
41
88
|
}
|
|
42
89
|
}
|
|
43
90
|
exports.Catalog = Catalog;
|
|
@@ -49,7 +96,9 @@ exports.Catalog = Catalog;
|
|
|
49
96
|
* ```ts
|
|
50
97
|
* const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" });
|
|
51
98
|
* const db = await catalog.createDatabase("analytics", { existOk: true });
|
|
52
|
-
* const
|
|
99
|
+
* for await (const name of catalog.listDatabases({ pageLimit: 20 })) {
|
|
100
|
+
* console.log(name);
|
|
101
|
+
* }
|
|
53
102
|
* ```
|
|
54
103
|
*/
|
|
55
104
|
async function connectCatalog(endpoint, options = {}) {
|
package/dist/connection.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
2
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
3
3
|
import { Job } from "./job";
|
|
4
|
+
import { Listing, ListingOptions } from "./listing";
|
|
4
5
|
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
5
6
|
import { Connection as LanceDbConnection } from "./native";
|
|
6
7
|
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
|
|
@@ -107,7 +108,7 @@ export interface TableNamesOptions {
|
|
|
107
108
|
* the last table name from the previous page.
|
|
108
109
|
*/
|
|
109
110
|
startAfter?: string;
|
|
110
|
-
/**
|
|
111
|
+
/** The maximum number of names to return. Omitted returns all names; zero returns none. */
|
|
111
112
|
limit?: number;
|
|
112
113
|
}
|
|
113
114
|
export interface ListTablesOptions {
|
|
@@ -120,6 +121,7 @@ export interface ListTablesOptions {
|
|
|
120
121
|
pageToken?: string;
|
|
121
122
|
/**
|
|
122
123
|
* An upper bound on how many tables to return.
|
|
124
|
+
* Defaults to 100. Zero returns an empty page without a continuation token.
|
|
123
125
|
*
|
|
124
126
|
* A page may hold fewer than this and still not be the last one, so keep
|
|
125
127
|
* going while the response carries a page token rather than while pages are
|
|
@@ -260,10 +262,6 @@ export declare abstract class Connection {
|
|
|
260
262
|
* The view is populated before creation returns. Set `withNoData` to create
|
|
261
263
|
* only its definition and empty backing table. The view is a normal table:
|
|
262
264
|
* it can be queried, indexed and searched, and it appears in `tableNames`.
|
|
263
|
-
* The source table must have stable row ids (create it with
|
|
264
|
-
* the `newTableEnableStableRowIds` storage option); they keep the view's
|
|
265
|
-
* provenance valid across source compactions and cannot be enabled after
|
|
266
|
-
* a table exists.
|
|
267
265
|
*/
|
|
268
266
|
abstract createMaterializedView(name: string, source: string, options?: {
|
|
269
267
|
select?: MaterializedViewSelect;
|
|
@@ -280,9 +278,11 @@ export declare abstract class Connection {
|
|
|
280
278
|
/**
|
|
281
279
|
* The names of the materialized views in this database.
|
|
282
280
|
*
|
|
283
|
-
*
|
|
281
|
+
* Iterate lazily with `for await...of`. Local listings inspect table schemas
|
|
282
|
+
* one page at a time. Options control page size and the starting token.
|
|
283
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
284
284
|
*/
|
|
285
|
-
abstract listMaterializedViews():
|
|
285
|
+
abstract listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
286
286
|
/**
|
|
287
287
|
* Drop the materialized view named `name`.
|
|
288
288
|
*
|
|
@@ -334,8 +334,10 @@ export declare abstract class Connection {
|
|
|
334
334
|
* The names of the views in one namespace.
|
|
335
335
|
*
|
|
336
336
|
* Names only; a definition comes from {@link describeView}.
|
|
337
|
+
* Iterate lazily with `for await...of`. See {@link Listing} for pagination
|
|
338
|
+
* state and error behavior; options control page size and the starting token.
|
|
337
339
|
*/
|
|
338
|
-
abstract listViews(namespacePath?: string[]):
|
|
340
|
+
abstract listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
339
341
|
abstract openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
340
342
|
/**
|
|
341
343
|
* Creates a new Table and initialize it with new data.
|
|
@@ -492,8 +494,12 @@ export declare abstract class Connection {
|
|
|
492
494
|
* connection-level call for any of them.
|
|
493
495
|
*/
|
|
494
496
|
abstract openJob(jobId: string): Promise<Job>;
|
|
495
|
-
/**
|
|
496
|
-
|
|
497
|
+
/**
|
|
498
|
+
* Iterate lazily over server-side jobs across the database's tables.
|
|
499
|
+
* Use `for await...of`. Options control page size and the starting token.
|
|
500
|
+
* See {@link Listing} for pagination state and error behavior.
|
|
501
|
+
*/
|
|
502
|
+
abstract listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
497
503
|
/**
|
|
498
504
|
* Request cancellation of a server-side job by id.
|
|
499
505
|
*
|
|
@@ -534,14 +540,14 @@ export declare class LocalConnection extends Connection {
|
|
|
534
540
|
withNoData?: boolean;
|
|
535
541
|
}): Promise<MaterializedView>;
|
|
536
542
|
openMaterializedView(name: string): Promise<MaterializedView>;
|
|
537
|
-
listMaterializedViews():
|
|
543
|
+
listMaterializedViews(options?: ListingOptions): Listing<string>;
|
|
538
544
|
dropMaterializedView(name: string, namespacePath?: string[]): Promise<void>;
|
|
539
545
|
dropMaterializedViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
540
546
|
createView(name: string, query: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
541
547
|
describeView(name: string, namespacePath?: string[]): Promise<ViewDescription>;
|
|
542
548
|
dropView(name: string, namespacePath?: string[]): Promise<void>;
|
|
543
549
|
dropViewAsync(name: string, namespacePath?: string[]): Promise<Job>;
|
|
544
|
-
listViews(namespacePath?: string[]):
|
|
550
|
+
listViews(namespacePath?: string[], options?: ListingOptions): Listing<string>;
|
|
545
551
|
listTables(namespacePathOrOptions?: string[] | Partial<ListTablesOptions>, options?: Partial<ListTablesOptions>): Promise<ListTablesResponse>;
|
|
546
552
|
openTable(name: string, namespacePath?: string[], options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
547
553
|
cloneTable(targetTableName: string, sourceUri: string, options?: {
|
|
@@ -566,7 +572,7 @@ export declare class LocalConnection extends Connection {
|
|
|
566
572
|
dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
|
|
567
573
|
renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
|
568
574
|
openJob(jobId: string): Promise<Job>;
|
|
569
|
-
listJobs():
|
|
575
|
+
listJobs(options?: ListingOptions): Listing<JobInfo>;
|
|
570
576
|
cancelJob(jobId: string): Promise<boolean>;
|
|
571
577
|
pauseJob(jobId: string): Promise<string>;
|
|
572
578
|
resumeJob(jobId: string): Promise<string>;
|
package/dist/connection.js
CHANGED
|
@@ -8,6 +8,7 @@ const arrow_1 = require("./arrow");
|
|
|
8
8
|
const arrow_2 = require("./arrow");
|
|
9
9
|
const registry_1 = require("./embedding/registry");
|
|
10
10
|
const job_1 = require("./job");
|
|
11
|
+
const listing_1 = require("./listing");
|
|
11
12
|
const materialized_view_1 = require("./materialized_view");
|
|
12
13
|
const view_1 = require("./view");
|
|
13
14
|
const sanitize_1 = require("./sanitize");
|
|
@@ -79,8 +80,9 @@ class LocalConnection extends Connection {
|
|
|
79
80
|
const innerTable = await this.inner.openMaterializedView(name);
|
|
80
81
|
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
81
82
|
}
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
listMaterializedViews(options = {}) {
|
|
84
|
+
(0, listing_1.validateListingOptions)(options);
|
|
85
|
+
return new listing_1.Listing(this.inner.listMaterializedViews(options.pageToken, options.pageLimit));
|
|
84
86
|
}
|
|
85
87
|
async dropMaterializedView(name, namespacePath) {
|
|
86
88
|
return this.inner.dropMaterializedView(name, namespacePath ?? []);
|
|
@@ -100,8 +102,9 @@ class LocalConnection extends Connection {
|
|
|
100
102
|
async dropViewAsync(name, namespacePath) {
|
|
101
103
|
return new job_1.Job(await this.inner.dropViewAsync(name, namespacePath ?? []));
|
|
102
104
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
+
listViews(namespacePath, options = {}) {
|
|
106
|
+
(0, listing_1.validateListingOptions)(options);
|
|
107
|
+
return new listing_1.Listing(this.inner.listViews(namespacePath ?? [], options.pageToken, options.pageLimit));
|
|
105
108
|
}
|
|
106
109
|
async listTables(namespacePathOrOptions, options) {
|
|
107
110
|
// Detect if first argument is namespacePath array or options object
|
|
@@ -243,8 +246,9 @@ class LocalConnection extends Connection {
|
|
|
243
246
|
async openJob(jobId) {
|
|
244
247
|
return new job_1.Job(await this.inner.openJob(jobId));
|
|
245
248
|
}
|
|
246
|
-
|
|
247
|
-
|
|
249
|
+
listJobs(options = {}) {
|
|
250
|
+
(0, listing_1.validateListingOptions)(options);
|
|
251
|
+
return new listing_1.Listing(this.inner.listJobs(options.pageToken, options.pageLimit));
|
|
248
252
|
}
|
|
249
253
|
async cancelJob(jobId) {
|
|
250
254
|
return this.inner.cancelJob(jobId);
|
|
@@ -303,7 +307,8 @@ async function parseTableData(data, options, streaming = false) {
|
|
|
303
307
|
table = (0, sanitize_1.sanitizeTable)(data);
|
|
304
308
|
}
|
|
305
309
|
else {
|
|
306
|
-
|
|
310
|
+
const records = await (0, arrow_1.resolveBlobInputs)(data, options?.schema);
|
|
311
|
+
table = (0, arrow_1.makeArrowTable)(records, options);
|
|
307
312
|
}
|
|
308
313
|
if (streaming) {
|
|
309
314
|
const buf = await (0, arrow_1.fromTableToStreamBuffer)(table, options?.embeddingFunction, options?.schema);
|
package/dist/index.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export { instrumentLanceDbMetrics } from "./otel";
|
|
|
10
10
|
export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
|
|
11
11
|
export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
12
12
|
export { blob, isBlobField, BlobFile } from "./blob";
|
|
13
|
-
export type { BlobOptions, BlobRange, BlobReadOptions } from "./blob";
|
|
13
|
+
export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, BlobUri, } from "./blob";
|
|
14
14
|
export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
15
15
|
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
16
16
|
export { Job, JobEventsOptions } from "./job";
|
|
@@ -24,7 +24,7 @@ export * as embedding from "./embedding";
|
|
|
24
24
|
export { permutationBuilder, PermutationBuilder } from "./permutation";
|
|
25
25
|
export { Scannable, ScannableOptions } from "./scannable";
|
|
26
26
|
export * as rerankers from "./rerankers";
|
|
27
|
-
export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
|
|
27
|
+
export { SchemaLike, TableLike, DataTypeLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
|
|
28
28
|
export { IntoSql, packBits } from "./util";
|
|
29
29
|
/**
|
|
30
30
|
* Options for tokenizing a full-text search query without a table index.
|
|
@@ -248,4 +248,5 @@ export declare function connectNamespace(implName: "rest", config: RestNamespace
|
|
|
248
248
|
* ```
|
|
249
249
|
*/
|
|
250
250
|
export declare function connectNamespace(implName: string, properties: Record<string, string>, options?: Partial<ConnectNamespaceOptions>): Promise<Connection>;
|
|
251
|
-
export { Catalog, CatalogOptions,
|
|
251
|
+
export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
|
|
252
|
+
export { Listing, ListingOptions } from "./listing";
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.connectCatalog = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
5
|
+
exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
6
6
|
exports.tokenize = tokenize;
|
|
7
7
|
exports.connect = connect;
|
|
8
8
|
exports.connectNamespace = connectNamespace;
|
|
@@ -184,4 +184,7 @@ async function connectNamespace(implName, configOrProperties, options) {
|
|
|
184
184
|
}
|
|
185
185
|
var catalog_1 = require("./catalog");
|
|
186
186
|
Object.defineProperty(exports, "Catalog", { enumerable: true, get: function () { return catalog_1.Catalog; } });
|
|
187
|
+
Object.defineProperty(exports, "DatabaseNames", { enumerable: true, get: function () { return catalog_1.DatabaseNames; } });
|
|
187
188
|
Object.defineProperty(exports, "connectCatalog", { enumerable: true, get: function () { return catalog_1.connectCatalog; } });
|
|
189
|
+
var listing_1 = require("./listing");
|
|
190
|
+
Object.defineProperty(exports, "Listing", { enumerable: true, get: function () { return listing_1.Listing; } });
|
package/dist/indices.d.ts
CHANGED
|
@@ -453,7 +453,7 @@ export interface IvfFlatOptions {
|
|
|
453
453
|
*/
|
|
454
454
|
sampleRate?: number;
|
|
455
455
|
}
|
|
456
|
-
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
456
|
+
export type BaseTokenizer = "simple" | "whitespace" | "raw" | "ngram" | "code" | "icu" | "icu/split" | `jieba/${string}` | `lindera/${string}`;
|
|
457
457
|
/**
|
|
458
458
|
* Options to create a full text search index
|
|
459
459
|
*/
|
|
@@ -477,6 +477,8 @@ export interface FtsOptions {
|
|
|
477
477
|
*
|
|
478
478
|
* "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
|
|
479
479
|
*
|
|
480
|
+
* "code" - Code tokenizer. This tokenizer keeps snake_case and camelCase identifiers as single tokens.
|
|
481
|
+
*
|
|
480
482
|
* "icu" - ICU dictionary-based word segmentation.
|
|
481
483
|
*
|
|
482
484
|
* "icu/split" - ICU segmentation with simple-style delimiter splitting.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/** Pagination controls for lazy resource listings. */
|
|
2
|
+
export interface ListingOptions {
|
|
3
|
+
/** Starting continuation token; omitted starts at the beginning. */
|
|
4
|
+
pageToken?: string;
|
|
5
|
+
/** Maximum results per request, not a limit on the whole listing. */
|
|
6
|
+
pageLimit?: number;
|
|
7
|
+
}
|
|
8
|
+
/** @internal */
|
|
9
|
+
export declare function validateListingOptions(options: ListingOptions): void;
|
|
10
|
+
/**
|
|
11
|
+
* A lazy async iterator over database resources with pagination state.
|
|
12
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```ts
|
|
16
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
17
|
+
* console.log(name);
|
|
18
|
+
* }
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
export declare class Listing<T> implements AsyncIterableIterator<T> {
|
|
22
|
+
private readonly inner;
|
|
23
|
+
/** @hidden */
|
|
24
|
+
constructor(inner: {
|
|
25
|
+
next(): Promise<T | null | undefined>;
|
|
26
|
+
numPageResults(): number;
|
|
27
|
+
pageToken(): string | null | undefined;
|
|
28
|
+
});
|
|
29
|
+
[Symbol.asyncIterator](): AsyncIterableIterator<T>;
|
|
30
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
31
|
+
next(): Promise<IteratorResult<T>>;
|
|
32
|
+
/** Number of cached items available without another request. */
|
|
33
|
+
numPageResults(): number;
|
|
34
|
+
/**
|
|
35
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
36
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
37
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
38
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
39
|
+
*/
|
|
40
|
+
pageToken(): string | undefined;
|
|
41
|
+
}
|
package/dist/listing.js
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.Listing = void 0;
|
|
6
|
+
exports.validateListingOptions = validateListingOptions;
|
|
7
|
+
/** @internal */
|
|
8
|
+
function validateListingOptions(options) {
|
|
9
|
+
if (options.pageLimit !== undefined &&
|
|
10
|
+
(!Number.isInteger(options.pageLimit) ||
|
|
11
|
+
options.pageLimit <= 0 ||
|
|
12
|
+
options.pageLimit > 2147483647)) {
|
|
13
|
+
throw new Error("Listing page limit must be an integer between 1 and 2147483647");
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A lazy async iterator over database resources with pagination state.
|
|
18
|
+
* Requests occur during iteration; request errors terminate the iterator.
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```ts
|
|
22
|
+
* for await (const name of connection.listViews([], { pageLimit: 20 })) {
|
|
23
|
+
* console.log(name);
|
|
24
|
+
* }
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
class Listing {
|
|
28
|
+
inner;
|
|
29
|
+
/** @hidden */
|
|
30
|
+
constructor(inner) {
|
|
31
|
+
this.inner = inner;
|
|
32
|
+
}
|
|
33
|
+
[Symbol.asyncIterator]() {
|
|
34
|
+
return this;
|
|
35
|
+
}
|
|
36
|
+
/** Fetch the next item, requesting another page only when needed. */
|
|
37
|
+
async next() {
|
|
38
|
+
const value = await this.inner.next();
|
|
39
|
+
return value == null
|
|
40
|
+
? { done: true, value: undefined }
|
|
41
|
+
: { done: false, value };
|
|
42
|
+
}
|
|
43
|
+
/** Number of cached items available without another request. */
|
|
44
|
+
numPageResults() {
|
|
45
|
+
return this.inner.numPageResults();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Token for the next request. Initially this is the supplied starting token.
|
|
49
|
+
* Returns undefined after fetching the final page, even with cached items.
|
|
50
|
+
* Drain the cache before saving a token to avoid skipping items on resumption.
|
|
51
|
+
* Failed requests terminate iteration and retain their token for a new iterator.
|
|
52
|
+
*/
|
|
53
|
+
pageToken() {
|
|
54
|
+
return this.inner.pageToken() ?? undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
exports.Listing = Listing;
|
|
@@ -4,10 +4,9 @@ import { Table } from "./table";
|
|
|
4
4
|
export declare const DEFINITION_META_KEY = "mv.definition";
|
|
5
5
|
/**
|
|
6
6
|
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
|
|
7
|
-
* format 2 being a query with `GROUP BY
|
|
8
|
-
* native dedup results.
|
|
7
|
+
* format 2 being a query with `GROUP BY`.
|
|
9
8
|
*/
|
|
10
|
-
export declare const DEFINITION_FORMAT =
|
|
9
|
+
export declare const DEFINITION_FORMAT = 2;
|
|
11
10
|
/**
|
|
12
11
|
* The query that defines a materialized view, as stored:
|
|
13
12
|
* `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
|
|
@@ -58,17 +57,11 @@ export declare class MaterializedView {
|
|
|
58
57
|
/**
|
|
59
58
|
* Recompute the view from its source.
|
|
60
59
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* refreshes to that source version instead of the latest.
|
|
65
|
-
*
|
|
66
|
-
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
|
67
|
-
* plan the same source rows conflict on commit, and the loser throws
|
|
68
|
-
* rather than writing them a second time.
|
|
60
|
+
* Every refresh rebuilds the complete view. For local views,
|
|
61
|
+
* `sourceVersion` refreshes to that source version instead of the latest;
|
|
62
|
+
* remote SQL refreshes do not support source-version pinning.
|
|
69
63
|
*/
|
|
70
64
|
refresh(options?: {
|
|
71
|
-
full?: boolean;
|
|
72
65
|
sourceVersion?: number;
|
|
73
66
|
}): Promise<RefreshMaterializedViewResult>;
|
|
74
67
|
}
|
|
@@ -11,10 +11,9 @@ exports.definitionFromJson = definitionFromJson;
|
|
|
11
11
|
exports.DEFINITION_META_KEY = "mv.definition";
|
|
12
12
|
/**
|
|
13
13
|
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
|
|
14
|
-
* format 2 being a query with `GROUP BY
|
|
15
|
-
* native dedup results.
|
|
14
|
+
* format 2 being a query with `GROUP BY`.
|
|
16
15
|
*/
|
|
17
|
-
exports.DEFINITION_FORMAT =
|
|
16
|
+
exports.DEFINITION_FORMAT = 2;
|
|
18
17
|
/**
|
|
19
18
|
* @internal Reject a numeric option N-API would otherwise silently coerce:
|
|
20
19
|
* `Infinity` reaches Rust as 0, `1.5` as 1.
|
|
@@ -133,18 +132,13 @@ class MaterializedView {
|
|
|
133
132
|
/**
|
|
134
133
|
* Recompute the view from its source.
|
|
135
134
|
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
* refreshes to that source version instead of the latest.
|
|
140
|
-
*
|
|
141
|
-
* Concurrent refreshes of one view do not duplicate its rows. Two that
|
|
142
|
-
* plan the same source rows conflict on commit, and the loser throws
|
|
143
|
-
* rather than writing them a second time.
|
|
135
|
+
* Every refresh rebuilds the complete view. For local views,
|
|
136
|
+
* `sourceVersion` refreshes to that source version instead of the latest;
|
|
137
|
+
* remote SQL refreshes do not support source-version pinning.
|
|
144
138
|
*/
|
|
145
139
|
async refresh(options) {
|
|
146
140
|
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
|
|
147
|
-
return await this.inner.refreshMaterializedView(options?.
|
|
141
|
+
return await this.inner.refreshMaterializedView(options?.sourceVersion);
|
|
148
142
|
}
|
|
149
143
|
}
|
|
150
144
|
exports.MaterializedView = MaterializedView;
|
package/dist/native.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export declare class Catalog {
|
|
|
32
32
|
createDatabase(name: string, existOk?: boolean | undefined | null): Promise<Connection>
|
|
33
33
|
connectDatabase(name: string): Promise<Connection>
|
|
34
34
|
dropDatabase(name: string, ignoreMissing?: boolean | undefined | null): Promise<void>
|
|
35
|
-
listDatabases(
|
|
35
|
+
listDatabases(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): DatabaseNames
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export declare class Connection {
|
|
@@ -58,7 +58,7 @@ export declare class Connection {
|
|
|
58
58
|
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
59
59
|
createMaterializedView(name: string, source: string, projections: Array<Array<string>> | undefined | null, filter: string | undefined | null, limit: number | undefined | null, withNoData: boolean): Promise<Table>
|
|
60
60
|
openMaterializedView(name: string): Promise<Table>
|
|
61
|
-
listMaterializedViews():
|
|
61
|
+
listMaterializedViews(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
|
|
62
62
|
/** Drop a materialized view. */
|
|
63
63
|
dropMaterializedView(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
|
|
64
64
|
/** Create a view: a named query planned on every read. */
|
|
@@ -73,7 +73,7 @@ export declare class Connection {
|
|
|
73
73
|
/** Start dropping a view and return the job deleting its definition. */
|
|
74
74
|
dropViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
|
|
75
75
|
/** The names of the views in one namespace. */
|
|
76
|
-
listViews(namespacePath?: Array<string> | undefined | null):
|
|
76
|
+
listViews(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
|
|
77
77
|
/** Start dropping a materialized view and return its cleanup job. */
|
|
78
78
|
dropMaterializedViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
|
|
79
79
|
openTable(name: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
|
|
@@ -93,7 +93,7 @@ export declare class Connection {
|
|
|
93
93
|
*/
|
|
94
94
|
openJob(jobId: string): Promise<Job>
|
|
95
95
|
/** List server-side jobs across the database's tables. */
|
|
96
|
-
listJobs():
|
|
96
|
+
listJobs(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): JobListing
|
|
97
97
|
/**
|
|
98
98
|
* Request cancellation of a server-side job by id. Returns true if the
|
|
99
99
|
* server accepted the cancellation, false if no such job exists.
|
|
@@ -125,6 +125,16 @@ export declare class Connection {
|
|
|
125
125
|
renameTable(currentName: string, newName: string, currentNamespacePath?: Array<string> | undefined | null, newNamespacePath?: Array<string> | undefined | null): Promise<void>
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
/** A lazy iterator over database names with inspectable pagination state. */
|
|
129
|
+
export declare class DatabaseNames {
|
|
130
|
+
/** Number of names cached without another REST request. */
|
|
131
|
+
numPageResults(): number
|
|
132
|
+
/** Continuation token for the next REST request. */
|
|
133
|
+
pageToken(): string | null
|
|
134
|
+
/** Fetch the next name, returning None when exhausted. */
|
|
135
|
+
next(): Promise<string | null>
|
|
136
|
+
}
|
|
137
|
+
|
|
128
138
|
export declare class Index {
|
|
129
139
|
static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
|
|
130
140
|
static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
|
|
@@ -193,6 +203,12 @@ export declare class Job {
|
|
|
193
203
|
events(limit?: number | undefined | null, filter?: string | undefined | null): Promise<Buffer>
|
|
194
204
|
}
|
|
195
205
|
|
|
206
|
+
export declare class JobListing {
|
|
207
|
+
numPageResults(): number
|
|
208
|
+
pageToken(): string | null
|
|
209
|
+
next(): Promise<JobInfo | null>
|
|
210
|
+
}
|
|
211
|
+
|
|
196
212
|
export declare class JsFullTextQuery {
|
|
197
213
|
static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
|
|
198
214
|
static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
|
|
@@ -212,6 +228,12 @@ export declare class JsHeaderProvider {
|
|
|
212
228
|
constructor(getHeadersCallback: () => Promise<Record<string, string>>)
|
|
213
229
|
}
|
|
214
230
|
|
|
231
|
+
export declare class NameListing {
|
|
232
|
+
numPageResults(): number
|
|
233
|
+
pageToken(): string | null
|
|
234
|
+
next(): Promise<string | null>
|
|
235
|
+
}
|
|
236
|
+
|
|
215
237
|
/**
|
|
216
238
|
* A Rust-side view of a JS-constructed `Scannable`.
|
|
217
239
|
*
|
|
@@ -331,6 +353,8 @@ export declare class Query {
|
|
|
331
353
|
|
|
332
354
|
/** Typescript-style Async Iterator over RecordBatches */
|
|
333
355
|
export declare class RecordBatchIterator {
|
|
356
|
+
/** Return the executed stream's output schema as an Arrow IPC file. */
|
|
357
|
+
schema(): Buffer
|
|
334
358
|
next(): Promise<Buffer | null>
|
|
335
359
|
}
|
|
336
360
|
|
|
@@ -410,7 +434,7 @@ export declare class Table {
|
|
|
410
434
|
refreshColumn(column: string): Promise<RefreshColumnResult>
|
|
411
435
|
refreshColumnAsync(column: string): Promise<Job>
|
|
412
436
|
functionErrors(options?: FunctionErrorsOptions | undefined | null): Promise<FunctionErrors>
|
|
413
|
-
refreshMaterializedView(
|
|
437
|
+
refreshMaterializedView(sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
|
|
414
438
|
materializedViewDefinition(): Promise<string>
|
|
415
439
|
addColumnsWithSchema(schemaBuf: Buffer): Promise<AddColumnsResult>
|
|
416
440
|
alterColumns(alterations: Array<ColumnAlteration>): Promise<AlterColumnsResult>
|
|
@@ -987,11 +1011,6 @@ export interface JobInfo {
|
|
|
987
1011
|
/** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
|
|
988
1012
|
export declare function lancedbMetricsCatalog(): Array<MetricDescription>
|
|
989
1013
|
|
|
990
|
-
export interface ListDatabasesResponse {
|
|
991
|
-
databases: Array<string>
|
|
992
|
-
pageToken?: string
|
|
993
|
-
}
|
|
994
|
-
|
|
995
1014
|
export interface ListNamespacesResponse {
|
|
996
1015
|
namespaces: Array<string>
|
|
997
1016
|
pageToken?: string
|
|
@@ -1175,7 +1194,7 @@ export interface RefreshColumnResult {
|
|
|
1175
1194
|
}
|
|
1176
1195
|
|
|
1177
1196
|
export interface RefreshMaterializedViewResult {
|
|
1178
|
-
/** How the view was brought up to date: "rebuild"
|
|
1197
|
+
/** How the view was brought up to date: "rebuild". */
|
|
1179
1198
|
mode: string
|
|
1180
1199
|
rowsWritten: number
|
|
1181
1200
|
sourceVersion: number
|
|
@@ -1338,6 +1357,17 @@ export interface TableStatistics {
|
|
|
1338
1357
|
totalBytes: number
|
|
1339
1358
|
/** The number of rows in the table */
|
|
1340
1359
|
numRows: number
|
|
1360
|
+
/**
|
|
1361
|
+
* The number of rows marked as deleted across all fragments of the table
|
|
1362
|
+
*
|
|
1363
|
+
* These rows are not included in `numRows`, but still occupy space on disk
|
|
1364
|
+
* until the table is compacted, so a large value here indicates that the
|
|
1365
|
+
* table should be optimized. Fragments in which every row was deleted are
|
|
1366
|
+
* dropped outright, so their rows are not counted here.
|
|
1367
|
+
*
|
|
1368
|
+
* Absent (`undefined`) when the backend does not report deletion counts.
|
|
1369
|
+
*/
|
|
1370
|
+
numDeletedRows?: number
|
|
1341
1371
|
/** The number of indices in the table */
|
|
1342
1372
|
numIndices: number
|
|
1343
1373
|
/** Statistics on table fragments */
|