@lancedb/lancedb 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +3 -3
- package/biome.json +19 -3
- package/dist/arrow.d.ts +42 -7
- package/dist/arrow.js +6 -5
- package/dist/connection.d.ts +55 -29
- package/dist/connection.js +22 -74
- package/dist/embedding/embedding_function.d.ts +11 -3
- package/dist/embedding/embedding_function.js +36 -12
- package/dist/embedding/openai.d.ts +6 -5
- package/dist/embedding/openai.js +4 -2
- package/dist/embedding/registry.d.ts +10 -11
- package/dist/embedding/registry.js +4 -0
- package/dist/index.d.ts +51 -3
- package/dist/index.js +28 -4
- package/dist/merge.d.ts +54 -0
- package/dist/merge.js +64 -0
- package/dist/native.d.ts +34 -7
- package/dist/native.js +26 -9
- package/dist/query.d.ts +51 -16
- package/dist/query.js +122 -21
- package/dist/remote/client.d.ts +28 -0
- package/dist/remote/client.js +172 -0
- package/dist/remote/connection.d.ts +25 -0
- package/dist/remote/connection.js +110 -0
- package/dist/remote/index.d.ts +3 -0
- package/dist/remote/index.js +9 -0
- package/dist/remote/table.d.ts +42 -0
- package/dist/remote/table.js +179 -0
- package/dist/sanitize.d.ts +3 -2
- package/dist/sanitize.js +55 -1
- package/dist/table.d.ts +116 -25
- package/dist/table.js +117 -233
- package/dist/util.d.ts +14 -0
- package/dist/util.js +65 -0
- package/examples/ann_indexes.ts +49 -0
- package/examples/basic.ts +149 -0
- package/examples/embedding.ts +83 -0
- package/examples/filtering.ts +34 -0
- package/examples/jsconfig.json +27 -0
- package/examples/package-lock.json +79 -0
- package/examples/package.json +18 -0
- package/examples/search.ts +37 -0
- package/lancedb/arrow.ts +87 -24
- package/lancedb/connection.ts +115 -92
- package/lancedb/embedding/embedding_function.ts +48 -16
- package/lancedb/embedding/openai.ts +11 -6
- package/lancedb/embedding/registry.ts +38 -22
- package/lancedb/index.ts +101 -2
- package/lancedb/merge.ts +70 -0
- package/lancedb/query.ts +168 -39
- package/lancedb/remote/client.ts +221 -0
- package/lancedb/remote/connection.ts +201 -0
- package/lancedb/remote/index.ts +3 -0
- package/lancedb/remote/table.ts +226 -0
- package/lancedb/sanitize.ts +73 -1
- package/lancedb/table.ts +344 -101
- package/lancedb/util.ts +69 -0
- package/native.d.ts +208 -0
- package/nodejs-artifacts/arrow.d.ts +42 -7
- package/nodejs-artifacts/arrow.js +6 -5
- package/nodejs-artifacts/connection.d.ts +55 -29
- package/nodejs-artifacts/connection.js +22 -74
- package/nodejs-artifacts/embedding/embedding_function.d.ts +11 -3
- package/nodejs-artifacts/embedding/embedding_function.js +36 -12
- package/nodejs-artifacts/embedding/openai.d.ts +6 -5
- package/nodejs-artifacts/embedding/openai.js +4 -2
- package/nodejs-artifacts/embedding/registry.d.ts +10 -11
- package/nodejs-artifacts/embedding/registry.js +4 -0
- package/nodejs-artifacts/index.d.ts +51 -3
- package/nodejs-artifacts/index.js +28 -4
- package/nodejs-artifacts/merge.d.ts +54 -0
- package/nodejs-artifacts/merge.js +64 -0
- package/nodejs-artifacts/native.d.ts +34 -7
- package/nodejs-artifacts/native.js +26 -9
- package/nodejs-artifacts/query.d.ts +51 -16
- package/nodejs-artifacts/query.js +122 -21
- package/nodejs-artifacts/remote/client.d.ts +28 -0
- package/nodejs-artifacts/remote/client.js +172 -0
- package/nodejs-artifacts/remote/connection.d.ts +25 -0
- package/nodejs-artifacts/remote/connection.js +110 -0
- package/nodejs-artifacts/remote/index.d.ts +3 -0
- package/nodejs-artifacts/remote/index.js +9 -0
- package/nodejs-artifacts/remote/table.d.ts +42 -0
- package/nodejs-artifacts/remote/table.js +179 -0
- package/nodejs-artifacts/sanitize.d.ts +3 -2
- package/nodejs-artifacts/sanitize.js +55 -1
- package/nodejs-artifacts/table.d.ts +116 -25
- package/nodejs-artifacts/table.js +117 -233
- package/nodejs-artifacts/util.d.ts +14 -0
- package/nodejs-artifacts/util.js +65 -0
- package/package.json +25 -11
|
@@ -1,13 +1,8 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
|
|
2
2
|
import "reflect-metadata";
|
|
3
|
-
|
|
4
|
-
[key: string]: unknown;
|
|
5
|
-
}
|
|
6
|
-
export interface EmbeddingFunctionFactory<T extends EmbeddingFunction = EmbeddingFunction> {
|
|
7
|
-
new (modelOptions?: EmbeddingFunctionOptions): T;
|
|
8
|
-
}
|
|
3
|
+
import { OpenAIEmbeddingFunction } from "./openai";
|
|
9
4
|
interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
|
|
10
|
-
create(options?:
|
|
5
|
+
create(options?: T["TOptions"]): T;
|
|
11
6
|
}
|
|
12
7
|
/**
|
|
13
8
|
* This is a singleton class used to register embedding functions
|
|
@@ -21,22 +16,26 @@ export declare class EmbeddingFunctionRegistry {
|
|
|
21
16
|
* Register an embedding function
|
|
22
17
|
* @param name The name of the function
|
|
23
18
|
* @param func The function to register
|
|
19
|
+
* @throws Error if the function is already registered
|
|
24
20
|
*/
|
|
25
|
-
register<T extends
|
|
21
|
+
register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
|
|
26
22
|
/**
|
|
27
23
|
* Fetch an embedding function by name
|
|
28
24
|
* @param name The name of the function
|
|
29
25
|
*/
|
|
30
|
-
get<T extends EmbeddingFunction<unknown
|
|
26
|
+
get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
|
|
31
27
|
/**
|
|
32
28
|
* reset the registry to the initial state
|
|
33
29
|
*/
|
|
34
30
|
reset(this: EmbeddingFunctionRegistry): void;
|
|
31
|
+
/**
|
|
32
|
+
* @ignore
|
|
33
|
+
*/
|
|
35
34
|
parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Map<string, EmbeddingFunctionConfig>;
|
|
36
35
|
functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
|
|
37
36
|
getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
|
|
38
37
|
}
|
|
39
|
-
export declare function register(name?: string): (ctor:
|
|
38
|
+
export declare function register(name?: string): (ctor: EmbeddingFunctionConstructor<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
|
|
40
39
|
/**
|
|
41
40
|
* Utility function to get the global instance of the registry
|
|
42
41
|
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
|
@@ -27,6 +27,7 @@ class EmbeddingFunctionRegistry {
|
|
|
27
27
|
* Register an embedding function
|
|
28
28
|
* @param name The name of the function
|
|
29
29
|
* @param func The function to register
|
|
30
|
+
* @throws Error if the function is already registered
|
|
30
31
|
*/
|
|
31
32
|
register(alias) {
|
|
32
33
|
const self = this;
|
|
@@ -63,6 +64,9 @@ class EmbeddingFunctionRegistry {
|
|
|
63
64
|
reset() {
|
|
64
65
|
this.#functions.clear();
|
|
65
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* @ignore
|
|
69
|
+
*/
|
|
66
70
|
parseFunctions(metadata) {
|
|
67
71
|
if (!metadata.has("embedding_functions")) {
|
|
68
72
|
return new Map();
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,55 @@
|
|
|
1
|
-
|
|
1
|
+
import { Connection } from "./connection";
|
|
2
|
+
import { ConnectionOptions } from "./native.js";
|
|
3
|
+
import { RemoteConnectionOptions } from "./remote";
|
|
4
|
+
export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, IndexStatistics, IndexMetadata, IndexConfig, } from "./native.js";
|
|
2
5
|
export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
3
|
-
export {
|
|
6
|
+
export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
|
|
4
7
|
export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
|
|
5
8
|
export { Index, IndexOptions, IvfPqOptions } from "./indices";
|
|
6
|
-
export { Table, AddDataOptions,
|
|
9
|
+
export { Table, AddDataOptions, UpdateOptions } from "./table";
|
|
7
10
|
export * as embedding from "./embedding";
|
|
11
|
+
/**
|
|
12
|
+
* Connect to a LanceDB instance at the given URI.
|
|
13
|
+
*
|
|
14
|
+
* Accepted formats:
|
|
15
|
+
*
|
|
16
|
+
* - `/path/to/database` - local database
|
|
17
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
18
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
19
|
+
* @param {string} uri - The uri of the database. If the database uri starts
|
|
20
|
+
* with `db://` then it connects to a remote database.
|
|
21
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
22
|
+
* @example
|
|
23
|
+
* ```ts
|
|
24
|
+
* const conn = await connect("/path/to/database");
|
|
25
|
+
* ```
|
|
26
|
+
* @example
|
|
27
|
+
* ```ts
|
|
28
|
+
* const conn = await connect(
|
|
29
|
+
* "s3://bucket/path/to/database",
|
|
30
|
+
* {storageOptions: {timeout: "60s"}
|
|
31
|
+
* });
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
export declare function connect(uri: string, opts?: Partial<ConnectionOptions | RemoteConnectionOptions>): Promise<Connection>;
|
|
35
|
+
/**
|
|
36
|
+
* Connect to a LanceDB instance at the given URI.
|
|
37
|
+
*
|
|
38
|
+
* Accepted formats:
|
|
39
|
+
*
|
|
40
|
+
* - `/path/to/database` - local database
|
|
41
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
42
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
43
|
+
* @param options - The options to use when connecting to the database
|
|
44
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* const conn = await connect({
|
|
48
|
+
* uri: "/path/to/database",
|
|
49
|
+
* storageOptions: {timeout: "60s"}
|
|
50
|
+
* });
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
export declare function connect(opts: Partial<RemoteConnectionOptions | ConnectionOptions> & {
|
|
54
|
+
uri: string;
|
|
55
|
+
}): Promise<Connection>;
|
package/dist/index.js
CHANGED
|
@@ -13,14 +13,16 @@
|
|
|
13
13
|
// See the License for the specific language governing permissions and
|
|
14
14
|
// limitations under the License.
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.
|
|
16
|
+
exports.connect = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
|
|
17
|
+
const connection_1 = require("./connection");
|
|
18
|
+
const native_js_1 = require("./native.js");
|
|
19
|
+
const remote_1 = require("./remote");
|
|
17
20
|
var arrow_1 = require("./arrow");
|
|
18
21
|
Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
|
|
19
22
|
Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
|
|
20
23
|
Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
|
|
21
|
-
var
|
|
22
|
-
Object.defineProperty(exports, "
|
|
23
|
-
Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_1.Connection; } });
|
|
24
|
+
var connection_2 = require("./connection");
|
|
25
|
+
Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
|
|
24
26
|
var query_1 = require("./query");
|
|
25
27
|
Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
|
|
26
28
|
Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
|
|
@@ -31,3 +33,25 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
|
|
|
31
33
|
var table_1 = require("./table");
|
|
32
34
|
Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
|
|
33
35
|
exports.embedding = require("./embedding");
|
|
36
|
+
async function connect(uriOrOptions, opts = {}) {
|
|
37
|
+
let uri;
|
|
38
|
+
if (typeof uriOrOptions !== "string") {
|
|
39
|
+
const { uri: uri_, ...options } = uriOrOptions;
|
|
40
|
+
uri = uri_;
|
|
41
|
+
opts = options;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
uri = uriOrOptions;
|
|
45
|
+
}
|
|
46
|
+
if (!uri) {
|
|
47
|
+
throw new Error("uri is required");
|
|
48
|
+
}
|
|
49
|
+
if (uri?.startsWith("db://")) {
|
|
50
|
+
return new remote_1.RemoteConnection(uri, opts);
|
|
51
|
+
}
|
|
52
|
+
opts = opts ?? {};
|
|
53
|
+
opts.storageOptions = (0, connection_1.cleanseStorageOptions)(opts.storageOptions);
|
|
54
|
+
const nativeConn = await native_js_1.Connection.new(uri, opts);
|
|
55
|
+
return new connection_1.LocalConnection(nativeConn);
|
|
56
|
+
}
|
|
57
|
+
exports.connect = connect;
|
package/dist/merge.d.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Data } from "./arrow";
|
|
2
|
+
import { NativeMergeInsertBuilder } from "./native";
|
|
3
|
+
/** A builder used to create and run a merge insert operation */
|
|
4
|
+
export declare class MergeInsertBuilder {
|
|
5
|
+
#private;
|
|
6
|
+
/** Construct a MergeInsertBuilder. __Internal use only.__ */
|
|
7
|
+
constructor(native: NativeMergeInsertBuilder);
|
|
8
|
+
/**
|
|
9
|
+
* Rows that exist in both the source table (new data) and
|
|
10
|
+
* the target table (old data) will be updated, replacing
|
|
11
|
+
* the old row with the corresponding matching row.
|
|
12
|
+
*
|
|
13
|
+
* If there are multiple matches then the behavior is undefined.
|
|
14
|
+
* Currently this causes multiple copies of the row to be created
|
|
15
|
+
* but that behavior is subject to change.
|
|
16
|
+
*
|
|
17
|
+
* An optional condition may be specified. If it is, then only
|
|
18
|
+
* matched rows that satisfy the condtion will be updated. Any
|
|
19
|
+
* rows that do not satisfy the condition will be left as they
|
|
20
|
+
* are. Failing to satisfy the condition does not cause a
|
|
21
|
+
* "matched row" to become a "not matched" row.
|
|
22
|
+
*
|
|
23
|
+
* The condition should be an SQL string. Use the prefix
|
|
24
|
+
* target. to refer to rows in the target table (old data)
|
|
25
|
+
* and the prefix source. to refer to rows in the source
|
|
26
|
+
* table (new data).
|
|
27
|
+
*
|
|
28
|
+
* For example, "target.last_update < source.last_update"
|
|
29
|
+
*/
|
|
30
|
+
whenMatchedUpdateAll(options?: {
|
|
31
|
+
where: string;
|
|
32
|
+
}): MergeInsertBuilder;
|
|
33
|
+
/**
|
|
34
|
+
* Rows that exist only in the source table (new data) should
|
|
35
|
+
* be inserted into the target table.
|
|
36
|
+
*/
|
|
37
|
+
whenNotMatchedInsertAll(): MergeInsertBuilder;
|
|
38
|
+
/**
|
|
39
|
+
* Rows that exist only in the target table (old data) will be
|
|
40
|
+
* deleted. An optional condition can be provided to limit what
|
|
41
|
+
* data is deleted.
|
|
42
|
+
*
|
|
43
|
+
* @param options.where - An optional condition to limit what data is deleted
|
|
44
|
+
*/
|
|
45
|
+
whenNotMatchedBySourceDelete(options?: {
|
|
46
|
+
where: string;
|
|
47
|
+
}): MergeInsertBuilder;
|
|
48
|
+
/**
|
|
49
|
+
* Executes the merge insert operation
|
|
50
|
+
*
|
|
51
|
+
* Nothing is returned but the `Table` is updated
|
|
52
|
+
*/
|
|
53
|
+
execute(data: Data): Promise<void>;
|
|
54
|
+
}
|
package/dist/merge.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MergeInsertBuilder = void 0;
|
|
4
|
+
const arrow_1 = require("./arrow");
|
|
5
|
+
/** A builder used to create and run a merge insert operation */
|
|
6
|
+
class MergeInsertBuilder {
|
|
7
|
+
#native;
|
|
8
|
+
/** Construct a MergeInsertBuilder. __Internal use only.__ */
|
|
9
|
+
constructor(native) {
|
|
10
|
+
this.#native = native;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Rows that exist in both the source table (new data) and
|
|
14
|
+
* the target table (old data) will be updated, replacing
|
|
15
|
+
* the old row with the corresponding matching row.
|
|
16
|
+
*
|
|
17
|
+
* If there are multiple matches then the behavior is undefined.
|
|
18
|
+
* Currently this causes multiple copies of the row to be created
|
|
19
|
+
* but that behavior is subject to change.
|
|
20
|
+
*
|
|
21
|
+
* An optional condition may be specified. If it is, then only
|
|
22
|
+
* matched rows that satisfy the condtion will be updated. Any
|
|
23
|
+
* rows that do not satisfy the condition will be left as they
|
|
24
|
+
* are. Failing to satisfy the condition does not cause a
|
|
25
|
+
* "matched row" to become a "not matched" row.
|
|
26
|
+
*
|
|
27
|
+
* The condition should be an SQL string. Use the prefix
|
|
28
|
+
* target. to refer to rows in the target table (old data)
|
|
29
|
+
* and the prefix source. to refer to rows in the source
|
|
30
|
+
* table (new data).
|
|
31
|
+
*
|
|
32
|
+
* For example, "target.last_update < source.last_update"
|
|
33
|
+
*/
|
|
34
|
+
whenMatchedUpdateAll(options) {
|
|
35
|
+
return new MergeInsertBuilder(this.#native.whenMatchedUpdateAll(options?.where));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Rows that exist only in the source table (new data) should
|
|
39
|
+
* be inserted into the target table.
|
|
40
|
+
*/
|
|
41
|
+
whenNotMatchedInsertAll() {
|
|
42
|
+
return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Rows that exist only in the target table (old data) will be
|
|
46
|
+
* deleted. An optional condition can be provided to limit what
|
|
47
|
+
* data is deleted.
|
|
48
|
+
*
|
|
49
|
+
* @param options.where - An optional condition to limit what data is deleted
|
|
50
|
+
*/
|
|
51
|
+
whenNotMatchedBySourceDelete(options) {
|
|
52
|
+
return new MergeInsertBuilder(this.#native.whenNotMatchedBySourceDelete(options?.where));
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Executes the merge insert operation
|
|
56
|
+
*
|
|
57
|
+
* Nothing is returned but the `Table` is updated
|
|
58
|
+
*/
|
|
59
|
+
async execute(data) {
|
|
60
|
+
const buffer = await (0, arrow_1.fromDataToBuffer)(data);
|
|
61
|
+
await this.#native.execute(buffer);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
exports.MergeInsertBuilder = MergeInsertBuilder;
|
package/dist/native.d.ts
CHANGED
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
|
|
6
6
|
/** A description of an index currently configured on a column */
|
|
7
7
|
export interface IndexConfig {
|
|
8
|
+
/** The name of the index */
|
|
9
|
+
name: string
|
|
8
10
|
/** The type of the index */
|
|
9
11
|
indexType: string
|
|
10
12
|
/**
|
|
11
13
|
* The columns in the index
|
|
12
14
|
*
|
|
13
|
-
* Currently this is always an array of size 1.
|
|
15
|
+
* Currently this is always an array of size 1. In the future there may
|
|
14
16
|
* be more columns to represent composite indices.
|
|
15
17
|
*/
|
|
16
18
|
columns: Array<string>
|
|
@@ -72,9 +74,21 @@ export interface AddColumnsSql {
|
|
|
72
74
|
*/
|
|
73
75
|
valueSql: string
|
|
74
76
|
}
|
|
77
|
+
export interface IndexStatistics {
|
|
78
|
+
/** The number of rows indexed by the index */
|
|
79
|
+
numIndexedRows: number
|
|
80
|
+
/** The number of rows not indexed */
|
|
81
|
+
numUnindexedRows: number
|
|
82
|
+
/** The type of the index */
|
|
83
|
+
indexType?: string
|
|
84
|
+
/** The metadata for each index */
|
|
85
|
+
indices: Array<IndexMetadata>
|
|
86
|
+
}
|
|
87
|
+
export interface IndexMetadata {
|
|
88
|
+
metricType?: string
|
|
89
|
+
indexType?: string
|
|
90
|
+
}
|
|
75
91
|
export interface ConnectionOptions {
|
|
76
|
-
apiKey?: string
|
|
77
|
-
hostOverride?: string
|
|
78
92
|
/**
|
|
79
93
|
* (For LanceDB OSS only): The interval, in seconds, at which to check for
|
|
80
94
|
* updates to the table from other processes. If None, then consistency is not
|
|
@@ -102,6 +116,7 @@ export const enum WriteMode {
|
|
|
102
116
|
}
|
|
103
117
|
/** Write options when creating a Table. */
|
|
104
118
|
export interface WriteOptions {
|
|
119
|
+
/** Write mode for writing to a table. */
|
|
105
120
|
mode?: WriteMode
|
|
106
121
|
}
|
|
107
122
|
export interface OpenTableOptions {
|
|
@@ -123,8 +138,8 @@ export class Connection {
|
|
|
123
138
|
* - buf: The buffer containing the IPC file.
|
|
124
139
|
*
|
|
125
140
|
*/
|
|
126
|
-
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
127
|
-
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
141
|
+
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
|
|
142
|
+
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
|
|
128
143
|
openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
|
|
129
144
|
/** Drop table with the name. Or raise an error if the table does not exist. */
|
|
130
145
|
dropTable(name: string): Promise<void>
|
|
@@ -137,12 +152,20 @@ export class Index {
|
|
|
137
152
|
export class RecordBatchIterator {
|
|
138
153
|
next(): Promise<Buffer | null>
|
|
139
154
|
}
|
|
155
|
+
/** A builder used to create and run a merge insert operation */
|
|
156
|
+
export class NativeMergeInsertBuilder {
|
|
157
|
+
whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
|
|
158
|
+
whenNotMatchedInsertAll(): NativeMergeInsertBuilder
|
|
159
|
+
whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
|
|
160
|
+
execute(buf: Buffer): Promise<void>
|
|
161
|
+
}
|
|
140
162
|
export class Query {
|
|
141
163
|
onlyIf(predicate: string): void
|
|
142
164
|
select(columns: Array<[string, string]>): void
|
|
143
165
|
limit(limit: number): void
|
|
144
166
|
nearestTo(vector: Float32Array): VectorQuery
|
|
145
|
-
execute(): Promise<RecordBatchIterator>
|
|
167
|
+
execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
|
|
168
|
+
explainPlan(verbose: boolean): Promise<string>
|
|
146
169
|
}
|
|
147
170
|
export class VectorQuery {
|
|
148
171
|
column(column: string): void
|
|
@@ -154,9 +177,11 @@ export class VectorQuery {
|
|
|
154
177
|
onlyIf(predicate: string): void
|
|
155
178
|
select(columns: Array<[string, string]>): void
|
|
156
179
|
limit(limit: number): void
|
|
157
|
-
execute(): Promise<RecordBatchIterator>
|
|
180
|
+
execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
|
|
181
|
+
explainPlan(verbose: boolean): Promise<string>
|
|
158
182
|
}
|
|
159
183
|
export class Table {
|
|
184
|
+
name: string
|
|
160
185
|
display(): string
|
|
161
186
|
isOpen(): boolean
|
|
162
187
|
close(): void
|
|
@@ -178,4 +203,6 @@ export class Table {
|
|
|
178
203
|
restore(): Promise<void>
|
|
179
204
|
optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
|
|
180
205
|
listIndices(): Promise<Array<IndexConfig>>
|
|
206
|
+
indexStats(indexName: string): Promise<IndexStatistics | null>
|
|
207
|
+
mergeInsert(on: Array<string>): NativeMergeInsertBuilder
|
|
181
208
|
}
|
package/dist/native.js
CHANGED
|
@@ -233,17 +233,33 @@ switch (platform) {
|
|
|
233
233
|
}
|
|
234
234
|
break;
|
|
235
235
|
case 'arm':
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
236
|
+
if (isMusl()) {
|
|
237
|
+
localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-musleabihf.node'));
|
|
238
|
+
try {
|
|
239
|
+
if (localFileExisted) {
|
|
240
|
+
nativeBinding = require('./lancedb.linux-arm-musleabihf.node');
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
nativeBinding = require('@lancedb/lancedb-linux-arm-musleabihf');
|
|
244
|
+
}
|
|
240
245
|
}
|
|
241
|
-
|
|
242
|
-
|
|
246
|
+
catch (e) {
|
|
247
|
+
loadError = e;
|
|
243
248
|
}
|
|
244
249
|
}
|
|
245
|
-
|
|
246
|
-
|
|
250
|
+
else {
|
|
251
|
+
localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-gnueabihf.node'));
|
|
252
|
+
try {
|
|
253
|
+
if (localFileExisted) {
|
|
254
|
+
nativeBinding = require('./lancedb.linux-arm-gnueabihf.node');
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
nativeBinding = require('@lancedb/lancedb-linux-arm-gnueabihf');
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
catch (e) {
|
|
261
|
+
loadError = e;
|
|
262
|
+
}
|
|
247
263
|
}
|
|
248
264
|
break;
|
|
249
265
|
case 'riscv64':
|
|
@@ -303,10 +319,11 @@ if (!nativeBinding) {
|
|
|
303
319
|
}
|
|
304
320
|
throw new Error(`Failed to load native binding`);
|
|
305
321
|
}
|
|
306
|
-
const { Connection, Index, RecordBatchIterator, Query, VectorQuery, Table, WriteMode } = nativeBinding;
|
|
322
|
+
const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Table, WriteMode } = nativeBinding;
|
|
307
323
|
module.exports.Connection = Connection;
|
|
308
324
|
module.exports.Index = Index;
|
|
309
325
|
module.exports.RecordBatchIterator = RecordBatchIterator;
|
|
326
|
+
module.exports.NativeMergeInsertBuilder = NativeMergeInsertBuilder;
|
|
310
327
|
module.exports.Query = Query;
|
|
311
328
|
module.exports.VectorQuery = VectorQuery;
|
|
312
329
|
module.exports.Table = Table;
|
package/dist/query.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { Table as ArrowTable, RecordBatch } from "./arrow";
|
|
1
|
+
import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
|
|
2
|
+
import { type IvfPqOptions } from "./indices";
|
|
2
3
|
import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
|
|
3
4
|
export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
|
|
4
5
|
private promisedInner?;
|
|
@@ -6,10 +7,23 @@ export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
|
|
|
6
7
|
constructor(promise?: Promise<NativeBatchIterator>);
|
|
7
8
|
next(): Promise<IteratorResult<RecordBatch<any>>>;
|
|
8
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* Options that control the behavior of a particular query execution
|
|
12
|
+
*/
|
|
13
|
+
export interface QueryExecutionOptions {
|
|
14
|
+
/**
|
|
15
|
+
* The maximum number of rows to return in a single batch
|
|
16
|
+
*
|
|
17
|
+
* Batches may have fewer rows if the underlying data is stored
|
|
18
|
+
* in smaller chunks.
|
|
19
|
+
*/
|
|
20
|
+
maxBatchLength?: number;
|
|
21
|
+
}
|
|
9
22
|
/** Common methods supported by all query types */
|
|
10
|
-
export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery
|
|
11
|
-
protected inner: NativeQueryType
|
|
12
|
-
protected constructor(inner: NativeQueryType);
|
|
23
|
+
export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery> implements AsyncIterable<RecordBatch> {
|
|
24
|
+
protected inner: NativeQueryType | Promise<NativeQueryType>;
|
|
25
|
+
protected constructor(inner: NativeQueryType | Promise<NativeQueryType>);
|
|
26
|
+
protected doCall(fn: (inner: NativeQueryType) => void): void;
|
|
13
27
|
/**
|
|
14
28
|
* A filter statement to be applied to this query.
|
|
15
29
|
*
|
|
@@ -22,7 +36,13 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
22
36
|
* Filtering performance can often be improved by creating a scalar index
|
|
23
37
|
* on the filter column(s).
|
|
24
38
|
*/
|
|
25
|
-
where(predicate: string):
|
|
39
|
+
where(predicate: string): this;
|
|
40
|
+
/**
|
|
41
|
+
* A filter statement to be applied to this query.
|
|
42
|
+
* @alias where
|
|
43
|
+
* @deprecated Use `where` instead
|
|
44
|
+
*/
|
|
45
|
+
filter(predicate: string): this;
|
|
26
46
|
/**
|
|
27
47
|
* Return only the specified columns.
|
|
28
48
|
*
|
|
@@ -53,15 +73,15 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
53
73
|
* uses `Object.entries` which should preserve the insertion order of the object. However,
|
|
54
74
|
* object insertion order is easy to get wrong and `Map` is more foolproof.
|
|
55
75
|
*/
|
|
56
|
-
select(columns: string[] | Map<string, string> | Record<string, string>):
|
|
76
|
+
select(columns: string[] | Map<string, string> | Record<string, string> | string): this;
|
|
57
77
|
/**
|
|
58
78
|
* Set the maximum number of results to return.
|
|
59
79
|
*
|
|
60
80
|
* By default, a plain search has no limit. If this method is not
|
|
61
81
|
* called then every valid row from the table will be returned.
|
|
62
82
|
*/
|
|
63
|
-
limit(limit: number):
|
|
64
|
-
protected nativeExecute(): Promise<NativeBatchIterator>;
|
|
83
|
+
limit(limit: number): this;
|
|
84
|
+
protected nativeExecute(options?: Partial<QueryExecutionOptions>): Promise<NativeBatchIterator>;
|
|
65
85
|
/**
|
|
66
86
|
* Execute the query and return the results as an @see {@link AsyncIterator}
|
|
67
87
|
* of @see {@link RecordBatch}.
|
|
@@ -73,12 +93,27 @@ export declare class QueryBase<NativeQueryType extends NativeQuery | NativeVecto
|
|
|
73
93
|
* single query)
|
|
74
94
|
*
|
|
75
95
|
*/
|
|
76
|
-
protected execute(): RecordBatchIterator;
|
|
96
|
+
protected execute(options?: Partial<QueryExecutionOptions>): RecordBatchIterator;
|
|
77
97
|
[Symbol.asyncIterator](): AsyncIterator<RecordBatch<any>>;
|
|
78
98
|
/** Collect the results as an Arrow @see {@link ArrowTable}. */
|
|
79
|
-
toArrow(): Promise<ArrowTable>;
|
|
99
|
+
toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable>;
|
|
80
100
|
/** Collect the results as an array of objects. */
|
|
81
|
-
toArray(): Promise<
|
|
101
|
+
toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]>;
|
|
102
|
+
/**
|
|
103
|
+
* Generates an explanation of the query execution plan.
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* import * as lancedb from "@lancedb/lancedb"
|
|
107
|
+
* const db = await lancedb.connect("./.lancedb");
|
|
108
|
+
* const table = await db.createTable("my_table", [
|
|
109
|
+
* { vector: [1.1, 0.9], id: "1" },
|
|
110
|
+
* ]);
|
|
111
|
+
* const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
|
|
112
|
+
*
|
|
113
|
+
* @param verbose - If true, provides a more detailed explanation. Defaults to false.
|
|
114
|
+
* @returns A Promise that resolves to a string containing the query execution plan explanation.
|
|
115
|
+
*/
|
|
116
|
+
explainPlan(verbose?: boolean): Promise<string>;
|
|
82
117
|
}
|
|
83
118
|
/**
|
|
84
119
|
* An interface for a query that can be executed
|
|
@@ -92,8 +127,8 @@ export interface ExecutableQuery {
|
|
|
92
127
|
*
|
|
93
128
|
* This builder can be reused to execute the query many times.
|
|
94
129
|
*/
|
|
95
|
-
export declare class VectorQuery extends QueryBase<NativeVectorQuery
|
|
96
|
-
constructor(inner: NativeVectorQuery);
|
|
130
|
+
export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
|
|
131
|
+
constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>);
|
|
97
132
|
/**
|
|
98
133
|
* Set the number of partitions to search (probe)
|
|
99
134
|
*
|
|
@@ -141,7 +176,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
|
|
|
141
176
|
*
|
|
142
177
|
* By default "l2" is used.
|
|
143
178
|
*/
|
|
144
|
-
distanceType(distanceType:
|
|
179
|
+
distanceType(distanceType: Required<IvfPqOptions>["distanceType"]): VectorQuery;
|
|
145
180
|
/**
|
|
146
181
|
* A multiplier to control how many additional rows are taken during the refine step
|
|
147
182
|
*
|
|
@@ -205,7 +240,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuer
|
|
|
205
240
|
bypassVectorIndex(): VectorQuery;
|
|
206
241
|
}
|
|
207
242
|
/** A builder for LanceDB queries. */
|
|
208
|
-
export declare class Query extends QueryBase<NativeQuery
|
|
243
|
+
export declare class Query extends QueryBase<NativeQuery> {
|
|
209
244
|
constructor(tbl: NativeTable);
|
|
210
245
|
/**
|
|
211
246
|
* Find the nearest vectors to the given query vector.
|
|
@@ -244,5 +279,5 @@ export declare class Query extends QueryBase<NativeQuery, Query> {
|
|
|
244
279
|
* Vector searches always have a `limit`. If `limit` has not been called then
|
|
245
280
|
* a default `limit` of 10 will be used. @see {@link Query#limit}
|
|
246
281
|
*/
|
|
247
|
-
nearestTo(vector:
|
|
282
|
+
nearestTo(vector: IntoVector): VectorQuery;
|
|
248
283
|
}
|