@lancedb/lancedb 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +3 -3
- package/biome.json +19 -3
- package/dist/arrow.d.ts +42 -7
- package/dist/arrow.js +6 -5
- package/dist/connection.d.ts +55 -29
- package/dist/connection.js +22 -74
- package/dist/embedding/embedding_function.d.ts +11 -3
- package/dist/embedding/embedding_function.js +36 -12
- package/dist/embedding/openai.d.ts +6 -5
- package/dist/embedding/openai.js +4 -2
- package/dist/embedding/registry.d.ts +10 -11
- package/dist/embedding/registry.js +4 -0
- package/dist/index.d.ts +51 -3
- package/dist/index.js +28 -4
- package/dist/merge.d.ts +54 -0
- package/dist/merge.js +64 -0
- package/dist/native.d.ts +34 -7
- package/dist/native.js +26 -9
- package/dist/query.d.ts +51 -16
- package/dist/query.js +122 -21
- package/dist/remote/client.d.ts +28 -0
- package/dist/remote/client.js +172 -0
- package/dist/remote/connection.d.ts +25 -0
- package/dist/remote/connection.js +110 -0
- package/dist/remote/index.d.ts +3 -0
- package/dist/remote/index.js +9 -0
- package/dist/remote/table.d.ts +42 -0
- package/dist/remote/table.js +179 -0
- package/dist/sanitize.d.ts +3 -2
- package/dist/sanitize.js +55 -1
- package/dist/table.d.ts +116 -25
- package/dist/table.js +117 -233
- package/dist/util.d.ts +14 -0
- package/dist/util.js +65 -0
- package/examples/ann_indexes.ts +49 -0
- package/examples/basic.ts +149 -0
- package/examples/embedding.ts +83 -0
- package/examples/filtering.ts +34 -0
- package/examples/jsconfig.json +27 -0
- package/examples/package-lock.json +79 -0
- package/examples/package.json +18 -0
- package/examples/search.ts +37 -0
- package/lancedb/arrow.ts +87 -24
- package/lancedb/connection.ts +115 -92
- package/lancedb/embedding/embedding_function.ts +48 -16
- package/lancedb/embedding/openai.ts +11 -6
- package/lancedb/embedding/registry.ts +38 -22
- package/lancedb/index.ts +101 -2
- package/lancedb/merge.ts +70 -0
- package/lancedb/query.ts +168 -39
- package/lancedb/remote/client.ts +221 -0
- package/lancedb/remote/connection.ts +201 -0
- package/lancedb/remote/index.ts +3 -0
- package/lancedb/remote/table.ts +226 -0
- package/lancedb/sanitize.ts +73 -1
- package/lancedb/table.ts +344 -101
- package/lancedb/util.ts +69 -0
- package/native.d.ts +208 -0
- package/nodejs-artifacts/arrow.d.ts +42 -7
- package/nodejs-artifacts/arrow.js +6 -5
- package/nodejs-artifacts/connection.d.ts +55 -29
- package/nodejs-artifacts/connection.js +22 -74
- package/nodejs-artifacts/embedding/embedding_function.d.ts +11 -3
- package/nodejs-artifacts/embedding/embedding_function.js +36 -12
- package/nodejs-artifacts/embedding/openai.d.ts +6 -5
- package/nodejs-artifacts/embedding/openai.js +4 -2
- package/nodejs-artifacts/embedding/registry.d.ts +10 -11
- package/nodejs-artifacts/embedding/registry.js +4 -0
- package/nodejs-artifacts/index.d.ts +51 -3
- package/nodejs-artifacts/index.js +28 -4
- package/nodejs-artifacts/merge.d.ts +54 -0
- package/nodejs-artifacts/merge.js +64 -0
- package/nodejs-artifacts/native.d.ts +34 -7
- package/nodejs-artifacts/native.js +26 -9
- package/nodejs-artifacts/query.d.ts +51 -16
- package/nodejs-artifacts/query.js +122 -21
- package/nodejs-artifacts/remote/client.d.ts +28 -0
- package/nodejs-artifacts/remote/client.js +172 -0
- package/nodejs-artifacts/remote/connection.d.ts +25 -0
- package/nodejs-artifacts/remote/connection.js +110 -0
- package/nodejs-artifacts/remote/index.d.ts +3 -0
- package/nodejs-artifacts/remote/index.js +9 -0
- package/nodejs-artifacts/remote/table.d.ts +42 -0
- package/nodejs-artifacts/remote/table.js +179 -0
- package/nodejs-artifacts/sanitize.d.ts +3 -2
- package/nodejs-artifacts/sanitize.js +55 -1
- package/nodejs-artifacts/table.d.ts +116 -25
- package/nodejs-artifacts/table.js +117 -233
- package/nodejs-artifacts/util.d.ts +14 -0
- package/nodejs-artifacts/util.js +65 -0
- package/package.json +25 -11
|
@@ -21,6 +21,12 @@ const sanitize_1 = require("../sanitize");
|
|
|
21
21
|
* An embedding function that automatically creates vector representation for a given column.
|
|
22
22
|
*/
|
|
23
23
|
class EmbeddingFunction {
|
|
24
|
+
/**
|
|
25
|
+
* @ignore
|
|
26
|
+
* This is only used for associating the options type with the class for type checking
|
|
27
|
+
*/
|
|
28
|
+
// biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
|
|
29
|
+
TOptions;
|
|
24
30
|
/**
|
|
25
31
|
* sourceField is used in combination with `LanceSchema` to provide a declarative data model
|
|
26
32
|
*
|
|
@@ -47,32 +53,50 @@ class EmbeddingFunction {
|
|
|
47
53
|
*
|
|
48
54
|
* @see {@link lancedb.LanceSchema}
|
|
49
55
|
*/
|
|
50
|
-
vectorField(
|
|
56
|
+
vectorField(optionsOrDatatype) {
|
|
51
57
|
let dtype;
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
dtype = new arrow_1.FixedSizeList(dims, new arrow_1.Field("item", new arrow_1.Float32(), true));
|
|
58
|
+
let vectorType;
|
|
59
|
+
let dims = this.ndims();
|
|
60
|
+
// `func.vectorField(new Float32())`
|
|
61
|
+
if ((0, arrow_1.isDataType)(optionsOrDatatype)) {
|
|
62
|
+
dtype = optionsOrDatatype;
|
|
58
63
|
}
|
|
59
64
|
else {
|
|
60
|
-
|
|
61
|
-
|
|
65
|
+
// `func.vectorField({
|
|
66
|
+
// datatype: new Float32(),
|
|
67
|
+
// dims: 10
|
|
68
|
+
// })`
|
|
69
|
+
dims = dims ?? optionsOrDatatype?.dims;
|
|
70
|
+
dtype = optionsOrDatatype?.datatype;
|
|
71
|
+
}
|
|
72
|
+
if (dtype !== undefined) {
|
|
73
|
+
// `func.vectorField(new FixedSizeList(dims, new Field("item", new Float32(), true)))`
|
|
74
|
+
// or `func.vectorField({datatype: new FixedSizeList(dims, new Field("item", new Float32(), true))})`
|
|
75
|
+
if ((0, arrow_1.isFixedSizeList)(dtype)) {
|
|
76
|
+
vectorType = dtype;
|
|
77
|
+
// `func.vectorField(new Float32())`
|
|
78
|
+
// or `func.vectorField({datatype: new Float32()})`
|
|
62
79
|
}
|
|
63
|
-
else if ((0, arrow_1.isFloat)(
|
|
80
|
+
else if ((0, arrow_1.isFloat)(dtype)) {
|
|
81
|
+
// No `ndims` impl and no `{dims: n}` provided;
|
|
64
82
|
if (dims === undefined) {
|
|
65
83
|
throw new Error("ndims is required for vector field");
|
|
66
84
|
}
|
|
67
|
-
|
|
85
|
+
vectorType = (0, arrow_1.newVectorType)(dims, dtype);
|
|
68
86
|
}
|
|
69
87
|
else {
|
|
70
88
|
throw new Error("Expected FixedSizeList or Float as datatype for vector field");
|
|
71
89
|
}
|
|
72
90
|
}
|
|
91
|
+
else {
|
|
92
|
+
if (dims === undefined) {
|
|
93
|
+
throw new Error("ndims is required for vector field");
|
|
94
|
+
}
|
|
95
|
+
vectorType = new arrow_1.FixedSizeList(dims, new arrow_1.Field("item", new arrow_1.Float32(), true));
|
|
96
|
+
}
|
|
73
97
|
const metadata = new Map();
|
|
74
98
|
metadata.set("vector_column_for", this);
|
|
75
|
-
return [
|
|
99
|
+
return [vectorType, metadata];
|
|
76
100
|
}
|
|
77
101
|
/** The number of dimensions of the embeddings */
|
|
78
102
|
ndims() {
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { type EmbeddingCreateParams } from "openai/resources";
|
|
1
2
|
import { Float } from "../arrow";
|
|
2
3
|
import { EmbeddingFunction } from "./embedding_function";
|
|
3
4
|
export type OpenAIOptions = {
|
|
4
|
-
apiKey
|
|
5
|
-
model
|
|
5
|
+
apiKey: string;
|
|
6
|
+
model: EmbeddingCreateParams["model"];
|
|
6
7
|
};
|
|
7
|
-
export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, OpenAIOptions
|
|
8
|
+
export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, Partial<OpenAIOptions>> {
|
|
8
9
|
#private;
|
|
9
|
-
constructor(options?: OpenAIOptions);
|
|
10
|
+
constructor(options?: Partial<OpenAIOptions>);
|
|
10
11
|
toJSON(): {
|
|
11
|
-
model: string;
|
|
12
|
+
model: (string & {}) | "text-embedding-ada-002" | "text-embedding-3-small" | "text-embedding-3-large";
|
|
12
13
|
};
|
|
13
14
|
ndims(): number;
|
|
14
15
|
embeddingDataType(): Float;
|
|
@@ -29,7 +29,9 @@ const registry_1 = require("./registry");
|
|
|
29
29
|
let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
|
|
30
30
|
#openai;
|
|
31
31
|
#modelName;
|
|
32
|
-
constructor(options = {
|
|
32
|
+
constructor(options = {
|
|
33
|
+
model: "text-embedding-ada-002",
|
|
34
|
+
}) {
|
|
33
35
|
super();
|
|
34
36
|
const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
|
|
35
37
|
if (!openAIKey) {
|
|
@@ -68,7 +70,7 @@ let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_fu
|
|
|
68
70
|
case "text-embedding-3-small":
|
|
69
71
|
return 1536;
|
|
70
72
|
default:
|
|
71
|
-
|
|
73
|
+
throw new Error(`Unknown model: ${this.#modelName}`);
|
|
72
74
|
}
|
|
73
75
|
}
|
|
74
76
|
embeddingDataType() {
|
|
@@ -1,13 +1,8 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
|
|
2
2
|
import "reflect-metadata";
|
|
3
|
-
|
|
4
|
-
[key: string]: unknown;
|
|
5
|
-
}
|
|
6
|
-
export interface EmbeddingFunctionFactory<T extends EmbeddingFunction = EmbeddingFunction> {
|
|
7
|
-
new (modelOptions?: EmbeddingFunctionOptions): T;
|
|
8
|
-
}
|
|
3
|
+
import { OpenAIEmbeddingFunction } from "./openai";
|
|
9
4
|
interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
|
|
10
|
-
create(options?:
|
|
5
|
+
create(options?: T["TOptions"]): T;
|
|
11
6
|
}
|
|
12
7
|
/**
|
|
13
8
|
* This is a singleton class used to register embedding functions
|
|
@@ -21,22 +16,26 @@ export declare class EmbeddingFunctionRegistry {
|
|
|
21
16
|
* Register an embedding function
|
|
22
17
|
* @param name The name of the function
|
|
23
18
|
* @param func The function to register
|
|
19
|
+
* @throws Error if the function is already registered
|
|
24
20
|
*/
|
|
25
|
-
register<T extends
|
|
21
|
+
register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
|
|
26
22
|
/**
|
|
27
23
|
* Fetch an embedding function by name
|
|
28
24
|
* @param name The name of the function
|
|
29
25
|
*/
|
|
30
|
-
get<T extends EmbeddingFunction<unknown
|
|
26
|
+
get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
|
|
31
27
|
/**
|
|
32
28
|
* reset the registry to the initial state
|
|
33
29
|
*/
|
|
34
30
|
reset(this: EmbeddingFunctionRegistry): void;
|
|
31
|
+
/**
|
|
32
|
+
* @ignore
|
|
33
|
+
*/
|
|
35
34
|
parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Map<string, EmbeddingFunctionConfig>;
|
|
36
35
|
functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
|
|
37
36
|
getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
|
|
38
37
|
}
|
|
39
|
-
export declare function register(name?: string): (ctor:
|
|
38
|
+
export declare function register(name?: string): (ctor: EmbeddingFunctionConstructor<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
|
|
40
39
|
/**
|
|
41
40
|
* Utility function to get the global instance of the registry
|
|
42
41
|
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
|
@@ -27,6 +27,7 @@ class EmbeddingFunctionRegistry {
|
|
|
27
27
|
* Register an embedding function
|
|
28
28
|
* @param name The name of the function
|
|
29
29
|
* @param func The function to register
|
|
30
|
+
* @throws Error if the function is already registered
|
|
30
31
|
*/
|
|
31
32
|
register(alias) {
|
|
32
33
|
const self = this;
|
|
@@ -63,6 +64,9 @@ class EmbeddingFunctionRegistry {
|
|
|
63
64
|
reset() {
|
|
64
65
|
this.#functions.clear();
|
|
65
66
|
}
|
|
67
|
+
/**
|
|
68
|
+
* @ignore
|
|
69
|
+
*/
|
|
66
70
|
parseFunctions(metadata) {
|
|
67
71
|
if (!metadata.has("embedding_functions")) {
|
|
68
72
|
return new Map();
|
|
@@ -1,7 +1,55 @@
|
|
|
1
|
-
|
|
1
|
+
import { Connection } from "./connection";
|
|
2
|
+
import { ConnectionOptions } from "./native.js";
|
|
3
|
+
import { RemoteConnectionOptions } from "./remote";
|
|
4
|
+
export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, IndexStatistics, IndexMetadata, IndexConfig, } from "./native.js";
|
|
2
5
|
export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
3
|
-
export {
|
|
6
|
+
export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
|
|
4
7
|
export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
|
|
5
8
|
export { Index, IndexOptions, IvfPqOptions } from "./indices";
|
|
6
|
-
export { Table, AddDataOptions,
|
|
9
|
+
export { Table, AddDataOptions, UpdateOptions } from "./table";
|
|
7
10
|
export * as embedding from "./embedding";
|
|
11
|
+
/**
|
|
12
|
+
* Connect to a LanceDB instance at the given URI.
|
|
13
|
+
*
|
|
14
|
+
* Accepted formats:
|
|
15
|
+
*
|
|
16
|
+
* - `/path/to/database` - local database
|
|
17
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
18
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
19
|
+
* @param {string} uri - The uri of the database. If the database uri starts
|
|
20
|
+
* with `db://` then it connects to a remote database.
|
|
21
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
22
|
+
* @example
|
|
23
|
+
* ```ts
|
|
24
|
+
* const conn = await connect("/path/to/database");
|
|
25
|
+
* ```
|
|
26
|
+
* @example
|
|
27
|
+
* ```ts
|
|
28
|
+
* const conn = await connect(
|
|
29
|
+
* "s3://bucket/path/to/database",
|
|
30
|
+
* {storageOptions: {timeout: "60s"}
|
|
31
|
+
* });
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
export declare function connect(uri: string, opts?: Partial<ConnectionOptions | RemoteConnectionOptions>): Promise<Connection>;
|
|
35
|
+
/**
|
|
36
|
+
* Connect to a LanceDB instance at the given URI.
|
|
37
|
+
*
|
|
38
|
+
* Accepted formats:
|
|
39
|
+
*
|
|
40
|
+
* - `/path/to/database` - local database
|
|
41
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
42
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
43
|
+
* @param options - The options to use when connecting to the database
|
|
44
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* const conn = await connect({
|
|
48
|
+
* uri: "/path/to/database",
|
|
49
|
+
* storageOptions: {timeout: "60s"}
|
|
50
|
+
* });
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
export declare function connect(opts: Partial<RemoteConnectionOptions | ConnectionOptions> & {
|
|
54
|
+
uri: string;
|
|
55
|
+
}): Promise<Connection>;
|
|
@@ -13,14 +13,16 @@
|
|
|
13
13
|
// See the License for the specific language governing permissions and
|
|
14
14
|
// limitations under the License.
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.
|
|
16
|
+
exports.connect = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
|
|
17
|
+
const connection_1 = require("./connection");
|
|
18
|
+
const native_js_1 = require("./native.js");
|
|
19
|
+
const remote_1 = require("./remote");
|
|
17
20
|
var arrow_1 = require("./arrow");
|
|
18
21
|
Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
|
|
19
22
|
Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
|
|
20
23
|
Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
|
|
21
|
-
var
|
|
22
|
-
Object.defineProperty(exports, "
|
|
23
|
-
Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_1.Connection; } });
|
|
24
|
+
var connection_2 = require("./connection");
|
|
25
|
+
Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
|
|
24
26
|
var query_1 = require("./query");
|
|
25
27
|
Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
|
|
26
28
|
Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
|
|
@@ -31,3 +33,25 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
|
|
|
31
33
|
var table_1 = require("./table");
|
|
32
34
|
Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
|
|
33
35
|
exports.embedding = require("./embedding");
|
|
36
|
+
async function connect(uriOrOptions, opts = {}) {
|
|
37
|
+
let uri;
|
|
38
|
+
if (typeof uriOrOptions !== "string") {
|
|
39
|
+
const { uri: uri_, ...options } = uriOrOptions;
|
|
40
|
+
uri = uri_;
|
|
41
|
+
opts = options;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
uri = uriOrOptions;
|
|
45
|
+
}
|
|
46
|
+
if (!uri) {
|
|
47
|
+
throw new Error("uri is required");
|
|
48
|
+
}
|
|
49
|
+
if (uri?.startsWith("db://")) {
|
|
50
|
+
return new remote_1.RemoteConnection(uri, opts);
|
|
51
|
+
}
|
|
52
|
+
opts = opts ?? {};
|
|
53
|
+
opts.storageOptions = (0, connection_1.cleanseStorageOptions)(opts.storageOptions);
|
|
54
|
+
const nativeConn = await native_js_1.Connection.new(uri, opts);
|
|
55
|
+
return new connection_1.LocalConnection(nativeConn);
|
|
56
|
+
}
|
|
57
|
+
exports.connect = connect;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Data } from "./arrow";
|
|
2
|
+
import { NativeMergeInsertBuilder } from "./native";
|
|
3
|
+
/** A builder used to create and run a merge insert operation */
|
|
4
|
+
export declare class MergeInsertBuilder {
|
|
5
|
+
#private;
|
|
6
|
+
/** Construct a MergeInsertBuilder. __Internal use only.__ */
|
|
7
|
+
constructor(native: NativeMergeInsertBuilder);
|
|
8
|
+
/**
|
|
9
|
+
* Rows that exist in both the source table (new data) and
|
|
10
|
+
* the target table (old data) will be updated, replacing
|
|
11
|
+
* the old row with the corresponding matching row.
|
|
12
|
+
*
|
|
13
|
+
* If there are multiple matches then the behavior is undefined.
|
|
14
|
+
* Currently this causes multiple copies of the row to be created
|
|
15
|
+
* but that behavior is subject to change.
|
|
16
|
+
*
|
|
17
|
+
* An optional condition may be specified. If it is, then only
|
|
18
|
+
* matched rows that satisfy the condtion will be updated. Any
|
|
19
|
+
* rows that do not satisfy the condition will be left as they
|
|
20
|
+
* are. Failing to satisfy the condition does not cause a
|
|
21
|
+
* "matched row" to become a "not matched" row.
|
|
22
|
+
*
|
|
23
|
+
* The condition should be an SQL string. Use the prefix
|
|
24
|
+
* target. to refer to rows in the target table (old data)
|
|
25
|
+
* and the prefix source. to refer to rows in the source
|
|
26
|
+
* table (new data).
|
|
27
|
+
*
|
|
28
|
+
* For example, "target.last_update < source.last_update"
|
|
29
|
+
*/
|
|
30
|
+
whenMatchedUpdateAll(options?: {
|
|
31
|
+
where: string;
|
|
32
|
+
}): MergeInsertBuilder;
|
|
33
|
+
/**
|
|
34
|
+
* Rows that exist only in the source table (new data) should
|
|
35
|
+
* be inserted into the target table.
|
|
36
|
+
*/
|
|
37
|
+
whenNotMatchedInsertAll(): MergeInsertBuilder;
|
|
38
|
+
/**
|
|
39
|
+
* Rows that exist only in the target table (old data) will be
|
|
40
|
+
* deleted. An optional condition can be provided to limit what
|
|
41
|
+
* data is deleted.
|
|
42
|
+
*
|
|
43
|
+
* @param options.where - An optional condition to limit what data is deleted
|
|
44
|
+
*/
|
|
45
|
+
whenNotMatchedBySourceDelete(options?: {
|
|
46
|
+
where: string;
|
|
47
|
+
}): MergeInsertBuilder;
|
|
48
|
+
/**
|
|
49
|
+
* Executes the merge insert operation
|
|
50
|
+
*
|
|
51
|
+
* Nothing is returned but the `Table` is updated
|
|
52
|
+
*/
|
|
53
|
+
execute(data: Data): Promise<void>;
|
|
54
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MergeInsertBuilder = void 0;
|
|
4
|
+
const arrow_1 = require("./arrow");
|
|
5
|
+
/** A builder used to create and run a merge insert operation */
|
|
6
|
+
class MergeInsertBuilder {
|
|
7
|
+
#native;
|
|
8
|
+
/** Construct a MergeInsertBuilder. __Internal use only.__ */
|
|
9
|
+
constructor(native) {
|
|
10
|
+
this.#native = native;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Rows that exist in both the source table (new data) and
|
|
14
|
+
* the target table (old data) will be updated, replacing
|
|
15
|
+
* the old row with the corresponding matching row.
|
|
16
|
+
*
|
|
17
|
+
* If there are multiple matches then the behavior is undefined.
|
|
18
|
+
* Currently this causes multiple copies of the row to be created
|
|
19
|
+
* but that behavior is subject to change.
|
|
20
|
+
*
|
|
21
|
+
* An optional condition may be specified. If it is, then only
|
|
22
|
+
* matched rows that satisfy the condtion will be updated. Any
|
|
23
|
+
* rows that do not satisfy the condition will be left as they
|
|
24
|
+
* are. Failing to satisfy the condition does not cause a
|
|
25
|
+
* "matched row" to become a "not matched" row.
|
|
26
|
+
*
|
|
27
|
+
* The condition should be an SQL string. Use the prefix
|
|
28
|
+
* target. to refer to rows in the target table (old data)
|
|
29
|
+
* and the prefix source. to refer to rows in the source
|
|
30
|
+
* table (new data).
|
|
31
|
+
*
|
|
32
|
+
* For example, "target.last_update < source.last_update"
|
|
33
|
+
*/
|
|
34
|
+
whenMatchedUpdateAll(options) {
|
|
35
|
+
return new MergeInsertBuilder(this.#native.whenMatchedUpdateAll(options?.where));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Rows that exist only in the source table (new data) should
|
|
39
|
+
* be inserted into the target table.
|
|
40
|
+
*/
|
|
41
|
+
whenNotMatchedInsertAll() {
|
|
42
|
+
return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Rows that exist only in the target table (old data) will be
|
|
46
|
+
* deleted. An optional condition can be provided to limit what
|
|
47
|
+
* data is deleted.
|
|
48
|
+
*
|
|
49
|
+
* @param options.where - An optional condition to limit what data is deleted
|
|
50
|
+
*/
|
|
51
|
+
whenNotMatchedBySourceDelete(options) {
|
|
52
|
+
return new MergeInsertBuilder(this.#native.whenNotMatchedBySourceDelete(options?.where));
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Executes the merge insert operation
|
|
56
|
+
*
|
|
57
|
+
* Nothing is returned but the `Table` is updated
|
|
58
|
+
*/
|
|
59
|
+
async execute(data) {
|
|
60
|
+
const buffer = await (0, arrow_1.fromDataToBuffer)(data);
|
|
61
|
+
await this.#native.execute(buffer);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
exports.MergeInsertBuilder = MergeInsertBuilder;
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
|
|
6
6
|
/** A description of an index currently configured on a column */
|
|
7
7
|
export interface IndexConfig {
|
|
8
|
+
/** The name of the index */
|
|
9
|
+
name: string
|
|
8
10
|
/** The type of the index */
|
|
9
11
|
indexType: string
|
|
10
12
|
/**
|
|
11
13
|
* The columns in the index
|
|
12
14
|
*
|
|
13
|
-
* Currently this is always an array of size 1.
|
|
15
|
+
* Currently this is always an array of size 1. In the future there may
|
|
14
16
|
* be more columns to represent composite indices.
|
|
15
17
|
*/
|
|
16
18
|
columns: Array<string>
|
|
@@ -72,9 +74,21 @@ export interface AddColumnsSql {
|
|
|
72
74
|
*/
|
|
73
75
|
valueSql: string
|
|
74
76
|
}
|
|
77
|
+
export interface IndexStatistics {
|
|
78
|
+
/** The number of rows indexed by the index */
|
|
79
|
+
numIndexedRows: number
|
|
80
|
+
/** The number of rows not indexed */
|
|
81
|
+
numUnindexedRows: number
|
|
82
|
+
/** The type of the index */
|
|
83
|
+
indexType?: string
|
|
84
|
+
/** The metadata for each index */
|
|
85
|
+
indices: Array<IndexMetadata>
|
|
86
|
+
}
|
|
87
|
+
export interface IndexMetadata {
|
|
88
|
+
metricType?: string
|
|
89
|
+
indexType?: string
|
|
90
|
+
}
|
|
75
91
|
export interface ConnectionOptions {
|
|
76
|
-
apiKey?: string
|
|
77
|
-
hostOverride?: string
|
|
78
92
|
/**
|
|
79
93
|
* (For LanceDB OSS only): The interval, in seconds, at which to check for
|
|
80
94
|
* updates to the table from other processes. If None, then consistency is not
|
|
@@ -102,6 +116,7 @@ export const enum WriteMode {
|
|
|
102
116
|
}
|
|
103
117
|
/** Write options when creating a Table. */
|
|
104
118
|
export interface WriteOptions {
|
|
119
|
+
/** Write mode for writing to a table. */
|
|
105
120
|
mode?: WriteMode
|
|
106
121
|
}
|
|
107
122
|
export interface OpenTableOptions {
|
|
@@ -123,8 +138,8 @@ export class Connection {
|
|
|
123
138
|
* - buf: The buffer containing the IPC file.
|
|
124
139
|
*
|
|
125
140
|
*/
|
|
126
|
-
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
127
|
-
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
|
|
141
|
+
createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
|
|
142
|
+
createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
|
|
128
143
|
openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
|
|
129
144
|
/** Drop table with the name. Or raise an error if the table does not exist. */
|
|
130
145
|
dropTable(name: string): Promise<void>
|
|
@@ -137,12 +152,20 @@ export class Index {
|
|
|
137
152
|
export class RecordBatchIterator {
|
|
138
153
|
next(): Promise<Buffer | null>
|
|
139
154
|
}
|
|
155
|
+
/** A builder used to create and run a merge insert operation */
|
|
156
|
+
export class NativeMergeInsertBuilder {
|
|
157
|
+
whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
|
|
158
|
+
whenNotMatchedInsertAll(): NativeMergeInsertBuilder
|
|
159
|
+
whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
|
|
160
|
+
execute(buf: Buffer): Promise<void>
|
|
161
|
+
}
|
|
140
162
|
export class Query {
|
|
141
163
|
onlyIf(predicate: string): void
|
|
142
164
|
select(columns: Array<[string, string]>): void
|
|
143
165
|
limit(limit: number): void
|
|
144
166
|
nearestTo(vector: Float32Array): VectorQuery
|
|
145
|
-
execute(): Promise<RecordBatchIterator>
|
|
167
|
+
execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
|
|
168
|
+
explainPlan(verbose: boolean): Promise<string>
|
|
146
169
|
}
|
|
147
170
|
export class VectorQuery {
|
|
148
171
|
column(column: string): void
|
|
@@ -154,9 +177,11 @@ export class VectorQuery {
|
|
|
154
177
|
onlyIf(predicate: string): void
|
|
155
178
|
select(columns: Array<[string, string]>): void
|
|
156
179
|
limit(limit: number): void
|
|
157
|
-
execute(): Promise<RecordBatchIterator>
|
|
180
|
+
execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
|
|
181
|
+
explainPlan(verbose: boolean): Promise<string>
|
|
158
182
|
}
|
|
159
183
|
export class Table {
|
|
184
|
+
name: string
|
|
160
185
|
display(): string
|
|
161
186
|
isOpen(): boolean
|
|
162
187
|
close(): void
|
|
@@ -178,4 +203,6 @@ export class Table {
|
|
|
178
203
|
restore(): Promise<void>
|
|
179
204
|
optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
|
|
180
205
|
listIndices(): Promise<Array<IndexConfig>>
|
|
206
|
+
indexStats(indexName: string): Promise<IndexStatistics | null>
|
|
207
|
+
mergeInsert(on: Array<string>): NativeMergeInsertBuilder
|
|
181
208
|
}
|
|
@@ -233,17 +233,33 @@ switch (platform) {
|
|
|
233
233
|
}
|
|
234
234
|
break;
|
|
235
235
|
case 'arm':
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
236
|
+
if (isMusl()) {
|
|
237
|
+
localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-musleabihf.node'));
|
|
238
|
+
try {
|
|
239
|
+
if (localFileExisted) {
|
|
240
|
+
nativeBinding = require('./lancedb.linux-arm-musleabihf.node');
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
nativeBinding = require('@lancedb/lancedb-linux-arm-musleabihf');
|
|
244
|
+
}
|
|
240
245
|
}
|
|
241
|
-
|
|
242
|
-
|
|
246
|
+
catch (e) {
|
|
247
|
+
loadError = e;
|
|
243
248
|
}
|
|
244
249
|
}
|
|
245
|
-
|
|
246
|
-
|
|
250
|
+
else {
|
|
251
|
+
localFileExisted = existsSync(join(__dirname, 'lancedb.linux-arm-gnueabihf.node'));
|
|
252
|
+
try {
|
|
253
|
+
if (localFileExisted) {
|
|
254
|
+
nativeBinding = require('./lancedb.linux-arm-gnueabihf.node');
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
nativeBinding = require('@lancedb/lancedb-linux-arm-gnueabihf');
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
catch (e) {
|
|
261
|
+
loadError = e;
|
|
262
|
+
}
|
|
247
263
|
}
|
|
248
264
|
break;
|
|
249
265
|
case 'riscv64':
|
|
@@ -303,10 +319,11 @@ if (!nativeBinding) {
|
|
|
303
319
|
}
|
|
304
320
|
throw new Error(`Failed to load native binding`);
|
|
305
321
|
}
|
|
306
|
-
const { Connection, Index, RecordBatchIterator, Query, VectorQuery, Table, WriteMode } = nativeBinding;
|
|
322
|
+
const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Table, WriteMode } = nativeBinding;
|
|
307
323
|
module.exports.Connection = Connection;
|
|
308
324
|
module.exports.Index = Index;
|
|
309
325
|
module.exports.RecordBatchIterator = RecordBatchIterator;
|
|
326
|
+
module.exports.NativeMergeInsertBuilder = NativeMergeInsertBuilder;
|
|
310
327
|
module.exports.Query = Query;
|
|
311
328
|
module.exports.VectorQuery = VectorQuery;
|
|
312
329
|
module.exports.Table = Table;
|