@lancedb/lancedb 0.5.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +3 -3
- package/biome.json +19 -3
- package/dist/arrow.d.ts +41 -8
- package/dist/arrow.js +4 -4
- package/dist/connection.d.ts +49 -29
- package/dist/connection.js +21 -73
- package/dist/embedding/embedding_function.d.ts +9 -1
- package/dist/embedding/embedding_function.js +6 -0
- package/dist/embedding/openai.d.ts +6 -5
- package/dist/embedding/openai.js +4 -2
- package/dist/embedding/registry.d.ts +6 -11
- package/dist/index.d.ts +51 -3
- package/dist/index.js +28 -4
- package/dist/merge.d.ts +54 -0
- package/dist/merge.js +64 -0
- package/dist/native.d.ts +29 -3
- package/dist/native.js +26 -9
- package/dist/query.d.ts +33 -10
- package/dist/query.js +100 -13
- package/dist/remote/client.d.ts +28 -0
- package/dist/remote/client.js +172 -0
- package/dist/remote/connection.d.ts +25 -0
- package/dist/remote/connection.js +110 -0
- package/dist/remote/index.d.ts +3 -0
- package/dist/remote/index.js +9 -0
- package/dist/remote/table.d.ts +42 -0
- package/dist/remote/table.js +179 -0
- package/dist/sanitize.d.ts +3 -2
- package/dist/sanitize.js +55 -1
- package/dist/table.d.ts +105 -30
- package/dist/table.js +94 -237
- package/dist/util.d.ts +14 -0
- package/dist/util.js +65 -0
- package/examples/ann_indexes.ts +49 -0
- package/examples/basic.ts +149 -0
- package/examples/embedding.ts +83 -0
- package/examples/filtering.ts +34 -0
- package/examples/jsconfig.json +27 -0
- package/examples/package-lock.json +79 -0
- package/examples/package.json +18 -0
- package/examples/search.ts +37 -0
- package/lancedb/arrow.ts +80 -23
- package/lancedb/connection.ts +107 -92
- package/lancedb/embedding/embedding_function.ts +12 -1
- package/lancedb/embedding/openai.ts +11 -6
- package/lancedb/embedding/registry.ts +34 -22
- package/lancedb/index.ts +101 -2
- package/lancedb/merge.ts +70 -0
- package/lancedb/query.ts +114 -28
- package/lancedb/remote/client.ts +221 -0
- package/lancedb/remote/connection.ts +201 -0
- package/lancedb/remote/index.ts +3 -0
- package/lancedb/remote/table.ts +226 -0
- package/lancedb/sanitize.ts +73 -1
- package/lancedb/table.ts +320 -132
- package/lancedb/util.ts +69 -0
- package/native.d.ts +208 -0
- package/nodejs-artifacts/arrow.d.ts +41 -8
- package/nodejs-artifacts/arrow.js +4 -4
- package/nodejs-artifacts/connection.d.ts +49 -29
- package/nodejs-artifacts/connection.js +21 -73
- package/nodejs-artifacts/embedding/embedding_function.d.ts +9 -1
- package/nodejs-artifacts/embedding/embedding_function.js +6 -0
- package/nodejs-artifacts/embedding/openai.d.ts +6 -5
- package/nodejs-artifacts/embedding/openai.js +4 -2
- package/nodejs-artifacts/embedding/registry.d.ts +6 -11
- package/nodejs-artifacts/index.d.ts +51 -3
- package/nodejs-artifacts/index.js +28 -4
- package/nodejs-artifacts/merge.d.ts +54 -0
- package/nodejs-artifacts/merge.js +64 -0
- package/nodejs-artifacts/native.d.ts +29 -3
- package/nodejs-artifacts/native.js +26 -9
- package/nodejs-artifacts/query.d.ts +33 -10
- package/nodejs-artifacts/query.js +100 -13
- package/nodejs-artifacts/remote/client.d.ts +28 -0
- package/nodejs-artifacts/remote/client.js +172 -0
- package/nodejs-artifacts/remote/connection.d.ts +25 -0
- package/nodejs-artifacts/remote/connection.js +110 -0
- package/nodejs-artifacts/remote/index.d.ts +3 -0
- package/nodejs-artifacts/remote/index.js +9 -0
- package/nodejs-artifacts/remote/table.d.ts +42 -0
- package/nodejs-artifacts/remote/table.js +179 -0
- package/nodejs-artifacts/sanitize.d.ts +3 -2
- package/nodejs-artifacts/sanitize.js +55 -1
- package/nodejs-artifacts/table.d.ts +105 -30
- package/nodejs-artifacts/table.js +94 -237
- package/nodejs-artifacts/util.d.ts +14 -0
- package/nodejs-artifacts/util.js +65 -0
- package/package.json +25 -11
package/Cargo.toml
CHANGED
|
@@ -15,11 +15,11 @@ crate-type = ["cdylib"]
|
|
|
15
15
|
arrow-ipc.workspace = true
|
|
16
16
|
futures.workspace = true
|
|
17
17
|
lancedb = { path = "../rust/lancedb" }
|
|
18
|
-
napi = { version = "2.
|
|
19
|
-
"
|
|
18
|
+
napi = { version = "2.16.8", default-features = false, features = [
|
|
19
|
+
"napi9",
|
|
20
20
|
"async",
|
|
21
21
|
] }
|
|
22
|
-
napi-derive = "2"
|
|
22
|
+
napi-derive = "2.16.4"
|
|
23
23
|
|
|
24
24
|
# Prevent dynamic linking of lzma, which comes from datafusion
|
|
25
25
|
lzma-sys = { version = "*", features = ["static"] }
|
package/biome.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"$schema": "https://biomejs.dev/schemas/1.
|
|
2
|
+
"$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
|
|
3
3
|
"organizeImports": {
|
|
4
4
|
"enabled": true
|
|
5
5
|
},
|
|
@@ -77,7 +77,7 @@
|
|
|
77
77
|
"noDuplicateObjectKeys": "error",
|
|
78
78
|
"noDuplicateParameters": "error",
|
|
79
79
|
"noEmptyBlockStatements": "error",
|
|
80
|
-
"noExplicitAny": "
|
|
80
|
+
"noExplicitAny": "warn",
|
|
81
81
|
"noExtraNonNullAssertion": "error",
|
|
82
82
|
"noFallthroughSwitchClause": "error",
|
|
83
83
|
"noFunctionAssign": "error",
|
|
@@ -94,12 +94,28 @@
|
|
|
94
94
|
"useValidTypeof": "error"
|
|
95
95
|
}
|
|
96
96
|
},
|
|
97
|
-
"ignore": [
|
|
97
|
+
"ignore": [
|
|
98
|
+
"**/dist/**/*",
|
|
99
|
+
"**/native.js",
|
|
100
|
+
"**/native.d.ts",
|
|
101
|
+
"__test__/docs/**/*",
|
|
102
|
+
"examples/**/*"
|
|
103
|
+
]
|
|
98
104
|
},
|
|
99
105
|
"javascript": {
|
|
100
106
|
"globals": []
|
|
101
107
|
},
|
|
102
108
|
"overrides": [
|
|
109
|
+
{
|
|
110
|
+
"include": ["__test__/s3_integration.test.ts"],
|
|
111
|
+
"linter": {
|
|
112
|
+
"rules": {
|
|
113
|
+
"style": {
|
|
114
|
+
"useNamingConvention": "off"
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
},
|
|
103
119
|
{
|
|
104
120
|
"include": [
|
|
105
121
|
"**/*.ts",
|
package/dist/arrow.d.ts
CHANGED
|
@@ -1,10 +1,43 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { Table as ArrowTable, Binary, DataType, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, Schema, Struct, Utf8 } from "apache-arrow";
|
|
2
|
+
import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
|
|
3
|
+
import { Buffers } from "apache-arrow/data";
|
|
3
4
|
import { type EmbeddingFunction } from "./embedding/embedding_function";
|
|
4
5
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
5
6
|
export * from "apache-arrow";
|
|
6
|
-
export type
|
|
7
|
-
|
|
7
|
+
export type SchemaLike = Schema | {
|
|
8
|
+
fields: FieldLike[];
|
|
9
|
+
metadata: Map<string, string>;
|
|
10
|
+
get names(): unknown[];
|
|
11
|
+
};
|
|
12
|
+
export type FieldLike = Field | {
|
|
13
|
+
type: string;
|
|
14
|
+
name: string;
|
|
15
|
+
nullable?: boolean;
|
|
16
|
+
metadata?: Map<string, string>;
|
|
17
|
+
};
|
|
18
|
+
export type DataLike = import("apache-arrow").Data<Struct<any>> | {
|
|
19
|
+
type: any;
|
|
20
|
+
length: number;
|
|
21
|
+
offset: number;
|
|
22
|
+
stride: number;
|
|
23
|
+
nullable: boolean;
|
|
24
|
+
children: DataLike[];
|
|
25
|
+
get nullCount(): number;
|
|
26
|
+
values: Buffers<any>[BufferType.DATA];
|
|
27
|
+
typeIds: Buffers<any>[BufferType.TYPE];
|
|
28
|
+
nullBitmap: Buffers<any>[BufferType.VALIDITY];
|
|
29
|
+
valueOffsets: Buffers<any>[BufferType.OFFSET];
|
|
30
|
+
};
|
|
31
|
+
export type RecordBatchLike = RecordBatch | {
|
|
32
|
+
schema: SchemaLike;
|
|
33
|
+
data: DataLike;
|
|
34
|
+
};
|
|
35
|
+
export type TableLike = ArrowTable | {
|
|
36
|
+
schema: SchemaLike;
|
|
37
|
+
batches: RecordBatchLike[];
|
|
38
|
+
};
|
|
39
|
+
export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
|
|
40
|
+
export declare function isArrowTable(value: object): value is TableLike;
|
|
8
41
|
export declare function isDataType(value: unknown): value is DataType;
|
|
9
42
|
export declare function isNull(value: unknown): value is Null;
|
|
10
43
|
export declare function isInt(value: unknown): value is Int;
|
|
@@ -26,7 +59,7 @@ export declare function isUnion(value: unknown): value is Struct;
|
|
|
26
59
|
export declare function isFixedSizeBinary(value: unknown): value is FixedSizeBinary;
|
|
27
60
|
export declare function isFixedSizeList(value: unknown): value is FixedSizeList;
|
|
28
61
|
/** Data type accepted by NodeJS SDK */
|
|
29
|
-
export type Data = Record<string, unknown>[] |
|
|
62
|
+
export type Data = Record<string, unknown>[] | TableLike;
|
|
30
63
|
export declare class VectorColumnOptions {
|
|
31
64
|
/** Vector column type. */
|
|
32
65
|
type: Float;
|
|
@@ -34,7 +67,7 @@ export declare class VectorColumnOptions {
|
|
|
34
67
|
}
|
|
35
68
|
/** Options to control the makeArrowTable call. */
|
|
36
69
|
export declare class MakeArrowTableOptions {
|
|
37
|
-
schema?:
|
|
70
|
+
schema?: SchemaLike;
|
|
38
71
|
vectorColumns: Record<string, VectorColumnOptions>;
|
|
39
72
|
embeddings?: EmbeddingFunction<unknown>;
|
|
40
73
|
embeddingFunction?: EmbeddingFunctionConfig;
|
|
@@ -146,7 +179,7 @@ export declare function makeArrowTable(data: Array<Record<string, unknown>>, opt
|
|
|
146
179
|
/**
|
|
147
180
|
* Create an empty Arrow table with the provided schema
|
|
148
181
|
*/
|
|
149
|
-
export declare function makeEmptyTable(schema:
|
|
182
|
+
export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string, string>): ArrowTable;
|
|
150
183
|
/**
|
|
151
184
|
* Convert an Array of records into an Arrow Table, optionally applying an
|
|
152
185
|
* embeddings function to it.
|
|
@@ -192,7 +225,7 @@ export declare function fromRecordsToStreamBuffer(data: Array<Record<string, unk
|
|
|
192
225
|
*
|
|
193
226
|
* `schema` is required if the table is empty
|
|
194
227
|
*/
|
|
195
|
-
export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?:
|
|
228
|
+
export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
|
|
196
229
|
/**
|
|
197
230
|
* Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
|
|
198
231
|
*
|
|
@@ -210,7 +243,7 @@ export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunct
|
|
|
210
243
|
*
|
|
211
244
|
* `schema` is required if the table is empty
|
|
212
245
|
*/
|
|
213
|
-
export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?:
|
|
246
|
+
export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike): Promise<Buffer>;
|
|
214
247
|
/**
|
|
215
248
|
* Create an empty table with the given schema
|
|
216
249
|
*/
|
package/dist/arrow.js
CHANGED
|
@@ -505,15 +505,15 @@ async function applyEmbeddingsFromMetadata(table, schema) {
|
|
|
505
505
|
}
|
|
506
506
|
/** Helper function to apply embeddings to an input table */
|
|
507
507
|
async function applyEmbeddings(table, embeddings, schema) {
|
|
508
|
+
if (schema !== undefined && schema !== null) {
|
|
509
|
+
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
510
|
+
}
|
|
508
511
|
if (schema?.metadata.has("embedding_functions")) {
|
|
509
512
|
return applyEmbeddingsFromMetadata(table, schema);
|
|
510
513
|
}
|
|
511
514
|
else if (embeddings == null || embeddings === undefined) {
|
|
512
515
|
return table;
|
|
513
516
|
}
|
|
514
|
-
if (schema !== undefined && schema !== null) {
|
|
515
|
-
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
516
|
-
}
|
|
517
517
|
// Convert from ArrowTable to Record<String, Vector>
|
|
518
518
|
const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
|
|
519
519
|
const name = table.schema.fields[idx].name;
|
|
@@ -670,7 +670,7 @@ async function fromDataToBuffer(data, embeddings, schema) {
|
|
|
670
670
|
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
671
671
|
}
|
|
672
672
|
if (isArrowTable(data)) {
|
|
673
|
-
return fromTableToBuffer(data, embeddings, schema);
|
|
673
|
+
return fromTableToBuffer((0, sanitize_1.sanitizeTable)(data), embeddings, schema);
|
|
674
674
|
}
|
|
675
675
|
else {
|
|
676
676
|
const table = await convertToTable(data, embeddings, { schema });
|
package/dist/connection.d.ts
CHANGED
|
@@ -1,20 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
2
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
3
|
-
import {
|
|
3
|
+
import { Connection as LanceDbConnection } from "./native";
|
|
4
4
|
import { Table } from "./table";
|
|
5
|
-
/**
|
|
6
|
-
* Connect to a LanceDB instance at the given URI.
|
|
7
|
-
*
|
|
8
|
-
* Accepted formats:
|
|
9
|
-
*
|
|
10
|
-
* - `/path/to/database` - local database
|
|
11
|
-
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
12
|
-
* - `db://host:port` - remote database (LanceDB cloud)
|
|
13
|
-
* @param {string} uri - The uri of the database. If the database uri starts
|
|
14
|
-
* with `db://` then it connects to a remote database.
|
|
15
|
-
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
16
|
-
*/
|
|
17
|
-
export declare function connect(uri: string, opts?: Partial<ConnectionOptions>): Promise<Connection>;
|
|
18
5
|
export interface CreateTableOptions {
|
|
19
6
|
/**
|
|
20
7
|
* The mode to use when creating the table.
|
|
@@ -46,7 +33,7 @@ export interface CreateTableOptions {
|
|
|
46
33
|
* The default is true while the new format is in beta
|
|
47
34
|
*/
|
|
48
35
|
useLegacyFormat?: boolean;
|
|
49
|
-
schema?:
|
|
36
|
+
schema?: SchemaLike;
|
|
50
37
|
embeddingFunction?: EmbeddingFunctionConfig;
|
|
51
38
|
}
|
|
52
39
|
export interface OpenTableOptions {
|
|
@@ -102,11 +89,11 @@ export interface TableNamesOptions {
|
|
|
102
89
|
* Any created tables are independent and will continue to work even if
|
|
103
90
|
* the underlying connection has been closed.
|
|
104
91
|
*/
|
|
105
|
-
export declare class Connection {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
isOpen(): boolean;
|
|
92
|
+
export declare abstract class Connection {
|
|
93
|
+
/**
|
|
94
|
+
* Return true if the connection has not been closed
|
|
95
|
+
*/
|
|
96
|
+
abstract isOpen(): boolean;
|
|
110
97
|
/**
|
|
111
98
|
* Close the connection, releasing any underlying resources.
|
|
112
99
|
*
|
|
@@ -114,38 +101,71 @@ export declare class Connection {
|
|
|
114
101
|
*
|
|
115
102
|
* Any attempt to use the connection after it is closed will result in an error.
|
|
116
103
|
*/
|
|
117
|
-
close(): void;
|
|
118
|
-
/**
|
|
119
|
-
|
|
104
|
+
abstract close(): void;
|
|
105
|
+
/**
|
|
106
|
+
* Return a brief description of the connection
|
|
107
|
+
*/
|
|
108
|
+
abstract display(): string;
|
|
120
109
|
/**
|
|
121
110
|
* List all the table names in this database.
|
|
122
111
|
*
|
|
123
112
|
* Tables will be returned in lexicographical order.
|
|
124
113
|
* @param {Partial<TableNamesOptions>} options - options to control the
|
|
125
114
|
* paging / start point
|
|
115
|
+
*
|
|
126
116
|
*/
|
|
127
|
-
tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
117
|
+
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
128
118
|
/**
|
|
129
119
|
* Open a table in the database.
|
|
130
120
|
* @param {string} name - The name of the table
|
|
131
121
|
*/
|
|
132
|
-
openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
122
|
+
abstract openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
123
|
+
/**
|
|
124
|
+
* Creates a new Table and initialize it with new data.
|
|
125
|
+
* @param {object} options - The options object.
|
|
126
|
+
* @param {string} options.name - The name of the table.
|
|
127
|
+
* @param {Data} options.data - Non-empty Array of Records to be inserted into the table
|
|
128
|
+
*
|
|
129
|
+
*/
|
|
130
|
+
abstract createTable(options: {
|
|
131
|
+
name: string;
|
|
132
|
+
data: Data;
|
|
133
|
+
} & Partial<CreateTableOptions>): Promise<Table>;
|
|
133
134
|
/**
|
|
134
135
|
* Creates a new Table and initialize it with new data.
|
|
135
136
|
* @param {string} name - The name of the table.
|
|
136
|
-
* @param {Record<string, unknown>[] |
|
|
137
|
+
* @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
|
|
137
138
|
* to be inserted into the table
|
|
138
139
|
*/
|
|
139
|
-
createTable(name: string, data: Record<string, unknown>[] |
|
|
140
|
+
abstract createTable(name: string, data: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
140
141
|
/**
|
|
141
142
|
* Creates a new empty Table
|
|
142
143
|
* @param {string} name - The name of the table.
|
|
143
144
|
* @param {Schema} schema - The schema of the table
|
|
144
145
|
*/
|
|
145
|
-
createEmptyTable(name: string, schema:
|
|
146
|
+
abstract createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
146
147
|
/**
|
|
147
148
|
* Drop an existing table.
|
|
148
149
|
* @param {string} name The name of the table to drop.
|
|
149
150
|
*/
|
|
151
|
+
abstract dropTable(name: string): Promise<void>;
|
|
152
|
+
}
|
|
153
|
+
export declare class LocalConnection extends Connection {
|
|
154
|
+
readonly inner: LanceDbConnection;
|
|
155
|
+
constructor(inner: LanceDbConnection);
|
|
156
|
+
isOpen(): boolean;
|
|
157
|
+
close(): void;
|
|
158
|
+
display(): string;
|
|
159
|
+
tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
|
|
160
|
+
openTable(name: string, options?: Partial<OpenTableOptions>): Promise<Table>;
|
|
161
|
+
createTable(nameOrOptions: string | ({
|
|
162
|
+
name: string;
|
|
163
|
+
data: Data;
|
|
164
|
+
} & Partial<CreateTableOptions>), data?: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
165
|
+
createEmptyTable(name: string, schema: import("./arrow").SchemaLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
150
166
|
dropTable(name: string): Promise<void>;
|
|
151
167
|
}
|
|
168
|
+
/**
|
|
169
|
+
* Takes storage options and makes all the keys snake case.
|
|
170
|
+
*/
|
|
171
|
+
export declare function cleanseStorageOptions(options?: Record<string, string>): Record<string, string> | undefined;
|
package/dist/connection.js
CHANGED
|
@@ -13,30 +13,10 @@
|
|
|
13
13
|
// See the License for the specific language governing permissions and
|
|
14
14
|
// limitations under the License.
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.
|
|
16
|
+
exports.cleanseStorageOptions = exports.LocalConnection = exports.Connection = void 0;
|
|
17
17
|
const arrow_1 = require("./arrow");
|
|
18
18
|
const registry_1 = require("./embedding/registry");
|
|
19
|
-
const native_1 = require("./native");
|
|
20
19
|
const table_1 = require("./table");
|
|
21
|
-
/**
|
|
22
|
-
* Connect to a LanceDB instance at the given URI.
|
|
23
|
-
*
|
|
24
|
-
* Accepted formats:
|
|
25
|
-
*
|
|
26
|
-
* - `/path/to/database` - local database
|
|
27
|
-
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
28
|
-
* - `db://host:port` - remote database (LanceDB cloud)
|
|
29
|
-
* @param {string} uri - The uri of the database. If the database uri starts
|
|
30
|
-
* with `db://` then it connects to a remote database.
|
|
31
|
-
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
32
|
-
*/
|
|
33
|
-
async function connect(uri, opts) {
|
|
34
|
-
opts = opts ?? {};
|
|
35
|
-
opts.storageOptions = cleanseStorageOptions(opts.storageOptions);
|
|
36
|
-
const nativeConn = await native_1.Connection.new(uri, opts);
|
|
37
|
-
return new Connection(nativeConn);
|
|
38
|
-
}
|
|
39
|
-
exports.connect = connect;
|
|
40
20
|
/**
|
|
41
21
|
* A LanceDB Connection that allows you to open tables and create new ones.
|
|
42
22
|
*
|
|
@@ -56,74 +36,45 @@ exports.connect = connect;
|
|
|
56
36
|
* the underlying connection has been closed.
|
|
57
37
|
*/
|
|
58
38
|
class Connection {
|
|
39
|
+
[Symbol.for("nodejs.util.inspect.custom")]() {
|
|
40
|
+
return this.display();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
exports.Connection = Connection;
|
|
44
|
+
class LocalConnection extends Connection {
|
|
59
45
|
inner;
|
|
60
46
|
constructor(inner) {
|
|
47
|
+
super();
|
|
61
48
|
this.inner = inner;
|
|
62
49
|
}
|
|
63
|
-
/** Return true if the connection has not been closed */
|
|
64
50
|
isOpen() {
|
|
65
51
|
return this.inner.isOpen();
|
|
66
52
|
}
|
|
67
|
-
/**
|
|
68
|
-
* Close the connection, releasing any underlying resources.
|
|
69
|
-
*
|
|
70
|
-
* It is safe to call this method multiple times.
|
|
71
|
-
*
|
|
72
|
-
* Any attempt to use the connection after it is closed will result in an error.
|
|
73
|
-
*/
|
|
74
53
|
close() {
|
|
75
54
|
this.inner.close();
|
|
76
55
|
}
|
|
77
|
-
/** Return a brief description of the connection */
|
|
78
56
|
display() {
|
|
79
57
|
return this.inner.display();
|
|
80
58
|
}
|
|
81
|
-
/**
|
|
82
|
-
* List all the table names in this database.
|
|
83
|
-
*
|
|
84
|
-
* Tables will be returned in lexicographical order.
|
|
85
|
-
* @param {Partial<TableNamesOptions>} options - options to control the
|
|
86
|
-
* paging / start point
|
|
87
|
-
*/
|
|
88
59
|
async tableNames(options) {
|
|
89
60
|
return this.inner.tableNames(options?.startAfter, options?.limit);
|
|
90
61
|
}
|
|
91
|
-
/**
|
|
92
|
-
* Open a table in the database.
|
|
93
|
-
* @param {string} name - The name of the table
|
|
94
|
-
*/
|
|
95
62
|
async openTable(name, options) {
|
|
96
63
|
const innerTable = await this.inner.openTable(name, cleanseStorageOptions(options?.storageOptions), options?.indexCacheSize);
|
|
97
|
-
return new table_1.
|
|
64
|
+
return new table_1.LocalTable(innerTable);
|
|
98
65
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
* to be inserted into the table
|
|
104
|
-
*/
|
|
105
|
-
async createTable(name, data, options) {
|
|
106
|
-
let mode = options?.mode ?? "create";
|
|
107
|
-
const existOk = options?.existOk ?? false;
|
|
108
|
-
if (mode === "create" && existOk) {
|
|
109
|
-
mode = "exist_ok";
|
|
110
|
-
}
|
|
111
|
-
let table;
|
|
112
|
-
if ((0, arrow_1.isArrowTable)(data)) {
|
|
113
|
-
table = data;
|
|
66
|
+
async createTable(nameOrOptions, data, options) {
|
|
67
|
+
if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
|
|
68
|
+
const { name, data, ...options } = nameOrOptions;
|
|
69
|
+
return this.createTable(name, data, options);
|
|
114
70
|
}
|
|
115
|
-
|
|
116
|
-
|
|
71
|
+
if (data === undefined) {
|
|
72
|
+
throw new Error("data is required");
|
|
117
73
|
}
|
|
118
|
-
const buf = await
|
|
119
|
-
const innerTable = await this.inner.createTable(
|
|
120
|
-
return new table_1.
|
|
74
|
+
const { buf, mode } = await table_1.Table.parseTableData(data, options);
|
|
75
|
+
const innerTable = await this.inner.createTable(nameOrOptions, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
|
|
76
|
+
return new table_1.LocalTable(innerTable);
|
|
121
77
|
}
|
|
122
|
-
/**
|
|
123
|
-
* Creates a new empty Table
|
|
124
|
-
* @param {string} name - The name of the table.
|
|
125
|
-
* @param {Schema} schema - The schema of the table
|
|
126
|
-
*/
|
|
127
78
|
async createEmptyTable(name, schema, options) {
|
|
128
79
|
let mode = options?.mode ?? "create";
|
|
129
80
|
const existOk = options?.existOk ?? false;
|
|
@@ -139,17 +90,13 @@ class Connection {
|
|
|
139
90
|
const table = (0, arrow_1.makeEmptyTable)(schema, metadata);
|
|
140
91
|
const buf = await (0, arrow_1.fromTableToBuffer)(table);
|
|
141
92
|
const innerTable = await this.inner.createEmptyTable(name, buf, mode, cleanseStorageOptions(options?.storageOptions), options?.useLegacyFormat);
|
|
142
|
-
return new table_1.
|
|
93
|
+
return new table_1.LocalTable(innerTable);
|
|
143
94
|
}
|
|
144
|
-
/**
|
|
145
|
-
* Drop an existing table.
|
|
146
|
-
* @param {string} name The name of the table to drop.
|
|
147
|
-
*/
|
|
148
95
|
async dropTable(name) {
|
|
149
96
|
return this.inner.dropTable(name);
|
|
150
97
|
}
|
|
151
98
|
}
|
|
152
|
-
exports.
|
|
99
|
+
exports.LocalConnection = LocalConnection;
|
|
153
100
|
/**
|
|
154
101
|
* Takes storage options and makes all the keys snake case.
|
|
155
102
|
*/
|
|
@@ -166,6 +113,7 @@ function cleanseStorageOptions(options) {
|
|
|
166
113
|
}
|
|
167
114
|
return result;
|
|
168
115
|
}
|
|
116
|
+
exports.cleanseStorageOptions = cleanseStorageOptions;
|
|
169
117
|
/**
|
|
170
118
|
* Convert a string to snake case. It might already be snake case, in which case it is
|
|
171
119
|
* returned unchanged.
|
|
@@ -6,10 +6,18 @@ import { DataType, Float, type IntoVector } from "../arrow";
|
|
|
6
6
|
export interface FunctionOptions {
|
|
7
7
|
[key: string]: any;
|
|
8
8
|
}
|
|
9
|
+
export interface EmbeddingFunctionConstructor<T extends EmbeddingFunction = EmbeddingFunction> {
|
|
10
|
+
new (modelOptions?: T["TOptions"]): T;
|
|
11
|
+
}
|
|
9
12
|
/**
|
|
10
13
|
* An embedding function that automatically creates vector representation for a given column.
|
|
11
14
|
*/
|
|
12
15
|
export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptions = FunctionOptions> {
|
|
16
|
+
/**
|
|
17
|
+
* @ignore
|
|
18
|
+
* This is only used for associating the options type with the class for type checking
|
|
19
|
+
*/
|
|
20
|
+
readonly TOptions: M;
|
|
13
21
|
/**
|
|
14
22
|
* Convert the embedding function to a JSON object
|
|
15
23
|
* It is used to serialize the embedding function to the schema
|
|
@@ -63,7 +71,7 @@ export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptio
|
|
|
63
71
|
/**
|
|
64
72
|
Compute the embeddings for a single query
|
|
65
73
|
*/
|
|
66
|
-
computeQueryEmbeddings(data: T): Promise<IntoVector
|
|
74
|
+
computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>>;
|
|
67
75
|
}
|
|
68
76
|
export interface FieldOptions<T extends DataType = DataType> {
|
|
69
77
|
datatype: T;
|
|
@@ -21,6 +21,12 @@ const sanitize_1 = require("../sanitize");
|
|
|
21
21
|
* An embedding function that automatically creates vector representation for a given column.
|
|
22
22
|
*/
|
|
23
23
|
class EmbeddingFunction {
|
|
24
|
+
/**
|
|
25
|
+
* @ignore
|
|
26
|
+
* This is only used for associating the options type with the class for type checking
|
|
27
|
+
*/
|
|
28
|
+
// biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
|
|
29
|
+
TOptions;
|
|
24
30
|
/**
|
|
25
31
|
* sourceField is used in combination with `LanceSchema` to provide a declarative data model
|
|
26
32
|
*
|
|
@@ -1,14 +1,15 @@
|
|
|
1
|
+
import { type EmbeddingCreateParams } from "openai/resources";
|
|
1
2
|
import { Float } from "../arrow";
|
|
2
3
|
import { EmbeddingFunction } from "./embedding_function";
|
|
3
4
|
export type OpenAIOptions = {
|
|
4
|
-
apiKey
|
|
5
|
-
model
|
|
5
|
+
apiKey: string;
|
|
6
|
+
model: EmbeddingCreateParams["model"];
|
|
6
7
|
};
|
|
7
|
-
export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, OpenAIOptions
|
|
8
|
+
export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, Partial<OpenAIOptions>> {
|
|
8
9
|
#private;
|
|
9
|
-
constructor(options?: OpenAIOptions);
|
|
10
|
+
constructor(options?: Partial<OpenAIOptions>);
|
|
10
11
|
toJSON(): {
|
|
11
|
-
model: string;
|
|
12
|
+
model: (string & {}) | "text-embedding-ada-002" | "text-embedding-3-small" | "text-embedding-3-large";
|
|
12
13
|
};
|
|
13
14
|
ndims(): number;
|
|
14
15
|
embeddingDataType(): Float;
|
package/dist/embedding/openai.js
CHANGED
|
@@ -29,7 +29,9 @@ const registry_1 = require("./registry");
|
|
|
29
29
|
let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
|
|
30
30
|
#openai;
|
|
31
31
|
#modelName;
|
|
32
|
-
constructor(options = {
|
|
32
|
+
constructor(options = {
|
|
33
|
+
model: "text-embedding-ada-002",
|
|
34
|
+
}) {
|
|
33
35
|
super();
|
|
34
36
|
const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
|
|
35
37
|
if (!openAIKey) {
|
|
@@ -68,7 +70,7 @@ let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_fu
|
|
|
68
70
|
case "text-embedding-3-small":
|
|
69
71
|
return 1536;
|
|
70
72
|
default:
|
|
71
|
-
|
|
73
|
+
throw new Error(`Unknown model: ${this.#modelName}`);
|
|
72
74
|
}
|
|
73
75
|
}
|
|
74
76
|
embeddingDataType() {
|
|
@@ -1,13 +1,8 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
|
|
2
2
|
import "reflect-metadata";
|
|
3
|
-
|
|
4
|
-
[key: string]: unknown;
|
|
5
|
-
}
|
|
6
|
-
export interface EmbeddingFunctionFactory<T extends EmbeddingFunction = EmbeddingFunction> {
|
|
7
|
-
new (modelOptions?: EmbeddingFunctionOptions): T;
|
|
8
|
-
}
|
|
3
|
+
import { OpenAIEmbeddingFunction } from "./openai";
|
|
9
4
|
interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
|
|
10
|
-
create(options?:
|
|
5
|
+
create(options?: T["TOptions"]): T;
|
|
11
6
|
}
|
|
12
7
|
/**
|
|
13
8
|
* This is a singleton class used to register embedding functions
|
|
@@ -23,12 +18,12 @@ export declare class EmbeddingFunctionRegistry {
|
|
|
23
18
|
* @param func The function to register
|
|
24
19
|
* @throws Error if the function is already registered
|
|
25
20
|
*/
|
|
26
|
-
register<T extends
|
|
21
|
+
register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
|
|
27
22
|
/**
|
|
28
23
|
* Fetch an embedding function by name
|
|
29
24
|
* @param name The name of the function
|
|
30
25
|
*/
|
|
31
|
-
get<T extends EmbeddingFunction<unknown
|
|
26
|
+
get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
|
|
32
27
|
/**
|
|
33
28
|
* reset the registry to the initial state
|
|
34
29
|
*/
|
|
@@ -40,7 +35,7 @@ export declare class EmbeddingFunctionRegistry {
|
|
|
40
35
|
functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
|
|
41
36
|
getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
|
|
42
37
|
}
|
|
43
|
-
export declare function register(name?: string): (ctor:
|
|
38
|
+
export declare function register(name?: string): (ctor: EmbeddingFunctionConstructor<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
|
|
44
39
|
/**
|
|
45
40
|
* Utility function to get the global instance of the registry
|
|
46
41
|
* @returns `EmbeddingFunctionRegistry` The global instance of the registry
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,55 @@
|
|
|
1
|
-
|
|
1
|
+
import { Connection } from "./connection";
|
|
2
|
+
import { ConnectionOptions } from "./native.js";
|
|
3
|
+
import { RemoteConnectionOptions } from "./remote";
|
|
4
|
+
export { WriteOptions, WriteMode, AddColumnsSql, ColumnAlteration, ConnectionOptions, IndexStatistics, IndexMetadata, IndexConfig, } from "./native.js";
|
|
2
5
|
export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
3
|
-
export {
|
|
6
|
+
export { Connection, CreateTableOptions, TableNamesOptions, } from "./connection";
|
|
4
7
|
export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, } from "./query";
|
|
5
8
|
export { Index, IndexOptions, IvfPqOptions } from "./indices";
|
|
6
|
-
export { Table, AddDataOptions,
|
|
9
|
+
export { Table, AddDataOptions, UpdateOptions } from "./table";
|
|
7
10
|
export * as embedding from "./embedding";
|
|
11
|
+
/**
|
|
12
|
+
* Connect to a LanceDB instance at the given URI.
|
|
13
|
+
*
|
|
14
|
+
* Accepted formats:
|
|
15
|
+
*
|
|
16
|
+
* - `/path/to/database` - local database
|
|
17
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
18
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
19
|
+
* @param {string} uri - The uri of the database. If the database uri starts
|
|
20
|
+
* with `db://` then it connects to a remote database.
|
|
21
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
22
|
+
* @example
|
|
23
|
+
* ```ts
|
|
24
|
+
* const conn = await connect("/path/to/database");
|
|
25
|
+
* ```
|
|
26
|
+
* @example
|
|
27
|
+
* ```ts
|
|
28
|
+
* const conn = await connect(
|
|
29
|
+
* "s3://bucket/path/to/database",
|
|
30
|
+
* {storageOptions: {timeout: "60s"}
|
|
31
|
+
* });
|
|
32
|
+
* ```
|
|
33
|
+
*/
|
|
34
|
+
export declare function connect(uri: string, opts?: Partial<ConnectionOptions | RemoteConnectionOptions>): Promise<Connection>;
|
|
35
|
+
/**
|
|
36
|
+
* Connect to a LanceDB instance at the given URI.
|
|
37
|
+
*
|
|
38
|
+
* Accepted formats:
|
|
39
|
+
*
|
|
40
|
+
* - `/path/to/database` - local database
|
|
41
|
+
* - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
|
|
42
|
+
* - `db://host:port` - remote database (LanceDB cloud)
|
|
43
|
+
* @param options - The options to use when connecting to the database
|
|
44
|
+
* @see {@link ConnectionOptions} for more details on the URI format.
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* const conn = await connect({
|
|
48
|
+
* uri: "/path/to/database",
|
|
49
|
+
* storageOptions: {timeout: "60s"}
|
|
50
|
+
* });
|
|
51
|
+
* ```
|
|
52
|
+
*/
|
|
53
|
+
export declare function connect(opts: Partial<RemoteConnectionOptions | ConnectionOptions> & {
|
|
54
|
+
uri: string;
|
|
55
|
+
}): Promise<Connection>;
|