@mastra/lance 0.1.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +23 -0
- package/CHANGELOG.md +13 -0
- package/LICENSE.md +46 -0
- package/README.md +263 -0
- package/dist/_tsup-dts-rollup.d.cts +369 -0
- package/dist/_tsup-dts-rollup.d.ts +369 -0
- package/dist/index.cjs +1564 -0
- package/dist/index.d.cts +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1561 -0
- package/eslint.config.js +6 -0
- package/package.json +45 -0
- package/src/index.ts +2 -0
- package/src/storage/index.test.ts +1269 -0
- package/src/storage/index.ts +999 -0
- package/src/vector/filter.test.ts +295 -0
- package/src/vector/filter.ts +423 -0
- package/src/vector/index.test.ts +1493 -0
- package/src/vector/index.ts +700 -0
- package/src/vector/types.ts +16 -0
- package/tsconfig.json +5 -0
- package/vitest.config.ts +11 -0
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import { ArrayOperator } from '@mastra/core/vector/filter';
|
|
2
|
+
import { BaseFilterTranslator } from '@mastra/core/vector/filter';
|
|
3
|
+
import { BasicOperator } from '@mastra/core/vector/filter';
|
|
4
|
+
import type { ConnectionOptions } from '@lancedb/lancedb';
|
|
5
|
+
import type { CreateIndexParams } from '@mastra/core';
|
|
6
|
+
import type { CreateTableOptions } from '@lancedb/lancedb';
|
|
7
|
+
import type { DeleteIndexParams } from '@mastra/core';
|
|
8
|
+
import type { DeleteVectorParams } from '@mastra/core';
|
|
9
|
+
import type { DescribeIndexParams } from '@mastra/core';
|
|
10
|
+
import type { EvalRow } from '@mastra/core';
|
|
11
|
+
import type { IndexStats } from '@mastra/core';
|
|
12
|
+
import { LogicalOperator } from '@mastra/core/vector/filter';
|
|
13
|
+
import type { MastraMessageV1 } from '@mastra/core';
|
|
14
|
+
import type { MastraMessageV2 } from '@mastra/core';
|
|
15
|
+
import { MastraStorage } from '@mastra/core/storage';
|
|
16
|
+
import { MastraVector } from '@mastra/core/vector';
|
|
17
|
+
import { NumericOperator } from '@mastra/core/vector/filter';
|
|
18
|
+
import type { QueryResult } from '@mastra/core';
|
|
19
|
+
import type { QueryVectorParams } from '@mastra/core';
|
|
20
|
+
import { RegexOperator } from '@mastra/core/vector/filter';
|
|
21
|
+
import type { SchemaLike } from '@lancedb/lancedb';
|
|
22
|
+
import type { StorageColumn } from '@mastra/core';
|
|
23
|
+
import type { StorageGetMessagesArg } from '@mastra/core';
|
|
24
|
+
import type { StorageThreadType } from '@mastra/core';
|
|
25
|
+
import type { Table } from '@lancedb/lancedb';
|
|
26
|
+
import type { TABLE_NAMES } from '@mastra/core/storage';
|
|
27
|
+
import type { TableLike } from '@lancedb/lancedb';
|
|
28
|
+
import type { TraceType } from '@mastra/core';
|
|
29
|
+
import type { UpdateVectorParams } from '@mastra/core';
|
|
30
|
+
import type { UpsertVectorParams } from '@mastra/core';
|
|
31
|
+
import type { VectorFilter } from '@mastra/core/vector/filter';
|
|
32
|
+
import type { WorkflowRuns } from '@mastra/core';
|
|
33
|
+
import type { WorkflowRunState } from '@mastra/core';
|
|
34
|
+
|
|
35
|
+
declare interface HNSWConfig {
|
|
36
|
+
m?: number;
|
|
37
|
+
efConstruction?: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export declare interface IndexConfig {
|
|
41
|
+
type?: IndexType;
|
|
42
|
+
ivf?: IVFConfig;
|
|
43
|
+
hnsw?: HNSWConfig;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export declare type IndexType = 'ivfflat' | 'hnsw';
|
|
47
|
+
|
|
48
|
+
declare interface IVFConfig {
|
|
49
|
+
lists?: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
declare interface LanceCreateIndexParams extends CreateIndexParams {
|
|
53
|
+
indexConfig?: LanceIndexConfig;
|
|
54
|
+
tableName?: string;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export declare class LanceFilterTranslator extends BaseFilterTranslator {
|
|
58
|
+
translate(filter: VectorFilter): string;
|
|
59
|
+
private processFilter;
|
|
60
|
+
private processLogicalOperator;
|
|
61
|
+
private processNestedObject;
|
|
62
|
+
private processField;
|
|
63
|
+
private processOperators;
|
|
64
|
+
private formatValue;
|
|
65
|
+
private formatArrayValues;
|
|
66
|
+
normalizeArrayValues(array: unknown[]): unknown[];
|
|
67
|
+
normalizeComparisonValue(value: unknown): unknown;
|
|
68
|
+
private isOperatorObject;
|
|
69
|
+
private isNestedObject;
|
|
70
|
+
private isNormalNestedField;
|
|
71
|
+
private escapeFieldName;
|
|
72
|
+
private isSqlKeyword;
|
|
73
|
+
private isDateObject;
|
|
74
|
+
/**
|
|
75
|
+
* Override getSupportedOperators to add custom operators for LanceDB
|
|
76
|
+
*/
|
|
77
|
+
protected getSupportedOperators(): {
|
|
78
|
+
custom: string[];
|
|
79
|
+
logical: LogicalOperator[];
|
|
80
|
+
basic: BasicOperator[];
|
|
81
|
+
numeric: NumericOperator[];
|
|
82
|
+
array: ArrayOperator[];
|
|
83
|
+
element: "$exists"[];
|
|
84
|
+
regex: RegexOperator[];
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
declare interface LanceIndexConfig extends IndexConfig {
|
|
89
|
+
numPartitions?: number;
|
|
90
|
+
numSubVectors?: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
declare interface LanceQueryVectorParams extends QueryVectorParams {
|
|
94
|
+
tableName: string;
|
|
95
|
+
columns?: string[];
|
|
96
|
+
includeAllColumns?: boolean;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
declare class LanceStorage extends MastraStorage {
|
|
100
|
+
private lanceClient;
|
|
101
|
+
/**
|
|
102
|
+
* Creates a new instance of LanceStorage
|
|
103
|
+
* @param uri The URI to connect to LanceDB
|
|
104
|
+
* @param options connection options
|
|
105
|
+
*
|
|
106
|
+
* Usage:
|
|
107
|
+
*
|
|
108
|
+
* Connect to a local database
|
|
109
|
+
* ```ts
|
|
110
|
+
* const store = await LanceStorage.create('/path/to/db');
|
|
111
|
+
* ```
|
|
112
|
+
*
|
|
113
|
+
* Connect to a LanceDB cloud database
|
|
114
|
+
* ```ts
|
|
115
|
+
* const store = await LanceStorage.create('db://host:port');
|
|
116
|
+
* ```
|
|
117
|
+
*
|
|
118
|
+
* Connect to a cloud database
|
|
119
|
+
* ```ts
|
|
120
|
+
* const store = await LanceStorage.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
121
|
+
* ```
|
|
122
|
+
*/
|
|
123
|
+
static create(name: string, uri: string, options?: ConnectionOptions): Promise<LanceStorage>;
|
|
124
|
+
/**
|
|
125
|
+
* @internal
|
|
126
|
+
* Private constructor to enforce using the create factory method
|
|
127
|
+
*/
|
|
128
|
+
private constructor();
|
|
129
|
+
createTable({ tableName, schema, }: {
|
|
130
|
+
tableName: TABLE_NAMES;
|
|
131
|
+
schema: Record<string, StorageColumn>;
|
|
132
|
+
}): Promise<void>;
|
|
133
|
+
private translateSchema;
|
|
134
|
+
/**
|
|
135
|
+
* Drop a table if it exists
|
|
136
|
+
* @param tableName Name of the table to drop
|
|
137
|
+
*/
|
|
138
|
+
dropTable(tableName: TABLE_NAMES): Promise<void>;
|
|
139
|
+
/**
|
|
140
|
+
* Get table schema
|
|
141
|
+
* @param tableName Name of the table
|
|
142
|
+
* @returns Table schema
|
|
143
|
+
*/
|
|
144
|
+
getTableSchema(tableName: TABLE_NAMES): Promise<SchemaLike>;
|
|
145
|
+
protected getDefaultValue(type: StorageColumn['type']): string;
|
|
146
|
+
/**
|
|
147
|
+
* Alters table schema to add columns if they don't exist
|
|
148
|
+
* @param tableName Name of the table
|
|
149
|
+
* @param schema Schema of the table
|
|
150
|
+
* @param ifNotExists Array of column names to add if they don't exist
|
|
151
|
+
*/
|
|
152
|
+
alterTable({ tableName, schema, ifNotExists, }: {
|
|
153
|
+
tableName: string;
|
|
154
|
+
schema: Record<string, StorageColumn>;
|
|
155
|
+
ifNotExists: string[];
|
|
156
|
+
}): Promise<void>;
|
|
157
|
+
clearTable({ tableName }: {
|
|
158
|
+
tableName: TABLE_NAMES;
|
|
159
|
+
}): Promise<void>;
|
|
160
|
+
/**
|
|
161
|
+
* Insert a single record into a table. This function overwrites the existing record if it exists. Use this function for inserting records into tables with custom schemas.
|
|
162
|
+
* @param tableName The name of the table to insert into.
|
|
163
|
+
* @param record The record to insert.
|
|
164
|
+
*/
|
|
165
|
+
insert({ tableName, record }: {
|
|
166
|
+
tableName: string;
|
|
167
|
+
record: Record<string, any>;
|
|
168
|
+
}): Promise<void>;
|
|
169
|
+
/**
|
|
170
|
+
* Insert multiple records into a table. This function overwrites the existing records if they exist. Use this function for inserting records into tables with custom schemas.
|
|
171
|
+
* @param tableName The name of the table to insert into.
|
|
172
|
+
* @param records The records to insert.
|
|
173
|
+
*/
|
|
174
|
+
batchInsert({ tableName, records }: {
|
|
175
|
+
tableName: string;
|
|
176
|
+
records: Record<string, any>[];
|
|
177
|
+
}): Promise<void>;
|
|
178
|
+
/**
|
|
179
|
+
* Load a record from the database by its key(s)
|
|
180
|
+
* @param tableName The name of the table to query
|
|
181
|
+
* @param keys Record of key-value pairs to use for lookup
|
|
182
|
+
* @throws Error if invalid types are provided for keys
|
|
183
|
+
* @returns The loaded record with proper type conversions, or null if not found
|
|
184
|
+
*/
|
|
185
|
+
load({ tableName, keys }: {
|
|
186
|
+
tableName: TABLE_NAMES;
|
|
187
|
+
keys: Record<string, any>;
|
|
188
|
+
}): Promise<any>;
|
|
189
|
+
/**
|
|
190
|
+
* Validates that key types match the schema definition
|
|
191
|
+
* @param keys The keys to validate
|
|
192
|
+
* @param tableSchema The table schema to validate against
|
|
193
|
+
* @throws Error if a key has an incompatible type
|
|
194
|
+
*/
|
|
195
|
+
private validateKeyTypes;
|
|
196
|
+
/**
|
|
197
|
+
* Process a database result with appropriate type conversions based on the table schema
|
|
198
|
+
* @param rawResult The raw result object from the database
|
|
199
|
+
* @param tableSchema The schema of the table containing type information
|
|
200
|
+
* @returns Processed result with correct data types
|
|
201
|
+
*/
|
|
202
|
+
private processResultWithTypeConversion;
|
|
203
|
+
getThreadById({ threadId }: {
|
|
204
|
+
threadId: string;
|
|
205
|
+
}): Promise<StorageThreadType | null>;
|
|
206
|
+
getThreadsByResourceId({ resourceId }: {
|
|
207
|
+
resourceId: string;
|
|
208
|
+
}): Promise<StorageThreadType[]>;
|
|
209
|
+
/**
|
|
210
|
+
* Saves a thread to the database. This function doesn't overwrite existing threads.
|
|
211
|
+
* @param thread - The thread to save
|
|
212
|
+
* @returns The saved thread
|
|
213
|
+
*/
|
|
214
|
+
saveThread({ thread }: {
|
|
215
|
+
thread: StorageThreadType;
|
|
216
|
+
}): Promise<StorageThreadType>;
|
|
217
|
+
updateThread({ id, title, metadata, }: {
|
|
218
|
+
id: string;
|
|
219
|
+
title: string;
|
|
220
|
+
metadata: Record<string, unknown>;
|
|
221
|
+
}): Promise<StorageThreadType>;
|
|
222
|
+
deleteThread({ threadId }: {
|
|
223
|
+
threadId: string;
|
|
224
|
+
}): Promise<void>;
|
|
225
|
+
/**
|
|
226
|
+
* Processes messages to include context messages based on withPreviousMessages and withNextMessages
|
|
227
|
+
* @param records - The sorted array of records to process
|
|
228
|
+
* @param include - The array of include specifications with context parameters
|
|
229
|
+
* @returns The processed array with context messages included
|
|
230
|
+
*/
|
|
231
|
+
private processMessagesWithContext;
|
|
232
|
+
getMessages(args: StorageGetMessagesArg & {
|
|
233
|
+
format?: 'v1';
|
|
234
|
+
}): Promise<MastraMessageV1[]>;
|
|
235
|
+
getMessages(args: StorageGetMessagesArg & {
|
|
236
|
+
format: 'v2';
|
|
237
|
+
}): Promise<MastraMessageV2[]>;
|
|
238
|
+
saveMessages(args: {
|
|
239
|
+
messages: MastraMessageV1[];
|
|
240
|
+
format?: undefined | 'v1';
|
|
241
|
+
}): Promise<MastraMessageV1[]>;
|
|
242
|
+
saveMessages(args: {
|
|
243
|
+
messages: MastraMessageV2[];
|
|
244
|
+
format: 'v2';
|
|
245
|
+
}): Promise<MastraMessageV2[]>;
|
|
246
|
+
saveTrace({ trace }: {
|
|
247
|
+
trace: TraceType;
|
|
248
|
+
}): Promise<TraceType>;
|
|
249
|
+
getTraceById({ traceId }: {
|
|
250
|
+
traceId: string;
|
|
251
|
+
}): Promise<TraceType>;
|
|
252
|
+
getTraces({ name, scope, page, perPage, attributes, }: {
|
|
253
|
+
name?: string;
|
|
254
|
+
scope?: string;
|
|
255
|
+
page: number;
|
|
256
|
+
perPage: number;
|
|
257
|
+
attributes?: Record<string, string>;
|
|
258
|
+
}): Promise<TraceType[]>;
|
|
259
|
+
saveEvals({ evals }: {
|
|
260
|
+
evals: EvalRow[];
|
|
261
|
+
}): Promise<EvalRow[]>;
|
|
262
|
+
getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]>;
|
|
263
|
+
private parseWorkflowRun;
|
|
264
|
+
getWorkflowRuns(args?: {
|
|
265
|
+
namespace?: string;
|
|
266
|
+
workflowName?: string;
|
|
267
|
+
fromDate?: Date;
|
|
268
|
+
toDate?: Date;
|
|
269
|
+
limit?: number;
|
|
270
|
+
offset?: number;
|
|
271
|
+
}): Promise<WorkflowRuns>;
|
|
272
|
+
/**
|
|
273
|
+
* Retrieve a single workflow run by its runId.
|
|
274
|
+
* @param args The ID of the workflow run to retrieve
|
|
275
|
+
* @returns The workflow run object or null if not found
|
|
276
|
+
*/
|
|
277
|
+
getWorkflowRunById(args: {
|
|
278
|
+
runId: string;
|
|
279
|
+
workflowName?: string;
|
|
280
|
+
}): Promise<{
|
|
281
|
+
workflowName: string;
|
|
282
|
+
runId: string;
|
|
283
|
+
snapshot: any;
|
|
284
|
+
createdAt: Date;
|
|
285
|
+
updatedAt: Date;
|
|
286
|
+
} | null>;
|
|
287
|
+
persistWorkflowSnapshot({ workflowName, runId, snapshot, }: {
|
|
288
|
+
workflowName: string;
|
|
289
|
+
runId: string;
|
|
290
|
+
snapshot: WorkflowRunState;
|
|
291
|
+
}): Promise<void>;
|
|
292
|
+
loadWorkflowSnapshot({ workflowName, runId, }: {
|
|
293
|
+
workflowName: string;
|
|
294
|
+
runId: string;
|
|
295
|
+
}): Promise<WorkflowRunState | null>;
|
|
296
|
+
}
|
|
297
|
+
export { LanceStorage }
|
|
298
|
+
export { LanceStorage as LanceStorage_alias_1 }
|
|
299
|
+
|
|
300
|
+
declare interface LanceUpsertVectorParams extends UpsertVectorParams {
|
|
301
|
+
tableName: string;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
declare class LanceVectorStore extends MastraVector {
|
|
305
|
+
private lanceClient;
|
|
306
|
+
/**
|
|
307
|
+
* Creates a new instance of LanceVectorStore
|
|
308
|
+
* @param uri The URI to connect to LanceDB
|
|
309
|
+
* @param options connection options
|
|
310
|
+
*
|
|
311
|
+
* Usage:
|
|
312
|
+
*
|
|
313
|
+
* Connect to a local database
|
|
314
|
+
* ```ts
|
|
315
|
+
* const store = await LanceVectorStore.create('/path/to/db');
|
|
316
|
+
* ```
|
|
317
|
+
*
|
|
318
|
+
* Connect to a LanceDB cloud database
|
|
319
|
+
* ```ts
|
|
320
|
+
* const store = await LanceVectorStore.create('db://host:port');
|
|
321
|
+
* ```
|
|
322
|
+
*
|
|
323
|
+
* Connect to a cloud database
|
|
324
|
+
* ```ts
|
|
325
|
+
* const store = await LanceVectorStore.create('s3://bucket/db', { storageOptions: { timeout: '60s' } });
|
|
326
|
+
* ```
|
|
327
|
+
*/
|
|
328
|
+
static create(uri: string, options?: ConnectionOptions): Promise<LanceVectorStore>;
|
|
329
|
+
/**
|
|
330
|
+
* @internal
|
|
331
|
+
* Private constructor to enforce using the create factory method
|
|
332
|
+
*/
|
|
333
|
+
private constructor();
|
|
334
|
+
close(): void;
|
|
335
|
+
query({ tableName, queryVector, filter, includeVector, topK, columns, includeAllColumns, }: LanceQueryVectorParams): Promise<QueryResult[]>;
|
|
336
|
+
private filterTranslator;
|
|
337
|
+
upsert({ tableName, vectors, metadata, ids }: LanceUpsertVectorParams): Promise<string[]>;
|
|
338
|
+
/**
|
|
339
|
+
* Flattens a nested object, creating new keys with underscores for nested properties.
|
|
340
|
+
* Example: { metadata: { text: 'test' } } → { metadata_text: 'test' }
|
|
341
|
+
*/
|
|
342
|
+
private flattenObject;
|
|
343
|
+
createTable(tableName: string, data: Record<string, unknown>[] | TableLike, options?: Partial<CreateTableOptions>): Promise<Table>;
|
|
344
|
+
listTables(): Promise<string[]>;
|
|
345
|
+
getTableSchema(tableName: string): Promise<any>;
|
|
346
|
+
/**
|
|
347
|
+
* indexName is actually a column name in a table in lanceDB
|
|
348
|
+
*/
|
|
349
|
+
createIndex({ tableName, indexName, dimension, metric, indexConfig, }: LanceCreateIndexParams): Promise<void>;
|
|
350
|
+
listIndexes(): Promise<string[]>;
|
|
351
|
+
describeIndex({ indexName }: DescribeIndexParams): Promise<IndexStats>;
|
|
352
|
+
deleteIndex({ indexName }: DeleteIndexParams): Promise<void>;
|
|
353
|
+
/**
|
|
354
|
+
* Deletes all tables in the database
|
|
355
|
+
*/
|
|
356
|
+
deleteAllTables(): Promise<void>;
|
|
357
|
+
deleteTable(tableName: string): Promise<void>;
|
|
358
|
+
updateVector({ indexName, id, update }: UpdateVectorParams): Promise<void>;
|
|
359
|
+
deleteVector({ indexName, id }: DeleteVectorParams): Promise<void>;
|
|
360
|
+
/**
|
|
361
|
+
* Converts a flattened object with keys using underscore notation back to a nested object.
|
|
362
|
+
* Example: { name: 'test', details_text: 'test' } → { name: 'test', details: { text: 'test' } }
|
|
363
|
+
*/
|
|
364
|
+
private unflattenObject;
|
|
365
|
+
}
|
|
366
|
+
export { LanceVectorStore }
|
|
367
|
+
export { LanceVectorStore as LanceVectorStore_alias_1 }
|
|
368
|
+
|
|
369
|
+
export { }
|