agentlang 0.10.2 → 0.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -14
- package/out/api/http.d.ts +4 -0
- package/out/api/http.d.ts.map +1 -1
- package/out/api/http.js +171 -26
- package/out/api/http.js.map +1 -1
- package/out/cli/main.d.ts.map +1 -1
- package/out/cli/main.js +3 -0
- package/out/cli/main.js.map +1 -1
- package/out/extension/main.cjs +250 -250
- package/out/extension/main.cjs.map +2 -2
- package/out/language/agentlang-validator.d.ts.map +1 -1
- package/out/language/agentlang-validator.js +4 -0
- package/out/language/agentlang-validator.js.map +1 -1
- package/out/language/error-reporter.d.ts +53 -0
- package/out/language/error-reporter.d.ts.map +1 -0
- package/out/language/error-reporter.js +879 -0
- package/out/language/error-reporter.js.map +1 -0
- package/out/language/generated/ast.d.ts +51 -1
- package/out/language/generated/ast.d.ts.map +1 -1
- package/out/language/generated/ast.js +40 -0
- package/out/language/generated/ast.js.map +1 -1
- package/out/language/generated/grammar.d.ts.map +1 -1
- package/out/language/generated/grammar.js +286 -190
- package/out/language/generated/grammar.js.map +1 -1
- package/out/language/main.cjs +828 -694
- package/out/language/main.cjs.map +3 -3
- package/out/language/parser.d.ts +4 -2
- package/out/language/parser.d.ts.map +1 -1
- package/out/language/parser.js +30 -97
- package/out/language/parser.js.map +1 -1
- package/out/language/syntax.d.ts +2 -0
- package/out/language/syntax.d.ts.map +1 -1
- package/out/language/syntax.js +6 -0
- package/out/language/syntax.js.map +1 -1
- package/out/runtime/api.d.ts.map +1 -1
- package/out/runtime/api.js +22 -0
- package/out/runtime/api.js.map +1 -1
- package/out/runtime/defs.d.ts +1 -0
- package/out/runtime/defs.d.ts.map +1 -1
- package/out/runtime/defs.js +2 -1
- package/out/runtime/defs.js.map +1 -1
- package/out/runtime/document-retriever.d.ts +24 -0
- package/out/runtime/document-retriever.d.ts.map +1 -0
- package/out/runtime/document-retriever.js +258 -0
- package/out/runtime/document-retriever.js.map +1 -0
- package/out/runtime/embeddings/chunker.d.ts +18 -0
- package/out/runtime/embeddings/chunker.d.ts.map +1 -1
- package/out/runtime/embeddings/chunker.js +47 -15
- package/out/runtime/embeddings/chunker.js.map +1 -1
- package/out/runtime/embeddings/openai.d.ts.map +1 -1
- package/out/runtime/embeddings/openai.js +22 -9
- package/out/runtime/embeddings/openai.js.map +1 -1
- package/out/runtime/embeddings/provider.d.ts +1 -0
- package/out/runtime/embeddings/provider.d.ts.map +1 -1
- package/out/runtime/embeddings/provider.js +20 -1
- package/out/runtime/embeddings/provider.js.map +1 -1
- package/out/runtime/integration-client.d.ts +21 -0
- package/out/runtime/integration-client.d.ts.map +1 -0
- package/out/runtime/integration-client.js +112 -0
- package/out/runtime/integration-client.js.map +1 -0
- package/out/runtime/integrations.d.ts.map +1 -1
- package/out/runtime/integrations.js +20 -9
- package/out/runtime/integrations.js.map +1 -1
- package/out/runtime/interpreter.d.ts +1 -0
- package/out/runtime/interpreter.d.ts.map +1 -1
- package/out/runtime/interpreter.js +152 -17
- package/out/runtime/interpreter.js.map +1 -1
- package/out/runtime/loader.d.ts.map +1 -1
- package/out/runtime/loader.js +70 -7
- package/out/runtime/loader.js.map +1 -1
- package/out/runtime/logger.d.ts.map +1 -1
- package/out/runtime/logger.js +8 -1
- package/out/runtime/logger.js.map +1 -1
- package/out/runtime/module.d.ts +10 -0
- package/out/runtime/module.d.ts.map +1 -1
- package/out/runtime/module.js +68 -3
- package/out/runtime/module.js.map +1 -1
- package/out/runtime/modules/ai.d.ts +9 -2
- package/out/runtime/modules/ai.d.ts.map +1 -1
- package/out/runtime/modules/ai.js +219 -67
- package/out/runtime/modules/ai.js.map +1 -1
- package/out/runtime/resolvers/interface.d.ts +4 -0
- package/out/runtime/resolvers/interface.d.ts.map +1 -1
- package/out/runtime/resolvers/interface.js +14 -1
- package/out/runtime/resolvers/interface.js.map +1 -1
- package/out/runtime/resolvers/sqldb/database.d.ts +2 -0
- package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/database.js +142 -126
- package/out/runtime/resolvers/sqldb/database.js.map +1 -1
- package/out/runtime/resolvers/sqldb/dbutil.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/dbutil.js +8 -0
- package/out/runtime/resolvers/sqldb/dbutil.js.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts +1 -0
- package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.js +7 -0
- package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
- package/out/runtime/resolvers/vector/lancedb-store.d.ts +16 -0
- package/out/runtime/resolvers/vector/lancedb-store.d.ts.map +1 -0
- package/out/runtime/resolvers/vector/lancedb-store.js +159 -0
- package/out/runtime/resolvers/vector/lancedb-store.js.map +1 -0
- package/out/runtime/resolvers/vector/types.d.ts +32 -0
- package/out/runtime/resolvers/vector/types.d.ts.map +1 -0
- package/out/runtime/resolvers/vector/types.js +2 -0
- package/out/runtime/resolvers/vector/types.js.map +1 -0
- package/out/runtime/services/documentFetcher.d.ts.map +1 -1
- package/out/runtime/services/documentFetcher.js +21 -6
- package/out/runtime/services/documentFetcher.js.map +1 -1
- package/out/runtime/state.d.ts +19 -1
- package/out/runtime/state.d.ts.map +1 -1
- package/out/runtime/state.js +36 -1
- package/out/runtime/state.js.map +1 -1
- package/out/syntaxes/agentlang.monarch.js +1 -1
- package/out/syntaxes/agentlang.monarch.js.map +1 -1
- package/package.json +19 -19
- package/src/api/http.ts +197 -37
- package/src/cli/main.ts +3 -0
- package/src/language/agentlang-validator.ts +3 -0
- package/src/language/agentlang.langium +3 -1
- package/src/language/error-reporter.ts +1028 -0
- package/src/language/generated/ast.ts +62 -0
- package/src/language/generated/grammar.ts +286 -190
- package/src/language/parser.ts +31 -100
- package/src/language/syntax.ts +8 -0
- package/src/runtime/api.ts +31 -0
- package/src/runtime/defs.ts +2 -1
- package/src/runtime/document-retriever.ts +311 -0
- package/src/runtime/embeddings/chunker.ts +52 -14
- package/src/runtime/embeddings/openai.ts +27 -9
- package/src/runtime/embeddings/provider.ts +22 -1
- package/src/runtime/integration-client.ts +158 -0
- package/src/runtime/integrations.ts +20 -11
- package/src/runtime/interpreter.ts +142 -12
- package/src/runtime/loader.ts +83 -5
- package/src/runtime/logger.ts +12 -1
- package/src/runtime/module.ts +78 -3
- package/src/runtime/modules/ai.ts +263 -76
- package/src/runtime/resolvers/interface.ts +19 -1
- package/src/runtime/resolvers/sqldb/database.ts +158 -130
- package/src/runtime/resolvers/sqldb/dbutil.ts +8 -0
- package/src/runtime/resolvers/sqldb/impl.ts +8 -0
- package/src/runtime/resolvers/vector/lancedb-store.ts +187 -0
- package/src/runtime/resolvers/vector/types.ts +39 -0
- package/src/runtime/services/documentFetcher.ts +21 -6
- package/src/runtime/state.ts +40 -1
- package/src/syntaxes/agentlang.monarch.ts +1 -1
|
@@ -46,6 +46,8 @@ import { saveMigration } from '../../modules/core.js';
|
|
|
46
46
|
import { getAppSpec } from '../../loader.js';
|
|
47
47
|
import { WhereClause } from '../interface.js';
|
|
48
48
|
import { AppConfig } from '../../state.js';
|
|
49
|
+
import { createLanceDBStore } from '../vector/lancedb-store.js';
|
|
50
|
+
import type { VectorStore } from '../vector/types.js';
|
|
49
51
|
|
|
50
52
|
export let defaultDataSource: DataSource | undefined;
|
|
51
53
|
|
|
@@ -58,57 +60,8 @@ function isBrowser(): boolean {
|
|
|
58
60
|
);
|
|
59
61
|
}
|
|
60
62
|
|
|
61
|
-
//
|
|
62
|
-
|
|
63
|
-
let sqliteVecWasmModule: any = null;
|
|
64
|
-
|
|
65
|
-
async function loadSqliteVecWasm(): Promise<any> {
|
|
66
|
-
if (sqliteVecWasmModule) {
|
|
67
|
-
return sqliteVecWasmModule;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
try {
|
|
71
|
-
// Use dynamic import with string to prevent bundlers from analyzing
|
|
72
|
-
const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
|
|
73
|
-
const module = await import(/* @vite-ignore */ cdnUrl);
|
|
74
|
-
sqliteVecWasmModule = await module.default();
|
|
75
|
-
return sqliteVecWasmModule;
|
|
76
|
-
} catch (err) {
|
|
77
|
-
logger.warn('Failed to load sqlite-vec WASM:', err);
|
|
78
|
-
return null;
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Helper to load sqlite-vec based on environment
|
|
83
|
-
async function loadSqliteVec(): Promise<any> {
|
|
84
|
-
// In browser, use WASM version with built-in sqlite-vec
|
|
85
|
-
// In Node.js, use the npm package
|
|
86
|
-
if (isBrowser()) {
|
|
87
|
-
const wasmModule = await loadSqliteVecWasm();
|
|
88
|
-
if (wasmModule) {
|
|
89
|
-
// WASM version has sqlite-vec built-in, no need to call load()
|
|
90
|
-
// Return a compatible interface
|
|
91
|
-
return {
|
|
92
|
-
load: () => {
|
|
93
|
-
// No-op: sqlite-vec is already loaded in WASM version
|
|
94
|
-
logger.info('sqlite-vec WASM loaded (built-in)');
|
|
95
|
-
},
|
|
96
|
-
// Expose the WASM module for direct use if needed
|
|
97
|
-
_wasmModule: wasmModule,
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
return null;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// Node.js: use npm package
|
|
104
|
-
try {
|
|
105
|
-
// Use variable to prevent bundlers from statically analyzing this import
|
|
106
|
-
const moduleName = 'sqlite-vec';
|
|
107
|
-
return await import(/* @vite-ignore */ moduleName);
|
|
108
|
-
} catch {
|
|
109
|
-
return null;
|
|
110
|
-
}
|
|
111
|
-
}
|
|
63
|
+
// LanceDB vector store cache - keyed by module name
|
|
64
|
+
const lanceDBStores: Map<string, VectorStore> = new Map();
|
|
112
65
|
|
|
113
66
|
export class DbContext {
|
|
114
67
|
txnId: string | undefined;
|
|
@@ -306,17 +259,27 @@ function makeSqliteDataSource(
|
|
|
306
259
|
ds.initialize = async () => {
|
|
307
260
|
const res = await originalInit();
|
|
308
261
|
try {
|
|
309
|
-
const sqliteVec = await loadSqliteVec();
|
|
310
262
|
const driver = ds.driver as any;
|
|
311
263
|
const db = driver.databaseConnection || driver.nativeDatabase;
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
264
|
+
// Enable WAL mode and additional pragmas for better write performance
|
|
265
|
+
if (db?.pragma) {
|
|
266
|
+
db.pragma('journal_mode = WAL');
|
|
267
|
+
|
|
268
|
+
const syncMode = process.env.SQLITE_SYNC_MODE || 'NORMAL';
|
|
269
|
+
const busyTimeout = process.env.SQLITE_BUSY_TIMEOUT || '5000';
|
|
270
|
+
const cacheSize = process.env.SQLITE_CACHE_SIZE || '-20000';
|
|
271
|
+
|
|
272
|
+
db.pragma(`synchronous = ${syncMode}`);
|
|
273
|
+
db.pragma(`busy_timeout = ${busyTimeout}`);
|
|
274
|
+
db.pragma(`cache_size = ${cacheSize}`);
|
|
275
|
+
db.pragma('temp_store = MEMORY');
|
|
276
|
+
|
|
277
|
+
logger.info(
|
|
278
|
+
`SQLite pragmas enabled: WAL mode, synchronous=${syncMode}, busy_timeout=${busyTimeout}, cache_size=${cacheSize}, temp_store=MEMORY`
|
|
279
|
+
);
|
|
315
280
|
}
|
|
316
281
|
} catch (err: any) {
|
|
317
|
-
logger.warn(
|
|
318
|
-
`Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
|
|
319
|
-
);
|
|
282
|
+
logger.warn(`Failed to enable SQLite pragmas: ${err.message}.`);
|
|
320
283
|
}
|
|
321
284
|
return res;
|
|
322
285
|
};
|
|
@@ -412,6 +375,20 @@ function forceGetDbType(config: DatabaseConfig | undefined): string {
|
|
|
412
375
|
|
|
413
376
|
let DbType: string | undefined;
|
|
414
377
|
|
|
378
|
+
function getVectorStoreType(): string {
|
|
379
|
+
// Check explicit vectorStore config first
|
|
380
|
+
const vectorStoreConfig = AppConfig?.vectorStore;
|
|
381
|
+
if (vectorStoreConfig?.type) {
|
|
382
|
+
if (vectorStoreConfig.type === 'pgvector') return 'postgres';
|
|
383
|
+
if (vectorStoreConfig.type === 'lancedb') return 'lancedb';
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Fallback to main store type
|
|
387
|
+
const dbType = getDbType(AppConfig?.store);
|
|
388
|
+
if (dbType === 'postgres') return 'postgres';
|
|
389
|
+
return 'lancedb';
|
|
390
|
+
}
|
|
391
|
+
|
|
415
392
|
function getDbType(config: DatabaseConfig | undefined): string {
|
|
416
393
|
if (DbType === undefined) DbType = forceGetDbType(config);
|
|
417
394
|
return DbType;
|
|
@@ -445,12 +422,9 @@ export function isUsingSqljs(): boolean {
|
|
|
445
422
|
}
|
|
446
423
|
|
|
447
424
|
export async function isVectorStoreSupported(): Promise<boolean> {
|
|
448
|
-
const
|
|
449
|
-
if (
|
|
450
|
-
if (
|
|
451
|
-
const sqliteVecModule = await loadSqliteVec();
|
|
452
|
-
return !!sqliteVecModule;
|
|
453
|
-
}
|
|
425
|
+
const vectorStoreType = getVectorStoreType();
|
|
426
|
+
if (vectorStoreType === 'postgres') return true;
|
|
427
|
+
if (vectorStoreType === 'lancedb') return true;
|
|
454
428
|
return false;
|
|
455
429
|
}
|
|
456
430
|
|
|
@@ -509,9 +483,14 @@ async function insertRowsHelper(
|
|
|
509
483
|
ctx: DbContext,
|
|
510
484
|
doUpsert: boolean
|
|
511
485
|
): Promise<void> {
|
|
512
|
-
const
|
|
513
|
-
|
|
514
|
-
|
|
486
|
+
const ds = getDatasourceForTransaction(ctx.txnId);
|
|
487
|
+
const repo = ds.getRepository(tableName);
|
|
488
|
+
|
|
489
|
+
if (doUpsert) {
|
|
490
|
+
await repo.save(rows);
|
|
491
|
+
} else {
|
|
492
|
+
await repo.insert(rows);
|
|
493
|
+
}
|
|
515
494
|
}
|
|
516
495
|
|
|
517
496
|
export async function addRowForFullTextSearch(
|
|
@@ -520,28 +499,45 @@ export async function addRowForFullTextSearch(
|
|
|
520
499
|
vect: number[],
|
|
521
500
|
ctx: DbContext
|
|
522
501
|
) {
|
|
523
|
-
if (!(await isVectorStoreSupported()))
|
|
502
|
+
if (!(await isVectorStoreSupported())) {
|
|
503
|
+
logger.warn(`[VECTOR] Vector store not supported, skipping save for ${id}`);
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
524
506
|
try {
|
|
525
507
|
const vecTableName = tableName + VectorSuffix;
|
|
526
|
-
|
|
508
|
+
logger.info(
|
|
509
|
+
`[VECTOR] Saving embedding to ${vecTableName} for ${id} (${vect.length} dimensions)`
|
|
510
|
+
);
|
|
511
|
+
const dbType = getVectorStoreType();
|
|
527
512
|
const tenantId = await ctx.getTenantId();
|
|
528
|
-
|
|
529
|
-
if (dbType === '
|
|
513
|
+
|
|
514
|
+
if (dbType === 'lancedb') {
|
|
515
|
+
let store = lanceDBStores.get(tableName);
|
|
516
|
+
if (!store) {
|
|
517
|
+
store = createLanceDBStore({
|
|
518
|
+
moduleName: tableName,
|
|
519
|
+
vectorDimension: vect.length,
|
|
520
|
+
});
|
|
521
|
+
await store.init();
|
|
522
|
+
lanceDBStores.set(tableName, store);
|
|
523
|
+
}
|
|
524
|
+
await store.addEmbedding({
|
|
525
|
+
id,
|
|
526
|
+
embedding: Array.from(vect),
|
|
527
|
+
tenantId,
|
|
528
|
+
});
|
|
529
|
+
} else if (dbType === 'postgres') {
|
|
530
|
+
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
530
531
|
const { default: pgvector } = await import('pgvector');
|
|
531
532
|
await qb
|
|
532
533
|
.insert()
|
|
533
534
|
.into(vecTableName)
|
|
534
|
-
.values([{ id: id, embedding: pgvector.toSql(vect),
|
|
535
|
-
.execute();
|
|
536
|
-
} else {
|
|
537
|
-
await qb
|
|
538
|
-
.insert()
|
|
539
|
-
.into(vecTableName)
|
|
540
|
-
.values([{ id: id, embedding: new Float32Array(vect) }])
|
|
535
|
+
.values([{ id: id, embedding: pgvector.toSql(vect), agentId: tenantId }])
|
|
541
536
|
.execute();
|
|
542
537
|
}
|
|
538
|
+
logger.info(`[VECTOR] Successfully saved embedding to ${vecTableName} for ${id}`);
|
|
543
539
|
} catch (err: any) {
|
|
544
|
-
logger.error(`Failed to add row to vector store - ${err}`);
|
|
540
|
+
logger.error(`[VECTOR] Failed to add row to vector store - ${err}`);
|
|
545
541
|
}
|
|
546
542
|
}
|
|
547
543
|
|
|
@@ -550,11 +546,21 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
550
546
|
logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
|
|
551
547
|
return;
|
|
552
548
|
}
|
|
553
|
-
const dbType =
|
|
549
|
+
const dbType = getVectorStoreType();
|
|
554
550
|
let notInited = true;
|
|
555
551
|
for (const vecTableName of tableNames) {
|
|
556
|
-
|
|
557
|
-
|
|
552
|
+
if (dbType === 'lancedb') {
|
|
553
|
+
if (!lanceDBStores.has(vecTableName)) {
|
|
554
|
+
const store = createLanceDBStore({
|
|
555
|
+
moduleName: vecTableName,
|
|
556
|
+
vectorDimension: DefaultVectorDimension,
|
|
557
|
+
});
|
|
558
|
+
await store.init();
|
|
559
|
+
lanceDBStores.set(vecTableName, store);
|
|
560
|
+
logger.info(`[VECTOR] Initialized LanceDB store for ${vecTableName}`);
|
|
561
|
+
}
|
|
562
|
+
} else if (dbType === 'postgres') {
|
|
563
|
+
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
558
564
|
if (notInited) {
|
|
559
565
|
let failure = false;
|
|
560
566
|
try {
|
|
@@ -568,18 +574,11 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
568
574
|
}
|
|
569
575
|
await vecRepo.query(
|
|
570
576
|
`CREATE TABLE IF NOT EXISTS ${vecTableName} (
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
);
|
|
577
|
-
} else {
|
|
578
|
-
// sqlite-vec - vec0 doesn't support type declarations for metadata columns
|
|
579
|
-
await vecRepo.query(
|
|
580
|
-
`CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
|
|
581
|
-
id TEXT PRIMARY KEY,
|
|
582
|
-
embedding FLOAT[${DefaultVectorDimension}])`
|
|
577
|
+
id varchar PRIMARY KEY,
|
|
578
|
+
embedding vector(${DefaultVectorDimension}),
|
|
579
|
+
${TenantAttributeName} varchar,
|
|
580
|
+
__is_deleted__ boolean default false
|
|
581
|
+
)`
|
|
583
582
|
);
|
|
584
583
|
}
|
|
585
584
|
}
|
|
@@ -596,6 +595,21 @@ export async function vectorStoreSearch(
|
|
|
596
595
|
return [];
|
|
597
596
|
}
|
|
598
597
|
try {
|
|
598
|
+
const dbType = getVectorStoreType();
|
|
599
|
+
const tenantId = await ctx.getTenantId();
|
|
600
|
+
|
|
601
|
+
if (dbType === 'lancedb') {
|
|
602
|
+
const store = lanceDBStores.get(tableName);
|
|
603
|
+
if (!store) {
|
|
604
|
+
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
605
|
+
return [];
|
|
606
|
+
}
|
|
607
|
+
// Extract agentId from resourceFqName for agent-level filtering
|
|
608
|
+
const agentId = ctx.resourceFqName || undefined;
|
|
609
|
+
const results = await store.search(searchVec, tenantId, agentId, limit);
|
|
610
|
+
return results.map(r => ({ id: r.id }));
|
|
611
|
+
}
|
|
612
|
+
|
|
599
613
|
let hasGlobalPerms = ctx.isPermitted();
|
|
600
614
|
if (!hasGlobalPerms) {
|
|
601
615
|
const userId = ctx.getUserId();
|
|
@@ -605,8 +619,6 @@ export async function vectorStoreSearch(
|
|
|
605
619
|
}
|
|
606
620
|
const vecTableName = tableName + VectorSuffix;
|
|
607
621
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
608
|
-
const dbType = getDbType(AppConfig?.store);
|
|
609
|
-
const tenantId = await ctx.getTenantId();
|
|
610
622
|
let ownersJoinCond: string = '';
|
|
611
623
|
if (!hasGlobalPerms) {
|
|
612
624
|
const ot = ownersTable(tableName);
|
|
@@ -619,16 +631,6 @@ export async function vectorStoreSearch(
|
|
|
619
631
|
const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
|
|
620
632
|
const args = pgvector.toSql(searchVec);
|
|
621
633
|
return await qb.query(sql, [args]);
|
|
622
|
-
} else {
|
|
623
|
-
// sqlite-vec - join with main table to filter by tenant
|
|
624
|
-
const alias = tableName.toLowerCase();
|
|
625
|
-
const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
626
|
-
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
627
|
-
${ownersJoinCond}
|
|
628
|
-
WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
|
|
629
|
-
LIMIT ${limit}`;
|
|
630
|
-
const args = new Float32Array(searchVec);
|
|
631
|
-
return await qb.query(sql, [args]);
|
|
632
634
|
}
|
|
633
635
|
} catch (err: any) {
|
|
634
636
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -643,9 +645,19 @@ export async function vectorStoreSearchEntryExists(
|
|
|
643
645
|
): Promise<boolean> {
|
|
644
646
|
if (!(await isVectorStoreSupported())) return false;
|
|
645
647
|
try {
|
|
648
|
+
const dbType = getVectorStoreType();
|
|
649
|
+
|
|
650
|
+
if (dbType === 'lancedb') {
|
|
651
|
+
const store = lanceDBStores.get(tableName);
|
|
652
|
+
if (!store) {
|
|
653
|
+
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
return await store.exists(id);
|
|
657
|
+
}
|
|
658
|
+
|
|
646
659
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
647
660
|
const vecTableName = tableName + VectorSuffix;
|
|
648
|
-
const dbType = getDbType(AppConfig?.store);
|
|
649
661
|
const tenantId = await ctx.getTenantId();
|
|
650
662
|
|
|
651
663
|
if (dbType === 'postgres') {
|
|
@@ -654,16 +666,6 @@ export async function vectorStoreSearchEntryExists(
|
|
|
654
666
|
[id]
|
|
655
667
|
);
|
|
656
668
|
return result !== null && result.length > 0;
|
|
657
|
-
} else {
|
|
658
|
-
// sqlite-vec - join with main table to verify tenant
|
|
659
|
-
const alias = tableName.toLowerCase();
|
|
660
|
-
const result: any[] = await qb.query(
|
|
661
|
-
`SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
662
|
-
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
663
|
-
WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
|
|
664
|
-
[id]
|
|
665
|
-
);
|
|
666
|
-
return result !== null && result.length > 0;
|
|
667
669
|
}
|
|
668
670
|
} catch (err: any) {
|
|
669
671
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -674,9 +676,20 @@ export async function vectorStoreSearchEntryExists(
|
|
|
674
676
|
export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
|
|
675
677
|
if (!(await isVectorStoreSupported())) return;
|
|
676
678
|
try {
|
|
679
|
+
const dbType = getVectorStoreType();
|
|
680
|
+
|
|
681
|
+
if (dbType === 'lancedb') {
|
|
682
|
+
const store = lanceDBStores.get(tableName);
|
|
683
|
+
if (!store) {
|
|
684
|
+
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
await store.delete(id);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
|
|
677
691
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
678
692
|
const vecTableName = tableName + VectorSuffix;
|
|
679
|
-
const dbType = getDbType(AppConfig?.store);
|
|
680
693
|
const tenantId = await ctx.getTenantId();
|
|
681
694
|
|
|
682
695
|
if (dbType === 'postgres') {
|
|
@@ -684,9 +697,6 @@ export async function deleteFullTextSearchEntry(tableName: string, id: string, c
|
|
|
684
697
|
`delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
|
|
685
698
|
[id]
|
|
686
699
|
);
|
|
687
|
-
} else {
|
|
688
|
-
// sqlite-vec - delete just by id (ownership verified by caller)
|
|
689
|
-
await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
|
|
690
700
|
}
|
|
691
701
|
} catch (err: any) {
|
|
692
702
|
logger.error(`Vector store delete failed - ${err}`);
|
|
@@ -862,7 +872,7 @@ async function createLimitedOwnership(
|
|
|
862
872
|
r: perms.has(RbacPermissionFlag.READ),
|
|
863
873
|
d: perms.has(RbacPermissionFlag.DELETE),
|
|
864
874
|
u: perms.has(RbacPermissionFlag.UPDATE),
|
|
865
|
-
|
|
875
|
+
agentId: tenantId,
|
|
866
876
|
});
|
|
867
877
|
});
|
|
868
878
|
const tname = ownersTable(tableName);
|
|
@@ -1055,6 +1065,8 @@ export type QuerySpec = {
|
|
|
1055
1065
|
intoSpec: Map<string, string> | undefined;
|
|
1056
1066
|
whereClauses: WhereClause[] | undefined;
|
|
1057
1067
|
distinct: boolean;
|
|
1068
|
+
limit: number | undefined;
|
|
1069
|
+
offset: number | undefined;
|
|
1058
1070
|
};
|
|
1059
1071
|
|
|
1060
1072
|
export function makeSimpleQuerySpec(queryObj: object, queryVals: object): QuerySpec {
|
|
@@ -1069,6 +1081,8 @@ export function makeSimpleQuerySpec(queryObj: object, queryVals: object): QueryS
|
|
|
1069
1081
|
intoSpec: undefined,
|
|
1070
1082
|
whereClauses: undefined,
|
|
1071
1083
|
distinct: false,
|
|
1084
|
+
limit: undefined,
|
|
1085
|
+
offset: undefined,
|
|
1072
1086
|
};
|
|
1073
1087
|
}
|
|
1074
1088
|
|
|
@@ -1144,6 +1158,12 @@ export async function getMany(
|
|
|
1144
1158
|
if (querySpec.distinct) {
|
|
1145
1159
|
qb.distinct(true);
|
|
1146
1160
|
}
|
|
1161
|
+
if (querySpec.limit !== undefined) {
|
|
1162
|
+
qb.take(querySpec.limit);
|
|
1163
|
+
}
|
|
1164
|
+
if (querySpec.offset !== undefined) {
|
|
1165
|
+
qb.skip(querySpec.offset);
|
|
1166
|
+
}
|
|
1147
1167
|
qb.where(queryStr, querySpec.queryVals);
|
|
1148
1168
|
if (hasAggregates) return await qb.getRawMany();
|
|
1149
1169
|
else return await qb.getMany();
|
|
@@ -1233,6 +1253,12 @@ export async function getManyByJoin(
|
|
|
1233
1253
|
if (querySpec.orderBy !== undefined) {
|
|
1234
1254
|
sql = `${sql} ORDER BY ${querySpec.orderBy.join(', ')} ${querySpec.orderByDesc}`;
|
|
1235
1255
|
}
|
|
1256
|
+
if (querySpec.limit !== undefined) {
|
|
1257
|
+
sql = `${sql} LIMIT ${querySpec.limit}`;
|
|
1258
|
+
}
|
|
1259
|
+
if (querySpec.offset !== undefined) {
|
|
1260
|
+
sql = `${sql} OFFSET ${querySpec.offset}`;
|
|
1261
|
+
}
|
|
1236
1262
|
logger.debug(`Join Query: ${sql}`);
|
|
1237
1263
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
1238
1264
|
return await qb.query(sql);
|
|
@@ -1348,14 +1374,16 @@ async function endTransaction(txnId: string, commit: boolean): Promise<void> {
|
|
|
1348
1374
|
const qr: QueryRunner | undefined = transactionsDb.get(txnId);
|
|
1349
1375
|
if (qr && qr.isTransactionActive) {
|
|
1350
1376
|
try {
|
|
1351
|
-
if (commit)
|
|
1352
|
-
await qr.commitTransaction()
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
}
|
|
1377
|
+
if (commit) {
|
|
1378
|
+
await qr.commitTransaction();
|
|
1379
|
+
} else {
|
|
1380
|
+
await qr.rollbackTransaction();
|
|
1381
|
+
}
|
|
1382
|
+
} catch (err: any) {
|
|
1383
|
+
logger.error(
|
|
1384
|
+
`Failed to ${commit ? 'commit' : 'rollback'} transaction ${txnId}: ${err.message}`
|
|
1385
|
+
);
|
|
1386
|
+
throw err;
|
|
1359
1387
|
} finally {
|
|
1360
1388
|
await qr.release();
|
|
1361
1389
|
transactionsDb.delete(txnId);
|
|
@@ -99,6 +99,12 @@ export type OrmSchema = {
|
|
|
99
99
|
fkSpecs: FkSpec[];
|
|
100
100
|
};
|
|
101
101
|
|
|
102
|
+
const SqlDefaultFunctions = new Set(['uuid()', 'autoincrement()', 'now()']);
|
|
103
|
+
|
|
104
|
+
function isSqlDefault(d: any): boolean {
|
|
105
|
+
return typeof d !== 'string' || !d.endsWith('()') || SqlDefaultFunctions.has(d);
|
|
106
|
+
}
|
|
107
|
+
|
|
102
108
|
export function modulesAsOrmSchema(): OrmSchema {
|
|
103
109
|
const ents: EntitySchema[] = [];
|
|
104
110
|
const vects: EntitySchema[] = [];
|
|
@@ -147,6 +153,7 @@ function ormSchemaFromRecordSchema(
|
|
|
147
153
|
const autoUuid: boolean = d && d == 'uuid()' ? true : false;
|
|
148
154
|
const autoIncr: boolean = !autoUuid && d && d == 'autoincrement()' ? true : false;
|
|
149
155
|
if (autoUuid || autoIncr) d = undefined;
|
|
156
|
+
else if (!isSqlDefault(d)) d = undefined;
|
|
150
157
|
let genStrat: 'uuid' | 'increment' | undefined = undefined;
|
|
151
158
|
if (autoIncr) genStrat = 'increment';
|
|
152
159
|
else if (autoUuid) genStrat = 'uuid';
|
|
@@ -273,6 +280,7 @@ function entitySchemaToTable(scm: RecordSchema): TableSpec {
|
|
|
273
280
|
const autoUuid: boolean = d && d == 'uuid()' ? true : false;
|
|
274
281
|
const autoIncr: boolean = !autoUuid && d && d == 'autoincrement()' ? true : false;
|
|
275
282
|
if (autoUuid || autoIncr) d = undefined;
|
|
283
|
+
else if (!isSqlDefault(d)) d = undefined;
|
|
276
284
|
let genStrat: 'uuid' | 'increment' | 'rowid' | 'identity' = 'identity';
|
|
277
285
|
if (autoIncr) genStrat = 'increment';
|
|
278
286
|
else if (autoUuid) genStrat = 'uuid';
|
|
@@ -101,6 +101,10 @@ export class EmbeddingService {
|
|
|
101
101
|
return await this.provider.embedText(query);
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
async embedTexts(texts: string[]): Promise<number[][]> {
|
|
105
|
+
return await this.provider.embedTexts(texts);
|
|
106
|
+
}
|
|
107
|
+
|
|
104
108
|
private averageEmbeddings(embeddings: number[][]): number[] {
|
|
105
109
|
if (embeddings.length === 0) return [];
|
|
106
110
|
const dimension = embeddings[0].length;
|
|
@@ -402,6 +406,8 @@ export class SqlDbResolver extends Resolver {
|
|
|
402
406
|
joinClauses: undefined,
|
|
403
407
|
intoSpec: undefined,
|
|
404
408
|
whereClauses: undefined,
|
|
409
|
+
limit: inst.limit,
|
|
410
|
+
offset: inst.offset,
|
|
405
411
|
};
|
|
406
412
|
const readOnlyAttrs = inst.record.getWriteOnlyAttributes();
|
|
407
413
|
const rslt: any =
|
|
@@ -567,6 +573,8 @@ export class SqlDbResolver extends Resolver {
|
|
|
567
573
|
joinClauses,
|
|
568
574
|
whereClauses,
|
|
569
575
|
intoSpec,
|
|
576
|
+
limit: inst.limit,
|
|
577
|
+
offset: inst.offset,
|
|
570
578
|
};
|
|
571
579
|
const rslt: any = await getManyByJoin(tableName, qspec, this.getDbContext(inst.getFqName()));
|
|
572
580
|
return rslt;
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import * as lancedb from '@lancedb/lancedb';
|
|
2
|
+
import { Schema, Field, Float32, Utf8, FixedSizeList } from 'apache-arrow';
|
|
3
|
+
import { logger } from '../../logger.js';
|
|
4
|
+
import { VectorStore, VectorRecord, SearchResult, VectorStoreConfig } from './types.js';
|
|
5
|
+
|
|
6
|
+
export class LanceDBVectorStore implements VectorStore {
|
|
7
|
+
private db: lancedb.Connection | null = null;
|
|
8
|
+
private table: lancedb.Table | null = null;
|
|
9
|
+
private config: VectorStoreConfig;
|
|
10
|
+
|
|
11
|
+
constructor(config: VectorStoreConfig) {
|
|
12
|
+
this.config = config;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async init(): Promise<void> {
|
|
16
|
+
try {
|
|
17
|
+
const dbPath = this.config.inMemory
|
|
18
|
+
? 'memory://'
|
|
19
|
+
: this.config.dbname || `./data/vector-store/${this.config.moduleName}.lance`;
|
|
20
|
+
|
|
21
|
+
this.db = await lancedb.connect(dbPath);
|
|
22
|
+
|
|
23
|
+
const tableName = 'embeddings';
|
|
24
|
+
const tableNames = await this.db.tableNames();
|
|
25
|
+
|
|
26
|
+
if (tableNames.includes(tableName)) {
|
|
27
|
+
this.table = await this.db.openTable(tableName);
|
|
28
|
+
logger.info(`LanceDB table ${tableName} opened`);
|
|
29
|
+
} else {
|
|
30
|
+
const schema = new Schema([
|
|
31
|
+
new Field('id', new Utf8(), false),
|
|
32
|
+
new Field(
|
|
33
|
+
'embedding',
|
|
34
|
+
new FixedSizeList(this.config.vectorDimension, new Field('item', new Float32())),
|
|
35
|
+
false
|
|
36
|
+
),
|
|
37
|
+
new Field('tenantId', new Utf8(), true),
|
|
38
|
+
new Field('agentId', new Utf8(), true),
|
|
39
|
+
new Field('documentId', new Utf8(), true),
|
|
40
|
+
]);
|
|
41
|
+
|
|
42
|
+
this.table = await this.db.createEmptyTable(tableName, schema);
|
|
43
|
+
logger.info(`LanceDB table ${tableName} created`);
|
|
44
|
+
}
|
|
45
|
+
} catch (error) {
|
|
46
|
+
logger.error('Failed to initialize LanceDB vector store:', error);
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async addEmbedding(record: VectorRecord): Promise<void> {
|
|
52
|
+
if (!this.table) {
|
|
53
|
+
throw new Error('Vector store not initialized. Call init() first.');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
await this.table.add([
|
|
58
|
+
{
|
|
59
|
+
id: record.id,
|
|
60
|
+
embedding: record.embedding,
|
|
61
|
+
tenantId: record.tenantId || null,
|
|
62
|
+
agentId: record.agentId || null,
|
|
63
|
+
documentId: record.documentId || null,
|
|
64
|
+
},
|
|
65
|
+
]);
|
|
66
|
+
} catch (error) {
|
|
67
|
+
logger.error(`Failed to add embedding ${record.id}:`, error);
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async addEmbeddings(records: VectorRecord[]): Promise<void> {
|
|
73
|
+
if (!this.table) {
|
|
74
|
+
throw new Error('Vector store not initialized. Call init() first.');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
try {
|
|
78
|
+
const data = records.map(record => ({
|
|
79
|
+
id: record.id,
|
|
80
|
+
embedding: record.embedding,
|
|
81
|
+
tenantId: record.tenantId || null,
|
|
82
|
+
agentId: record.agentId || null,
|
|
83
|
+
documentId: record.documentId || null,
|
|
84
|
+
}));
|
|
85
|
+
|
|
86
|
+
await this.table.add(data);
|
|
87
|
+
} catch (error) {
|
|
88
|
+
logger.error(`Failed to add ${records.length} embeddings:`, error);
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async search(
|
|
94
|
+
embedding: number[],
|
|
95
|
+
tenantId?: string,
|
|
96
|
+
agentId?: string,
|
|
97
|
+
limit: number = 10
|
|
98
|
+
): Promise<SearchResult[]> {
|
|
99
|
+
if (!this.table) {
|
|
100
|
+
throw new Error('Vector store not initialized. Call init() first.');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
let query = this.table.vectorSearch(embedding).limit(limit);
|
|
105
|
+
|
|
106
|
+
// Build filter conditions for agent-level isolation
|
|
107
|
+
const filters: string[] = [];
|
|
108
|
+
|
|
109
|
+
if (tenantId) {
|
|
110
|
+
// Use parameterized filtering to prevent SQL injection
|
|
111
|
+
const escapedTenantId = tenantId.replace(/'/g, "''");
|
|
112
|
+
filters.push(`tenantId = '${escapedTenantId}'`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (agentId) {
|
|
116
|
+
// Add agent-level filtering for strict agent isolation
|
|
117
|
+
const escapedAgentId = agentId.replace(/'/g, "''");
|
|
118
|
+
filters.push(`agentId = '${escapedAgentId}'`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (filters.length > 0) {
|
|
122
|
+
query = query.where(filters.join(' AND '));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const results = await query.toArray();
|
|
126
|
+
|
|
127
|
+
return results.map((row: any) => ({
|
|
128
|
+
id: row.id,
|
|
129
|
+
distance: row._distance || 0,
|
|
130
|
+
tenantId: row.tenantId,
|
|
131
|
+
agentId: row.agentId,
|
|
132
|
+
documentId: row.documentId,
|
|
133
|
+
}));
|
|
134
|
+
} catch (error) {
|
|
135
|
+
logger.error('Failed to search embeddings:', error);
|
|
136
|
+
throw error;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async delete(id: string): Promise<void> {
|
|
141
|
+
if (!this.table) {
|
|
142
|
+
throw new Error('Vector store not initialized. Call init() first.');
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
await this.table.delete(`id = '${id}'`);
|
|
147
|
+
} catch (error) {
|
|
148
|
+
logger.error(`Failed to delete embedding ${id}:`, error);
|
|
149
|
+
throw error;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async exists(id: string): Promise<boolean> {
|
|
154
|
+
if (!this.table) {
|
|
155
|
+
throw new Error('Vector store not initialized. Call init() first.');
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
const results = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
|
|
160
|
+
return results.length > 0;
|
|
161
|
+
} catch (error) {
|
|
162
|
+
logger.error(`Failed to check existence of ${id}:`, error);
|
|
163
|
+
throw error;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async close(): Promise<void> {
|
|
168
|
+
try {
|
|
169
|
+
if (this.table) {
|
|
170
|
+
// LanceDB tables don't have explicit close method
|
|
171
|
+
this.table = null;
|
|
172
|
+
}
|
|
173
|
+
if (this.db) {
|
|
174
|
+
// Note: LanceDB connection doesn't have explicit close method
|
|
175
|
+
this.db = null;
|
|
176
|
+
}
|
|
177
|
+
logger.info('LanceDB vector store closed');
|
|
178
|
+
} catch (error) {
|
|
179
|
+
logger.error('Failed to close LanceDB vector store:', error);
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
export function createLanceDBStore(config: VectorStoreConfig): VectorStore {
|
|
186
|
+
return new LanceDBVectorStore(config);
|
|
187
|
+
}
|