agentlang 0.10.4 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -9
- package/out/extension/main.cjs +250 -250
- package/out/extension/main.cjs.map +2 -2
- package/out/language/generated/ast.d.ts +2 -12
- package/out/language/generated/ast.d.ts.map +1 -1
- package/out/language/generated/ast.js +0 -8
- package/out/language/generated/ast.js.map +1 -1
- package/out/language/generated/grammar.d.ts.map +1 -1
- package/out/language/generated/grammar.js +18 -52
- package/out/language/generated/grammar.js.map +1 -1
- package/out/language/main.cjs +522 -564
- package/out/language/main.cjs.map +3 -3
- package/out/language/parser.d.ts.map +1 -1
- package/out/language/parser.js +10 -3
- package/out/language/parser.js.map +1 -1
- package/out/runtime/api.d.ts.map +1 -1
- package/out/runtime/api.js +0 -14
- package/out/runtime/api.js.map +1 -1
- package/out/runtime/defs.d.ts +0 -1
- package/out/runtime/defs.d.ts.map +1 -1
- package/out/runtime/defs.js +1 -2
- package/out/runtime/defs.js.map +1 -1
- package/out/runtime/embeddings/chunker.d.ts +0 -18
- package/out/runtime/embeddings/chunker.d.ts.map +1 -1
- package/out/runtime/embeddings/chunker.js +15 -47
- package/out/runtime/embeddings/chunker.js.map +1 -1
- package/out/runtime/embeddings/openai.d.ts.map +1 -1
- package/out/runtime/embeddings/openai.js +11 -22
- package/out/runtime/embeddings/openai.js.map +1 -1
- package/out/runtime/embeddings/provider.d.ts +0 -1
- package/out/runtime/embeddings/provider.d.ts.map +1 -1
- package/out/runtime/embeddings/provider.js +1 -20
- package/out/runtime/embeddings/provider.js.map +1 -1
- package/out/runtime/exec-graph.js +5 -5
- package/out/runtime/exec-graph.js.map +1 -1
- package/out/runtime/interpreter.d.ts +4 -4
- package/out/runtime/interpreter.d.ts.map +1 -1
- package/out/runtime/interpreter.js +23 -30
- package/out/runtime/interpreter.js.map +1 -1
- package/out/runtime/loader.d.ts.map +1 -1
- package/out/runtime/loader.js +6 -2
- package/out/runtime/loader.js.map +1 -1
- package/out/runtime/logger.d.ts.map +1 -1
- package/out/runtime/logger.js +1 -8
- package/out/runtime/logger.js.map +1 -1
- package/out/runtime/module.d.ts +0 -6
- package/out/runtime/module.d.ts.map +1 -1
- package/out/runtime/module.js +1 -58
- package/out/runtime/module.js.map +1 -1
- package/out/runtime/modules/ai.d.ts +4 -4
- package/out/runtime/modules/ai.d.ts.map +1 -1
- package/out/runtime/modules/ai.js +70 -166
- package/out/runtime/modules/ai.js.map +1 -1
- package/out/runtime/modules/auth.d.ts.map +1 -1
- package/out/runtime/modules/auth.js +6 -4
- package/out/runtime/modules/auth.js.map +1 -1
- package/out/runtime/monitor.d.ts +2 -1
- package/out/runtime/monitor.d.ts.map +1 -1
- package/out/runtime/monitor.js +5 -1
- package/out/runtime/monitor.js.map +1 -1
- package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/database.js +126 -128
- package/out/runtime/resolvers/sqldb/database.js.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts +0 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.js +0 -3
- package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
- package/out/runtime/services/documentFetcher.d.ts.map +1 -1
- package/out/runtime/services/documentFetcher.js +6 -21
- package/out/runtime/services/documentFetcher.js.map +1 -1
- package/out/runtime/state.d.ts +0 -14
- package/out/runtime/state.d.ts.map +1 -1
- package/out/runtime/state.js +0 -28
- package/out/runtime/state.js.map +1 -1
- package/package.json +19 -19
- package/src/language/agentlang.langium +2 -2
- package/src/language/generated/ast.ts +2 -12
- package/src/language/generated/grammar.ts +18 -52
- package/src/language/parser.ts +9 -2
- package/src/runtime/api.ts +0 -15
- package/src/runtime/defs.ts +1 -2
- package/src/runtime/embeddings/chunker.ts +14 -52
- package/src/runtime/embeddings/openai.ts +9 -27
- package/src/runtime/embeddings/provider.ts +1 -22
- package/src/runtime/exec-graph.ts +4 -4
- package/src/runtime/interpreter.ts +24 -29
- package/src/runtime/loader.ts +10 -2
- package/src/runtime/logger.ts +1 -12
- package/src/runtime/module.ts +1 -64
- package/src/runtime/modules/ai.ts +81 -206
- package/src/runtime/modules/auth.ts +6 -4
- package/src/runtime/monitor.ts +10 -1
- package/src/runtime/resolvers/sqldb/database.ts +130 -142
- package/src/runtime/resolvers/sqldb/impl.ts +0 -4
- package/src/runtime/services/documentFetcher.ts +6 -21
- package/src/runtime/state.ts +0 -29
- package/out/runtime/document-retriever.d.ts +0 -24
- package/out/runtime/document-retriever.d.ts.map +0 -1
- package/out/runtime/document-retriever.js +0 -258
- package/out/runtime/document-retriever.js.map +0 -1
- package/out/runtime/resolvers/vector/lancedb-store.d.ts +0 -16
- package/out/runtime/resolvers/vector/lancedb-store.d.ts.map +0 -1
- package/out/runtime/resolvers/vector/lancedb-store.js +0 -159
- package/out/runtime/resolvers/vector/lancedb-store.js.map +0 -1
- package/out/runtime/resolvers/vector/types.d.ts +0 -32
- package/out/runtime/resolvers/vector/types.d.ts.map +0 -1
- package/out/runtime/resolvers/vector/types.js +0 -2
- package/out/runtime/resolvers/vector/types.js.map +0 -1
- package/src/runtime/document-retriever.ts +0 -311
- package/src/runtime/resolvers/vector/lancedb-store.ts +0 -187
- package/src/runtime/resolvers/vector/types.ts +0 -39
|
@@ -46,8 +46,6 @@ import { saveMigration } from '../../modules/core.js';
|
|
|
46
46
|
import { getAppSpec } from '../../loader.js';
|
|
47
47
|
import { WhereClause } from '../interface.js';
|
|
48
48
|
import { AppConfig } from '../../state.js';
|
|
49
|
-
import { createLanceDBStore } from '../vector/lancedb-store.js';
|
|
50
|
-
import type { VectorStore } from '../vector/types.js';
|
|
51
49
|
|
|
52
50
|
export let defaultDataSource: DataSource | undefined;
|
|
53
51
|
|
|
@@ -60,8 +58,57 @@ function isBrowser(): boolean {
|
|
|
60
58
|
);
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
//
|
|
64
|
-
|
|
61
|
+
// SQLite WASM with built-in sqlite-vec for browsers
|
|
62
|
+
// Loaded from CDN - this has sqlite-vec statically compiled in
|
|
63
|
+
let sqliteVecWasmModule: any = null;
|
|
64
|
+
|
|
65
|
+
async function loadSqliteVecWasm(): Promise<any> {
|
|
66
|
+
if (sqliteVecWasmModule) {
|
|
67
|
+
return sqliteVecWasmModule;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
// Use dynamic import with string to prevent bundlers from analyzing
|
|
72
|
+
const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
|
|
73
|
+
const module = await import(/* @vite-ignore */ cdnUrl);
|
|
74
|
+
sqliteVecWasmModule = await module.default();
|
|
75
|
+
return sqliteVecWasmModule;
|
|
76
|
+
} catch (err) {
|
|
77
|
+
logger.warn('Failed to load sqlite-vec WASM:', err);
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Helper to load sqlite-vec based on environment
|
|
83
|
+
async function loadSqliteVec(): Promise<any> {
|
|
84
|
+
// In browser, use WASM version with built-in sqlite-vec
|
|
85
|
+
// In Node.js, use the npm package
|
|
86
|
+
if (isBrowser()) {
|
|
87
|
+
const wasmModule = await loadSqliteVecWasm();
|
|
88
|
+
if (wasmModule) {
|
|
89
|
+
// WASM version has sqlite-vec built-in, no need to call load()
|
|
90
|
+
// Return a compatible interface
|
|
91
|
+
return {
|
|
92
|
+
load: () => {
|
|
93
|
+
// No-op: sqlite-vec is already loaded in WASM version
|
|
94
|
+
logger.info('sqlite-vec WASM loaded (built-in)');
|
|
95
|
+
},
|
|
96
|
+
// Expose the WASM module for direct use if needed
|
|
97
|
+
_wasmModule: wasmModule,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Node.js: use npm package
|
|
104
|
+
try {
|
|
105
|
+
// Use variable to prevent bundlers from statically analyzing this import
|
|
106
|
+
const moduleName = 'sqlite-vec';
|
|
107
|
+
return await import(/* @vite-ignore */ moduleName);
|
|
108
|
+
} catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
65
112
|
|
|
66
113
|
export class DbContext {
|
|
67
114
|
txnId: string | undefined;
|
|
@@ -259,27 +306,17 @@ function makeSqliteDataSource(
|
|
|
259
306
|
ds.initialize = async () => {
|
|
260
307
|
const res = await originalInit();
|
|
261
308
|
try {
|
|
309
|
+
const sqliteVec = await loadSqliteVec();
|
|
262
310
|
const driver = ds.driver as any;
|
|
263
311
|
const db = driver.databaseConnection || driver.nativeDatabase;
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const syncMode = process.env.SQLITE_SYNC_MODE || 'NORMAL';
|
|
269
|
-
const busyTimeout = process.env.SQLITE_BUSY_TIMEOUT || '5000';
|
|
270
|
-
const cacheSize = process.env.SQLITE_CACHE_SIZE || '-20000';
|
|
271
|
-
|
|
272
|
-
db.pragma(`synchronous = ${syncMode}`);
|
|
273
|
-
db.pragma(`busy_timeout = ${busyTimeout}`);
|
|
274
|
-
db.pragma(`cache_size = ${cacheSize}`);
|
|
275
|
-
db.pragma('temp_store = MEMORY');
|
|
276
|
-
|
|
277
|
-
logger.info(
|
|
278
|
-
`SQLite pragmas enabled: WAL mode, synchronous=${syncMode}, busy_timeout=${busyTimeout}, cache_size=${cacheSize}, temp_store=MEMORY`
|
|
279
|
-
);
|
|
312
|
+
if (db && sqliteVec?.load) {
|
|
313
|
+
sqliteVec.load(db);
|
|
314
|
+
logger.info('sqlite-vec extension loaded successfully');
|
|
280
315
|
}
|
|
281
316
|
} catch (err: any) {
|
|
282
|
-
logger.warn(
|
|
317
|
+
logger.warn(
|
|
318
|
+
`Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
|
|
319
|
+
);
|
|
283
320
|
}
|
|
284
321
|
return res;
|
|
285
322
|
};
|
|
@@ -375,20 +412,6 @@ function forceGetDbType(config: DatabaseConfig | undefined): string {
|
|
|
375
412
|
|
|
376
413
|
let DbType: string | undefined;
|
|
377
414
|
|
|
378
|
-
function getVectorStoreType(): string {
|
|
379
|
-
// Check explicit vectorStore config first
|
|
380
|
-
const vectorStoreConfig = AppConfig?.vectorStore;
|
|
381
|
-
if (vectorStoreConfig?.type) {
|
|
382
|
-
if (vectorStoreConfig.type === 'pgvector') return 'postgres';
|
|
383
|
-
if (vectorStoreConfig.type === 'lancedb') return 'lancedb';
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
// Fallback to main store type
|
|
387
|
-
const dbType = getDbType(AppConfig?.store);
|
|
388
|
-
if (dbType === 'postgres') return 'postgres';
|
|
389
|
-
return 'lancedb';
|
|
390
|
-
}
|
|
391
|
-
|
|
392
415
|
function getDbType(config: DatabaseConfig | undefined): string {
|
|
393
416
|
if (DbType === undefined) DbType = forceGetDbType(config);
|
|
394
417
|
return DbType;
|
|
@@ -422,9 +445,12 @@ export function isUsingSqljs(): boolean {
|
|
|
422
445
|
}
|
|
423
446
|
|
|
424
447
|
export async function isVectorStoreSupported(): Promise<boolean> {
|
|
425
|
-
const
|
|
426
|
-
if (
|
|
427
|
-
if (
|
|
448
|
+
const dbType = getDbType(AppConfig?.store);
|
|
449
|
+
if (dbType === 'postgres') return true;
|
|
450
|
+
if (dbType === 'sqlite') {
|
|
451
|
+
const sqliteVecModule = await loadSqliteVec();
|
|
452
|
+
return !!sqliteVecModule;
|
|
453
|
+
}
|
|
428
454
|
return false;
|
|
429
455
|
}
|
|
430
456
|
|
|
@@ -483,14 +509,9 @@ async function insertRowsHelper(
|
|
|
483
509
|
ctx: DbContext,
|
|
484
510
|
doUpsert: boolean
|
|
485
511
|
): Promise<void> {
|
|
486
|
-
const
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
if (doUpsert) {
|
|
490
|
-
await repo.save(rows);
|
|
491
|
-
} else {
|
|
492
|
-
await repo.insert(rows);
|
|
493
|
-
}
|
|
512
|
+
const repo = getDatasourceForTransaction(ctx.txnId).getRepository(tableName);
|
|
513
|
+
if (doUpsert) await repo.save(rows);
|
|
514
|
+
else await repo.insert(rows);
|
|
494
515
|
}
|
|
495
516
|
|
|
496
517
|
export async function addRowForFullTextSearch(
|
|
@@ -499,45 +520,28 @@ export async function addRowForFullTextSearch(
|
|
|
499
520
|
vect: number[],
|
|
500
521
|
ctx: DbContext
|
|
501
522
|
) {
|
|
502
|
-
if (!(await isVectorStoreSupported()))
|
|
503
|
-
logger.warn(`[VECTOR] Vector store not supported, skipping save for ${id}`);
|
|
504
|
-
return;
|
|
505
|
-
}
|
|
523
|
+
if (!(await isVectorStoreSupported())) return;
|
|
506
524
|
try {
|
|
507
525
|
const vecTableName = tableName + VectorSuffix;
|
|
508
|
-
|
|
509
|
-
`[VECTOR] Saving embedding to ${vecTableName} for ${id} (${vect.length} dimensions)`
|
|
510
|
-
);
|
|
511
|
-
const dbType = getVectorStoreType();
|
|
526
|
+
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
512
527
|
const tenantId = await ctx.getTenantId();
|
|
513
|
-
|
|
514
|
-
if (dbType === '
|
|
515
|
-
let store = lanceDBStores.get(tableName);
|
|
516
|
-
if (!store) {
|
|
517
|
-
store = createLanceDBStore({
|
|
518
|
-
moduleName: tableName,
|
|
519
|
-
vectorDimension: vect.length,
|
|
520
|
-
});
|
|
521
|
-
await store.init();
|
|
522
|
-
lanceDBStores.set(tableName, store);
|
|
523
|
-
}
|
|
524
|
-
await store.addEmbedding({
|
|
525
|
-
id,
|
|
526
|
-
embedding: Array.from(vect),
|
|
527
|
-
tenantId,
|
|
528
|
-
});
|
|
529
|
-
} else if (dbType === 'postgres') {
|
|
530
|
-
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
528
|
+
const dbType = getDbType(AppConfig?.store);
|
|
529
|
+
if (dbType === 'postgres') {
|
|
531
530
|
const { default: pgvector } = await import('pgvector');
|
|
532
531
|
await qb
|
|
533
532
|
.insert()
|
|
534
533
|
.into(vecTableName)
|
|
535
|
-
.values([{ id: id, embedding: pgvector.toSql(vect),
|
|
534
|
+
.values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
|
|
535
|
+
.execute();
|
|
536
|
+
} else {
|
|
537
|
+
await qb
|
|
538
|
+
.insert()
|
|
539
|
+
.into(vecTableName)
|
|
540
|
+
.values([{ id: id, embedding: new Float32Array(vect) }])
|
|
536
541
|
.execute();
|
|
537
542
|
}
|
|
538
|
-
logger.info(`[VECTOR] Successfully saved embedding to ${vecTableName} for ${id}`);
|
|
539
543
|
} catch (err: any) {
|
|
540
|
-
logger.error(`
|
|
544
|
+
logger.error(`Failed to add row to vector store - ${err}`);
|
|
541
545
|
}
|
|
542
546
|
}
|
|
543
547
|
|
|
@@ -546,21 +550,11 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
546
550
|
logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
|
|
547
551
|
return;
|
|
548
552
|
}
|
|
549
|
-
const dbType =
|
|
553
|
+
const dbType = getDbType(AppConfig?.store);
|
|
550
554
|
let notInited = true;
|
|
551
555
|
for (const vecTableName of tableNames) {
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
const store = createLanceDBStore({
|
|
555
|
-
moduleName: vecTableName,
|
|
556
|
-
vectorDimension: DefaultVectorDimension,
|
|
557
|
-
});
|
|
558
|
-
await store.init();
|
|
559
|
-
lanceDBStores.set(vecTableName, store);
|
|
560
|
-
logger.info(`[VECTOR] Initialized LanceDB store for ${vecTableName}`);
|
|
561
|
-
}
|
|
562
|
-
} else if (dbType === 'postgres') {
|
|
563
|
-
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
556
|
+
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
557
|
+
if (dbType === 'postgres') {
|
|
564
558
|
if (notInited) {
|
|
565
559
|
let failure = false;
|
|
566
560
|
try {
|
|
@@ -574,11 +568,18 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
|
574
568
|
}
|
|
575
569
|
await vecRepo.query(
|
|
576
570
|
`CREATE TABLE IF NOT EXISTS ${vecTableName} (
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
571
|
+
id varchar PRIMARY KEY,
|
|
572
|
+
embedding vector(${DefaultVectorDimension}),
|
|
573
|
+
${TenantAttributeName} varchar,
|
|
574
|
+
__is_deleted__ boolean default false
|
|
575
|
+
)`
|
|
576
|
+
);
|
|
577
|
+
} else {
|
|
578
|
+
// sqlite-vec - vec0 doesn't support type declarations for metadata columns
|
|
579
|
+
await vecRepo.query(
|
|
580
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
|
|
581
|
+
id TEXT PRIMARY KEY,
|
|
582
|
+
embedding FLOAT[${DefaultVectorDimension}])`
|
|
582
583
|
);
|
|
583
584
|
}
|
|
584
585
|
}
|
|
@@ -595,21 +596,6 @@ export async function vectorStoreSearch(
|
|
|
595
596
|
return [];
|
|
596
597
|
}
|
|
597
598
|
try {
|
|
598
|
-
const dbType = getVectorStoreType();
|
|
599
|
-
const tenantId = await ctx.getTenantId();
|
|
600
|
-
|
|
601
|
-
if (dbType === 'lancedb') {
|
|
602
|
-
const store = lanceDBStores.get(tableName);
|
|
603
|
-
if (!store) {
|
|
604
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
605
|
-
return [];
|
|
606
|
-
}
|
|
607
|
-
// Extract agentId from resourceFqName for agent-level filtering
|
|
608
|
-
const agentId = ctx.resourceFqName || undefined;
|
|
609
|
-
const results = await store.search(searchVec, tenantId, agentId, limit);
|
|
610
|
-
return results.map(r => ({ id: r.id }));
|
|
611
|
-
}
|
|
612
|
-
|
|
613
599
|
let hasGlobalPerms = ctx.isPermitted();
|
|
614
600
|
if (!hasGlobalPerms) {
|
|
615
601
|
const userId = ctx.getUserId();
|
|
@@ -619,6 +605,8 @@ export async function vectorStoreSearch(
|
|
|
619
605
|
}
|
|
620
606
|
const vecTableName = tableName + VectorSuffix;
|
|
621
607
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
608
|
+
const dbType = getDbType(AppConfig?.store);
|
|
609
|
+
const tenantId = await ctx.getTenantId();
|
|
622
610
|
let ownersJoinCond: string = '';
|
|
623
611
|
if (!hasGlobalPerms) {
|
|
624
612
|
const ot = ownersTable(tableName);
|
|
@@ -631,6 +619,16 @@ export async function vectorStoreSearch(
|
|
|
631
619
|
const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
|
|
632
620
|
const args = pgvector.toSql(searchVec);
|
|
633
621
|
return await qb.query(sql, [args]);
|
|
622
|
+
} else {
|
|
623
|
+
// sqlite-vec - join with main table to filter by tenant
|
|
624
|
+
const alias = tableName.toLowerCase();
|
|
625
|
+
const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
626
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
627
|
+
${ownersJoinCond}
|
|
628
|
+
WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
|
|
629
|
+
LIMIT ${limit}`;
|
|
630
|
+
const args = new Float32Array(searchVec);
|
|
631
|
+
return await qb.query(sql, [args]);
|
|
634
632
|
}
|
|
635
633
|
} catch (err: any) {
|
|
636
634
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -645,19 +643,9 @@ export async function vectorStoreSearchEntryExists(
|
|
|
645
643
|
): Promise<boolean> {
|
|
646
644
|
if (!(await isVectorStoreSupported())) return false;
|
|
647
645
|
try {
|
|
648
|
-
const dbType = getVectorStoreType();
|
|
649
|
-
|
|
650
|
-
if (dbType === 'lancedb') {
|
|
651
|
-
const store = lanceDBStores.get(tableName);
|
|
652
|
-
if (!store) {
|
|
653
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
654
|
-
return false;
|
|
655
|
-
}
|
|
656
|
-
return await store.exists(id);
|
|
657
|
-
}
|
|
658
|
-
|
|
659
646
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
660
647
|
const vecTableName = tableName + VectorSuffix;
|
|
648
|
+
const dbType = getDbType(AppConfig?.store);
|
|
661
649
|
const tenantId = await ctx.getTenantId();
|
|
662
650
|
|
|
663
651
|
if (dbType === 'postgres') {
|
|
@@ -666,6 +654,16 @@ export async function vectorStoreSearchEntryExists(
|
|
|
666
654
|
[id]
|
|
667
655
|
);
|
|
668
656
|
return result !== null && result.length > 0;
|
|
657
|
+
} else {
|
|
658
|
+
// sqlite-vec - join with main table to verify tenant
|
|
659
|
+
const alias = tableName.toLowerCase();
|
|
660
|
+
const result: any[] = await qb.query(
|
|
661
|
+
`SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
662
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
663
|
+
WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
|
|
664
|
+
[id]
|
|
665
|
+
);
|
|
666
|
+
return result !== null && result.length > 0;
|
|
669
667
|
}
|
|
670
668
|
} catch (err: any) {
|
|
671
669
|
logger.error(`Vector store search failed - ${err}`);
|
|
@@ -676,20 +674,9 @@ export async function vectorStoreSearchEntryExists(
|
|
|
676
674
|
export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
|
|
677
675
|
if (!(await isVectorStoreSupported())) return;
|
|
678
676
|
try {
|
|
679
|
-
const dbType = getVectorStoreType();
|
|
680
|
-
|
|
681
|
-
if (dbType === 'lancedb') {
|
|
682
|
-
const store = lanceDBStores.get(tableName);
|
|
683
|
-
if (!store) {
|
|
684
|
-
logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
|
|
685
|
-
return;
|
|
686
|
-
}
|
|
687
|
-
await store.delete(id);
|
|
688
|
-
return;
|
|
689
|
-
}
|
|
690
|
-
|
|
691
677
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
692
678
|
const vecTableName = tableName + VectorSuffix;
|
|
679
|
+
const dbType = getDbType(AppConfig?.store);
|
|
693
680
|
const tenantId = await ctx.getTenantId();
|
|
694
681
|
|
|
695
682
|
if (dbType === 'postgres') {
|
|
@@ -697,6 +684,9 @@ export async function deleteFullTextSearchEntry(tableName: string, id: string, c
|
|
|
697
684
|
`delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
|
|
698
685
|
[id]
|
|
699
686
|
);
|
|
687
|
+
} else {
|
|
688
|
+
// sqlite-vec - delete just by id (ownership verified by caller)
|
|
689
|
+
await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
|
|
700
690
|
}
|
|
701
691
|
} catch (err: any) {
|
|
702
692
|
logger.error(`Vector store delete failed - ${err}`);
|
|
@@ -872,7 +862,7 @@ async function createLimitedOwnership(
|
|
|
872
862
|
r: perms.has(RbacPermissionFlag.READ),
|
|
873
863
|
d: perms.has(RbacPermissionFlag.DELETE),
|
|
874
864
|
u: perms.has(RbacPermissionFlag.UPDATE),
|
|
875
|
-
|
|
865
|
+
__tenant__: tenantId,
|
|
876
866
|
});
|
|
877
867
|
});
|
|
878
868
|
const tname = ownersTable(tableName);
|
|
@@ -1374,16 +1364,14 @@ async function endTransaction(txnId: string, commit: boolean): Promise<void> {
|
|
|
1374
1364
|
const qr: QueryRunner | undefined = transactionsDb.get(txnId);
|
|
1375
1365
|
if (qr && qr.isTransactionActive) {
|
|
1376
1366
|
try {
|
|
1377
|
-
if (commit)
|
|
1378
|
-
await qr.commitTransaction()
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
);
|
|
1386
|
-
throw err;
|
|
1367
|
+
if (commit)
|
|
1368
|
+
await qr.commitTransaction().catch((reason: any) => {
|
|
1369
|
+
logger.error(`failed to commit transaction ${txnId} - ${reason}`);
|
|
1370
|
+
});
|
|
1371
|
+
else
|
|
1372
|
+
await qr.rollbackTransaction().catch((reason: any) => {
|
|
1373
|
+
logger.error(`failed to rollback transaction ${txnId} - ${reason}`);
|
|
1374
|
+
});
|
|
1387
1375
|
} finally {
|
|
1388
1376
|
await qr.release();
|
|
1389
1377
|
transactionsDb.delete(txnId);
|
|
@@ -101,10 +101,6 @@ export class EmbeddingService {
|
|
|
101
101
|
return await this.provider.embedText(query);
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
async embedTexts(texts: string[]): Promise<number[][]> {
|
|
105
|
-
return await this.provider.embedTexts(texts);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
104
|
private averageEmbeddings(embeddings: number[][]): number[] {
|
|
109
105
|
if (embeddings.length === 0) return [];
|
|
110
106
|
const dimension = embeddings[0].length;
|
|
@@ -68,7 +68,6 @@ class DocumentFetcherService {
|
|
|
68
68
|
|
|
69
69
|
async fetchDocument(config: DocumentConfig): Promise<FetchedDocument | null> {
|
|
70
70
|
this.ensureNodeEnv();
|
|
71
|
-
logger.info(`[DOC-FETCH] Starting fetch for: ${config.title}`);
|
|
72
71
|
const cacheKey = `${config.title}:${config.url || config.documentServiceId}`;
|
|
73
72
|
const cached = this.documentCache.get(cacheKey);
|
|
74
73
|
|
|
@@ -80,7 +79,6 @@ class DocumentFetcherService {
|
|
|
80
79
|
try {
|
|
81
80
|
let content: string;
|
|
82
81
|
let sourceUrl: string;
|
|
83
|
-
logger.info(`[DOC-FETCH] Fetching from URL: ${config.url}`);
|
|
84
82
|
|
|
85
83
|
if (config.url?.startsWith('document-service://')) {
|
|
86
84
|
if (!config.retrievalConfig || config.retrievalConfig.provider !== 'document-service') {
|
|
@@ -146,10 +144,8 @@ class DocumentFetcherService {
|
|
|
146
144
|
content = await this.fetchFromUrl(config.url);
|
|
147
145
|
sourceUrl = config.url;
|
|
148
146
|
} else if (config.url) {
|
|
149
|
-
logger.info(`[DOC-FETCH] Fetching from local file: ${config.url}`);
|
|
150
147
|
content = await this.fetchFromLocal(config.url);
|
|
151
148
|
sourceUrl = config.url;
|
|
152
|
-
logger.info(`[DOC-FETCH] Local file fetched successfully: ${content.length} chars`);
|
|
153
149
|
} else {
|
|
154
150
|
if (this.documentServiceConfig) {
|
|
155
151
|
const docId = await this.lookupDocumentByTitle(config.title);
|
|
@@ -175,9 +171,7 @@ class DocumentFetcherService {
|
|
|
175
171
|
|
|
176
172
|
this.documentCache.set(cacheKey, document);
|
|
177
173
|
|
|
178
|
-
logger.info(`[DOC-FETCH] Creating Document entity for: ${config.title}`);
|
|
179
174
|
await this.createDocumentEntity(document);
|
|
180
|
-
logger.info(`[DOC-FETCH] Document entity created successfully for: ${config.title}`);
|
|
181
175
|
|
|
182
176
|
return document;
|
|
183
177
|
} catch (error) {
|
|
@@ -449,14 +443,11 @@ class DocumentFetcherService {
|
|
|
449
443
|
|
|
450
444
|
private async fetchFromLocal(filePath: string): Promise<string> {
|
|
451
445
|
try {
|
|
452
|
-
logger.info(`[DOC-FETCH] Reading local file: ${filePath}`);
|
|
453
446
|
const content = await readFile(filePath);
|
|
454
|
-
logger.info(`[DOC-FETCH] File read successfully: ${content.length} chars`);
|
|
455
447
|
const lowerPath = filePath.toLowerCase();
|
|
456
448
|
const isMarkdown = lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown');
|
|
457
449
|
|
|
458
450
|
if (isMarkdown) {
|
|
459
|
-
logger.info(`[DOC-FETCH] Parsing markdown...`);
|
|
460
451
|
return this.parseMarkdownText(content);
|
|
461
452
|
}
|
|
462
453
|
|
|
@@ -556,11 +547,10 @@ class DocumentFetcherService {
|
|
|
556
547
|
// Lazy load PDF parser
|
|
557
548
|
if (!this.pdfParser) {
|
|
558
549
|
try {
|
|
559
|
-
const
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
this.pdfParser = PDFParse;
|
|
550
|
+
const pdfParse = await import('pdf-parse');
|
|
551
|
+
// Handle both ESM and CSM module formats
|
|
552
|
+
const parser = (pdfParse as any).default || pdfParse;
|
|
553
|
+
this.pdfParser = parser;
|
|
564
554
|
} catch (error) {
|
|
565
555
|
logger.error('Failed to load PDF parser', { error });
|
|
566
556
|
throw new Error(
|
|
@@ -570,16 +560,11 @@ class DocumentFetcherService {
|
|
|
570
560
|
}
|
|
571
561
|
|
|
572
562
|
try {
|
|
573
|
-
const
|
|
574
|
-
const result = await parser.getText();
|
|
575
|
-
if (typeof parser.destroy === 'function') {
|
|
576
|
-
await parser.destroy();
|
|
577
|
-
}
|
|
563
|
+
const result = await this.pdfParser(buffer);
|
|
578
564
|
return result.text || '';
|
|
579
565
|
} catch (error) {
|
|
580
566
|
logger.error('PDF parsing failed', { error });
|
|
581
|
-
|
|
582
|
-
throw new Error(`Failed to parse PDF: ${message}`);
|
|
567
|
+
throw new Error(`Failed to parse PDF: ${error}`);
|
|
583
568
|
}
|
|
584
569
|
}
|
|
585
570
|
|
package/src/runtime/state.ts
CHANGED
|
@@ -35,21 +35,6 @@ export const ConfigSchema = z.object({
|
|
|
35
35
|
type: z.literal('sqlite'),
|
|
36
36
|
dbname: z.string().optional(),
|
|
37
37
|
}),
|
|
38
|
-
z.object({
|
|
39
|
-
type: z.literal('lancedb'),
|
|
40
|
-
path: z.string().optional(),
|
|
41
|
-
}),
|
|
42
|
-
])
|
|
43
|
-
.optional(),
|
|
44
|
-
vectorStore: z
|
|
45
|
-
.discriminatedUnion('type', [
|
|
46
|
-
z.object({
|
|
47
|
-
type: z.literal('pgvector'),
|
|
48
|
-
}),
|
|
49
|
-
z.object({
|
|
50
|
-
type: z.literal('lancedb'),
|
|
51
|
-
dbname: z.string().optional(),
|
|
52
|
-
}),
|
|
53
38
|
])
|
|
54
39
|
.optional(),
|
|
55
40
|
integrations: z
|
|
@@ -96,11 +81,6 @@ export const ConfigSchema = z.object({
|
|
|
96
81
|
enabled: z.boolean().default(false),
|
|
97
82
|
})
|
|
98
83
|
.optional(),
|
|
99
|
-
knowledgeGraph: z
|
|
100
|
-
.object({
|
|
101
|
-
serviceUrl: z.string().default('#js process.env.KNOWLEDGE_SERVICE_URL || ""'),
|
|
102
|
-
})
|
|
103
|
-
.optional(),
|
|
104
84
|
authentication: z
|
|
105
85
|
.discriminatedUnion('service', [
|
|
106
86
|
z.object({
|
|
@@ -185,15 +165,6 @@ export function isMonitoringEnabled(): boolean {
|
|
|
185
165
|
return internalMonitoringEnabled || AppConfig?.monitoring?.enabled === true;
|
|
186
166
|
}
|
|
187
167
|
|
|
188
|
-
export function isKnowledgeGraphEnabled(): boolean {
|
|
189
|
-
const url = AppConfig?.knowledgeGraph?.serviceUrl?.trim();
|
|
190
|
-
return !!(url && url.length > 0) || !!process.env.KNOWLEDGE_SERVICE_URL;
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
export function getKnowledgeGraphConfig(): Config['knowledgeGraph'] | undefined {
|
|
194
|
-
return AppConfig?.knowledgeGraph;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
168
|
type TtlCacheEntry<T> = {
|
|
198
169
|
value: T;
|
|
199
170
|
expireTime: number;
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Local document retriever — embeds documents into pgvector or LanceDB
|
|
3
|
-
* and retrieves relevant chunks via vector similarity search.
|
|
4
|
-
*/
|
|
5
|
-
declare class DocumentRetriever {
|
|
6
|
-
private vectorStore;
|
|
7
|
-
private embeddingProvider;
|
|
8
|
-
private chunker;
|
|
9
|
-
private localChunks;
|
|
10
|
-
private processedDocuments;
|
|
11
|
-
private initialized;
|
|
12
|
-
private ensureInit;
|
|
13
|
-
processDocument(title: string, url: string): Promise<void>;
|
|
14
|
-
private storePgvectorChunks;
|
|
15
|
-
private storeLanceDBChunks;
|
|
16
|
-
query(queryText: string, documentTitles?: string[], limit?: number): Promise<string>;
|
|
17
|
-
private queryPgvector;
|
|
18
|
-
private queryLanceDB;
|
|
19
|
-
close(): Promise<void>;
|
|
20
|
-
}
|
|
21
|
-
export declare function getDocumentRetriever(): DocumentRetriever;
|
|
22
|
-
export declare function resetDocumentRetriever(): void;
|
|
23
|
-
export {};
|
|
24
|
-
//# sourceMappingURL=document-retriever.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"document-retriever.d.ts","sourceRoot":"","sources":["../../src/runtime/document-retriever.ts"],"names":[],"mappings":"AAyBA;;;GAGG;AACH,cAAM,iBAAiB;IACrB,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,iBAAiB,CAAwC;IACjE,OAAO,CAAC,OAAO,CAA4B;IAC3C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,kBAAkB,CAA0B;IACpD,OAAO,CAAC,WAAW,CAAS;YAEd,UAAU;IAoDlB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAyDlD,mBAAmB;YA0BnB,kBAAkB;IAuB1B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAiBhF,aAAa;YAsCb,YAAY;IAmCpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B;AAID,wBAAgB,oBAAoB,IAAI,iBAAiB,CAKxD;AAED,wBAAgB,sBAAsB,IAAI,IAAI,CAK7C"}
|