agentlang 0.10.2 → 0.10.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +7 -14
  2. package/out/api/http.d.ts +4 -0
  3. package/out/api/http.d.ts.map +1 -1
  4. package/out/api/http.js +171 -26
  5. package/out/api/http.js.map +1 -1
  6. package/out/cli/main.d.ts.map +1 -1
  7. package/out/cli/main.js +3 -0
  8. package/out/cli/main.js.map +1 -1
  9. package/out/extension/main.cjs +250 -250
  10. package/out/extension/main.cjs.map +2 -2
  11. package/out/language/agentlang-validator.d.ts.map +1 -1
  12. package/out/language/agentlang-validator.js +4 -0
  13. package/out/language/agentlang-validator.js.map +1 -1
  14. package/out/language/error-reporter.d.ts +53 -0
  15. package/out/language/error-reporter.d.ts.map +1 -0
  16. package/out/language/error-reporter.js +879 -0
  17. package/out/language/error-reporter.js.map +1 -0
  18. package/out/language/generated/ast.d.ts +51 -1
  19. package/out/language/generated/ast.d.ts.map +1 -1
  20. package/out/language/generated/ast.js +40 -0
  21. package/out/language/generated/ast.js.map +1 -1
  22. package/out/language/generated/grammar.d.ts.map +1 -1
  23. package/out/language/generated/grammar.js +286 -190
  24. package/out/language/generated/grammar.js.map +1 -1
  25. package/out/language/main.cjs +828 -694
  26. package/out/language/main.cjs.map +3 -3
  27. package/out/language/parser.d.ts +4 -2
  28. package/out/language/parser.d.ts.map +1 -1
  29. package/out/language/parser.js +30 -97
  30. package/out/language/parser.js.map +1 -1
  31. package/out/language/syntax.d.ts +2 -0
  32. package/out/language/syntax.d.ts.map +1 -1
  33. package/out/language/syntax.js +6 -0
  34. package/out/language/syntax.js.map +1 -1
  35. package/out/runtime/api.d.ts.map +1 -1
  36. package/out/runtime/api.js +22 -0
  37. package/out/runtime/api.js.map +1 -1
  38. package/out/runtime/defs.d.ts +1 -0
  39. package/out/runtime/defs.d.ts.map +1 -1
  40. package/out/runtime/defs.js +2 -1
  41. package/out/runtime/defs.js.map +1 -1
  42. package/out/runtime/document-retriever.d.ts +24 -0
  43. package/out/runtime/document-retriever.d.ts.map +1 -0
  44. package/out/runtime/document-retriever.js +258 -0
  45. package/out/runtime/document-retriever.js.map +1 -0
  46. package/out/runtime/embeddings/chunker.d.ts +18 -0
  47. package/out/runtime/embeddings/chunker.d.ts.map +1 -1
  48. package/out/runtime/embeddings/chunker.js +47 -15
  49. package/out/runtime/embeddings/chunker.js.map +1 -1
  50. package/out/runtime/embeddings/openai.d.ts.map +1 -1
  51. package/out/runtime/embeddings/openai.js +22 -9
  52. package/out/runtime/embeddings/openai.js.map +1 -1
  53. package/out/runtime/embeddings/provider.d.ts +1 -0
  54. package/out/runtime/embeddings/provider.d.ts.map +1 -1
  55. package/out/runtime/embeddings/provider.js +20 -1
  56. package/out/runtime/embeddings/provider.js.map +1 -1
  57. package/out/runtime/integration-client.d.ts +21 -0
  58. package/out/runtime/integration-client.d.ts.map +1 -0
  59. package/out/runtime/integration-client.js +112 -0
  60. package/out/runtime/integration-client.js.map +1 -0
  61. package/out/runtime/integrations.d.ts.map +1 -1
  62. package/out/runtime/integrations.js +20 -9
  63. package/out/runtime/integrations.js.map +1 -1
  64. package/out/runtime/interpreter.d.ts +1 -0
  65. package/out/runtime/interpreter.d.ts.map +1 -1
  66. package/out/runtime/interpreter.js +152 -17
  67. package/out/runtime/interpreter.js.map +1 -1
  68. package/out/runtime/loader.d.ts.map +1 -1
  69. package/out/runtime/loader.js +70 -7
  70. package/out/runtime/loader.js.map +1 -1
  71. package/out/runtime/logger.d.ts.map +1 -1
  72. package/out/runtime/logger.js +8 -1
  73. package/out/runtime/logger.js.map +1 -1
  74. package/out/runtime/module.d.ts +10 -0
  75. package/out/runtime/module.d.ts.map +1 -1
  76. package/out/runtime/module.js +68 -3
  77. package/out/runtime/module.js.map +1 -1
  78. package/out/runtime/modules/ai.d.ts +9 -2
  79. package/out/runtime/modules/ai.d.ts.map +1 -1
  80. package/out/runtime/modules/ai.js +219 -67
  81. package/out/runtime/modules/ai.js.map +1 -1
  82. package/out/runtime/resolvers/interface.d.ts +4 -0
  83. package/out/runtime/resolvers/interface.d.ts.map +1 -1
  84. package/out/runtime/resolvers/interface.js +14 -1
  85. package/out/runtime/resolvers/interface.js.map +1 -1
  86. package/out/runtime/resolvers/sqldb/database.d.ts +2 -0
  87. package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
  88. package/out/runtime/resolvers/sqldb/database.js +142 -126
  89. package/out/runtime/resolvers/sqldb/database.js.map +1 -1
  90. package/out/runtime/resolvers/sqldb/dbutil.d.ts.map +1 -1
  91. package/out/runtime/resolvers/sqldb/dbutil.js +8 -0
  92. package/out/runtime/resolvers/sqldb/dbutil.js.map +1 -1
  93. package/out/runtime/resolvers/sqldb/impl.d.ts +1 -0
  94. package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
  95. package/out/runtime/resolvers/sqldb/impl.js +7 -0
  96. package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
  97. package/out/runtime/resolvers/vector/lancedb-store.d.ts +16 -0
  98. package/out/runtime/resolvers/vector/lancedb-store.d.ts.map +1 -0
  99. package/out/runtime/resolvers/vector/lancedb-store.js +159 -0
  100. package/out/runtime/resolvers/vector/lancedb-store.js.map +1 -0
  101. package/out/runtime/resolvers/vector/types.d.ts +32 -0
  102. package/out/runtime/resolvers/vector/types.d.ts.map +1 -0
  103. package/out/runtime/resolvers/vector/types.js +2 -0
  104. package/out/runtime/resolvers/vector/types.js.map +1 -0
  105. package/out/runtime/services/documentFetcher.d.ts.map +1 -1
  106. package/out/runtime/services/documentFetcher.js +21 -6
  107. package/out/runtime/services/documentFetcher.js.map +1 -1
  108. package/out/runtime/state.d.ts +19 -1
  109. package/out/runtime/state.d.ts.map +1 -1
  110. package/out/runtime/state.js +36 -1
  111. package/out/runtime/state.js.map +1 -1
  112. package/out/syntaxes/agentlang.monarch.js +1 -1
  113. package/out/syntaxes/agentlang.monarch.js.map +1 -1
  114. package/package.json +19 -19
  115. package/src/api/http.ts +197 -37
  116. package/src/cli/main.ts +3 -0
  117. package/src/language/agentlang-validator.ts +3 -0
  118. package/src/language/agentlang.langium +3 -1
  119. package/src/language/error-reporter.ts +1028 -0
  120. package/src/language/generated/ast.ts +62 -0
  121. package/src/language/generated/grammar.ts +286 -190
  122. package/src/language/parser.ts +31 -100
  123. package/src/language/syntax.ts +8 -0
  124. package/src/runtime/api.ts +31 -0
  125. package/src/runtime/defs.ts +2 -1
  126. package/src/runtime/document-retriever.ts +311 -0
  127. package/src/runtime/embeddings/chunker.ts +52 -14
  128. package/src/runtime/embeddings/openai.ts +27 -9
  129. package/src/runtime/embeddings/provider.ts +22 -1
  130. package/src/runtime/integration-client.ts +158 -0
  131. package/src/runtime/integrations.ts +20 -11
  132. package/src/runtime/interpreter.ts +142 -12
  133. package/src/runtime/loader.ts +83 -5
  134. package/src/runtime/logger.ts +12 -1
  135. package/src/runtime/module.ts +78 -3
  136. package/src/runtime/modules/ai.ts +263 -76
  137. package/src/runtime/resolvers/interface.ts +19 -1
  138. package/src/runtime/resolvers/sqldb/database.ts +158 -130
  139. package/src/runtime/resolvers/sqldb/dbutil.ts +8 -0
  140. package/src/runtime/resolvers/sqldb/impl.ts +8 -0
  141. package/src/runtime/resolvers/vector/lancedb-store.ts +187 -0
  142. package/src/runtime/resolvers/vector/types.ts +39 -0
  143. package/src/runtime/services/documentFetcher.ts +21 -6
  144. package/src/runtime/state.ts +40 -1
  145. package/src/syntaxes/agentlang.monarch.ts +1 -1
@@ -46,6 +46,8 @@ import { saveMigration } from '../../modules/core.js';
46
46
  import { getAppSpec } from '../../loader.js';
47
47
  import { WhereClause } from '../interface.js';
48
48
  import { AppConfig } from '../../state.js';
49
+ import { createLanceDBStore } from '../vector/lancedb-store.js';
50
+ import type { VectorStore } from '../vector/types.js';
49
51
 
50
52
  export let defaultDataSource: DataSource | undefined;
51
53
 
@@ -58,57 +60,8 @@ function isBrowser(): boolean {
58
60
  );
59
61
  }
60
62
 
61
- // SQLite WASM with built-in sqlite-vec for browsers
62
- // Loaded from CDN - this has sqlite-vec statically compiled in
63
- let sqliteVecWasmModule: any = null;
64
-
65
- async function loadSqliteVecWasm(): Promise<any> {
66
- if (sqliteVecWasmModule) {
67
- return sqliteVecWasmModule;
68
- }
69
-
70
- try {
71
- // Use dynamic import with string to prevent bundlers from analyzing
72
- const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
73
- const module = await import(/* @vite-ignore */ cdnUrl);
74
- sqliteVecWasmModule = await module.default();
75
- return sqliteVecWasmModule;
76
- } catch (err) {
77
- logger.warn('Failed to load sqlite-vec WASM:', err);
78
- return null;
79
- }
80
- }
81
-
82
- // Helper to load sqlite-vec based on environment
83
- async function loadSqliteVec(): Promise<any> {
84
- // In browser, use WASM version with built-in sqlite-vec
85
- // In Node.js, use the npm package
86
- if (isBrowser()) {
87
- const wasmModule = await loadSqliteVecWasm();
88
- if (wasmModule) {
89
- // WASM version has sqlite-vec built-in, no need to call load()
90
- // Return a compatible interface
91
- return {
92
- load: () => {
93
- // No-op: sqlite-vec is already loaded in WASM version
94
- logger.info('sqlite-vec WASM loaded (built-in)');
95
- },
96
- // Expose the WASM module for direct use if needed
97
- _wasmModule: wasmModule,
98
- };
99
- }
100
- return null;
101
- }
102
-
103
- // Node.js: use npm package
104
- try {
105
- // Use variable to prevent bundlers from statically analyzing this import
106
- const moduleName = 'sqlite-vec';
107
- return await import(/* @vite-ignore */ moduleName);
108
- } catch {
109
- return null;
110
- }
111
- }
63
+ // LanceDB vector store cache - keyed by module name
64
+ const lanceDBStores: Map<string, VectorStore> = new Map();
112
65
 
113
66
  export class DbContext {
114
67
  txnId: string | undefined;
@@ -306,17 +259,27 @@ function makeSqliteDataSource(
306
259
  ds.initialize = async () => {
307
260
  const res = await originalInit();
308
261
  try {
309
- const sqliteVec = await loadSqliteVec();
310
262
  const driver = ds.driver as any;
311
263
  const db = driver.databaseConnection || driver.nativeDatabase;
312
- if (db && sqliteVec?.load) {
313
- sqliteVec.load(db);
314
- logger.info('sqlite-vec extension loaded successfully');
264
+ // Enable WAL mode and additional pragmas for better write performance
265
+ if (db?.pragma) {
266
+ db.pragma('journal_mode = WAL');
267
+
268
+ const syncMode = process.env.SQLITE_SYNC_MODE || 'NORMAL';
269
+ const busyTimeout = process.env.SQLITE_BUSY_TIMEOUT || '5000';
270
+ const cacheSize = process.env.SQLITE_CACHE_SIZE || '-20000';
271
+
272
+ db.pragma(`synchronous = ${syncMode}`);
273
+ db.pragma(`busy_timeout = ${busyTimeout}`);
274
+ db.pragma(`cache_size = ${cacheSize}`);
275
+ db.pragma('temp_store = MEMORY');
276
+
277
+ logger.info(
278
+ `SQLite pragmas enabled: WAL mode, synchronous=${syncMode}, busy_timeout=${busyTimeout}, cache_size=${cacheSize}, temp_store=MEMORY`
279
+ );
315
280
  }
316
281
  } catch (err: any) {
317
- logger.warn(
318
- `Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
319
- );
282
+ logger.warn(`Failed to enable SQLite pragmas: ${err.message}.`);
320
283
  }
321
284
  return res;
322
285
  };
@@ -412,6 +375,20 @@ function forceGetDbType(config: DatabaseConfig | undefined): string {
412
375
 
413
376
  let DbType: string | undefined;
414
377
 
378
+ function getVectorStoreType(): string {
379
+ // Check explicit vectorStore config first
380
+ const vectorStoreConfig = AppConfig?.vectorStore;
381
+ if (vectorStoreConfig?.type) {
382
+ if (vectorStoreConfig.type === 'pgvector') return 'postgres';
383
+ if (vectorStoreConfig.type === 'lancedb') return 'lancedb';
384
+ }
385
+
386
+ // Fallback to main store type
387
+ const dbType = getDbType(AppConfig?.store);
388
+ if (dbType === 'postgres') return 'postgres';
389
+ return 'lancedb';
390
+ }
391
+
415
392
  function getDbType(config: DatabaseConfig | undefined): string {
416
393
  if (DbType === undefined) DbType = forceGetDbType(config);
417
394
  return DbType;
@@ -445,12 +422,9 @@ export function isUsingSqljs(): boolean {
445
422
  }
446
423
 
447
424
  export async function isVectorStoreSupported(): Promise<boolean> {
448
- const dbType = getDbType(AppConfig?.store);
449
- if (dbType === 'postgres') return true;
450
- if (dbType === 'sqlite') {
451
- const sqliteVecModule = await loadSqliteVec();
452
- return !!sqliteVecModule;
453
- }
425
+ const vectorStoreType = getVectorStoreType();
426
+ if (vectorStoreType === 'postgres') return true;
427
+ if (vectorStoreType === 'lancedb') return true;
454
428
  return false;
455
429
  }
456
430
 
@@ -509,9 +483,14 @@ async function insertRowsHelper(
509
483
  ctx: DbContext,
510
484
  doUpsert: boolean
511
485
  ): Promise<void> {
512
- const repo = getDatasourceForTransaction(ctx.txnId).getRepository(tableName);
513
- if (doUpsert) await repo.save(rows);
514
- else await repo.insert(rows);
486
+ const ds = getDatasourceForTransaction(ctx.txnId);
487
+ const repo = ds.getRepository(tableName);
488
+
489
+ if (doUpsert) {
490
+ await repo.save(rows);
491
+ } else {
492
+ await repo.insert(rows);
493
+ }
515
494
  }
516
495
 
517
496
  export async function addRowForFullTextSearch(
@@ -520,28 +499,45 @@ export async function addRowForFullTextSearch(
520
499
  vect: number[],
521
500
  ctx: DbContext
522
501
  ) {
523
- if (!(await isVectorStoreSupported())) return;
502
+ if (!(await isVectorStoreSupported())) {
503
+ logger.warn(`[VECTOR] Vector store not supported, skipping save for ${id}`);
504
+ return;
505
+ }
524
506
  try {
525
507
  const vecTableName = tableName + VectorSuffix;
526
- const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
508
+ logger.info(
509
+ `[VECTOR] Saving embedding to ${vecTableName} for ${id} (${vect.length} dimensions)`
510
+ );
511
+ const dbType = getVectorStoreType();
527
512
  const tenantId = await ctx.getTenantId();
528
- const dbType = getDbType(AppConfig?.store);
529
- if (dbType === 'postgres') {
513
+
514
+ if (dbType === 'lancedb') {
515
+ let store = lanceDBStores.get(tableName);
516
+ if (!store) {
517
+ store = createLanceDBStore({
518
+ moduleName: tableName,
519
+ vectorDimension: vect.length,
520
+ });
521
+ await store.init();
522
+ lanceDBStores.set(tableName, store);
523
+ }
524
+ await store.addEmbedding({
525
+ id,
526
+ embedding: Array.from(vect),
527
+ tenantId,
528
+ });
529
+ } else if (dbType === 'postgres') {
530
+ const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
530
531
  const { default: pgvector } = await import('pgvector');
531
532
  await qb
532
533
  .insert()
533
534
  .into(vecTableName)
534
- .values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
535
- .execute();
536
- } else {
537
- await qb
538
- .insert()
539
- .into(vecTableName)
540
- .values([{ id: id, embedding: new Float32Array(vect) }])
535
+ .values([{ id: id, embedding: pgvector.toSql(vect), agentId: tenantId }])
541
536
  .execute();
542
537
  }
538
+ logger.info(`[VECTOR] Successfully saved embedding to ${vecTableName} for ${id}`);
543
539
  } catch (err: any) {
544
- logger.error(`Failed to add row to vector store - ${err}`);
540
+ logger.error(`[VECTOR] Failed to add row to vector store - ${err}`);
545
541
  }
546
542
  }
547
543
 
@@ -550,11 +546,21 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
550
546
  logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
551
547
  return;
552
548
  }
553
- const dbType = getDbType(AppConfig?.store);
549
+ const dbType = getVectorStoreType();
554
550
  let notInited = true;
555
551
  for (const vecTableName of tableNames) {
556
- const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
557
- if (dbType === 'postgres') {
552
+ if (dbType === 'lancedb') {
553
+ if (!lanceDBStores.has(vecTableName)) {
554
+ const store = createLanceDBStore({
555
+ moduleName: vecTableName,
556
+ vectorDimension: DefaultVectorDimension,
557
+ });
558
+ await store.init();
559
+ lanceDBStores.set(vecTableName, store);
560
+ logger.info(`[VECTOR] Initialized LanceDB store for ${vecTableName}`);
561
+ }
562
+ } else if (dbType === 'postgres') {
563
+ const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
558
564
  if (notInited) {
559
565
  let failure = false;
560
566
  try {
@@ -568,18 +574,11 @@ export async function initVectorStore(tableNames: string[], ctx: DbContext) {
568
574
  }
569
575
  await vecRepo.query(
570
576
  `CREATE TABLE IF NOT EXISTS ${vecTableName} (
571
- id varchar PRIMARY KEY,
572
- embedding vector(${DefaultVectorDimension}),
573
- ${TenantAttributeName} varchar,
574
- __is_deleted__ boolean default false
575
- )`
576
- );
577
- } else {
578
- // sqlite-vec - vec0 doesn't support type declarations for metadata columns
579
- await vecRepo.query(
580
- `CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
581
- id TEXT PRIMARY KEY,
582
- embedding FLOAT[${DefaultVectorDimension}])`
577
+ id varchar PRIMARY KEY,
578
+ embedding vector(${DefaultVectorDimension}),
579
+ ${TenantAttributeName} varchar,
580
+ __is_deleted__ boolean default false
581
+ )`
583
582
  );
584
583
  }
585
584
  }
@@ -596,6 +595,21 @@ export async function vectorStoreSearch(
596
595
  return [];
597
596
  }
598
597
  try {
598
+ const dbType = getVectorStoreType();
599
+ const tenantId = await ctx.getTenantId();
600
+
601
+ if (dbType === 'lancedb') {
602
+ const store = lanceDBStores.get(tableName);
603
+ if (!store) {
604
+ logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
605
+ return [];
606
+ }
607
+ // Extract agentId from resourceFqName for agent-level filtering
608
+ const agentId = ctx.resourceFqName || undefined;
609
+ const results = await store.search(searchVec, tenantId, agentId, limit);
610
+ return results.map(r => ({ id: r.id }));
611
+ }
612
+
599
613
  let hasGlobalPerms = ctx.isPermitted();
600
614
  if (!hasGlobalPerms) {
601
615
  const userId = ctx.getUserId();
@@ -605,8 +619,6 @@ export async function vectorStoreSearch(
605
619
  }
606
620
  const vecTableName = tableName + VectorSuffix;
607
621
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
608
- const dbType = getDbType(AppConfig?.store);
609
- const tenantId = await ctx.getTenantId();
610
622
  let ownersJoinCond: string = '';
611
623
  if (!hasGlobalPerms) {
612
624
  const ot = ownersTable(tableName);
@@ -619,16 +631,6 @@ export async function vectorStoreSearch(
619
631
  const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
620
632
  const args = pgvector.toSql(searchVec);
621
633
  return await qb.query(sql, [args]);
622
- } else {
623
- // sqlite-vec - join with main table to filter by tenant
624
- const alias = tableName.toLowerCase();
625
- const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
626
- INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
627
- ${ownersJoinCond}
628
- WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
629
- LIMIT ${limit}`;
630
- const args = new Float32Array(searchVec);
631
- return await qb.query(sql, [args]);
632
634
  }
633
635
  } catch (err: any) {
634
636
  logger.error(`Vector store search failed - ${err}`);
@@ -643,9 +645,19 @@ export async function vectorStoreSearchEntryExists(
643
645
  ): Promise<boolean> {
644
646
  if (!(await isVectorStoreSupported())) return false;
645
647
  try {
648
+ const dbType = getVectorStoreType();
649
+
650
+ if (dbType === 'lancedb') {
651
+ const store = lanceDBStores.get(tableName);
652
+ if (!store) {
653
+ logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
654
+ return false;
655
+ }
656
+ return await store.exists(id);
657
+ }
658
+
646
659
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
647
660
  const vecTableName = tableName + VectorSuffix;
648
- const dbType = getDbType(AppConfig?.store);
649
661
  const tenantId = await ctx.getTenantId();
650
662
 
651
663
  if (dbType === 'postgres') {
@@ -654,16 +666,6 @@ export async function vectorStoreSearchEntryExists(
654
666
  [id]
655
667
  );
656
668
  return result !== null && result.length > 0;
657
- } else {
658
- // sqlite-vec - join with main table to verify tenant
659
- const alias = tableName.toLowerCase();
660
- const result: any[] = await qb.query(
661
- `SELECT ${vecTableName}.id FROM ${vecTableName}
662
- INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
663
- WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
664
- [id]
665
- );
666
- return result !== null && result.length > 0;
667
669
  }
668
670
  } catch (err: any) {
669
671
  logger.error(`Vector store search failed - ${err}`);
@@ -674,9 +676,20 @@ export async function vectorStoreSearchEntryExists(
674
676
  export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
675
677
  if (!(await isVectorStoreSupported())) return;
676
678
  try {
679
+ const dbType = getVectorStoreType();
680
+
681
+ if (dbType === 'lancedb') {
682
+ const store = lanceDBStores.get(tableName);
683
+ if (!store) {
684
+ logger.warn(`[VECTOR] LanceDB store not found for ${tableName}`);
685
+ return;
686
+ }
687
+ await store.delete(id);
688
+ return;
689
+ }
690
+
677
691
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
678
692
  const vecTableName = tableName + VectorSuffix;
679
- const dbType = getDbType(AppConfig?.store);
680
693
  const tenantId = await ctx.getTenantId();
681
694
 
682
695
  if (dbType === 'postgres') {
@@ -684,9 +697,6 @@ export async function deleteFullTextSearchEntry(tableName: string, id: string, c
684
697
  `delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
685
698
  [id]
686
699
  );
687
- } else {
688
- // sqlite-vec - delete just by id (ownership verified by caller)
689
- await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
690
700
  }
691
701
  } catch (err: any) {
692
702
  logger.error(`Vector store delete failed - ${err}`);
@@ -862,7 +872,7 @@ async function createLimitedOwnership(
862
872
  r: perms.has(RbacPermissionFlag.READ),
863
873
  d: perms.has(RbacPermissionFlag.DELETE),
864
874
  u: perms.has(RbacPermissionFlag.UPDATE),
865
- __tenant__: tenantId,
875
+ agentId: tenantId,
866
876
  });
867
877
  });
868
878
  const tname = ownersTable(tableName);
@@ -1055,6 +1065,8 @@ export type QuerySpec = {
1055
1065
  intoSpec: Map<string, string> | undefined;
1056
1066
  whereClauses: WhereClause[] | undefined;
1057
1067
  distinct: boolean;
1068
+ limit: number | undefined;
1069
+ offset: number | undefined;
1058
1070
  };
1059
1071
 
1060
1072
  export function makeSimpleQuerySpec(queryObj: object, queryVals: object): QuerySpec {
@@ -1069,6 +1081,8 @@ export function makeSimpleQuerySpec(queryObj: object, queryVals: object): QueryS
1069
1081
  intoSpec: undefined,
1070
1082
  whereClauses: undefined,
1071
1083
  distinct: false,
1084
+ limit: undefined,
1085
+ offset: undefined,
1072
1086
  };
1073
1087
  }
1074
1088
 
@@ -1144,6 +1158,12 @@ export async function getMany(
1144
1158
  if (querySpec.distinct) {
1145
1159
  qb.distinct(true);
1146
1160
  }
1161
+ if (querySpec.limit !== undefined) {
1162
+ qb.take(querySpec.limit);
1163
+ }
1164
+ if (querySpec.offset !== undefined) {
1165
+ qb.skip(querySpec.offset);
1166
+ }
1147
1167
  qb.where(queryStr, querySpec.queryVals);
1148
1168
  if (hasAggregates) return await qb.getRawMany();
1149
1169
  else return await qb.getMany();
@@ -1233,6 +1253,12 @@ export async function getManyByJoin(
1233
1253
  if (querySpec.orderBy !== undefined) {
1234
1254
  sql = `${sql} ORDER BY ${querySpec.orderBy.join(', ')} ${querySpec.orderByDesc}`;
1235
1255
  }
1256
+ if (querySpec.limit !== undefined) {
1257
+ sql = `${sql} LIMIT ${querySpec.limit}`;
1258
+ }
1259
+ if (querySpec.offset !== undefined) {
1260
+ sql = `${sql} OFFSET ${querySpec.offset}`;
1261
+ }
1236
1262
  logger.debug(`Join Query: ${sql}`);
1237
1263
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
1238
1264
  return await qb.query(sql);
@@ -1348,14 +1374,16 @@ async function endTransaction(txnId: string, commit: boolean): Promise<void> {
1348
1374
  const qr: QueryRunner | undefined = transactionsDb.get(txnId);
1349
1375
  if (qr && qr.isTransactionActive) {
1350
1376
  try {
1351
- if (commit)
1352
- await qr.commitTransaction().catch((reason: any) => {
1353
- logger.error(`failed to commit transaction ${txnId} - ${reason}`);
1354
- });
1355
- else
1356
- await qr.rollbackTransaction().catch((reason: any) => {
1357
- logger.error(`failed to rollback transaction ${txnId} - ${reason}`);
1358
- });
1377
+ if (commit) {
1378
+ await qr.commitTransaction();
1379
+ } else {
1380
+ await qr.rollbackTransaction();
1381
+ }
1382
+ } catch (err: any) {
1383
+ logger.error(
1384
+ `Failed to ${commit ? 'commit' : 'rollback'} transaction ${txnId}: ${err.message}`
1385
+ );
1386
+ throw err;
1359
1387
  } finally {
1360
1388
  await qr.release();
1361
1389
  transactionsDb.delete(txnId);
@@ -99,6 +99,12 @@ export type OrmSchema = {
99
99
  fkSpecs: FkSpec[];
100
100
  };
101
101
 
102
+ const SqlDefaultFunctions = new Set(['uuid()', 'autoincrement()', 'now()']);
103
+
104
+ function isSqlDefault(d: any): boolean {
105
+ return typeof d !== 'string' || !d.endsWith('()') || SqlDefaultFunctions.has(d);
106
+ }
107
+
102
108
  export function modulesAsOrmSchema(): OrmSchema {
103
109
  const ents: EntitySchema[] = [];
104
110
  const vects: EntitySchema[] = [];
@@ -147,6 +153,7 @@ function ormSchemaFromRecordSchema(
147
153
  const autoUuid: boolean = d && d == 'uuid()' ? true : false;
148
154
  const autoIncr: boolean = !autoUuid && d && d == 'autoincrement()' ? true : false;
149
155
  if (autoUuid || autoIncr) d = undefined;
156
+ else if (!isSqlDefault(d)) d = undefined;
150
157
  let genStrat: 'uuid' | 'increment' | undefined = undefined;
151
158
  if (autoIncr) genStrat = 'increment';
152
159
  else if (autoUuid) genStrat = 'uuid';
@@ -273,6 +280,7 @@ function entitySchemaToTable(scm: RecordSchema): TableSpec {
273
280
  const autoUuid: boolean = d && d == 'uuid()' ? true : false;
274
281
  const autoIncr: boolean = !autoUuid && d && d == 'autoincrement()' ? true : false;
275
282
  if (autoUuid || autoIncr) d = undefined;
283
+ else if (!isSqlDefault(d)) d = undefined;
276
284
  let genStrat: 'uuid' | 'increment' | 'rowid' | 'identity' = 'identity';
277
285
  if (autoIncr) genStrat = 'increment';
278
286
  else if (autoUuid) genStrat = 'uuid';
@@ -101,6 +101,10 @@ export class EmbeddingService {
101
101
  return await this.provider.embedText(query);
102
102
  }
103
103
 
104
+ async embedTexts(texts: string[]): Promise<number[][]> {
105
+ return await this.provider.embedTexts(texts);
106
+ }
107
+
104
108
  private averageEmbeddings(embeddings: number[][]): number[] {
105
109
  if (embeddings.length === 0) return [];
106
110
  const dimension = embeddings[0].length;
@@ -402,6 +406,8 @@ export class SqlDbResolver extends Resolver {
402
406
  joinClauses: undefined,
403
407
  intoSpec: undefined,
404
408
  whereClauses: undefined,
409
+ limit: inst.limit,
410
+ offset: inst.offset,
405
411
  };
406
412
  const readOnlyAttrs = inst.record.getWriteOnlyAttributes();
407
413
  const rslt: any =
@@ -567,6 +573,8 @@ export class SqlDbResolver extends Resolver {
567
573
  joinClauses,
568
574
  whereClauses,
569
575
  intoSpec,
576
+ limit: inst.limit,
577
+ offset: inst.offset,
570
578
  };
571
579
  const rslt: any = await getManyByJoin(tableName, qspec, this.getDbContext(inst.getFqName()));
572
580
  return rslt;
@@ -0,0 +1,187 @@
1
+ import * as lancedb from '@lancedb/lancedb';
2
+ import { Schema, Field, Float32, Utf8, FixedSizeList } from 'apache-arrow';
3
+ import { logger } from '../../logger.js';
4
+ import { VectorStore, VectorRecord, SearchResult, VectorStoreConfig } from './types.js';
5
+
6
+ export class LanceDBVectorStore implements VectorStore {
7
+ private db: lancedb.Connection | null = null;
8
+ private table: lancedb.Table | null = null;
9
+ private config: VectorStoreConfig;
10
+
11
+ constructor(config: VectorStoreConfig) {
12
+ this.config = config;
13
+ }
14
+
15
+ async init(): Promise<void> {
16
+ try {
17
+ const dbPath = this.config.inMemory
18
+ ? 'memory://'
19
+ : this.config.dbname || `./data/vector-store/${this.config.moduleName}.lance`;
20
+
21
+ this.db = await lancedb.connect(dbPath);
22
+
23
+ const tableName = 'embeddings';
24
+ const tableNames = await this.db.tableNames();
25
+
26
+ if (tableNames.includes(tableName)) {
27
+ this.table = await this.db.openTable(tableName);
28
+ logger.info(`LanceDB table ${tableName} opened`);
29
+ } else {
30
+ const schema = new Schema([
31
+ new Field('id', new Utf8(), false),
32
+ new Field(
33
+ 'embedding',
34
+ new FixedSizeList(this.config.vectorDimension, new Field('item', new Float32())),
35
+ false
36
+ ),
37
+ new Field('tenantId', new Utf8(), true),
38
+ new Field('agentId', new Utf8(), true),
39
+ new Field('documentId', new Utf8(), true),
40
+ ]);
41
+
42
+ this.table = await this.db.createEmptyTable(tableName, schema);
43
+ logger.info(`LanceDB table ${tableName} created`);
44
+ }
45
+ } catch (error) {
46
+ logger.error('Failed to initialize LanceDB vector store:', error);
47
+ throw error;
48
+ }
49
+ }
50
+
51
+ async addEmbedding(record: VectorRecord): Promise<void> {
52
+ if (!this.table) {
53
+ throw new Error('Vector store not initialized. Call init() first.');
54
+ }
55
+
56
+ try {
57
+ await this.table.add([
58
+ {
59
+ id: record.id,
60
+ embedding: record.embedding,
61
+ tenantId: record.tenantId || null,
62
+ agentId: record.agentId || null,
63
+ documentId: record.documentId || null,
64
+ },
65
+ ]);
66
+ } catch (error) {
67
+ logger.error(`Failed to add embedding ${record.id}:`, error);
68
+ throw error;
69
+ }
70
+ }
71
+
72
+ async addEmbeddings(records: VectorRecord[]): Promise<void> {
73
+ if (!this.table) {
74
+ throw new Error('Vector store not initialized. Call init() first.');
75
+ }
76
+
77
+ try {
78
+ const data = records.map(record => ({
79
+ id: record.id,
80
+ embedding: record.embedding,
81
+ tenantId: record.tenantId || null,
82
+ agentId: record.agentId || null,
83
+ documentId: record.documentId || null,
84
+ }));
85
+
86
+ await this.table.add(data);
87
+ } catch (error) {
88
+ logger.error(`Failed to add ${records.length} embeddings:`, error);
89
+ throw error;
90
+ }
91
+ }
92
+
93
+ async search(
94
+ embedding: number[],
95
+ tenantId?: string,
96
+ agentId?: string,
97
+ limit: number = 10
98
+ ): Promise<SearchResult[]> {
99
+ if (!this.table) {
100
+ throw new Error('Vector store not initialized. Call init() first.');
101
+ }
102
+
103
+ try {
104
+ let query = this.table.vectorSearch(embedding).limit(limit);
105
+
106
+ // Build filter conditions for agent-level isolation
107
+ const filters: string[] = [];
108
+
109
+ if (tenantId) {
110
+ // Use parameterized filtering to prevent SQL injection
111
+ const escapedTenantId = tenantId.replace(/'/g, "''");
112
+ filters.push(`tenantId = '${escapedTenantId}'`);
113
+ }
114
+
115
+ if (agentId) {
116
+ // Add agent-level filtering for strict agent isolation
117
+ const escapedAgentId = agentId.replace(/'/g, "''");
118
+ filters.push(`agentId = '${escapedAgentId}'`);
119
+ }
120
+
121
+ if (filters.length > 0) {
122
+ query = query.where(filters.join(' AND '));
123
+ }
124
+
125
+ const results = await query.toArray();
126
+
127
+ return results.map((row: any) => ({
128
+ id: row.id,
129
+ distance: row._distance || 0,
130
+ tenantId: row.tenantId,
131
+ agentId: row.agentId,
132
+ documentId: row.documentId,
133
+ }));
134
+ } catch (error) {
135
+ logger.error('Failed to search embeddings:', error);
136
+ throw error;
137
+ }
138
+ }
139
+
140
+ async delete(id: string): Promise<void> {
141
+ if (!this.table) {
142
+ throw new Error('Vector store not initialized. Call init() first.');
143
+ }
144
+
145
+ try {
146
+ await this.table.delete(`id = '${id}'`);
147
+ } catch (error) {
148
+ logger.error(`Failed to delete embedding ${id}:`, error);
149
+ throw error;
150
+ }
151
+ }
152
+
153
+ async exists(id: string): Promise<boolean> {
154
+ if (!this.table) {
155
+ throw new Error('Vector store not initialized. Call init() first.');
156
+ }
157
+
158
+ try {
159
+ const results = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
160
+ return results.length > 0;
161
+ } catch (error) {
162
+ logger.error(`Failed to check existence of ${id}:`, error);
163
+ throw error;
164
+ }
165
+ }
166
+
167
+ async close(): Promise<void> {
168
+ try {
169
+ if (this.table) {
170
+ // LanceDB tables don't have explicit close method
171
+ this.table = null;
172
+ }
173
+ if (this.db) {
174
+ // Note: LanceDB connection doesn't have explicit close method
175
+ this.db = null;
176
+ }
177
+ logger.info('LanceDB vector store closed');
178
+ } catch (error) {
179
+ logger.error('Failed to close LanceDB vector store:', error);
180
+ throw error;
181
+ }
182
+ }
183
+ }
184
+
185
+ export function createLanceDBStore(config: VectorStoreConfig): VectorStore {
186
+ return new LanceDBVectorStore(config);
187
+ }