agentlang 0.9.6 → 0.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/out/cli/main.d.ts.map +1 -1
  2. package/out/cli/main.js +8 -3
  3. package/out/cli/main.js.map +1 -1
  4. package/out/extension/main.cjs +250 -250
  5. package/out/extension/main.cjs.map +2 -2
  6. package/out/language/generated/ast.js +1 -0
  7. package/out/language/generated/ast.js.map +1 -1
  8. package/out/language/main.cjs +2420 -776
  9. package/out/language/main.cjs.map +4 -4
  10. package/out/runtime/docs.d.ts.map +1 -1
  11. package/out/runtime/docs.js +109 -7
  12. package/out/runtime/docs.js.map +1 -1
  13. package/out/runtime/embeddings/chunker.d.ts +9 -0
  14. package/out/runtime/embeddings/chunker.d.ts.map +1 -0
  15. package/out/runtime/embeddings/chunker.js +41 -0
  16. package/out/runtime/embeddings/chunker.js.map +1 -0
  17. package/out/runtime/embeddings/index.d.ts +6 -0
  18. package/out/runtime/embeddings/index.d.ts.map +1 -0
  19. package/out/runtime/embeddings/index.js +6 -0
  20. package/out/runtime/embeddings/index.js.map +1 -0
  21. package/out/runtime/embeddings/openai.d.ts +15 -0
  22. package/out/runtime/embeddings/openai.d.ts.map +1 -0
  23. package/out/runtime/embeddings/openai.js +34 -0
  24. package/out/runtime/embeddings/openai.js.map +1 -0
  25. package/out/runtime/embeddings/provider.d.ts +20 -0
  26. package/out/runtime/embeddings/provider.d.ts.map +1 -0
  27. package/out/runtime/embeddings/provider.js +17 -0
  28. package/out/runtime/embeddings/provider.js.map +1 -0
  29. package/out/runtime/embeddings/registry.d.ts +3 -0
  30. package/out/runtime/embeddings/registry.d.ts.map +1 -0
  31. package/out/runtime/embeddings/registry.js +16 -0
  32. package/out/runtime/embeddings/registry.js.map +1 -0
  33. package/out/runtime/interpreter.d.ts.map +1 -1
  34. package/out/runtime/interpreter.js +8 -4
  35. package/out/runtime/interpreter.js.map +1 -1
  36. package/out/runtime/module.d.ts +5 -2
  37. package/out/runtime/module.d.ts.map +1 -1
  38. package/out/runtime/module.js +14 -6
  39. package/out/runtime/module.js.map +1 -1
  40. package/out/runtime/modules/ai.d.ts +2 -0
  41. package/out/runtime/modules/ai.d.ts.map +1 -1
  42. package/out/runtime/modules/ai.js +62 -16
  43. package/out/runtime/modules/ai.js.map +1 -1
  44. package/out/runtime/resolvers/sqldb/database.d.ts +1 -1
  45. package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
  46. package/out/runtime/resolvers/sqldb/database.js +127 -46
  47. package/out/runtime/resolvers/sqldb/database.js.map +1 -1
  48. package/out/runtime/resolvers/sqldb/impl.d.ts +21 -1
  49. package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
  50. package/out/runtime/resolvers/sqldb/impl.js +176 -45
  51. package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
  52. package/package.json +188 -185
  53. package/public/pdf.worker.mjs +65152 -0
  54. package/src/cli/main.ts +7 -2
  55. package/src/language/generated/ast.ts +1 -1
  56. package/src/runtime/docs.ts +120 -9
  57. package/src/runtime/embeddings/chunker.ts +50 -0
  58. package/src/runtime/embeddings/index.ts +5 -0
  59. package/src/runtime/embeddings/openai.ts +49 -0
  60. package/src/runtime/embeddings/provider.ts +37 -0
  61. package/src/runtime/embeddings/registry.ts +17 -0
  62. package/src/runtime/interpreter.ts +16 -12
  63. package/src/runtime/module.ts +48 -39
  64. package/src/runtime/modules/ai.ts +76 -19
  65. package/src/runtime/resolvers/sqldb/database.ts +133 -51
  66. package/src/runtime/resolvers/sqldb/impl.ts +235 -58
  67. package/out/setupClassic.d.ts +0 -98
  68. package/out/setupClassic.d.ts.map +0 -1
  69. package/out/setupClassic.js +0 -38
  70. package/out/setupClassic.js.map +0 -1
  71. package/out/setupCommon.d.ts +0 -2
  72. package/out/setupCommon.d.ts.map +0 -1
  73. package/out/setupCommon.js +0 -33
  74. package/out/setupCommon.js.map +0 -1
  75. package/out/setupExtended.d.ts +0 -40
  76. package/out/setupExtended.d.ts.map +0 -1
  77. package/out/setupExtended.js +0 -67
  78. package/out/setupExtended.js.map +0 -1
@@ -228,9 +228,10 @@ function makeSqliteDataSource(
228
228
  ): DataSource {
229
229
  const synchronize = needSync();
230
230
  //const runMigrations = isRuntimeMode_migration() || isRuntimeMode_undo_migration() || !synchronize;
231
- return new DataSource({
231
+ const dbPath = config?.dbname || mkDbName();
232
+ const ds = new DataSource({
232
233
  type: 'better-sqlite3',
233
- database: config?.dbname || mkDbName(),
234
+ database: dbPath,
234
235
  synchronize: synchronize,
235
236
  entities: entities,
236
237
  migrationsRun: false,
@@ -240,6 +241,25 @@ function makeSqliteDataSource(
240
241
  undefined: 'ignore',
241
242
  },
242
243
  });
244
+ const originalInit = ds.initialize.bind(ds);
245
+ ds.initialize = async () => {
246
+ const res = await originalInit();
247
+ try {
248
+ const { load } = await import('sqlite-vec');
249
+ const driver = ds.driver as any;
250
+ const db = driver.databaseConnection || driver.nativeDatabase;
251
+ if (db) {
252
+ load(db);
253
+ logger.info('sqlite-vec extension loaded successfully');
254
+ }
255
+ } catch (err: any) {
256
+ logger.warn(
257
+ `Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
258
+ );
259
+ }
260
+ return res;
261
+ };
262
+ return ds;
243
263
  }
244
264
 
245
265
  async function execMigrationSql(dataSource: DataSource, sql: string[]) {
@@ -371,9 +391,18 @@ export function isUsingSqljs(): boolean {
371
391
  return getDbType(AppConfig?.store) == 'sqljs';
372
392
  }
373
393
 
374
- export function isVectorStoreSupported(): boolean {
375
- // Only Postgres supports pgvector
376
- return getDbType(AppConfig?.store) === 'postgres';
394
+ export async function isVectorStoreSupported(): Promise<boolean> {
395
+ const dbType = getDbType(AppConfig?.store);
396
+ if (dbType === 'postgres') return true;
397
+ if (dbType === 'sqlite') {
398
+ try {
399
+ const sqliteVecModule = await import('sqlite-vec');
400
+ return !!sqliteVecModule;
401
+ } catch {
402
+ return false;
403
+ }
404
+ }
405
+ return false;
377
406
  }
378
407
 
379
408
  export async function initDatabase(config: DatabaseConfig | undefined) {
@@ -442,50 +471,69 @@ export async function addRowForFullTextSearch(
442
471
  vect: number[],
443
472
  ctx: DbContext
444
473
  ) {
445
- if (!isVectorStoreSupported()) return;
474
+ if (!(await isVectorStoreSupported())) return;
446
475
  try {
447
476
  const vecTableName = tableName + VectorSuffix;
448
477
  const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
449
478
  const tenantId = await ctx.getTenantId();
450
- const { default: pgvector } = await import('pgvector');
451
- await qb
452
- .insert()
453
- .into(vecTableName)
454
- .values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
455
- .execute();
479
+ const dbType = getDbType(AppConfig?.store);
480
+ if (dbType === 'postgres') {
481
+ const { default: pgvector } = await import('pgvector');
482
+ await qb
483
+ .insert()
484
+ .into(vecTableName)
485
+ .values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
486
+ .execute();
487
+ } else {
488
+ await qb
489
+ .insert()
490
+ .into(vecTableName)
491
+ .values([{ id: id, embedding: new Float32Array(vect) }])
492
+ .execute();
493
+ }
456
494
  } catch (err: any) {
457
495
  logger.error(`Failed to add row to vector store - ${err}`);
458
496
  }
459
497
  }
460
498
 
461
499
  export async function initVectorStore(tableNames: string[], ctx: DbContext) {
462
- if (!isVectorStoreSupported()) {
500
+ if (!(await isVectorStoreSupported())) {
463
501
  logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
464
502
  return;
465
503
  }
504
+ const dbType = getDbType(AppConfig?.store);
466
505
  let notInited = true;
467
- tableNames.forEach(async (vecTableName: string) => {
506
+ for (const vecTableName of tableNames) {
468
507
  const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
469
- if (notInited) {
470
- let failure = false;
471
- try {
472
- await vecRepo.query('CREATE EXTENSION IF NOT EXISTS vector');
473
- } catch (err: any) {
474
- logger.error(`Failed to initialize vector store - ${err}`);
475
- failure = true;
508
+ if (dbType === 'postgres') {
509
+ if (notInited) {
510
+ let failure = false;
511
+ try {
512
+ await vecRepo.query('CREATE EXTENSION IF NOT EXISTS vector');
513
+ } catch (err: any) {
514
+ logger.error(`Failed to initialize vector store - ${err}`);
515
+ failure = true;
516
+ }
517
+ if (failure) continue;
518
+ notInited = false;
476
519
  }
477
- if (failure) return;
478
- notInited = false;
520
+ await vecRepo.query(
521
+ `CREATE TABLE IF NOT EXISTS ${vecTableName} (
522
+ id varchar PRIMARY KEY,
523
+ embedding vector(${DefaultVectorDimension}),
524
+ ${TenantAttributeName} varchar,
525
+ __is_deleted__ boolean default false
526
+ )`
527
+ );
528
+ } else {
529
+ // sqlite-vec - vec0 doesn't support type declarations for metadata columns
530
+ await vecRepo.query(
531
+ `CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
532
+ id TEXT PRIMARY KEY,
533
+ embedding FLOAT[${DefaultVectorDimension}])`
534
+ );
479
535
  }
480
- await vecRepo.query(
481
- `CREATE TABLE IF NOT EXISTS ${vecTableName} (
482
- id varchar PRIMARY KEY,
483
- embedding vector(${DefaultVectorDimension}),
484
- ${TenantAttributeName} varchar,
485
- __is_deleted__ boolean default false
486
- )`
487
- );
488
- });
536
+ }
489
537
  }
490
538
 
491
539
  export async function vectorStoreSearch(
@@ -494,7 +542,7 @@ export async function vectorStoreSearch(
494
542
  limit: number,
495
543
  ctx: DbContext
496
544
  ): Promise<any> {
497
- if (!isVectorStoreSupported()) {
545
+ if (!(await isVectorStoreSupported())) {
498
546
  // Not supported on sqljs/sqlite
499
547
  return [];
500
548
  }
@@ -508,18 +556,31 @@ export async function vectorStoreSearch(
508
556
  }
509
557
  const vecTableName = tableName + VectorSuffix;
510
558
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
511
- const { default: pgvector } = await import('pgvector');
512
- let ownersJoinCond: string = '';
559
+ const dbType = getDbType(AppConfig?.store);
513
560
  const tenantId = await ctx.getTenantId();
561
+ let ownersJoinCond: string = '';
514
562
  if (!hasGlobalPerms) {
515
563
  const ot = ownersTable(tableName);
516
564
  ownersJoinCond = `inner join ${ot} on
517
- ${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
518
- and ${ot}.${TenantAttributeName} = '${tenantId}' and ${vecTableName}.${TenantAttributeName} = '${tenantId}'`;
565
+ ${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
566
+ and ${ot}.${TenantAttributeName} = '${tenantId}'`;
567
+ }
568
+ if (dbType === 'postgres') {
569
+ const { default: pgvector } = await import('pgvector');
570
+ const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
571
+ const args = pgvector.toSql(searchVec);
572
+ return await qb.query(sql, [args]);
573
+ } else {
574
+ // sqlite-vec - join with main table to filter by tenant
575
+ const alias = tableName.toLowerCase();
576
+ const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
577
+ INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
578
+ ${ownersJoinCond}
579
+ WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
580
+ LIMIT ${limit}`;
581
+ const args = new Float32Array(searchVec);
582
+ return await qb.query(sql, [args]);
519
583
  }
520
- const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
521
- const args = pgvector.toSql(searchVec);
522
- return await qb.query(sql, [args]);
523
584
  } catch (err: any) {
524
585
  logger.error(`Vector store search failed - ${err}`);
525
586
  return [];
@@ -531,16 +592,30 @@ export async function vectorStoreSearchEntryExists(
531
592
  id: string,
532
593
  ctx: DbContext
533
594
  ): Promise<boolean> {
534
- if (!isVectorStoreSupported()) return false;
595
+ if (!(await isVectorStoreSupported())) return false;
535
596
  try {
536
597
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
537
598
  const vecTableName = tableName + VectorSuffix;
599
+ const dbType = getDbType(AppConfig?.store);
538
600
  const tenantId = await ctx.getTenantId();
539
- const result: any[] = await qb.query(
540
- `select id from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
541
- [id]
542
- );
543
- return result !== null && result.length > 0;
601
+
602
+ if (dbType === 'postgres') {
603
+ const result: any[] = await qb.query(
604
+ `select id from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
605
+ [id]
606
+ );
607
+ return result !== null && result.length > 0;
608
+ } else {
609
+ // sqlite-vec - join with main table to verify tenant
610
+ const alias = tableName.toLowerCase();
611
+ const result: any[] = await qb.query(
612
+ `SELECT ${vecTableName}.id FROM ${vecTableName}
613
+ INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
614
+ WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
615
+ [id]
616
+ );
617
+ return result !== null && result.length > 0;
618
+ }
544
619
  } catch (err: any) {
545
620
  logger.error(`Vector store search failed - ${err}`);
546
621
  }
@@ -548,15 +623,22 @@ export async function vectorStoreSearchEntryExists(
548
623
  }
549
624
 
550
625
  export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
551
- if (!isVectorStoreSupported()) return;
626
+ if (!(await isVectorStoreSupported())) return;
552
627
  try {
553
628
  const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
554
629
  const vecTableName = tableName + VectorSuffix;
630
+ const dbType = getDbType(AppConfig?.store);
555
631
  const tenantId = await ctx.getTenantId();
556
- await qb.query(
557
- `delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
558
- [id]
559
- );
632
+
633
+ if (dbType === 'postgres') {
634
+ await qb.query(
635
+ `delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
636
+ [id]
637
+ );
638
+ } else {
639
+ // sqlite-vec - delete just by id (ownership verified by caller)
640
+ await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
641
+ }
560
642
  } catch (err: any) {
561
643
  logger.error(`Vector store delete failed - ${err}`);
562
644
  }
@@ -25,32 +25,96 @@ import {
25
25
  import { JoinInfo, Resolver, WhereClause } from '../interface.js';
26
26
  import { asColumnReference, asTableReference } from './dbutil.js';
27
27
  import {
28
- getMany,
29
- insertRow,
30
- updateRow,
31
- getAllConnected,
32
- startDbTransaction,
28
+ addRowForFullTextSearch,
33
29
  commitDbTransaction,
34
- rollbackDbTransaction,
35
- hardDeleteRow,
36
30
  DbContext,
37
- insertBetweenRow,
38
- addRowForFullTextSearch,
39
- vectorStoreSearch,
40
- vectorStoreSearchEntryExists,
41
31
  deleteFullTextSearchEntry,
32
+ getAllConnected,
33
+ getMany,
34
+ getManyByJoin,
35
+ hardDeleteRow,
36
+ insertBetweenRow,
37
+ insertRow,
38
+ isVectorStoreSupported,
42
39
  JoinClause,
43
40
  JoinOn,
44
41
  makeJoinOn,
45
- getManyByJoin,
46
42
  QuerySpec,
43
+ rollbackDbTransaction,
44
+ startDbTransaction,
45
+ updateRow,
46
+ vectorStoreSearch,
47
+ vectorStoreSearchEntryExists,
47
48
  } from './database.js';
48
49
  import { AggregateFunctionCall, Environment } from '../../interpreter.js';
49
- import { OpenAIEmbeddings } from '@langchain/openai';
50
- import { Embeddings } from '@langchain/core/embeddings';
51
- import { DeletedFlagAttributeName, ParentAttributeName, PathAttributeName } from '../../defs.js';
50
+ import {
51
+ DeletedFlagAttributeName,
52
+ ParentAttributeName,
53
+ PathAttributeName,
54
+ TenantAttributeName,
55
+ } from '../../defs.js';
52
56
  import { logger } from '../../logger.js';
53
57
  import { JoinSpec } from '../../../language/generated/ast.js';
58
+ import { EmbeddingProvider, EmbeddingProviderConfig } from '../../embeddings/provider.js';
59
+ import { embeddingProvider } from '../../embeddings/registry.js';
60
+ import { TextChunker } from '../../embeddings/chunker.js';
61
+
62
+ interface EmbeddingServiceConfig extends EmbeddingProviderConfig {
63
+ provider?: string;
64
+ }
65
+
66
+ export class EmbeddingService {
67
+ private provider: EmbeddingProvider;
68
+ private config: EmbeddingServiceConfig;
69
+ private chunker: TextChunker;
70
+
71
+ constructor(config?: EmbeddingServiceConfig) {
72
+ this.config = config || {};
73
+ const providerClass = embeddingProvider(this.config.provider || 'openai');
74
+ this.provider = new providerClass(this.config);
75
+ this.chunker = new TextChunker(this.getChunkSize(), this.getChunkOverlap());
76
+ }
77
+
78
+ private getChunkSize(): number {
79
+ return this.config.chunkSize || 1000;
80
+ }
81
+
82
+ private getChunkOverlap(): number {
83
+ return this.config.chunkOverlap || 200;
84
+ }
85
+
86
+ async embedText(text: string): Promise<number[]> {
87
+ const chunks = this.chunker.splitText(text);
88
+
89
+ if (chunks.length === 1) {
90
+ return await this.provider.embedText(chunks[0]);
91
+ }
92
+
93
+ const chunkEmbeddings = await Promise.all(
94
+ chunks.map((chunk: string) => this.provider.embedText(chunk))
95
+ );
96
+
97
+ return this.averageEmbeddings(chunkEmbeddings);
98
+ }
99
+
100
+ async embedQuery(query: string): Promise<number[]> {
101
+ return await this.provider.embedText(query);
102
+ }
103
+
104
+ private averageEmbeddings(embeddings: number[][]): number[] {
105
+ if (embeddings.length === 0) return [];
106
+ const dimension = embeddings[0].length;
107
+ const averaged = new Array(dimension).fill(0);
108
+
109
+ for (const embedding of embeddings) {
110
+ for (let i = 0; i < dimension; i++) {
111
+ averaged[i] += embedding[i];
112
+ }
113
+ }
114
+
115
+ return averaged.map((v: number) => v / embeddings.length);
116
+ }
117
+ }
54
118
 
55
119
  function maybeFindIdAttributeName(inst: Instance): string | undefined {
56
120
  const attrEntry: AttributeEntry | undefined = findIdAttribute(inst);
@@ -62,12 +126,18 @@ function maybeFindIdAttributeName(inst: Instance): string | undefined {
62
126
 
63
127
  export class SqlDbResolver extends Resolver {
64
128
  private txnId: string | undefined;
65
- private embeddings: Embeddings;
129
+ private _embeddingService: EmbeddingService | undefined;
66
130
 
67
131
  constructor(name: string) {
68
132
  super();
69
133
  this.name = name;
70
- this.embeddings = new OpenAIEmbeddings();
134
+ }
135
+
136
+ private get embeddingService(): EmbeddingService {
137
+ if (!this._embeddingService) {
138
+ this._embeddingService = new EmbeddingService();
139
+ }
140
+ return this._embeddingService;
71
141
  }
72
142
 
73
143
  public override getName(): string {
@@ -93,6 +163,45 @@ export class SqlDbResolver extends Resolver {
93
163
  return entryName;
94
164
  }
95
165
 
166
+ private extractTextForEmbedding(rowObj: object, searchAttributes: string[] | undefined): string {
167
+ const obj = rowObj as Record<string, any>;
168
+ const ftsAttrs =
169
+ !searchAttributes || searchAttributes.length === 0 || searchAttributes[0] === '*'
170
+ ? Object.keys(obj).filter(k => this.shouldIncludeAttribute(k))
171
+ : searchAttributes;
172
+
173
+ const parts: string[] = [];
174
+ for (const attr of ftsAttrs) {
175
+ const value = obj[attr];
176
+ if (value !== undefined && value !== null) {
177
+ parts.push(this.valueToString(value));
178
+ }
179
+ }
180
+
181
+ return parts.join(' ');
182
+ }
183
+
184
+ private shouldIncludeAttribute(key: string): boolean {
185
+ const excludedAttrs = [
186
+ PathAttributeName,
187
+ DeletedFlagAttributeName,
188
+ TenantAttributeName,
189
+ '__tenant__',
190
+ '__is_deleted__',
191
+ ];
192
+ return !excludedAttrs.includes(key);
193
+ }
194
+
195
+ private valueToString(value: any): string {
196
+ if (Array.isArray(value)) {
197
+ return value.join(' ');
198
+ }
199
+ if (typeof value === 'object' && value !== null) {
200
+ return JSON.stringify(value);
201
+ }
202
+ return String(value);
203
+ }
204
+
96
205
  private async insertInstance(inst: Instance, orUpdate = false): Promise<Instance> {
97
206
  const ctx = this.getDbContext(inst.getFqName());
98
207
  if (isBetweenRelationship(inst.name, inst.moduleName)) {
@@ -107,35 +216,43 @@ export class SqlDbResolver extends Resolver {
107
216
  ctx.activeEnv.isInDeleteMode()
108
217
  );
109
218
  return inst;
110
- } else {
111
- const idAttrName: string | undefined = maybeFindIdAttributeName(inst);
112
- ensureOneToOneAttributes(inst);
113
- const attrs: InstanceAttributes = inst.attributes;
114
- const idAttrVal: any = idAttrName ? attrs.get(idAttrName) : crypto.randomUUID();
115
- if (idAttrVal !== undefined) {
116
- const pp: string | undefined = attrs.get(PathAttributeName);
117
- const n: string = `${inst.moduleName}/${inst.name}`;
118
- let p: string = '';
119
- if (pp !== undefined) p = `${pp}/${escapeFqName(n)}/${idAttrVal}`;
120
- else p = `${n.replace('/', '$')}/${idAttrVal}`;
121
- attrs.set(PathAttributeName, p);
122
- }
123
- const n: string = asTableReference(inst.moduleName, inst.name);
124
- const rowObj: object = inst.attributesWithStringifiedObjects();
125
- await insertRow(n, rowObj, ctx, orUpdate);
126
- if (inst.record.getFullTextSearchAttributes()) {
127
- const path = attrs.get(PathAttributeName);
128
- try {
129
- if (!(await vectorStoreSearchEntryExists(n, path, ctx))) {
130
- const res = await this.embeddings.embedQuery(JSON.stringify(rowObj));
131
- await addRowForFullTextSearch(n, path, res, ctx);
132
- }
133
- } catch (reason: any) {
134
- logger.warn(`Full text indexing failed for ${path} - ${reason}`);
219
+ }
220
+ const idAttrName: string | undefined = maybeFindIdAttributeName(inst);
221
+ ensureOneToOneAttributes(inst);
222
+ const attrs: InstanceAttributes = inst.attributes;
223
+ const idAttrVal: any = idAttrName ? attrs.get(idAttrName) : crypto.randomUUID();
224
+ if (idAttrVal !== undefined) {
225
+ const pp: string | undefined = attrs.get(PathAttributeName);
226
+ const n: string = `${inst.moduleName}/${inst.name}`;
227
+ let p: string = '';
228
+ if (pp !== undefined) p = `${pp}/${escapeFqName(n)}/${idAttrVal}`;
229
+ else p = `${n.replace('/', '$')}/${idAttrVal}`;
230
+ attrs.set(PathAttributeName, p);
231
+ }
232
+ const n: string = asTableReference(inst.moduleName, inst.name);
233
+ const rowObj: object = inst.attributesWithStringifiedObjects();
234
+ await insertRow(n, rowObj, ctx, orUpdate);
235
+ if (inst.record.getEmbeddingConfig() || inst.record.getFullTextSearchAttributes()) {
236
+ const path = attrs.get(PathAttributeName);
237
+ try {
238
+ if (
239
+ (await isVectorStoreSupported()) &&
240
+ !(await vectorStoreSearchEntryExists(n, path, ctx))
241
+ ) {
242
+ const ftsAttrs = inst.record.getFullTextSearchAttributes() || ['*'];
243
+ const textToEmbed = this.extractTextForEmbedding(rowObj, ftsAttrs);
244
+ const embeddingConfig = inst.record.getEmbeddingConfig();
245
+ const embeddingService = embeddingConfig
246
+ ? new EmbeddingService(embeddingConfig)
247
+ : this.embeddingService;
248
+ const res = await embeddingService.embedText(textToEmbed);
249
+ await addRowForFullTextSearch(n, path, res, ctx);
135
250
  }
251
+ } catch (reason: any) {
252
+ logger.warn(`Full text indexing failed for ${path} - ${reason}`);
136
253
  }
137
- return inst;
138
254
  }
255
+ return inst;
139
256
  }
140
257
 
141
258
  public override async createInstance(inst: Instance): Promise<Instance> {
@@ -211,9 +328,42 @@ export class SqlDbResolver extends Resolver {
211
328
  : undefined;
212
329
  const orderByDesc = inst.orderByDesc ? 'DESC' : 'ASC';
213
330
  const aggregates = SqlDbResolver.normalizedAggregates(inst, tableName);
331
+
332
+ let vectorResult: Instance[] | undefined;
333
+ const embeddingConfig = inst.record.getEmbeddingConfig();
334
+ const ftsAttrs = inst.record.getFullTextSearchAttributes();
335
+ if (
336
+ (await isVectorStoreSupported()) &&
337
+ embeddingConfig &&
338
+ qattrs &&
339
+ (ftsAttrs || Object.keys(qattrs).some(k => k.endsWith('?')))
340
+ ) {
341
+ const vectorSearchAttr = Object.keys(qattrs).find(k => k.endsWith('?'));
342
+ if (vectorSearchAttr) {
343
+ const queryVal = qvals[vectorSearchAttr];
344
+ const searchString = this.valueToString(queryVal);
345
+ const embeddingService = new EmbeddingService(embeddingConfig);
346
+ const queryVec = await embeddingService.embedQuery(searchString);
347
+ const rslt: any = await vectorStoreSearch(tableName, queryVec, 10, ctx);
348
+ if (rslt instanceof Array) {
349
+ vectorResult = new Array<Instance>();
350
+ rslt.forEach((r: any) => {
351
+ const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
352
+ tableName,
353
+ new Map(Object.entries(r))
354
+ );
355
+ attrs.delete(DeletedFlagAttributeName);
356
+ vectorResult!.push(Instance.newWithAttributes(inst, attrs));
357
+ });
358
+ }
359
+ delete qattrs[vectorSearchAttr];
360
+ delete qvals[vectorSearchAttr];
361
+ }
362
+ }
363
+
214
364
  const qspec: QuerySpec = {
215
- queryObj: qattrs,
216
- queryVals: qvals,
365
+ queryObj: Object.keys(qattrs || {}).length > 0 ? qattrs : undefined,
366
+ queryVals: Object.keys(qvals || {}).length > 0 ? qvals : undefined,
217
367
  distinct,
218
368
  groupBy,
219
369
  orderBy,
@@ -224,20 +374,47 @@ export class SqlDbResolver extends Resolver {
224
374
  whereClauses: undefined,
225
375
  };
226
376
  const readOnlyAttrs = inst.record.getWriteOnlyAttributes();
227
- const rslt: any = await getMany(tableName, qspec, ctx);
377
+ const rslt: any =
378
+ vectorResult !== undefined && qspec.queryObj === undefined
379
+ ? vectorResult
380
+ : await getMany(tableName, qspec, ctx);
228
381
  if (rslt instanceof Array) {
229
- result = new Array<Instance>();
230
- rslt.forEach((r: object) => {
231
- const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
232
- tableName,
233
- new Map(Object.entries(r))
234
- );
235
- attrs.delete(DeletedFlagAttributeName);
236
- readOnlyAttrs?.forEach((n: string) => {
237
- attrs.delete(n);
382
+ if (vectorResult !== undefined && qspec.queryObj !== undefined) {
383
+ // Merge results if both vector and standard queries were performed
384
+ const vectorPaths = new Set(vectorResult.map(i => i.lookup(PathAttributeName)));
385
+ result = new Array<Instance>();
386
+ rslt.forEach((r: any) => {
387
+ const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
388
+ tableName,
389
+ new Map(Object.entries(r))
390
+ );
391
+ if (vectorPaths.has(attrs.get(PathAttributeName))) {
392
+ attrs.delete(DeletedFlagAttributeName);
393
+ readOnlyAttrs?.forEach((n: string) => {
394
+ attrs.delete(n);
395
+ });
396
+ result.push(Instance.newWithAttributes(inst, attrs));
397
+ }
238
398
  });
239
- result.push(Instance.newWithAttributes(inst, attrs));
240
- });
399
+ } else {
400
+ result =
401
+ vectorResult !== undefined && qspec.queryObj === undefined
402
+ ? vectorResult
403
+ : new Array<Instance>();
404
+ if (vectorResult === undefined || qspec.queryObj !== undefined) {
405
+ rslt.forEach((r: any) => {
406
+ const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
407
+ tableName,
408
+ new Map(Object.entries(r))
409
+ );
410
+ attrs.delete(DeletedFlagAttributeName);
411
+ readOnlyAttrs?.forEach((n: string) => {
412
+ attrs.delete(n);
413
+ });
414
+ result.push(Instance.newWithAttributes(inst, attrs));
415
+ });
416
+ }
417
+ }
241
418
  }
242
419
  return result;
243
420
  }
@@ -555,7 +732,7 @@ export class SqlDbResolver extends Resolver {
555
732
  query: string,
556
733
  options?: any
557
734
  ): Promise<any> {
558
- const queryVec = await this.embeddings.embedQuery(query);
735
+ const queryVec = await this.embeddingService.embedQuery(query);
559
736
  const ctx = this.getDbContext(makeFqName(moduleName, entryName));
560
737
  let limit = 5;
561
738
  if (options && options.limit) {