agentlang 0.9.6 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/cli/main.d.ts.map +1 -1
- package/out/cli/main.js +8 -3
- package/out/cli/main.js.map +1 -1
- package/out/extension/main.cjs +250 -250
- package/out/extension/main.cjs.map +2 -2
- package/out/language/generated/ast.js +1 -0
- package/out/language/generated/ast.js.map +1 -1
- package/out/language/main.cjs +2420 -776
- package/out/language/main.cjs.map +4 -4
- package/out/runtime/docs.d.ts.map +1 -1
- package/out/runtime/docs.js +109 -7
- package/out/runtime/docs.js.map +1 -1
- package/out/runtime/embeddings/chunker.d.ts +9 -0
- package/out/runtime/embeddings/chunker.d.ts.map +1 -0
- package/out/runtime/embeddings/chunker.js +41 -0
- package/out/runtime/embeddings/chunker.js.map +1 -0
- package/out/runtime/embeddings/index.d.ts +6 -0
- package/out/runtime/embeddings/index.d.ts.map +1 -0
- package/out/runtime/embeddings/index.js +6 -0
- package/out/runtime/embeddings/index.js.map +1 -0
- package/out/runtime/embeddings/openai.d.ts +15 -0
- package/out/runtime/embeddings/openai.d.ts.map +1 -0
- package/out/runtime/embeddings/openai.js +34 -0
- package/out/runtime/embeddings/openai.js.map +1 -0
- package/out/runtime/embeddings/provider.d.ts +20 -0
- package/out/runtime/embeddings/provider.d.ts.map +1 -0
- package/out/runtime/embeddings/provider.js +17 -0
- package/out/runtime/embeddings/provider.js.map +1 -0
- package/out/runtime/embeddings/registry.d.ts +3 -0
- package/out/runtime/embeddings/registry.d.ts.map +1 -0
- package/out/runtime/embeddings/registry.js +16 -0
- package/out/runtime/embeddings/registry.js.map +1 -0
- package/out/runtime/interpreter.d.ts.map +1 -1
- package/out/runtime/interpreter.js +8 -4
- package/out/runtime/interpreter.js.map +1 -1
- package/out/runtime/module.d.ts +5 -2
- package/out/runtime/module.d.ts.map +1 -1
- package/out/runtime/module.js +14 -6
- package/out/runtime/module.js.map +1 -1
- package/out/runtime/modules/ai.d.ts +2 -0
- package/out/runtime/modules/ai.d.ts.map +1 -1
- package/out/runtime/modules/ai.js +62 -16
- package/out/runtime/modules/ai.js.map +1 -1
- package/out/runtime/resolvers/sqldb/database.d.ts +1 -1
- package/out/runtime/resolvers/sqldb/database.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/database.js +127 -46
- package/out/runtime/resolvers/sqldb/database.js.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts +21 -1
- package/out/runtime/resolvers/sqldb/impl.d.ts.map +1 -1
- package/out/runtime/resolvers/sqldb/impl.js +176 -45
- package/out/runtime/resolvers/sqldb/impl.js.map +1 -1
- package/package.json +188 -185
- package/public/pdf.worker.mjs +65152 -0
- package/src/cli/main.ts +7 -2
- package/src/language/generated/ast.ts +1 -1
- package/src/runtime/docs.ts +120 -9
- package/src/runtime/embeddings/chunker.ts +50 -0
- package/src/runtime/embeddings/index.ts +5 -0
- package/src/runtime/embeddings/openai.ts +49 -0
- package/src/runtime/embeddings/provider.ts +37 -0
- package/src/runtime/embeddings/registry.ts +17 -0
- package/src/runtime/interpreter.ts +16 -12
- package/src/runtime/module.ts +48 -39
- package/src/runtime/modules/ai.ts +76 -19
- package/src/runtime/resolvers/sqldb/database.ts +133 -51
- package/src/runtime/resolvers/sqldb/impl.ts +235 -58
- package/out/setupClassic.d.ts +0 -98
- package/out/setupClassic.d.ts.map +0 -1
- package/out/setupClassic.js +0 -38
- package/out/setupClassic.js.map +0 -1
- package/out/setupCommon.d.ts +0 -2
- package/out/setupCommon.d.ts.map +0 -1
- package/out/setupCommon.js +0 -33
- package/out/setupCommon.js.map +0 -1
- package/out/setupExtended.d.ts +0 -40
- package/out/setupExtended.d.ts.map +0 -1
- package/out/setupExtended.js +0 -67
- package/out/setupExtended.js.map +0 -1
|
@@ -228,9 +228,10 @@ function makeSqliteDataSource(
|
|
|
228
228
|
): DataSource {
|
|
229
229
|
const synchronize = needSync();
|
|
230
230
|
//const runMigrations = isRuntimeMode_migration() || isRuntimeMode_undo_migration() || !synchronize;
|
|
231
|
-
|
|
231
|
+
const dbPath = config?.dbname || mkDbName();
|
|
232
|
+
const ds = new DataSource({
|
|
232
233
|
type: 'better-sqlite3',
|
|
233
|
-
database:
|
|
234
|
+
database: dbPath,
|
|
234
235
|
synchronize: synchronize,
|
|
235
236
|
entities: entities,
|
|
236
237
|
migrationsRun: false,
|
|
@@ -240,6 +241,25 @@ function makeSqliteDataSource(
|
|
|
240
241
|
undefined: 'ignore',
|
|
241
242
|
},
|
|
242
243
|
});
|
|
244
|
+
const originalInit = ds.initialize.bind(ds);
|
|
245
|
+
ds.initialize = async () => {
|
|
246
|
+
const res = await originalInit();
|
|
247
|
+
try {
|
|
248
|
+
const { load } = await import('sqlite-vec');
|
|
249
|
+
const driver = ds.driver as any;
|
|
250
|
+
const db = driver.databaseConnection || driver.nativeDatabase;
|
|
251
|
+
if (db) {
|
|
252
|
+
load(db);
|
|
253
|
+
logger.info('sqlite-vec extension loaded successfully');
|
|
254
|
+
}
|
|
255
|
+
} catch (err: any) {
|
|
256
|
+
logger.warn(
|
|
257
|
+
`Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
return res;
|
|
261
|
+
};
|
|
262
|
+
return ds;
|
|
243
263
|
}
|
|
244
264
|
|
|
245
265
|
async function execMigrationSql(dataSource: DataSource, sql: string[]) {
|
|
@@ -371,9 +391,18 @@ export function isUsingSqljs(): boolean {
|
|
|
371
391
|
return getDbType(AppConfig?.store) == 'sqljs';
|
|
372
392
|
}
|
|
373
393
|
|
|
374
|
-
export function isVectorStoreSupported(): boolean {
|
|
375
|
-
|
|
376
|
-
|
|
394
|
+
export async function isVectorStoreSupported(): Promise<boolean> {
|
|
395
|
+
const dbType = getDbType(AppConfig?.store);
|
|
396
|
+
if (dbType === 'postgres') return true;
|
|
397
|
+
if (dbType === 'sqlite') {
|
|
398
|
+
try {
|
|
399
|
+
const sqliteVecModule = await import('sqlite-vec');
|
|
400
|
+
return !!sqliteVecModule;
|
|
401
|
+
} catch {
|
|
402
|
+
return false;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return false;
|
|
377
406
|
}
|
|
378
407
|
|
|
379
408
|
export async function initDatabase(config: DatabaseConfig | undefined) {
|
|
@@ -442,50 +471,69 @@ export async function addRowForFullTextSearch(
|
|
|
442
471
|
vect: number[],
|
|
443
472
|
ctx: DbContext
|
|
444
473
|
) {
|
|
445
|
-
if (!isVectorStoreSupported()) return;
|
|
474
|
+
if (!(await isVectorStoreSupported())) return;
|
|
446
475
|
try {
|
|
447
476
|
const vecTableName = tableName + VectorSuffix;
|
|
448
477
|
const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
|
|
449
478
|
const tenantId = await ctx.getTenantId();
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
479
|
+
const dbType = getDbType(AppConfig?.store);
|
|
480
|
+
if (dbType === 'postgres') {
|
|
481
|
+
const { default: pgvector } = await import('pgvector');
|
|
482
|
+
await qb
|
|
483
|
+
.insert()
|
|
484
|
+
.into(vecTableName)
|
|
485
|
+
.values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
|
|
486
|
+
.execute();
|
|
487
|
+
} else {
|
|
488
|
+
await qb
|
|
489
|
+
.insert()
|
|
490
|
+
.into(vecTableName)
|
|
491
|
+
.values([{ id: id, embedding: new Float32Array(vect) }])
|
|
492
|
+
.execute();
|
|
493
|
+
}
|
|
456
494
|
} catch (err: any) {
|
|
457
495
|
logger.error(`Failed to add row to vector store - ${err}`);
|
|
458
496
|
}
|
|
459
497
|
}
|
|
460
498
|
|
|
461
499
|
export async function initVectorStore(tableNames: string[], ctx: DbContext) {
|
|
462
|
-
if (!isVectorStoreSupported()) {
|
|
500
|
+
if (!(await isVectorStoreSupported())) {
|
|
463
501
|
logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
|
|
464
502
|
return;
|
|
465
503
|
}
|
|
504
|
+
const dbType = getDbType(AppConfig?.store);
|
|
466
505
|
let notInited = true;
|
|
467
|
-
|
|
506
|
+
for (const vecTableName of tableNames) {
|
|
468
507
|
const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
|
|
469
|
-
if (
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
508
|
+
if (dbType === 'postgres') {
|
|
509
|
+
if (notInited) {
|
|
510
|
+
let failure = false;
|
|
511
|
+
try {
|
|
512
|
+
await vecRepo.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
513
|
+
} catch (err: any) {
|
|
514
|
+
logger.error(`Failed to initialize vector store - ${err}`);
|
|
515
|
+
failure = true;
|
|
516
|
+
}
|
|
517
|
+
if (failure) continue;
|
|
518
|
+
notInited = false;
|
|
476
519
|
}
|
|
477
|
-
|
|
478
|
-
|
|
520
|
+
await vecRepo.query(
|
|
521
|
+
`CREATE TABLE IF NOT EXISTS ${vecTableName} (
|
|
522
|
+
id varchar PRIMARY KEY,
|
|
523
|
+
embedding vector(${DefaultVectorDimension}),
|
|
524
|
+
${TenantAttributeName} varchar,
|
|
525
|
+
__is_deleted__ boolean default false
|
|
526
|
+
)`
|
|
527
|
+
);
|
|
528
|
+
} else {
|
|
529
|
+
// sqlite-vec - vec0 doesn't support type declarations for metadata columns
|
|
530
|
+
await vecRepo.query(
|
|
531
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
|
|
532
|
+
id TEXT PRIMARY KEY,
|
|
533
|
+
embedding FLOAT[${DefaultVectorDimension}])`
|
|
534
|
+
);
|
|
479
535
|
}
|
|
480
|
-
|
|
481
|
-
`CREATE TABLE IF NOT EXISTS ${vecTableName} (
|
|
482
|
-
id varchar PRIMARY KEY,
|
|
483
|
-
embedding vector(${DefaultVectorDimension}),
|
|
484
|
-
${TenantAttributeName} varchar,
|
|
485
|
-
__is_deleted__ boolean default false
|
|
486
|
-
)`
|
|
487
|
-
);
|
|
488
|
-
});
|
|
536
|
+
}
|
|
489
537
|
}
|
|
490
538
|
|
|
491
539
|
export async function vectorStoreSearch(
|
|
@@ -494,7 +542,7 @@ export async function vectorStoreSearch(
|
|
|
494
542
|
limit: number,
|
|
495
543
|
ctx: DbContext
|
|
496
544
|
): Promise<any> {
|
|
497
|
-
if (!isVectorStoreSupported()) {
|
|
545
|
+
if (!(await isVectorStoreSupported())) {
|
|
498
546
|
// Not supported on sqljs/sqlite
|
|
499
547
|
return [];
|
|
500
548
|
}
|
|
@@ -508,18 +556,31 @@ export async function vectorStoreSearch(
|
|
|
508
556
|
}
|
|
509
557
|
const vecTableName = tableName + VectorSuffix;
|
|
510
558
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
511
|
-
const
|
|
512
|
-
let ownersJoinCond: string = '';
|
|
559
|
+
const dbType = getDbType(AppConfig?.store);
|
|
513
560
|
const tenantId = await ctx.getTenantId();
|
|
561
|
+
let ownersJoinCond: string = '';
|
|
514
562
|
if (!hasGlobalPerms) {
|
|
515
563
|
const ot = ownersTable(tableName);
|
|
516
564
|
ownersJoinCond = `inner join ${ot} on
|
|
517
|
-
${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
|
|
518
|
-
and ${ot}.${TenantAttributeName} = '${tenantId}'
|
|
565
|
+
${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
|
|
566
|
+
and ${ot}.${TenantAttributeName} = '${tenantId}'`;
|
|
567
|
+
}
|
|
568
|
+
if (dbType === 'postgres') {
|
|
569
|
+
const { default: pgvector } = await import('pgvector');
|
|
570
|
+
const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
|
|
571
|
+
const args = pgvector.toSql(searchVec);
|
|
572
|
+
return await qb.query(sql, [args]);
|
|
573
|
+
} else {
|
|
574
|
+
// sqlite-vec - join with main table to filter by tenant
|
|
575
|
+
const alias = tableName.toLowerCase();
|
|
576
|
+
const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
577
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
578
|
+
${ownersJoinCond}
|
|
579
|
+
WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
|
|
580
|
+
LIMIT ${limit}`;
|
|
581
|
+
const args = new Float32Array(searchVec);
|
|
582
|
+
return await qb.query(sql, [args]);
|
|
519
583
|
}
|
|
520
|
-
const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
|
|
521
|
-
const args = pgvector.toSql(searchVec);
|
|
522
|
-
return await qb.query(sql, [args]);
|
|
523
584
|
} catch (err: any) {
|
|
524
585
|
logger.error(`Vector store search failed - ${err}`);
|
|
525
586
|
return [];
|
|
@@ -531,16 +592,30 @@ export async function vectorStoreSearchEntryExists(
|
|
|
531
592
|
id: string,
|
|
532
593
|
ctx: DbContext
|
|
533
594
|
): Promise<boolean> {
|
|
534
|
-
if (!isVectorStoreSupported()) return false;
|
|
595
|
+
if (!(await isVectorStoreSupported())) return false;
|
|
535
596
|
try {
|
|
536
597
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
537
598
|
const vecTableName = tableName + VectorSuffix;
|
|
599
|
+
const dbType = getDbType(AppConfig?.store);
|
|
538
600
|
const tenantId = await ctx.getTenantId();
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
[
|
|
542
|
-
|
|
543
|
-
|
|
601
|
+
|
|
602
|
+
if (dbType === 'postgres') {
|
|
603
|
+
const result: any[] = await qb.query(
|
|
604
|
+
`select id from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
|
|
605
|
+
[id]
|
|
606
|
+
);
|
|
607
|
+
return result !== null && result.length > 0;
|
|
608
|
+
} else {
|
|
609
|
+
// sqlite-vec - join with main table to verify tenant
|
|
610
|
+
const alias = tableName.toLowerCase();
|
|
611
|
+
const result: any[] = await qb.query(
|
|
612
|
+
`SELECT ${vecTableName}.id FROM ${vecTableName}
|
|
613
|
+
INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
|
|
614
|
+
WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
|
|
615
|
+
[id]
|
|
616
|
+
);
|
|
617
|
+
return result !== null && result.length > 0;
|
|
618
|
+
}
|
|
544
619
|
} catch (err: any) {
|
|
545
620
|
logger.error(`Vector store search failed - ${err}`);
|
|
546
621
|
}
|
|
@@ -548,15 +623,22 @@ export async function vectorStoreSearchEntryExists(
|
|
|
548
623
|
}
|
|
549
624
|
|
|
550
625
|
export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
|
|
551
|
-
if (!isVectorStoreSupported()) return;
|
|
626
|
+
if (!(await isVectorStoreSupported())) return;
|
|
552
627
|
try {
|
|
553
628
|
const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
|
|
554
629
|
const vecTableName = tableName + VectorSuffix;
|
|
630
|
+
const dbType = getDbType(AppConfig?.store);
|
|
555
631
|
const tenantId = await ctx.getTenantId();
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
632
|
+
|
|
633
|
+
if (dbType === 'postgres') {
|
|
634
|
+
await qb.query(
|
|
635
|
+
`delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
|
|
636
|
+
[id]
|
|
637
|
+
);
|
|
638
|
+
} else {
|
|
639
|
+
// sqlite-vec - delete just by id (ownership verified by caller)
|
|
640
|
+
await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
|
|
641
|
+
}
|
|
560
642
|
} catch (err: any) {
|
|
561
643
|
logger.error(`Vector store delete failed - ${err}`);
|
|
562
644
|
}
|
|
@@ -25,32 +25,96 @@ import {
|
|
|
25
25
|
import { JoinInfo, Resolver, WhereClause } from '../interface.js';
|
|
26
26
|
import { asColumnReference, asTableReference } from './dbutil.js';
|
|
27
27
|
import {
|
|
28
|
-
|
|
29
|
-
insertRow,
|
|
30
|
-
updateRow,
|
|
31
|
-
getAllConnected,
|
|
32
|
-
startDbTransaction,
|
|
28
|
+
addRowForFullTextSearch,
|
|
33
29
|
commitDbTransaction,
|
|
34
|
-
rollbackDbTransaction,
|
|
35
|
-
hardDeleteRow,
|
|
36
30
|
DbContext,
|
|
37
|
-
insertBetweenRow,
|
|
38
|
-
addRowForFullTextSearch,
|
|
39
|
-
vectorStoreSearch,
|
|
40
|
-
vectorStoreSearchEntryExists,
|
|
41
31
|
deleteFullTextSearchEntry,
|
|
32
|
+
getAllConnected,
|
|
33
|
+
getMany,
|
|
34
|
+
getManyByJoin,
|
|
35
|
+
hardDeleteRow,
|
|
36
|
+
insertBetweenRow,
|
|
37
|
+
insertRow,
|
|
38
|
+
isVectorStoreSupported,
|
|
42
39
|
JoinClause,
|
|
43
40
|
JoinOn,
|
|
44
41
|
makeJoinOn,
|
|
45
|
-
getManyByJoin,
|
|
46
42
|
QuerySpec,
|
|
43
|
+
rollbackDbTransaction,
|
|
44
|
+
startDbTransaction,
|
|
45
|
+
updateRow,
|
|
46
|
+
vectorStoreSearch,
|
|
47
|
+
vectorStoreSearchEntryExists,
|
|
47
48
|
} from './database.js';
|
|
48
49
|
import { AggregateFunctionCall, Environment } from '../../interpreter.js';
|
|
49
|
-
import {
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
import {
|
|
51
|
+
DeletedFlagAttributeName,
|
|
52
|
+
ParentAttributeName,
|
|
53
|
+
PathAttributeName,
|
|
54
|
+
TenantAttributeName,
|
|
55
|
+
} from '../../defs.js';
|
|
52
56
|
import { logger } from '../../logger.js';
|
|
53
57
|
import { JoinSpec } from '../../../language/generated/ast.js';
|
|
58
|
+
import { EmbeddingProvider, EmbeddingProviderConfig } from '../../embeddings/provider.js';
|
|
59
|
+
import { embeddingProvider } from '../../embeddings/registry.js';
|
|
60
|
+
import { TextChunker } from '../../embeddings/chunker.js';
|
|
61
|
+
|
|
62
|
+
interface EmbeddingServiceConfig extends EmbeddingProviderConfig {
|
|
63
|
+
provider?: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export class EmbeddingService {
|
|
67
|
+
private provider: EmbeddingProvider;
|
|
68
|
+
private config: EmbeddingServiceConfig;
|
|
69
|
+
private chunker: TextChunker;
|
|
70
|
+
|
|
71
|
+
constructor(config?: EmbeddingServiceConfig) {
|
|
72
|
+
this.config = config || {};
|
|
73
|
+
const providerClass = embeddingProvider(this.config.provider || 'openai');
|
|
74
|
+
this.provider = new providerClass(this.config);
|
|
75
|
+
this.chunker = new TextChunker(this.getChunkSize(), this.getChunkOverlap());
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private getChunkSize(): number {
|
|
79
|
+
return this.config.chunkSize || 1000;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
private getChunkOverlap(): number {
|
|
83
|
+
return this.config.chunkOverlap || 200;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async embedText(text: string): Promise<number[]> {
|
|
87
|
+
const chunks = this.chunker.splitText(text);
|
|
88
|
+
|
|
89
|
+
if (chunks.length === 1) {
|
|
90
|
+
return await this.provider.embedText(chunks[0]);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const chunkEmbeddings = await Promise.all(
|
|
94
|
+
chunks.map((chunk: string) => this.provider.embedText(chunk))
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
return this.averageEmbeddings(chunkEmbeddings);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async embedQuery(query: string): Promise<number[]> {
|
|
101
|
+
return await this.provider.embedText(query);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
private averageEmbeddings(embeddings: number[][]): number[] {
|
|
105
|
+
if (embeddings.length === 0) return [];
|
|
106
|
+
const dimension = embeddings[0].length;
|
|
107
|
+
const averaged = new Array(dimension).fill(0);
|
|
108
|
+
|
|
109
|
+
for (const embedding of embeddings) {
|
|
110
|
+
for (let i = 0; i < dimension; i++) {
|
|
111
|
+
averaged[i] += embedding[i];
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return averaged.map((v: number) => v / embeddings.length);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
54
118
|
|
|
55
119
|
function maybeFindIdAttributeName(inst: Instance): string | undefined {
|
|
56
120
|
const attrEntry: AttributeEntry | undefined = findIdAttribute(inst);
|
|
@@ -62,12 +126,18 @@ function maybeFindIdAttributeName(inst: Instance): string | undefined {
|
|
|
62
126
|
|
|
63
127
|
export class SqlDbResolver extends Resolver {
|
|
64
128
|
private txnId: string | undefined;
|
|
65
|
-
private
|
|
129
|
+
private _embeddingService: EmbeddingService | undefined;
|
|
66
130
|
|
|
67
131
|
constructor(name: string) {
|
|
68
132
|
super();
|
|
69
133
|
this.name = name;
|
|
70
|
-
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
private get embeddingService(): EmbeddingService {
|
|
137
|
+
if (!this._embeddingService) {
|
|
138
|
+
this._embeddingService = new EmbeddingService();
|
|
139
|
+
}
|
|
140
|
+
return this._embeddingService;
|
|
71
141
|
}
|
|
72
142
|
|
|
73
143
|
public override getName(): string {
|
|
@@ -93,6 +163,45 @@ export class SqlDbResolver extends Resolver {
|
|
|
93
163
|
return entryName;
|
|
94
164
|
}
|
|
95
165
|
|
|
166
|
+
private extractTextForEmbedding(rowObj: object, searchAttributes: string[] | undefined): string {
|
|
167
|
+
const obj = rowObj as Record<string, any>;
|
|
168
|
+
const ftsAttrs =
|
|
169
|
+
!searchAttributes || searchAttributes.length === 0 || searchAttributes[0] === '*'
|
|
170
|
+
? Object.keys(obj).filter(k => this.shouldIncludeAttribute(k))
|
|
171
|
+
: searchAttributes;
|
|
172
|
+
|
|
173
|
+
const parts: string[] = [];
|
|
174
|
+
for (const attr of ftsAttrs) {
|
|
175
|
+
const value = obj[attr];
|
|
176
|
+
if (value !== undefined && value !== null) {
|
|
177
|
+
parts.push(this.valueToString(value));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return parts.join(' ');
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private shouldIncludeAttribute(key: string): boolean {
|
|
185
|
+
const excludedAttrs = [
|
|
186
|
+
PathAttributeName,
|
|
187
|
+
DeletedFlagAttributeName,
|
|
188
|
+
TenantAttributeName,
|
|
189
|
+
'__tenant__',
|
|
190
|
+
'__is_deleted__',
|
|
191
|
+
];
|
|
192
|
+
return !excludedAttrs.includes(key);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private valueToString(value: any): string {
|
|
196
|
+
if (Array.isArray(value)) {
|
|
197
|
+
return value.join(' ');
|
|
198
|
+
}
|
|
199
|
+
if (typeof value === 'object' && value !== null) {
|
|
200
|
+
return JSON.stringify(value);
|
|
201
|
+
}
|
|
202
|
+
return String(value);
|
|
203
|
+
}
|
|
204
|
+
|
|
96
205
|
private async insertInstance(inst: Instance, orUpdate = false): Promise<Instance> {
|
|
97
206
|
const ctx = this.getDbContext(inst.getFqName());
|
|
98
207
|
if (isBetweenRelationship(inst.name, inst.moduleName)) {
|
|
@@ -107,35 +216,43 @@ export class SqlDbResolver extends Resolver {
|
|
|
107
216
|
ctx.activeEnv.isInDeleteMode()
|
|
108
217
|
);
|
|
109
218
|
return inst;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
219
|
+
}
|
|
220
|
+
const idAttrName: string | undefined = maybeFindIdAttributeName(inst);
|
|
221
|
+
ensureOneToOneAttributes(inst);
|
|
222
|
+
const attrs: InstanceAttributes = inst.attributes;
|
|
223
|
+
const idAttrVal: any = idAttrName ? attrs.get(idAttrName) : crypto.randomUUID();
|
|
224
|
+
if (idAttrVal !== undefined) {
|
|
225
|
+
const pp: string | undefined = attrs.get(PathAttributeName);
|
|
226
|
+
const n: string = `${inst.moduleName}/${inst.name}`;
|
|
227
|
+
let p: string = '';
|
|
228
|
+
if (pp !== undefined) p = `${pp}/${escapeFqName(n)}/${idAttrVal}`;
|
|
229
|
+
else p = `${n.replace('/', '$')}/${idAttrVal}`;
|
|
230
|
+
attrs.set(PathAttributeName, p);
|
|
231
|
+
}
|
|
232
|
+
const n: string = asTableReference(inst.moduleName, inst.name);
|
|
233
|
+
const rowObj: object = inst.attributesWithStringifiedObjects();
|
|
234
|
+
await insertRow(n, rowObj, ctx, orUpdate);
|
|
235
|
+
if (inst.record.getEmbeddingConfig() || inst.record.getFullTextSearchAttributes()) {
|
|
236
|
+
const path = attrs.get(PathAttributeName);
|
|
237
|
+
try {
|
|
238
|
+
if (
|
|
239
|
+
(await isVectorStoreSupported()) &&
|
|
240
|
+
!(await vectorStoreSearchEntryExists(n, path, ctx))
|
|
241
|
+
) {
|
|
242
|
+
const ftsAttrs = inst.record.getFullTextSearchAttributes() || ['*'];
|
|
243
|
+
const textToEmbed = this.extractTextForEmbedding(rowObj, ftsAttrs);
|
|
244
|
+
const embeddingConfig = inst.record.getEmbeddingConfig();
|
|
245
|
+
const embeddingService = embeddingConfig
|
|
246
|
+
? new EmbeddingService(embeddingConfig)
|
|
247
|
+
: this.embeddingService;
|
|
248
|
+
const res = await embeddingService.embedText(textToEmbed);
|
|
249
|
+
await addRowForFullTextSearch(n, path, res, ctx);
|
|
135
250
|
}
|
|
251
|
+
} catch (reason: any) {
|
|
252
|
+
logger.warn(`Full text indexing failed for ${path} - ${reason}`);
|
|
136
253
|
}
|
|
137
|
-
return inst;
|
|
138
254
|
}
|
|
255
|
+
return inst;
|
|
139
256
|
}
|
|
140
257
|
|
|
141
258
|
public override async createInstance(inst: Instance): Promise<Instance> {
|
|
@@ -211,9 +328,42 @@ export class SqlDbResolver extends Resolver {
|
|
|
211
328
|
: undefined;
|
|
212
329
|
const orderByDesc = inst.orderByDesc ? 'DESC' : 'ASC';
|
|
213
330
|
const aggregates = SqlDbResolver.normalizedAggregates(inst, tableName);
|
|
331
|
+
|
|
332
|
+
let vectorResult: Instance[] | undefined;
|
|
333
|
+
const embeddingConfig = inst.record.getEmbeddingConfig();
|
|
334
|
+
const ftsAttrs = inst.record.getFullTextSearchAttributes();
|
|
335
|
+
if (
|
|
336
|
+
(await isVectorStoreSupported()) &&
|
|
337
|
+
embeddingConfig &&
|
|
338
|
+
qattrs &&
|
|
339
|
+
(ftsAttrs || Object.keys(qattrs).some(k => k.endsWith('?')))
|
|
340
|
+
) {
|
|
341
|
+
const vectorSearchAttr = Object.keys(qattrs).find(k => k.endsWith('?'));
|
|
342
|
+
if (vectorSearchAttr) {
|
|
343
|
+
const queryVal = qvals[vectorSearchAttr];
|
|
344
|
+
const searchString = this.valueToString(queryVal);
|
|
345
|
+
const embeddingService = new EmbeddingService(embeddingConfig);
|
|
346
|
+
const queryVec = await embeddingService.embedQuery(searchString);
|
|
347
|
+
const rslt: any = await vectorStoreSearch(tableName, queryVec, 10, ctx);
|
|
348
|
+
if (rslt instanceof Array) {
|
|
349
|
+
vectorResult = new Array<Instance>();
|
|
350
|
+
rslt.forEach((r: any) => {
|
|
351
|
+
const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
|
|
352
|
+
tableName,
|
|
353
|
+
new Map(Object.entries(r))
|
|
354
|
+
);
|
|
355
|
+
attrs.delete(DeletedFlagAttributeName);
|
|
356
|
+
vectorResult!.push(Instance.newWithAttributes(inst, attrs));
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
delete qattrs[vectorSearchAttr];
|
|
360
|
+
delete qvals[vectorSearchAttr];
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
214
364
|
const qspec: QuerySpec = {
|
|
215
|
-
queryObj: qattrs,
|
|
216
|
-
queryVals: qvals,
|
|
365
|
+
queryObj: Object.keys(qattrs || {}).length > 0 ? qattrs : undefined,
|
|
366
|
+
queryVals: Object.keys(qvals || {}).length > 0 ? qvals : undefined,
|
|
217
367
|
distinct,
|
|
218
368
|
groupBy,
|
|
219
369
|
orderBy,
|
|
@@ -224,20 +374,47 @@ export class SqlDbResolver extends Resolver {
|
|
|
224
374
|
whereClauses: undefined,
|
|
225
375
|
};
|
|
226
376
|
const readOnlyAttrs = inst.record.getWriteOnlyAttributes();
|
|
227
|
-
const rslt: any =
|
|
377
|
+
const rslt: any =
|
|
378
|
+
vectorResult !== undefined && qspec.queryObj === undefined
|
|
379
|
+
? vectorResult
|
|
380
|
+
: await getMany(tableName, qspec, ctx);
|
|
228
381
|
if (rslt instanceof Array) {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
const
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
382
|
+
if (vectorResult !== undefined && qspec.queryObj !== undefined) {
|
|
383
|
+
// Merge results if both vector and standard queries were performed
|
|
384
|
+
const vectorPaths = new Set(vectorResult.map(i => i.lookup(PathAttributeName)));
|
|
385
|
+
result = new Array<Instance>();
|
|
386
|
+
rslt.forEach((r: any) => {
|
|
387
|
+
const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
|
|
388
|
+
tableName,
|
|
389
|
+
new Map(Object.entries(r))
|
|
390
|
+
);
|
|
391
|
+
if (vectorPaths.has(attrs.get(PathAttributeName))) {
|
|
392
|
+
attrs.delete(DeletedFlagAttributeName);
|
|
393
|
+
readOnlyAttrs?.forEach((n: string) => {
|
|
394
|
+
attrs.delete(n);
|
|
395
|
+
});
|
|
396
|
+
result.push(Instance.newWithAttributes(inst, attrs));
|
|
397
|
+
}
|
|
238
398
|
});
|
|
239
|
-
|
|
240
|
-
|
|
399
|
+
} else {
|
|
400
|
+
result =
|
|
401
|
+
vectorResult !== undefined && qspec.queryObj === undefined
|
|
402
|
+
? vectorResult
|
|
403
|
+
: new Array<Instance>();
|
|
404
|
+
if (vectorResult === undefined || qspec.queryObj !== undefined) {
|
|
405
|
+
rslt.forEach((r: any) => {
|
|
406
|
+
const attrs: InstanceAttributes = maybeNormalizeAttributeNames(
|
|
407
|
+
tableName,
|
|
408
|
+
new Map(Object.entries(r))
|
|
409
|
+
);
|
|
410
|
+
attrs.delete(DeletedFlagAttributeName);
|
|
411
|
+
readOnlyAttrs?.forEach((n: string) => {
|
|
412
|
+
attrs.delete(n);
|
|
413
|
+
});
|
|
414
|
+
result.push(Instance.newWithAttributes(inst, attrs));
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
}
|
|
241
418
|
}
|
|
242
419
|
return result;
|
|
243
420
|
}
|
|
@@ -555,7 +732,7 @@ export class SqlDbResolver extends Resolver {
|
|
|
555
732
|
query: string,
|
|
556
733
|
options?: any
|
|
557
734
|
): Promise<any> {
|
|
558
|
-
const queryVec = await this.
|
|
735
|
+
const queryVec = await this.embeddingService.embedQuery(query);
|
|
559
736
|
const ctx = this.getDbContext(makeFqName(moduleName, entryName));
|
|
560
737
|
let limit = 5;
|
|
561
738
|
if (options && options.limit) {
|