expxagents 0.25.2 → 0.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/src/commands/info.d.ts +2 -1
- package/dist/cli/src/commands/login.d.ts +2 -1
- package/dist/cli/src/commands/logout.d.ts +2 -1
- package/dist/cli/src/commands/outdated.d.ts +2 -1
- package/dist/cli/src/commands/publish.d.ts +2 -1
- package/dist/cli/src/commands/registry-install.d.ts +2 -1
- package/dist/cli/src/commands/search.d.ts +2 -1
- package/dist/cli/src/commands/update.d.ts +2 -1
- package/dist/cli/src/commands/whoami.d.ts +2 -1
- package/dist/cli/src/utils/server-paths.js +17 -5
- package/node_modules/expxagents-knowledge/dist/config.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/config.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/config.js +37 -0
- package/node_modules/expxagents-knowledge/dist/config.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.d.ts +6 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.js +69 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.d.ts +3 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.js +46 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.d.ts +3 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.js +79 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/index.d.ts +16 -0
- package/node_modules/expxagents-knowledge/dist/index.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/index.js +16 -0
- package/node_modules/expxagents-knowledge/dist/index.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.d.ts +10 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.js +221 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js +56 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.js +25 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts +21 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js +54 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts +16 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js +49 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.d.ts +9 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.js +116 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pdf-extractor.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.d.ts +27 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js +92 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.d.ts +41 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.js +62 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.d.ts +31 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.js +106 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.d.ts +26 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.js +57 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.d.ts +10 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.js +29 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.d.ts +6 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.js +42 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/types.d.ts +138 -0
- package/node_modules/expxagents-knowledge/dist/types.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/types.js +2 -0
- package/node_modules/expxagents-knowledge/dist/types.js.map +1 -0
- package/node_modules/expxagents-knowledge/package.json +7 -0
- package/package.json +2 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf-extractor.js","sourceRoot":"","sources":["../../src/ingest/pdf-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAQzB,IAAI,SAAS,GAAmB,IAAI,CAAC;AAErC,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,QAAgB,EAChB,MAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEvD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAEjE,IAAI,GAAG,CAAC;IACR,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;IAC3E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,KAAK,OAAO,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC;IAE/B,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,cAAc,GAAG,SAAS,CAAC;IAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK;aAC3B,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;aAC5C,IAAI,CAAC,GAAG,CAAC;aACT,IAAI,EAAE,CAAC;QAEV,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC5C,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;gBAChC,IAAI,OAAO;oBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,IAAI,OAAe,CAAC;IACpB,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,SAAS;aAChB,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aAC/D,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC7C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,cAAc;YAClC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,OAAO,CAAC;IAEd,OAAO;QACL,OAAO;QACP,QAAQ,EAAE;YACR,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE;YAC1D,iBAAiB;YACjB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1C;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAa,EAAE,MAAiB;IACrD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,MAAM,GAAG,SAAmF,CAAC;QAEnG,MAAM,QAAQ,GAAI,IAA8D,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC7G,gFAAgF;QAChF,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,QAAkB,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,YAAY,CACxB,QAA8B,CAAC,KAAK,EACpC,QAA+B,CAAC,MAAM,CACxC,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAExC,MAAO,IAAkE,CAAC,MAAM,CAAC;YAC/E,aAAa,EAAE,OAAO;YACtB,QAAQ;SACT,CAAC,CAAC,OAAO,CAAC;QAEX,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,IAAI,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACnF,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;YAC9E,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAO,SAAgD,CAAC,SAAS,EAAE,CAAC;QACtE,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;QAC9B,CAAC;QACD,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { Embedder, SourceType } from '../types.js';
|
|
3
|
+
interface ChunkerConfig {
|
|
4
|
+
max_tokens: number;
|
|
5
|
+
overlap: number;
|
|
6
|
+
}
|
|
7
|
+
interface IngestOptions {
|
|
8
|
+
squad_code?: string;
|
|
9
|
+
agent_id?: string;
|
|
10
|
+
source_type?: SourceType;
|
|
11
|
+
tags?: string[];
|
|
12
|
+
}
|
|
13
|
+
interface IngestResult {
|
|
14
|
+
document_id: string;
|
|
15
|
+
chunks_count: number;
|
|
16
|
+
skipped: boolean;
|
|
17
|
+
}
|
|
18
|
+
export declare class IngestPipeline {
|
|
19
|
+
private db;
|
|
20
|
+
private embedder;
|
|
21
|
+
private chunkerConfig;
|
|
22
|
+
constructor(db: Database.Database, embedder: Embedder, chunkerConfig: ChunkerConfig);
|
|
23
|
+
ingest(filePath: string, options?: IngestOptions): Promise<IngestResult>;
|
|
24
|
+
ingestText(content: string, title: string, options?: IngestOptions): Promise<IngestResult>;
|
|
25
|
+
}
|
|
26
|
+
export {};
|
|
27
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAI3C,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAe,MAAM,aAAa,CAAC;AAErE,UAAU,aAAa;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,aAAa;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,UAAU,YAAY;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,qBAAa,cAAc;IAEvB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,aAAa;gBAFb,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa;IAGhC,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAmFxE,UAAU,CACd,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,YAAY,CAAC;CA6CzB"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
3
|
+
import { loadDocument, computeHash } from './document-loader.js';
|
|
4
|
+
import { chunkMarkdown, chunkPlainText } from './chunker.js';
|
|
5
|
+
export class IngestPipeline {
|
|
6
|
+
db;
|
|
7
|
+
embedder;
|
|
8
|
+
chunkerConfig;
|
|
9
|
+
constructor(db, embedder, chunkerConfig) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
this.embedder = embedder;
|
|
12
|
+
this.chunkerConfig = chunkerConfig;
|
|
13
|
+
}
|
|
14
|
+
async ingest(filePath, options) {
|
|
15
|
+
const parsed = await loadDocument(filePath);
|
|
16
|
+
const contentHash = parsed.mime_type === 'application/pdf'
|
|
17
|
+
? computeHash(fs.readFileSync(filePath))
|
|
18
|
+
: computeHash(parsed.content);
|
|
19
|
+
const sourceType = options?.source_type ?? 'external';
|
|
20
|
+
// Check for existing document with same path and hash
|
|
21
|
+
const existing = this.db
|
|
22
|
+
.prepare('SELECT id FROM documents WHERE source_path = ? AND content_hash = ?')
|
|
23
|
+
.get(filePath, contentHash);
|
|
24
|
+
if (existing) {
|
|
25
|
+
return { document_id: existing.id, chunks_count: 0, skipped: true };
|
|
26
|
+
}
|
|
27
|
+
// Upsert: delete old document with same source_path (CASCADE deletes chunks)
|
|
28
|
+
this.db.prepare('DELETE FROM documents WHERE source_path = ?').run(filePath);
|
|
29
|
+
const docId = nanoid();
|
|
30
|
+
const now = new Date().toISOString();
|
|
31
|
+
const docMetadata = {
|
|
32
|
+
...(options?.tags ? { tags: options.tags } : {}),
|
|
33
|
+
...(parsed.metadata ? parsed.metadata : {}),
|
|
34
|
+
};
|
|
35
|
+
const metadata = Object.keys(docMetadata).length > 0 ? JSON.stringify(docMetadata) : null;
|
|
36
|
+
this.db
|
|
37
|
+
.prepare(`INSERT INTO documents (id, source_type, source_path, squad_code, agent_id, title, mime_type, content_hash, extraction_status, metadata, ingested_at, updated_at)
|
|
38
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`)
|
|
39
|
+
.run(docId, sourceType, filePath, options?.squad_code ?? null, options?.agent_id ?? null, parsed.title, parsed.mime_type, contentHash, metadata, now, now);
|
|
40
|
+
// Chunk
|
|
41
|
+
const chunks = parsed.mime_type === 'application/pdf'
|
|
42
|
+
? chunkPlainText(parsed.content, this.chunkerConfig)
|
|
43
|
+
: chunkMarkdown(parsed.content, this.chunkerConfig);
|
|
44
|
+
// Store chunks + embeddings
|
|
45
|
+
const insertChunk = this.db.prepare('INSERT INTO chunks (id, document_id, content, chunk_index, token_count, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)');
|
|
46
|
+
const insertEmbedding = this.db.prepare('INSERT INTO chunk_embeddings_store (chunk_id, embedding) VALUES (?, ?)');
|
|
47
|
+
// Batch embed
|
|
48
|
+
const texts = chunks.map((c) => c.content);
|
|
49
|
+
const embeddings = texts.length > 0 ? await this.embedder.embed(texts) : [];
|
|
50
|
+
const transaction = this.db.transaction(() => {
|
|
51
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
52
|
+
const chunkId = nanoid();
|
|
53
|
+
insertChunk.run(chunkId, docId, chunks[i].content, chunks[i].chunk_index, chunks[i].token_count, JSON.stringify(chunks[i].metadata), now);
|
|
54
|
+
insertEmbedding.run(chunkId, JSON.stringify(embeddings[i]));
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
transaction();
|
|
58
|
+
return { document_id: docId, chunks_count: chunks.length, skipped: false };
|
|
59
|
+
}
|
|
60
|
+
async ingestText(content, title, options) {
|
|
61
|
+
const contentHash = computeHash(content);
|
|
62
|
+
const existing = this.db
|
|
63
|
+
.prepare('SELECT id FROM documents WHERE content_hash = ? AND source_path IS NULL AND title = ?')
|
|
64
|
+
.get(contentHash, title);
|
|
65
|
+
if (existing) {
|
|
66
|
+
return { document_id: existing.id, chunks_count: 0, skipped: true };
|
|
67
|
+
}
|
|
68
|
+
const docId = nanoid();
|
|
69
|
+
const now = new Date().toISOString();
|
|
70
|
+
const sourceType = options?.source_type ?? 'external';
|
|
71
|
+
const metadata = options?.tags ? JSON.stringify({ tags: options.tags }) : null;
|
|
72
|
+
this.db
|
|
73
|
+
.prepare(`INSERT INTO documents (id, source_type, source_path, squad_code, agent_id, title, mime_type, content_hash, extraction_status, metadata, ingested_at, updated_at)
|
|
74
|
+
VALUES (?, ?, NULL, ?, ?, ?, 'text/markdown', ?, 'pending', ?, ?, ?)`)
|
|
75
|
+
.run(docId, sourceType, options?.squad_code ?? null, options?.agent_id ?? null, title, contentHash, metadata, now, now);
|
|
76
|
+
const chunks = chunkMarkdown(content, this.chunkerConfig);
|
|
77
|
+
const texts = chunks.map((c) => c.content);
|
|
78
|
+
const embeddings = texts.length > 0 ? await this.embedder.embed(texts) : [];
|
|
79
|
+
const insertChunk = this.db.prepare('INSERT INTO chunks (id, document_id, content, chunk_index, token_count, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)');
|
|
80
|
+
const insertEmbedding = this.db.prepare('INSERT INTO chunk_embeddings_store (chunk_id, embedding) VALUES (?, ?)');
|
|
81
|
+
const transaction = this.db.transaction(() => {
|
|
82
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
83
|
+
const chunkId = nanoid();
|
|
84
|
+
insertChunk.run(chunkId, docId, chunks[i].content, chunks[i].chunk_index, chunks[i].token_count, JSON.stringify(chunks[i].metadata), now);
|
|
85
|
+
insertEmbedding.run(chunkId, JSON.stringify(embeddings[i]));
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
transaction();
|
|
89
|
+
return { document_id: docId, chunks_count: chunks.length, skipped: false };
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAqB7D,MAAM,OAAO,cAAc;IAEf;IACA;IACA;IAHV,YACU,EAAqB,EACrB,QAAkB,EAClB,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAAmB;QACrB,aAAQ,GAAR,QAAQ,CAAU;QAClB,kBAAa,GAAb,aAAa,CAAe;IACnC,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,OAAuB;QACpD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,KAAK,iBAAiB;YACxD,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;QAEtD,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,OAAO,CAAC,qEAAqE,CAAC;aAC9E,GAAG,CAAC,QAAQ,EAAE,WAAW,CAA+B,CAAC;QAE5D,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACtE,CAAC;QAED,6EAA6E;QAC7E,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE7E,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG;YAClB,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1F,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;6DACqD,CACtD;aACA,GAAG,CACF,KAAK,EACL,UAAU,EACV,QAAQ,EACR,OAAO,EAAE,UAAU,IAAI,IAAI,EAC3B,OAAO,EAAE,QAAQ,IAAI,IAAI,EACzB,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,SAAS,EAChB,WAAW,EACX,QAAQ,EACR,GAAG,EACH,GAAG,CACJ,CAAC;QAEJ,QAAQ;QACR,MAAM,MAAM,GAAkB,MAAM,CAAC,SAAS,KAAK,iBAAiB;YAClE,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YACpD,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAEtD,4BAA4B;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACjC,4HAA4H,CAC7H,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACrC,wEAAwE,CACzE,CAAC;QAEF,cAAc;QACd,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5E,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CACb,OAAO,EACP,KAAK,EACL,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EACjB,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EACrB,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAClC,GAAG,CACJ,CAAC;gBACF,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;QACH,WAAW,EAAE,CAAC;QAEd,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,UAAU,CACd,OAAe,EACf,KAAa,EACb,OAAuB;QAEvB,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,OAAO,CAAC,uFAAuF,CAAC;aAChG,GAAG,CAAC,WAAW,EAAE,KAAK,CAA+B,CAAC;QAEzD,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;QACtD,MAAM,QAAQ,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE/E,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;8EACsE,CACvE;aACA,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,IAAI,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAE1H,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5E,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACjC,4HAA4H,CAC7H,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACrC,wEAAwE,CACzE,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1I,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;QACH,WAAW,EAAE,CAAC;QAEd,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7E,CAAC;CACF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { RelationType } from '../types.js';
|
|
3
|
+
interface TraversalOptions {
|
|
4
|
+
maxDepth: number;
|
|
5
|
+
relationType?: RelationType;
|
|
6
|
+
}
|
|
7
|
+
interface TraversalResult {
|
|
8
|
+
source: {
|
|
9
|
+
id: string;
|
|
10
|
+
name: string;
|
|
11
|
+
entity_type: string;
|
|
12
|
+
};
|
|
13
|
+
target: {
|
|
14
|
+
id: string;
|
|
15
|
+
name: string;
|
|
16
|
+
entity_type: string;
|
|
17
|
+
};
|
|
18
|
+
relation_type: string;
|
|
19
|
+
description: string | null;
|
|
20
|
+
weight: number;
|
|
21
|
+
depth: number;
|
|
22
|
+
}
|
|
23
|
+
export declare class GraphTraversal {
|
|
24
|
+
private db;
|
|
25
|
+
constructor(db: Database.Database);
|
|
26
|
+
findEntityByName(name: string): {
|
|
27
|
+
id: string;
|
|
28
|
+
name: string;
|
|
29
|
+
entity_type: string;
|
|
30
|
+
} | null;
|
|
31
|
+
findEntitiesByType(type: string): Array<{
|
|
32
|
+
id: string;
|
|
33
|
+
name: string;
|
|
34
|
+
entity_type: string;
|
|
35
|
+
}>;
|
|
36
|
+
traverse(entityId: string, options: TraversalOptions): TraversalResult[];
|
|
37
|
+
private dfs;
|
|
38
|
+
buildNarrative(results: TraversalResult[]): string;
|
|
39
|
+
}
|
|
40
|
+
export {};
|
|
41
|
+
//# sourceMappingURL=graph-traversal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-traversal.d.ts","sourceRoot":"","sources":["../../src/query/graph-traversal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,cAAc;IACb,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAEzC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAOxF,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAM1F,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,eAAe,EAAE;IAOxE,OAAO,CAAC,GAAG;IAsDX,cAAc,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM;CAUnD"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
export class GraphTraversal {
|
|
2
|
+
db;
|
|
3
|
+
constructor(db) {
|
|
4
|
+
this.db = db;
|
|
5
|
+
}
|
|
6
|
+
findEntityByName(name) {
|
|
7
|
+
const row = this.db
|
|
8
|
+
.prepare('SELECT id, name, entity_type FROM entities WHERE name = ? COLLATE NOCASE')
|
|
9
|
+
.get(name);
|
|
10
|
+
return row ?? null;
|
|
11
|
+
}
|
|
12
|
+
findEntitiesByType(type) {
|
|
13
|
+
return this.db
|
|
14
|
+
.prepare('SELECT id, name, entity_type FROM entities WHERE entity_type = ?')
|
|
15
|
+
.all(type);
|
|
16
|
+
}
|
|
17
|
+
traverse(entityId, options) {
|
|
18
|
+
const results = [];
|
|
19
|
+
const visited = new Set();
|
|
20
|
+
this.dfs(entityId, options, results, visited, 1);
|
|
21
|
+
return results;
|
|
22
|
+
}
|
|
23
|
+
dfs(entityId, options, results, visited, depth) {
|
|
24
|
+
if (depth > options.maxDepth)
|
|
25
|
+
return;
|
|
26
|
+
visited.add(entityId);
|
|
27
|
+
let sql = `
|
|
28
|
+
SELECT r.relation_type, r.description, r.weight,
|
|
29
|
+
s.id as source_id, s.name as source_name, s.entity_type as source_type,
|
|
30
|
+
t.id as target_id, t.name as target_name, t.entity_type as target_type
|
|
31
|
+
FROM relations r
|
|
32
|
+
JOIN entities s ON s.id = r.source_id
|
|
33
|
+
JOIN entities t ON t.id = r.target_id
|
|
34
|
+
WHERE r.source_id = ?`;
|
|
35
|
+
const params = [entityId];
|
|
36
|
+
if (options.relationType) {
|
|
37
|
+
sql += ' AND r.relation_type = ?';
|
|
38
|
+
params.push(options.relationType);
|
|
39
|
+
}
|
|
40
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
41
|
+
for (const row of rows) {
|
|
42
|
+
results.push({
|
|
43
|
+
source: { id: row.source_id, name: row.source_name, entity_type: row.source_type },
|
|
44
|
+
target: { id: row.target_id, name: row.target_name, entity_type: row.target_type },
|
|
45
|
+
relation_type: row.relation_type,
|
|
46
|
+
description: row.description,
|
|
47
|
+
weight: row.weight,
|
|
48
|
+
depth,
|
|
49
|
+
});
|
|
50
|
+
if (!visited.has(row.target_id)) {
|
|
51
|
+
this.dfs(row.target_id, options, results, visited, depth + 1);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
buildNarrative(results) {
|
|
56
|
+
if (results.length === 0)
|
|
57
|
+
return 'No relationships found.';
|
|
58
|
+
const lines = results.map((r) => `${r.source.name} (${r.source.entity_type}) —[${r.relation_type}]→ ${r.target.name} (${r.target.entity_type})${r.description ? `: ${r.description}` : ''}`);
|
|
59
|
+
return lines.join('\n');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=graph-traversal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graph-traversal.js","sourceRoot":"","sources":["../../src/query/graph-traversal.ts"],"names":[],"mappings":"AAiBA,MAAM,OAAO,cAAc;IACL;IAApB,YAAoB,EAAqB;QAArB,OAAE,GAAF,EAAE,CAAmB;IAAG,CAAC;IAE7C,gBAAgB,CAAC,IAAY;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,0EAA0E,CAAC;aACnF,GAAG,CAAC,IAAI,CAAkE,CAAC;QAC9E,OAAO,GAAG,IAAI,IAAI,CAAC;IACrB,CAAC;IAED,kBAAkB,CAAC,IAAY;QAC7B,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CAAC,kEAAkE,CAAC;aAC3E,GAAG,CAAC,IAAI,CAA6D,CAAC;IAC3E,CAAC;IAED,QAAQ,CAAC,QAAgB,EAAE,OAAyB;QAClD,MAAM,OAAO,GAAsB,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,GAAG,CACT,QAAgB,EAChB,OAAyB,EACzB,OAA0B,EAC1B,OAAoB,EACpB,KAAa;QAEb,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ;YAAE,OAAO;QACrC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,IAAI,GAAG,GAAG;;;;;;;4BAOc,CAAC;QAEzB,MAAM,MAAM,GAA8B,CAAC,QAAQ,CAAC,CAAC;QAErD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,GAAG,IAAI,0BAA0B,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAU7C,CAAC;QAEH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE;gBAClF,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE;gBAClF,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK;aACN,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA0B;QACvC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,yBAAyB,CAAC;QAE3D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CACvB,CAAC,CAAC,EAAE,EAAE,CACJ,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,OAAO,CAAC,CAAC,aAAa,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7J,CAAC;QAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { Embedder, QueryScope, QueryMode, KnowledgeResult } from '../types.js';
|
|
3
|
+
interface QueryOptions {
|
|
4
|
+
mode?: QueryMode;
|
|
5
|
+
scope?: QueryScope;
|
|
6
|
+
limit?: number;
|
|
7
|
+
}
|
|
8
|
+
interface KnowledgeStats {
|
|
9
|
+
documents: number;
|
|
10
|
+
chunks: number;
|
|
11
|
+
entities: number;
|
|
12
|
+
relations: number;
|
|
13
|
+
}
|
|
14
|
+
export declare class KnowledgeQuery {
|
|
15
|
+
private db;
|
|
16
|
+
private vectorSearch;
|
|
17
|
+
private graphTraversal;
|
|
18
|
+
constructor(db: Database.Database, embedder: Embedder);
|
|
19
|
+
query(queryText: string, options?: QueryOptions): Promise<KnowledgeResult>;
|
|
20
|
+
private graphMode;
|
|
21
|
+
private autoMode;
|
|
22
|
+
private searchMode;
|
|
23
|
+
getStats(): KnowledgeStats;
|
|
24
|
+
listDocuments(limit?: number, offset?: number): unknown[];
|
|
25
|
+
getDocument(id: string): unknown;
|
|
26
|
+
deleteDocument(id: string): boolean;
|
|
27
|
+
listEntities(type?: string, limit?: number): unknown[];
|
|
28
|
+
getEntityRelations(entityId: string): unknown[];
|
|
29
|
+
}
|
|
30
|
+
export {};
|
|
31
|
+
//# sourceMappingURL=knowledge-query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-query.d.ts","sourceRoot":"","sources":["../../src/query/knowledge-query.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAIpF,UAAU,YAAY;IACpB,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,cAAc;IAKvB,OAAO,CAAC,EAAE;IAJZ,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,cAAc,CAAiB;gBAG7B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAC7B,QAAQ,EAAE,QAAQ;IAMd,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC;YAalE,SAAS;YAYT,QAAQ;YAaR,UAAU;IAkBxB,QAAQ,IAAI,cAAc;IAgB1B,aAAa,CAAC,KAAK,SAAK,EAAE,MAAM,SAAI,GAAG,OAAO,EAAE;IAUhD,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIhC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAKnC,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,EAAE;IAOlD,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,EAAE;CAUhD"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { VectorSearch } from './vector-search.js';
|
|
2
|
+
import { GraphTraversal } from './graph-traversal.js';
|
|
3
|
+
export class KnowledgeQuery {
|
|
4
|
+
db;
|
|
5
|
+
vectorSearch;
|
|
6
|
+
graphTraversal;
|
|
7
|
+
constructor(db, embedder) {
|
|
8
|
+
this.db = db;
|
|
9
|
+
this.vectorSearch = new VectorSearch(db, embedder);
|
|
10
|
+
this.graphTraversal = new GraphTraversal(db);
|
|
11
|
+
}
|
|
12
|
+
async query(queryText, options) {
|
|
13
|
+
const mode = options?.mode ?? 'auto';
|
|
14
|
+
const limit = options?.limit ?? 10;
|
|
15
|
+
if (mode === 'graph') {
|
|
16
|
+
return this.graphMode(queryText, limit, options?.scope);
|
|
17
|
+
}
|
|
18
|
+
if (mode === 'auto') {
|
|
19
|
+
return this.autoMode(queryText, limit, options?.scope);
|
|
20
|
+
}
|
|
21
|
+
return this.searchMode(queryText, limit, options?.scope);
|
|
22
|
+
}
|
|
23
|
+
async graphMode(queryText, limit, scope) {
|
|
24
|
+
const entity = this.graphTraversal.findEntityByName(queryText);
|
|
25
|
+
if (entity) {
|
|
26
|
+
const traversal = this.graphTraversal.traverse(entity.id, { maxDepth: 2 });
|
|
27
|
+
return {
|
|
28
|
+
results: [],
|
|
29
|
+
graph_context: this.graphTraversal.buildNarrative(traversal),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
return this.searchMode(queryText, limit, scope);
|
|
33
|
+
}
|
|
34
|
+
async autoMode(queryText, limit, scope) {
|
|
35
|
+
const GRAPH_KEYWORDS = ['depende', 'depends', 'impacta', 'impacts', 'contradiz', 'contradicts', 'relação', 'relationship', 'levou a', 'led to'];
|
|
36
|
+
const isGraphQuery = GRAPH_KEYWORDS.some(kw => queryText.toLowerCase().includes(kw));
|
|
37
|
+
if (isGraphQuery) {
|
|
38
|
+
const graphResult = await this.graphMode(queryText, limit, scope);
|
|
39
|
+
if (graphResult.graph_context && graphResult.graph_context !== 'No relationships found.') {
|
|
40
|
+
return graphResult;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return this.searchMode(queryText, limit, scope);
|
|
44
|
+
}
|
|
45
|
+
async searchMode(queryText, limit, scope) {
|
|
46
|
+
const searchResults = await this.vectorSearch.search(queryText, { limit, scope });
|
|
47
|
+
return {
|
|
48
|
+
results: searchResults.map((r) => ({
|
|
49
|
+
content: r.content,
|
|
50
|
+
source: {
|
|
51
|
+
document_title: r.document_title,
|
|
52
|
+
source_type: r.source_type,
|
|
53
|
+
squad_code: r.squad_code ?? undefined,
|
|
54
|
+
agent_id: r.agent_id ?? undefined,
|
|
55
|
+
path: r.source_path ?? undefined,
|
|
56
|
+
},
|
|
57
|
+
score: r.score,
|
|
58
|
+
})),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
getStats() {
|
|
62
|
+
const docs = this.db.prepare('SELECT COUNT(*) as count FROM documents').get().count;
|
|
63
|
+
const chunks = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get().count;
|
|
64
|
+
let entities = 0;
|
|
65
|
+
let relations = 0;
|
|
66
|
+
try {
|
|
67
|
+
entities = this.db.prepare('SELECT COUNT(*) as count FROM entities').get().count;
|
|
68
|
+
relations = this.db.prepare('SELECT COUNT(*) as count FROM relations').get().count;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Tables may not exist yet (Phase 1 only)
|
|
72
|
+
}
|
|
73
|
+
return { documents: docs, chunks, entities, relations };
|
|
74
|
+
}
|
|
75
|
+
listDocuments(limit = 50, offset = 0) {
|
|
76
|
+
return this.db
|
|
77
|
+
.prepare(`SELECT id, source_type, source_path, squad_code, title, content_hash,
|
|
78
|
+
extraction_status, ingested_at, updated_at
|
|
79
|
+
FROM documents ORDER BY ingested_at DESC LIMIT ? OFFSET ?`)
|
|
80
|
+
.all(limit, offset);
|
|
81
|
+
}
|
|
82
|
+
getDocument(id) {
|
|
83
|
+
return this.db.prepare('SELECT * FROM documents WHERE id = ?').get(id);
|
|
84
|
+
}
|
|
85
|
+
deleteDocument(id) {
|
|
86
|
+
const result = this.db.prepare('DELETE FROM documents WHERE id = ?').run(id);
|
|
87
|
+
return result.changes > 0;
|
|
88
|
+
}
|
|
89
|
+
listEntities(type, limit = 50) {
|
|
90
|
+
if (type) {
|
|
91
|
+
return this.db.prepare('SELECT * FROM entities WHERE entity_type = ? ORDER BY mention_count DESC LIMIT ?').all(type, limit);
|
|
92
|
+
}
|
|
93
|
+
return this.db.prepare('SELECT * FROM entities ORDER BY mention_count DESC LIMIT ?').all(limit);
|
|
94
|
+
}
|
|
95
|
+
getEntityRelations(entityId) {
|
|
96
|
+
return this.db.prepare(`
|
|
97
|
+
SELECT r.*, s.name as source_name, s.entity_type as source_type,
|
|
98
|
+
t.name as target_name, t.entity_type as target_type
|
|
99
|
+
FROM relations r
|
|
100
|
+
JOIN entities s ON s.id = r.source_id
|
|
101
|
+
JOIN entities t ON t.id = r.target_id
|
|
102
|
+
WHERE r.source_id = ? OR r.target_id = ?
|
|
103
|
+
`).all(entityId, entityId);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=knowledge-query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-query.js","sourceRoot":"","sources":["../../src/query/knowledge-query.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAetD,MAAM,OAAO,cAAc;IAKf;IAJF,YAAY,CAAe;IAC3B,cAAc,CAAiB;IAEvC,YACU,EAAqB,EAC7B,QAAkB;QADV,OAAE,GAAF,EAAE,CAAmB;QAG7B,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,OAAsB;QACnD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,MAAM,CAAC;QACrC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEnC,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,KAAa,EAAE,KAAkB;QAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC/D,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;YAC3E,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,aAAa,EAAE,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,SAAS,CAAC;aAC7D,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,KAAa,EAAE,KAAkB;QACzE,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChJ,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QAErF,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAClE,IAAI,WAAW,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,KAAK,yBAAyB,EAAE,CAAC;gBACzF,OAAO,WAAW,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,SAAiB,EAAE,KAAa,EAAE,KAAkB;QAC3E,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAElF,OAAO;YACL,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACjC,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE;oBACN,cAAc,EAAE,CAAC,CAAC,cAAc;oBAChC,WAAW,EAAE,CAAC,CAAC,WAAW;oBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,SAAS;oBACrC,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,SAAS;oBACjC,IAAI,EAAE,CAAC,CAAC,WAAW,IAAI,SAAS;iBACjC;gBACD,KAAK,EAAE,CAAC,CAAC,KAAK;aACf,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;IAED,QAAQ;QACN,MAAM,IAAI,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC3G,MAAM,MAAM,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAE1G,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC;YACH,QAAQ,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;YACxG,SAAS,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC5G,CAAC;QAAC,MAAM,CAAC;YACP,0CAA0C;QAC5C,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC1D,CAAC;IAED,aAAa,CAAC,KAAK,GAAG,EAAE,EAAE,MAAM,GAAG,CAAC;QAClC,OAAO,IAAI,CAAC,EAAE;aACX,OAAO,CACN;;mEAE2D,CAC5D;aACA,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACxB,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,YAAY,CAAC,IAAa,EAAE,KAAK,GAAG,EAAE;QACpC,IAAI,IAAI,EAAE,CAAC;YACT,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9H,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClG,CAAC;IAED,kBAAkB,CAAC,QAAgB;QACjC,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;KAOtB,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { Embedder, QueryScope } from '../types.js';
|
|
3
|
+
export interface SearchResult {
|
|
4
|
+
chunk_id: string;
|
|
5
|
+
document_id: string;
|
|
6
|
+
content: string;
|
|
7
|
+
score: number;
|
|
8
|
+
document_title: string;
|
|
9
|
+
source_type: string;
|
|
10
|
+
squad_code: string | null;
|
|
11
|
+
agent_id: string | null;
|
|
12
|
+
source_path: string | null;
|
|
13
|
+
}
|
|
14
|
+
interface SearchOptions {
|
|
15
|
+
limit: number;
|
|
16
|
+
scope?: QueryScope;
|
|
17
|
+
}
|
|
18
|
+
export declare class VectorSearch {
|
|
19
|
+
private db;
|
|
20
|
+
private embedder;
|
|
21
|
+
constructor(db: Database.Database, embedder: Embedder);
|
|
22
|
+
search(query: string, options: SearchOptions): Promise<SearchResult[]>;
|
|
23
|
+
private bruteForceSearch;
|
|
24
|
+
}
|
|
25
|
+
export {};
|
|
26
|
+
//# sourceMappingURL=vector-search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-search.d.ts","sourceRoot":"","sources":["../../src/query/vector-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAExD,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,UAAU,aAAa;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB;AAED,qBAAa,YAAY;IAErB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,QAAQ;gBADR,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,QAAQ,EAAE,QAAQ;IAGtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAK5E,OAAO,CAAC,gBAAgB;CA+CzB"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
export class VectorSearch {
|
|
2
|
+
db;
|
|
3
|
+
embedder;
|
|
4
|
+
constructor(db, embedder) {
|
|
5
|
+
this.db = db;
|
|
6
|
+
this.embedder = embedder;
|
|
7
|
+
}
|
|
8
|
+
async search(query, options) {
|
|
9
|
+
const [queryVec] = await this.embedder.embed([query]);
|
|
10
|
+
return this.bruteForceSearch(queryVec, options);
|
|
11
|
+
}
|
|
12
|
+
bruteForceSearch(queryVec, options) {
|
|
13
|
+
const rows = this.db
|
|
14
|
+
.prepare(`SELECT ces.chunk_id, ces.embedding, c.document_id, c.content,
|
|
15
|
+
d.title as document_title, d.source_type, d.squad_code, d.agent_id, d.source_path
|
|
16
|
+
FROM chunk_embeddings_store ces
|
|
17
|
+
JOIN chunks c ON c.id = ces.chunk_id
|
|
18
|
+
JOIN documents d ON d.id = c.document_id`)
|
|
19
|
+
.all();
|
|
20
|
+
let results = rows.map((row) => {
|
|
21
|
+
const embedding = JSON.parse(row.embedding);
|
|
22
|
+
const score = cosineSimilarity(queryVec, embedding);
|
|
23
|
+
return {
|
|
24
|
+
chunk_id: row.chunk_id,
|
|
25
|
+
document_id: row.document_id,
|
|
26
|
+
content: row.content,
|
|
27
|
+
score,
|
|
28
|
+
document_title: row.document_title,
|
|
29
|
+
source_type: row.source_type,
|
|
30
|
+
squad_code: row.squad_code,
|
|
31
|
+
agent_id: row.agent_id,
|
|
32
|
+
source_path: row.source_path,
|
|
33
|
+
};
|
|
34
|
+
});
|
|
35
|
+
if (options.scope?.squad) {
|
|
36
|
+
results = results.filter((r) => r.squad_code === options.scope.squad);
|
|
37
|
+
}
|
|
38
|
+
if (options.scope?.source_type) {
|
|
39
|
+
results = results.filter((r) => r.source_type === options.scope.source_type);
|
|
40
|
+
}
|
|
41
|
+
results.sort((a, b) => b.score - a.score);
|
|
42
|
+
return results.slice(0, options.limit);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function cosineSimilarity(a, b) {
|
|
46
|
+
let dot = 0;
|
|
47
|
+
let normA = 0;
|
|
48
|
+
let normB = 0;
|
|
49
|
+
for (let i = 0; i < a.length; i++) {
|
|
50
|
+
dot += a[i] * b[i];
|
|
51
|
+
normA += a[i] * a[i];
|
|
52
|
+
normB += b[i] * b[i];
|
|
53
|
+
}
|
|
54
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
55
|
+
return denom === 0 ? 0 : dot / denom;
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=vector-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-search.js","sourceRoot":"","sources":["../../src/query/vector-search.ts"],"names":[],"mappings":"AAoBA,MAAM,OAAO,YAAY;IAEb;IACA;IAFV,YACU,EAAqB,EACrB,QAAkB;QADlB,OAAE,GAAF,EAAE,CAAmB;QACrB,aAAQ,GAAR,QAAQ,CAAU;IACzB,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAEO,gBAAgB,CAAC,QAAkB,EAAE,OAAsB;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CACN;;;;kDAI0C,CAC3C;aACA,GAAG,EAUF,CAAC;QAEL,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAa,CAAC;YACxD,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YACpD,OAAO;gBACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK;gBACL,cAAc,EAAE,GAAG,CAAC,cAAc;gBAClC,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,WAAW;aAC7B,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;YACzB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,OAAO,CAAC,KAAM,CAAC,KAAK,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC;YAC/B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { IngestPipeline } from '../ingest/pipeline.js';
|
|
2
|
+
interface AgentOutputEvent {
|
|
3
|
+
squad_code: string;
|
|
4
|
+
agent_id: string;
|
|
5
|
+
result_summary?: string;
|
|
6
|
+
output_files?: string[];
|
|
7
|
+
}
|
|
8
|
+
export declare function createAgentOutputHook(pipeline: IngestPipeline): (event: AgentOutputEvent) => Promise<void>;
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=agent-output.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-output.d.ts","sourceRoot":"","sources":["../../src/sources/agent-output.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,UAAU,gBAAgB;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,cAAc,IACtB,OAAO,gBAAgB,KAAG,OAAO,CAAC,IAAI,CAAC,CAyB9E"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
export function createAgentOutputHook(pipeline) {
|
|
3
|
+
return async function onAgentComplete(event) {
|
|
4
|
+
try {
|
|
5
|
+
if (event.output_files) {
|
|
6
|
+
for (const file of event.output_files) {
|
|
7
|
+
if (fs.existsSync(file)) {
|
|
8
|
+
await pipeline.ingest(file, {
|
|
9
|
+
squad_code: event.squad_code,
|
|
10
|
+
agent_id: event.agent_id,
|
|
11
|
+
source_type: 'agent_output',
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
if (event.result_summary?.trim()) {
|
|
17
|
+
await pipeline.ingestText(event.result_summary, `Agent output: ${event.agent_id}`, {
|
|
18
|
+
squad_code: event.squad_code,
|
|
19
|
+
agent_id: event.agent_id,
|
|
20
|
+
source_type: 'agent_output',
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
console.error('[knowledge] auto-ingest failed:', err);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=agent-output.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-output.js","sourceRoot":"","sources":["../../src/sources/agent-output.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAUzB,MAAM,UAAU,qBAAqB,CAAC,QAAwB;IAC5D,OAAO,KAAK,UAAU,eAAe,CAAC,KAAuB;QAC3D,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;gBACvB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;oBACtC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxB,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE;4BAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;4BAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;4BACxB,WAAW,EAAE,cAAc;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,KAAK,CAAC,cAAc,EAAE,IAAI,EAAE,EAAE,CAAC;gBACjC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,iBAAiB,KAAK,CAAC,QAAQ,EAAE,EAAE;oBACjF,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,WAAW,EAAE,cAAc;iBAC5B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;QACxD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watcher.d.ts","sourceRoot":"","sources":["../../src/sources/watcher.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAI5D,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAAE,EACpB,QAAQ,EAAE,cAAc,GACvB;IAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAAC,IAAI,EAAE,MAAM,IAAI,CAAA;CAAE,CA2C5C"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import chokidar from 'chokidar';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
const SUPPORTED_EXTENSIONS = new Set(['.md', '.txt', '.json', '.yaml', '.yml', '.pdf']);
|
|
4
|
+
export function createKnowledgeWatcher(watchPaths, pipeline) {
|
|
5
|
+
const debounceTimers = new Map();
|
|
6
|
+
const watcher = chokidar.watch(watchPaths, {
|
|
7
|
+
ignoreInitial: false,
|
|
8
|
+
depth: 5,
|
|
9
|
+
ignored: (filePath) => {
|
|
10
|
+
const base = path.basename(filePath);
|
|
11
|
+
if (base.startsWith('.') || base.startsWith('_'))
|
|
12
|
+
return true;
|
|
13
|
+
return false;
|
|
14
|
+
},
|
|
15
|
+
});
|
|
16
|
+
function handleFile(filePath) {
|
|
17
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
18
|
+
if (!SUPPORTED_EXTENSIONS.has(ext))
|
|
19
|
+
return;
|
|
20
|
+
clearTimeout(debounceTimers.get(filePath));
|
|
21
|
+
debounceTimers.set(filePath, setTimeout(async () => {
|
|
22
|
+
try {
|
|
23
|
+
await pipeline.ingest(filePath);
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
console.error(`[knowledge-watcher] Failed to ingest ${filePath}:`, err);
|
|
27
|
+
}
|
|
28
|
+
}, 50));
|
|
29
|
+
}
|
|
30
|
+
watcher.on('add', handleFile);
|
|
31
|
+
watcher.on('change', handleFile);
|
|
32
|
+
const ready = new Promise((resolve) => {
|
|
33
|
+
watcher.on('ready', resolve);
|
|
34
|
+
});
|
|
35
|
+
const stop = () => {
|
|
36
|
+
for (const timer of debounceTimers.values())
|
|
37
|
+
clearTimeout(timer);
|
|
38
|
+
watcher.close();
|
|
39
|
+
};
|
|
40
|
+
return { ready, stop };
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=watcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../src/sources/watcher.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAExF,MAAM,UAAU,sBAAsB,CACpC,UAAoB,EACpB,QAAwB;IAExB,MAAM,cAAc,GAAG,IAAI,GAAG,EAAyC,CAAC;IAExE,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE;QACzC,aAAa,EAAE,KAAK;QACpB,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,CAAC,QAAgB,EAAE,EAAE;YAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC9D,OAAO,KAAK,CAAC;QACf,CAAC;KACF,CAAC,CAAC;IAEH,SAAS,UAAU,CAAC,QAAgB;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QAE3C,YAAY,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,cAAc,CAAC,GAAG,CAChB,QAAQ,EACR,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC,EAAE,EAAE,CAAC,CACP,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAEjC,MAAM,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC1C,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,MAAM,EAAE;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC,CAAC;IAEF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC"}
|