expxagents 0.30.6 → 0.30.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dashboard/assets/{BufferResource-CjsnSf42.js → BufferResource-hsxtSN5M.js} +1 -1
- package/dist/dashboard/assets/{CanvasRenderer-B_BGtDGv.js → CanvasRenderer-CLlk_5v1.js} +1 -1
- package/dist/dashboard/assets/{JarvisView-CjQi3aoo.js → JarvisView-KyeSi1xX.js} +1 -1
- package/dist/dashboard/assets/{RenderTargetSystem-CvRMI3ss.js → RenderTargetSystem-B_xRfycy.js} +1 -1
- package/dist/dashboard/assets/{ThreeBackground-DxwBr08u.js → ThreeBackground-juOXHCKO.js} +1 -1
- package/dist/dashboard/assets/{WebGLRenderer-yAiRn_ne.js → WebGLRenderer-DYj9Hx4C.js} +1 -1
- package/dist/dashboard/assets/{WebGPURenderer-BxFqovmL.js → WebGPURenderer-BlwJkjd4.js} +1 -1
- package/dist/dashboard/assets/{browserAll-DMNx9Ffc.js → browserAll-KZ-3VUYt.js} +1 -1
- package/dist/dashboard/assets/{index-B9UJ5mUd.js → index-DyGKrw_p.js} +12 -12
- package/dist/dashboard/assets/{webworkerAll-Dfvzix_M.js → webworkerAll-BWnP0xU2.js} +1 -1
- package/dist/dashboard/index.html +1 -1
- package/dist/server/api/knowledge-routes.d.ts.map +1 -1
- package/dist/server/api/knowledge-routes.js +46 -0
- package/dist/server/api/knowledge-routes.js.map +1 -1
- package/dist/server/services/knowledge-extractor.d.ts +18 -0
- package/dist/server/services/knowledge-extractor.d.ts.map +1 -0
- package/dist/server/services/knowledge-extractor.js +116 -0
- package/dist/server/services/knowledge-extractor.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js +2 -2
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a1 as G,a3 as I,a4 as B,l as _,M as k,W as O,N as A,a9 as m,T as v,as as C,R as E,w as z,a8 as U,v as w}from"./index-
|
|
1
|
+
import{a1 as G,a3 as I,a4 as B,l as _,M as k,W as O,N as A,a9 as m,T as v,as as C,R as E,w as z,a8 as U,v as w}from"./index-DyGKrw_p.js";var M=`in vec2 aPosition;
|
|
2
2
|
out vec2 vTextureCoord;
|
|
3
3
|
|
|
4
4
|
uniform vec4 uInputSize;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>ExpxAgents — Mission Control</title>
|
|
7
|
-
<script type="module" crossorigin src="/assets/index-
|
|
7
|
+
<script type="module" crossorigin src="/assets/index-DyGKrw_p.js"></script>
|
|
8
8
|
<link rel="stylesheet" crossorigin href="/assets/index-DtbIzZ5n.css">
|
|
9
9
|
</head>
|
|
10
10
|
<body>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-routes.d.ts","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-routes.d.ts","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAK3C,UAAU,sBAAuB,SAAQ,oBAAoB;IAC3D,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC;CACvB;AAED,wBAAsB,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkMvG"}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import multipart from '@fastify/multipart';
|
|
2
2
|
import { IngestPipeline, KnowledgeQuery, createEmbedder } from 'expxagents-knowledge';
|
|
3
|
+
import { KnowledgeExtractor } from '../services/knowledge-extractor.js';
|
|
3
4
|
export async function knowledgeRoutes(app, opts) {
|
|
4
5
|
const { db } = opts;
|
|
5
6
|
const embedder = createEmbedder('local');
|
|
6
7
|
const pipeline = new IngestPipeline(db, embedder, { max_tokens: 512, overlap: 0.2 });
|
|
7
8
|
const query = new KnowledgeQuery(db, embedder);
|
|
9
|
+
const extractor = new KnowledgeExtractor(db);
|
|
8
10
|
await app.register(multipart, { limits: { fileSize: 10 * 1024 * 1024 } });
|
|
9
11
|
// POST /api/knowledge/ingest — multipart file upload
|
|
10
12
|
app.post('/api/knowledge/ingest', { preHandler: [app.requireAuth] }, async (request, reply) => {
|
|
@@ -78,5 +80,49 @@ export async function knowledgeRoutes(app, opts) {
|
|
|
78
80
|
app.get('/api/knowledge/stats', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
79
81
|
return reply.send(query.getStats());
|
|
80
82
|
});
|
|
83
|
+
// POST /api/knowledge/rebuild — reprocess pending/failed documents
|
|
84
|
+
app.post('/api/knowledge/rebuild', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
85
|
+
const updated = db.prepare("UPDATE documents SET extraction_status = 'complete' WHERE extraction_status IN ('pending', 'failed')").run();
|
|
86
|
+
return reply.send({ updated: updated.changes });
|
|
87
|
+
});
|
|
88
|
+
// POST /api/knowledge/extract — extract entities from a specific document
|
|
89
|
+
app.post('/api/knowledge/extract', { preHandler: [app.requireAuth] }, async (request, reply) => {
|
|
90
|
+
const { document_id } = request.body;
|
|
91
|
+
if (!document_id) {
|
|
92
|
+
return reply.status(400).send({ error: 'document_id is required' });
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
const result = await extractor.extractDocument(document_id);
|
|
96
|
+
return reply.send({ ok: true, ...result });
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
100
|
+
return reply.status(500).send({ error: msg });
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
// POST /api/knowledge/extract-all — extract entities from all documents without entities
|
|
104
|
+
app.post('/api/knowledge/extract-all', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
105
|
+
const docs = db.prepare("SELECT id, title FROM documents WHERE extraction_status = 'complete'").all();
|
|
106
|
+
if (docs.length === 0) {
|
|
107
|
+
return reply.send({ processed: 0, results: [] });
|
|
108
|
+
}
|
|
109
|
+
const results = [];
|
|
110
|
+
for (const doc of docs) {
|
|
111
|
+
try {
|
|
112
|
+
const result = await extractor.extractDocument(doc.id);
|
|
113
|
+
results.push({ id: doc.id, title: doc.title, ...result });
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
117
|
+
results.push({ id: doc.id, title: doc.title, entities: 0, relations: 0, error: msg });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const stats = extractor.getStats();
|
|
121
|
+
return reply.send({ processed: results.length, results, stats });
|
|
122
|
+
});
|
|
123
|
+
// GET /api/knowledge/stats — use extractor for accurate counts
|
|
124
|
+
app.get('/api/knowledge/stats', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
125
|
+
return reply.send(extractor.getStats());
|
|
126
|
+
});
|
|
81
127
|
}
|
|
82
128
|
//# sourceMappingURL=knowledge-routes.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-routes.js","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-routes.js","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AAMxE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAoB,EAAE,IAA4B;IACtF,MAAM,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC;IACpB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE7C,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC;IAE1E,qDAAqD;IACrD,GAAG,CAAC,IAAI,CACN,uBAAuB,EACvB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAErE,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAa,CAAC;QAClC,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CACF,CAAC;IAEF,kCAAkC;IAClC,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1D,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CACF,CAAC;IAEF,2BAA2B;IAC3B,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE;YAClC,IAAI,EAAG,IAAY,IAAI,MAAM;YAC7B,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,WAAkB,EAAE;YACjD,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;SACxC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC,CACF,CAAC;IAEF,+BAA+B;IAC/B,GAAG,CAAC,GAAG,CACL,0BAA0B,EAC1B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC,CACF,CAAC;IAEF,mCAAmC;IACnC,GAAG,CAAC,GAAG,CACL,8BAA8B,EAC9B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACzE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC,CACF,CAAC;IAEF,mDAAmD;IACnD,GAAG,CAAC,MAAM,CACR,8BAA8B,EAC9B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,IAAK,OAAe,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC,CACF,CAAC;IAEF,8BAA8B;IAC9B,GAAG,CAAC,GAAG,CACL,yBAAyB,EACzB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClC,CAAC,CACF,CAAC;IAEF,4CAA4C;IAC5C,GAAG,CAAC,GAAG,CACL,uCAAuC,EACvC,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,SAAS,GAAG,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IACnC,CAAC,CACF,CAAC;IAEF,2BAA2B;IAC3B,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtC,CAAC,CACF,CAAC;IAEF,mEAAmE;IACnE,GAAG,CAAC,IAAI,CACN,wBAAwB,EACxB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB,sGAAsG,CACvG,CAAC,GAAG,EAAE,CAAC;QACR,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAClD,CAAC,CACF,CAAC;IAEF,0EAA0E;IAC1E,GAAG,CAAC,IAAI,CACN,wBAAwB,EACxB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QACrC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CACF,CAAC;IAEF,yFAAyF;IACzF,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,sEAAsE,CACvE,CAAC,GAAG,EAAqC,CAAC;QAE3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,OAAO,GAAyF,EAAE,CAAC;QAEzG,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC,CACF,CAAC;IAEF,+DAA+D;IAC/D,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1C,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export declare class KnowledgeExtractor {
|
|
3
|
+
private db;
|
|
4
|
+
constructor(db: Database.Database);
|
|
5
|
+
private callLLM;
|
|
6
|
+
extractDocumentContent(documentId: string): string | null;
|
|
7
|
+
extractDocument(documentId: string): Promise<{
|
|
8
|
+
entities: number;
|
|
9
|
+
relations: number;
|
|
10
|
+
}>;
|
|
11
|
+
getStats(): {
|
|
12
|
+
documents: number;
|
|
13
|
+
chunks: number;
|
|
14
|
+
entities: number;
|
|
15
|
+
relations: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=knowledge-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-extractor.d.ts","sourceRoot":"","sources":["../../src/services/knowledge-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAI3C,qBAAa,kBAAkB;IACjB,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;YAE3B,OAAO;IA2DrB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAWnD,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAwD3F,QAAQ,IAAI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAOvF"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { nanoid } from 'nanoid';
|
|
2
|
+
import { buildExtractionPrompt, parseExtractionResponse, validateEntities, validateRelations } from 'expxagents-knowledge';
|
|
3
|
+
export class KnowledgeExtractor {
|
|
4
|
+
db;
|
|
5
|
+
constructor(db) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
}
|
|
8
|
+
async callLLM(prompt) {
|
|
9
|
+
const openrouterKey = process.env.OPENROUTER_API_KEY;
|
|
10
|
+
const openrouterModel = process.env.OPENROUTER_DEFAULT_MODEL ?? 'anthropic/claude-sonnet-4-20250514';
|
|
11
|
+
if (openrouterKey) {
|
|
12
|
+
console.log('[knowledge-extractor] Using OpenRouter for extraction');
|
|
13
|
+
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
|
14
|
+
method: 'POST',
|
|
15
|
+
headers: {
|
|
16
|
+
'Content-Type': 'application/json',
|
|
17
|
+
'Authorization': `Bearer ${openrouterKey}`,
|
|
18
|
+
'HTTP-Referer': 'https://expxagents.local',
|
|
19
|
+
'X-Title': 'ExpxAgents Knowledge',
|
|
20
|
+
},
|
|
21
|
+
body: JSON.stringify({
|
|
22
|
+
model: openrouterModel,
|
|
23
|
+
messages: [{ role: 'user', content: prompt }],
|
|
24
|
+
max_tokens: 4096,
|
|
25
|
+
}),
|
|
26
|
+
});
|
|
27
|
+
if (!response.ok) {
|
|
28
|
+
const err = await response.text();
|
|
29
|
+
throw new Error(`OpenRouter error: ${response.status} - ${err}`);
|
|
30
|
+
}
|
|
31
|
+
const data = await response.json();
|
|
32
|
+
return data.choices[0].message.content;
|
|
33
|
+
}
|
|
34
|
+
const anthropicKey = process.env.ANTHROPIC_API_KEY;
|
|
35
|
+
if (anthropicKey) {
|
|
36
|
+
console.log('[knowledge-extractor] Using Anthropic Claude for extraction');
|
|
37
|
+
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
headers: {
|
|
40
|
+
'Content-Type': 'application/json',
|
|
41
|
+
'x-api-key': anthropicKey,
|
|
42
|
+
'anthropic-version': '2023-06-01',
|
|
43
|
+
},
|
|
44
|
+
body: JSON.stringify({
|
|
45
|
+
model: 'claude-sonnet-4-20250514',
|
|
46
|
+
max_tokens: 4096,
|
|
47
|
+
messages: [{ role: 'user', content: prompt }],
|
|
48
|
+
}),
|
|
49
|
+
});
|
|
50
|
+
if (!response.ok) {
|
|
51
|
+
const err = await response.text();
|
|
52
|
+
throw new Error(`Anthropic error: ${response.status} - ${err}`);
|
|
53
|
+
}
|
|
54
|
+
const data = await response.json();
|
|
55
|
+
return data.content[0].text;
|
|
56
|
+
}
|
|
57
|
+
throw new Error('No LLM available: set OPENROUTER_API_KEY or ANTHROPIC_API_KEY');
|
|
58
|
+
}
|
|
59
|
+
extractDocumentContent(documentId) {
|
|
60
|
+
const doc = this.db.prepare('SELECT title, source_type FROM documents WHERE id = ?').get(documentId);
|
|
61
|
+
if (!doc)
|
|
62
|
+
return null;
|
|
63
|
+
const chunks = this.db.prepare('SELECT content FROM chunks WHERE document_id = ? ORDER BY chunk_index').all(documentId);
|
|
64
|
+
return chunks.map(c => c.content).join('\n\n');
|
|
65
|
+
}
|
|
66
|
+
async extractDocument(documentId) {
|
|
67
|
+
const content = this.extractDocumentContent(documentId);
|
|
68
|
+
if (!content) {
|
|
69
|
+
throw new Error('Document not found or has no chunks');
|
|
70
|
+
}
|
|
71
|
+
const prompt = buildExtractionPrompt(content);
|
|
72
|
+
let response;
|
|
73
|
+
try {
|
|
74
|
+
response = await this.callLLM(prompt);
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
console.error('[knowledge-extractor] LLM call failed:', err);
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
const result = parseExtractionResponse(response);
|
|
81
|
+
const validEntities = validateEntities(result.entities);
|
|
82
|
+
const validRelations = validateRelations(result.relations);
|
|
83
|
+
console.log(`[knowledge-extractor] Extracted ${validEntities.length} entities and ${validRelations.length} relations`);
|
|
84
|
+
const now = new Date().toISOString();
|
|
85
|
+
const insertEntity = this.db.prepare('INSERT OR IGNORE INTO entities (id, name, entity_type, description, first_seen_in, mention_count, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 1, ?, ?)');
|
|
86
|
+
const insertRelation = this.db.prepare('INSERT OR REPLACE INTO relations (id, source_id, target_id, relation_type, description, weight, document_id, created_at) VALUES (?, ?, ?, ?, ?, 1.0, ?, ?, ?)');
|
|
87
|
+
const getEntityId = (name, type) => {
|
|
88
|
+
const existing = this.db.prepare('SELECT id FROM entities WHERE name = ? AND entity_type = ?').get(name, type);
|
|
89
|
+
if (existing) {
|
|
90
|
+
this.db.prepare('UPDATE entities SET mention_count = mention_count + 1, updated_at = ? WHERE id = ?').run(now, existing.id);
|
|
91
|
+
return existing.id;
|
|
92
|
+
}
|
|
93
|
+
const id = nanoid();
|
|
94
|
+
insertEntity.run(id, name, type, null, documentId, now, now);
|
|
95
|
+
return id;
|
|
96
|
+
};
|
|
97
|
+
for (const entity of validEntities) {
|
|
98
|
+
getEntityId(entity.name, entity.type);
|
|
99
|
+
}
|
|
100
|
+
for (const rel of validRelations) {
|
|
101
|
+
const sourceId = getEntityId(rel.source, 'concept');
|
|
102
|
+
const targetId = getEntityId(rel.target, 'concept');
|
|
103
|
+
insertRelation.run(nanoid(), sourceId, targetId, rel.type, rel.description ?? null, documentId, now, now);
|
|
104
|
+
}
|
|
105
|
+
this.db.prepare("UPDATE documents SET extraction_status = 'complete', updated_at = ? WHERE id = ?").run(now, documentId);
|
|
106
|
+
return { entities: validEntities.length, relations: validRelations.length };
|
|
107
|
+
}
|
|
108
|
+
getStats() {
|
|
109
|
+
const documents = this.db.prepare('SELECT COUNT(*) as count FROM documents').get().count;
|
|
110
|
+
const chunks = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get().count;
|
|
111
|
+
const entities = this.db.prepare('SELECT COUNT(*) as count FROM entities').get().count;
|
|
112
|
+
const relations = this.db.prepare('SELECT COUNT(*) as count FROM relations').get().count;
|
|
113
|
+
return { documents, chunks, entities, relations };
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=knowledge-extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-extractor.js","sourceRoot":"","sources":["../../src/services/knowledge-extractor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAE3H,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,EAAqB;QAArB,OAAE,GAAF,EAAE,CAAmB;IAAG,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,MAAc;QAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACrD,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,oCAAoC,CAAC;QAErG,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;YACrE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,aAAa,EAAE;oBAC1C,cAAc,EAAE,0BAA0B;oBAC1C,SAAS,EAAE,sBAAsB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,eAAe;oBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;oBAC7C,UAAU,EAAE,IAAI;iBACjB,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACnD,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;YAC3E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;gBACpE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,WAAW,EAAE,YAAY;oBACzB,mBAAmB,EAAE,YAAY;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,0BAA0B;oBACjC,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;iBAC9C,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IAED,sBAAsB,CAAC,UAAkB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC,GAAG,CAAC,UAAU,CAAuD,CAAC;QAC3J,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC5B,uEAAuE,CACxE,CAAC,GAAG,CAAC,UAAU,CAA0B,CAAC;QAE3C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,UAAkB;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAE9C,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;YAC7D,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,cAAc,GAAG,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO,CAAC,GAAG,CAAC,mCAAmC,aAAa,CAAC,MAAM,iBAAiB,cAAc,CAAC,MAAM,YAAY,CAAC,CAAC;QAEvH,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAClC,2JAA2J,CAC5J,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACpC,+JAA+J,CAChK,CAAC;QAEF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAA+B,CAAC;YAC7I,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oFAAoF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC5H,OAAO,QAAQ,CAAC,EAAE,CAAC;YACrB,CAAC;YACD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;YACpB,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC7D,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpD,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAEzH,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;IAC9E,CAAC;IAED,QAAQ;QACN,MAAM,SAAS,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAChH,MAAM,MAAM,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC1G,MAAM,QAAQ,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC9G,MAAM,SAAS,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAChH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACpD,CAAC;CACF"}
|
|
@@ -35,7 +35,7 @@ export class IngestPipeline {
|
|
|
35
35
|
const metadata = Object.keys(docMetadata).length > 0 ? JSON.stringify(docMetadata) : null;
|
|
36
36
|
this.db
|
|
37
37
|
.prepare(`INSERT INTO documents (id, source_type, source_path, squad_code, agent_id, title, mime_type, content_hash, extraction_status, metadata, ingested_at, updated_at)
|
|
38
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, '
|
|
38
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'complete', ?, ?, ?)`)
|
|
39
39
|
.run(docId, sourceType, filePath, options?.squad_code ?? null, options?.agent_id ?? null, parsed.title, parsed.mime_type, contentHash, metadata, now, now);
|
|
40
40
|
// Chunk
|
|
41
41
|
const chunks = parsed.mime_type === 'application/pdf'
|
|
@@ -71,7 +71,7 @@ export class IngestPipeline {
|
|
|
71
71
|
const metadata = options?.tags ? JSON.stringify({ tags: options.tags }) : null;
|
|
72
72
|
this.db
|
|
73
73
|
.prepare(`INSERT INTO documents (id, source_type, source_path, squad_code, agent_id, title, mime_type, content_hash, extraction_status, metadata, ingested_at, updated_at)
|
|
74
|
-
VALUES (?, ?, NULL, ?, ?, ?, 'text/markdown', ?, '
|
|
74
|
+
VALUES (?, ?, NULL, ?, ?, ?, 'text/markdown', ?, 'complete', ?, ?, ?)`)
|
|
75
75
|
.run(docId, sourceType, options?.squad_code ?? null, options?.agent_id ?? null, title, contentHash, metadata, now, now);
|
|
76
76
|
const chunks = chunkMarkdown(content, this.chunkerConfig);
|
|
77
77
|
const texts = chunks.map((c) => c.content);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAqB7D,MAAM,OAAO,cAAc;IAEf;IACA;IACA;IAHV,YACU,EAAqB,EACrB,QAAkB,EAClB,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAAmB;QACrB,aAAQ,GAAR,QAAQ,CAAU;QAClB,kBAAa,GAAb,aAAa,CAAe;IACnC,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,OAAuB;QACpD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,KAAK,iBAAiB;YACxD,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;QAEtD,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,OAAO,CAAC,qEAAqE,CAAC;aAC9E,GAAG,CAAC,QAAQ,EAAE,WAAW,CAA+B,CAAC;QAE5D,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACtE,CAAC;QAED,6EAA6E;QAC7E,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE7E,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG;YAClB,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1F,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;
|
|
1
|
+
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AAEzB,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAqB7D,MAAM,OAAO,cAAc;IAEf;IACA;IACA;IAHV,YACU,EAAqB,EACrB,QAAkB,EAClB,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAAmB;QACrB,aAAQ,GAAR,QAAQ,CAAU;QAClB,kBAAa,GAAb,aAAa,CAAe;IACnC,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,QAAgB,EAAE,OAAuB;QACpD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,KAAK,iBAAiB;YACxD,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YACxC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,UAAU,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;QAEtD,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,OAAO,CAAC,qEAAqE,CAAC;aAC9E,GAAG,CAAC,QAAQ,EAAE,WAAW,CAA+B,CAAC;QAE5D,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACtE,CAAC;QAED,6EAA6E;QAC7E,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE7E,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG;YAClB,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE1F,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;8DACsD,CACvD;aACA,GAAG,CACF,KAAK,EACL,UAAU,EACV,QAAQ,EACR,OAAO,EAAE,UAAU,IAAI,IAAI,EAC3B,OAAO,EAAE,QAAQ,IAAI,IAAI,EACzB,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,SAAS,EAChB,WAAW,EACX,QAAQ,EACR,GAAG,EACH,GAAG,CACJ,CAAC;QAEJ,QAAQ;QACR,MAAM,MAAM,GAAkB,MAAM,CAAC,SAAS,KAAK,iBAAiB;YAClE,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC;YACpD,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAEtD,4BAA4B;QAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACjC,4HAA4H,CAC7H,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACrC,wEAAwE,CACzE,CAAC;QAEF,cAAc;QACd,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5E,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CACb,OAAO,EACP,KAAK,EACL,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EACjB,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EACrB,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EACrB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAClC,GAAG,CACJ,CAAC;gBACF,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;QACH,WAAW,EAAE,CAAC;QAEd,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,UAAU,CACd,OAAe,EACf,KAAa,EACb,OAAuB;QAEvB,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEzC,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,OAAO,CAAC,uFAAuF,CAAC;aAChG,GAAG,CAAC,WAAW,EAAE,KAAK,CAA+B,CAAC;QAEzD,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;QACtD,MAAM,QAAQ,GAAG,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE/E,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;+EACuE,CACxE;aACA,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,IAAI,IAAI,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAE1H,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5E,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACjC,4HAA4H,CAC7H,CAAC;QACF,MAAM,eAAe,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACrC,wEAAwE,CACzE,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;gBAC1I,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;QACH,WAAW,EAAE,CAAC;QAEd,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC7E,CAAC;CACF"}
|