expxagents 0.30.7 → 0.30.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dashboard/assets/{BufferResource-5t0IawpH.js → BufferResource-hsxtSN5M.js} +1 -1
- package/dist/dashboard/assets/{CanvasRenderer-DucjAUv8.js → CanvasRenderer-CLlk_5v1.js} +1 -1
- package/dist/dashboard/assets/{JarvisView-DoUEtcGD.js → JarvisView-KyeSi1xX.js} +1 -1
- package/dist/dashboard/assets/{RenderTargetSystem-kqAucAnb.js → RenderTargetSystem-B_xRfycy.js} +1 -1
- package/dist/dashboard/assets/{ThreeBackground-BbUm_C_i.js → ThreeBackground-juOXHCKO.js} +1 -1
- package/dist/dashboard/assets/{WebGLRenderer-DgaLTjkO.js → WebGLRenderer-DYj9Hx4C.js} +1 -1
- package/dist/dashboard/assets/{WebGPURenderer-B4w0l2xv.js → WebGPURenderer-BlwJkjd4.js} +1 -1
- package/dist/dashboard/assets/{browserAll-JjSpWKy6.js → browserAll-KZ-3VUYt.js} +1 -1
- package/dist/dashboard/assets/{index-CXgCZOr4.js → index-DyGKrw_p.js} +12 -12
- package/dist/dashboard/assets/{webworkerAll-CRdlrERU.js → webworkerAll-BWnP0xU2.js} +1 -1
- package/dist/dashboard/index.html +1 -1
- package/dist/server/api/knowledge-routes.d.ts.map +1 -1
- package/dist/server/api/knowledge-routes.js +41 -0
- package/dist/server/api/knowledge-routes.js.map +1 -1
- package/dist/server/services/knowledge-extractor.d.ts +18 -0
- package/dist/server/services/knowledge-extractor.d.ts.map +1 -0
- package/dist/server/services/knowledge-extractor.js +116 -0
- package/dist/server/services/knowledge-extractor.js.map +1 -0
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{a1 as G,a3 as I,a4 as B,l as _,M as k,W as O,N as A,a9 as m,T as v,as as C,R as E,w as z,a8 as U,v as w}from"./index-
|
|
1
|
+
import{a1 as G,a3 as I,a4 as B,l as _,M as k,W as O,N as A,a9 as m,T as v,as as C,R as E,w as z,a8 as U,v as w}from"./index-DyGKrw_p.js";var M=`in vec2 aPosition;
|
|
2
2
|
out vec2 vTextureCoord;
|
|
3
3
|
|
|
4
4
|
uniform vec4 uInputSize;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>ExpxAgents — Mission Control</title>
|
|
7
|
-
<script type="module" crossorigin src="/assets/index-
|
|
7
|
+
<script type="module" crossorigin src="/assets/index-DyGKrw_p.js"></script>
|
|
8
8
|
<link rel="stylesheet" crossorigin href="/assets/index-DtbIzZ5n.css">
|
|
9
9
|
</head>
|
|
10
10
|
<body>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-routes.d.ts","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-routes.d.ts","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACrE,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAK3C,UAAU,sBAAuB,SAAQ,oBAAoB;IAC3D,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC;CACvB;AAED,wBAAsB,eAAe,CAAC,GAAG,EAAE,eAAe,EAAE,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkMvG"}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import multipart from '@fastify/multipart';
|
|
2
2
|
import { IngestPipeline, KnowledgeQuery, createEmbedder } from 'expxagents-knowledge';
|
|
3
|
+
import { KnowledgeExtractor } from '../services/knowledge-extractor.js';
|
|
3
4
|
export async function knowledgeRoutes(app, opts) {
|
|
4
5
|
const { db } = opts;
|
|
5
6
|
const embedder = createEmbedder('local');
|
|
6
7
|
const pipeline = new IngestPipeline(db, embedder, { max_tokens: 512, overlap: 0.2 });
|
|
7
8
|
const query = new KnowledgeQuery(db, embedder);
|
|
9
|
+
const extractor = new KnowledgeExtractor(db);
|
|
8
10
|
await app.register(multipart, { limits: { fileSize: 10 * 1024 * 1024 } });
|
|
9
11
|
// POST /api/knowledge/ingest — multipart file upload
|
|
10
12
|
app.post('/api/knowledge/ingest', { preHandler: [app.requireAuth] }, async (request, reply) => {
|
|
@@ -83,5 +85,44 @@ export async function knowledgeRoutes(app, opts) {
|
|
|
83
85
|
const updated = db.prepare("UPDATE documents SET extraction_status = 'complete' WHERE extraction_status IN ('pending', 'failed')").run();
|
|
84
86
|
return reply.send({ updated: updated.changes });
|
|
85
87
|
});
|
|
88
|
+
// POST /api/knowledge/extract — extract entities from a specific document
|
|
89
|
+
app.post('/api/knowledge/extract', { preHandler: [app.requireAuth] }, async (request, reply) => {
|
|
90
|
+
const { document_id } = request.body;
|
|
91
|
+
if (!document_id) {
|
|
92
|
+
return reply.status(400).send({ error: 'document_id is required' });
|
|
93
|
+
}
|
|
94
|
+
try {
|
|
95
|
+
const result = await extractor.extractDocument(document_id);
|
|
96
|
+
return reply.send({ ok: true, ...result });
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
100
|
+
return reply.status(500).send({ error: msg });
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
// POST /api/knowledge/extract-all — extract entities from all documents without entities
|
|
104
|
+
app.post('/api/knowledge/extract-all', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
105
|
+
const docs = db.prepare("SELECT id, title FROM documents WHERE extraction_status = 'complete'").all();
|
|
106
|
+
if (docs.length === 0) {
|
|
107
|
+
return reply.send({ processed: 0, results: [] });
|
|
108
|
+
}
|
|
109
|
+
const results = [];
|
|
110
|
+
for (const doc of docs) {
|
|
111
|
+
try {
|
|
112
|
+
const result = await extractor.extractDocument(doc.id);
|
|
113
|
+
results.push({ id: doc.id, title: doc.title, ...result });
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
117
|
+
results.push({ id: doc.id, title: doc.title, entities: 0, relations: 0, error: msg });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const stats = extractor.getStats();
|
|
121
|
+
return reply.send({ processed: results.length, results, stats });
|
|
122
|
+
});
|
|
123
|
+
// GET /api/knowledge/stats — use extractor for accurate counts
|
|
124
|
+
app.get('/api/knowledge/stats', { preHandler: [app.requireAuth] }, async (_request, reply) => {
|
|
125
|
+
return reply.send(extractor.getStats());
|
|
126
|
+
});
|
|
86
127
|
}
|
|
87
128
|
//# sourceMappingURL=knowledge-routes.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-routes.js","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-routes.js","sourceRoot":"","sources":["../../src/api/knowledge-routes.ts"],"names":[],"mappings":"AAEA,OAAO,SAAS,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtF,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AAMxE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAoB,EAAE,IAA4B;IACtF,MAAM,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC;IACpB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IAE7C,MAAM,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC;IAE1E,qDAAqD;IACrD,GAAG,CAAC,IAAI,CACN,uBAAuB,EACvB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QAErE,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAa,CAAC;QAClC,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CACF,CAAC;IAEF,kCAAkC;IAClC,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1D,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC,CACF,CAAC;IAEF,2BAA2B;IAC3B,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;QACpF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE;YAClC,IAAI,EAAG,IAAY,IAAI,MAAM;YAC7B,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,WAAkB,EAAE;YACjD,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE;SACxC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC,CACF,CAAC;IAEF,+BAA+B;IAC/B,GAAG,CAAC,GAAG,CACL,0BAA0B,EAC1B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACvE,CAAC,CACF,CAAC;IAEF,mCAAmC;IACnC,GAAG,CAAC,GAAG,CACL,8BAA8B,EAC9B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACzE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC,CACF,CAAC;IAEF,mDAAmD;IACnD,GAAG,CAAC,MAAM,CACR,8BAA8B,EAC9B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,IAAK,OAAe,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5C,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC,CACF,CAAC;IAEF,8BAA8B;IAC9B,GAAG,CAAC,GAAG,CACL,yBAAyB,EACzB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClC,CAAC,CACF,CAAC;IAEF,4CAA4C;IAC5C,GAAG,CAAC,GAAG,CACL,uCAAuC,EACvC,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,SAAS,GAAG,KAAK,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IACnC,CAAC,CACF,CAAC;IAEF,2BAA2B;IAC3B,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACtC,CAAC,CACF,CAAC;IAEF,mEAAmE;IACnE,GAAG,CAAC,IAAI,CACN,wBAAwB,EACxB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB,sGAAsG,CACvG,CAAC,GAAG,EAAE,CAAC;QACR,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAClD,CAAC,CACF,CAAC;IAEF,0EAA0E;IAC1E,GAAG,CAAC,IAAI,CACN,wBAAwB,EACxB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACvB,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;QACrC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;YAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CACF,CAAC;IAEF,yFAAyF;IACzF,GAAG,CAAC,IAAI,CACN,4BAA4B,EAC5B,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,sEAAsE,CACvE,CAAC,GAAG,EAAqC,CAAC;QAE3C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,OAAO,GAAyF,EAAE,CAAC;QAEzG,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;YAC5D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC,CACF,CAAC;IAEF,+DAA+D;IAC/D,GAAG,CAAC,GAAG,CACL,sBAAsB,EACtB,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EACjC,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;QACxB,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1C,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export declare class KnowledgeExtractor {
|
|
3
|
+
private db;
|
|
4
|
+
constructor(db: Database.Database);
|
|
5
|
+
private callLLM;
|
|
6
|
+
extractDocumentContent(documentId: string): string | null;
|
|
7
|
+
extractDocument(documentId: string): Promise<{
|
|
8
|
+
entities: number;
|
|
9
|
+
relations: number;
|
|
10
|
+
}>;
|
|
11
|
+
getStats(): {
|
|
12
|
+
documents: number;
|
|
13
|
+
chunks: number;
|
|
14
|
+
entities: number;
|
|
15
|
+
relations: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=knowledge-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-extractor.d.ts","sourceRoot":"","sources":["../../src/services/knowledge-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAI3C,qBAAa,kBAAkB;IACjB,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;YAE3B,OAAO;IA2DrB,sBAAsB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAWnD,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAwD3F,QAAQ,IAAI;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;CAOvF"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { nanoid } from 'nanoid';
|
|
2
|
+
import { buildExtractionPrompt, parseExtractionResponse, validateEntities, validateRelations } from 'expxagents-knowledge';
|
|
3
|
+
export class KnowledgeExtractor {
|
|
4
|
+
db;
|
|
5
|
+
constructor(db) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
}
|
|
8
|
+
async callLLM(prompt) {
|
|
9
|
+
const openrouterKey = process.env.OPENROUTER_API_KEY;
|
|
10
|
+
const openrouterModel = process.env.OPENROUTER_DEFAULT_MODEL ?? 'anthropic/claude-sonnet-4-20250514';
|
|
11
|
+
if (openrouterKey) {
|
|
12
|
+
console.log('[knowledge-extractor] Using OpenRouter for extraction');
|
|
13
|
+
const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
|
|
14
|
+
method: 'POST',
|
|
15
|
+
headers: {
|
|
16
|
+
'Content-Type': 'application/json',
|
|
17
|
+
'Authorization': `Bearer ${openrouterKey}`,
|
|
18
|
+
'HTTP-Referer': 'https://expxagents.local',
|
|
19
|
+
'X-Title': 'ExpxAgents Knowledge',
|
|
20
|
+
},
|
|
21
|
+
body: JSON.stringify({
|
|
22
|
+
model: openrouterModel,
|
|
23
|
+
messages: [{ role: 'user', content: prompt }],
|
|
24
|
+
max_tokens: 4096,
|
|
25
|
+
}),
|
|
26
|
+
});
|
|
27
|
+
if (!response.ok) {
|
|
28
|
+
const err = await response.text();
|
|
29
|
+
throw new Error(`OpenRouter error: ${response.status} - ${err}`);
|
|
30
|
+
}
|
|
31
|
+
const data = await response.json();
|
|
32
|
+
return data.choices[0].message.content;
|
|
33
|
+
}
|
|
34
|
+
const anthropicKey = process.env.ANTHROPIC_API_KEY;
|
|
35
|
+
if (anthropicKey) {
|
|
36
|
+
console.log('[knowledge-extractor] Using Anthropic Claude for extraction');
|
|
37
|
+
const response = await fetch('https://api.anthropic.com/v1/messages', {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
headers: {
|
|
40
|
+
'Content-Type': 'application/json',
|
|
41
|
+
'x-api-key': anthropicKey,
|
|
42
|
+
'anthropic-version': '2023-06-01',
|
|
43
|
+
},
|
|
44
|
+
body: JSON.stringify({
|
|
45
|
+
model: 'claude-sonnet-4-20250514',
|
|
46
|
+
max_tokens: 4096,
|
|
47
|
+
messages: [{ role: 'user', content: prompt }],
|
|
48
|
+
}),
|
|
49
|
+
});
|
|
50
|
+
if (!response.ok) {
|
|
51
|
+
const err = await response.text();
|
|
52
|
+
throw new Error(`Anthropic error: ${response.status} - ${err}`);
|
|
53
|
+
}
|
|
54
|
+
const data = await response.json();
|
|
55
|
+
return data.content[0].text;
|
|
56
|
+
}
|
|
57
|
+
throw new Error('No LLM available: set OPENROUTER_API_KEY or ANTHROPIC_API_KEY');
|
|
58
|
+
}
|
|
59
|
+
extractDocumentContent(documentId) {
|
|
60
|
+
const doc = this.db.prepare('SELECT title, source_type FROM documents WHERE id = ?').get(documentId);
|
|
61
|
+
if (!doc)
|
|
62
|
+
return null;
|
|
63
|
+
const chunks = this.db.prepare('SELECT content FROM chunks WHERE document_id = ? ORDER BY chunk_index').all(documentId);
|
|
64
|
+
return chunks.map(c => c.content).join('\n\n');
|
|
65
|
+
}
|
|
66
|
+
async extractDocument(documentId) {
|
|
67
|
+
const content = this.extractDocumentContent(documentId);
|
|
68
|
+
if (!content) {
|
|
69
|
+
throw new Error('Document not found or has no chunks');
|
|
70
|
+
}
|
|
71
|
+
const prompt = buildExtractionPrompt(content);
|
|
72
|
+
let response;
|
|
73
|
+
try {
|
|
74
|
+
response = await this.callLLM(prompt);
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
console.error('[knowledge-extractor] LLM call failed:', err);
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
const result = parseExtractionResponse(response);
|
|
81
|
+
const validEntities = validateEntities(result.entities);
|
|
82
|
+
const validRelations = validateRelations(result.relations);
|
|
83
|
+
console.log(`[knowledge-extractor] Extracted ${validEntities.length} entities and ${validRelations.length} relations`);
|
|
84
|
+
const now = new Date().toISOString();
|
|
85
|
+
const insertEntity = this.db.prepare('INSERT OR IGNORE INTO entities (id, name, entity_type, description, first_seen_in, mention_count, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 1, ?, ?)');
|
|
86
|
+
const insertRelation = this.db.prepare('INSERT OR REPLACE INTO relations (id, source_id, target_id, relation_type, description, weight, document_id, created_at) VALUES (?, ?, ?, ?, ?, 1.0, ?, ?, ?)');
|
|
87
|
+
const getEntityId = (name, type) => {
|
|
88
|
+
const existing = this.db.prepare('SELECT id FROM entities WHERE name = ? AND entity_type = ?').get(name, type);
|
|
89
|
+
if (existing) {
|
|
90
|
+
this.db.prepare('UPDATE entities SET mention_count = mention_count + 1, updated_at = ? WHERE id = ?').run(now, existing.id);
|
|
91
|
+
return existing.id;
|
|
92
|
+
}
|
|
93
|
+
const id = nanoid();
|
|
94
|
+
insertEntity.run(id, name, type, null, documentId, now, now);
|
|
95
|
+
return id;
|
|
96
|
+
};
|
|
97
|
+
for (const entity of validEntities) {
|
|
98
|
+
getEntityId(entity.name, entity.type);
|
|
99
|
+
}
|
|
100
|
+
for (const rel of validRelations) {
|
|
101
|
+
const sourceId = getEntityId(rel.source, 'concept');
|
|
102
|
+
const targetId = getEntityId(rel.target, 'concept');
|
|
103
|
+
insertRelation.run(nanoid(), sourceId, targetId, rel.type, rel.description ?? null, documentId, now, now);
|
|
104
|
+
}
|
|
105
|
+
this.db.prepare("UPDATE documents SET extraction_status = 'complete', updated_at = ? WHERE id = ?").run(now, documentId);
|
|
106
|
+
return { entities: validEntities.length, relations: validRelations.length };
|
|
107
|
+
}
|
|
108
|
+
getStats() {
|
|
109
|
+
const documents = this.db.prepare('SELECT COUNT(*) as count FROM documents').get().count;
|
|
110
|
+
const chunks = this.db.prepare('SELECT COUNT(*) as count FROM chunks').get().count;
|
|
111
|
+
const entities = this.db.prepare('SELECT COUNT(*) as count FROM entities').get().count;
|
|
112
|
+
const relations = this.db.prepare('SELECT COUNT(*) as count FROM relations').get().count;
|
|
113
|
+
return { documents, chunks, entities, relations };
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=knowledge-extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledge-extractor.js","sourceRoot":"","sources":["../../src/services/knowledge-extractor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAE3H,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,EAAqB;QAArB,OAAE,GAAF,EAAE,CAAmB;IAAG,CAAC;IAErC,KAAK,CAAC,OAAO,CAAC,MAAc;QAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC;QACrD,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,oCAAoC,CAAC;QAErG,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;YACrE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+CAA+C,EAAE;gBAC5E,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,aAAa,EAAE;oBAC1C,cAAc,EAAE,0BAA0B;oBAC1C,SAAS,EAAE,sBAAsB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,eAAe;oBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;oBAC7C,UAAU,EAAE,IAAI;iBACjB,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QACnD,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;YAC3E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;gBACpE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,WAAW,EAAE,YAAY;oBACzB,mBAAmB,EAAE,YAAY;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,0BAA0B;oBACjC,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;iBAC9C,CAAC;aACH,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAS,CAAC;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,CAAC;IAED,sBAAsB,CAAC,UAAkB;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC,GAAG,CAAC,UAAU,CAAuD,CAAC;QAC3J,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC5B,uEAAuE,CACxE,CAAC,GAAG,CAAC,UAAU,CAA0B,CAAC;QAE3C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,UAAkB;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;QACxD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAE9C,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;YAC7D,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,MAAM,MAAM,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACxD,MAAM,cAAc,GAAG,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO,CAAC,GAAG,CAAC,mCAAmC,aAAa,CAAC,MAAM,iBAAiB,cAAc,CAAC,MAAM,YAAY,CAAC,CAAC;QAEvH,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAClC,2JAA2J,CAC5J,CAAC;QACF,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACpC,+JAA+J,CAChK,CAAC;QAEF,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,EAAE;YACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAA+B,CAAC;YAC7I,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oFAAoF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC5H,OAAO,QAAQ,CAAC,EAAE,CAAC;YACrB,CAAC;YACD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;YACpB,YAAY,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC7D,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpD,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC5G,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAEzH,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,CAAC;IAC9E,CAAC;IAED,QAAQ;QACN,MAAM,SAAS,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAChH,MAAM,MAAM,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC1G,MAAM,QAAQ,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAC9G,MAAM,SAAS,GAAI,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC,GAAG,EAAwB,CAAC,KAAK,CAAC;QAChH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACpD,CAAC;CACF"}
|