expxagents 0.21.1 → 0.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/expxagents-knowledge/dist/config.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/config.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/config.js +23 -0
- package/node_modules/expxagents-knowledge/dist/config.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.d.ts +6 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.js +69 -0
- package/node_modules/expxagents-knowledge/dist/db/connection.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.d.ts +3 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.js +46 -0
- package/node_modules/expxagents-knowledge/dist/db/migrations.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.d.ts +3 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.js +79 -0
- package/node_modules/expxagents-knowledge/dist/db/schema.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/index.d.ts +15 -0
- package/node_modules/expxagents-knowledge/dist/index.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/index.js +15 -0
- package/node_modules/expxagents-knowledge/dist/index.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.d.ts +8 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.js +139 -0
- package/node_modules/expxagents-knowledge/dist/ingest/chunker.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js +39 -0
- package/node_modules/expxagents-knowledge/dist/ingest/document-loader.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts +4 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.js +25 -0
- package/node_modules/expxagents-knowledge/dist/ingest/embedder.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts +21 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js +54 -0
- package/node_modules/expxagents-knowledge/dist/ingest/entity-extractor.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts +16 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js +49 -0
- package/node_modules/expxagents-knowledge/dist/ingest/extraction-queue.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.d.ts +27 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js +83 -0
- package/node_modules/expxagents-knowledge/dist/ingest/pipeline.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.d.ts +41 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.js +62 -0
- package/node_modules/expxagents-knowledge/dist/query/graph-traversal.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.d.ts +31 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.js +106 -0
- package/node_modules/expxagents-knowledge/dist/query/knowledge-query.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.d.ts +26 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.js +57 -0
- package/node_modules/expxagents-knowledge/dist/query/vector-search.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.d.ts +10 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.js +29 -0
- package/node_modules/expxagents-knowledge/dist/sources/agent-output.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.d.ts +6 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.js +42 -0
- package/node_modules/expxagents-knowledge/dist/sources/watcher.js.map +1 -0
- package/node_modules/expxagents-knowledge/dist/types.d.ts +123 -0
- package/node_modules/expxagents-knowledge/dist/types.d.ts.map +1 -0
- package/node_modules/expxagents-knowledge/dist/types.js +2 -0
- package/node_modules/expxagents-knowledge/dist/types.js.map +1 -0
- package/node_modules/expxagents-knowledge/package.json +7 -0
- package/package.json +8 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,cAAc,EAAE,eAQ5B,CAAC;AAEF,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,eAAe,CAa1F"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
export const DEFAULT_CONFIG = {
|
|
3
|
+
enabled: true,
|
|
4
|
+
db_path: '',
|
|
5
|
+
embedder: { provider: 'auto', batch_size: 32 },
|
|
6
|
+
chunker: { max_tokens: 512, overlap: 0.2 },
|
|
7
|
+
graph: { enabled: true, extract_on_ingest: true, dedup_threshold: 0.85 },
|
|
8
|
+
auto_ingest: { agent_outputs: true, activity_logs: false, watch_paths: [] },
|
|
9
|
+
limits: { max_document_size_mb: 10, max_documents: 10000, max_entities: 50000 },
|
|
10
|
+
};
|
|
11
|
+
export function loadKnowledgeConfig(projectRoot, dataDir) {
|
|
12
|
+
const config = structuredClone(DEFAULT_CONFIG);
|
|
13
|
+
config.db_path = path.join(dataDir ?? path.join(projectRoot, '.expxagents', 'data'), 'knowledge.db');
|
|
14
|
+
// Env var overrides (take precedence over yaml)
|
|
15
|
+
if (process.env.KNOWLEDGE_ENABLED !== undefined) {
|
|
16
|
+
config.enabled = process.env.KNOWLEDGE_ENABLED !== 'false';
|
|
17
|
+
}
|
|
18
|
+
if (process.env.KNOWLEDGE_EMBEDDER_PROVIDER) {
|
|
19
|
+
config.embedder.provider = process.env.KNOWLEDGE_EMBEDDER_PROVIDER;
|
|
20
|
+
}
|
|
21
|
+
return config;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,CAAC,MAAM,cAAc,GAAoB;IAC7C,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,EAAE;IACX,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE;IAC9C,OAAO,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE;IAC1C,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE;IACxE,WAAW,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE;IAC3E,MAAM,EAAE,EAAE,oBAAoB,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE;CAChF,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,WAAmB,EAAE,OAAgB;IACvE,MAAM,MAAM,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAC/C,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,cAAc,CAAC,CAAC;IAErG,gDAAgD;IAChD,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAChD,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,OAAO,CAAC;IAC7D,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,CAAC;QAC5C,MAAM,CAAC,QAAQ,CAAC,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,2BAAsE,CAAC;IAChH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
export declare function getKnowledgeDb(dbPath: string): Database.Database;
|
|
3
|
+
export declare function isVecLoaded(): boolean;
|
|
4
|
+
export declare function closeKnowledgeDb(): void;
|
|
5
|
+
export declare function createTestDb(): Database.Database;
|
|
6
|
+
//# sourceMappingURL=connection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAsBtC,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAsChE;AAED,wBAAgB,WAAW,IAAI,OAAO,CAAsB;AAE5D,wBAAgB,gBAAgB,IAAI,IAAI,CAOvC;AAID,wBAAgB,YAAY,IAAI,QAAQ,CAAC,QAAQ,CAKhD"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import { createRequire } from 'node:module';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { runKnowledgeMigrations } from './migrations.js';
|
|
6
|
+
const _require = createRequire(import.meta.url);
|
|
7
|
+
let knowledgeDb = null;
|
|
8
|
+
let currentDbPath = null;
|
|
9
|
+
let vecLoaded = false;
|
|
10
|
+
function tryLoadSqliteVec(db) {
|
|
11
|
+
try {
|
|
12
|
+
const sqliteVec = _require('sqlite-vec');
|
|
13
|
+
sqliteVec.load(db);
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export function getKnowledgeDb(dbPath) {
|
|
21
|
+
if (knowledgeDb) {
|
|
22
|
+
if (currentDbPath && currentDbPath !== dbPath) {
|
|
23
|
+
throw new Error(`Knowledge DB already initialized at ${currentDbPath}, cannot reinitialize at ${dbPath}. Call closeKnowledgeDb() first.`);
|
|
24
|
+
}
|
|
25
|
+
return knowledgeDb;
|
|
26
|
+
}
|
|
27
|
+
const dir = path.dirname(dbPath);
|
|
28
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
29
|
+
currentDbPath = dbPath;
|
|
30
|
+
knowledgeDb = new Database(dbPath);
|
|
31
|
+
runKnowledgeMigrations(knowledgeDb);
|
|
32
|
+
vecLoaded = tryLoadSqliteVec(knowledgeDb);
|
|
33
|
+
// chunk_embeddings tables are created outside the migration system because
|
|
34
|
+
// the virtual table dimensions may vary by embedder configuration at runtime.
|
|
35
|
+
if (vecLoaded) {
|
|
36
|
+
knowledgeDb.exec(`
|
|
37
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0(
|
|
38
|
+
chunk_id TEXT PRIMARY KEY,
|
|
39
|
+
embedding float[384]
|
|
40
|
+
);
|
|
41
|
+
`);
|
|
42
|
+
}
|
|
43
|
+
// Always create fallback store table
|
|
44
|
+
knowledgeDb.exec(`
|
|
45
|
+
CREATE TABLE IF NOT EXISTS chunk_embeddings_store (
|
|
46
|
+
chunk_id TEXT PRIMARY KEY,
|
|
47
|
+
embedding TEXT NOT NULL
|
|
48
|
+
);
|
|
49
|
+
`);
|
|
50
|
+
return knowledgeDb;
|
|
51
|
+
}
|
|
52
|
+
export function isVecLoaded() { return vecLoaded; }
|
|
53
|
+
export function closeKnowledgeDb() {
|
|
54
|
+
if (knowledgeDb) {
|
|
55
|
+
knowledgeDb.close();
|
|
56
|
+
knowledgeDb = null;
|
|
57
|
+
currentDbPath = null;
|
|
58
|
+
vecLoaded = false;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// For tests — create an isolated in-memory DB with migrations + fallback store.
|
|
62
|
+
// Note: this bypasses the singleton, so isVecLoaded() is not meaningful after createTestDb().
|
|
63
|
+
export function createTestDb() {
|
|
64
|
+
const db = new Database(':memory:');
|
|
65
|
+
runKnowledgeMigrations(db);
|
|
66
|
+
db.exec('CREATE TABLE IF NOT EXISTS chunk_embeddings_store (chunk_id TEXT PRIMARY KEY, embedding TEXT NOT NULL)');
|
|
67
|
+
return db;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=connection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEhD,IAAI,WAAW,GAA6B,IAAI,CAAC;AACjD,IAAI,aAAa,GAAkB,IAAI,CAAC;AACxC,IAAI,SAAS,GAAG,KAAK,CAAC;AAEtB,SAAS,gBAAgB,CAAC,EAAqB;IAC7C,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;QACzC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,aAAa,IAAI,aAAa,KAAK,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CACb,uCAAuC,aAAa,4BAA4B,MAAM,kCAAkC,CACzH,CAAC;QACJ,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvC,aAAa,GAAG,MAAM,CAAC;IACvB,WAAW,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnC,sBAAsB,CAAC,WAAW,CAAC,CAAC;IACpC,SAAS,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAE1C,2EAA2E;IAC3E,8EAA8E;IAC9E,IAAI,SAAS,EAAE,CAAC;QACd,WAAW,CAAC,IAAI,CAAC;;;;;KAKhB,CAAC,CAAC;IACL,CAAC;IAED,qCAAqC;IACrC,WAAW,CAAC,IAAI,CAAC;;;;;GAKhB,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,WAAW,KAAc,OAAO,SAAS,CAAC,CAAC,CAAC;AAE5D,MAAM,UAAU,gBAAgB;IAC9B,IAAI,WAAW,EAAE,CAAC;QAChB,WAAW,CAAC,KAAK,EAAE,CAAC;QACpB,WAAW,GAAG,IAAI,CAAC;QACnB,aAAa,GAAG,IAAI,CAAC;QACrB,SAAS,GAAG,KAAK,CAAC;IACpB,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,8FAA8F;AAC9F,MAAM,UAAU,YAAY;IAC1B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;IACpC,sBAAsB,CAAC,EAAE,CAAC,CAAC;IAC3B,EAAE,CAAC,IAAI,CAAC,wGAAwG,CAAC,CAAC;IAClH,OAAO,EAAE,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/db/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAuB3C,wBAAgB,sBAAsB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAuClE"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { KNOWLEDGE_SCHEMA_V1, KNOWLEDGE_SCHEMA_V2 } from './schema.js';
|
|
2
|
+
const migrations = [
|
|
3
|
+
{
|
|
4
|
+
version: 1,
|
|
5
|
+
up: (db) => {
|
|
6
|
+
db.exec(KNOWLEDGE_SCHEMA_V1);
|
|
7
|
+
},
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
version: 2,
|
|
11
|
+
up: (db) => {
|
|
12
|
+
db.exec(KNOWLEDGE_SCHEMA_V2);
|
|
13
|
+
},
|
|
14
|
+
},
|
|
15
|
+
];
|
|
16
|
+
export function runKnowledgeMigrations(db) {
|
|
17
|
+
db.pragma('journal_mode = WAL');
|
|
18
|
+
db.pragma('foreign_keys = ON');
|
|
19
|
+
// Ensure meta table exists
|
|
20
|
+
db.exec(`
|
|
21
|
+
CREATE TABLE IF NOT EXISTS knowledge_meta (
|
|
22
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
23
|
+
schema_version INTEGER NOT NULL DEFAULT 0,
|
|
24
|
+
embedder_provider TEXT,
|
|
25
|
+
embedding_dimensions INTEGER,
|
|
26
|
+
updated_at TEXT NOT NULL
|
|
27
|
+
);
|
|
28
|
+
`);
|
|
29
|
+
const meta = db.prepare('SELECT schema_version FROM knowledge_meta WHERE id = 1').get();
|
|
30
|
+
let currentVersion = meta?.schema_version ?? 0;
|
|
31
|
+
if (!meta) {
|
|
32
|
+
db.prepare('INSERT INTO knowledge_meta (id, schema_version, updated_at) VALUES (1, 0, ?)').run(new Date().toISOString());
|
|
33
|
+
}
|
|
34
|
+
const applyMigration = db.transaction((m) => {
|
|
35
|
+
m.up(db);
|
|
36
|
+
db.prepare('UPDATE knowledge_meta SET schema_version = ?, updated_at = ? WHERE id = 1')
|
|
37
|
+
.run(m.version, new Date().toISOString());
|
|
38
|
+
});
|
|
39
|
+
for (const migration of migrations) {
|
|
40
|
+
if (migration.version > currentVersion) {
|
|
41
|
+
applyMigration(migration);
|
|
42
|
+
currentVersion = migration.version;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=migrations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/db/migrations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAOvE,MAAM,UAAU,GAAgB;IAC9B;QACE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACT,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,CAAC;KACF;IACD;QACE,OAAO,EAAE,CAAC;QACV,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACT,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC/B,CAAC;KACF;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,EAAqB;IAC1D,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAChC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAE/B,2BAA2B;IAC3B,EAAE,CAAC,IAAI,CAAC;;;;;;;;GAQP,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC,GAAG,EAExE,CAAC;IAEd,IAAI,cAAc,GAAG,IAAI,EAAE,cAAc,IAAI,CAAC,CAAC;IAE/C,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,EAAE,CAAC,OAAO,CAAC,8EAA8E,CAAC,CAAC,GAAG,CAC5F,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACzB,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAY,EAAE,EAAE;QACrD,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACT,EAAE,CAAC,OAAO,CAAC,2EAA2E,CAAC;aACpF,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,SAAS,CAAC,OAAO,GAAG,cAAc,EAAE,CAAC;YACvC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC1B,cAAc,GAAG,SAAS,CAAC,OAAO,CAAC;QACrC,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export declare const KNOWLEDGE_SCHEMA_V1 = "\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_type TEXT NOT NULL CHECK (source_type IN ('external', 'agent_output', 'squad_memory', 'activity_log')),\n source_path TEXT,\n squad_code TEXT,\n agent_id TEXT,\n title TEXT NOT NULL,\n mime_type TEXT,\n content_hash TEXT NOT NULL,\n extraction_status TEXT NOT NULL DEFAULT 'pending' CHECK (extraction_status IN ('pending', 'complete', 'failed', 'skipped')),\n metadata TEXT,\n ingested_at TEXT NOT NULL,\n updated_at TEXT NOT NULL,\n UNIQUE(source_path, content_hash)\n);\n\nCREATE TABLE IF NOT EXISTS chunks (\n id TEXT PRIMARY KEY,\n document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,\n content TEXT NOT NULL,\n chunk_index INTEGER NOT NULL,\n token_count INTEGER,\n metadata TEXT,\n created_at TEXT NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_documents_squad ON documents(squad_code);\nCREATE INDEX IF NOT EXISTS idx_documents_source_type ON documents(source_type);\nCREATE INDEX IF NOT EXISTS idx_documents_content_hash ON documents(content_hash);\nCREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_id);\n";
|
|
2
|
+
export declare const KNOWLEDGE_SCHEMA_V2 = "\nCREATE TABLE IF NOT EXISTS entities (\n id TEXT PRIMARY KEY,\n name TEXT NOT NULL,\n entity_type TEXT NOT NULL CHECK (entity_type IN ('technology', 'person', 'decision', 'concept', 'squad', 'agent', 'learning')),\n description TEXT,\n properties TEXT,\n first_seen_in TEXT REFERENCES documents(id),\n mention_count INTEGER NOT NULL DEFAULT 1,\n created_at TEXT NOT NULL,\n updated_at TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS relations (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,\n target_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,\n relation_type TEXT NOT NULL CHECK (relation_type IN ('produced', 'depends_on', 'learned_from', 'contradicts', 'decided', 'uses', 'replaced')),\n description TEXT,\n weight REAL NOT NULL DEFAULT 1.0,\n document_id TEXT REFERENCES documents(id),\n created_at TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS extraction_jobs (\n id TEXT PRIMARY KEY,\n document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,\n status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'complete', 'failed')),\n attempts INTEGER NOT NULL DEFAULT 0,\n error TEXT,\n created_at TEXT NOT NULL,\n updated_at TEXT NOT NULL\n);\n\nCREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);\nCREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);\nCREATE INDEX IF NOT EXISTS idx_relations_source ON relations(source_id);\nCREATE INDEX IF NOT EXISTS idx_relations_target ON relations(target_id);\nCREATE INDEX IF NOT EXISTS idx_relations_type ON relations(relation_type);\nCREATE INDEX IF NOT EXISTS idx_extraction_jobs_status ON extraction_jobs(status);\n\nCREATE TABLE IF NOT EXISTS entity_embeddings_store (\n entity_id TEXT PRIMARY KEY,\n embedding TEXT NOT NULL\n);\n";
|
|
3
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,ynCA+B/B,CAAC;AAEF,eAAO,MAAM,mBAAmB,8yDA6C/B,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export const KNOWLEDGE_SCHEMA_V1 = `
|
|
2
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
3
|
+
id TEXT PRIMARY KEY,
|
|
4
|
+
source_type TEXT NOT NULL CHECK (source_type IN ('external', 'agent_output', 'squad_memory', 'activity_log')),
|
|
5
|
+
source_path TEXT,
|
|
6
|
+
squad_code TEXT,
|
|
7
|
+
agent_id TEXT,
|
|
8
|
+
title TEXT NOT NULL,
|
|
9
|
+
mime_type TEXT,
|
|
10
|
+
content_hash TEXT NOT NULL,
|
|
11
|
+
extraction_status TEXT NOT NULL DEFAULT 'pending' CHECK (extraction_status IN ('pending', 'complete', 'failed', 'skipped')),
|
|
12
|
+
metadata TEXT,
|
|
13
|
+
ingested_at TEXT NOT NULL,
|
|
14
|
+
updated_at TEXT NOT NULL,
|
|
15
|
+
UNIQUE(source_path, content_hash)
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
19
|
+
id TEXT PRIMARY KEY,
|
|
20
|
+
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
21
|
+
content TEXT NOT NULL,
|
|
22
|
+
chunk_index INTEGER NOT NULL,
|
|
23
|
+
token_count INTEGER,
|
|
24
|
+
metadata TEXT,
|
|
25
|
+
created_at TEXT NOT NULL
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
CREATE INDEX IF NOT EXISTS idx_documents_squad ON documents(squad_code);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS idx_documents_source_type ON documents(source_type);
|
|
30
|
+
CREATE INDEX IF NOT EXISTS idx_documents_content_hash ON documents(content_hash);
|
|
31
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_document ON chunks(document_id);
|
|
32
|
+
`;
|
|
33
|
+
export const KNOWLEDGE_SCHEMA_V2 = `
|
|
34
|
+
CREATE TABLE IF NOT EXISTS entities (
|
|
35
|
+
id TEXT PRIMARY KEY,
|
|
36
|
+
name TEXT NOT NULL,
|
|
37
|
+
entity_type TEXT NOT NULL CHECK (entity_type IN ('technology', 'person', 'decision', 'concept', 'squad', 'agent', 'learning')),
|
|
38
|
+
description TEXT,
|
|
39
|
+
properties TEXT,
|
|
40
|
+
first_seen_in TEXT REFERENCES documents(id),
|
|
41
|
+
mention_count INTEGER NOT NULL DEFAULT 1,
|
|
42
|
+
created_at TEXT NOT NULL,
|
|
43
|
+
updated_at TEXT NOT NULL
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
CREATE TABLE IF NOT EXISTS relations (
|
|
47
|
+
id TEXT PRIMARY KEY,
|
|
48
|
+
source_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
|
49
|
+
target_id TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
|
50
|
+
relation_type TEXT NOT NULL CHECK (relation_type IN ('produced', 'depends_on', 'learned_from', 'contradicts', 'decided', 'uses', 'replaced')),
|
|
51
|
+
description TEXT,
|
|
52
|
+
weight REAL NOT NULL DEFAULT 1.0,
|
|
53
|
+
document_id TEXT REFERENCES documents(id),
|
|
54
|
+
created_at TEXT NOT NULL
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
CREATE TABLE IF NOT EXISTS extraction_jobs (
|
|
58
|
+
id TEXT PRIMARY KEY,
|
|
59
|
+
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
60
|
+
status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'complete', 'failed')),
|
|
61
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
62
|
+
error TEXT,
|
|
63
|
+
created_at TEXT NOT NULL,
|
|
64
|
+
updated_at TEXT NOT NULL
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);
|
|
68
|
+
CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);
|
|
69
|
+
CREATE INDEX IF NOT EXISTS idx_relations_source ON relations(source_id);
|
|
70
|
+
CREATE INDEX IF NOT EXISTS idx_relations_target ON relations(target_id);
|
|
71
|
+
CREATE INDEX IF NOT EXISTS idx_relations_type ON relations(relation_type);
|
|
72
|
+
CREATE INDEX IF NOT EXISTS idx_extraction_jobs_status ON extraction_jobs(status);
|
|
73
|
+
|
|
74
|
+
CREATE TABLE IF NOT EXISTS entity_embeddings_store (
|
|
75
|
+
entity_id TEXT PRIMARY KEY,
|
|
76
|
+
embedding TEXT NOT NULL
|
|
77
|
+
);
|
|
78
|
+
`;
|
|
79
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BlC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6ClC,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export * from './types.js';
|
|
2
|
+
export { IngestPipeline } from './ingest/pipeline.js';
|
|
3
|
+
export { KnowledgeQuery } from './query/knowledge-query.js';
|
|
4
|
+
export { GraphTraversal } from './query/graph-traversal.js';
|
|
5
|
+
export { createMockEmbedder, createEmbedder } from './ingest/embedder.js';
|
|
6
|
+
export { getKnowledgeDb, closeKnowledgeDb, createTestDb } from './db/connection.js';
|
|
7
|
+
export { runKnowledgeMigrations } from './db/migrations.js';
|
|
8
|
+
export { loadDocument, computeHash } from './ingest/document-loader.js';
|
|
9
|
+
export { chunkMarkdown } from './ingest/chunker.js';
|
|
10
|
+
export { VectorSearch } from './query/vector-search.js';
|
|
11
|
+
export { parseExtractionResponse, validateEntities, validateRelations, buildExtractionPrompt } from './ingest/entity-extractor.js';
|
|
12
|
+
export { ExtractionQueue } from './ingest/extraction-queue.js';
|
|
13
|
+
export { createAgentOutputHook } from './sources/agent-output.js';
|
|
14
|
+
export { createKnowledgeWatcher } from './sources/watcher.js';
|
|
15
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACnI,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export * from './types.js';
|
|
2
|
+
export { IngestPipeline } from './ingest/pipeline.js';
|
|
3
|
+
export { KnowledgeQuery } from './query/knowledge-query.js';
|
|
4
|
+
export { GraphTraversal } from './query/graph-traversal.js';
|
|
5
|
+
export { createMockEmbedder, createEmbedder } from './ingest/embedder.js';
|
|
6
|
+
export { getKnowledgeDb, closeKnowledgeDb, createTestDb } from './db/connection.js';
|
|
7
|
+
export { runKnowledgeMigrations } from './db/migrations.js';
|
|
8
|
+
export { loadDocument, computeHash } from './ingest/document-loader.js';
|
|
9
|
+
export { chunkMarkdown } from './ingest/chunker.js';
|
|
10
|
+
export { VectorSearch } from './query/vector-search.js';
|
|
11
|
+
export { parseExtractionResponse, validateEntities, validateRelations, buildExtractionPrompt } from './ingest/entity-extractor.js';
|
|
12
|
+
export { ExtractionQueue } from './ingest/extraction-queue.js';
|
|
13
|
+
export { createAgentOutputHook } from './sources/agent-output.js';
|
|
14
|
+
export { createKnowledgeWatcher } from './sources/watcher.js';
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACpF,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACnI,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ChunkResult } from '../types.js';
|
|
2
|
+
interface ChunkerOptions {
|
|
3
|
+
max_tokens: number;
|
|
4
|
+
overlap: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function chunkMarkdown(content: string, options: ChunkerOptions): ChunkResult[];
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/ingest/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,UAAU,cAAc;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACjB;AAWD,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,WAAW,EAAE,CA+BrF"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
function estimateTokens(text) {
|
|
2
|
+
return Math.ceil(text.length / 4);
|
|
3
|
+
}
|
|
4
|
+
export function chunkMarkdown(content, options) {
|
|
5
|
+
if (!content.trim())
|
|
6
|
+
return [];
|
|
7
|
+
const sections = splitByHeadings(content);
|
|
8
|
+
const results = [];
|
|
9
|
+
let chunkIndex = 0;
|
|
10
|
+
for (const section of sections) {
|
|
11
|
+
const sectionTokens = estimateTokens(section.content);
|
|
12
|
+
if (sectionTokens <= options.max_tokens) {
|
|
13
|
+
results.push({
|
|
14
|
+
content: section.content,
|
|
15
|
+
chunk_index: chunkIndex++,
|
|
16
|
+
token_count: sectionTokens,
|
|
17
|
+
metadata: { heading_path: section.heading_path },
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
const subChunks = splitByParagraphs(section.content, options);
|
|
22
|
+
for (const sub of subChunks) {
|
|
23
|
+
results.push({
|
|
24
|
+
content: sub,
|
|
25
|
+
chunk_index: chunkIndex++,
|
|
26
|
+
token_count: estimateTokens(sub),
|
|
27
|
+
metadata: { heading_path: section.heading_path },
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return results;
|
|
33
|
+
}
|
|
34
|
+
function splitByHeadings(content) {
|
|
35
|
+
const lines = content.split('\n');
|
|
36
|
+
const sections = [];
|
|
37
|
+
let currentHeadings = [];
|
|
38
|
+
let currentLines = [];
|
|
39
|
+
for (const line of lines) {
|
|
40
|
+
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
41
|
+
if (headingMatch) {
|
|
42
|
+
if (currentLines.length > 0) {
|
|
43
|
+
const text = currentLines.join('\n').trim();
|
|
44
|
+
if (text) {
|
|
45
|
+
sections.push({
|
|
46
|
+
heading_path: currentHeadings.join(' > ') || '',
|
|
47
|
+
content: text,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
currentLines = [];
|
|
51
|
+
}
|
|
52
|
+
const level = headingMatch[1].length;
|
|
53
|
+
const title = headingMatch[2].trim();
|
|
54
|
+
currentHeadings = currentHeadings.slice(0, level - 1);
|
|
55
|
+
currentHeadings[level - 1] = title;
|
|
56
|
+
currentHeadings = currentHeadings.slice(0, level);
|
|
57
|
+
currentLines.push(line);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
currentLines.push(line);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (currentLines.length > 0) {
|
|
64
|
+
const text = currentLines.join('\n').trim();
|
|
65
|
+
if (text) {
|
|
66
|
+
sections.push({
|
|
67
|
+
heading_path: currentHeadings.join(' > ') || '',
|
|
68
|
+
content: text,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return sections;
|
|
73
|
+
}
|
|
74
|
+
function splitByParagraphs(content, options) {
|
|
75
|
+
const blocks = splitPreservingCodeBlocks(content);
|
|
76
|
+
const chunks = [];
|
|
77
|
+
let current = [];
|
|
78
|
+
let currentTokens = 0;
|
|
79
|
+
for (const block of blocks) {
|
|
80
|
+
const blockTokens = estimateTokens(block);
|
|
81
|
+
if (currentTokens + blockTokens > options.max_tokens && current.length > 0) {
|
|
82
|
+
chunks.push(current.join('\n\n'));
|
|
83
|
+
if (options.overlap > 0) {
|
|
84
|
+
const overlapTokens = Math.floor(options.max_tokens * options.overlap);
|
|
85
|
+
const kept = [];
|
|
86
|
+
let keptTokens = 0;
|
|
87
|
+
for (let i = current.length - 1; i >= 0; i--) {
|
|
88
|
+
const t = estimateTokens(current[i]);
|
|
89
|
+
if (keptTokens + t > overlapTokens)
|
|
90
|
+
break;
|
|
91
|
+
kept.unshift(current[i]);
|
|
92
|
+
keptTokens += t;
|
|
93
|
+
}
|
|
94
|
+
current = kept;
|
|
95
|
+
currentTokens = keptTokens;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
current = [];
|
|
99
|
+
currentTokens = 0;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
current.push(block);
|
|
103
|
+
currentTokens += blockTokens;
|
|
104
|
+
}
|
|
105
|
+
if (current.length > 0) {
|
|
106
|
+
chunks.push(current.join('\n\n'));
|
|
107
|
+
}
|
|
108
|
+
return chunks;
|
|
109
|
+
}
|
|
110
|
+
function splitPreservingCodeBlocks(content) {
|
|
111
|
+
const blocks = [];
|
|
112
|
+
const parts = content.split(/\n\n/);
|
|
113
|
+
let inCodeBlock = false;
|
|
114
|
+
let codeBuffer = [];
|
|
115
|
+
for (const part of parts) {
|
|
116
|
+
const fenceCount = (part.match(/```/g) || []).length;
|
|
117
|
+
if (inCodeBlock) {
|
|
118
|
+
codeBuffer.push(part);
|
|
119
|
+
if (fenceCount % 2 === 1) {
|
|
120
|
+
blocks.push(codeBuffer.join('\n\n'));
|
|
121
|
+
codeBuffer = [];
|
|
122
|
+
inCodeBlock = false;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
else if (fenceCount % 2 === 1) {
|
|
126
|
+
inCodeBlock = true;
|
|
127
|
+
codeBuffer.push(part);
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
if (part.trim())
|
|
131
|
+
blocks.push(part);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (codeBuffer.length > 0) {
|
|
135
|
+
blocks.push(codeBuffer.join('\n\n'));
|
|
136
|
+
}
|
|
137
|
+
return blocks;
|
|
138
|
+
}
|
|
139
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/ingest/chunker.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAOD,MAAM,UAAU,aAAa,CAAC,OAAe,EAAE,OAAuB;IACpE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAE/B,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEtD,IAAI,aAAa,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,WAAW,EAAE,UAAU,EAAE;gBACzB,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE;aACjD,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9D,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,GAAG;oBACZ,WAAW,EAAE,UAAU,EAAE;oBACzB,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC;oBAChC,QAAQ,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,eAAe,GAAa,EAAE,CAAC;IACnC,IAAI,YAAY,GAAa,EAAE,CAAC;IAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,CAAC,IAAI,CAAC;wBACZ,YAAY,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;wBAC/C,OAAO,EAAE,IAAI;qBACd,CAAC,CAAC;gBACL,CAAC;gBACD,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACrC,eAAe,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACtD,eAAe,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;YACnC,eAAe,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAClD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC;gBACZ,YAAY,EAAE,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;gBAC/C,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,OAAe,EAAE,OAAuB;IACjE,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QAE1C,IAAI,aAAa,GAAG,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAClC,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBACvE,MAAM,IAAI,GAAa,EAAE,CAAC;gBAC1B,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,UAAU,GAAG,CAAC,GAAG,aAAa;wBAAE,MAAM;oBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzB,UAAU,IAAI,CAAC,CAAC;gBAClB,CAAC;gBACD,OAAO,GAAG,IAAI,CAAC;gBACf,aAAa,GAAG,UAAU,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,EAAE,CAAC;gBACb,aAAa,GAAG,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,aAAa,IAAI,WAAW,CAAC;IAC/B,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,yBAAyB,CAAC,OAAe;IAChD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAErD,IAAI,WAAW,EAAE,CAAC;YAChB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBACrC,UAAU,GAAG,EAAE,CAAC;gBAChB,WAAW,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;aAAM,IAAI,UAAU,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,GAAG,IAAI,CAAC;YACnB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,IAAI,IAAI,CAAC,IAAI,EAAE;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-loader.d.ts","sourceRoot":"","sources":["../../src/ingest/document-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAalD,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,SAAK,GAAG,cAAc,CAiB7E"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
const MIME_MAP = {
|
|
5
|
+
'.md': 'text/markdown',
|
|
6
|
+
'.markdown': 'text/markdown',
|
|
7
|
+
'.txt': 'text/plain',
|
|
8
|
+
'.json': 'application/json',
|
|
9
|
+
'.yaml': 'text/yaml',
|
|
10
|
+
'.yml': 'text/yaml',
|
|
11
|
+
};
|
|
12
|
+
const SUPPORTED_EXTENSIONS = new Set(Object.keys(MIME_MAP));
|
|
13
|
+
export function computeHash(content) {
|
|
14
|
+
return crypto.createHash('sha256').update(content).digest('hex');
|
|
15
|
+
}
|
|
16
|
+
export function loadDocument(filePath, maxSizeMb = 10) {
|
|
17
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
18
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
19
|
+
throw new Error(`Unsupported file type: ${ext}`);
|
|
20
|
+
}
|
|
21
|
+
const stat = fs.statSync(filePath);
|
|
22
|
+
const sizeMb = stat.size / (1024 * 1024);
|
|
23
|
+
if (sizeMb > maxSizeMb) {
|
|
24
|
+
throw new Error(`File too large: ${sizeMb.toFixed(1)}MB exceeds limit of ${maxSizeMb}MB`);
|
|
25
|
+
}
|
|
26
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
27
|
+
const mime_type = MIME_MAP[ext];
|
|
28
|
+
const title = extractTitle(content, filePath, mime_type);
|
|
29
|
+
return { title, content, mime_type };
|
|
30
|
+
}
|
|
31
|
+
function extractTitle(content, filePath, mimeType) {
|
|
32
|
+
if (mimeType === 'text/markdown') {
|
|
33
|
+
const match = content.match(/^#{1,6}\s+(.+)$/m);
|
|
34
|
+
if (match)
|
|
35
|
+
return match[1].trim();
|
|
36
|
+
}
|
|
37
|
+
return path.basename(filePath);
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=document-loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-loader.js","sourceRoot":"","sources":["../../src/ingest/document-loader.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,QAAQ,GAA2B;IACvC,KAAK,EAAE,eAAe;IACtB,WAAW,EAAE,eAAe;IAC5B,MAAM,EAAE,YAAY;IACpB,OAAO,EAAE,kBAAkB;IAC3B,OAAO,EAAE,WAAW;IACpB,MAAM,EAAE,WAAW;CACpB,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE5D,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,SAAS,GAAG,EAAE;IAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACzC,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,SAAS,IAAI,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAE,CAAC;IACjC,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAEzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,OAAe,EAAE,QAAgB,EAAE,QAAgB;IACvE,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAChD,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/ingest/embedder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAiBzD;AAGD,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAGzD"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
2
|
+
export function createMockEmbedder(dims) {
|
|
3
|
+
return {
|
|
4
|
+
dimensions: dims,
|
|
5
|
+
async embed(texts) {
|
|
6
|
+
return texts.map((text) => {
|
|
7
|
+
// Deterministic hash-based vector for testing
|
|
8
|
+
const hash = crypto.createHash('sha256').update(text).digest();
|
|
9
|
+
const vector = [];
|
|
10
|
+
for (let i = 0; i < dims; i++) {
|
|
11
|
+
vector.push((hash[i % hash.length] / 255) * 2 - 1);
|
|
12
|
+
}
|
|
13
|
+
// Normalize
|
|
14
|
+
const norm = Math.sqrt(vector.reduce((s, v) => s + v * v, 0));
|
|
15
|
+
return vector.map((v) => v / norm);
|
|
16
|
+
});
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
// Placeholder for future real implementations
|
|
21
|
+
export function createEmbedder(provider) {
|
|
22
|
+
// For now, always use mock. Real providers added in follow-up.
|
|
23
|
+
return createMockEmbedder(384);
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/ingest/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AAGjC,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,KAAK,CAAC,KAAK,CAAC,KAAe;YACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACxB,8CAA8C;gBAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC/D,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACrD,CAAC;gBACD,YAAY;gBACZ,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,+DAA+D;IAC/D,OAAO,kBAAkB,CAAC,GAAG,CAAC,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
interface RawEntity {
|
|
2
|
+
name: string;
|
|
3
|
+
type: string;
|
|
4
|
+
description?: string;
|
|
5
|
+
}
|
|
6
|
+
interface RawRelation {
|
|
7
|
+
source: string;
|
|
8
|
+
target: string;
|
|
9
|
+
type: string;
|
|
10
|
+
description?: string;
|
|
11
|
+
}
|
|
12
|
+
interface ExtractionResult {
|
|
13
|
+
entities: RawEntity[];
|
|
14
|
+
relations: RawRelation[];
|
|
15
|
+
}
|
|
16
|
+
export declare function parseExtractionResponse(response: string): ExtractionResult;
|
|
17
|
+
export declare function validateEntities(entities: RawEntity[]): RawEntity[];
|
|
18
|
+
export declare function validateRelations(relations: RawRelation[]): RawRelation[];
|
|
19
|
+
export declare function buildExtractionPrompt(content: string): string;
|
|
20
|
+
export {};
|
|
21
|
+
//# sourceMappingURL=entity-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entity-extractor.d.ts","sourceRoot":"","sources":["../../src/ingest/entity-extractor.ts"],"names":[],"mappings":"AAwBA,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,WAAW;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,UAAU,gBAAgB;IACxB,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtB,SAAS,EAAE,WAAW,EAAE,CAAC;CAC1B;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,gBAAgB,CAW1E;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAEnE;AAED,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,WAAW,EAAE,CAIzE;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAgB7D"}
|