@alextis59/athena 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +147 -0
- package/dist/agent/ask.d.ts +7 -0
- package/dist/agent/ask.d.ts.map +1 -0
- package/dist/agent/ask.js +35 -0
- package/dist/agent/ask.js.map +1 -0
- package/dist/agent/citations.d.ts +27 -0
- package/dist/agent/citations.d.ts.map +1 -0
- package/dist/agent/citations.js +190 -0
- package/dist/agent/citations.js.map +1 -0
- package/dist/agent/fake-provider.d.ts +9 -0
- package/dist/agent/fake-provider.d.ts.map +1 -0
- package/dist/agent/fake-provider.js +51 -0
- package/dist/agent/fake-provider.js.map +1 -0
- package/dist/agent/index.d.ts +10 -0
- package/dist/agent/index.d.ts.map +1 -0
- package/dist/agent/index.js +6 -0
- package/dist/agent/index.js.map +1 -0
- package/dist/agent/loop.d.ts +11 -0
- package/dist/agent/loop.d.ts.map +1 -0
- package/dist/agent/loop.js +156 -0
- package/dist/agent/loop.js.map +1 -0
- package/dist/agent/providers.d.ts +17 -0
- package/dist/agent/providers.d.ts.map +1 -0
- package/dist/agent/providers.js +171 -0
- package/dist/agent/providers.js.map +1 -0
- package/dist/agent/tools.d.ts +8 -0
- package/dist/agent/tools.d.ts.map +1 -0
- package/dist/agent/tools.js +233 -0
- package/dist/agent/tools.js.map +1 -0
- package/dist/agent/types.d.ts +84 -0
- package/dist/agent/types.d.ts.map +1 -0
- package/dist/agent/types.js +2 -0
- package/dist/agent/types.js.map +1 -0
- package/dist/cli/help.d.ts +2 -0
- package/dist/cli/help.d.ts.map +1 -0
- package/dist/cli/help.js +28 -0
- package/dist/cli/help.js.map +1 -0
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +5 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/options.d.ts +21 -0
- package/dist/cli/options.d.ts.map +1 -0
- package/dist/cli/options.js +97 -0
- package/dist/cli/options.js.map +1 -0
- package/dist/cli/run.d.ts +21 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +144 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +3 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +14 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +352 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/schema.d.ts +6 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +72 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/types.d.ts +53 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +2 -0
- package/dist/config/types.js.map +1 -0
- package/dist/doctor/doctor.d.ts +47 -0
- package/dist/doctor/doctor.d.ts.map +1 -0
- package/dist/doctor/doctor.js +159 -0
- package/dist/doctor/doctor.js.map +1 -0
- package/dist/embeddings/index.d.ts +117 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +515 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/errors.d.ts +19 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +37 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer/discovery.d.ts +10 -0
- package/dist/indexer/discovery.d.ts.map +1 -0
- package/dist/indexer/discovery.js +38 -0
- package/dist/indexer/discovery.js.map +1 -0
- package/dist/indexer/hash.d.ts +3 -0
- package/dist/indexer/hash.d.ts.map +1 -0
- package/dist/indexer/hash.js +11 -0
- package/dist/indexer/hash.js.map +1 -0
- package/dist/indexer/index.d.ts +7 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +5 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/kinds.d.ts +4 -0
- package/dist/indexer/kinds.d.ts.map +1 -0
- package/dist/indexer/kinds.js +38 -0
- package/dist/indexer/kinds.js.map +1 -0
- package/dist/indexer/run.d.ts +42 -0
- package/dist/indexer/run.d.ts.map +1 -0
- package/dist/indexer/run.js +563 -0
- package/dist/indexer/run.js.map +1 -0
- package/dist/logging/logger.d.ts +11 -0
- package/dist/logging/logger.d.ts.map +1 -0
- package/dist/logging/logger.js +18 -0
- package/dist/logging/logger.js.map +1 -0
- package/dist/mcp/index.d.ts +35 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +392 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/parsers/chunking.d.ts +4 -0
- package/dist/parsers/chunking.d.ts.map +1 -0
- package/dist/parsers/chunking.js +112 -0
- package/dist/parsers/chunking.js.map +1 -0
- package/dist/parsers/index.d.ts +8 -0
- package/dist/parsers/index.d.ts.map +1 -0
- package/dist/parsers/index.js +5 -0
- package/dist/parsers/index.js.map +1 -0
- package/dist/parsers/line-parser.d.ts +13 -0
- package/dist/parsers/line-parser.d.ts.map +1 -0
- package/dist/parsers/line-parser.js +242 -0
- package/dist/parsers/line-parser.js.map +1 -0
- package/dist/parsers/normalize.d.ts +4 -0
- package/dist/parsers/normalize.d.ts.map +1 -0
- package/dist/parsers/normalize.js +17 -0
- package/dist/parsers/normalize.js.map +1 -0
- package/dist/parsers/parser.d.ts +14 -0
- package/dist/parsers/parser.d.ts.map +1 -0
- package/dist/parsers/parser.js +77 -0
- package/dist/parsers/parser.js.map +1 -0
- package/dist/parsers/pdf2llm.d.ts +14 -0
- package/dist/parsers/pdf2llm.d.ts.map +1 -0
- package/dist/parsers/pdf2llm.js +215 -0
- package/dist/parsers/pdf2llm.js.map +1 -0
- package/dist/parsers/types.d.ts +48 -0
- package/dist/parsers/types.d.ts.map +1 -0
- package/dist/parsers/types.js +2 -0
- package/dist/parsers/types.js.map +1 -0
- package/dist/retrieval/index.d.ts +4 -0
- package/dist/retrieval/index.d.ts.map +1 -0
- package/dist/retrieval/index.js +3 -0
- package/dist/retrieval/index.js.map +1 -0
- package/dist/retrieval/schemas.d.ts +127 -0
- package/dist/retrieval/schemas.d.ts.map +1 -0
- package/dist/retrieval/schemas.js +65 -0
- package/dist/retrieval/schemas.js.map +1 -0
- package/dist/retrieval/tools.d.ts +31 -0
- package/dist/retrieval/tools.d.ts.map +1 -0
- package/dist/retrieval/tools.js +803 -0
- package/dist/retrieval/tools.js.map +1 -0
- package/dist/retrieval/types.d.ts +104 -0
- package/dist/retrieval/types.d.ts.map +1 -0
- package/dist/retrieval/types.js +2 -0
- package/dist/retrieval/types.js.map +1 -0
- package/dist/server/index.d.ts +3 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +2 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/server.d.ts +49 -0
- package/dist/server/server.d.ts.map +1 -0
- package/dist/server/server.js +517 -0
- package/dist/server/server.js.map +1 -0
- package/dist/shared/types.d.ts +98 -0
- package/dist/shared/types.d.ts.map +1 -0
- package/dist/shared/types.js +2 -0
- package/dist/shared/types.js.map +1 -0
- package/dist/storage/database.d.ts +18 -0
- package/dist/storage/database.d.ts.map +1 -0
- package/dist/storage/database.js +66 -0
- package/dist/storage/database.js.map +1 -0
- package/dist/storage/index.d.ts +8 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +5 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/migrations.d.ts +15 -0
- package/dist/storage/migrations.d.ts.map +1 -0
- package/dist/storage/migrations.js +170 -0
- package/dist/storage/migrations.js.map +1 -0
- package/dist/storage/paths.d.ts +15 -0
- package/dist/storage/paths.d.ts.map +1 -0
- package/dist/storage/paths.js +27 -0
- package/dist/storage/paths.js.map +1 -0
- package/dist/storage/schema.d.ts +11 -0
- package/dist/storage/schema.d.ts.map +1 -0
- package/dist/storage/schema.js +40 -0
- package/dist/storage/schema.js.map +1 -0
- package/dist/ui/index.d.ts +2 -0
- package/dist/ui/index.d.ts.map +1 -0
- package/dist/ui/index.js +2 -0
- package/dist/ui/index.js.map +1 -0
- package/dist/ui/static-app.d.ts +2 -0
- package/dist/ui/static-app.d.ts.map +1 -0
- package/dist/ui/static-app.js +735 -0
- package/dist/ui/static-app.js.map +1 -0
- package/docs/adr/0001-sqlite-driver.md +23 -0
- package/docs/agentic-rag-specifications.md +659 -0
- package/docs/agentic-rag-study.md +1056 -0
- package/docs/implementation-plan.md +1425 -0
- package/docs/mcp-integration.md +154 -0
- package/docs/validation-corpus.md +71 -0
- package/package.json +79 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { createServer } from "node:net";
|
|
4
|
+
import { loadConfig } from "../config/index.js";
|
|
5
|
+
import { countAvailable, inspectDatabase, openDatabase, requiredTables } from "../storage/index.js";
|
|
6
|
+
const require = createRequire(import.meta.url);
|
|
7
|
+
export async function runDoctor(runOptions) {
|
|
8
|
+
const report = await collectDoctorReport(runOptions.options);
|
|
9
|
+
if (runOptions.options.json) {
|
|
10
|
+
runOptions.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
runOptions.stdout.write(renderDoctorReport(report));
|
|
14
|
+
}
|
|
15
|
+
export async function collectDoctorReport(options) {
|
|
16
|
+
const loadedConfig = await loadConfig(options.config === undefined
|
|
17
|
+
? { root: options.root }
|
|
18
|
+
: { root: options.root, configPath: options.config });
|
|
19
|
+
const host = options.hostProvided
|
|
20
|
+
? options.host
|
|
21
|
+
: loadedConfig.config.server.host;
|
|
22
|
+
const port = options.portProvided
|
|
23
|
+
? options.port
|
|
24
|
+
: loadedConfig.config.server.port;
|
|
25
|
+
const database = inspectDatabase(options.root, options.db);
|
|
26
|
+
const portStatus = await checkPortAvailability(host, port);
|
|
27
|
+
return {
|
|
28
|
+
root: database.paths.root,
|
|
29
|
+
config: renderConfigSummary(loadedConfig),
|
|
30
|
+
database,
|
|
31
|
+
index: collectIndexSummary(database),
|
|
32
|
+
parsers: {
|
|
33
|
+
pdf2llm: resolvePackage("pdf-2-llm/node")
|
|
34
|
+
},
|
|
35
|
+
embeddings: {
|
|
36
|
+
provider: loadedConfig.config.embeddings.provider,
|
|
37
|
+
model: loadedConfig.config.embeddings.model,
|
|
38
|
+
modelCacheDir: database.paths.modelCacheDir,
|
|
39
|
+
modelCacheState: existsSync(database.paths.modelCacheDir)
|
|
40
|
+
? "present"
|
|
41
|
+
: "missing"
|
|
42
|
+
},
|
|
43
|
+
server: {
|
|
44
|
+
host,
|
|
45
|
+
port,
|
|
46
|
+
available: portStatus.available,
|
|
47
|
+
error: portStatus.error
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export function renderDoctorReport(report) {
|
|
52
|
+
const tableCount = report.database.schema === null
|
|
53
|
+
? 0
|
|
54
|
+
: countAvailable(report.database.schema.tables);
|
|
55
|
+
const schemaVersion = report.database.schemaVersion === null
|
|
56
|
+
? "unavailable"
|
|
57
|
+
: String(report.database.schemaVersion);
|
|
58
|
+
const pdfStatus = report.parsers.pdf2llm.available
|
|
59
|
+
? `available (${report.parsers.pdf2llm.resolvedPath ?? "resolved"})`
|
|
60
|
+
: "not installed";
|
|
61
|
+
const serverStatus = report.server.available
|
|
62
|
+
? "available"
|
|
63
|
+
: `unavailable${report.server.error === null ? "" : ` (${report.server.error})`}`;
|
|
64
|
+
return [
|
|
65
|
+
"Athena doctor",
|
|
66
|
+
`Root: ${report.root}`,
|
|
67
|
+
`Config: ${report.config.path ?? "defaults"}`,
|
|
68
|
+
`Database: ${report.database.status} (${report.database.paths.databasePath})`,
|
|
69
|
+
`Schema version: ${schemaVersion}`,
|
|
70
|
+
`Tables: ${String(tableCount)}/${String(requiredTables.length)}`,
|
|
71
|
+
`Sources: ${String(report.index.sources)}`,
|
|
72
|
+
`Documents: ${String(report.index.documents)}`,
|
|
73
|
+
`Chunks: ${String(report.index.chunks)}`,
|
|
74
|
+
`Warnings: ${String(report.index.warnings)}`,
|
|
75
|
+
`PDF parser: pdf-2-llm ${pdfStatus}`,
|
|
76
|
+
`Embeddings: ${report.embeddings.provider} / ${report.embeddings.model}`,
|
|
77
|
+
`Model cache: ${report.embeddings.modelCacheState} (${report.embeddings.modelCacheDir})`,
|
|
78
|
+
`Server port: ${report.server.host}:${String(report.server.port)} ${serverStatus}`,
|
|
79
|
+
""
|
|
80
|
+
].join("\n");
|
|
81
|
+
}
|
|
82
|
+
function renderConfigSummary(loadedConfig) {
|
|
83
|
+
return {
|
|
84
|
+
path: loadedConfig.configPath,
|
|
85
|
+
server: {
|
|
86
|
+
host: loadedConfig.config.server.host,
|
|
87
|
+
port: loadedConfig.config.server.port
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function collectIndexSummary(databaseReport) {
|
|
92
|
+
if (databaseReport.status !== "ready") {
|
|
93
|
+
return {
|
|
94
|
+
sources: 0,
|
|
95
|
+
documents: 0,
|
|
96
|
+
chunks: 0,
|
|
97
|
+
warnings: 0
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
const database = openDatabase(databaseReport.paths.databasePath, {
|
|
101
|
+
readonly: true,
|
|
102
|
+
fileMustExist: true
|
|
103
|
+
});
|
|
104
|
+
try {
|
|
105
|
+
return {
|
|
106
|
+
sources: countRows(database, "sources"),
|
|
107
|
+
documents: countRows(database, "documents"),
|
|
108
|
+
chunks: countRows(database, "chunks"),
|
|
109
|
+
warnings: countParserWarnings(database)
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
database.close();
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
function countRows(database, tableName) {
|
|
117
|
+
const row = database
|
|
118
|
+
.prepare(`SELECT COUNT(*) AS count FROM ${tableName}`)
|
|
119
|
+
.get();
|
|
120
|
+
return row?.count ?? 0;
|
|
121
|
+
}
|
|
122
|
+
function countParserWarnings(database) {
|
|
123
|
+
const row = database
|
|
124
|
+
.prepare("SELECT COUNT(*) AS count FROM parser_artifacts WHERE kind = 'warning'")
|
|
125
|
+
.get();
|
|
126
|
+
return row?.count ?? 0;
|
|
127
|
+
}
|
|
128
|
+
function resolvePackage(specifier) {
|
|
129
|
+
try {
|
|
130
|
+
return {
|
|
131
|
+
available: true,
|
|
132
|
+
resolvedPath: require.resolve(specifier)
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return {
|
|
137
|
+
available: false,
|
|
138
|
+
resolvedPath: null
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function checkPortAvailability(host, port) {
|
|
143
|
+
return new Promise((resolve) => {
|
|
144
|
+
const server = createServer();
|
|
145
|
+
server.once("error", (error) => {
|
|
146
|
+
resolve({
|
|
147
|
+
available: false,
|
|
148
|
+
error: error.code ?? error.message
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
server.once("listening", () => {
|
|
152
|
+
server.close(() => {
|
|
153
|
+
resolve({ available: true, error: null });
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
server.listen(port, host);
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
//# sourceMappingURL=doctor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../src/doctor/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EACL,cAAc,EACd,eAAe,EACf,YAAY,EACZ,cAAc,EACf,MAAM,qBAAqB,CAAC;AAiD7B,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,UAA4B;IAC1D,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAE7D,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO;IACT,CAAC;IAED,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAmB;IAEnB,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,OAAO,CAAC,MAAM,KAAK,SAAS;QAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;QACxB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,CACvD,CAAC;IACF,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY;QAC/B,CAAC,CAAC,OAAO,CAAC,IAAI;QACd,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY;QAC/B,CAAC,CAAC,OAAO,CAAC,IAAI;QACd,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IACpC,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAE3D,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI;QACzB,MAAM,EAAE,mBAAmB,CAAC,YAAY,CAAC;QACzC,QAAQ;QACR,KAAK,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QACpC,OAAO,EAAE;YACP,OAAO,EAAE,cAAc,CAAC,gBAAgB,CAAC;SAC1C;QACD,UAAU,EAAE;YACV,QAAQ,EAAE,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ;YACjD,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK;YAC3C,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;YAC3C,eAAe,EAAE,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;gBACvD,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,SAAS;SACd;QACD,MAAM,EAAE;YACN,IAAI;YACJ,IAAI;YACJ,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,KAAK,EAAE,UAAU,CAAC,KAAK;SACxB;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAoB;IACrD,MAAM,UAAU,GACd,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,IAAI;QAC7B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,aAAa,GACjB,MAAM,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI;QACpC,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS;QAChD,CAAC,CAAC,cAAc,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,IAAI,UAAU,GAAG;QACpE,CAAC,CAAC,eAAe,CAAC;IACpB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS;QAC1C,CAAC,CAAC,WAAW;QACb,CAAC,CAAC,cAAc,MAAM,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC;IAEpF,OAAO;QACL,eAAe;QACf,SAAS,MAAM,CAAC,IAAI,EAAE;QACtB,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,UAAU,EAAE;QAC7C,aAAa,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,GAAG;QAC7E,mBAAmB,aAAa,EAAE;QAClC,WAAW,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE;QAChE,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE;QAC1C,cAAc,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE;QAC9C,WAAW,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE;QACxC,aAAa,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC5C,yBAAyB,SAAS,EAAE;QACpC,eAAe,MAAM,CAAC,UAAU,CAAC,QAAQ,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE;QACxE,gBAAgB,MAAM,CAAC,UAAU,CAAC,eAAe,KAAK,MAAM,CAAC,UAAU,CAAC,aAAa,GAAG;QACxF,gBAAgB,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,EAAE;QAClF,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAC1B,YAA0B;IAE1B,OAAO;QACL,IAAI,EAAE,YAAY,CAAC,UAAU;QAC7B,MAAM,EAAE;YACN,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;YACrC,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI;SACtC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,cAAoC;IAEpC,IAAI,cAAc,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;QACtC,OAAO;YACL,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,cAAc,CAAC,KAAK,CAAC,YAAY,EAAE;QAC/D,QAAQ,EAAE,IAAI;QACd,aAAa,EAAE,IAAI;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,OAAO;YACL,OAAO,EAAE,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC;YACvC,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE,WAAW,CAAC;YAC3C,MAAM,EAAE,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC;YACrC,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;SACxC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,QAAyC,EACzC,SAA6C;IAE7C,MAAM,GAAG,GAAG,QAAQ;SACjB,OAAO,CACN,iCAAiC,SAAS,EAAE,CAC7C;SACA,GAAG,EAAE,CAAC;IAET,OAAO,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAC1B,QAAyC;IAEzC,MAAM,GAAG,GAAG,QAAQ;SACjB,OAAO,CACN,uEAAuE,CACxE;SACA,GAAG,EAAE,CAAC;IAET,OAAO,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB;IACvC,IAAI,CAAC;QACH,OAAO;YACL,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;SACzC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,SAAS,EAAE,KAAK;YAChB,YAAY,EAAE,IAAI;SACnB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,IAAY,EACZ,IAAY;IAEZ,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAE9B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAA4B,EAAE,EAAE;YACpD,OAAO,CAAC;gBACN,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE;YAC5B,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;gBAChB,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import type { EmbeddingProviderConfig } from "../config/index.js";
|
|
2
|
+
export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
3
|
+
export interface EmbeddingInput {
|
|
4
|
+
chunkId: string;
|
|
5
|
+
text: string;
|
|
6
|
+
normalizedText: string;
|
|
7
|
+
}
|
|
8
|
+
export interface EmbeddingVector {
|
|
9
|
+
chunkId: string;
|
|
10
|
+
vector: Float32Array;
|
|
11
|
+
metadata: EmbeddingVectorMetadata;
|
|
12
|
+
}
|
|
13
|
+
export interface EmbeddingVectorMetadata {
|
|
14
|
+
provider: string;
|
|
15
|
+
model: string;
|
|
16
|
+
dimensions: number;
|
|
17
|
+
pooling: "mean" | "cls";
|
|
18
|
+
normalize: boolean;
|
|
19
|
+
quantization: string;
|
|
20
|
+
revision: string;
|
|
21
|
+
implementation: string;
|
|
22
|
+
}
|
|
23
|
+
export interface EmbeddingVectorProvider {
|
|
24
|
+
metadata: EmbeddingVectorMetadata;
|
|
25
|
+
prepare?(): Promise<void>;
|
|
26
|
+
embed(inputs: readonly EmbeddingInput[]): Promise<EmbeddingVector[]>;
|
|
27
|
+
}
|
|
28
|
+
export interface EmbeddingCacheKeyInput {
|
|
29
|
+
chunkId: string;
|
|
30
|
+
normalizedText: string;
|
|
31
|
+
provider: string;
|
|
32
|
+
model: string;
|
|
33
|
+
dimensions: number;
|
|
34
|
+
pooling: "mean" | "cls";
|
|
35
|
+
normalize: boolean;
|
|
36
|
+
quantization: string;
|
|
37
|
+
revision: string;
|
|
38
|
+
chunkingVersion: string;
|
|
39
|
+
}
|
|
40
|
+
export declare const DEFAULT_EMBEDDING_REVISION = "local-deterministic-v2";
|
|
41
|
+
export interface CreateEmbeddingProviderOptions {
|
|
42
|
+
env?: NodeJS.ProcessEnv;
|
|
43
|
+
fetch?: FetchLike;
|
|
44
|
+
ollamaBaseUrl?: string;
|
|
45
|
+
openaiBaseUrl?: string;
|
|
46
|
+
voyageBaseUrl?: string;
|
|
47
|
+
cohereBaseUrl?: string;
|
|
48
|
+
geminiBaseUrl?: string;
|
|
49
|
+
}
|
|
50
|
+
type TransformersJsPooling = "mean" | "cls";
|
|
51
|
+
interface TransformersJsTensorLike {
|
|
52
|
+
data: ArrayLike<number>;
|
|
53
|
+
dims: readonly number[];
|
|
54
|
+
}
|
|
55
|
+
interface TransformersJsFeatureExtractor {
|
|
56
|
+
(texts: string | string[], options: {
|
|
57
|
+
pooling: TransformersJsPooling;
|
|
58
|
+
normalize: boolean;
|
|
59
|
+
}): Promise<TransformersJsTensorLike>;
|
|
60
|
+
}
|
|
61
|
+
type TransformersJsPipelineLoader = (task: "feature-extraction", model: string, options: {
|
|
62
|
+
local_files_only: true;
|
|
63
|
+
}) => Promise<TransformersJsFeatureExtractor>;
|
|
64
|
+
export declare function createEmbeddingProvider(config: EmbeddingProviderConfig, options?: CreateEmbeddingProviderOptions): EmbeddingVectorProvider | null;
|
|
65
|
+
export declare function validateEmbeddingProviderConfig(config: EmbeddingProviderConfig, env?: NodeJS.ProcessEnv): void;
|
|
66
|
+
export declare class DeterministicEmbeddingProvider implements EmbeddingVectorProvider {
|
|
67
|
+
readonly metadata: EmbeddingVectorMetadata;
|
|
68
|
+
constructor(config: EmbeddingProviderConfig, options?: {
|
|
69
|
+
implementation?: string;
|
|
70
|
+
revision?: string;
|
|
71
|
+
});
|
|
72
|
+
embed(inputs: readonly EmbeddingInput[]): Promise<EmbeddingVector[]>;
|
|
73
|
+
}
|
|
74
|
+
export declare class TransformersJsEmbeddingProvider implements EmbeddingVectorProvider {
|
|
75
|
+
private readonly config;
|
|
76
|
+
private readonly transformersMetadata;
|
|
77
|
+
private readonly fallbackProvider;
|
|
78
|
+
private readonly loader;
|
|
79
|
+
private extractor;
|
|
80
|
+
private prepared;
|
|
81
|
+
private usingFallback;
|
|
82
|
+
constructor(config: EmbeddingProviderConfig, options?: {
|
|
83
|
+
loader?: TransformersJsPipelineLoader;
|
|
84
|
+
revision?: string;
|
|
85
|
+
fallbackRevision?: string;
|
|
86
|
+
});
|
|
87
|
+
get metadata(): EmbeddingVectorMetadata;
|
|
88
|
+
prepare(): Promise<void>;
|
|
89
|
+
embed(inputs: readonly EmbeddingInput[]): Promise<EmbeddingVector[]>;
|
|
90
|
+
}
|
|
91
|
+
export declare class OllamaEmbeddingProvider implements EmbeddingVectorProvider {
|
|
92
|
+
private readonly config;
|
|
93
|
+
readonly metadata: EmbeddingVectorMetadata;
|
|
94
|
+
private readonly fetchImpl;
|
|
95
|
+
private readonly endpoint;
|
|
96
|
+
constructor(config: EmbeddingProviderConfig, options?: CreateEmbeddingProviderOptions);
|
|
97
|
+
embed(inputs: readonly EmbeddingInput[]): Promise<EmbeddingVector[]>;
|
|
98
|
+
}
|
|
99
|
+
export declare class RemoteRestEmbeddingProvider implements EmbeddingVectorProvider {
|
|
100
|
+
private readonly config;
|
|
101
|
+
private readonly options;
|
|
102
|
+
readonly metadata: EmbeddingVectorMetadata;
|
|
103
|
+
private readonly fetchImpl;
|
|
104
|
+
private readonly env;
|
|
105
|
+
constructor(config: EmbeddingProviderConfig, options?: CreateEmbeddingProviderOptions);
|
|
106
|
+
embed(inputs: readonly EmbeddingInput[]): Promise<EmbeddingVector[]>;
|
|
107
|
+
private embedOpenAi;
|
|
108
|
+
private embedVoyage;
|
|
109
|
+
private embedCohere;
|
|
110
|
+
private embedGemini;
|
|
111
|
+
}
|
|
112
|
+
export declare function createEmbeddingCacheKey(input: EmbeddingCacheKeyInput): string;
|
|
113
|
+
export declare function encodeVector(vector: Float32Array): Buffer;
|
|
114
|
+
export declare function decodeVector(buffer: Buffer): Float32Array;
|
|
115
|
+
export declare function createDeterministicEmbeddingVector(text: string, dimensions: number): Float32Array;
|
|
116
|
+
export {};
|
|
117
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,uBAAuB,EACxB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,SAAS,GAAG,CACtB,KAAK,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,EAC7B,IAAI,CAAC,EAAE,WAAW,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;IACrB,QAAQ,EAAE,uBAAuB,CAAC;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,uBAAuB,CAAC;IAClC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,KAAK,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,0BAA0B,2BAA2B,CAAC;AAQnE,MAAM,WAAW,8BAA8B;IAC7C,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,KAAK,qBAAqB,GAAG,MAAM,GAAG,KAAK,CAAC;AAE5C,UAAU,wBAAwB;IAChC,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IACxB,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED,UAAU,8BAA8B;IACtC,CACE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EACxB,OAAO,EAAE;QAAE,OAAO,EAAE,qBAAqB,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,GAC9D,OAAO,CAAC,wBAAwB,CAAC,CAAC;CACtC;AAED,KAAK,4BAA4B,GAAG,CAClC,IAAI,EAAE,oBAAoB,EAC1B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,gBAAgB,EAAE,IAAI,CAAA;CAAE,KAChC,OAAO,CAAC,8BAA8B,CAAC,CAAC;AAoB7C,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,uBAAuB,EAC/B,OAAO,GAAE,8BAAmC,GAC3C,uBAAuB,GAAG,IAAI,CAgBhC;AAED,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,uBAAuB,EAC/B,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,IAAI,CA0BN;AAED,qBAAa,8BAA+B,YAAW,uBAAuB;IAC5E,SAAgB,QAAQ,EAAE,uBAAuB,CAAC;gBAGhD,MAAM,EAAE,uBAAuB,EAC/B,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAO;IAcvD,KAAK,CAAC,MAAM,EAAE,SAAS,cAAc,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAgB5E;AAED,qBAAa,+BAAgC,YAAW,uBAAuB;IAS3E,OAAO,CAAC,QAAQ,CAAC,MAAM;IARzB,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAA0B;IAC/D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAiC;IAClE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+B;IACtD,OAAO,CAAC,SAAS,CAA+C;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;gBAGX,MAAM,EAAE,uBAAuB,EAChD,OAAO,GAAE;QACP,MAAM,CAAC,EAAE,4BAA4B,CAAC;QACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KACtB;IA2BR,IAAW,QAAQ,IAAI,uBAAuB,CAI7C;IAEY,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBxB,KAAK,CAChB,MAAM,EAAE,SAAS,cAAc,EAAE,GAChC,OAAO,CAAC,eAAe,EAAE,CAAC;CA0B9B;AAED,qBAAa,uBAAwB,YAAW,uBAAuB;IAMnE,OAAO,CAAC,QAAQ,CAAC,MAAM;IALzB,SAAgB,QAAQ,EAAE,uBAAuB,CAAC;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAGf,MAAM,EAAE,uBAAuB,EAChD,OAAO,GAAE,8BAAmC;IAejC,KAAK,CAChB,MAAM,EAAE,SAAS,cAAc,EAAE,GAChC,OAAO,CAAC,eAAe,EAAE,CAAC;CAkB9B;AAED,qBAAa,2BAA4B,YAAW,uBAAuB;IAMvE,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAN1B,SAAgB,QAAQ,EAAE,uBAAuB,CAAC;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAGrB,MAAM,EAAE,uBAAuB,EAC/B,OAAO,GAAE,8BAAmC;IAoBlD,KAAK,CAChB,MAAM,EAAE,SAAS,cAAc,EAAE,GAChC,OAAO,CAAC,eAAe,EAAE,CAAC;YAqBf,WAAW;YAyBX,WAAW;YA0BX,WAAW;YA+BX,WAAW;CA+B1B;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,sBAAsB,GAAG,MAAM,CAe7E;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAEzD;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAOzD;AAqSD,wBAAgB,kCAAkC,CAChD,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,GACjB,YAAY,CAiBd"}
|