agent-rag-mcp 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/dist/extensions/agent-rag.mjs +39 -14
- package/dist/src/adapters/image-ocr.mjs +16 -4
- package/dist/src/adapters/pdf.mjs +41 -35
- package/dist/src/pi/cache-status.mjs +18 -0
- package/dist/src/pi/cli-bridge.mjs +40 -0
- package/dist/src/retrieve/find.mjs +16 -10
- package/dist/src/retrieve/semantic.mjs +236 -0
- package/dist/src/server.mjs +16 -4
- package/dist/src/store/backup.mjs +279 -19
- package/dist/src/tools/schema.mjs +21 -1
- package/package.json +3 -1
- package/skills/agent-rag/SKILL.md +11 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { DatabaseSync } from "node:sqlite";
|
|
5
|
+
import { clientDbPath } from "../store/db.mjs";
|
|
6
|
+
const MAX_CHUNK_CHARS = 8192;
|
|
7
|
+
const TIMEOUT_MS = 3e4;
|
|
8
|
+
const MAX_RANKED_RESULTS = 100;
|
|
9
|
+
async function ollamaEmbed(model, texts, signal) {
|
|
10
|
+
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
|
11
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
12
|
+
const response = await fetch("http://127.0.0.1:11434/api/embed", {
|
|
13
|
+
method: "POST",
|
|
14
|
+
headers: { "content-type": "application/json" },
|
|
15
|
+
body: JSON.stringify({ model, input: texts, truncate: false }),
|
|
16
|
+
signal: combined,
|
|
17
|
+
redirect: "error"
|
|
18
|
+
});
|
|
19
|
+
if (!response.ok) throw new Error(`Local embedding service returned HTTP ${response.status}.`);
|
|
20
|
+
const data = await response.json();
|
|
21
|
+
if (!data || typeof data !== "object" || !("embeddings" in data)) throw new Error("Embedding response is invalid.");
|
|
22
|
+
return data.embeddings;
|
|
23
|
+
}
|
|
24
|
+
function azureEmbeddingEndpoint(value) {
|
|
25
|
+
let url;
|
|
26
|
+
try {
|
|
27
|
+
url = new URL(value);
|
|
28
|
+
} catch {
|
|
29
|
+
throw new Error("invalid_azure_endpoint");
|
|
30
|
+
}
|
|
31
|
+
const validHost = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.(?:openai\.azure\.com|services\.ai\.azure\.com|cognitiveservices\.azure\.com)$/i.test(url.hostname);
|
|
32
|
+
const validPath = ["", "/", "/openai/v1", "/openai/v1/"].includes(url.pathname);
|
|
33
|
+
if (url.protocol !== "https:" || !validHost || !validPath || url.username || url.password || url.port || url.search || url.hash) throw new Error("invalid_azure_endpoint");
|
|
34
|
+
return `${url.origin}/openai/v1/embeddings`;
|
|
35
|
+
}
|
|
36
|
+
function createAzureEmbed(endpoint, apiKey) {
|
|
37
|
+
const target = azureEmbeddingEndpoint(endpoint);
|
|
38
|
+
if (!apiKey.trim()) throw new Error("missing_azure_api_key");
|
|
39
|
+
return async (deployment, texts, signal) => {
|
|
40
|
+
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
|
41
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
42
|
+
const response = await fetch(target, {
|
|
43
|
+
method: "POST",
|
|
44
|
+
headers: { "content-type": "application/json", "api-key": apiKey },
|
|
45
|
+
body: JSON.stringify({ model: deployment, input: texts, encoding_format: "float" }),
|
|
46
|
+
signal: combined,
|
|
47
|
+
redirect: "error"
|
|
48
|
+
});
|
|
49
|
+
if (!response.ok) throw new Error("Azure embedding request failed.");
|
|
50
|
+
const body = await response.json();
|
|
51
|
+
if (!body || typeof body !== "object" || !("data" in body) || !Array.isArray(body.data) || body.data.length !== texts.length) throw new Error("Invalid Azure embedding response.");
|
|
52
|
+
const vectors = new Array(texts.length);
|
|
53
|
+
for (const entry of body.data) {
|
|
54
|
+
if (!entry || typeof entry !== "object" || !Number.isInteger(entry.index) || entry.index < 0 || entry.index >= texts.length || vectors[entry.index] !== void 0 || !Array.isArray(entry.embedding)) throw new Error("Invalid Azure embedding response.");
|
|
55
|
+
vectors[entry.index] = entry.embedding;
|
|
56
|
+
}
|
|
57
|
+
return vectors;
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
async function findConfiguredSemantically(input, env = process.env, signal) {
|
|
61
|
+
const provider = env.RAG_EMBEDDING_PROVIDER ?? "ollama";
|
|
62
|
+
const batchSize = env.RAG_EMBEDDING_BATCH_SIZE === void 0 ? 8 : Number(env.RAG_EMBEDDING_BATCH_SIZE);
|
|
63
|
+
if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 32) return { namespace: input.namespace, error: { code: "invalid_embedding_batch_size", message: "RAG_EMBEDDING_BATCH_SIZE must be an integer from 1 to 32." } };
|
|
64
|
+
const maxBatchBytes = env.RAG_EMBEDDING_BATCH_BYTES === void 0 ? 65536 : Number(env.RAG_EMBEDDING_BATCH_BYTES);
|
|
65
|
+
if (!Number.isInteger(maxBatchBytes) || maxBatchBytes < 1024 || maxBatchBytes > 1048576) return { namespace: input.namespace, error: { code: "invalid_embedding_batch_bytes", message: "RAG_EMBEDDING_BATCH_BYTES must be an integer from 1024 to 1048576." } };
|
|
66
|
+
const revision = env.RAG_EMBEDDING_REVISION ?? "";
|
|
67
|
+
if (provider === "ollama") return findSemantically({ ...input, model: env.RAG_EMBEDDING_MODEL ?? "", cacheIdentity: JSON.stringify(["ollama", env.RAG_EMBEDDING_MODEL ?? "", revision]), batchSize, maxBatchBytes }, ollamaEmbed, signal);
|
|
68
|
+
if (provider !== "azure") return { namespace: input.namespace, error: { code: "invalid_embedding_provider", message: "RAG_EMBEDDING_PROVIDER must be ollama or azure." } };
|
|
69
|
+
const endpoint = env.RAG_AZURE_OPENAI_ENDPOINT;
|
|
70
|
+
const key = env.RAG_AZURE_OPENAI_API_KEY;
|
|
71
|
+
const deployment = env.RAG_AZURE_OPENAI_DEPLOYMENT;
|
|
72
|
+
if (!endpoint || !key || !deployment?.trim()) return { namespace: input.namespace, error: { code: "semantic_not_configured", message: "Azure embeddings require endpoint, deployment and API key configuration." } };
|
|
73
|
+
let embed;
|
|
74
|
+
try {
|
|
75
|
+
embed = createAzureEmbed(endpoint, key);
|
|
76
|
+
} catch {
|
|
77
|
+
return { namespace: input.namespace, error: { code: "invalid_azure_endpoint", message: "Azure endpoint must be a supported HTTPS resource URL." } };
|
|
78
|
+
}
|
|
79
|
+
return findSemantically({ ...input, model: deployment, cacheIdentity: JSON.stringify(["azure", azureEmbeddingEndpoint(endpoint), deployment, revision]), batchSize, maxBatchBytes }, embed, signal);
|
|
80
|
+
}
|
|
81
|
+
function cosine(a, b) {
|
|
82
|
+
let dot = 0;
|
|
83
|
+
let aNorm = 0;
|
|
84
|
+
let bNorm = 0;
|
|
85
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
86
|
+
dot += a[i] * b[i];
|
|
87
|
+
aNorm += a[i] ** 2;
|
|
88
|
+
bNorm += b[i] ** 2;
|
|
89
|
+
}
|
|
90
|
+
if (!aNorm || !bNorm || !Number.isFinite(dot) || !Number.isFinite(aNorm) || !Number.isFinite(bNorm)) throw new Error("Embedding vector must be finite and nonzero.");
|
|
91
|
+
return dot / Math.sqrt(aNorm * bNorm);
|
|
92
|
+
}
|
|
93
|
+
async function findSemantically(input, embed = ollamaEmbed, signal) {
|
|
94
|
+
const { root, namespace, query, includeHistorical, excerptLength, model } = input;
|
|
95
|
+
const batchSize = input.batchSize ?? 8;
|
|
96
|
+
const maxBatchBytes = input.maxBatchBytes ?? 65536;
|
|
97
|
+
if (!model.trim()) return { namespace, error: { code: "semantic_not_configured", message: "Configure an embedding model." } };
|
|
98
|
+
if (!Number.isInteger(batchSize) || batchSize < 1 || batchSize > 32) return { namespace, error: { code: "invalid_embedding_batch_size", message: "Embedding batch size must be an integer from 1 to 32." } };
|
|
99
|
+
if (!Number.isInteger(maxBatchBytes) || maxBatchBytes < 1024 || maxBatchBytes > 1048576) return { namespace, error: { code: "invalid_embedding_batch_bytes", message: "Embedding batch bytes must be an integer from 1024 to 1048576." } };
|
|
100
|
+
if (query.length > MAX_CHUNK_CHARS) return { namespace, error: { code: "semantic_query_too_large", message: `Semantic queries must not exceed ${MAX_CHUNK_CHARS} characters.` } };
|
|
101
|
+
let db;
|
|
102
|
+
let cache;
|
|
103
|
+
let phase = "database";
|
|
104
|
+
try {
|
|
105
|
+
const dbPath = clientDbPath(root, namespace.client);
|
|
106
|
+
if (!fs.existsSync(dbPath) || !fs.statSync(dbPath).isFile()) return { namespace, error: { code: "database_not_found", message: "Namespace database was not found." } };
|
|
107
|
+
db = new DatabaseSync(dbPath, { readOnly: true });
|
|
108
|
+
const rows = db.prepare(`SELECT dv.doc_version_id AS docVersionId, d.document_id AS documentId,
|
|
109
|
+
e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel, dv.is_current AS isCurrent,
|
|
110
|
+
s.ordinal AS sectionOrdinal, c.ordinal AS chunkOrdinal, c.char_start AS charStart, c.char_end AS charEnd, c.text AS text
|
|
111
|
+
FROM chunk AS c JOIN doc_version AS dv ON dv.doc_version_id = c.doc_version_id
|
|
112
|
+
JOIN document AS d ON d.document_id = dv.document_id JOIN entity AS e ON e.entity_id = d.entity_id
|
|
113
|
+
JOIN project AS p ON p.project_id = d.project_id JOIN client AS cl ON cl.client_id = p.client_id
|
|
114
|
+
LEFT JOIN section AS s ON s.section_id = c.section_id
|
|
115
|
+
WHERE cl.name = ? AND p.slug = ? AND d.deleted_at IS NULL AND e.deleted_at IS NULL AND (dv.is_current = 1 OR ? = 1)
|
|
116
|
+
ORDER BY dv.doc_version_id COLLATE BINARY, c.ordinal`).iterate(namespace.client, namespace.project, includeHistorical ? 1 : 0);
|
|
117
|
+
const iterator = rows[Symbol.iterator]();
|
|
118
|
+
const first = iterator.next();
|
|
119
|
+
if (first.done) return { namespace, result: { citations: [], total: 0, searchMode: "semantic" } };
|
|
120
|
+
phase = "cache";
|
|
121
|
+
cache = new DatabaseSync(path.join(path.dirname(dbPath), "semantic-cache.sqlite"));
|
|
122
|
+
cache.exec("PRAGMA busy_timeout = 5000; PRAGMA journal_mode = WAL;");
|
|
123
|
+
cache.exec(`CREATE TABLE IF NOT EXISTS vector_cache (
|
|
124
|
+
project TEXT NOT NULL, identity TEXT NOT NULL, doc_version_id TEXT NOT NULL, ordinal INTEGER NOT NULL,
|
|
125
|
+
text_hash TEXT NOT NULL, dimensions INTEGER NOT NULL, vector BLOB NOT NULL,
|
|
126
|
+
PRIMARY KEY(project, identity, doc_version_id, ordinal)
|
|
127
|
+
) WITHOUT ROWID`);
|
|
128
|
+
cache.exec("CREATE INDEX IF NOT EXISTS idx_vector_content ON vector_cache(project, identity, text_hash)");
|
|
129
|
+
const cacheIdentity = createHash("sha256").update(input.cacheIdentity ?? JSON.stringify(["direct", model])).digest("hex");
|
|
130
|
+
const select = cache.prepare("SELECT dimensions, vector FROM vector_cache WHERE project = ? AND identity = ? AND doc_version_id = ? AND ordinal = ? AND text_hash = ?");
|
|
131
|
+
const selectByContent = cache.prepare("SELECT dimensions, vector FROM vector_cache WHERE project = ? AND identity = ? AND text_hash = ? LIMIT 1");
|
|
132
|
+
const upsert = cache.prepare(`INSERT INTO vector_cache(project, identity, doc_version_id, ordinal, text_hash, dimensions, vector) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
133
|
+
ON CONFLICT(project, identity, doc_version_id, ordinal) DO UPDATE SET text_hash=excluded.text_hash, dimensions=excluded.dimensions, vector=excluded.vector`);
|
|
134
|
+
const validateVectors = (vectors, count, dimensions2) => {
|
|
135
|
+
if (!Array.isArray(vectors) || vectors.length !== count || !Array.isArray(vectors[0])) throw new Error("Invalid vector count.");
|
|
136
|
+
const size = dimensions2 ?? vectors[0].length;
|
|
137
|
+
if (size < 1 || size > 8192 || vectors.some((vector) => !Array.isArray(vector) || vector.length !== size || vector.some((n) => typeof n !== "number" || !Number.isFinite(n)))) throw new Error("Invalid vector dimensions.");
|
|
138
|
+
return size;
|
|
139
|
+
};
|
|
140
|
+
phase = "embedding";
|
|
141
|
+
const queryVectors = await embed(model, [query], signal);
|
|
142
|
+
const dimensions = validateVectors(queryVectors, 1);
|
|
143
|
+
const queryVector = queryVectors[0];
|
|
144
|
+
cosine(queryVector, queryVector);
|
|
145
|
+
const citations = [];
|
|
146
|
+
let total = 0;
|
|
147
|
+
const rank = (row, vector) => {
|
|
148
|
+
citations.push({ docVersionId: row.docVersionId, documentId: row.documentId, entityKey: row.entityKey, label: row.label, versionLabel: row.versionLabel, isCurrent: row.isCurrent === 1, sectionOrdinal: row.sectionOrdinal, chunkOrdinal: row.chunkOrdinal, charStart: row.charStart, charEnd: row.charEnd, matchStart: null, matchEnd: null, score: cosine(queryVector, vector), excerpt: row.text.slice(0, excerptLength) });
|
|
149
|
+
citations.sort((a, b) => b.score - a.score || a.docVersionId.localeCompare(b.docVersionId) || a.chunkOrdinal - b.chunkOrdinal);
|
|
150
|
+
if (citations.length > MAX_RANKED_RESULTS) citations.pop();
|
|
151
|
+
};
|
|
152
|
+
let pending = [];
|
|
153
|
+
let pendingBytes = 0;
|
|
154
|
+
const flush = async () => {
|
|
155
|
+
if (!pending.length) return;
|
|
156
|
+
phase = "embedding";
|
|
157
|
+
const vectors = await embed(model, pending.map((entry) => entry.row.text), signal);
|
|
158
|
+
validateVectors(vectors, pending.length, dimensions);
|
|
159
|
+
if (signal?.aborted) throw new Error("cancelled");
|
|
160
|
+
for (const vector of vectors) cosine(vector, vector);
|
|
161
|
+
phase = "cache";
|
|
162
|
+
cache.exec("BEGIN IMMEDIATE");
|
|
163
|
+
try {
|
|
164
|
+
for (const [index, entry] of pending.entries()) {
|
|
165
|
+
const vector = vectors[index];
|
|
166
|
+
const bytes = Buffer.allocUnsafe(dimensions * 4);
|
|
167
|
+
vector.forEach((value, offset) => {
|
|
168
|
+
if (!Number.isFinite(Math.fround(value))) throw new Error("Embedding value exceeds float32.");
|
|
169
|
+
bytes.writeFloatLE(value, offset * 4);
|
|
170
|
+
});
|
|
171
|
+
upsert.run(namespace.project, cacheIdentity, entry.row.docVersionId, entry.row.chunkOrdinal, entry.hash, dimensions, bytes);
|
|
172
|
+
rank(entry.row, Array.from({ length: dimensions }, (_, i) => bytes.readFloatLE(i * 4)));
|
|
173
|
+
}
|
|
174
|
+
cache.exec("COMMIT");
|
|
175
|
+
} catch (error) {
|
|
176
|
+
cache.exec("ROLLBACK");
|
|
177
|
+
throw error;
|
|
178
|
+
}
|
|
179
|
+
pending = [];
|
|
180
|
+
pendingBytes = 0;
|
|
181
|
+
};
|
|
182
|
+
for (const row of { *[Symbol.iterator]() {
|
|
183
|
+
yield first.value;
|
|
184
|
+
yield* { [Symbol.iterator]: () => iterator };
|
|
185
|
+
} }) {
|
|
186
|
+
if (signal?.aborted) throw new Error("cancelled");
|
|
187
|
+
if (row.text.length > MAX_CHUNK_CHARS) return { namespace, error: { code: "semantic_chunk_too_large", message: `A chunk exceeds ${MAX_CHUNK_CHARS} characters; semantic search did not finish.` } };
|
|
188
|
+
total += 1;
|
|
189
|
+
const hash = createHash("sha256").update(row.text).digest("hex");
|
|
190
|
+
phase = "cache";
|
|
191
|
+
const cached = select.get(namespace.project, cacheIdentity, row.docVersionId, row.chunkOrdinal, hash) ?? selectByContent.get(namespace.project, cacheIdentity, hash);
|
|
192
|
+
if (cached) {
|
|
193
|
+
if (cached.dimensions !== dimensions || cached.vector.byteLength !== dimensions * 4) return { namespace, error: { code: "embedding_dimension_mismatch", message: "Cached vectors do not match this model. Set a new RAG_EMBEDDING_REVISION before rebuilding." } };
|
|
194
|
+
const bytes = Buffer.from(cached.vector);
|
|
195
|
+
rank(row, Array.from({ length: dimensions }, (_, i) => bytes.readFloatLE(i * 4)));
|
|
196
|
+
} else {
|
|
197
|
+
const textBytes = Buffer.byteLength(row.text, "utf8");
|
|
198
|
+
if (textBytes > maxBatchBytes) return { namespace, error: { code: "semantic_chunk_too_large", message: "A chunk exceeds RAG_EMBEDDING_BATCH_BYTES." } };
|
|
199
|
+
if (pendingBytes + textBytes > maxBatchBytes) await flush();
|
|
200
|
+
pending.push({ row, hash });
|
|
201
|
+
pendingBytes += textBytes;
|
|
202
|
+
if (pending.length >= batchSize) await flush();
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
await flush();
|
|
206
|
+
return { namespace, result: { citations, total, searchMode: "semantic" } };
|
|
207
|
+
} catch {
|
|
208
|
+
if (signal?.aborted) return { namespace, error: { code: "cancelled", message: "Search was cancelled." } };
|
|
209
|
+
if (phase === "embedding") return { namespace, error: { code: "embedding_unavailable", message: "Embedding request failed; check the configured provider, model and access." } };
|
|
210
|
+
if (phase === "cache") return { namespace, error: { code: "semantic_cache_unavailable", message: "Semantic cache could not be read or written." } };
|
|
211
|
+
return { namespace, error: { code: "database_unavailable", message: "Could not load namespace chunks." } };
|
|
212
|
+
} finally {
|
|
213
|
+
cache?.close();
|
|
214
|
+
db?.close();
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
function combineHybrid(lexical, semantic) {
|
|
218
|
+
const merged = /* @__PURE__ */ new Map();
|
|
219
|
+
for (const list of [lexical, semantic]) {
|
|
220
|
+
list.forEach((citation, index) => {
|
|
221
|
+
const key = JSON.stringify([citation.docVersionId, citation.chunkOrdinal]);
|
|
222
|
+
const current = merged.get(key);
|
|
223
|
+
if (current) current.score += 1 / (60 + index + 1);
|
|
224
|
+
else merged.set(key, { citation, score: 1 / (60 + index + 1) });
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
return [...merged.values()].map(({ citation, score }) => ({ ...citation, score })).sort((a, b) => b.score - a.score || a.docVersionId.localeCompare(b.docVersionId) || a.chunkOrdinal - b.chunkOrdinal);
|
|
228
|
+
}
|
|
229
|
+
export {
|
|
230
|
+
azureEmbeddingEndpoint,
|
|
231
|
+
combineHybrid,
|
|
232
|
+
createAzureEmbed,
|
|
233
|
+
findConfiguredSemantically,
|
|
234
|
+
findSemantically,
|
|
235
|
+
ollamaEmbed
|
|
236
|
+
};
|
package/dist/src/server.mjs
CHANGED
|
@@ -7,9 +7,12 @@ import {
|
|
|
7
7
|
import { discoverConfig } from "./config/discovery.mjs";
|
|
8
8
|
import { findInNamespace } from "./retrieve/find.mjs";
|
|
9
9
|
import { readFromCitation } from "./retrieve/read.mjs";
|
|
10
|
+
import { combineHybrid, findConfiguredSemantically } from "./retrieve/semantic.mjs";
|
|
10
11
|
import {
|
|
11
12
|
RAG_TOOLS,
|
|
12
13
|
parseCapabilities,
|
|
14
|
+
parseNamespaceAllowlist,
|
|
15
|
+
isNamespaceAllowed,
|
|
13
16
|
namespaceFromToolInput,
|
|
14
17
|
validateRagFindInput,
|
|
15
18
|
validateRagReadInput
|
|
@@ -24,12 +27,13 @@ function toolError(namespace, error) {
|
|
|
24
27
|
}
|
|
25
28
|
async function main() {
|
|
26
29
|
const capabilities = parseCapabilities(process.env.RAG_CAPABILITIES);
|
|
30
|
+
const allowlist = parseNamespaceAllowlist(process.env.RAG_ALLOWED_NAMESPACES);
|
|
27
31
|
const discovery = discoverConfig({ cwd: process.cwd(), homeDir: process.cwd(), argv: [], env: process.env });
|
|
28
32
|
const root = discovery.root;
|
|
29
33
|
const tools = RAG_TOOLS.filter((tool) => capabilities.has(tool.name === "rag_find" ? "find" : "read"));
|
|
30
34
|
const server = new Server({ name: "agent-rag", version: "0.1.0" }, { capabilities: { tools: { listChanged: false } } });
|
|
31
35
|
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools }));
|
|
32
|
-
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
36
|
+
server.setRequestHandler(CallToolRequestSchema, async (request, extra) => {
|
|
33
37
|
try {
|
|
34
38
|
const { name, arguments: args } = request.params;
|
|
35
39
|
if (name !== "rag_find" && name !== "rag_read") {
|
|
@@ -42,14 +46,22 @@ async function main() {
|
|
|
42
46
|
if (name === "rag_find") {
|
|
43
47
|
const check2 = validateRagFindInput(args);
|
|
44
48
|
if (!check2.ok) return toolError(check2.namespace, check2.error);
|
|
45
|
-
const { namespace: namespace2, query, limit, excerptChars } = check2.input;
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
const { namespace: namespace2, query, limit, excerptChars, includeHistorical, matchMode, searchMode } = check2.input;
|
|
50
|
+
if (!isNamespaceAllowed(allowlist, namespace2)) return toolError(namespace2, { code: "namespace_denied", message: "Namespace is not enabled for this process." });
|
|
51
|
+
const found = searchMode !== "lexical" ? await findConfiguredSemantically({ root, namespace: namespace2, query, excerptLength: excerptChars, includeHistorical }, process.env, extra.signal) : findInNamespace({ root, namespace: namespace2, query, excerptLength: excerptChars, includeHistorical, matchMode, maxResults: limit });
|
|
52
|
+
if ("error" in found && found.error) return toolError(namespace2, found.error);
|
|
53
|
+
if (searchMode === "hybrid") {
|
|
54
|
+
const lexical = findInNamespace({ root, namespace: namespace2, query, excerptLength: excerptChars, includeHistorical, matchMode, maxResults: 100 });
|
|
55
|
+
if ("error" in lexical) return toolError(namespace2, lexical.error);
|
|
56
|
+
const citations = combineHybrid(lexical.result.citations, found.result.citations);
|
|
57
|
+
return toolResult({ namespace: namespace2, result: { citations: citations.slice(0, limit), total: found.result.total, searchMode: "hybrid" } });
|
|
58
|
+
}
|
|
48
59
|
return toolResult({ namespace: namespace2, result: { ...found.result, citations: found.result.citations.slice(0, limit) } });
|
|
49
60
|
}
|
|
50
61
|
const check = validateRagReadInput(args);
|
|
51
62
|
if (!check.ok) return toolError(check.namespace, check.error);
|
|
52
63
|
const { namespace, docVersionId, chunkOrdinal, maxBytes, maxLines, startLine } = check.input;
|
|
64
|
+
if (!isNamespaceAllowed(allowlist, namespace)) return toolError(namespace, { code: "namespace_denied", message: "Namespace is not enabled for this process." });
|
|
53
65
|
const read = await readFromCitation({ root, namespace, docVersionId, chunkOrdinal, maxBytes, maxLines, startLine });
|
|
54
66
|
if ("error" in read) return toolError(namespace, read.error);
|
|
55
67
|
return toolResult({ namespace, result: read.result });
|
|
@@ -33,25 +33,285 @@ async function renameWithRetry(source, destination) {
|
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
const MANIFEST_LOCK_TIMEOUT_MS = 5e3;
|
|
37
|
+
const MANIFEST_LOCK_POLL_MS = 25;
|
|
38
|
+
const MANIFEST_LOCK_STALE_MS = 3e4;
|
|
39
|
+
async function processIsAlive(pid) {
|
|
40
|
+
try {
|
|
41
|
+
process.kill(pid, 0);
|
|
42
|
+
return true;
|
|
43
|
+
} catch (error) {
|
|
44
|
+
const code = error.code;
|
|
45
|
+
return code === "EPERM";
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async function observeManifestLock(lockPath) {
|
|
49
|
+
let handle;
|
|
50
|
+
try {
|
|
51
|
+
handle = await fs.open(lockPath, "r");
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (error.code === "ENOENT") return void 0;
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
const [stat, contents] = await Promise.all([handle.stat(), handle.readFile("utf8")]);
|
|
58
|
+
return { dev: stat.dev, ino: stat.ino, mtimeMs: stat.mtimeMs, contents };
|
|
59
|
+
} finally {
|
|
60
|
+
await handle.close();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function sameLockIdentity(left, right) {
|
|
64
|
+
return left.dev === right.dev && left.ino === right.ino && left.contents === right.contents;
|
|
65
|
+
}
|
|
66
|
+
function parsedLockRecord(observation) {
|
|
67
|
+
try {
|
|
68
|
+
return JSON.parse(observation.contents);
|
|
69
|
+
} catch {
|
|
70
|
+
return void 0;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async function isStaleManifestLock(observation) {
|
|
74
|
+
const record = parsedLockRecord(observation);
|
|
75
|
+
if (!Number.isSafeInteger(record?.pid) || (record?.pid ?? 0) <= 0 || typeof record?.token !== "string" || record.token.length === 0) {
|
|
76
|
+
return Date.now() - observation.mtimeMs > MANIFEST_LOCK_STALE_MS;
|
|
77
|
+
}
|
|
78
|
+
return !await processIsAlive(record.pid);
|
|
79
|
+
}
|
|
80
|
+
async function acquireRecoveryGuard(lockPath, deadline) {
|
|
81
|
+
const guardPath = `${lockPath}.recovery`;
|
|
82
|
+
while (true) {
|
|
83
|
+
const token = randomUUID();
|
|
84
|
+
let handle;
|
|
85
|
+
try {
|
|
86
|
+
handle = await fs.open(guardPath, "wx");
|
|
87
|
+
} catch (error) {
|
|
88
|
+
if (error.code !== "EEXIST") throw error;
|
|
89
|
+
if (Date.now() >= deadline) return void 0;
|
|
90
|
+
await new Promise((resolve) => setTimeout(resolve, MANIFEST_LOCK_POLL_MS));
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
await handle.writeFile(JSON.stringify({ pid: process.pid, token, createdAt: Date.now() }), "utf8");
|
|
95
|
+
} catch (error) {
|
|
96
|
+
await handle.close().catch(() => void 0);
|
|
97
|
+
await fs.unlink(guardPath).catch(() => void 0);
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
await handle.close();
|
|
101
|
+
return async () => {
|
|
102
|
+
try {
|
|
103
|
+
const current = JSON.parse(await fs.readFile(guardPath, "utf8"));
|
|
104
|
+
if (current.token === token) await fs.unlink(guardPath);
|
|
105
|
+
} catch (error) {
|
|
106
|
+
if (error.code !== "ENOENT") throw error;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
async function createManifestLock(lockPath, token) {
|
|
112
|
+
let handle;
|
|
113
|
+
try {
|
|
114
|
+
handle = await fs.open(lockPath, "wx");
|
|
115
|
+
} catch (error) {
|
|
116
|
+
if (error.code === "EEXIST") return void 0;
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
try {
|
|
120
|
+
await handle.writeFile(JSON.stringify({ pid: process.pid, token, createdAt: Date.now() }), "utf8");
|
|
121
|
+
} finally {
|
|
122
|
+
await handle.close();
|
|
123
|
+
}
|
|
124
|
+
const observation = await observeManifestLock(lockPath);
|
|
125
|
+
if (!observation) throw new Error(`Manifest lock disappeared after exclusive creation: ${lockPath}`);
|
|
126
|
+
return observation;
|
|
127
|
+
}
|
|
128
|
+
async function takeoverStaleManifestLock(lockPath, observed, token = randomUUID()) {
|
|
129
|
+
const releaseGuard = await acquireRecoveryGuard(lockPath, Date.now());
|
|
130
|
+
if (!releaseGuard) return false;
|
|
131
|
+
try {
|
|
132
|
+
const current = await observeManifestLock(lockPath);
|
|
133
|
+
if (!current || !sameLockIdentity(current, observed) || !await isStaleManifestLock(current)) return false;
|
|
134
|
+
const quarantinedPath = `${lockPath}.${randomUUID()}.stale`;
|
|
135
|
+
await fs.rename(lockPath, quarantinedPath);
|
|
136
|
+
const moved = await observeManifestLock(quarantinedPath);
|
|
137
|
+
if (!moved || !sameLockIdentity(moved, current)) {
|
|
138
|
+
try {
|
|
139
|
+
await fs.link(quarantinedPath, lockPath);
|
|
140
|
+
} catch (restoreError) {
|
|
141
|
+
throw new Error(`Manifest lock identity changed during takeover; preserved ${quarantinedPath} and could not restore ${lockPath}: ${String(restoreError)}`);
|
|
142
|
+
}
|
|
143
|
+
throw new Error(`Manifest lock identity changed during takeover; restored ${lockPath} and preserved ${quarantinedPath}.`);
|
|
144
|
+
}
|
|
145
|
+
await fs.unlink(quarantinedPath);
|
|
146
|
+
const replacement = await createManifestLock(lockPath, token);
|
|
147
|
+
return replacement !== void 0;
|
|
148
|
+
} finally {
|
|
149
|
+
await releaseGuard();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async function acquireManifestLock(root, timeoutMs = MANIFEST_LOCK_TIMEOUT_MS) {
|
|
153
|
+
const lockPath = path.join(root, ".manifest.lock");
|
|
154
|
+
const deadline = Date.now() + timeoutMs;
|
|
155
|
+
while (true) {
|
|
156
|
+
const token = randomUUID();
|
|
157
|
+
const created = await createManifestLock(lockPath, token);
|
|
158
|
+
if (created) return async () => {
|
|
159
|
+
const releaseGuard = await acquireRecoveryGuard(lockPath, Date.now() + timeoutMs);
|
|
160
|
+
if (!releaseGuard) throw new Error(`Timed out waiting to release the root manifest lock at ${lockPath}.`);
|
|
161
|
+
try {
|
|
162
|
+
const current = await observeManifestLock(lockPath);
|
|
163
|
+
const record = current && parsedLockRecord(current);
|
|
164
|
+
if (current && sameLockIdentity(current, created) && record?.token === token) await fs.unlink(lockPath);
|
|
165
|
+
} finally {
|
|
166
|
+
await releaseGuard();
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
const observed = await observeManifestLock(lockPath);
|
|
170
|
+
if (!observed) continue;
|
|
171
|
+
if (await isStaleManifestLock(observed)) {
|
|
172
|
+
if (await takeoverStaleManifestLock(lockPath, observed, token)) {
|
|
173
|
+
const createdLock = await observeManifestLock(lockPath);
|
|
174
|
+
if (!createdLock) throw new Error(`Manifest lock disappeared after stale recovery: ${lockPath}`);
|
|
175
|
+
return async () => {
|
|
176
|
+
const releaseGuard = await acquireRecoveryGuard(lockPath, Date.now() + timeoutMs);
|
|
177
|
+
if (!releaseGuard) throw new Error(`Timed out waiting to release the root manifest lock at ${lockPath}.`);
|
|
178
|
+
try {
|
|
179
|
+
const current = await observeManifestLock(lockPath);
|
|
180
|
+
const record = current && parsedLockRecord(current);
|
|
181
|
+
if (current && sameLockIdentity(current, createdLock) && record?.token === token) await fs.unlink(lockPath);
|
|
182
|
+
} finally {
|
|
183
|
+
await releaseGuard();
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (Date.now() >= deadline) {
|
|
189
|
+
throw new Error(`Timed out waiting for the root manifest lock at ${lockPath} (recovery guard ${lockPath}.recovery may require manual verification).`);
|
|
190
|
+
}
|
|
191
|
+
await new Promise((resolve) => setTimeout(resolve, MANIFEST_LOCK_POLL_MS));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async function readRootManifestIfPresent(root) {
|
|
195
|
+
let contents;
|
|
196
|
+
try {
|
|
197
|
+
contents = await fs.readFile(path.join(root, "manifest.json"), "utf8");
|
|
198
|
+
} catch (error) {
|
|
199
|
+
if (error.code === "ENOENT") return null;
|
|
200
|
+
throw ioError("manifest_read_failed", `Unable to read manifest under ${root}.`);
|
|
201
|
+
}
|
|
202
|
+
let value;
|
|
203
|
+
try {
|
|
204
|
+
value = JSON.parse(contents);
|
|
205
|
+
} catch {
|
|
206
|
+
throw ioError("manifest_invalid", `Manifest under ${root} is not valid JSON.`);
|
|
207
|
+
}
|
|
208
|
+
validateManifest(value);
|
|
209
|
+
return value;
|
|
210
|
+
}
|
|
211
|
+
function subsetManifest(manifest, clientIds) {
|
|
212
|
+
const clients = manifest.clients.filter((client) => clientIds.has(client.client_id));
|
|
213
|
+
const projects = manifest.projects.filter((project) => clientIds.has(project.client_id));
|
|
214
|
+
const projectIds = new Set(projects.map((project) => project.project_id));
|
|
215
|
+
const ingestRuns = manifest.ingestRuns.filter((run) => projectIds.has(String(run.project_id)));
|
|
216
|
+
const entities = manifest.entities.filter((entity) => projectIds.has(entity.project_id));
|
|
217
|
+
const documents = manifest.documents.filter((document) => projectIds.has(document.project_id));
|
|
218
|
+
const documentIds = new Set(documents.map((document) => document.document_id));
|
|
219
|
+
const versions = manifest.versions.filter((version) => documentIds.has(version.document_id));
|
|
220
|
+
const versionIds = new Set(versions.map((version) => version.doc_version_id));
|
|
221
|
+
const objectHashes = new Set(versions.map((version) => version.sha256));
|
|
222
|
+
return {
|
|
223
|
+
version: 1,
|
|
224
|
+
clients,
|
|
225
|
+
projects,
|
|
226
|
+
ingestRuns,
|
|
227
|
+
entities,
|
|
228
|
+
documents,
|
|
229
|
+
versions,
|
|
230
|
+
sections: manifest.sections.filter((section) => versionIds.has(section.doc_version_id)),
|
|
231
|
+
chunks: manifest.chunks.filter((chunk) => versionIds.has(chunk.doc_version_id)),
|
|
232
|
+
relations: manifest.relations.filter((relation) => versionIds.has(relation.from_doc_version_id) && versionIds.has(relation.to_doc_version_id)),
|
|
233
|
+
objects: manifest.objects.filter((object) => objectHashes.has(object.sha256)),
|
|
234
|
+
namespaceAcls: manifest.namespaceAcls.filter((acl) => clientIds.has(String(acl.client_id)) || acl.project_id !== null && projectIds.has(String(acl.project_id)))
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
function selectManifestClient(manifest, clientName) {
|
|
238
|
+
const client = manifest.clients.find((row) => row.name === clientName);
|
|
239
|
+
if (!client) throw ioError("namespace_mismatch", `Manifest does not contain client ${clientName}.`);
|
|
240
|
+
return subsetManifest(manifest, /* @__PURE__ */ new Set([client.client_id]));
|
|
241
|
+
}
|
|
242
|
+
function mergeRootManifests(existing, replacement) {
|
|
243
|
+
if (replacement.clients.length === 0) return existing ?? replacement;
|
|
244
|
+
if (!existing) return replacement;
|
|
245
|
+
const replacedClientIds = new Set(replacement.clients.map((client) => client.client_id));
|
|
246
|
+
const oldTarget = subsetManifest(existing, replacedClientIds);
|
|
247
|
+
const retainedClientIds = new Set(existing.clients.filter((client) => !replacedClientIds.has(client.client_id)).map((client) => client.client_id));
|
|
248
|
+
const retained = subsetManifest(existing, retainedClientIds);
|
|
249
|
+
const retainedVersionHashes = new Set(retained.versions.map((version) => version.sha256));
|
|
250
|
+
const oldTargetVersionHashes = new Set(oldTarget.versions.map((version) => version.sha256));
|
|
251
|
+
const unownedOrSharedObjects = existing.objects.filter((object) => !oldTargetVersionHashes.has(object.sha256) || retainedVersionHashes.has(object.sha256));
|
|
252
|
+
const combinedObjectHashes = new Set(unownedOrSharedObjects.map((object) => object.sha256));
|
|
253
|
+
const objects = [
|
|
254
|
+
...unownedOrSharedObjects,
|
|
255
|
+
...replacement.objects.filter((object) => !combinedObjectHashes.has(object.sha256))
|
|
256
|
+
];
|
|
257
|
+
const merged = {
|
|
258
|
+
version: 1,
|
|
259
|
+
clients: [...retained.clients, ...replacement.clients],
|
|
260
|
+
projects: [...retained.projects, ...replacement.projects],
|
|
261
|
+
ingestRuns: [...retained.ingestRuns, ...replacement.ingestRuns],
|
|
262
|
+
entities: [...retained.entities, ...replacement.entities],
|
|
263
|
+
documents: [...retained.documents, ...replacement.documents],
|
|
264
|
+
versions: [...retained.versions, ...replacement.versions],
|
|
265
|
+
sections: [...retained.sections, ...replacement.sections],
|
|
266
|
+
chunks: [...retained.chunks, ...replacement.chunks],
|
|
267
|
+
relations: [...retained.relations, ...replacement.relations],
|
|
268
|
+
objects,
|
|
269
|
+
namespaceAcls: [...retained.namespaceAcls, ...replacement.namespaceAcls]
|
|
270
|
+
};
|
|
271
|
+
validateManifest(merged);
|
|
272
|
+
return merged;
|
|
273
|
+
}
|
|
274
|
+
async function publishRootManifest(root, manifestOrFactory, hooks = {}) {
|
|
39
275
|
const destination = path.join(root, "manifest.json");
|
|
40
276
|
const temporary = path.join(root, `.manifest.${randomUUID()}.tmp`);
|
|
277
|
+
let releaseLock;
|
|
278
|
+
let manifest;
|
|
279
|
+
let writeFailure;
|
|
280
|
+
let cleanupFailed = false;
|
|
41
281
|
try {
|
|
42
282
|
await fs.mkdir(root, { recursive: true });
|
|
283
|
+
releaseLock = await acquireManifestLock(root);
|
|
284
|
+
manifest = typeof manifestOrFactory === "function" ? await manifestOrFactory() : manifestOrFactory;
|
|
285
|
+
validateManifest(manifest);
|
|
43
286
|
await fs.writeFile(temporary, serializeManifest(manifest), { encoding: "utf8", flag: "wx" });
|
|
287
|
+
await hooks.afterTemporaryWrite?.();
|
|
44
288
|
await renameWithRetry(temporary, destination);
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
let cleanupFailed = false;
|
|
289
|
+
} catch (error) {
|
|
290
|
+
writeFailure = error;
|
|
48
291
|
try {
|
|
49
292
|
await fs.rm(temporary, { force: true });
|
|
50
293
|
} catch {
|
|
51
294
|
cleanupFailed = true;
|
|
52
295
|
}
|
|
296
|
+
}
|
|
297
|
+
try {
|
|
298
|
+
await releaseLock?.();
|
|
299
|
+
} catch (error) {
|
|
300
|
+
writeFailure ??= error;
|
|
301
|
+
}
|
|
302
|
+
if (writeFailure !== void 0) {
|
|
53
303
|
throw ioError("manifest_write_failed", `Unable to atomically write manifest under ${root}.${cleanupFailed ? " Temporary manifest cleanup failed." : ""}`);
|
|
54
304
|
}
|
|
305
|
+
return manifest;
|
|
306
|
+
}
|
|
307
|
+
async function writeRootManifest(root, db, hooks = {}) {
|
|
308
|
+
return publishRootManifest(root, async () => {
|
|
309
|
+
const replacement = buildManifest(db);
|
|
310
|
+
validateManifest(replacement);
|
|
311
|
+
const manifest = mergeRootManifests(await readRootManifestIfPresent(root), replacement);
|
|
312
|
+
validateManifest(manifest);
|
|
313
|
+
return manifest;
|
|
314
|
+
}, hooks);
|
|
55
315
|
}
|
|
56
316
|
function verifyObjectBytes(bytes, object) {
|
|
57
317
|
if (bytes.byteLength !== object.byte_size || createHash("sha256").update(bytes).digest("hex") !== object.sha256) {
|
|
@@ -157,13 +417,12 @@ async function backupRoot(sourceRoot, db, destinationRoot) {
|
|
|
157
417
|
}
|
|
158
418
|
async function restoreRoot(sourceRoot, destinationRoot, client) {
|
|
159
419
|
const manifest = await readRootManifest(sourceRoot);
|
|
160
|
-
const
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
if (!manifest.clients.some((row) => row.name === client)) {
|
|
165
|
-
throw ioError("namespace_mismatch", `Manifest does not contain client ${client}.`);
|
|
420
|
+
const projectClientIds = new Set(manifest.projects.map((project) => project.client_id));
|
|
421
|
+
const unscopedForeignClient = manifest.clients.find((row) => row.name !== client && !projectClientIds.has(row.client_id));
|
|
422
|
+
if (unscopedForeignClient) {
|
|
423
|
+
throw ioError("foreign_client_in_manifest", `Manifest contains foreign client ${unscopedForeignClient.name}.`);
|
|
166
424
|
}
|
|
425
|
+
const clientManifest = selectManifestClient(manifest, client);
|
|
167
426
|
const destination = path.resolve(destinationRoot);
|
|
168
427
|
const parent = path.dirname(destination);
|
|
169
428
|
let destinationExists = false;
|
|
@@ -184,15 +443,15 @@ async function restoreRoot(sourceRoot, destinationRoot, client) {
|
|
|
184
443
|
try {
|
|
185
444
|
await fs.mkdir(parent, { recursive: true });
|
|
186
445
|
await fs.mkdir(temporary);
|
|
187
|
-
await copyVerifiedObjects(
|
|
446
|
+
await copyVerifiedObjects(clientManifest, sourceRoot, temporary);
|
|
188
447
|
db = openDatabase(clientDbPath(temporary, client));
|
|
189
|
-
importManifest(db,
|
|
448
|
+
importManifest(db, clientManifest);
|
|
190
449
|
await writeRootManifest(temporary, db);
|
|
191
450
|
db.close();
|
|
192
451
|
db = void 0;
|
|
193
452
|
if (destinationExists) await fs.rmdir(destination);
|
|
194
453
|
await renameWithRetry(temporary, destination);
|
|
195
|
-
return counts(
|
|
454
|
+
return counts(clientManifest);
|
|
196
455
|
} catch (error) {
|
|
197
456
|
try {
|
|
198
457
|
db?.close();
|
|
@@ -223,10 +482,7 @@ async function removeDatabaseFiles(dbPath) {
|
|
|
223
482
|
return failures.length > 0 ? ` Database cleanup failed: ${failures.join("; ")}` : "";
|
|
224
483
|
}
|
|
225
484
|
async function reindexRoot(root, client) {
|
|
226
|
-
const manifest = await readRootManifest(root);
|
|
227
|
-
if (!manifest.clients.some((row) => row.name === client)) {
|
|
228
|
-
throw ioError("namespace_mismatch", `Manifest does not contain client ${client}.`);
|
|
229
|
-
}
|
|
485
|
+
const manifest = selectManifestClient(await readRootManifest(root), client);
|
|
230
486
|
for (const object of manifest.objects) await readAndVerifyObject(root, object);
|
|
231
487
|
const dbPath = clientDbPath(root, client);
|
|
232
488
|
try {
|
|
@@ -259,9 +515,13 @@ async function reindexRoot(root, client) {
|
|
|
259
515
|
}
|
|
260
516
|
}
|
|
261
517
|
export {
|
|
518
|
+
acquireManifestLock,
|
|
262
519
|
backupRoot,
|
|
520
|
+
observeManifestLock,
|
|
521
|
+
publishRootManifest,
|
|
263
522
|
readRootManifest,
|
|
264
523
|
reindexRoot,
|
|
265
524
|
restoreRoot,
|
|
525
|
+
takeoverStaleManifestLock,
|
|
266
526
|
writeRootManifest
|
|
267
527
|
};
|