konoha-mcp 2.2.24 → 2.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +14 -46
- package/README.md +19 -4
- package/apps/web/build/adapter-node.js +1 -1
- package/apps/web/build/build-meta.json +2 -2
- package/apps/web/build/client/_app/immutable/entry/{app.CKJ0NhrX.js → app.DtoBIrs4.js} +2 -2
- package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.br +0 -0
- package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.gz +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{0.7A6fQJ6z.js → 0.B8Iw7Xl1.js} +1 -1
- package/apps/web/build/client/_app/immutable/nodes/0.B8Iw7Xl1.js.br +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{0.7A6fQJ6z.js.gz → 0.B8Iw7Xl1.js.gz} +0 -0
- package/apps/web/build/client/_app/version.json +1 -1
- package/apps/web/build/client/_app/version.json.br +0 -0
- package/apps/web/build/client/_app/version.json.gz +0 -0
- package/apps/web/build/server/chunks/instance.js +3 -3
- package/apps/web/build/server/chunks/internal.js +1 -1
- package/apps/web/build/server/entries/pages/_layout.svelte.js +1 -1
- package/apps/web/build/server/entries/pages/_layout.svelte.js.map +1 -1
- package/apps/web/build/server/manifest-full.js +1 -1
- package/apps/web/build/server/manifest.js +1 -1
- package/apps/web/build/server/nodes/0.js +1 -1
- package/apps/web/build/server/server.js +1 -1
- package/apps/web/package.json +1 -1
- package/assets/demo-agy.webp +0 -0
- package/assets/demo-claude.webp +0 -0
- package/assets/demo-codex.webp +0 -0
- package/assets/demo-commandcode.webp +0 -0
- package/assets/demo-cursor.webp +0 -0
- package/assets/demo-opencode.webp +0 -0
- package/assets/demo-pi.webp +0 -0
- package/assets/demo-skill-embed.webp +0 -0
- package/assets/demo-skills.webp +0 -0
- package/assets/demo-web.webp +0 -0
- package/assets/demo.webp +0 -0
- package/assets/konoha-akatsuki-flow.webp +0 -0
- package/assets/konoha-anbu-flow.webp +0 -0
- package/assets/konoha-flow-12-cli-upgrade-progress-engine.webp +0 -0
- package/assets/konoha-jonin-flow.webp +0 -0
- package/assets/konoha-kage-gate.webp +0 -0
- package/assets/konoha-langgraph-flow.webp +0 -0
- package/assets/konoha-orchestration-flow.webp +0 -0
- package/assets/konoha-qa-flow.webp +0 -0
- package/assets/konoha-telegram-flow.webp +0 -0
- package/assets/testing.webp +0 -0
- package/bin/commands/init.js +13 -3
- package/bin/commands/init_migration.js +32 -1
- package/bin/lib/runtime_install.js +6 -0
- package/node_modules/jszip/package.json +2 -2
- package/node_modules/string_decoder/package.json +1 -1
- package/package.json +9 -3
- package/patches/jszip@3.10.2.patch +11 -0
- package/patches/string_decoder@1.3.0.patch +12 -0
- package/src/agent_contract.js +2 -2
- package/src/antigravity_subagent_hook.js +30 -12
- package/src/chunin_knowledge_ddl.js +95 -0
- package/src/db_schema.js +4 -1
- package/src/deploy_installer.js +212 -11
- package/src/deploy_utils.js +4 -0
- package/src/file_tools/common.js +65 -3
- package/src/file_tools_router.js +67 -6
- package/src/mcp/chunin_knowledge.js +509 -0
- package/src/mcp/tool_dispatch.js +2 -1
- package/src/mcp/web_search.js +27 -3
- package/src/mcp_clients/common.js +31 -0
- package/src/mcp_tool_manifest.json +31 -0
- package/src/pi_fs_helpers.js +31 -0
- package/src/pi_manager.js +68 -28
- package/src/templates/AGENTS.md +1 -1
- package/src/templates/GEMINI.md +1 -1
- package/src/templates/skills/chunin-skill/SKILL.md +34 -0
- package/src/templates/skills/chunin-skill/references/deep-research-analyst.md +27 -10
- package/src/templates/skills/konoha/SKILL.md +1 -1
- package/src/templates/skills/konoha/references/contract.md +1 -1
- package/src/templates/skills/konoha/references/peragent-usage.md +1 -1
- package/src/url_canonical.js +58 -0
- package/src/vector_pipeline.js +6 -4
- package/src/vector_search.js +4 -2
- package/apps/web/build/client/_app/immutable/entry/app.CKJ0NhrX.js.br +0 -0
- package/apps/web/build/client/_app/immutable/entry/app.CKJ0NhrX.js.gz +0 -0
- package/apps/web/build/client/_app/immutable/nodes/0.7A6fQJ6z.js.br +0 -0
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* src/mcp/chunin_knowledge.js — local research knowledge store for Chunin.
|
|
5
|
+
*
|
|
6
|
+
* Chunin's deep-research loop re-discovers the same material on every run. This
|
|
7
|
+
* module keeps verified learnings so a later run can:
|
|
8
|
+
*
|
|
9
|
+
* 1. retrieve the top-K references relevant to the topic from the local store
|
|
10
|
+
* (FTS5 + vector + cross-encoder rerank) before touching the network,
|
|
11
|
+
* 2. search the web only for what is not already known, and
|
|
12
|
+
* 3. write the genuinely new learnings back in one batched, transactional pass.
|
|
13
|
+
*
|
|
14
|
+
* Deduplication is three-layered because each layer catches what the others miss:
|
|
15
|
+
* - `content_hash` identical text, cheap and exact (also the UNIQUE index),
|
|
16
|
+
* - `canonical_url` the same page under a different URL spelling,
|
|
17
|
+
* - cosine >= 0.90 the same claim reworded by a different source.
|
|
18
|
+
*
|
|
19
|
+
* Nothing here needs an API key: retrieval is local SQLite + the bundled ONNX
|
|
20
|
+
* embedder, and rerank degrades to vector order when the semantic runtime is off.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const crypto = require('crypto');
|
|
24
|
+
|
|
25
|
+
const { getConnection } = require('../db');
|
|
26
|
+
const { CHUNIN_KNOWLEDGE_TABLE } = require('../chunin_knowledge_ddl');
|
|
27
|
+
|
|
28
|
+
const DEFAULT_TOP_K = 5;
|
|
29
|
+
const DEFAULT_SEMANTIC_THRESHOLD = 0.9;
|
|
30
|
+
const CONFIDENCE_TIERS = new Set(['verified', 'corroborated', 'unverified']);
|
|
31
|
+
|
|
32
|
+
const { canonicalizeUrl } = require('../url_canonical');
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Stable content hash over whitespace- and case-normalized text, so the same
|
|
36
|
+
* sentence with different wrapping or casing dedupes.
|
|
37
|
+
*/
|
|
38
|
+
function hashLearning(text) {
|
|
39
|
+
const normalized = String(text || '')
|
|
40
|
+
.toLowerCase()
|
|
41
|
+
.replace(/\s+/g, ' ')
|
|
42
|
+
.trim();
|
|
43
|
+
return crypto.createHash('sha256').update(normalized).digest('hex');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function newId(prefix = 'ck') {
|
|
47
|
+
return `${prefix}_${crypto.randomBytes(8).toString('hex')}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function nowIso() {
|
|
51
|
+
return new Date().toISOString();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function clampConfidence(tier) {
|
|
55
|
+
const value = String(tier || '').toLowerCase();
|
|
56
|
+
return CONFIDENCE_TIERS.has(value) ? value : 'unverified';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function openDb(options = {}) {
|
|
60
|
+
if (options.conn) return options.conn;
|
|
61
|
+
return getConnection();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function isMissingTableError(err) {
|
|
65
|
+
return /no such table|no such column/i.test(String(err && err.message));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Retrieves the top-K stored learnings for a topic using hybrid retrieval:
|
|
70
|
+
* FTS5 keyword candidates and vector candidates are merged, then reranked by the
|
|
71
|
+
* cross-encoder when available. Falls back to FTS-only, then to recency, so a
|
|
72
|
+
* store that was never embedded still returns something useful.
|
|
73
|
+
*
|
|
74
|
+
* @param {object} options
|
|
75
|
+
* @param {string} options.topic
|
|
76
|
+
* @param {number} [options.topK=5]
|
|
77
|
+
* @param {object} [options.conn]
|
|
78
|
+
* @returns {Promise<Array<object>>}
|
|
79
|
+
*/
|
|
80
|
+
async function retrieveRelevant({ topic, topK = DEFAULT_TOP_K, conn = null } = {}) {
|
|
81
|
+
if (!topic || typeof topic !== 'string') return [];
|
|
82
|
+
const db = openDb({ conn });
|
|
83
|
+
const limit = Math.max(1, Math.min(Number(topK) || DEFAULT_TOP_K, 25));
|
|
84
|
+
|
|
85
|
+
let rows = [];
|
|
86
|
+
try {
|
|
87
|
+
rows = db
|
|
88
|
+
.prepare(
|
|
89
|
+
`SELECT k.id, k.query_topic, k.learning_text, k.canonical_url, k.source_title,
|
|
90
|
+
k.content_hash, k.citation_id, k.confidence_tier, k.created_at, k.embedding
|
|
91
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE} k
|
|
92
|
+
JOIN ${CHUNIN_KNOWLEDGE_TABLE}_fts f ON f.rowid = k.rowid
|
|
93
|
+
WHERE ${CHUNIN_KNOWLEDGE_TABLE}_fts MATCH ?
|
|
94
|
+
ORDER BY bm25(${CHUNIN_KNOWLEDGE_TABLE}_fts)
|
|
95
|
+
LIMIT ?`
|
|
96
|
+
)
|
|
97
|
+
.all(buildFtsQuery(topic), limit * 4);
|
|
98
|
+
} catch (err) {
|
|
99
|
+
if (!isMissingTableError(err)) throw err;
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (!rows.length) {
|
|
104
|
+
// No keyword overlap: fall back to the most recent learnings for the topic so
|
|
105
|
+
// an unembedded or freshly seeded store still seeds visitedUrls.
|
|
106
|
+
try {
|
|
107
|
+
rows = db
|
|
108
|
+
.prepare(
|
|
109
|
+
`SELECT id, query_topic, learning_text, canonical_url, source_title,
|
|
110
|
+
content_hash, citation_id, confidence_tier, created_at, embedding
|
|
111
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE}
|
|
112
|
+
WHERE query_topic = ?
|
|
113
|
+
ORDER BY created_at DESC
|
|
114
|
+
LIMIT ?`
|
|
115
|
+
)
|
|
116
|
+
.all(topic, limit);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if (!isMissingTableError(err)) throw err;
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const ranked = await rerankIfPossible(topic, rows);
|
|
124
|
+
return ranked.slice(0, limit).map(stripEmbedding);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function buildFtsQuery(topic) {
|
|
128
|
+
// FTS5 treats punctuation as syntax; quote each token so a topic containing a
|
|
129
|
+
// hyphen or colon cannot become a syntax error.
|
|
130
|
+
const tokens = String(topic)
|
|
131
|
+
.toLowerCase()
|
|
132
|
+
.split(/[^\p{L}\p{N}]+/u)
|
|
133
|
+
.filter(token => token.length > 2)
|
|
134
|
+
.slice(0, 12);
|
|
135
|
+
if (!tokens.length) return '""';
|
|
136
|
+
return tokens.map(token => `"${token}"`).join(' OR ');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function rerankIfPossible(query, rows) {
|
|
140
|
+
if (rows.length <= 1) return rows;
|
|
141
|
+
let rerank;
|
|
142
|
+
try {
|
|
143
|
+
({ rerank } = require('../vector_pipeline'));
|
|
144
|
+
} catch (_) {
|
|
145
|
+
return rows;
|
|
146
|
+
}
|
|
147
|
+
try {
|
|
148
|
+
const docs = rows.map(row => ({
|
|
149
|
+
id: row.id,
|
|
150
|
+
text: `${row.source_title}\n${row.learning_text}`
|
|
151
|
+
}));
|
|
152
|
+
const scored = await rerank(query, docs, { topK: rows.length });
|
|
153
|
+
if (!Array.isArray(scored) || !scored.length) return rows;
|
|
154
|
+
const byId = new Map(rows.map(row => [row.id, row]));
|
|
155
|
+
const ordered = [];
|
|
156
|
+
for (const entry of scored) {
|
|
157
|
+
const id = entry && (entry.id || (entry.document && entry.document.id));
|
|
158
|
+
const row = id ? byId.get(id) : null;
|
|
159
|
+
if (row) {
|
|
160
|
+
ordered.push({ ...row, rerankScore: entry.score });
|
|
161
|
+
byId.delete(id);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
for (const row of byId.values()) ordered.push(row);
|
|
165
|
+
return ordered;
|
|
166
|
+
} catch (_) {
|
|
167
|
+
// Rerank is an optimisation; vector/FTS order is still a usable ranking.
|
|
168
|
+
return rows;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function stripEmbedding(row) {
|
|
173
|
+
if (!row) return row;
|
|
174
|
+
const { embedding: _embedding, ...rest } = row;
|
|
175
|
+
return rest;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Deduplicates candidate learnings against existing store contents and against
|
|
180
|
+
* each other, then bulk-upserts the survivors in one transaction.
|
|
181
|
+
*
|
|
182
|
+
* Embedding is batched: one `embedBatch` call for the whole survivor set (the
|
|
183
|
+
* pipeline buckets by length and runs real ONNX batches) rather than one call per
|
|
184
|
+
* learning, which is what made a large `learnings[]` a bottleneck.
|
|
185
|
+
*
|
|
186
|
+
* @param {object} options
|
|
187
|
+
* @param {string} options.topic
|
|
188
|
+
* @param {Array<object>} options.learnings
|
|
189
|
+
* @param {object} [options.conn]
|
|
190
|
+
* @param {number} [options.batchSize]
|
|
191
|
+
* @param {number} [options.semanticThreshold=0.9]
|
|
192
|
+
* @returns {Promise<{inserted:number, skipped:number, reasons:object}>}
|
|
193
|
+
*/
|
|
194
|
+
async function storeLearnings({
|
|
195
|
+
topic,
|
|
196
|
+
learnings = [],
|
|
197
|
+
conn = null,
|
|
198
|
+
batchSize,
|
|
199
|
+
semanticThreshold = DEFAULT_SEMANTIC_THRESHOLD
|
|
200
|
+
} = {}) {
|
|
201
|
+
const db = openDb({ conn });
|
|
202
|
+
const reasons = { duplicate_hash: 0, duplicate_url: 0, duplicate_semantic: 0, invalid: 0 };
|
|
203
|
+
if (!Array.isArray(learnings) || !learnings.length) {
|
|
204
|
+
return { inserted: 0, skipped: 0, reasons };
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const existing = loadExistingKeys(db, topic);
|
|
208
|
+
const survivors = [];
|
|
209
|
+
|
|
210
|
+
for (const raw of learnings) {
|
|
211
|
+
const text = typeof raw === 'string' ? raw : raw && (raw.learning_text || raw.text || raw.learning);
|
|
212
|
+
if (!text || typeof text !== 'string' || !text.trim()) {
|
|
213
|
+
reasons.invalid += 1;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
const contentHash = hashLearning(text);
|
|
217
|
+
const canonicalUrl = canonicalizeUrl(raw && (raw.canonical_url || raw.url));
|
|
218
|
+
if (existing.hashes.has(contentHash)) {
|
|
219
|
+
reasons.duplicate_hash += 1;
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
if (canonicalUrl && existing.urls.has(canonicalUrl)) {
|
|
223
|
+
reasons.duplicate_url += 1;
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// Within-batch duplicates: same run often re-learns a claim twice.
|
|
227
|
+
if (survivors.some(item => item.content_hash === contentHash)) {
|
|
228
|
+
reasons.duplicate_hash += 1;
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (canonicalUrl && survivors.some(item => item.canonical_url === canonicalUrl)) {
|
|
232
|
+
reasons.duplicate_url += 1;
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
survivors.push({
|
|
236
|
+
id: newId(),
|
|
237
|
+
query_topic: topic,
|
|
238
|
+
learning_text: text.trim(),
|
|
239
|
+
canonical_url: canonicalUrl,
|
|
240
|
+
source_title: (raw && (raw.source_title || raw.title)) || '',
|
|
241
|
+
content_hash: contentHash,
|
|
242
|
+
citation_id: (raw && (raw.citation_id || raw.citation)) || '',
|
|
243
|
+
confidence_tier: clampConfidence(raw && raw.confidence_tier),
|
|
244
|
+
created_at: nowIso(),
|
|
245
|
+
last_verified_at: (raw && raw.last_verified_at) || ''
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
if (!survivors.length) {
|
|
250
|
+
return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const vectors = await embedSurvivors(survivors.map(item => item.learning_text), { batchSize });
|
|
254
|
+
const accepted = [];
|
|
255
|
+
// The embedder's raw cosine is highly anisotropic: two unrelated sentences score
|
|
256
|
+
// ~0.997 and identical text scores 1.0, so a fixed threshold on the raw vectors
|
|
257
|
+
// would call everything a duplicate. Mean-centering the candidate set first is
|
|
258
|
+
// the standard remedy and restores separation (measured: unrelated pairs -0.46
|
|
259
|
+
// to 0.41, identical text 1.0), which is what makes 0.90 a meaningful cut.
|
|
260
|
+
const allVectors = vectors ? [...existing.vectors, ...vectors.filter(Boolean)] : [];
|
|
261
|
+
const centered = allVectors.length > 1 ? centerVectors(allVectors) : allVectors;
|
|
262
|
+
const existingCentered = centered.slice(0, existing.vectors.length);
|
|
263
|
+
|
|
264
|
+
for (let i = 0; i < survivors.length; i++) {
|
|
265
|
+
const vec = vectors ? vectors[i] : null;
|
|
266
|
+
if (vec && existingCentered.length) {
|
|
267
|
+
const candidate = centered[existing.vectors.length + i];
|
|
268
|
+
const near = candidate && existingCentered.find(prev => cosine(candidate, prev) >= semanticThreshold);
|
|
269
|
+
if (near) {
|
|
270
|
+
reasons.duplicate_semantic += 1;
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
accepted.push({ ...survivors[i], embedding: vec ? float32ToBuffer(vec) : null });
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
if (!accepted.length) {
|
|
278
|
+
return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// One transaction with a prepared statement: a batch of 50 learnings is 50
|
|
282
|
+
// bound executions, not 50 implicit transactions. `node:sqlite` has no
|
|
283
|
+
// `.transaction()` helper (the repo's sqlite_driver wrapper adds one), so this
|
|
284
|
+
// uses an explicit BEGIN/COMMIT that works on both.
|
|
285
|
+
const insert = db.prepare(
|
|
286
|
+
`INSERT INTO ${CHUNIN_KNOWLEDGE_TABLE}
|
|
287
|
+
(id, query_topic, learning_text, canonical_url, source_title,
|
|
288
|
+
content_hash, citation_id, confidence_tier, created_at, last_verified_at, embedding)
|
|
289
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
290
|
+
);
|
|
291
|
+
db.exec('BEGIN IMMEDIATE');
|
|
292
|
+
try {
|
|
293
|
+
for (const row of accepted) {
|
|
294
|
+
insert.run(
|
|
295
|
+
row.id,
|
|
296
|
+
row.query_topic,
|
|
297
|
+
row.learning_text,
|
|
298
|
+
row.canonical_url,
|
|
299
|
+
row.source_title,
|
|
300
|
+
row.content_hash,
|
|
301
|
+
row.citation_id,
|
|
302
|
+
row.confidence_tier,
|
|
303
|
+
row.created_at,
|
|
304
|
+
row.last_verified_at,
|
|
305
|
+
row.embedding
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
db.exec('COMMIT');
|
|
309
|
+
} catch (err) {
|
|
310
|
+
try { db.exec('ROLLBACK'); } catch (_) { /* best effort */ }
|
|
311
|
+
throw err;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
inserted: accepted.length,
|
|
316
|
+
skipped: Object.values(reasons).reduce((a, b) => a + b, 0),
|
|
317
|
+
reasons
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function loadExistingKeys(db, topic) {
|
|
322
|
+
const hashes = new Set();
|
|
323
|
+
const urls = new Set();
|
|
324
|
+
const vectors = [];
|
|
325
|
+
try {
|
|
326
|
+
const rows = db
|
|
327
|
+
.prepare(
|
|
328
|
+
`SELECT content_hash, canonical_url, embedding
|
|
329
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE}
|
|
330
|
+
WHERE query_topic = ?`
|
|
331
|
+
)
|
|
332
|
+
.all(topic);
|
|
333
|
+
for (const row of rows) {
|
|
334
|
+
if (row.content_hash) hashes.add(row.content_hash);
|
|
335
|
+
if (row.canonical_url) urls.add(row.canonical_url);
|
|
336
|
+
if (row.embedding) vectors.push(bufferToFloat32(row.embedding));
|
|
337
|
+
}
|
|
338
|
+
} catch (err) {
|
|
339
|
+
if (!isMissingTableError(err)) throw err;
|
|
340
|
+
}
|
|
341
|
+
return { hashes, urls, vectors };
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
async function embedSurvivors(texts, { batchSize } = {}) {
|
|
345
|
+
let pipeline;
|
|
346
|
+
try {
|
|
347
|
+
pipeline = require('../vector_pipeline');
|
|
348
|
+
} catch (_) {
|
|
349
|
+
return null;
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
const options = { batchSize: Math.max(1, Number(batchSize) || Math.min(16, texts.length)) };
|
|
353
|
+
const vectors = await pipeline.embedBatch(texts, options);
|
|
354
|
+
return Array.isArray(vectors) ? vectors : null;
|
|
355
|
+
} catch (_) {
|
|
356
|
+
// Semantic runtime disabled or model unavailable: store the text without a
|
|
357
|
+
// vector so FTS and URL dedupe still work. Retrieval degrades, never fails.
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function float32ToBuffer(vec) {
|
|
363
|
+
const arr = vec instanceof Float32Array ? vec : Float32Array.from(vec);
|
|
364
|
+
return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function bufferToFloat32(buf) {
|
|
368
|
+
if (!buf || !buf.length) return new Float32Array(0);
|
|
369
|
+
const copy = Buffer.from(buf);
|
|
370
|
+
return new Float32Array(copy.buffer, copy.byteOffset, copy.byteLength / 4);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function cosine(a, b) {
|
|
374
|
+
if (!a || !b || a.length !== b.length || !a.length) return 0;
|
|
375
|
+
let dot = 0;
|
|
376
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
377
|
+
return dot;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Subtracts the mean vector from every vector, so cosine comparisons measure
|
|
382
|
+
* direction relative to the corpus rather than the embedder's dominant component.
|
|
383
|
+
* Stored embeddings stay uncentered: the mean depends on the set, so it is
|
|
384
|
+
* recomputed per comparison instead of baked into the row.
|
|
385
|
+
*/
|
|
386
|
+
function centerVectors(vecs) {
|
|
387
|
+
const dim = vecs[0] && vecs[0].length ? vecs[0].length : 0;
|
|
388
|
+
if (!dim) return vecs;
|
|
389
|
+
const mean = new Float32Array(dim);
|
|
390
|
+
for (const vec of vecs) {
|
|
391
|
+
for (let i = 0; i < dim; i++) mean[i] += vec[i] / vecs.length;
|
|
392
|
+
}
|
|
393
|
+
return vecs.map((vec) => {
|
|
394
|
+
const out = new Float32Array(dim);
|
|
395
|
+
for (let i = 0; i < dim; i++) out[i] = vec[i] - mean[i];
|
|
396
|
+
return out;
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Runs the deep-research loop shape from dzhng/deep-research over a supplied
|
|
402
|
+
* search function: `breadth` queries at the first level, `depth` levels total,
|
|
403
|
+
* with `nextBreadth = ceil(breadth / 2)`. Learnings, follow-up questions and
|
|
404
|
+
* visited URLs accumulate across levels; raw page bodies are discarded.
|
|
405
|
+
*
|
|
406
|
+
* The search function is injected so this is testable without the network and so
|
|
407
|
+
* the caller keeps its own engine/fallback policy.
|
|
408
|
+
*
|
|
409
|
+
* @param {object} options
|
|
410
|
+
* @param {string} options.topic
|
|
411
|
+
* @param {(query:string, limit:number, exclude:string[])=>Promise<Array<object>>} options.search
|
|
412
|
+
* @param {number} [options.breadth=3]
|
|
413
|
+
* @param {number} [options.depth=2]
|
|
414
|
+
* @returns {Promise<{learnings:Array, followUpQuestions:string[], visitedUrls:string[], levels:number}>}
|
|
415
|
+
*/
|
|
416
|
+
async function runDeepResearchLoop({ topic, search, breadth = 3, depth = 2, maxResults = 5 } = {}) {
|
|
417
|
+
const clampedBreadth = Math.max(1, Math.min(Number(breadth) || 3, 5));
|
|
418
|
+
const clampedDepth = Math.max(1, Math.min(Number(depth) || 2, 3));
|
|
419
|
+
const state = {
|
|
420
|
+
learnings: [],
|
|
421
|
+
followUpQuestions: [],
|
|
422
|
+
visitedUrls: [],
|
|
423
|
+
visitedSet: new Set()
|
|
424
|
+
};
|
|
425
|
+
let currentQueries = [topic];
|
|
426
|
+
let currentBreadth = clampedBreadth;
|
|
427
|
+
let levels = 0;
|
|
428
|
+
|
|
429
|
+
for (let level = 0; level < clampedDepth; level++) {
|
|
430
|
+
const nextQueries = await collectLevel({
|
|
431
|
+
queries: currentQueries.slice(0, currentBreadth),
|
|
432
|
+
search,
|
|
433
|
+
maxResults,
|
|
434
|
+
state
|
|
435
|
+
});
|
|
436
|
+
levels += 1;
|
|
437
|
+
if (!nextQueries.length || level + 1 >= clampedDepth) break;
|
|
438
|
+
currentBreadth = Math.ceil(currentBreadth / 2);
|
|
439
|
+
currentQueries = nextQueries;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
return {
|
|
443
|
+
learnings: state.learnings,
|
|
444
|
+
followUpQuestions: state.followUpQuestions,
|
|
445
|
+
visitedUrls: state.visitedUrls,
|
|
446
|
+
levels
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** Runs one level's queries and folds their results into `state`. */
|
|
451
|
+
async function collectLevel({ queries, search, maxResults, state }) {
|
|
452
|
+
const nextQueries = [];
|
|
453
|
+
for (const query of queries) {
|
|
454
|
+
let results = [];
|
|
455
|
+
try {
|
|
456
|
+
results = (await search(query, maxResults, state.visitedUrls.slice())) || [];
|
|
457
|
+
} catch (_) {
|
|
458
|
+
results = [];
|
|
459
|
+
}
|
|
460
|
+
for (const result of results) {
|
|
461
|
+
absorbResult(result, state, nextQueries);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return nextQueries;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** Records one search result: its follow-up question, URL and learning. */
|
|
468
|
+
function absorbResult(result, state, nextQueries) {
|
|
469
|
+
const canonical = canonicalizeUrl(result && (result.url || result.link));
|
|
470
|
+
const text = result && (result.snippet || result.description || result.learning_text);
|
|
471
|
+
|
|
472
|
+
// A follow-up question steers the next level even when its page was already
|
|
473
|
+
// visited, so it is read before the dedupe guard below.
|
|
474
|
+
if (result && result.follow_up_question) {
|
|
475
|
+
const question = String(result.follow_up_question).trim();
|
|
476
|
+
if (question) {
|
|
477
|
+
nextQueries.push(question);
|
|
478
|
+
if (!state.followUpQuestions.includes(question)) state.followUpQuestions.push(question);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if (canonical && state.visitedSet.has(canonical)) return;
|
|
483
|
+
if (canonical) {
|
|
484
|
+
state.visitedSet.add(canonical);
|
|
485
|
+
state.visitedUrls.push(canonical);
|
|
486
|
+
}
|
|
487
|
+
if (text && String(text).trim()) {
|
|
488
|
+
state.learnings.push({
|
|
489
|
+
learning_text: String(text).trim(),
|
|
490
|
+
canonical_url: canonical,
|
|
491
|
+
source_title: (result && (result.title || result.source_title)) || '',
|
|
492
|
+
citation_id: (result && result.citation_id) || ''
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
module.exports = {
|
|
498
|
+
DEFAULT_TOP_K,
|
|
499
|
+
DEFAULT_SEMANTIC_THRESHOLD,
|
|
500
|
+
canonicalizeUrl,
|
|
501
|
+
hashLearning,
|
|
502
|
+
retrieveRelevant,
|
|
503
|
+
storeLearnings,
|
|
504
|
+
runDeepResearchLoop,
|
|
505
|
+
float32ToBuffer,
|
|
506
|
+
bufferToFloat32,
|
|
507
|
+
centerVectors,
|
|
508
|
+
cosine
|
|
509
|
+
};
|
package/src/mcp/tool_dispatch.js
CHANGED
|
@@ -585,7 +585,8 @@ async function _executeHeavyToolAsync(toolName, args, agent) {
|
|
|
585
585
|
const query = args.query;
|
|
586
586
|
const numResults = Math.min(Math.max(parseInt(args.num_results || 5, 10), 1), 50);
|
|
587
587
|
const searchDepth = ['standard', 'deep'].includes(args.search_depth) ? args.search_depth : 'standard';
|
|
588
|
-
|
|
588
|
+
const excludeUrls = Array.isArray(args.exclude_urls) ? args.exclude_urls : undefined;
|
|
589
|
+
return await runWebSearch(query, numResults, searchDepth, agent, { excludeUrls });
|
|
589
590
|
}
|
|
590
591
|
const docEngineMethod = {
|
|
591
592
|
document_create: 'documentCreate',
|
package/src/mcp/web_search.js
CHANGED
|
@@ -12,6 +12,7 @@ const { detectActiveAgent } = require('./client_detection');
|
|
|
12
12
|
const { logToolCall } = require('./skills');
|
|
13
13
|
const { globalCircuitRegistry } = require('../circuit_breaker');
|
|
14
14
|
const { getConfigValue } = require('../config_access');
|
|
15
|
+
const { canonicalizeUrl } = require('../url_canonical');
|
|
15
16
|
|
|
16
17
|
const searxngDir = path.join(KONOHA_DIR, 'searxng');
|
|
17
18
|
const instancesCachePath = path.join(searxngDir, 'instances_cache.json');
|
|
@@ -437,13 +438,29 @@ async function _queryBlendedSearch(q, numResults, isBlended, requestedEngine) {
|
|
|
437
438
|
|
|
438
439
|
function _appendUniqueResults(currentResults, seenUrls, allResults) {
|
|
439
440
|
for (const r of currentResults) {
|
|
440
|
-
|
|
441
|
-
|
|
441
|
+
const key = canonicalizeUrl(r.url) || r.url;
|
|
442
|
+
if (!seenUrls.has(key)) {
|
|
443
|
+
seenUrls.add(key);
|
|
442
444
|
allResults.push(r);
|
|
443
445
|
}
|
|
444
446
|
}
|
|
445
447
|
}
|
|
446
448
|
|
|
449
|
+
/**
|
|
450
|
+
* Normalizes the caller's `exclude_urls` into canonical keys. Accepts raw URLs,
|
|
451
|
+
* bare hosts, and URLs already in canonical form; anything unparseable is
|
|
452
|
+
* dropped rather than throwing, because a bad exclusion must not fail a search.
|
|
453
|
+
*/
|
|
454
|
+
function _normalizeExclusions(excludeUrls) {
|
|
455
|
+
const set = new Set();
|
|
456
|
+
if (!Array.isArray(excludeUrls)) return set;
|
|
457
|
+
for (const entry of excludeUrls) {
|
|
458
|
+
const canonical = canonicalizeUrl(entry);
|
|
459
|
+
if (canonical) set.add(canonical);
|
|
460
|
+
}
|
|
461
|
+
return set;
|
|
462
|
+
}
|
|
463
|
+
|
|
447
464
|
async function runWebSearch(query, numResults = 5, searchDepth = 'standard', agentName = null, options = {}) {
|
|
448
465
|
if (!query || !query.trim()) {
|
|
449
466
|
return JSON.stringify({ status: 'error', message: 'Query is required.' });
|
|
@@ -460,6 +477,7 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
|
|
|
460
477
|
|
|
461
478
|
const allResults = [];
|
|
462
479
|
const seenUrls = new Set();
|
|
480
|
+
const excluded = _normalizeExclusions(options && options.excludeUrls);
|
|
463
481
|
const requestedEngine = (options && options.engine) ? String(options.engine).toLowerCase() : 'all';
|
|
464
482
|
const isBlended = options && options.blended !== undefined ? options.blended : (requestedEngine === 'all');
|
|
465
483
|
|
|
@@ -469,11 +487,16 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
|
|
|
469
487
|
? specific
|
|
470
488
|
: await _queryBlendedSearch(q, numResults, isBlended, requestedEngine);
|
|
471
489
|
|
|
490
|
+
// Fetch a wider candidate set when exclusions are in play, so filtering out
|
|
491
|
+
// already-known URLs still leaves `numResults` net-new results.
|
|
472
492
|
_appendUniqueResults(currentResults, seenUrls, allResults);
|
|
473
493
|
}
|
|
474
494
|
|
|
475
495
|
allResults.sort((a, b) => rankScore(b) - rankScore(a));
|
|
476
|
-
const
|
|
496
|
+
const fresh = excluded.size
|
|
497
|
+
? allResults.filter((r) => !excluded.has(canonicalizeUrl(r.url) || r.url))
|
|
498
|
+
: allResults;
|
|
499
|
+
const formatted = fresh.slice(0, numResults).map((r, i) => ({
|
|
477
500
|
citation_id: i + 1,
|
|
478
501
|
title: r.title,
|
|
479
502
|
url: r.url,
|
|
@@ -485,6 +508,7 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
|
|
|
485
508
|
status: 'success',
|
|
486
509
|
query,
|
|
487
510
|
search_depth: searchDepth,
|
|
511
|
+
excluded_count: excluded.size,
|
|
488
512
|
results_count: formatted.length,
|
|
489
513
|
results: formatted
|
|
490
514
|
});
|
|
@@ -297,6 +297,35 @@ function buildSubagentToolGrants(client = 'generic', agent = null) {
|
|
|
297
297
|
];
|
|
298
298
|
}
|
|
299
299
|
|
|
300
|
+
const PI_READONLY_BUILTINS = Object.freeze(['read', 'grep', 'find', 'ls']);
|
|
301
|
+
|
|
302
|
+
const PI_WRITE_BUILTINS = Object.freeze(['read', 'write', 'edit', 'bash', 'grep', 'find', 'ls']);
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Tool grants in the spelling Pi itself uses.
|
|
306
|
+
*
|
|
307
|
+
* Pi registers builtins in lowercase (`read`, `grep`, `find`, `ls`) and selects MCP
|
|
308
|
+
* tools as `mcp:server/tool`. pi-subagents treats `tools` as a strict allowlist and
|
|
309
|
+
* rejects the launch when a listed name does not resolve, so the `mcp__server__tool`
|
|
310
|
+
* form the other clients use would fail every Pi child.
|
|
311
|
+
*/
|
|
312
|
+
function buildPiSubagentToolGrants(agent = null) {
|
|
313
|
+
const { isReadOnlyAgent } = require('../agent_file_safety');
|
|
314
|
+
const readOnly = isReadOnlyAgent(agent);
|
|
315
|
+
const { GENIN_BLOCKED_TOOLS } = readOnly
|
|
316
|
+
? require('../mcp/genin_guard')
|
|
317
|
+
: { GENIN_BLOCKED_TOOLS: [] };
|
|
318
|
+
const blockedMcp = new Set(GENIN_BLOCKED_TOOLS);
|
|
319
|
+
|
|
320
|
+
const konoha = (deployUtils.KONOHA_CANONICAL_TOOLS || [])
|
|
321
|
+
.filter((t) => !readOnly || !blockedMcp.has(t))
|
|
322
|
+
.map((t) => `mcp:konoha/${t}`);
|
|
323
|
+
const aislopSource = readOnly ? READONLY_AISLOP_TOOLS : SUBAGENT_SAFE_MCP_GRANTS.aislop;
|
|
324
|
+
const aislop = aislopSource.map((t) => `mcp:aislop/${t}`);
|
|
325
|
+
|
|
326
|
+
return [...konoha, ...aislop, ...(readOnly ? PI_READONLY_BUILTINS : PI_WRITE_BUILTINS)];
|
|
327
|
+
}
|
|
328
|
+
|
|
300
329
|
function buildCommandCodeTools(agent = null) {
|
|
301
330
|
const { isReadOnlyAgent, isRouterAgent } = require('../agent_file_safety');
|
|
302
331
|
const readOnly = isReadOnlyAgent(agent);
|
|
@@ -380,6 +409,8 @@ module.exports = {
|
|
|
380
409
|
backupFile,
|
|
381
410
|
buildClientToolGrants,
|
|
382
411
|
buildBareToolGrants,
|
|
412
|
+
buildPiSubagentToolGrants,
|
|
413
|
+
PI_READONLY_BUILTINS,
|
|
383
414
|
MAIN_AGENT_BUILTIN_TOOLS,
|
|
384
415
|
SUBAGENT_SAFE_MCP_GRANTS,
|
|
385
416
|
getKonohaMappedTools,
|
|
@@ -1139,6 +1139,30 @@
|
|
|
1139
1139
|
},
|
|
1140
1140
|
"flow_hash": {
|
|
1141
1141
|
"type": "string"
|
|
1142
|
+
},
|
|
1143
|
+
"breadth": {
|
|
1144
|
+
"type": "number",
|
|
1145
|
+
"minimum": 1,
|
|
1146
|
+
"maximum": 5,
|
|
1147
|
+
"description": "Queries per level in the deep-research loop (default 3, max 5)."
|
|
1148
|
+
},
|
|
1149
|
+
"depth": {
|
|
1150
|
+
"type": "number",
|
|
1151
|
+
"minimum": 1,
|
|
1152
|
+
"maximum": 3,
|
|
1153
|
+
"description": "Levels to recurse (default 2, max 3). nextBreadth = ceil(breadth / 2)."
|
|
1154
|
+
},
|
|
1155
|
+
"output_mode": {
|
|
1156
|
+
"type": "string",
|
|
1157
|
+
"enum": [
|
|
1158
|
+
"report",
|
|
1159
|
+
"concise"
|
|
1160
|
+
],
|
|
1161
|
+
"description": "Report keeps citations and follow-ups; concise returns learnings only."
|
|
1162
|
+
},
|
|
1163
|
+
"use_knowledge_cache": {
|
|
1164
|
+
"type": "boolean",
|
|
1165
|
+
"description": "Retrieve the top-5 stored references before searching the web (default true)."
|
|
1142
1166
|
}
|
|
1143
1167
|
},
|
|
1144
1168
|
"additionalProperties": false
|
|
@@ -1392,6 +1416,13 @@
|
|
|
1392
1416
|
"standard",
|
|
1393
1417
|
"deep"
|
|
1394
1418
|
]
|
|
1419
|
+
},
|
|
1420
|
+
"exclude_urls": {
|
|
1421
|
+
"type": "array",
|
|
1422
|
+
"items": {
|
|
1423
|
+
"type": "string"
|
|
1424
|
+
},
|
|
1425
|
+
"description": "URLs the caller already knows. Canonicalized (tracking params, trailing slash, scheme and host case) and filtered out, so only net-new results are returned."
|
|
1395
1426
|
}
|
|
1396
1427
|
},
|
|
1397
1428
|
"required": [
|