konoha-mcp 2.2.24 → 2.2.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/NOTICE +14 -46
  2. package/README.md +19 -4
  3. package/apps/web/build/adapter-node.js +1 -1
  4. package/apps/web/build/build-meta.json +2 -2
  5. package/apps/web/build/client/_app/immutable/entry/{app.CKJ0NhrX.js → app.DtoBIrs4.js} +2 -2
  6. package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.br +0 -0
  7. package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.gz +0 -0
  8. package/apps/web/build/client/_app/immutable/nodes/{0.7A6fQJ6z.js → 0.B8Iw7Xl1.js} +1 -1
  9. package/apps/web/build/client/_app/immutable/nodes/0.B8Iw7Xl1.js.br +0 -0
  10. package/apps/web/build/client/_app/immutable/nodes/{0.7A6fQJ6z.js.gz → 0.B8Iw7Xl1.js.gz} +0 -0
  11. package/apps/web/build/client/_app/version.json +1 -1
  12. package/apps/web/build/client/_app/version.json.br +0 -0
  13. package/apps/web/build/client/_app/version.json.gz +0 -0
  14. package/apps/web/build/server/chunks/instance.js +3 -3
  15. package/apps/web/build/server/chunks/internal.js +1 -1
  16. package/apps/web/build/server/entries/pages/_layout.svelte.js +1 -1
  17. package/apps/web/build/server/entries/pages/_layout.svelte.js.map +1 -1
  18. package/apps/web/build/server/manifest-full.js +1 -1
  19. package/apps/web/build/server/manifest.js +1 -1
  20. package/apps/web/build/server/nodes/0.js +1 -1
  21. package/apps/web/build/server/server.js +1 -1
  22. package/apps/web/package.json +1 -1
  23. package/assets/demo-agy.webp +0 -0
  24. package/assets/demo-claude.webp +0 -0
  25. package/assets/demo-codex.webp +0 -0
  26. package/assets/demo-commandcode.webp +0 -0
  27. package/assets/demo-cursor.webp +0 -0
  28. package/assets/demo-opencode.webp +0 -0
  29. package/assets/demo-pi.webp +0 -0
  30. package/assets/demo-skill-embed.webp +0 -0
  31. package/assets/demo-skills.webp +0 -0
  32. package/assets/demo-web.webp +0 -0
  33. package/assets/demo.webp +0 -0
  34. package/assets/konoha-akatsuki-flow.webp +0 -0
  35. package/assets/konoha-anbu-flow.webp +0 -0
  36. package/assets/konoha-flow-12-cli-upgrade-progress-engine.webp +0 -0
  37. package/assets/konoha-jonin-flow.webp +0 -0
  38. package/assets/konoha-kage-gate.webp +0 -0
  39. package/assets/konoha-langgraph-flow.webp +0 -0
  40. package/assets/konoha-orchestration-flow.webp +0 -0
  41. package/assets/konoha-qa-flow.webp +0 -0
  42. package/assets/konoha-telegram-flow.webp +0 -0
  43. package/assets/testing.webp +0 -0
  44. package/bin/commands/init.js +13 -3
  45. package/bin/commands/init_migration.js +32 -1
  46. package/bin/lib/runtime_install.js +6 -0
  47. package/node_modules/jszip/package.json +2 -2
  48. package/node_modules/string_decoder/package.json +1 -1
  49. package/package.json +9 -3
  50. package/patches/jszip@3.10.2.patch +11 -0
  51. package/patches/string_decoder@1.3.0.patch +12 -0
  52. package/src/agent_contract.js +2 -2
  53. package/src/antigravity_subagent_hook.js +30 -12
  54. package/src/chunin_knowledge_ddl.js +95 -0
  55. package/src/db_schema.js +4 -1
  56. package/src/deploy_installer.js +212 -11
  57. package/src/deploy_utils.js +4 -0
  58. package/src/file_tools/common.js +65 -3
  59. package/src/file_tools_router.js +67 -6
  60. package/src/mcp/chunin_knowledge.js +509 -0
  61. package/src/mcp/tool_dispatch.js +2 -1
  62. package/src/mcp/web_search.js +27 -3
  63. package/src/mcp_clients/common.js +31 -0
  64. package/src/mcp_tool_manifest.json +31 -0
  65. package/src/pi_fs_helpers.js +31 -0
  66. package/src/pi_manager.js +68 -28
  67. package/src/templates/AGENTS.md +1 -1
  68. package/src/templates/GEMINI.md +1 -1
  69. package/src/templates/skills/chunin-skill/SKILL.md +34 -0
  70. package/src/templates/skills/chunin-skill/references/deep-research-analyst.md +27 -10
  71. package/src/templates/skills/konoha/SKILL.md +1 -1
  72. package/src/templates/skills/konoha/references/contract.md +1 -1
  73. package/src/templates/skills/konoha/references/peragent-usage.md +1 -1
  74. package/src/url_canonical.js +58 -0
  75. package/src/vector_pipeline.js +6 -4
  76. package/src/vector_search.js +4 -2
  77. package/apps/web/build/client/_app/immutable/entry/app.CKJ0NhrX.js.br +0 -0
  78. package/apps/web/build/client/_app/immutable/entry/app.CKJ0NhrX.js.gz +0 -0
  79. package/apps/web/build/client/_app/immutable/nodes/0.7A6fQJ6z.js.br +0 -0
@@ -0,0 +1,509 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * src/mcp/chunin_knowledge.js — local research knowledge store for Chunin.
5
+ *
6
+ * Chunin's deep-research loop re-discovers the same material on every run. This
7
+ * module keeps verified learnings so a later run can:
8
+ *
9
+ * 1. retrieve the top-K references relevant to the topic from the local store
10
+ * (FTS5 + vector + cross-encoder rerank) before touching the network,
11
+ * 2. search the web only for what is not already known, and
12
+ * 3. write the genuinely new learnings back in one batched, transactional pass.
13
+ *
14
+ * Deduplication is three-layered because each layer catches what the others miss:
15
+ * - `content_hash` identical text, cheap and exact (also the UNIQUE index),
16
+ * - `canonical_url` the same page under a different URL spelling,
17
+ * - cosine >= 0.90 the same claim reworded by a different source.
18
+ *
19
+ * Nothing here needs an API key: retrieval is local SQLite + the bundled ONNX
20
+ * embedder, and rerank degrades to vector order when the semantic runtime is off.
21
+ */
22
+
23
+ const crypto = require('crypto');
24
+
25
+ const { getConnection } = require('../db');
26
+ const { CHUNIN_KNOWLEDGE_TABLE } = require('../chunin_knowledge_ddl');
27
+
28
+ const DEFAULT_TOP_K = 5;
29
+ const DEFAULT_SEMANTIC_THRESHOLD = 0.9;
30
+ const CONFIDENCE_TIERS = new Set(['verified', 'corroborated', 'unverified']);
31
+
32
+ const { canonicalizeUrl } = require('../url_canonical');
33
+
34
+ /**
35
+ * Stable content hash over whitespace- and case-normalized text, so the same
36
+ * sentence with different wrapping or casing dedupes.
37
+ */
38
+ function hashLearning(text) {
39
+ const normalized = String(text || '')
40
+ .toLowerCase()
41
+ .replace(/\s+/g, ' ')
42
+ .trim();
43
+ return crypto.createHash('sha256').update(normalized).digest('hex');
44
+ }
45
+
46
+ function newId(prefix = 'ck') {
47
+ return `${prefix}_${crypto.randomBytes(8).toString('hex')}`;
48
+ }
49
+
50
+ function nowIso() {
51
+ return new Date().toISOString();
52
+ }
53
+
54
+ function clampConfidence(tier) {
55
+ const value = String(tier || '').toLowerCase();
56
+ return CONFIDENCE_TIERS.has(value) ? value : 'unverified';
57
+ }
58
+
59
+ function openDb(options = {}) {
60
+ if (options.conn) return options.conn;
61
+ return getConnection();
62
+ }
63
+
64
+ function isMissingTableError(err) {
65
+ return /no such table|no such column/i.test(String(err && err.message));
66
+ }
67
+
68
+ /**
69
+ * Retrieves the top-K stored learnings for a topic using hybrid retrieval:
70
+ * FTS5 keyword candidates and vector candidates are merged, then reranked by the
71
+ * cross-encoder when available. Falls back to FTS-only, then to recency, so a
72
+ * store that was never embedded still returns something useful.
73
+ *
74
+ * @param {object} options
75
+ * @param {string} options.topic
76
+ * @param {number} [options.topK=5]
77
+ * @param {object} [options.conn]
78
+ * @returns {Promise<Array<object>>}
79
+ */
80
+ async function retrieveRelevant({ topic, topK = DEFAULT_TOP_K, conn = null } = {}) {
81
+ if (!topic || typeof topic !== 'string') return [];
82
+ const db = openDb({ conn });
83
+ const limit = Math.max(1, Math.min(Number(topK) || DEFAULT_TOP_K, 25));
84
+
85
+ let rows = [];
86
+ try {
87
+ rows = db
88
+ .prepare(
89
+ `SELECT k.id, k.query_topic, k.learning_text, k.canonical_url, k.source_title,
90
+ k.content_hash, k.citation_id, k.confidence_tier, k.created_at, k.embedding
91
+ FROM ${CHUNIN_KNOWLEDGE_TABLE} k
92
+ JOIN ${CHUNIN_KNOWLEDGE_TABLE}_fts f ON f.rowid = k.rowid
93
+ WHERE ${CHUNIN_KNOWLEDGE_TABLE}_fts MATCH ?
94
+ ORDER BY bm25(${CHUNIN_KNOWLEDGE_TABLE}_fts)
95
+ LIMIT ?`
96
+ )
97
+ .all(buildFtsQuery(topic), limit * 4);
98
+ } catch (err) {
99
+ if (!isMissingTableError(err)) throw err;
100
+ return [];
101
+ }
102
+
103
+ if (!rows.length) {
104
+ // No keyword overlap: fall back to the most recent learnings for the topic so
105
+ // an unembedded or freshly seeded store still seeds visitedUrls.
106
+ try {
107
+ rows = db
108
+ .prepare(
109
+ `SELECT id, query_topic, learning_text, canonical_url, source_title,
110
+ content_hash, citation_id, confidence_tier, created_at, embedding
111
+ FROM ${CHUNIN_KNOWLEDGE_TABLE}
112
+ WHERE query_topic = ?
113
+ ORDER BY created_at DESC
114
+ LIMIT ?`
115
+ )
116
+ .all(topic, limit);
117
+ } catch (err) {
118
+ if (!isMissingTableError(err)) throw err;
119
+ return [];
120
+ }
121
+ }
122
+
123
+ const ranked = await rerankIfPossible(topic, rows);
124
+ return ranked.slice(0, limit).map(stripEmbedding);
125
+ }
126
+
127
+ function buildFtsQuery(topic) {
128
+ // FTS5 treats punctuation as syntax; quote each token so a topic containing a
129
+ // hyphen or colon cannot become a syntax error.
130
+ const tokens = String(topic)
131
+ .toLowerCase()
132
+ .split(/[^\p{L}\p{N}]+/u)
133
+ .filter(token => token.length > 2)
134
+ .slice(0, 12);
135
+ if (!tokens.length) return '""';
136
+ return tokens.map(token => `"${token}"`).join(' OR ');
137
+ }
138
+
139
+ async function rerankIfPossible(query, rows) {
140
+ if (rows.length <= 1) return rows;
141
+ let rerank;
142
+ try {
143
+ ({ rerank } = require('../vector_pipeline'));
144
+ } catch (_) {
145
+ return rows;
146
+ }
147
+ try {
148
+ const docs = rows.map(row => ({
149
+ id: row.id,
150
+ text: `${row.source_title}\n${row.learning_text}`
151
+ }));
152
+ const scored = await rerank(query, docs, { topK: rows.length });
153
+ if (!Array.isArray(scored) || !scored.length) return rows;
154
+ const byId = new Map(rows.map(row => [row.id, row]));
155
+ const ordered = [];
156
+ for (const entry of scored) {
157
+ const id = entry && (entry.id || (entry.document && entry.document.id));
158
+ const row = id ? byId.get(id) : null;
159
+ if (row) {
160
+ ordered.push({ ...row, rerankScore: entry.score });
161
+ byId.delete(id);
162
+ }
163
+ }
164
+ for (const row of byId.values()) ordered.push(row);
165
+ return ordered;
166
+ } catch (_) {
167
+ // Rerank is an optimisation; vector/FTS order is still a usable ranking.
168
+ return rows;
169
+ }
170
+ }
171
+
172
+ function stripEmbedding(row) {
173
+ if (!row) return row;
174
+ const { embedding: _embedding, ...rest } = row;
175
+ return rest;
176
+ }
177
+
178
+ /**
179
+ * Deduplicates candidate learnings against existing store contents and against
180
+ * each other, then bulk-upserts the survivors in one transaction.
181
+ *
182
+ * Embedding is batched: one `embedBatch` call for the whole survivor set (the
183
+ * pipeline buckets by length and runs real ONNX batches) rather than one call per
184
+ * learning, which is what made a large `learnings[]` a bottleneck.
185
+ *
186
+ * @param {object} options
187
+ * @param {string} options.topic
188
+ * @param {Array<object>} options.learnings
189
+ * @param {object} [options.conn]
190
+ * @param {number} [options.batchSize]
191
+ * @param {number} [options.semanticThreshold=0.9]
192
+ * @returns {Promise<{inserted:number, skipped:number, reasons:object}>}
193
+ */
194
+ async function storeLearnings({
195
+ topic,
196
+ learnings = [],
197
+ conn = null,
198
+ batchSize,
199
+ semanticThreshold = DEFAULT_SEMANTIC_THRESHOLD
200
+ } = {}) {
201
+ const db = openDb({ conn });
202
+ const reasons = { duplicate_hash: 0, duplicate_url: 0, duplicate_semantic: 0, invalid: 0 };
203
+ if (!Array.isArray(learnings) || !learnings.length) {
204
+ return { inserted: 0, skipped: 0, reasons };
205
+ }
206
+
207
+ const existing = loadExistingKeys(db, topic);
208
+ const survivors = [];
209
+
210
+ for (const raw of learnings) {
211
+ const text = typeof raw === 'string' ? raw : raw && (raw.learning_text || raw.text || raw.learning);
212
+ if (!text || typeof text !== 'string' || !text.trim()) {
213
+ reasons.invalid += 1;
214
+ continue;
215
+ }
216
+ const contentHash = hashLearning(text);
217
+ const canonicalUrl = canonicalizeUrl(raw && (raw.canonical_url || raw.url));
218
+ if (existing.hashes.has(contentHash)) {
219
+ reasons.duplicate_hash += 1;
220
+ continue;
221
+ }
222
+ if (canonicalUrl && existing.urls.has(canonicalUrl)) {
223
+ reasons.duplicate_url += 1;
224
+ continue;
225
+ }
226
+ // Within-batch duplicates: same run often re-learns a claim twice.
227
+ if (survivors.some(item => item.content_hash === contentHash)) {
228
+ reasons.duplicate_hash += 1;
229
+ continue;
230
+ }
231
+ if (canonicalUrl && survivors.some(item => item.canonical_url === canonicalUrl)) {
232
+ reasons.duplicate_url += 1;
233
+ continue;
234
+ }
235
+ survivors.push({
236
+ id: newId(),
237
+ query_topic: topic,
238
+ learning_text: text.trim(),
239
+ canonical_url: canonicalUrl,
240
+ source_title: (raw && (raw.source_title || raw.title)) || '',
241
+ content_hash: contentHash,
242
+ citation_id: (raw && (raw.citation_id || raw.citation)) || '',
243
+ confidence_tier: clampConfidence(raw && raw.confidence_tier),
244
+ created_at: nowIso(),
245
+ last_verified_at: (raw && raw.last_verified_at) || ''
246
+ });
247
+ }
248
+
249
+ if (!survivors.length) {
250
+ return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
251
+ }
252
+
253
+ const vectors = await embedSurvivors(survivors.map(item => item.learning_text), { batchSize });
254
+ const accepted = [];
255
+ // The embedder's raw cosine is highly anisotropic: two unrelated sentences score
256
+ // ~0.997 and identical text scores 1.0, so a fixed threshold on the raw vectors
257
+ // would call everything a duplicate. Mean-centering the candidate set first is
258
+ // the standard remedy and restores separation (measured: unrelated pairs -0.46
259
+ // to 0.41, identical text 1.0), which is what makes 0.90 a meaningful cut.
260
+ const allVectors = vectors ? [...existing.vectors, ...vectors.filter(Boolean)] : [];
261
+ const centered = allVectors.length > 1 ? centerVectors(allVectors) : allVectors;
262
+ const existingCentered = centered.slice(0, existing.vectors.length);
263
+
264
+ for (let i = 0; i < survivors.length; i++) {
265
+ const vec = vectors ? vectors[i] : null;
266
+ if (vec && existingCentered.length) {
267
+ const candidate = centered[existing.vectors.length + i];
268
+ const near = candidate && existingCentered.find(prev => cosine(candidate, prev) >= semanticThreshold);
269
+ if (near) {
270
+ reasons.duplicate_semantic += 1;
271
+ continue;
272
+ }
273
+ }
274
+ accepted.push({ ...survivors[i], embedding: vec ? float32ToBuffer(vec) : null });
275
+ }
276
+
277
+ if (!accepted.length) {
278
+ return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
279
+ }
280
+
281
+ // One transaction with a prepared statement: a batch of 50 learnings is 50
282
+ // bound executions, not 50 implicit transactions. `node:sqlite` has no
283
+ // `.transaction()` helper (the repo's sqlite_driver wrapper adds one), so this
284
+ // uses an explicit BEGIN/COMMIT that works on both.
285
+ const insert = db.prepare(
286
+ `INSERT INTO ${CHUNIN_KNOWLEDGE_TABLE}
287
+ (id, query_topic, learning_text, canonical_url, source_title,
288
+ content_hash, citation_id, confidence_tier, created_at, last_verified_at, embedding)
289
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
290
+ );
291
+ db.exec('BEGIN IMMEDIATE');
292
+ try {
293
+ for (const row of accepted) {
294
+ insert.run(
295
+ row.id,
296
+ row.query_topic,
297
+ row.learning_text,
298
+ row.canonical_url,
299
+ row.source_title,
300
+ row.content_hash,
301
+ row.citation_id,
302
+ row.confidence_tier,
303
+ row.created_at,
304
+ row.last_verified_at,
305
+ row.embedding
306
+ );
307
+ }
308
+ db.exec('COMMIT');
309
+ } catch (err) {
310
+ try { db.exec('ROLLBACK'); } catch (_) { /* best effort */ }
311
+ throw err;
312
+ }
313
+
314
+ return {
315
+ inserted: accepted.length,
316
+ skipped: Object.values(reasons).reduce((a, b) => a + b, 0),
317
+ reasons
318
+ };
319
+ }
320
+
321
+ function loadExistingKeys(db, topic) {
322
+ const hashes = new Set();
323
+ const urls = new Set();
324
+ const vectors = [];
325
+ try {
326
+ const rows = db
327
+ .prepare(
328
+ `SELECT content_hash, canonical_url, embedding
329
+ FROM ${CHUNIN_KNOWLEDGE_TABLE}
330
+ WHERE query_topic = ?`
331
+ )
332
+ .all(topic);
333
+ for (const row of rows) {
334
+ if (row.content_hash) hashes.add(row.content_hash);
335
+ if (row.canonical_url) urls.add(row.canonical_url);
336
+ if (row.embedding) vectors.push(bufferToFloat32(row.embedding));
337
+ }
338
+ } catch (err) {
339
+ if (!isMissingTableError(err)) throw err;
340
+ }
341
+ return { hashes, urls, vectors };
342
+ }
343
+
344
+ async function embedSurvivors(texts, { batchSize } = {}) {
345
+ let pipeline;
346
+ try {
347
+ pipeline = require('../vector_pipeline');
348
+ } catch (_) {
349
+ return null;
350
+ }
351
+ try {
352
+ const options = { batchSize: Math.max(1, Number(batchSize) || Math.min(16, texts.length)) };
353
+ const vectors = await pipeline.embedBatch(texts, options);
354
+ return Array.isArray(vectors) ? vectors : null;
355
+ } catch (_) {
356
+ // Semantic runtime disabled or model unavailable: store the text without a
357
+ // vector so FTS and URL dedupe still work. Retrieval degrades, never fails.
358
+ return null;
359
+ }
360
+ }
361
+
362
+ function float32ToBuffer(vec) {
363
+ const arr = vec instanceof Float32Array ? vec : Float32Array.from(vec);
364
+ return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
365
+ }
366
+
367
+ function bufferToFloat32(buf) {
368
+ if (!buf || !buf.length) return new Float32Array(0);
369
+ const copy = Buffer.from(buf);
370
+ return new Float32Array(copy.buffer, copy.byteOffset, copy.byteLength / 4);
371
+ }
372
+
373
+ function cosine(a, b) {
374
+ if (!a || !b || a.length !== b.length || !a.length) return 0;
375
+ let dot = 0;
376
+ for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
377
+ return dot;
378
+ }
379
+
380
+ /**
381
+ * Subtracts the mean vector from every vector, so cosine comparisons measure
382
+ * direction relative to the corpus rather than the embedder's dominant component.
383
+ * Stored embeddings stay uncentered: the mean depends on the set, so it is
384
+ * recomputed per comparison instead of baked into the row.
385
+ */
386
+ function centerVectors(vecs) {
387
+ const dim = vecs[0] && vecs[0].length ? vecs[0].length : 0;
388
+ if (!dim) return vecs;
389
+ const mean = new Float32Array(dim);
390
+ for (const vec of vecs) {
391
+ for (let i = 0; i < dim; i++) mean[i] += vec[i] / vecs.length;
392
+ }
393
+ return vecs.map((vec) => {
394
+ const out = new Float32Array(dim);
395
+ for (let i = 0; i < dim; i++) out[i] = vec[i] - mean[i];
396
+ return out;
397
+ });
398
+ }
399
+
400
+ /**
401
+ * Runs the deep-research loop shape from dzhng/deep-research over a supplied
402
+ * search function: `breadth` queries at the first level, `depth` levels total,
403
+ * with `nextBreadth = ceil(breadth / 2)`. Learnings, follow-up questions and
404
+ * visited URLs accumulate across levels; raw page bodies are discarded.
405
+ *
406
+ * The search function is injected so this is testable without the network and so
407
+ * the caller keeps its own engine/fallback policy.
408
+ *
409
+ * @param {object} options
410
+ * @param {string} options.topic
411
+ * @param {(query:string, limit:number, exclude:string[])=>Promise<Array<object>>} options.search
412
+ * @param {number} [options.breadth=3]
413
+ * @param {number} [options.depth=2]
414
+ * @returns {Promise<{learnings:Array, followUpQuestions:string[], visitedUrls:string[], levels:number}>}
415
+ */
416
+ async function runDeepResearchLoop({ topic, search, breadth = 3, depth = 2, maxResults = 5 } = {}) {
417
+ const clampedBreadth = Math.max(1, Math.min(Number(breadth) || 3, 5));
418
+ const clampedDepth = Math.max(1, Math.min(Number(depth) || 2, 3));
419
+ const state = {
420
+ learnings: [],
421
+ followUpQuestions: [],
422
+ visitedUrls: [],
423
+ visitedSet: new Set()
424
+ };
425
+ let currentQueries = [topic];
426
+ let currentBreadth = clampedBreadth;
427
+ let levels = 0;
428
+
429
+ for (let level = 0; level < clampedDepth; level++) {
430
+ const nextQueries = await collectLevel({
431
+ queries: currentQueries.slice(0, currentBreadth),
432
+ search,
433
+ maxResults,
434
+ state
435
+ });
436
+ levels += 1;
437
+ if (!nextQueries.length || level + 1 >= clampedDepth) break;
438
+ currentBreadth = Math.ceil(currentBreadth / 2);
439
+ currentQueries = nextQueries;
440
+ }
441
+
442
+ return {
443
+ learnings: state.learnings,
444
+ followUpQuestions: state.followUpQuestions,
445
+ visitedUrls: state.visitedUrls,
446
+ levels
447
+ };
448
+ }
449
+
450
+ /** Runs one level's queries and folds their results into `state`. */
451
+ async function collectLevel({ queries, search, maxResults, state }) {
452
+ const nextQueries = [];
453
+ for (const query of queries) {
454
+ let results = [];
455
+ try {
456
+ results = (await search(query, maxResults, state.visitedUrls.slice())) || [];
457
+ } catch (_) {
458
+ results = [];
459
+ }
460
+ for (const result of results) {
461
+ absorbResult(result, state, nextQueries);
462
+ }
463
+ }
464
+ return nextQueries;
465
+ }
466
+
467
+ /** Records one search result: its follow-up question, URL and learning. */
468
+ function absorbResult(result, state, nextQueries) {
469
+ const canonical = canonicalizeUrl(result && (result.url || result.link));
470
+ const text = result && (result.snippet || result.description || result.learning_text);
471
+
472
+ // A follow-up question steers the next level even when its page was already
473
+ // visited, so it is read before the dedupe guard below.
474
+ if (result && result.follow_up_question) {
475
+ const question = String(result.follow_up_question).trim();
476
+ if (question) {
477
+ nextQueries.push(question);
478
+ if (!state.followUpQuestions.includes(question)) state.followUpQuestions.push(question);
479
+ }
480
+ }
481
+
482
+ if (canonical && state.visitedSet.has(canonical)) return;
483
+ if (canonical) {
484
+ state.visitedSet.add(canonical);
485
+ state.visitedUrls.push(canonical);
486
+ }
487
+ if (text && String(text).trim()) {
488
+ state.learnings.push({
489
+ learning_text: String(text).trim(),
490
+ canonical_url: canonical,
491
+ source_title: (result && (result.title || result.source_title)) || '',
492
+ citation_id: (result && result.citation_id) || ''
493
+ });
494
+ }
495
+ }
496
+
497
+ module.exports = {
498
+ DEFAULT_TOP_K,
499
+ DEFAULT_SEMANTIC_THRESHOLD,
500
+ canonicalizeUrl,
501
+ hashLearning,
502
+ retrieveRelevant,
503
+ storeLearnings,
504
+ runDeepResearchLoop,
505
+ float32ToBuffer,
506
+ bufferToFloat32,
507
+ centerVectors,
508
+ cosine
509
+ };
@@ -585,7 +585,8 @@ async function _executeHeavyToolAsync(toolName, args, agent) {
585
585
  const query = args.query;
586
586
  const numResults = Math.min(Math.max(parseInt(args.num_results || 5, 10), 1), 50);
587
587
  const searchDepth = ['standard', 'deep'].includes(args.search_depth) ? args.search_depth : 'standard';
588
- return await runWebSearch(query, numResults, searchDepth, agent);
588
+ const excludeUrls = Array.isArray(args.exclude_urls) ? args.exclude_urls : undefined;
589
+ return await runWebSearch(query, numResults, searchDepth, agent, { excludeUrls });
589
590
  }
590
591
  const docEngineMethod = {
591
592
  document_create: 'documentCreate',
@@ -12,6 +12,7 @@ const { detectActiveAgent } = require('./client_detection');
12
12
  const { logToolCall } = require('./skills');
13
13
  const { globalCircuitRegistry } = require('../circuit_breaker');
14
14
  const { getConfigValue } = require('../config_access');
15
+ const { canonicalizeUrl } = require('../url_canonical');
15
16
 
16
17
  const searxngDir = path.join(KONOHA_DIR, 'searxng');
17
18
  const instancesCachePath = path.join(searxngDir, 'instances_cache.json');
@@ -437,13 +438,29 @@ async function _queryBlendedSearch(q, numResults, isBlended, requestedEngine) {
437
438
 
438
439
  function _appendUniqueResults(currentResults, seenUrls, allResults) {
439
440
  for (const r of currentResults) {
440
- if (!seenUrls.has(r.url)) {
441
- seenUrls.add(r.url);
441
+ const key = canonicalizeUrl(r.url) || r.url;
442
+ if (!seenUrls.has(key)) {
443
+ seenUrls.add(key);
442
444
  allResults.push(r);
443
445
  }
444
446
  }
445
447
  }
446
448
 
449
+ /**
450
+ * Normalizes the caller's `exclude_urls` into canonical keys. Accepts raw URLs,
451
+ * bare hosts, and URLs already in canonical form; anything unparseable is
452
+ * dropped rather than throwing, because a bad exclusion must not fail a search.
453
+ */
454
+ function _normalizeExclusions(excludeUrls) {
455
+ const set = new Set();
456
+ if (!Array.isArray(excludeUrls)) return set;
457
+ for (const entry of excludeUrls) {
458
+ const canonical = canonicalizeUrl(entry);
459
+ if (canonical) set.add(canonical);
460
+ }
461
+ return set;
462
+ }
463
+
447
464
  async function runWebSearch(query, numResults = 5, searchDepth = 'standard', agentName = null, options = {}) {
448
465
  if (!query || !query.trim()) {
449
466
  return JSON.stringify({ status: 'error', message: 'Query is required.' });
@@ -460,6 +477,7 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
460
477
 
461
478
  const allResults = [];
462
479
  const seenUrls = new Set();
480
+ const excluded = _normalizeExclusions(options && options.excludeUrls);
463
481
  const requestedEngine = (options && options.engine) ? String(options.engine).toLowerCase() : 'all';
464
482
  const isBlended = options && options.blended !== undefined ? options.blended : (requestedEngine === 'all');
465
483
 
@@ -469,11 +487,16 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
469
487
  ? specific
470
488
  : await _queryBlendedSearch(q, numResults, isBlended, requestedEngine);
471
489
 
490
+ // Fetch a wider candidate set when exclusions are in play, so filtering out
491
+ // already-known URLs still leaves `numResults` net-new results.
472
492
  _appendUniqueResults(currentResults, seenUrls, allResults);
473
493
  }
474
494
 
475
495
  allResults.sort((a, b) => rankScore(b) - rankScore(a));
476
- const formatted = allResults.slice(0, numResults).map((r, i) => ({
496
+ const fresh = excluded.size
497
+ ? allResults.filter((r) => !excluded.has(canonicalizeUrl(r.url) || r.url))
498
+ : allResults;
499
+ const formatted = fresh.slice(0, numResults).map((r, i) => ({
477
500
  citation_id: i + 1,
478
501
  title: r.title,
479
502
  url: r.url,
@@ -485,6 +508,7 @@ async function runWebSearch(query, numResults = 5, searchDepth = 'standard', age
485
508
  status: 'success',
486
509
  query,
487
510
  search_depth: searchDepth,
511
+ excluded_count: excluded.size,
488
512
  results_count: formatted.length,
489
513
  results: formatted
490
514
  });
@@ -297,6 +297,35 @@ function buildSubagentToolGrants(client = 'generic', agent = null) {
297
297
  ];
298
298
  }
299
299
 
300
+ const PI_READONLY_BUILTINS = Object.freeze(['read', 'grep', 'find', 'ls']);
301
+
302
+ const PI_WRITE_BUILTINS = Object.freeze(['read', 'write', 'edit', 'bash', 'grep', 'find', 'ls']);
303
+
304
+ /**
305
+ * Tool grants in the spelling Pi itself uses.
306
+ *
307
+ * Pi registers builtins in lowercase (`read`, `grep`, `find`, `ls`) and selects MCP
308
+ * tools as `mcp:server/tool`. pi-subagents treats `tools` as a strict allowlist and
309
+ * rejects the launch when a listed name does not resolve, so the `mcp__server__tool`
310
+ * form the other clients use would fail every Pi child.
311
+ */
312
+ function buildPiSubagentToolGrants(agent = null) {
313
+ const { isReadOnlyAgent } = require('../agent_file_safety');
314
+ const readOnly = isReadOnlyAgent(agent);
315
+ const { GENIN_BLOCKED_TOOLS } = readOnly
316
+ ? require('../mcp/genin_guard')
317
+ : { GENIN_BLOCKED_TOOLS: [] };
318
+ const blockedMcp = new Set(GENIN_BLOCKED_TOOLS);
319
+
320
+ const konoha = (deployUtils.KONOHA_CANONICAL_TOOLS || [])
321
+ .filter((t) => !readOnly || !blockedMcp.has(t))
322
+ .map((t) => `mcp:konoha/${t}`);
323
+ const aislopSource = readOnly ? READONLY_AISLOP_TOOLS : SUBAGENT_SAFE_MCP_GRANTS.aislop;
324
+ const aislop = aislopSource.map((t) => `mcp:aislop/${t}`);
325
+
326
+ return [...konoha, ...aislop, ...(readOnly ? PI_READONLY_BUILTINS : PI_WRITE_BUILTINS)];
327
+ }
328
+
300
329
  function buildCommandCodeTools(agent = null) {
301
330
  const { isReadOnlyAgent, isRouterAgent } = require('../agent_file_safety');
302
331
  const readOnly = isReadOnlyAgent(agent);
@@ -380,6 +409,8 @@ module.exports = {
380
409
  backupFile,
381
410
  buildClientToolGrants,
382
411
  buildBareToolGrants,
412
+ buildPiSubagentToolGrants,
413
+ PI_READONLY_BUILTINS,
383
414
  MAIN_AGENT_BUILTIN_TOOLS,
384
415
  SUBAGENT_SAFE_MCP_GRANTS,
385
416
  getKonohaMappedTools,
@@ -1139,6 +1139,30 @@
1139
1139
  },
1140
1140
  "flow_hash": {
1141
1141
  "type": "string"
1142
+ },
1143
+ "breadth": {
1144
+ "type": "number",
1145
+ "minimum": 1,
1146
+ "maximum": 5,
1147
+ "description": "Queries per level in the deep-research loop (default 3, max 5)."
1148
+ },
1149
+ "depth": {
1150
+ "type": "number",
1151
+ "minimum": 1,
1152
+ "maximum": 3,
1153
+ "description": "Levels to recurse (default 2, max 3). nextBreadth = ceil(breadth / 2)."
1154
+ },
1155
+ "output_mode": {
1156
+ "type": "string",
1157
+ "enum": [
1158
+ "report",
1159
+ "concise"
1160
+ ],
1161
+ "description": "Report keeps citations and follow-ups; concise returns learnings only."
1162
+ },
1163
+ "use_knowledge_cache": {
1164
+ "type": "boolean",
1165
+ "description": "Retrieve the top-5 stored references before searching the web (default true)."
1142
1166
  }
1143
1167
  },
1144
1168
  "additionalProperties": false
@@ -1392,6 +1416,13 @@
1392
1416
  "standard",
1393
1417
  "deep"
1394
1418
  ]
1419
+ },
1420
+ "exclude_urls": {
1421
+ "type": "array",
1422
+ "items": {
1423
+ "type": "string"
1424
+ },
1425
+ "description": "URLs the caller already knows. Canonicalized (tracking params, trailing slash, scheme and host case) and filtered out, so only net-new results are returned."
1395
1426
  }
1396
1427
  },
1397
1428
  "required": [