konoha-mcp 2.2.23 → 2.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +14 -8
- package/README.md +20 -5
- package/apps/web/build/adapter-node.js +1 -1
- package/apps/web/build/build-meta.json +2 -2
- package/apps/web/build/client/_app/immutable/entry/{app.TYH87iYy.js → app.DtoBIrs4.js} +2 -2
- package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.br +0 -0
- package/apps/web/build/client/_app/immutable/entry/app.DtoBIrs4.js.gz +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{0.D7dPIEWA.js → 0.B8Iw7Xl1.js} +1 -1
- package/apps/web/build/client/_app/immutable/nodes/0.B8Iw7Xl1.js.br +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{0.D7dPIEWA.js.gz → 0.B8Iw7Xl1.js.gz} +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{13.TOXVaTBr.js → 13.CQB_Ed6W.js} +1 -1
- package/apps/web/build/client/_app/immutable/nodes/13.CQB_Ed6W.js.br +0 -0
- package/apps/web/build/client/_app/immutable/nodes/{13.TOXVaTBr.js.gz → 13.CQB_Ed6W.js.gz} +0 -0
- package/apps/web/build/client/_app/version.json +1 -1
- package/apps/web/build/client/_app/version.json.br +0 -0
- package/apps/web/build/client/_app/version.json.gz +0 -0
- package/apps/web/build/server/chunks/instance.js +3 -3
- package/apps/web/build/server/chunks/internal.js +1 -1
- package/apps/web/build/server/entries/pages/_layout.svelte.js +1 -1
- package/apps/web/build/server/entries/pages/_layout.svelte.js.map +1 -1
- package/apps/web/build/server/entries/pages/docs/_page.svelte.js +1 -1
- package/apps/web/build/server/entries/pages/docs/_page.svelte.js.map +1 -1
- package/apps/web/build/server/manifest-full.js +1 -1
- package/apps/web/build/server/manifest.js +1 -1
- package/apps/web/build/server/nodes/0.js +1 -1
- package/apps/web/build/server/nodes/13.js +1 -1
- package/apps/web/build/server/server.js +1 -1
- package/apps/web/package.json +1 -1
- package/assets/demo-agy.webp +0 -0
- package/assets/demo-claude.webp +0 -0
- package/assets/demo-codex.webp +0 -0
- package/assets/demo-commandcode.webp +0 -0
- package/assets/demo-cursor.webp +0 -0
- package/assets/demo-opencode.webp +0 -0
- package/assets/demo-pi.webp +0 -0
- package/assets/demo-skill-embed.webp +0 -0
- package/assets/demo-skills.webp +0 -0
- package/assets/demo-web.webp +0 -0
- package/assets/demo.webp +0 -0
- package/assets/konoha-akatsuki-flow.webp +0 -0
- package/assets/konoha-anbu-flow.webp +0 -0
- package/assets/konoha-flow-12-cli-upgrade-progress-engine.webp +0 -0
- package/assets/konoha-jonin-flow.webp +0 -0
- package/assets/konoha-kage-gate.webp +0 -0
- package/assets/konoha-langgraph-flow.webp +0 -0
- package/assets/konoha-orchestration-flow.webp +0 -0
- package/assets/konoha-qa-flow.webp +0 -0
- package/assets/konoha-telegram-flow.webp +0 -0
- package/assets/testing.webp +0 -0
- package/bin/commands/bridge.js +62 -4
- package/bin/commands/doctor.js +2 -0
- package/bin/commands/doctor_services.js +57 -0
- package/bin/commands/init.js +29 -8
- package/bin/commands/init_clients.js +95 -8
- package/bin/commands/init_migration.js +32 -1
- package/bin/lib/config_defaults.js +3 -0
- package/bin/lib/runtime_install.js +147 -54
- package/node_modules/jszip/package.json +2 -2
- package/node_modules/string_decoder/package.json +1 -1
- package/package.json +10 -3
- package/patches/gpt-tokenizer@4.1.0.patch +15 -0
- package/patches/jszip@3.10.2.patch +11 -0
- package/patches/string_decoder@1.3.0.patch +12 -0
- package/src/agent_contract.js +2 -2
- package/src/agent_deployer.js +48 -35
- package/src/agent_file_safety.js +57 -0
- package/src/antigravity_subagent_hook.js +30 -12
- package/src/bridge/gateway.js +2 -0
- package/src/bridge/gateway_adapters.js +155 -29
- package/src/bridge/handlers/anthropic.js +95 -26
- package/src/bridge/handlers/openai.js +18 -5
- package/src/bridge/sidecar/raw.js +10 -2
- package/src/bridge/stream_usage.js +66 -0
- package/src/chunin_knowledge_ddl.js +95 -0
- package/src/client_commands.js +190 -0
- package/src/db_bridges.js +53 -7
- package/src/db_schema.js +14 -2
- package/src/deploy_installer.js +212 -11
- package/src/deploy_utils.js +4 -0
- package/src/file_tools/common.js +65 -3
- package/src/file_tools_mcp.js +17 -9
- package/src/file_tools_router.js +67 -6
- package/src/global_skill_sync.js +3 -1
- package/src/mcp/chunin_knowledge.js +509 -0
- package/src/mcp/langgraph/edges/router.js +3 -0
- package/src/mcp/langgraph/index.js +22 -3
- package/src/mcp/langgraph/nodes/dispatch.js +3 -0
- package/src/mcp/langgraph/nodes/execute.js +87 -13
- package/src/mcp/langgraph/nodes/review.js +13 -1
- package/src/mcp/langgraph/slice_executor_registry.js +22 -0
- package/src/mcp/langgraph/state.js +5 -0
- package/src/mcp/lazy_subagent.js +11 -1
- package/src/mcp/memory_project_context.js +4 -1
- package/src/mcp/memory_subagent_runner.js +2 -1
- package/src/mcp/skills_common.js +1 -0
- package/src/mcp/subagent_spawner.js +7 -0
- package/src/mcp/todo_walkthrough_sync.js +7 -0
- package/src/mcp/tool_dispatch.js +18 -1
- package/src/mcp/web_search.js +27 -3
- package/src/mcp/workflow_completion_helpers.js +6 -1
- package/src/mcp/workflow_gates.js +118 -5
- package/src/mcp/workflow_sannin.js +9 -3
- package/src/mcp_clients/common.js +31 -0
- package/src/mcp_tool_manifest.json +36 -1
- package/src/pi_fs_helpers.js +31 -0
- package/src/pi_manager.js +68 -28
- package/src/provider_usage_logger.js +92 -0
- package/src/schema_ddl.js +17 -1
- package/src/templates/AGENTS.md +1 -1
- package/src/templates/GEMINI.md +1 -1
- package/src/templates/skills/chunin-skill/SKILL.md +34 -0
- package/src/templates/skills/chunin-skill/references/deep-research-analyst.md +27 -10
- package/src/templates/skills/kage-skill/SKILL.md +1 -1
- package/src/templates/skills/konoha/SKILL.md +23 -165
- package/src/templates/skills/konoha/references/contract.md +2 -2
- package/src/templates/skills/konoha/references/peragent-usage.md +1 -1
- package/src/token_usage.js +57 -0
- package/src/tools_savings_logger.js +26 -8
- package/src/url_canonical.js +58 -0
- package/src/vector_pipeline.js +6 -4
- package/src/vector_search.js +4 -2
- package/src/web/controllers/core_controller.js +10 -1
- package/src/workflow/jonin-builder/live_browser_monitor.js +23 -7
- package/apps/web/build/client/_app/immutable/entry/app.TYH87iYy.js.br +0 -0
- package/apps/web/build/client/_app/immutable/entry/app.TYH87iYy.js.gz +0 -0
- package/apps/web/build/client/_app/immutable/nodes/0.D7dPIEWA.js.br +0 -0
- package/apps/web/build/client/_app/immutable/nodes/13.TOXVaTBr.js.br +0 -0
- package/src/templates/skills/konoha/references/architecture.md +0 -99
- package/src/templates/skills/konoha/references/benchmark.md +0 -123
- package/src/templates/skills/konoha/references/docs.md +0 -46
- package/src/templates/skills/konoha/references/test.md +0 -71
- package/src/templates/skills/konoha/references/workflow.md +0 -104
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* src/mcp/chunin_knowledge.js — local research knowledge store for Chunin.
|
|
5
|
+
*
|
|
6
|
+
* Chunin's deep-research loop re-discovers the same material on every run. This
|
|
7
|
+
* module keeps verified learnings so a later run can:
|
|
8
|
+
*
|
|
9
|
+
* 1. retrieve the top-K references relevant to the topic from the local store
|
|
10
|
+
* (FTS5 + vector + cross-encoder rerank) before touching the network,
|
|
11
|
+
* 2. search the web only for what is not already known, and
|
|
12
|
+
* 3. write the genuinely new learnings back in one batched, transactional pass.
|
|
13
|
+
*
|
|
14
|
+
* Deduplication is three-layered because each layer catches what the others miss:
|
|
15
|
+
* - `content_hash` identical text, cheap and exact (also the UNIQUE index),
|
|
16
|
+
* - `canonical_url` the same page under a different URL spelling,
|
|
17
|
+
* - cosine >= 0.90 the same claim reworded by a different source.
|
|
18
|
+
*
|
|
19
|
+
* Nothing here needs an API key: retrieval is local SQLite + the bundled ONNX
|
|
20
|
+
* embedder, and rerank degrades to vector order when the semantic runtime is off.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const crypto = require('crypto');
|
|
24
|
+
|
|
25
|
+
const { getConnection } = require('../db');
|
|
26
|
+
const { CHUNIN_KNOWLEDGE_TABLE } = require('../chunin_knowledge_ddl');
|
|
27
|
+
|
|
28
|
+
const DEFAULT_TOP_K = 5;
|
|
29
|
+
const DEFAULT_SEMANTIC_THRESHOLD = 0.9;
|
|
30
|
+
const CONFIDENCE_TIERS = new Set(['verified', 'corroborated', 'unverified']);
|
|
31
|
+
|
|
32
|
+
const { canonicalizeUrl } = require('../url_canonical');
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Stable content hash over whitespace- and case-normalized text, so the same
|
|
36
|
+
* sentence with different wrapping or casing dedupes.
|
|
37
|
+
*/
|
|
38
|
+
function hashLearning(text) {
|
|
39
|
+
const normalized = String(text || '')
|
|
40
|
+
.toLowerCase()
|
|
41
|
+
.replace(/\s+/g, ' ')
|
|
42
|
+
.trim();
|
|
43
|
+
return crypto.createHash('sha256').update(normalized).digest('hex');
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function newId(prefix = 'ck') {
|
|
47
|
+
return `${prefix}_${crypto.randomBytes(8).toString('hex')}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function nowIso() {
|
|
51
|
+
return new Date().toISOString();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function clampConfidence(tier) {
|
|
55
|
+
const value = String(tier || '').toLowerCase();
|
|
56
|
+
return CONFIDENCE_TIERS.has(value) ? value : 'unverified';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function openDb(options = {}) {
|
|
60
|
+
if (options.conn) return options.conn;
|
|
61
|
+
return getConnection();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function isMissingTableError(err) {
|
|
65
|
+
return /no such table|no such column/i.test(String(err && err.message));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Retrieves the top-K stored learnings for a topic using hybrid retrieval:
|
|
70
|
+
* FTS5 keyword candidates and vector candidates are merged, then reranked by the
|
|
71
|
+
* cross-encoder when available. Falls back to FTS-only, then to recency, so a
|
|
72
|
+
* store that was never embedded still returns something useful.
|
|
73
|
+
*
|
|
74
|
+
* @param {object} options
|
|
75
|
+
* @param {string} options.topic
|
|
76
|
+
* @param {number} [options.topK=5]
|
|
77
|
+
* @param {object} [options.conn]
|
|
78
|
+
* @returns {Promise<Array<object>>}
|
|
79
|
+
*/
|
|
80
|
+
async function retrieveRelevant({ topic, topK = DEFAULT_TOP_K, conn = null } = {}) {
|
|
81
|
+
if (!topic || typeof topic !== 'string') return [];
|
|
82
|
+
const db = openDb({ conn });
|
|
83
|
+
const limit = Math.max(1, Math.min(Number(topK) || DEFAULT_TOP_K, 25));
|
|
84
|
+
|
|
85
|
+
let rows = [];
|
|
86
|
+
try {
|
|
87
|
+
rows = db
|
|
88
|
+
.prepare(
|
|
89
|
+
`SELECT k.id, k.query_topic, k.learning_text, k.canonical_url, k.source_title,
|
|
90
|
+
k.content_hash, k.citation_id, k.confidence_tier, k.created_at, k.embedding
|
|
91
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE} k
|
|
92
|
+
JOIN ${CHUNIN_KNOWLEDGE_TABLE}_fts f ON f.rowid = k.rowid
|
|
93
|
+
WHERE ${CHUNIN_KNOWLEDGE_TABLE}_fts MATCH ?
|
|
94
|
+
ORDER BY bm25(${CHUNIN_KNOWLEDGE_TABLE}_fts)
|
|
95
|
+
LIMIT ?`
|
|
96
|
+
)
|
|
97
|
+
.all(buildFtsQuery(topic), limit * 4);
|
|
98
|
+
} catch (err) {
|
|
99
|
+
if (!isMissingTableError(err)) throw err;
|
|
100
|
+
return [];
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (!rows.length) {
|
|
104
|
+
// No keyword overlap: fall back to the most recent learnings for the topic so
|
|
105
|
+
// an unembedded or freshly seeded store still seeds visitedUrls.
|
|
106
|
+
try {
|
|
107
|
+
rows = db
|
|
108
|
+
.prepare(
|
|
109
|
+
`SELECT id, query_topic, learning_text, canonical_url, source_title,
|
|
110
|
+
content_hash, citation_id, confidence_tier, created_at, embedding
|
|
111
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE}
|
|
112
|
+
WHERE query_topic = ?
|
|
113
|
+
ORDER BY created_at DESC
|
|
114
|
+
LIMIT ?`
|
|
115
|
+
)
|
|
116
|
+
.all(topic, limit);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if (!isMissingTableError(err)) throw err;
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const ranked = await rerankIfPossible(topic, rows);
|
|
124
|
+
return ranked.slice(0, limit).map(stripEmbedding);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function buildFtsQuery(topic) {
|
|
128
|
+
// FTS5 treats punctuation as syntax; quote each token so a topic containing a
|
|
129
|
+
// hyphen or colon cannot become a syntax error.
|
|
130
|
+
const tokens = String(topic)
|
|
131
|
+
.toLowerCase()
|
|
132
|
+
.split(/[^\p{L}\p{N}]+/u)
|
|
133
|
+
.filter(token => token.length > 2)
|
|
134
|
+
.slice(0, 12);
|
|
135
|
+
if (!tokens.length) return '""';
|
|
136
|
+
return tokens.map(token => `"${token}"`).join(' OR ');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function rerankIfPossible(query, rows) {
|
|
140
|
+
if (rows.length <= 1) return rows;
|
|
141
|
+
let rerank;
|
|
142
|
+
try {
|
|
143
|
+
({ rerank } = require('../vector_pipeline'));
|
|
144
|
+
} catch (_) {
|
|
145
|
+
return rows;
|
|
146
|
+
}
|
|
147
|
+
try {
|
|
148
|
+
const docs = rows.map(row => ({
|
|
149
|
+
id: row.id,
|
|
150
|
+
text: `${row.source_title}\n${row.learning_text}`
|
|
151
|
+
}));
|
|
152
|
+
const scored = await rerank(query, docs, { topK: rows.length });
|
|
153
|
+
if (!Array.isArray(scored) || !scored.length) return rows;
|
|
154
|
+
const byId = new Map(rows.map(row => [row.id, row]));
|
|
155
|
+
const ordered = [];
|
|
156
|
+
for (const entry of scored) {
|
|
157
|
+
const id = entry && (entry.id || (entry.document && entry.document.id));
|
|
158
|
+
const row = id ? byId.get(id) : null;
|
|
159
|
+
if (row) {
|
|
160
|
+
ordered.push({ ...row, rerankScore: entry.score });
|
|
161
|
+
byId.delete(id);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
for (const row of byId.values()) ordered.push(row);
|
|
165
|
+
return ordered;
|
|
166
|
+
} catch (_) {
|
|
167
|
+
// Rerank is an optimisation; vector/FTS order is still a usable ranking.
|
|
168
|
+
return rows;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function stripEmbedding(row) {
|
|
173
|
+
if (!row) return row;
|
|
174
|
+
const { embedding: _embedding, ...rest } = row;
|
|
175
|
+
return rest;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Deduplicates candidate learnings against existing store contents and against
|
|
180
|
+
* each other, then bulk-upserts the survivors in one transaction.
|
|
181
|
+
*
|
|
182
|
+
* Embedding is batched: one `embedBatch` call for the whole survivor set (the
|
|
183
|
+
* pipeline buckets by length and runs real ONNX batches) rather than one call per
|
|
184
|
+
* learning, which is what made a large `learnings[]` a bottleneck.
|
|
185
|
+
*
|
|
186
|
+
* @param {object} options
|
|
187
|
+
* @param {string} options.topic
|
|
188
|
+
* @param {Array<object>} options.learnings
|
|
189
|
+
* @param {object} [options.conn]
|
|
190
|
+
* @param {number} [options.batchSize]
|
|
191
|
+
* @param {number} [options.semanticThreshold=0.9]
|
|
192
|
+
* @returns {Promise<{inserted:number, skipped:number, reasons:object}>}
|
|
193
|
+
*/
|
|
194
|
+
async function storeLearnings({
|
|
195
|
+
topic,
|
|
196
|
+
learnings = [],
|
|
197
|
+
conn = null,
|
|
198
|
+
batchSize,
|
|
199
|
+
semanticThreshold = DEFAULT_SEMANTIC_THRESHOLD
|
|
200
|
+
} = {}) {
|
|
201
|
+
const db = openDb({ conn });
|
|
202
|
+
const reasons = { duplicate_hash: 0, duplicate_url: 0, duplicate_semantic: 0, invalid: 0 };
|
|
203
|
+
if (!Array.isArray(learnings) || !learnings.length) {
|
|
204
|
+
return { inserted: 0, skipped: 0, reasons };
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const existing = loadExistingKeys(db, topic);
|
|
208
|
+
const survivors = [];
|
|
209
|
+
|
|
210
|
+
for (const raw of learnings) {
|
|
211
|
+
const text = typeof raw === 'string' ? raw : raw && (raw.learning_text || raw.text || raw.learning);
|
|
212
|
+
if (!text || typeof text !== 'string' || !text.trim()) {
|
|
213
|
+
reasons.invalid += 1;
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
216
|
+
const contentHash = hashLearning(text);
|
|
217
|
+
const canonicalUrl = canonicalizeUrl(raw && (raw.canonical_url || raw.url));
|
|
218
|
+
if (existing.hashes.has(contentHash)) {
|
|
219
|
+
reasons.duplicate_hash += 1;
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
if (canonicalUrl && existing.urls.has(canonicalUrl)) {
|
|
223
|
+
reasons.duplicate_url += 1;
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// Within-batch duplicates: same run often re-learns a claim twice.
|
|
227
|
+
if (survivors.some(item => item.content_hash === contentHash)) {
|
|
228
|
+
reasons.duplicate_hash += 1;
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
if (canonicalUrl && survivors.some(item => item.canonical_url === canonicalUrl)) {
|
|
232
|
+
reasons.duplicate_url += 1;
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
survivors.push({
|
|
236
|
+
id: newId(),
|
|
237
|
+
query_topic: topic,
|
|
238
|
+
learning_text: text.trim(),
|
|
239
|
+
canonical_url: canonicalUrl,
|
|
240
|
+
source_title: (raw && (raw.source_title || raw.title)) || '',
|
|
241
|
+
content_hash: contentHash,
|
|
242
|
+
citation_id: (raw && (raw.citation_id || raw.citation)) || '',
|
|
243
|
+
confidence_tier: clampConfidence(raw && raw.confidence_tier),
|
|
244
|
+
created_at: nowIso(),
|
|
245
|
+
last_verified_at: (raw && raw.last_verified_at) || ''
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
if (!survivors.length) {
|
|
250
|
+
return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const vectors = await embedSurvivors(survivors.map(item => item.learning_text), { batchSize });
|
|
254
|
+
const accepted = [];
|
|
255
|
+
// The embedder's raw cosine is highly anisotropic: two unrelated sentences score
|
|
256
|
+
// ~0.997 and identical text scores 1.0, so a fixed threshold on the raw vectors
|
|
257
|
+
// would call everything a duplicate. Mean-centering the candidate set first is
|
|
258
|
+
// the standard remedy and restores separation (measured: unrelated pairs -0.46
|
|
259
|
+
// to 0.41, identical text 1.0), which is what makes 0.90 a meaningful cut.
|
|
260
|
+
const allVectors = vectors ? [...existing.vectors, ...vectors.filter(Boolean)] : [];
|
|
261
|
+
const centered = allVectors.length > 1 ? centerVectors(allVectors) : allVectors;
|
|
262
|
+
const existingCentered = centered.slice(0, existing.vectors.length);
|
|
263
|
+
|
|
264
|
+
for (let i = 0; i < survivors.length; i++) {
|
|
265
|
+
const vec = vectors ? vectors[i] : null;
|
|
266
|
+
if (vec && existingCentered.length) {
|
|
267
|
+
const candidate = centered[existing.vectors.length + i];
|
|
268
|
+
const near = candidate && existingCentered.find(prev => cosine(candidate, prev) >= semanticThreshold);
|
|
269
|
+
if (near) {
|
|
270
|
+
reasons.duplicate_semantic += 1;
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
accepted.push({ ...survivors[i], embedding: vec ? float32ToBuffer(vec) : null });
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
if (!accepted.length) {
|
|
278
|
+
return { inserted: 0, skipped: Object.values(reasons).reduce((a, b) => a + b, 0), reasons };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// One transaction with a prepared statement: a batch of 50 learnings is 50
|
|
282
|
+
// bound executions, not 50 implicit transactions. `node:sqlite` has no
|
|
283
|
+
// `.transaction()` helper (the repo's sqlite_driver wrapper adds one), so this
|
|
284
|
+
// uses an explicit BEGIN/COMMIT that works on both.
|
|
285
|
+
const insert = db.prepare(
|
|
286
|
+
`INSERT INTO ${CHUNIN_KNOWLEDGE_TABLE}
|
|
287
|
+
(id, query_topic, learning_text, canonical_url, source_title,
|
|
288
|
+
content_hash, citation_id, confidence_tier, created_at, last_verified_at, embedding)
|
|
289
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
290
|
+
);
|
|
291
|
+
db.exec('BEGIN IMMEDIATE');
|
|
292
|
+
try {
|
|
293
|
+
for (const row of accepted) {
|
|
294
|
+
insert.run(
|
|
295
|
+
row.id,
|
|
296
|
+
row.query_topic,
|
|
297
|
+
row.learning_text,
|
|
298
|
+
row.canonical_url,
|
|
299
|
+
row.source_title,
|
|
300
|
+
row.content_hash,
|
|
301
|
+
row.citation_id,
|
|
302
|
+
row.confidence_tier,
|
|
303
|
+
row.created_at,
|
|
304
|
+
row.last_verified_at,
|
|
305
|
+
row.embedding
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
db.exec('COMMIT');
|
|
309
|
+
} catch (err) {
|
|
310
|
+
try { db.exec('ROLLBACK'); } catch (_) { /* best effort */ }
|
|
311
|
+
throw err;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
inserted: accepted.length,
|
|
316
|
+
skipped: Object.values(reasons).reduce((a, b) => a + b, 0),
|
|
317
|
+
reasons
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function loadExistingKeys(db, topic) {
|
|
322
|
+
const hashes = new Set();
|
|
323
|
+
const urls = new Set();
|
|
324
|
+
const vectors = [];
|
|
325
|
+
try {
|
|
326
|
+
const rows = db
|
|
327
|
+
.prepare(
|
|
328
|
+
`SELECT content_hash, canonical_url, embedding
|
|
329
|
+
FROM ${CHUNIN_KNOWLEDGE_TABLE}
|
|
330
|
+
WHERE query_topic = ?`
|
|
331
|
+
)
|
|
332
|
+
.all(topic);
|
|
333
|
+
for (const row of rows) {
|
|
334
|
+
if (row.content_hash) hashes.add(row.content_hash);
|
|
335
|
+
if (row.canonical_url) urls.add(row.canonical_url);
|
|
336
|
+
if (row.embedding) vectors.push(bufferToFloat32(row.embedding));
|
|
337
|
+
}
|
|
338
|
+
} catch (err) {
|
|
339
|
+
if (!isMissingTableError(err)) throw err;
|
|
340
|
+
}
|
|
341
|
+
return { hashes, urls, vectors };
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
async function embedSurvivors(texts, { batchSize } = {}) {
|
|
345
|
+
let pipeline;
|
|
346
|
+
try {
|
|
347
|
+
pipeline = require('../vector_pipeline');
|
|
348
|
+
} catch (_) {
|
|
349
|
+
return null;
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
const options = { batchSize: Math.max(1, Number(batchSize) || Math.min(16, texts.length)) };
|
|
353
|
+
const vectors = await pipeline.embedBatch(texts, options);
|
|
354
|
+
return Array.isArray(vectors) ? vectors : null;
|
|
355
|
+
} catch (_) {
|
|
356
|
+
// Semantic runtime disabled or model unavailable: store the text without a
|
|
357
|
+
// vector so FTS and URL dedupe still work. Retrieval degrades, never fails.
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function float32ToBuffer(vec) {
|
|
363
|
+
const arr = vec instanceof Float32Array ? vec : Float32Array.from(vec);
|
|
364
|
+
return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function bufferToFloat32(buf) {
|
|
368
|
+
if (!buf || !buf.length) return new Float32Array(0);
|
|
369
|
+
const copy = Buffer.from(buf);
|
|
370
|
+
return new Float32Array(copy.buffer, copy.byteOffset, copy.byteLength / 4);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function cosine(a, b) {
|
|
374
|
+
if (!a || !b || a.length !== b.length || !a.length) return 0;
|
|
375
|
+
let dot = 0;
|
|
376
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
377
|
+
return dot;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* Subtracts the mean vector from every vector, so cosine comparisons measure
|
|
382
|
+
* direction relative to the corpus rather than the embedder's dominant component.
|
|
383
|
+
* Stored embeddings stay uncentered: the mean depends on the set, so it is
|
|
384
|
+
* recomputed per comparison instead of baked into the row.
|
|
385
|
+
*/
|
|
386
|
+
function centerVectors(vecs) {
|
|
387
|
+
const dim = vecs[0] && vecs[0].length ? vecs[0].length : 0;
|
|
388
|
+
if (!dim) return vecs;
|
|
389
|
+
const mean = new Float32Array(dim);
|
|
390
|
+
for (const vec of vecs) {
|
|
391
|
+
for (let i = 0; i < dim; i++) mean[i] += vec[i] / vecs.length;
|
|
392
|
+
}
|
|
393
|
+
return vecs.map((vec) => {
|
|
394
|
+
const out = new Float32Array(dim);
|
|
395
|
+
for (let i = 0; i < dim; i++) out[i] = vec[i] - mean[i];
|
|
396
|
+
return out;
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Runs the deep-research loop shape from dzhng/deep-research over a supplied
|
|
402
|
+
* search function: `breadth` queries at the first level, `depth` levels total,
|
|
403
|
+
* with `nextBreadth = ceil(breadth / 2)`. Learnings, follow-up questions and
|
|
404
|
+
* visited URLs accumulate across levels; raw page bodies are discarded.
|
|
405
|
+
*
|
|
406
|
+
* The search function is injected so this is testable without the network and so
|
|
407
|
+
* the caller keeps its own engine/fallback policy.
|
|
408
|
+
*
|
|
409
|
+
* @param {object} options
|
|
410
|
+
* @param {string} options.topic
|
|
411
|
+
* @param {(query:string, limit:number, exclude:string[])=>Promise<Array<object>>} options.search
|
|
412
|
+
* @param {number} [options.breadth=3]
|
|
413
|
+
* @param {number} [options.depth=2]
|
|
414
|
+
* @returns {Promise<{learnings:Array, followUpQuestions:string[], visitedUrls:string[], levels:number}>}
|
|
415
|
+
*/
|
|
416
|
+
async function runDeepResearchLoop({ topic, search, breadth = 3, depth = 2, maxResults = 5 } = {}) {
|
|
417
|
+
const clampedBreadth = Math.max(1, Math.min(Number(breadth) || 3, 5));
|
|
418
|
+
const clampedDepth = Math.max(1, Math.min(Number(depth) || 2, 3));
|
|
419
|
+
const state = {
|
|
420
|
+
learnings: [],
|
|
421
|
+
followUpQuestions: [],
|
|
422
|
+
visitedUrls: [],
|
|
423
|
+
visitedSet: new Set()
|
|
424
|
+
};
|
|
425
|
+
let currentQueries = [topic];
|
|
426
|
+
let currentBreadth = clampedBreadth;
|
|
427
|
+
let levels = 0;
|
|
428
|
+
|
|
429
|
+
for (let level = 0; level < clampedDepth; level++) {
|
|
430
|
+
const nextQueries = await collectLevel({
|
|
431
|
+
queries: currentQueries.slice(0, currentBreadth),
|
|
432
|
+
search,
|
|
433
|
+
maxResults,
|
|
434
|
+
state
|
|
435
|
+
});
|
|
436
|
+
levels += 1;
|
|
437
|
+
if (!nextQueries.length || level + 1 >= clampedDepth) break;
|
|
438
|
+
currentBreadth = Math.ceil(currentBreadth / 2);
|
|
439
|
+
currentQueries = nextQueries;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
return {
|
|
443
|
+
learnings: state.learnings,
|
|
444
|
+
followUpQuestions: state.followUpQuestions,
|
|
445
|
+
visitedUrls: state.visitedUrls,
|
|
446
|
+
levels
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** Runs one level's queries and folds their results into `state`. */
|
|
451
|
+
async function collectLevel({ queries, search, maxResults, state }) {
|
|
452
|
+
const nextQueries = [];
|
|
453
|
+
for (const query of queries) {
|
|
454
|
+
let results = [];
|
|
455
|
+
try {
|
|
456
|
+
results = (await search(query, maxResults, state.visitedUrls.slice())) || [];
|
|
457
|
+
} catch (_) {
|
|
458
|
+
results = [];
|
|
459
|
+
}
|
|
460
|
+
for (const result of results) {
|
|
461
|
+
absorbResult(result, state, nextQueries);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return nextQueries;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** Records one search result: its follow-up question, URL and learning. */
|
|
468
|
+
function absorbResult(result, state, nextQueries) {
|
|
469
|
+
const canonical = canonicalizeUrl(result && (result.url || result.link));
|
|
470
|
+
const text = result && (result.snippet || result.description || result.learning_text);
|
|
471
|
+
|
|
472
|
+
// A follow-up question steers the next level even when its page was already
|
|
473
|
+
// visited, so it is read before the dedupe guard below.
|
|
474
|
+
if (result && result.follow_up_question) {
|
|
475
|
+
const question = String(result.follow_up_question).trim();
|
|
476
|
+
if (question) {
|
|
477
|
+
nextQueries.push(question);
|
|
478
|
+
if (!state.followUpQuestions.includes(question)) state.followUpQuestions.push(question);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
if (canonical && state.visitedSet.has(canonical)) return;
|
|
483
|
+
if (canonical) {
|
|
484
|
+
state.visitedSet.add(canonical);
|
|
485
|
+
state.visitedUrls.push(canonical);
|
|
486
|
+
}
|
|
487
|
+
if (text && String(text).trim()) {
|
|
488
|
+
state.learnings.push({
|
|
489
|
+
learning_text: String(text).trim(),
|
|
490
|
+
canonical_url: canonical,
|
|
491
|
+
source_title: (result && (result.title || result.source_title)) || '',
|
|
492
|
+
citation_id: (result && result.citation_id) || ''
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
module.exports = {
|
|
498
|
+
DEFAULT_TOP_K,
|
|
499
|
+
DEFAULT_SEMANTIC_THRESHOLD,
|
|
500
|
+
canonicalizeUrl,
|
|
501
|
+
hashLearning,
|
|
502
|
+
retrieveRelevant,
|
|
503
|
+
storeLearnings,
|
|
504
|
+
runDeepResearchLoop,
|
|
505
|
+
float32ToBuffer,
|
|
506
|
+
bufferToFloat32,
|
|
507
|
+
centerVectors,
|
|
508
|
+
cosine
|
|
509
|
+
};
|
|
@@ -55,6 +55,9 @@ function dispatchRouter(state) {
|
|
|
55
55
|
function executeRouter(state) {
|
|
56
56
|
if (state.error || state.phase === 'error') return 'node_error';
|
|
57
57
|
if (state.budgetExceeded) return 'node_budget_exceeded';
|
|
58
|
+
// No child ran, so there is nothing to review: end the turn and let the client
|
|
59
|
+
// spawn the directive and report back. Resuming re-enters node_execute.
|
|
60
|
+
if (state.spawnPending) return '__end__';
|
|
58
61
|
if (state.phase === 'document') return 'node_dispatch';
|
|
59
62
|
return 'node_review';
|
|
60
63
|
}
|
|
@@ -7,6 +7,7 @@ const { KonohaCheckpointSaver } = require('./checkpointer');
|
|
|
7
7
|
const { getDb } = require('../../db');
|
|
8
8
|
const { normalizeTokenBudget } = require('./token_guard');
|
|
9
9
|
const { wrapGuardedNode } = require('./node_guard');
|
|
10
|
+
const { setSliceExecutor, getSliceExecutor, clearSliceExecutor } = require('./slice_executor_registry');
|
|
10
11
|
|
|
11
12
|
// macOS and Windows use case-insensitive filesystems, so a require('./nodes/Plan')
|
|
12
13
|
// typed against a plan.js on disk resolves locally but fails on Linux CI and on
|
|
@@ -118,6 +119,7 @@ function createGraphBuilder() {
|
|
|
118
119
|
node_budget_exceeded: 'node_budget_exceeded',
|
|
119
120
|
node_dispatch: 'node_dispatch',
|
|
120
121
|
node_review: 'node_review',
|
|
122
|
+
__end__: '__end__',
|
|
121
123
|
})
|
|
122
124
|
|
|
123
125
|
.addConditionalEdges('node_review', reviewRouter, {
|
|
@@ -272,7 +274,8 @@ async function runKonohaGraph(input, options = {}) {
|
|
|
272
274
|
|
|
273
275
|
const turnIndex = await resolveTurnIndex(graph, config, threadId);
|
|
274
276
|
|
|
275
|
-
|
|
277
|
+
// eslint-disable-next-line no-unused-vars
|
|
278
|
+
const { resumeNode: _stripResume, sliceExecutor: _stripExecutor, ...restInput } = input || {};
|
|
276
279
|
|
|
277
280
|
const invokeInput = {
|
|
278
281
|
...restInput,
|
|
@@ -288,8 +291,24 @@ async function runKonohaGraph(input, options = {}) {
|
|
|
288
291
|
invokeInput.tokenBudget = explicitBudget;
|
|
289
292
|
}
|
|
290
293
|
|
|
291
|
-
|
|
292
|
-
|
|
294
|
+
// A caller may hand the executor either beside the input or inside it; both are
|
|
295
|
+
// accepted because the state schema cannot carry a function through the
|
|
296
|
+
// checkpointer, so it never becomes a channel either way.
|
|
297
|
+
const executor = typeof options.sliceExecutor === 'function'
|
|
298
|
+
? options.sliceExecutor
|
|
299
|
+
: (typeof input?.sliceExecutor === 'function' ? input.sliceExecutor : null);
|
|
300
|
+
const hasExecutor = executor !== null;
|
|
301
|
+
const priorExecutor = getSliceExecutor(threadId);
|
|
302
|
+
if (hasExecutor) setSliceExecutor(threadId, executor);
|
|
303
|
+
try {
|
|
304
|
+
const rawResult = await graph.invoke(invokeInput, config);
|
|
305
|
+
return formatGraphOutput(rawResult);
|
|
306
|
+
} finally {
|
|
307
|
+
if (hasExecutor) {
|
|
308
|
+
if (priorExecutor) setSliceExecutor(threadId, priorExecutor);
|
|
309
|
+
else clearSliceExecutor(threadId);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
293
312
|
}
|
|
294
313
|
|
|
295
314
|
module.exports = {
|
|
@@ -68,6 +68,8 @@ async function dispatchNode(state) {
|
|
|
68
68
|
skills,
|
|
69
69
|
turn_budget: state.tokenBudget || 8192,
|
|
70
70
|
spawn_directive: spawnDirective,
|
|
71
|
+
// The MCP runner prepares a directive; it never starts the child.
|
|
72
|
+
spawn_state: 'directive_only',
|
|
71
73
|
advisor_steers: steerCtx.pending.length > 0 ? steerCtx.pending : undefined,
|
|
72
74
|
todo: {
|
|
73
75
|
id: d.task_id,
|
|
@@ -131,6 +133,7 @@ async function dispatchNode(state) {
|
|
|
131
133
|
prompt: taskPrompt,
|
|
132
134
|
worktree: parsed.worktree_path ? { worktree_path: parsed.worktree_path, branch: parsed.branch, base_sha: parsed.base_sha } : null
|
|
133
135
|
});
|
|
136
|
+
parsed.spawn_state = 'directive_only';
|
|
134
137
|
if (steerCtx.pending.length > 0) parsed.advisor_steers = steerCtx.pending;
|
|
135
138
|
if (status.advisor_supervisor) parsed.advisor_supervisor = status.advisor_supervisor;
|
|
136
139
|
if (parsed.task_id || parsed.task) {
|