@hadooppei/hwcode 1.0.16 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/dist/lib/knowledge/evidence.js +302 -0
- package/.pi/dist/lib/knowledge/extractor.js +53 -76
- package/.pi/dist/lib/knowledge/review-worker.js +97 -20
- package/.pi/dist/lib/knowledge/sanitize.js +8 -0
- package/.pi/dist/lib/knowledge/session-scanner.js +66 -13
- package/.pi/dist/lib/knowledge/store.js +215 -69
- package/.pi/dist/lib/runtime/defaults.js +6 -0
- package/.pi/extensions/knowledge.ts +90 -15
- package/.pi/extensions/workflows/cloud/provider-tools.ts +37 -7
- package/.pi/lib/knowledge/evidence.ts +314 -0
- package/.pi/lib/knowledge/extractor.ts +61 -55
- package/.pi/lib/knowledge/review-status.ts +18 -0
- package/.pi/lib/knowledge/review-worker.ts +94 -20
- package/.pi/lib/knowledge/sanitize.ts +10 -0
- package/.pi/lib/knowledge/session-scanner.ts +65 -12
- package/.pi/lib/knowledge/store.ts +220 -69
- package/.pi/lib/knowledge/types.ts +59 -2
- package/.pi/lib/knowledge/worker-protocol.ts +1 -1
- package/.pi/lib/runtime/defaults.ts +6 -0
- package/.pi/lib/workflows/cloud/process.ts +19 -0
- package/package.json +1 -1
|
@@ -11,10 +11,6 @@ const SAFE_GENERATION_RE = /^[0-9TZ-]+-[a-f0-9]{12}$/u;
|
|
|
11
11
|
const PROJECT_SCOPE_RE = /^project:[a-f0-9]{16}$/u;
|
|
12
12
|
const UNVERIFIED_CLAIM_RE = /(?:待验证|尚未(?:验证|确认)|未经(?:验证|确认)|未(?:经)?证实|推测|猜测|疑似|可能是|可能由于|可能导致|或许|unverified|unconfirmed|speculat(?:e|ive|ion)|hypothes(?:is|ize)|suspect(?:ed|ion)?|possibly|probably)/iu;
|
|
13
13
|
const VOLATILE_ARGUMENT_RE = /--(?:namespace|name|server_name|security_group_id|subnet_id|vpc_id|image_id|project_id)(?:\.\d+)?=(?!<[^>]+>|\$?\{)[^\s`"']+/iu;
|
|
14
|
-
const DISTINCTIVE_COMMON_TERMS = new Set([
|
|
15
|
-
"cloud", "huawei", "huaweicloud", "hcloud", "topic", "rule", "sop", "project", "region", "create",
|
|
16
|
-
"deploy", "deployment", "service", "workflow", "cn", "south",
|
|
17
|
-
]);
|
|
18
14
|
export class KnowledgeCommitBusyError extends Error {
|
|
19
15
|
}
|
|
20
16
|
function emptyCatalog() {
|
|
@@ -105,47 +101,8 @@ function normalizeSlug(value) {
|
|
|
105
101
|
.replace(/^-+|-+$/gu, "").slice(0, STORAGE.slugMaxChars) || "knowledge";
|
|
106
102
|
}
|
|
107
103
|
function normalizedFingerprint(candidate) {
|
|
108
|
-
|
|
109
|
-
}
|
|
110
|
-
function comparisonTerms(value) {
|
|
111
|
-
return new Set((value.normalize("NFKC").toLowerCase().match(/\p{Script=Han}+|[\p{L}\p{N}]+/gu) ?? [])
|
|
112
|
-
.filter((term) => term.length >= 2));
|
|
113
|
-
}
|
|
114
|
-
function containment(left, right) {
|
|
115
|
-
const minimum = Math.min(left.size, right.size);
|
|
116
|
-
if (minimum === 0)
|
|
117
|
-
return 0;
|
|
118
|
-
let shared = 0;
|
|
119
|
-
for (const term of left)
|
|
120
|
-
if (right.has(term))
|
|
121
|
-
shared++;
|
|
122
|
-
return shared / minimum;
|
|
123
|
-
}
|
|
124
|
-
function semanticDuplicateScore(candidate, entry) {
|
|
125
|
-
if (candidate.scope !== entry.scope || candidate.storageHint !== entry.track)
|
|
126
|
-
return 0;
|
|
127
|
-
const titleScore = containment(comparisonTerms(candidate.title), comparisonTerms(entry.title));
|
|
128
|
-
const keywordScore = containment(comparisonTerms(candidate.keywords.join(" ")), comparisonTerms(entry.keywords.join(" ")));
|
|
129
|
-
if (titleScore >= 0.8 && keywordScore >= 0.5)
|
|
130
|
-
return titleScore * 0.7 + keywordScore * 0.3;
|
|
131
|
-
const candidateTerms = comparisonTerms(`${candidate.title} ${candidate.keywords.join(" ")}`);
|
|
132
|
-
const entryTerms = comparisonTerms(`${entry.title} ${entry.keywords.join(" ")}`);
|
|
133
|
-
const distinctiveShared = [...candidateTerms].filter((term) => (entryTerms.has(term) && term.length >= 3 && !DISTINCTIVE_COMMON_TERMS.has(term))).length;
|
|
134
|
-
if (keywordScore < 0.5 || distinctiveShared < 3)
|
|
135
|
-
return 0;
|
|
136
|
-
return keywordScore * 0.7 + Math.min(distinctiveShared / 6, 1) * 0.3;
|
|
137
|
-
}
|
|
138
|
-
function semanticDuplicateIndex(candidate, catalog) {
|
|
139
|
-
let bestIndex = -1;
|
|
140
|
-
let bestScore = 0;
|
|
141
|
-
for (const [index, entry] of catalog.items.entries()) {
|
|
142
|
-
const score = semanticDuplicateScore(candidate, entry);
|
|
143
|
-
if (score > bestScore) {
|
|
144
|
-
bestIndex = index;
|
|
145
|
-
bestScore = score;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
return bestIndex;
|
|
104
|
+
const identity = candidate.identityKey?.trim().toLowerCase() || candidate.key.trim().toLowerCase();
|
|
105
|
+
return createHash("sha256").update(`${candidate.scope}\0${candidate.storageHint}\0${identity}`).digest("hex");
|
|
149
106
|
}
|
|
150
107
|
function unsupportedOperations(value) {
|
|
151
108
|
return [...value.matchAll(/\bOperation\s+([A-Za-z][A-Za-z0-9]+)\s+is not supported\b/gu)].map((match) => {
|
|
@@ -178,6 +135,71 @@ function asStringArray(value, maxItems) {
|
|
|
178
135
|
return [...new Set(value.filter((item) => typeof item === "string")
|
|
179
136
|
.map((item) => compactKnowledgeText(item, 240)).filter(Boolean))].slice(0, maxItems);
|
|
180
137
|
}
|
|
138
|
+
function asSafeReferenceArray(value, maxItems) {
|
|
139
|
+
if (!Array.isArray(value))
|
|
140
|
+
return [];
|
|
141
|
+
return [...new Set(value.filter((item) => (typeof item === "string" && /^[a-z0-9][a-z0-9-]{0,127}$/u.test(item))))].slice(0, maxItems);
|
|
142
|
+
}
|
|
143
|
+
function normalizedClaimKey(value) {
|
|
144
|
+
return compactKnowledgeText(value, 160).toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ".").replace(/^\.+|\.+$/gu, "");
|
|
145
|
+
}
|
|
146
|
+
function parseClaims(value) {
|
|
147
|
+
if (!Array.isArray(value))
|
|
148
|
+
return [];
|
|
149
|
+
const claims = [];
|
|
150
|
+
for (const raw of value.slice(0, 12)) {
|
|
151
|
+
if (!raw || typeof raw !== "object")
|
|
152
|
+
continue;
|
|
153
|
+
const item = raw;
|
|
154
|
+
if (typeof item.text !== "string")
|
|
155
|
+
continue;
|
|
156
|
+
const text = compactKnowledgeText(item.text, 500);
|
|
157
|
+
const key = typeof item.key === "string" ? normalizedClaimKey(item.key) : normalizedClaimKey(text);
|
|
158
|
+
const evidenceIds = asSafeReferenceArray(item.evidenceIds, 12);
|
|
159
|
+
if (key && text && evidenceIds.length > 0)
|
|
160
|
+
claims.push({ key, text, evidenceIds });
|
|
161
|
+
}
|
|
162
|
+
return claims;
|
|
163
|
+
}
|
|
164
|
+
function evidenceMap(task) {
|
|
165
|
+
return new Map((task.evidence ?? []).map((item) => [item.id, item]));
|
|
166
|
+
}
|
|
167
|
+
const UNIVERSAL_CLAIM_RE = /(?:\b(?:all|always|any|every)\b|全部|所有|任何|始终|均需)/iu;
|
|
168
|
+
const BROAD_REQUIRED_CLAIM_RE = /(?:\b(?:apis?|operations?|services?)\b|接口|操作|服务)[^\n]{0,80}(?:\b(?:must|require[sd]?)\b|必须)/iu;
|
|
169
|
+
const TECHNICAL_TOKEN_RE = /\p{Script=Han}{2,}|[a-z][a-z0-9_.\/-]{2,}/giu;
|
|
170
|
+
const CLAIM_COMMON_TERMS = new Set(["should", "must", "always", "before", "after", "使用", "必须", "应该", "需要", "首先"]);
|
|
171
|
+
function claimTerms(value) {
|
|
172
|
+
return new Set((value.toLowerCase().match(TECHNICAL_TOKEN_RE) ?? [])
|
|
173
|
+
.filter((term) => !CLAIM_COMMON_TERMS.has(term)));
|
|
174
|
+
}
|
|
175
|
+
function claimHasDirectSupport(claim, ledger) {
|
|
176
|
+
const terms = claimTerms(claim.text);
|
|
177
|
+
if (terms.size === 0)
|
|
178
|
+
return false;
|
|
179
|
+
for (const id of claim.evidenceIds) {
|
|
180
|
+
const evidence = ledger.get(id);
|
|
181
|
+
if (!evidence || evidence.kind === "assistant-claim")
|
|
182
|
+
continue;
|
|
183
|
+
const haystack = `${evidence.subject}\n${evidence.operation ?? ""}\n${evidence.excerpt}`.toLowerCase();
|
|
184
|
+
if ([...terms].some((term) => haystack.includes(term)))
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
function claimIsOverbroad(claim, ledger) {
|
|
190
|
+
if (!UNIVERSAL_CLAIM_RE.test(claim.text) && !BROAD_REQUIRED_CLAIM_RE.test(claim.text))
|
|
191
|
+
return false;
|
|
192
|
+
const operations = new Set(claim.evidenceIds.map((id) => ledger.get(id)?.operation).filter(Boolean));
|
|
193
|
+
return operations.size <= 1;
|
|
194
|
+
}
|
|
195
|
+
function claimContradictsSuccess(claim, task) {
|
|
196
|
+
if (!/(?:not supported|unsupported|不支持|不可用|必须.*\/v\d|must.*\/v\d)/iu.test(claim.text))
|
|
197
|
+
return false;
|
|
198
|
+
const claimLower = claim.text.toLowerCase();
|
|
199
|
+
return (task.evidence ?? []).some((item) => item.outcome === "success" && item.operation
|
|
200
|
+
&& claimLower.includes(item.operation.split(/\s+/u).at(-1).toLowerCase().replace(/\/v\d$/u, ""))
|
|
201
|
+
&& !item.operation.toLowerCase().match(/\/v\d$/u));
|
|
202
|
+
}
|
|
181
203
|
function withoutLeadingTitle(body) {
|
|
182
204
|
return body.replace(/^#\s+[^\n]*(?:\n+|$)/u, "").trim();
|
|
183
205
|
}
|
|
@@ -190,7 +212,7 @@ function containsVolatileDetail(body, projectRoot) {
|
|
|
190
212
|
|| namedProject
|
|
191
213
|
|| /\.hwcode\/cloud\/runs\//iu.test(body);
|
|
192
214
|
}
|
|
193
|
-
function normalizeCandidate(value,
|
|
215
|
+
function normalizeCandidate(value, task, validationText) {
|
|
194
216
|
if (!value || typeof value !== "object")
|
|
195
217
|
return { reason: "invalid-schema" };
|
|
196
218
|
const raw = value;
|
|
@@ -205,15 +227,50 @@ function normalizeCandidate(value, projectKey, projectRoot, validationText) {
|
|
|
205
227
|
&& (typeof raw.targetId !== "string" || !SAFE_ID_RE.test(raw.targetId)))
|
|
206
228
|
return { reason: "unknown-target" };
|
|
207
229
|
const explicitUserDirective = raw.explicitUserDirective === true;
|
|
208
|
-
const scope = raw.scope === "global" && explicitUserDirective ? "global" : `project:${projectKey}`;
|
|
230
|
+
const scope = raw.scope === "global" && explicitUserDirective ? "global" : `project:${task.projectKey}`;
|
|
209
231
|
const body = withoutLeadingTitle(compactKnowledgeText(raw.body, 20_000));
|
|
210
|
-
const
|
|
232
|
+
const ledger = evidenceMap(task);
|
|
233
|
+
const claims = parseClaims(raw.claims);
|
|
234
|
+
const episodeIds = asSafeReferenceArray(raw.episodeIds, 8);
|
|
235
|
+
const evidenceIds = [...new Set([
|
|
236
|
+
...asSafeReferenceArray(raw.evidenceIds, 16),
|
|
237
|
+
...claims.flatMap((claim) => claim.evidenceIds),
|
|
238
|
+
])];
|
|
239
|
+
let evidence = asStringArray(raw.evidence, 8);
|
|
240
|
+
if (ledger.size > 0) {
|
|
241
|
+
if (typeof raw.identityKey !== "string" || !raw.identityKey.trim())
|
|
242
|
+
return { reason: "invalid-schema" };
|
|
243
|
+
if (claims.length === 0 || evidenceIds.length === 0)
|
|
244
|
+
return { reason: "missing-body-or-evidence" };
|
|
245
|
+
if (evidenceIds.some((id) => !ledger.has(id)))
|
|
246
|
+
return { reason: "unknown-evidence" };
|
|
247
|
+
if (episodeIds.length === 0 || episodeIds.some((id) => !(task.episodes ?? []).some((episode) => episode.id === id))) {
|
|
248
|
+
return { reason: "invalid-schema" };
|
|
249
|
+
}
|
|
250
|
+
const episodeEvidenceIds = new Set((task.episodes ?? [])
|
|
251
|
+
.filter((episode) => episodeIds.includes(episode.id)).flatMap((episode) => episode.evidenceIds));
|
|
252
|
+
if (evidenceIds.some((id) => !episodeEvidenceIds.has(id)))
|
|
253
|
+
return { reason: "unknown-evidence" };
|
|
254
|
+
if (!explicitUserDirective && claims.some((claim) => !claimHasDirectSupport(claim, ledger))) {
|
|
255
|
+
return { reason: "unsupported-claim" };
|
|
256
|
+
}
|
|
257
|
+
if (!explicitUserDirective && claims.some((claim) => claimIsOverbroad(claim, ledger))) {
|
|
258
|
+
return { reason: "overbroad-claim" };
|
|
259
|
+
}
|
|
260
|
+
if (claims.some((claim) => claimContradictsSuccess(claim, task))) {
|
|
261
|
+
return { reason: "contradicts-verified-failure" };
|
|
262
|
+
}
|
|
263
|
+
evidence = evidenceIds.slice(0, 8).map((id) => {
|
|
264
|
+
const item = ledger.get(id);
|
|
265
|
+
return `${id}: ${compactKnowledgeText(`${item.subject}: ${item.excerpt}`, 220)}`;
|
|
266
|
+
});
|
|
267
|
+
}
|
|
211
268
|
if (!body || evidence.length === 0)
|
|
212
269
|
return { reason: "missing-body-or-evidence" };
|
|
213
270
|
const candidateClaims = `${raw.title}\n${raw.summary}\n${body}`;
|
|
214
271
|
if (!explicitUserDirective && UNVERIFIED_CLAIM_RE.test(candidateClaims))
|
|
215
272
|
return { reason: "unverified-claim" };
|
|
216
|
-
if (containsVolatileDetail(body, projectRoot))
|
|
273
|
+
if (containsVolatileDetail(body, task.projectRoot))
|
|
217
274
|
return { reason: "volatile-detail" };
|
|
218
275
|
if (contradictsVerifiedFailures(body, validationText))
|
|
219
276
|
return { reason: "contradicts-verified-failure" };
|
|
@@ -222,14 +279,18 @@ function normalizeCandidate(value, projectKey, projectRoot, validationText) {
|
|
|
222
279
|
&& body.split(/\n\s*\n/gu).length === 1
|
|
223
280
|
&& (explicitUserDirective || raw.confidence >= 0.9);
|
|
224
281
|
const targetId = typeof raw.targetId === "string" && SAFE_ID_RE.test(raw.targetId) ? raw.targetId : undefined;
|
|
282
|
+
const action = raw.action === "reinforce" || raw.action === "extend" || raw.action === "correct"
|
|
283
|
+
|| raw.action === "retire" ? raw.action : "add";
|
|
225
284
|
return { candidate: {
|
|
226
|
-
key: compactKnowledgeText(raw.key, 160),
|
|
285
|
+
key: compactKnowledgeText(raw.key, 160),
|
|
286
|
+
identityKey: typeof raw.identityKey === "string" ? compactKnowledgeText(raw.identityKey, 240).toLowerCase() : undefined,
|
|
287
|
+
targetId,
|
|
227
288
|
title: compactKnowledgeText(raw.title, STORAGE.maxTitleChars),
|
|
228
289
|
summary: compactKnowledgeSummary(raw.summary, STORAGE.maxSummaryChars),
|
|
229
290
|
keywords: asStringArray(raw.keywords, STORAGE.maxKeywordCount).map((keyword) => keyword.toLowerCase()),
|
|
230
|
-
scope, body, evidence, confidence: Math.min(1, raw.confidence),
|
|
291
|
+
scope, body, evidence, evidenceIds, episodeIds, claims, confidence: Math.min(1, raw.confidence),
|
|
231
292
|
storageHint: requestedTrack === "rule" && ruleEligible ? "rule" : "topic",
|
|
232
|
-
action
|
|
293
|
+
action,
|
|
233
294
|
explicitUserDirective, durability: "stable",
|
|
234
295
|
} };
|
|
235
296
|
}
|
|
@@ -258,6 +319,17 @@ function renderKnowledgeFile(candidate) {
|
|
|
258
319
|
candidate.body, "", "## Evidence", "", ...candidate.evidence.map((item) => `- ${item}`), "",
|
|
259
320
|
].join("\n");
|
|
260
321
|
}
|
|
322
|
+
function extendKnowledgeFile(content, candidate) {
|
|
323
|
+
const evidenceHeading = "\n## Evidence\n";
|
|
324
|
+
const index = content.indexOf(evidenceHeading);
|
|
325
|
+
const addition = candidate.body.trim();
|
|
326
|
+
const evidenceLines = candidate.evidence.map((item) => `- ${item}`).join("\n");
|
|
327
|
+
if (index < 0)
|
|
328
|
+
return `${content.trim()}\n\n${addition}\n\n## Evidence\n\n${evidenceLines}\n`;
|
|
329
|
+
const before = content.slice(0, index).trim();
|
|
330
|
+
const after = content.slice(index + evidenceHeading.length).trim();
|
|
331
|
+
return `${before}\n\n${addition}\n\n## Evidence\n\n${after}${after ? "\n" : ""}${evidenceLines}\n`;
|
|
332
|
+
}
|
|
261
333
|
function applicable(entry, projectKey) {
|
|
262
334
|
return entry.scope === "global" || entry.scope === `project:${projectKey}`;
|
|
263
335
|
}
|
|
@@ -324,14 +396,14 @@ function newGenerationId() {
|
|
|
324
396
|
return `${new Date().toISOString().replace(/[:.]/gu, "-")}-${randomUUID().replace(/-/gu, "").slice(0, 12)}`;
|
|
325
397
|
}
|
|
326
398
|
function copyCurrentContent(manifest, target, home) {
|
|
327
|
-
for (const name of ["rules", "topics", "pending"])
|
|
399
|
+
for (const name of ["rules", "topics", "pending", "audits"])
|
|
328
400
|
ensureDirectory(join(target, name));
|
|
329
401
|
if (!manifest.generationId)
|
|
330
402
|
return;
|
|
331
403
|
const source = generationDirectory(manifest.generationId, home);
|
|
332
404
|
if (!source)
|
|
333
405
|
return;
|
|
334
|
-
for (const name of ["rules", "topics", "pending"]) {
|
|
406
|
+
for (const name of ["rules", "topics", "pending", "audits"]) {
|
|
335
407
|
const sourceDirectory = join(source, name);
|
|
336
408
|
if (!existsSync(sourceDirectory))
|
|
337
409
|
continue;
|
|
@@ -362,7 +434,17 @@ function reviewCursor(task) {
|
|
|
362
434
|
lastReviewKey: task.reviewKey, fileSize: task.fileSize, fileMtimeMs: task.fileMtimeMs,
|
|
363
435
|
};
|
|
364
436
|
}
|
|
365
|
-
|
|
437
|
+
function validateEpisodeDecisions(task, decisions) {
|
|
438
|
+
const mandatory = (task.episodes ?? []).filter((episode) => (episode.score >= KNOWLEDGE_RUNTIME_DEFAULTS.review.mandatoryEpisodeScore));
|
|
439
|
+
for (const episode of mandatory) {
|
|
440
|
+
const decision = decisions.find((item) => item.episodeId === episode.id);
|
|
441
|
+
if (!decision || typeof decision.reason !== "string" || !decision.reason.trim()) {
|
|
442
|
+
throw new Error(`Knowledge reviewer omitted mandatory episode ${episode.id}`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
export function commitKnowledgeReview(values, task, leaderToken, home = homedir(), decisions = []) {
|
|
447
|
+
validateEpisodeDecisions(task, decisions);
|
|
366
448
|
ensureKnowledgeDirectories(home);
|
|
367
449
|
const release = acquireCommitLock(leaderToken, home);
|
|
368
450
|
const paths = userRuntimePaths(home);
|
|
@@ -380,7 +462,7 @@ export function commitKnowledgeReview(values, task, leaderToken, home = homedir(
|
|
|
380
462
|
const result = { saved: 0, updated: 0, pending: 0, skipped: 0 };
|
|
381
463
|
const rejected = [];
|
|
382
464
|
for (const value of values.slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxCandidates)) {
|
|
383
|
-
const normalized = normalizeCandidate(value, task
|
|
465
|
+
const normalized = normalizeCandidate(value, task, `${task.context ?? ""}\n${task.delta}`);
|
|
384
466
|
if (!("candidate" in normalized)) {
|
|
385
467
|
recordSkip(result, normalized.reason);
|
|
386
468
|
rejected.push({ reason: normalized.reason, value });
|
|
@@ -399,20 +481,73 @@ export function commitKnowledgeReview(values, task, leaderToken, home = homedir(
|
|
|
399
481
|
}
|
|
400
482
|
let existingIndex = targetIndex >= 0
|
|
401
483
|
? targetIndex
|
|
402
|
-
: catalog.items.findIndex((item) => item.fingerprint === fingerprint
|
|
403
|
-
|
|
404
|
-
if (existingIndex < 0) {
|
|
405
|
-
existingIndex = semanticDuplicateIndex(candidate, catalog);
|
|
406
|
-
semanticReinforcement = existingIndex >= 0;
|
|
407
|
-
}
|
|
484
|
+
: catalog.items.findIndex((item) => item.fingerprint === fingerprint
|
|
485
|
+
|| Boolean(candidate.identityKey && item.identityKey === candidate.identityKey && item.track === candidate.storageHint));
|
|
408
486
|
const existing = existingIndex >= 0 ? catalog.items[existingIndex] : undefined;
|
|
409
|
-
if (
|
|
487
|
+
if (candidate.action === "reinforce") {
|
|
488
|
+
const claimKeys = candidate.claims?.map((claim) => claim.key) ?? [];
|
|
489
|
+
const covered = claimKeys.length > 0 && claimKeys.every((key) => existing?.claimKeys?.includes(key));
|
|
490
|
+
if (!existing || (!covered && existing.contentHash !== hash)) {
|
|
491
|
+
recordSkip(result, "unsafe-reinforcement");
|
|
492
|
+
rejected.push({ reason: "unsafe-reinforcement", value });
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
410
495
|
existing.evidenceCount += candidate.evidence.length;
|
|
411
496
|
existing.updatedAt = new Date().toISOString();
|
|
412
497
|
result.updated++;
|
|
413
498
|
continue;
|
|
414
499
|
}
|
|
415
|
-
if (
|
|
500
|
+
if (candidate.action === "retire") {
|
|
501
|
+
if (!existing || !candidate.targetId || (!candidate.explicitUserDirective && candidate.confidence < 0.98)) {
|
|
502
|
+
recordSkip(result, "unsafe-reinforcement");
|
|
503
|
+
rejected.push({ reason: "unsafe-reinforcement", value });
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
const existingPath = resolve(temporary, existing.file);
|
|
507
|
+
if (existingPath.startsWith(`${temporary}${sep}`))
|
|
508
|
+
rmSync(existingPath, { force: true });
|
|
509
|
+
catalog.items.splice(existingIndex, 1);
|
|
510
|
+
result.updated++;
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
if (candidate.action === "extend") {
|
|
514
|
+
if (!existing || !candidate.targetId || existing.track !== "topic") {
|
|
515
|
+
recordSkip(result, "unknown-target");
|
|
516
|
+
rejected.push({ reason: "unknown-target", value });
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
const newClaimKeys = (candidate.claims ?? []).map((claim) => claim.key)
|
|
520
|
+
.filter((key) => !existing.claimKeys?.includes(key));
|
|
521
|
+
if (newClaimKeys.length === 0 && existing.claimKeys?.length) {
|
|
522
|
+
existing.evidenceCount += candidate.evidence.length;
|
|
523
|
+
existing.updatedAt = new Date().toISOString();
|
|
524
|
+
result.updated++;
|
|
525
|
+
continue;
|
|
526
|
+
}
|
|
527
|
+
const existingPath = resolve(temporary, existing.file);
|
|
528
|
+
if (!existingPath.startsWith(`${temporary}${sep}`) || !existsSync(existingPath)) {
|
|
529
|
+
recordSkip(result, "unknown-target");
|
|
530
|
+
rejected.push({ reason: "unknown-target", value });
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
const content = extendKnowledgeFile(readFileSync(existingPath, "utf8"), candidate);
|
|
534
|
+
atomicWrite(existingPath, content);
|
|
535
|
+
existing.contentHash = createHash("sha256").update(content).digest("hex");
|
|
536
|
+
existing.keywords = [...new Set([...existing.keywords, ...candidate.keywords])].slice(0, STORAGE.maxKeywordCount);
|
|
537
|
+
existing.claimKeys = [...new Set([...(existing.claimKeys ?? []), ...newClaimKeys])];
|
|
538
|
+
existing.evidenceCount += candidate.evidence.length;
|
|
539
|
+
existing.updatedAt = new Date().toISOString();
|
|
540
|
+
result.updated++;
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
if (existing && candidate.action !== "correct") {
|
|
544
|
+
const name = `${Date.now()}-${normalizeSlug(candidate.title)}-${randomUUID().slice(0, 8)}.json`;
|
|
545
|
+
atomicWrite(join(temporary, "pending", name), `${JSON.stringify({ reason: "conflicting-or-ambiguous-update", candidate }, null, 2)}\n`);
|
|
546
|
+
result.pending++;
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
if (existing && candidate.action === "correct"
|
|
550
|
+
&& (!candidate.targetId || (!candidate.explicitUserDirective && candidate.confidence < 0.92))) {
|
|
416
551
|
const name = `${Date.now()}-${normalizeSlug(candidate.title)}-${randomUUID().slice(0, 8)}.json`;
|
|
417
552
|
atomicWrite(join(temporary, "pending", name), `${JSON.stringify({ reason: "conflicting-or-ambiguous-update", candidate }, null, 2)}\n`);
|
|
418
553
|
result.pending++;
|
|
@@ -433,6 +568,8 @@ export function commitKnowledgeReview(values, task, leaderToken, home = homedir(
|
|
|
433
568
|
title: candidate.title, summary: candidate.summary,
|
|
434
569
|
keywords: candidate.keywords, scope: existing?.scope ?? candidate.scope, track, file: relativeFile,
|
|
435
570
|
evidenceCount: (existing?.evidenceCount ?? 0) + candidate.evidence.length,
|
|
571
|
+
identityKey: candidate.identityKey ?? existing?.identityKey,
|
|
572
|
+
claimKeys: candidate.claims?.map((claim) => claim.key) ?? existing?.claimKeys,
|
|
436
573
|
createdAt: existing?.createdAt ?? now, updatedAt: now,
|
|
437
574
|
};
|
|
438
575
|
if (existingIndex >= 0) {
|
|
@@ -444,17 +581,26 @@ export function commitKnowledgeReview(values, task, leaderToken, home = homedir(
|
|
|
444
581
|
result.saved++;
|
|
445
582
|
}
|
|
446
583
|
}
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
result.pending += rejected.length;
|
|
451
|
-
}
|
|
584
|
+
for (const item of rejected)
|
|
585
|
+
writeRejectedPending(temporary, item.reason, item.value);
|
|
586
|
+
result.pending += rejected.length;
|
|
452
587
|
catalog.updatedAt = new Date().toISOString();
|
|
453
588
|
const manifest = {
|
|
454
589
|
version: 3, generationId, createdAt: new Date().toISOString(), leaderToken, catalog,
|
|
455
590
|
reviews: { ...current.reviews, [task.sessionKey]: reviewCursor(task) },
|
|
456
591
|
};
|
|
457
592
|
atomicWrite(join(temporary, "MEMORY.md"), generateMemory(catalog));
|
|
593
|
+
atomicWrite(join(temporary, "audits", `${task.reviewKey}.json`), `${JSON.stringify({
|
|
594
|
+
reviewKey: task.reviewKey,
|
|
595
|
+
sessionKey: task.sessionKey,
|
|
596
|
+
episodes: (task.episodes ?? []).map((episode) => ({
|
|
597
|
+
id: episode.id, title: episode.title, score: episode.score,
|
|
598
|
+
failureCount: episode.failureCount, successCount: episode.successCount,
|
|
599
|
+
})),
|
|
600
|
+
decisions,
|
|
601
|
+
result,
|
|
602
|
+
rejected: rejected.map((item) => ({ reason: item.reason })),
|
|
603
|
+
}, null, 2)}\n`);
|
|
458
604
|
atomicWrite(join(temporary, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
|
459
605
|
if (!validManifestAt(manifest, temporary))
|
|
460
606
|
throw new Error("Refusing to publish an incomplete knowledge generation");
|
|
@@ -31,8 +31,14 @@ export const KNOWLEDGE_RUNTIME_DEFAULTS = Object.freeze({
|
|
|
31
31
|
idleMs: 60_000,
|
|
32
32
|
capabilityPollMs: 5_000,
|
|
33
33
|
modelTimeoutMs: 180_000,
|
|
34
|
+
maxAttempts: 2,
|
|
35
|
+
maxFailureDetailChars: 4_000,
|
|
34
36
|
maxDeltaChars: 12_000,
|
|
35
37
|
maxPriorContextChars: 4_000,
|
|
38
|
+
maxNarrativeChars: 6_000,
|
|
39
|
+
maxEvidenceItems: 24,
|
|
40
|
+
maxEvidenceChars: 12_000,
|
|
41
|
+
mandatoryEpisodeScore: 45,
|
|
36
42
|
maxExistingContextItems: 5,
|
|
37
43
|
maxExistingBodyItems: 2,
|
|
38
44
|
maxExistingBodyChars: 600,
|
|
@@ -10,8 +10,9 @@ import { defineTool, type ExtensionAPI, type ExtensionContext } from "@earendil-
|
|
|
10
10
|
import {
|
|
11
11
|
buildKnowledgeExtractionPrompt, type ExistingKnowledgeContext, KNOWLEDGE_EXTRACTION_SYSTEM_PROMPT,
|
|
12
12
|
} from "../lib/knowledge/extractor.ts";
|
|
13
|
-
import { matchKnowledge
|
|
13
|
+
import { matchKnowledge } from "../lib/knowledge/matcher.ts";
|
|
14
14
|
import { loadKnowledgeById, loadKnowledgeSnapshot, projectKnowledgeKey } from "../lib/knowledge/store.ts";
|
|
15
|
+
import type { KnowledgeEpisode, KnowledgeEvidence, KnowledgeReviewTask } from "../lib/knowledge/types.ts";
|
|
15
16
|
import type { KnowledgeWorkerInput, KnowledgeWorkerOutput } from "../lib/knowledge/worker-protocol.ts";
|
|
16
17
|
import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../lib/runtime/defaults.ts";
|
|
17
18
|
import { getSessionProjectRoot } from "../lib/working-directory.ts";
|
|
@@ -21,6 +22,7 @@ interface ActiveReview {
|
|
|
21
22
|
leaderToken: string;
|
|
22
23
|
controller: AbortController;
|
|
23
24
|
abortReason?: string;
|
|
25
|
+
diagnostic?: string;
|
|
24
26
|
}
|
|
25
27
|
|
|
26
28
|
interface ProcessKnowledgeRuntime {
|
|
@@ -42,6 +44,44 @@ runtime.workerRetryAttempt ??= 0;
|
|
|
42
44
|
|
|
43
45
|
const WORKER_RETRY_DELAYS_MS = [1_000, 5_000, 30_000, 60_000] as const;
|
|
44
46
|
|
|
47
|
+
function compactReviewEvidence(task: KnowledgeReviewTask): {
|
|
48
|
+
evidence: KnowledgeEvidence[];
|
|
49
|
+
episodes: KnowledgeEpisode[];
|
|
50
|
+
} {
|
|
51
|
+
const ledger = new Map((task.evidence ?? []).map((item) => [item.id, item]));
|
|
52
|
+
const selected: KnowledgeEvidence[] = [];
|
|
53
|
+
const seen = new Set<string>();
|
|
54
|
+
let characters = 0;
|
|
55
|
+
const add = (item: KnowledgeEvidence | undefined): void => {
|
|
56
|
+
if (!item || seen.has(item.id) || selected.length >= KNOWLEDGE_RUNTIME_DEFAULTS.review.maxEvidenceItems) return;
|
|
57
|
+
const compact = { ...item, excerpt: item.excerpt.slice(0, 500) };
|
|
58
|
+
const size = JSON.stringify(compact).length;
|
|
59
|
+
if (characters + size > KNOWLEDGE_RUNTIME_DEFAULTS.review.maxEvidenceChars) return;
|
|
60
|
+
selected.push(compact);
|
|
61
|
+
seen.add(item.id);
|
|
62
|
+
characters += size;
|
|
63
|
+
};
|
|
64
|
+
for (const item of task.evidence ?? []) if (item.kind === "user-correction") add(item);
|
|
65
|
+
(task.episodes ?? []).forEach((episode, index) => {
|
|
66
|
+
const items = episode.evidenceIds.map((id) => ledger.get(id)).filter((item): item is KnowledgeEvidence => Boolean(item));
|
|
67
|
+
const quota = index === 0 ? 10 : index < 3 ? 4 : 2;
|
|
68
|
+
const representative = [
|
|
69
|
+
...items.filter((item) => item.outcome === "failure").slice(-Math.max(1, Math.floor(quota * 0.4))),
|
|
70
|
+
...items.filter((item) => item.kind === "assistant-claim").slice(-Math.max(1, Math.floor(quota * 0.2))),
|
|
71
|
+
...items.filter((item) => item.kind === "verification").slice(-Math.max(1, Math.floor(quota * 0.3))),
|
|
72
|
+
...items.filter((item) => item.outcome === "success").slice(-quota),
|
|
73
|
+
];
|
|
74
|
+
for (const item of representative.slice(0, quota)) add(item);
|
|
75
|
+
});
|
|
76
|
+
const selectedIds = new Set(selected.map((item) => item.id));
|
|
77
|
+
return {
|
|
78
|
+
evidence: selected,
|
|
79
|
+
episodes: (task.episodes ?? []).map((episode) => ({
|
|
80
|
+
...episode, evidenceIds: episode.evidenceIds.filter((id) => selectedIds.has(id)),
|
|
81
|
+
})),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
45
85
|
function post(message: KnowledgeWorkerInput): void {
|
|
46
86
|
runtime.worker?.postMessage(message);
|
|
47
87
|
}
|
|
@@ -78,7 +118,7 @@ function reportWorkerIssue(error: unknown): void {
|
|
|
78
118
|
const prior = runtime.lastWorkerIssue;
|
|
79
119
|
if (prior?.message === message && now - prior.reportedAt < 60_000) return;
|
|
80
120
|
runtime.lastWorkerIssue = { message, reportedAt: now };
|
|
81
|
-
const rendered = `HWCode knowledge worker failed: ${message}`;
|
|
121
|
+
const rendered = `HWCode knowledge worker failed: ${message.slice(0, 600)}`;
|
|
82
122
|
if (runtime.context?.hasUI) runtime.context.ui.notify(rendered, "error");
|
|
83
123
|
else console.error(rendered);
|
|
84
124
|
}
|
|
@@ -123,7 +163,7 @@ async function runReview(message: Extract<KnowledgeWorkerOutput, { type: "review
|
|
|
123
163
|
runtime.activeReview = review;
|
|
124
164
|
timeout = setTimeout(() => {
|
|
125
165
|
if (controller.signal.aborted) return;
|
|
126
|
-
review!.abortReason =
|
|
166
|
+
review!.abortReason = `knowledge-review-model-timeout after ${KNOWLEDGE_RUNTIME_DEFAULTS.review.modelTimeoutMs}ms${review!.diagnostic ? `; ${review!.diagnostic}` : ""}`;
|
|
127
167
|
controller.abort();
|
|
128
168
|
}, KNOWLEDGE_RUNTIME_DEFAULTS.review.modelTimeoutMs);
|
|
129
169
|
timeout.unref();
|
|
@@ -134,24 +174,60 @@ async function runReview(message: Extract<KnowledgeWorkerOutput, { type: "review
|
|
|
134
174
|
entry.scope === "global" || entry.scope === `project:${message.task.projectKey}`
|
|
135
175
|
)),
|
|
136
176
|
};
|
|
137
|
-
const
|
|
177
|
+
const relatedEntries = [] as typeof applicableCatalog.items;
|
|
178
|
+
const seenRelated = new Set<string>();
|
|
179
|
+
const addRelated = (entry: typeof applicableCatalog.items[number] | undefined): void => {
|
|
180
|
+
if (!entry || seenRelated.has(entry.id)
|
|
181
|
+
|| relatedEntries.length >= KNOWLEDGE_RUNTIME_DEFAULTS.review.maxExistingContextItems) return;
|
|
182
|
+
relatedEntries.push(entry);
|
|
183
|
+
seenRelated.add(entry.id);
|
|
184
|
+
};
|
|
185
|
+
const episodeMatches = (message.task.episodes ?? []).map((episode) => (
|
|
186
|
+
matchKnowledge(`${episode.title}\n${episode.query}`, applicableCatalog, 2)
|
|
187
|
+
));
|
|
188
|
+
for (const matches of episodeMatches.slice(0, 3)) addRelated(matches[0]);
|
|
189
|
+
for (const id of message.task.loadedKnowledgeIds ?? []) {
|
|
190
|
+
addRelated(applicableCatalog.items.find((entry) => entry.id === id));
|
|
191
|
+
}
|
|
192
|
+
for (const matches of episodeMatches) for (const entry of matches.slice(1)) addRelated(entry);
|
|
193
|
+
for (const entry of matchKnowledge(
|
|
138
194
|
message.task.recallQuery || message.task.delta,
|
|
139
195
|
applicableCatalog,
|
|
140
|
-
message.task.loadedKnowledgeIds ?? [],
|
|
141
196
|
KNOWLEDGE_RUNTIME_DEFAULTS.review.maxExistingContextItems,
|
|
142
|
-
)
|
|
197
|
+
)) addRelated(entry);
|
|
198
|
+
const promptContext = compactReviewEvidence(message.task);
|
|
199
|
+
const related: ExistingKnowledgeContext[] = relatedEntries.map((entry, index) => {
|
|
143
200
|
const reference: ExistingKnowledgeContext = {
|
|
144
201
|
id: entry.id,
|
|
145
202
|
title: entry.title,
|
|
146
203
|
summary: entry.summary,
|
|
147
204
|
keywords: entry.keywords,
|
|
148
205
|
track: entry.track,
|
|
206
|
+
identityKey: entry.identityKey,
|
|
207
|
+
claimKeys: entry.claimKeys,
|
|
149
208
|
};
|
|
150
209
|
if (index >= KNOWLEDGE_RUNTIME_DEFAULTS.review.maxExistingBodyItems) return reference;
|
|
151
210
|
const content = loadKnowledgeById(entry.id, message.task.projectKey)?.content
|
|
152
211
|
.slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxExistingBodyChars);
|
|
153
212
|
return content ? { ...reference, content } : reference;
|
|
154
213
|
});
|
|
214
|
+
const prompt = buildKnowledgeExtractionPrompt(
|
|
215
|
+
message.task.projectRoot,
|
|
216
|
+
message.task.delta.slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxNarrativeChars),
|
|
217
|
+
message.task.context,
|
|
218
|
+
related,
|
|
219
|
+
promptContext.evidence,
|
|
220
|
+
promptContext.episodes,
|
|
221
|
+
);
|
|
222
|
+
review.diagnostic = [
|
|
223
|
+
`reviewKey=${message.task.reviewKey}`,
|
|
224
|
+
`model=${ctx.model.provider}/${ctx.model.id}`,
|
|
225
|
+
`promptChars=${prompt.length}`,
|
|
226
|
+
`deltaChars=${message.task.delta.length}`,
|
|
227
|
+
`evidenceItems=${promptContext.evidence.length}`,
|
|
228
|
+
`episodeItems=${promptContext.episodes.length}`,
|
|
229
|
+
`existingItems=${related.length}`,
|
|
230
|
+
].join("; ");
|
|
155
231
|
const response = await ctx.modelRegistry.complete(
|
|
156
232
|
ctx.model,
|
|
157
233
|
{
|
|
@@ -160,12 +236,7 @@ async function runReview(message: Extract<KnowledgeWorkerOutput, { type: "review
|
|
|
160
236
|
role: "user",
|
|
161
237
|
content: [{
|
|
162
238
|
type: "text",
|
|
163
|
-
text:
|
|
164
|
-
message.task.projectRoot,
|
|
165
|
-
message.task.delta,
|
|
166
|
-
message.task.context,
|
|
167
|
-
related,
|
|
168
|
-
),
|
|
239
|
+
text: prompt,
|
|
169
240
|
}],
|
|
170
241
|
timestamp: Date.now(),
|
|
171
242
|
}],
|
|
@@ -177,10 +248,14 @@ async function runReview(message: Extract<KnowledgeWorkerOutput, { type: "review
|
|
|
177
248
|
.map((item) => item.text).join("\n");
|
|
178
249
|
post({ type: "review_result", leaderToken: message.leaderToken, requestId: message.requestId, raw });
|
|
179
250
|
} catch (error) {
|
|
180
|
-
const
|
|
251
|
+
const baseReason = review?.controller.signal.aborted
|
|
181
252
|
? review.abortReason ?? "knowledge-review-interrupted"
|
|
182
253
|
: error instanceof Error ? error.message : String(error);
|
|
183
|
-
|
|
254
|
+
const expectedInterruption = baseReason === "model-executor-is-busy" || isExpectedReviewInterruption(baseReason);
|
|
255
|
+
const reason = review?.diagnostic && !expectedInterruption && !baseReason.includes("reviewKey=")
|
|
256
|
+
? `${baseReason}; ${review.diagnostic}`
|
|
257
|
+
: baseReason;
|
|
258
|
+
if (expectedInterruption) {
|
|
184
259
|
post({ type: "review_deferred", leaderToken: message.leaderToken, requestId: message.requestId, reason });
|
|
185
260
|
} else {
|
|
186
261
|
post({ type: "review_result", leaderToken: message.leaderToken, requestId: message.requestId, error: reason });
|
|
@@ -203,7 +278,7 @@ function handleWorkerMessage(message: KnowledgeWorkerOutput): void {
|
|
|
203
278
|
cancelActiveReview(message.reason);
|
|
204
279
|
return;
|
|
205
280
|
}
|
|
206
|
-
if (message.type === "review_failed") reportWorkerIssue(message.error);
|
|
281
|
+
if (message.type === "review_failed" && message.terminal !== false) reportWorkerIssue(message.error);
|
|
207
282
|
}
|
|
208
283
|
|
|
209
284
|
function ensureWorker(): Worker | undefined {
|