@hadooppei/hwcode 1.0.16 → 1.0.20

package/.pi/lib/knowledge/review-worker.ts

@@ -6,8 +6,8 @@ import { parentPort, workerData } from "node:worker_threads";
 
 import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.ts";
 import { userRuntimePaths } from "../runtime/paths.ts";
-import { parseCandidateEnvelope } from "./extractor.ts";
-import { type KnowledgeReviewStatus, writeKnowledgeReviewStatus } from "./review-status.ts";
+import { parseKnowledgeReviewEnvelope } from "./extractor.ts";
+import { readKnowledgeReviewStatus, type KnowledgeReviewStatus, writeKnowledgeReviewStatus } from "./review-status.ts";
 import { sanitizeKnowledgeText } from "./sanitize.ts";
 import { findNextReviewTask } from "./session-scanner.ts";
 import { commitKnowledgeReview, ensureKnowledgeDirectories, loadCurrentManifest } from "./store.ts";
@@ -59,21 +59,44 @@ function publishReviewStatus(): void {
 }
 
 function boundedError(error: unknown): string {
-  return sanitizeKnowledgeText(error instanceof Error ? error.message : String(error)).slice(0, 1_000);
+  return sanitizeKnowledgeText(error instanceof Error ? error.message : String(error))
+    .slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxFailureDetailChars);
+}
+
+function clearSupersededReviewState(task: KnowledgeReviewTask): void {
+  if (!reviewStatus) return;
+  for (const field of ["blockedReviews", "reviewFailures"] as const) {
+    const records = reviewStatus[field];
+    if (!records) continue;
+    for (const [reviewKey, record] of Object.entries(records)) {
+      if (record.sessionKey === task.sessionKey && reviewKey !== task.reviewKey) {
+        delete records[reviewKey];
+        reviewAttempts.delete(reviewKey);
+      }
+    }
+    if (Object.keys(records).length === 0) reviewStatus[field] = undefined;
+  }
 }
 
 function beginReview(task: KnowledgeReviewTask): void {
+  clearSupersededReviewState(task);
   activeAttempt = (reviewAttempts.get(task.reviewKey) ?? 0) + 1;
   reviewAttempts.set(task.reviewKey, activeAttempt);
   activeStartedAt = Date.now();
   clearTimeout(activeDeadlineTimer);
   activeDeadlineTimer = setTimeout(() => {
     if (activeTask?.requestId !== task.requestId) return;
-    const error = "knowledge-review-result-timeout";
-    finishReview(task, "failed", { error });
+    const error = [
+      `knowledge-review-result-timeout after ${KNOWLEDGE_RUNTIME_DEFAULTS.review.modelTimeoutMs + 5_000}ms`,
+      `reviewKey=${task.reviewKey}`,
+      `deltaChars=${task.delta.length}`,
+      `evidenceItems=${task.evidence?.length ?? 0}`,
+      `episodeItems=${task.episodes?.length ?? 0}`,
+    ].join("; ");
+    const terminal = finishReview(task, "failed", { error });
     activeTask = undefined;
     send({ type: "review_cancel", requestId: task.requestId, reason: error });
-    send({ type: "review_failed", requestId: task.requestId, error });
+    send({ type: "review_failed", requestId: task.requestId, error, terminal });
   }, KNOWLEDGE_RUNTIME_DEFAULTS.review.modelTimeoutMs + 5_000);
   activeDeadlineTimer.unref();
   if (reviewStatus) {
@@ -93,24 +116,59 @@ function finishReview(
   task: KnowledgeReviewTask,
   outcome: "saved" | "failed",
   detail: { generationId?: string; result?: PersistKnowledgeResult; error?: unknown },
-): void {
+): boolean {
   clearTimeout(activeDeadlineTimer);
   activeDeadlineTimer = undefined;
-  if (!reviewStatus) return;
+  if (!reviewStatus) return false;
+  const error = detail.error === undefined ? undefined : boundedError(detail.error);
+  const exhausted = outcome === "failed" && activeAttempt >= KNOWLEDGE_RUNTIME_DEFAULTS.review.maxAttempts;
+  const completedAt = new Date().toISOString();
+  const durationMs = Math.max(0, Date.now() - activeStartedAt);
   reviewStatus.activeReview = undefined;
   reviewStatus.lastReview = {
     reviewKey: task.reviewKey,
     sessionKey: task.sessionKey,
     attempt: activeAttempt,
     outcome,
-    completedAt: new Date().toISOString(),
-    durationMs: Math.max(0, Date.now() - activeStartedAt),
+    completedAt,
+    durationMs,
     generationId: detail.generationId,
     result: detail.result,
-    error: detail.error === undefined ? undefined : boundedError(detail.error),
+    error,
   };
-  if (outcome === "saved") reviewAttempts.delete(task.reviewKey);
+  if (outcome === "saved") {
+    reviewAttempts.delete(task.reviewKey);
+    delete reviewStatus.reviewFailures?.[task.reviewKey];
+    clearSupersededReviewState(task);
+  } else {
+    const reviewFailures = reviewStatus.reviewFailures ?? {};
+    const prior = reviewFailures[task.reviewKey];
+    reviewFailures[task.reviewKey] = {
+      sessionKey: task.sessionKey,
+      sessionFileHash: task.sessionFileHash,
+      attempts: [...(prior?.attempts ?? []), {
+        attempt: activeAttempt,
+        failedAt: completedAt,
+        durationMs,
+        error: error ?? "unknown-review-failure",
+      }].slice(-KNOWLEDGE_RUNTIME_DEFAULTS.review.maxAttempts),
+    };
+    reviewStatus.reviewFailures = Object.fromEntries(Object.entries(reviewFailures).slice(-100));
+  }
+  if (exhausted) {
+    const blockedReviews = reviewStatus.blockedReviews ?? {};
+    blockedReviews[task.reviewKey] = {
+      reviewKey: task.reviewKey,
+      sessionKey: task.sessionKey,
+      sessionFileHash: task.sessionFileHash,
+      blockedAt: new Date().toISOString(),
+      attempts: activeAttempt,
+      error: error ?? "unknown-review-failure",
+    };
+    reviewStatus.blockedReviews = Object.fromEntries(Object.entries(blockedReviews).slice(-100));
+  }
   publishReviewStatus();
+  return exhausted;
 }
 
 function deferReview(task: KnowledgeReviewTask, reason: string): void {
@@ -155,7 +213,12 @@ function scheduleElection(delay = Math.floor(Math.random() * 500)): void {
 }
 
 function stepDown(reason: string): void {
-  if (activeTask) send({ type: "review_cancel", requestId: activeTask.requestId, reason });
+  if (activeTask) {
+    const attempts = reviewAttempts.get(activeTask.reviewKey) ?? 1;
+    if (attempts <= 1) reviewAttempts.delete(activeTask.reviewKey);
+    else reviewAttempts.set(activeTask.reviewKey, attempts - 1);
+    send({ type: "review_cancel", requestId: activeTask.requestId, reason });
+  }
   if (reviewStatus) {
     reviewStatus.state = "stepped-down";
     reviewStatus.stepDownReason = reason;
@@ -260,8 +323,16 @@ function becomeLeader(server: Server): void {
     scheduleElection();
   });
   writeLeaderMetadata(electedAt);
+  const priorStatus = readKnowledgeReviewStatus(home);
+  reviewAttempts.clear();
+  for (const [reviewKey, failure] of Object.entries(priorStatus?.reviewFailures ?? {})) {
+    const attempt = Math.max(0, ...failure.attempts.map((item) => item.attempt));
+    if (attempt > 0) reviewAttempts.set(reviewKey, attempt);
+  }
   reviewStatus = {
     version: 1, protocolVersion: 3, state: "leader", pid: process.pid, leaderToken, electedAt, updatedAt: electedAt,
+    blockedReviews: priorStatus?.blockedReviews,
+    reviewFailures: priorStatus?.reviewFailures,
   };
   publishReviewStatus();
   send({ type: "leadership", state: "leader", leaderToken });
@@ -296,7 +367,8 @@ async function scanForReview(): Promise<void> {
   }
   try {
     const manifest = loadCurrentManifest(home);
-    const task = findNextReviewTask(sessionsRoot, manifest.reviews);
+    const blockedReviewKeys = new Set(Object.keys(reviewStatus?.blockedReviews ?? {}));
+    const task = findNextReviewTask(sessionsRoot, manifest.reviews, Date.now(), blockedReviewKeys);
     if (!task) return;
     activeTask = task;
     beginReview(task);
@@ -315,13 +387,14 @@ async function scanForReview(): Promise<void> {
     }
     send({ type: "review_request", leaderToken, requestId: task.requestId, task });
   } catch (error) {
-    if (activeTask) finishReview(activeTask, "failed", { error });
-    else if (reviewStatus) {
+    const failedTask = activeTask;
+    const terminal = failedTask ? finishReview(failedTask, "failed", { error }) : true;
+    if (!failedTask && reviewStatus) {
       reviewStatus.lastScanError = { at: new Date().toISOString(), error: boundedError(error) };
       publishReviewStatus();
     }
     activeTask = undefined;
-    send({ type: "review_failed", error: boundedError(error) });
+    send({ type: "review_failed", requestId: failedTask?.requestId, error: boundedError(error), terminal });
   }
 }
 
@@ -331,7 +404,8 @@ function handleReviewResult(message: Extract<KnowledgeWorkerInput, { type: "revi
   let committed = false;
   try {
     if (message.error || message.raw === undefined) throw new Error(message.error || "Knowledge review returned no content");
-    const result = commitKnowledgeReview(parseCandidateEnvelope(message.raw), task, leaderToken, home);
+    const envelope = parseKnowledgeReviewEnvelope(message.raw);
+    const result = commitKnowledgeReview(envelope.candidates, task, leaderToken, home, envelope.episodeDecisions);
     const generationId = loadCurrentManifest(home).generationId;
     finishReview(task, "saved", { generationId, result });
     send({ type: "review_saved", requestId: task.requestId, generationId, result });
@@ -341,8 +415,8 @@ function handleReviewResult(message: Extract<KnowledgeWorkerInput, { type: "revi
     }
     committed = true;
   } catch (error) {
-    finishReview(task, "failed", { error });
-    send({ type: "review_failed", requestId: task.requestId, error: boundedError(error) });
+    const terminal = finishReview(task, "failed", { error });
+    send({ type: "review_failed", requestId: task.requestId, error: boundedError(error), terminal });
   } finally {
     activeTask = undefined;
     if (committed) queueMicrotask(() => void scanForReview());

package/.pi/lib/knowledge/sanitize.ts

@@ -3,6 +3,16 @@ const BEARER_RE = /\bBearer\s+[A-Za-z0-9._~+/=-]{12,}/giu;
 const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/gu;
 const SECRET_ASSIGNMENT_RE = /\b(access[_-]?key|secret(?:[_-]?(?:access|key))?|password|passwd|token|credential|client[_-]?secret)\b(\s*[:=]\s*)[^\s,;]+/giu;
 const URL_CREDENTIAL_RE = /(https?:\/\/)[^\s/@:]+:[^\s/@]+@/giu;
+const INJECTED_SKILL_BLOCK_RE = /<skill\b[^>]*>[\s\S]*?<\/skill>/giu;
+const INJECTED_SKILL_MARKER_RE = /<skill\b[^>]*>/iu;
+
+export function containsInjectedSkill(value: string): boolean {
+  return INJECTED_SKILL_MARKER_RE.test(value);
+}
+
+export function stripInjectedSkillBlocks(value: string): string {
+  return value.replace(INJECTED_SKILL_BLOCK_RE, "");
+}
 
 export function sanitizeKnowledgeText(value: string): string {
   return value

package/.pi/lib/knowledge/session-scanner.ts

@@ -8,8 +8,9 @@ import {
 
 import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.ts";
 import { canonicalizeDirectory } from "../working-directory.ts";
+import { buildKnowledgeEpisodes, buildKnowledgeEvidence } from "./evidence.ts";
 import { knowledgeDeltaDigest } from "./extractor.ts";
-import { sanitizeKnowledgeText } from "./sanitize.ts";
+import { containsInjectedSkill, sanitizeKnowledgeText, stripInjectedSkillBlocks } from "./sanitize.ts";
 import { projectKnowledgeKey } from "./store.ts";
 import type { KnowledgeReviewCursor, KnowledgeReviewTask } from "./types.ts";
 
@@ -21,7 +22,6 @@ interface ReviewPiece {
 }
 
 const FAILURE_EVIDENCE_RE = /(?:\b(?:error|failed|failure|invalid|unsupported|mistyp(?:e|ed)|typo)\b|not supported|失败|错误|不支持|无效)/iu;
-const INJECTED_SKILL_RE = /<skill\b[^>]*>[\s\S]*?<\/skill>/giu;
 const SAFE_KNOWLEDGE_ID_RE = /^[a-z0-9][a-z0-9-]{0,79}$/u;
 const MAX_RECALL_QUERY_CHARS = 4_000;
 const MAX_LOADED_KNOWLEDGE_IDS = 8;
@@ -53,8 +53,13 @@ function reviewPiece(entry: SessionEntry, index: number): ReviewPiece | undefine
   if (entry.type !== "message" || !entry.message || typeof entry.message !== "object") return undefined;
   const message = entry.message as unknown as Record<string, unknown>;
   const role = typeof message.role === "string" ? message.role : "message";
-  const content = sanitizeKnowledgeText(textContent(message.content).replace(INJECTED_SKILL_RE, "[loaded skill omitted]"));
+  const rawContent = textContent(message.content);
+  const injectedContext = role === "user" && containsInjectedSkill(rawContent);
+  const content = sanitizeKnowledgeText(injectedContext ? stripInjectedSkillBlocks(rawContent) : rawContent);
   if (!content) return undefined;
+  if (injectedContext) {
+    return { index, priority: 3, category: "assistant", text: `[injected_context]\n${content.slice(0, 4_000)}` };
+  }
   const priority = role === "user" ? 0 : role === "toolResult" && FAILURE_EVIDENCE_RE.test(content) ? 2
     : role === "toolResult" ? 4 : 3;
   const cap = role === "user" ? 8_000 : role === "toolResult" ? 1_500 : 4_000;
@@ -62,8 +67,39 @@ function reviewPiece(entry: SessionEntry, index: number): ReviewPiece | undefine
   return { index, priority, category, text: `[${role}]\n${content.slice(0, cap)}` };
 }
 
-function buildDelta(entries: SessionEntry[]): string {
-  const pieces = entries.map(reviewPiece).filter((piece): piece is ReviewPiece => Boolean(piece));
+function entryId(entry: SessionEntry): string | undefined {
+  const value = (entry as unknown as Record<string, unknown>).id;
+  return typeof value === "string" ? value : undefined;
+}
+
+function knowledgeLookupResultEntryIds(entries: SessionEntry[]): Set<string> {
+  const lookupCalls = new Set<string>();
+  const resultEntries = new Set<string>();
+  for (const entry of entries) {
+    const message = messageRecord(entry);
+    const content = message?.content;
+    if (Array.isArray(content)) {
+      for (const raw of content) {
+        if (!raw || typeof raw !== "object") continue;
+        const item = raw as Record<string, unknown>;
+        if (item.type === "toolCall" && item.name === "hwcode_knowledge_lookup" && typeof item.id === "string") {
+          lookupCalls.add(item.id);
+        }
+      }
+    }
+    if (message?.role === "toolResult" && typeof message.toolCallId === "string" && lookupCalls.has(message.toolCallId)) {
+      const id = entryId(entry);
+      if (id) resultEntries.add(id);
+    }
+  }
+  return resultEntries;
+}
+
+function buildDelta(entries: SessionEntry[], excludedEntryIds: ReadonlySet<string> = new Set()): string {
+  const pieces = entries.map((entry, index) => {
+    const id = entryId(entry);
+    return id && excludedEntryIds.has(id) ? undefined : reviewPiece(entry, index);
+  }).filter((piece): piece is ReviewPiece => Boolean(piece));
   const selected: ReviewPiece[] = [];
   const categoryRemaining: Record<ReviewPiece["category"], number> = {
     user: 14_000, workflow: 6_000, summary: 4_000, assistant: 8_000, tool: 4_000,
@@ -86,8 +122,11 @@ function buildDelta(entries: SessionEntry[]): string {
     .slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxDeltaChars);
 }
 
-function buildValidationContext(entries: SessionEntry[]): string {
-  const pieces = entries.map(reviewPiece).filter((piece): piece is ReviewPiece => Boolean(piece));
+function buildValidationContext(entries: SessionEntry[], excludedEntryIds: ReadonlySet<string> = new Set()): string {
+  const pieces = entries.map((entry, index) => {
+    const id = entryId(entry);
+    return id && excludedEntryIds.has(id) ? undefined : reviewPiece(entry, index);
+  }).filter((piece): piece is ReviewPiece => Boolean(piece));
   const latestWorkflow = pieces.filter((piece) => piece.category === "workflow").at(-1);
   const evidence = pieces.filter((piece) => (
     (piece.category === "tool" || piece.category === "assistant" || piece.category === "summary")
@@ -182,7 +221,9 @@ function buildRecallQuery(entries: SessionEntry[], ids: string[]): string {
   for (const entry of entries.slice().reverse()) {
     const message = messageRecord(entry);
     if (!message || message.role !== "user") continue;
-    const text = compactRecallValue(textContent(message.content).replace(INJECTED_SKILL_RE, ""), 800);
+    const raw = textContent(message.content);
+    if (containsInjectedSkill(raw)) continue;
+    const text = compactRecallValue(raw, 800);
     if (text) lines.push(`User request: ${text}`);
     if (lines.filter((line) => line.startsWith("User request:")).length >= 2) break;
   }
@@ -240,13 +281,23 @@ export function readReviewTask(
     : -1;
   const pending = branch.slice(cursorIndex + 1);
   if (pending.length === 0) return undefined;
-  const delta = buildDelta(pending);
+  const pendingEntryIds = new Set(pending.map((entry) => entry.id));
+  const evidenceWindow = cursorIndex >= 0 ? branch.slice(Math.max(0, cursorIndex - 40)) : pending;
+  const evidence = buildKnowledgeEvidence(evidenceWindow).filter((item) => pendingEntryIds.has(item.entryId));
+  const episodes = buildKnowledgeEpisodes(evidence);
+  const lookupResultEntryIds = knowledgeLookupResultEntryIds(branch);
+  const delta = buildDelta(pending, lookupResultEntryIds);
   const loadedIds = loadedKnowledgeIds(pending);
   const recallQuery = buildRecallQuery(pending, loadedIds);
-  const context = cursorIndex >= 0 ? buildValidationContext(branch.slice(0, cursorIndex + 1)) : "";
+  const context = cursorIndex >= 0
+    ? buildValidationContext(branch.slice(0, cursorIndex + 1), lookupResultEntryIds)
+    : "";
   const firstEntryId = pending[0].id;
   const lastEntryId = pending[pending.length - 1].id;
-  const deltaDigest = knowledgeDeltaDigest(`${delta || `${firstEntryId}\0${lastEntryId}`}\0${recallQuery}\0${loadedIds.join("\0")}`);
+  const deltaDigest = knowledgeDeltaDigest([
+    delta || `${firstEntryId}\0${lastEntryId}`, recallQuery, loadedIds.join("\0"),
+    JSON.stringify(evidence), JSON.stringify(episodes),
+  ].join("\0"));
   const sessionFileHash = hash(resolve(sessionFile), 16);
   const sessionKey = hash(header.id, 24);
   const reviewKey = hash(`${header.id}\0${firstEntryId}\0${lastEntryId}\0${deltaDigest}`);
@@ -255,6 +306,7 @@ export function readReviewTask(
     requestId: randomUUID(), reviewKey, sessionId: header.id, sessionKey, sessionFileHash,
     projectRoot, projectKey: projectKnowledgeKey(projectRoot),
     firstEntryId, lastEntryId, context, delta, recallQuery, loadedKnowledgeIds: loadedIds,
+    evidence, episodes,
     deltaDigest, fileSize: stat.size, fileMtimeMs: stat.mtimeMs,
   };
 }
@@ -263,6 +315,7 @@ export function findNextReviewTask(
   sessionsRoot: string,
   reviews: Record<string, KnowledgeReviewCursor>,
   now = Date.now(),
+  excludedReviewKeys: ReadonlySet<string> = new Set(),
 ): KnowledgeReviewTask | undefined {
   const files = discoverSessionFiles(sessionsRoot).map((path) => {
     try { return { path, mtimeMs: statSync(path).mtimeMs }; } catch { return undefined; }
@@ -274,7 +327,7 @@ export function findNextReviewTask(
     let task = readReviewTask(file.path, cursor, now);
     const stableCursor = task ? reviews[task.sessionKey] : undefined;
     if (task && stableCursor && stableCursor !== cursor) task = readReviewTask(file.path, stableCursor, now);
-    if (task) return task;
+    if (task && !excludedReviewKeys.has(task.reviewKey)) return task;
   }
   return undefined;
 }