@hadooppei/hwcode 1.0.8 → 1.0.9

package/.pi/lib/knowledge/review-worker.ts

@@ -1,64 +1,260 @@
-import { parentPort } from "node:worker_threads";
+import { randomUUID } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
+import { createConnection, createServer, type Server, type Socket } from "node:net";
+import { homedir } from "node:os";
+import { parentPort, workerData } from "node:worker_threads";
 
 import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.ts";
+import { userRuntimePaths } from "../runtime/paths.ts";
 import { parseCandidateEnvelope } from "./extractor.ts";
-import { persistKnowledgeCandidates } from "./store.ts";
-import type { KnowledgeWorkerInput, KnowledgeWorkerOutput } from "./worker-protocol.ts";
+import { findNextReviewTask } from "./session-scanner.ts";
+import { commitKnowledgeReview, ensureKnowledgeDirectories, loadCurrentManifest } from "./store.ts";
+import type { KnowledgeReviewTask } from "./types.ts";
+import type { CoordinatorBroadcast, KnowledgeWorkerInput, KnowledgeWorkerOutput } from "./worker-protocol.ts";
 
 if (!parentPort) throw new Error("Knowledge review worker requires a parent port");
 
-let sessionId = "";
-let projectKey = "";
-let dirty = false;
-let settled = false;
-let reviewing = false;
-let activeRequestId = "";
+const home = (workerData as { home?: string } | null)?.home ?? homedir();
+const paths = userRuntimePaths(home);
+const standbySockets = new Set<Socket>();
+let eligible = false;
+let stopped = false;
+let sessionsRoot = "";
+let leaderServer: Server | undefined;
+let leaderToken = "";
+let standbySocket: Socket | undefined;
+let electionPending = false;
+let activeTask: KnowledgeReviewTask | undefined;
 
 function send(message: KnowledgeWorkerOutput): void {
   parentPort!.postMessage(message);
 }
 
-function checkForReview(): void {
-  if (!sessionId || !projectKey || !dirty || !settled || reviewing) return;
-  reviewing = true;
-  activeRequestId = `${sessionId}-${Date.now()}`;
-  send({ type: "review_due", requestId: activeRequestId });
+function coordinatorPath(): string {
+  return process.platform === "win32"
+    ? `\\\\.\\pipe\\hwcode-knowledge-v3-${Buffer.from(home).toString("hex").slice(0, 16)}`
+    : paths.knowledgeCoordinatorSocket;
 }
 
-const timer = setInterval(checkForReview, KNOWLEDGE_RUNTIME_DEFAULTS.review.intervalMs);
-timer.unref();
+function writeLeaderMetadata(): void {
+  writeFileSync(paths.knowledgeLeader, `${JSON.stringify({
+    protocolVersion: 3, leaderToken, pid: process.pid, electedAt: new Date().toISOString(),
+  }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+  chmodSync(paths.knowledgeLeader, 0o600);
+}
 
-parentPort.on("message", (message: KnowledgeWorkerInput) => {
-  if (message.type === "configure") {
-    sessionId = message.sessionId;
-    projectKey = message.projectKey;
-    dirty = message.dirty;
-    settled = true;
-    reviewing = false;
-    activeRequestId = "";
-    return;
+function removeLeaderMetadata(token: string): void {
+  try {
+    const current = JSON.parse(readFileSync(paths.knowledgeLeader, "utf8")) as { leaderToken?: string };
+    if (current.leaderToken === token) rmSync(paths.knowledgeLeader, { force: true });
+  } catch { /* Another contender may already have replaced it. */ }
+}
+
+function broadcast(message: CoordinatorBroadcast): void {
+  const line = `${JSON.stringify(message)}\n`;
+  for (const socket of standbySockets) {
+    if (socket.writable) socket.write(line);
   }
-  if (message.type === "activity") {
-    settled = message.state === "settled";
-    if (message.dirty !== undefined) dirty = message.dirty;
+}
+
+function scheduleElection(delay = Math.floor(Math.random() * 500)): void {
+  if (!eligible || stopped || electionPending || leaderServer || standbySocket) return;
+  electionPending = true;
+  const timer = setTimeout(() => {
+    electionPending = false;
+    attemptElection();
+  }, delay);
+  timer.unref();
+}
+
+function stepDown(reason: string): void {
+  if (activeTask) send({ type: "review_cancel", requestId: activeTask.requestId, reason });
+  activeTask = undefined;
+  const token = leaderToken;
+  leaderToken = "";
+  for (const socket of standbySockets) socket.destroy();
+  standbySockets.clear();
+  const server = leaderServer;
+  leaderServer = undefined;
+  if (server) {
+    try { server.close(); } catch { /* The failing server may already be closed. */ }
+    if (process.platform !== "win32") {
+      try { unlinkSync(coordinatorPath()); } catch { /* Socket may already be gone. */ }
+    }
+  }
+  if (token) removeLeaderMetadata(token);
+}
+
+function becomeStandby(socket: Socket): void {
+  standbySocket = socket;
+  socket.unref();
+  send({ type: "leadership", state: "standby" });
+  let buffer = "";
+  socket.on("data", (chunk) => {
+    buffer += chunk.toString("utf8");
+    for (;;) {
+      const newline = buffer.indexOf("\n");
+      if (newline < 0) break;
+      const line = buffer.slice(0, newline);
+      buffer = buffer.slice(newline + 1);
+      try {
+        const message = JSON.parse(line) as CoordinatorBroadcast;
+        if (message.type === "generation_changed") send(message);
+      } catch { /* Ignore malformed coordinator notifications. */ }
+    }
+  });
+  socket.once("close", () => {
+    if (standbySocket === socket) standbySocket = undefined;
+    scheduleElection();
+  });
+}
+
+function cleanStaleSocketAndRetry(): void {
+  if (process.platform === "win32") { scheduleElection(250); return; }
+  try {
+    mkdirSync(paths.knowledgeElectionLock, { mode: 0o700 });
+  } catch {
+    scheduleElection(250);
     return;
   }
-  if (message.type === "review_result") {
-    if (!reviewing || message.requestId !== activeRequestId) return;
-    try {
-      if (message.error || message.raw === undefined) throw new Error(message.error || "Knowledge review returned no content");
-      const candidates = parseCandidateEnvelope(message.raw);
-      const result = persistKnowledgeCandidates(candidates, projectKey);
-      dirty = false;
-      send({ type: "review_saved", requestId: message.requestId, result });
-    } catch (error) {
-      send({ type: "review_failed", requestId: message.requestId, error: error instanceof Error ? error.message : String(error) });
-    } finally {
-      reviewing = false;
-      activeRequestId = "";
+  try {
+    if (existsSync(coordinatorPath())) unlinkSync(coordinatorPath());
+  } catch { /* Retrying listen will surface persistent failures. */ }
+  finally { rmSync(paths.knowledgeElectionLock, { recursive: true, force: true }); }
+  scheduleElection();
+}
+
+function connectToLeader(probe = 0): void {
+  if (!eligible || stopped || leaderServer || standbySocket) return;
+  const socket = createConnection(coordinatorPath());
+  let connected = false;
+  socket.once("connect", () => { connected = true; becomeStandby(socket); });
+  socket.once("error", (error: NodeJS.ErrnoException) => {
+    socket.destroy();
+    if (connected) return;
+    if (error.code === "ECONNREFUSED" || error.code === "ENOENT") {
+      if (probe < 2) {
+        const timer = setTimeout(() => connectToLeader(probe + 1), 100);
+        timer.unref();
+      } else cleanStaleSocketAndRetry();
+    } else {
+      send({ type: "review_failed", error: `Coordinator connection failed: ${error.message}` });
+      scheduleElection(1_000);
     }
+  });
+}
+
+function becomeLeader(server: Server): void {
+  leaderServer = server;
+  leaderToken = randomUUID();
+  server.unref();
+  server.on("connection", (socket) => {
+    standbySockets.add(socket);
+    socket.unref();
+    socket.once("close", () => standbySockets.delete(socket));
+    const generationId = loadCurrentManifest(home).generationId;
+    if (generationId) socket.write(`${JSON.stringify({ type: "generation_changed", generationId })}\n`);
+  });
+  server.on("error", (error) => {
+    send({ type: "review_failed", error: `Coordinator server failed: ${error.message}` });
+    stepDown("coordinator-server-failed");
+    scheduleElection();
+  });
+  writeLeaderMetadata();
+  send({ type: "leadership", state: "leader", leaderToken });
+  void scanForReview();
+}
+
+function attemptElection(): void {
+  if (!eligible || stopped || leaderServer || standbySocket) return;
+  ensureKnowledgeDirectories(home);
+  const server = createServer();
+  const electionError = (error: NodeJS.ErrnoException): void => {
+    if (error.code === "EADDRINUSE") connectToLeader();
+    else {
+      send({ type: "review_failed", error: `Coordinator election failed: ${error.message}` });
+      scheduleElection(1_000);
+    }
+  };
+  server.once("error", electionError);
+  server.listen(coordinatorPath(), () => {
+    server.off("error", electionError);
+    if (process.platform !== "win32") chmodSync(coordinatorPath(), 0o600);
+    becomeLeader(server);
+  });
+}
+
+async function scanForReview(): Promise<void> {
+  if (!leaderServer || !leaderToken || activeTask || !sessionsRoot || !eligible) return;
+  try {
+    const manifest = loadCurrentManifest(home);
+    const task = findNextReviewTask(sessionsRoot, manifest.reviews);
+    if (!task) return;
+    activeTask = task;
+    if (!task.delta) {
+      const result = commitKnowledgeReview([], task, leaderToken, home);
+      const generationId = loadCurrentManifest(home).generationId;
+      send({ type: "review_saved", requestId: task.requestId, generationId, result });
+      if (generationId) {
+        send({ type: "generation_changed", generationId });
+        broadcast({ type: "generation_changed", generationId });
+      }
+      activeTask = undefined;
+      queueMicrotask(() => void scanForReview());
+      return;
+    }
+    send({ type: "review_request", leaderToken, requestId: task.requestId, task });
+  } catch (error) {
+    activeTask = undefined;
+    send({ type: "review_failed", error: error instanceof Error ? error.message : String(error) });
+  }
+}
+
+function handleReviewResult(message: Extract<KnowledgeWorkerInput, { type: "review_result" }>): void {
+  const task = activeTask;
+  if (!task || message.requestId !== task.requestId || message.leaderToken !== leaderToken || !leaderServer) return;
+  let committed = false;
+  try {
+    if (message.error || message.raw === undefined) throw new Error(message.error || "Knowledge review returned no content");
+    const result = commitKnowledgeReview(parseCandidateEnvelope(message.raw), task, leaderToken, home);
+    const generationId = loadCurrentManifest(home).generationId;
+    send({ type: "review_saved", requestId: task.requestId, generationId, result });
+    if (generationId) {
+      send({ type: "generation_changed", generationId });
+      broadcast({ type: "generation_changed", generationId });
+    }
+    committed = true;
+  } catch (error) {
+    send({ type: "review_failed", requestId: task.requestId, error: error instanceof Error ? error.message : String(error) });
+  } finally {
+    activeTask = undefined;
+    if (committed) queueMicrotask(() => void scanForReview());
+  }
+}
+
+const scanTimer = setInterval(() => { void scanForReview(); }, KNOWLEDGE_RUNTIME_DEFAULTS.review.intervalMs);
+scanTimer.unref();
+
+parentPort.on("message", (message: KnowledgeWorkerInput) => {
+  if (message.type === "configure") {
+    sessionsRoot = message.sessionsRoot;
+    if (eligible === message.modelAvailable) return;
+    eligible = message.modelAvailable;
+    if (!eligible) {
+      standbySocket?.destroy();
+      standbySocket = undefined;
+      stepDown("model-unavailable");
+      send({ type: "leadership", state: "ineligible" });
+    } else scheduleElection(0);
     return;
   }
-  clearInterval(timer);
+  if (message.type === "review_result") { handleReviewResult(message); return; }
+  if (message.type === "scan_now") { void scanForReview(); return; }
+  stopped = true;
+  eligible = false;
+  clearInterval(scanTimer);
+  standbySocket?.destroy();
+  standbySocket = undefined;
+  stepDown("worker-stopped");
   process.exit(0);
 });

package/.pi/lib/knowledge/session-scanner.ts

@@ -0,0 +1,155 @@
+import { createHash, randomUUID } from "node:crypto";
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+
+import {
+  buildContextEntries, parseSessionEntries, type SessionEntry, type SessionHeader,
+} from "@earendil-works/pi-coding-agent";
+
+import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.ts";
+import { knowledgeDeltaDigest } from "./extractor.ts";
+import { sanitizeKnowledgeText } from "./sanitize.ts";
+import { projectKnowledgeKey } from "./store.ts";
+import type { KnowledgeReviewCursor, KnowledgeReviewTask } from "./types.ts";
+
+interface ReviewPiece {
+  index: number;
+  priority: number;
+  category: "user" | "workflow" | "summary" | "assistant" | "tool";
+  text: string;
+}
+
+function hash(value: string, length = 64): string {
+  return createHash("sha256").update(value).digest("hex").slice(0, length);
+}
+
+function textContent(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content.map((item) => {
+    if (!item || typeof item !== "object") return "";
+    const value = item as Record<string, unknown>;
+    return typeof value.text === "string" ? value.text : "";
+  }).filter(Boolean).join("\n");
+}
+
+function reviewPiece(entry: SessionEntry, index: number): ReviewPiece | undefined {
+  if (entry.type === "compaction" || entry.type === "branch_summary") {
+    return { index, priority: 1, category: "summary", text: `[summary]\n${sanitizeKnowledgeText(entry.summary).slice(0, 6_000)}` };
+  }
+  if (entry.type === "custom" && entry.customType.startsWith("hwcode-workflow")) {
+    return {
+      index, priority: 0, category: "workflow",
+      text: `[workflow_state:${entry.customType}]\n${sanitizeKnowledgeText(JSON.stringify(entry.data ?? {})).slice(0, 6_000)}`,
+    };
+  }
+  if (entry.type !== "message" || !entry.message || typeof entry.message !== "object") return undefined;
+  const message = entry.message as unknown as Record<string, unknown>;
+  const role = typeof message.role === "string" ? message.role : "message";
+  const content = sanitizeKnowledgeText(textContent(message.content));
+  if (!content) return undefined;
+  const priority = role === "user" ? 0 : role === "toolResult" ? 3 : 2;
+  const cap = role === "user" ? 8_000 : role === "toolResult" ? 1_500 : 4_000;
+  const category = role === "user" ? "user" : role === "toolResult" ? "tool" : "assistant";
+  return { index, priority, category, text: `[${role}]\n${content.slice(0, cap)}` };
+}
+
+function buildDelta(entries: SessionEntry[]): string {
+  const pieces = entries.map(reviewPiece).filter((piece): piece is ReviewPiece => Boolean(piece));
+  const selected: ReviewPiece[] = [];
+  const categoryRemaining: Record<ReviewPiece["category"], number> = {
+    user: 14_000, workflow: 6_000, summary: 4_000, assistant: 8_000, tool: 4_000,
+  };
+  let remaining = KNOWLEDGE_RUNTIME_DEFAULTS.review.maxDeltaChars;
+  for (const piece of pieces.slice().sort((left, right) => left.priority - right.priority || left.index - right.index)) {
+    if (remaining <= 0) break;
+    const text = piece.text.slice(0, Math.min(remaining, categoryRemaining[piece.category]));
+    if (text) selected.push({ ...piece, text });
+    remaining -= text.length;
+    categoryRemaining[piece.category] -= text.length;
+  }
+  return selected.sort((left, right) => left.index - right.index).map((piece) => piece.text).join("\n\n");
+}
+
+function projectRootForSession(header: SessionHeader, branch: SessionEntry[]): string {
+  for (const entry of branch.slice().reverse()) {
+    if (entry.type !== "custom" || entry.customType !== "hwcode-working-directory"
+      || !entry.data || typeof entry.data !== "object") continue;
+    const cwd = (entry.data as Record<string, unknown>).cwd;
+    if (typeof cwd === "string" && cwd) return resolve(cwd);
+  }
+  return resolve(header.cwd);
+}
+
+export function discoverSessionFiles(root: string): string[] {
+  const files: string[] = [];
+  const visit = (directory: string): void => {
+    let entries;
+    try { entries = readdirSync(directory, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      const path = resolve(directory, entry.name);
+      if (entry.isDirectory()) visit(path);
+      else if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(path);
+    }
+  };
+  visit(resolve(root));
+  return files.sort();
+}
+
+export function readReviewTask(
+  sessionFile: string,
+  cursor: KnowledgeReviewCursor | undefined,
+  now = Date.now(),
+): KnowledgeReviewTask | undefined {
+  let stat;
+  try { stat = statSync(sessionFile); } catch { return undefined; }
+  if (!stat.isFile() || now - stat.mtimeMs < KNOWLEDGE_RUNTIME_DEFAULTS.review.idleMs) return undefined;
+  let content: string;
+  try { content = readFileSync(sessionFile, "utf8"); } catch { return undefined; }
+  const lastNewline = content.lastIndexOf("\n");
+  if (lastNewline < 0) return undefined;
+  const parsed = parseSessionEntries(content.slice(0, lastNewline + 1));
+  const header = parsed.find((entry): entry is SessionHeader => entry.type === "session");
+  if (!header?.id || !header.cwd) return undefined;
+  const allEntries = parsed.filter((entry): entry is SessionEntry => entry.type !== "session");
+  const branch = buildContextEntries(allEntries);
+  if (branch.length === 0) return undefined;
+  const cursorIndex = cursor?.lastReviewedEntryId
+    ? branch.findIndex((entry) => entry.id === cursor.lastReviewedEntryId)
+    : -1;
+  const pending = branch.slice(cursorIndex + 1);
+  if (pending.length === 0) return undefined;
+  const delta = buildDelta(pending);
+  const firstEntryId = pending[0].id;
+  const lastEntryId = pending[pending.length - 1].id;
+  const deltaDigest = knowledgeDeltaDigest(delta || `${firstEntryId}\0${lastEntryId}`);
+  const sessionFileHash = hash(resolve(sessionFile), 16);
+  const sessionKey = hash(header.id, 24);
+  const reviewKey = hash(`${header.id}\0${firstEntryId}\0${lastEntryId}\0${deltaDigest}`);
+  const projectRoot = projectRootForSession(header, branch);
+  return {
+    requestId: randomUUID(), reviewKey, sessionId: header.id, sessionKey, sessionFileHash,
+    projectRoot, projectKey: projectKnowledgeKey(projectRoot),
+    firstEntryId, lastEntryId, delta, deltaDigest, fileSize: stat.size, fileMtimeMs: stat.mtimeMs,
+  };
+}
+
+export function findNextReviewTask(
+  sessionsRoot: string,
+  reviews: Record<string, KnowledgeReviewCursor>,
+  now = Date.now(),
+): KnowledgeReviewTask | undefined {
+  const files = discoverSessionFiles(sessionsRoot).map((path) => {
+    try { return { path, mtimeMs: statSync(path).mtimeMs }; } catch { return undefined; }
+  }).filter((item): item is { path: string; mtimeMs: number } => Boolean(item))
+    .sort((left, right) => left.mtimeMs - right.mtimeMs);
+  for (const file of files) {
+    const fileHash = hash(resolve(file.path), 16);
+    const cursor = Object.values(reviews).find((item) => item.sessionFileHash === fileHash);
+    let task = readReviewTask(file.path, cursor, now);
+    const stableCursor = task ? reviews[task.sessionKey] : undefined;
+    if (task && stableCursor && stableCursor !== cursor) task = readReviewTask(file.path, stableCursor, now);
+    if (task) return task;
+  }
+  return undefined;
+}