@hadooppei/hwcode 1.0.8 → 1.0.10

package/.pi/dist/lib/knowledge/extractor.js

@@ -0,0 +1,32 @@
+import { createHash } from "node:crypto";
+import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.js";
+export const KNOWLEDGE_EXTRACTION_SYSTEM_PROMPT = `You are HWCode's background knowledge reviewer.
+Review only the supplied conversation delta. Return strict JSON and no markdown.
+Persist only knowledge that is likely to be useful in future sessions:
+- explicit user corrections or stable preferences;
+- verified engineering rules, successful procedures, or expensive failed approaches;
+- reusable architecture, testing, debugging, deployment, or operational knowledge.
+Do not persist task summaries, guesses, temporary IDs, credentials, secrets, private data, or facts recoverable by simply reading the repository.
+Use storageHint "rule" only for short, precise, high-value instructions. Use "topic" for multi-step SOPs and detailed experience.
+Use action "revise" only when the delta explicitly corrects prior knowledge; otherwise use "add" or "reinforce".
+Scope may be "global" only for an explicit user directive intended across projects; otherwise use "project".
+Return: {"candidates":[{"key":"stable semantic key","title":"...","summary":"...","keywords":["..."],"scope":"global|project","body":"markdown body","evidence":["verified evidence"],"confidence":0.0,"storageHint":"rule|topic","action":"add|reinforce|revise","explicitUserDirective":false}]}
+Return {"candidates":[]} when nothing meets the threshold.`;
+export function buildKnowledgeExtractionPrompt(projectRoot, delta) {
+    return `Project root: ${projectRoot}\n\n<conversation_delta>\n${delta}\n</conversation_delta>`;
+}
+export function parseCandidateEnvelope(text) {
+    const trimmed = text.trim();
+    const unfenced = trimmed.replace(/^```(?:json)?\s*/iu, "").replace(/\s*```$/u, "");
+    const start = unfenced.indexOf("{");
+    const end = unfenced.lastIndexOf("}");
+    if (start < 0 || end <= start)
+        throw new Error("Knowledge reviewer did not return a JSON object");
+    const parsed = JSON.parse(unfenced.slice(start, end + 1));
+    if (!Array.isArray(parsed.candidates))
+        throw new Error("Knowledge reviewer response is missing candidates[]");
+    return parsed.candidates.slice(0, KNOWLEDGE_RUNTIME_DEFAULTS.review.maxCandidates);
+}
+export function knowledgeDeltaDigest(delta) {
+    return createHash("sha256").update(delta).digest("hex");
+}

package/.pi/dist/lib/knowledge/review-worker.js

@@ -0,0 +1,292 @@
+import { randomUUID } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
+import { createConnection, createServer } from "node:net";
+import { homedir } from "node:os";
+import { parentPort, workerData } from "node:worker_threads";
+import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.js";
+import { userRuntimePaths } from "../runtime/paths.js";
+import { parseCandidateEnvelope } from "./extractor.js";
+import { findNextReviewTask } from "./session-scanner.js";
+import { commitKnowledgeReview, ensureKnowledgeDirectories, loadCurrentManifest } from "./store.js";
+if (!parentPort)
+    throw new Error("Knowledge review worker requires a parent port");
+const home = workerData?.home ?? homedir();
+const paths = userRuntimePaths(home);
+const standbySockets = new Set();
+let eligible = false;
+let stopped = false;
+let sessionsRoot = "";
+let leaderServer;
+let leaderToken = "";
+let standbySocket;
+let electionPending = false;
+let activeTask;
+function send(message) {
+    parentPort.postMessage(message);
+}
+function coordinatorPath() {
+    return process.platform === "win32"
+        ? `\\\\.\\pipe\\hwcode-knowledge-v3-${Buffer.from(home).toString("hex").slice(0, 16)}`
+        : paths.knowledgeCoordinatorSocket;
+}
+function writeLeaderMetadata() {
+    writeFileSync(paths.knowledgeLeader, `${JSON.stringify({
+        protocolVersion: 3, leaderToken, pid: process.pid, electedAt: new Date().toISOString(),
+    }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+    chmodSync(paths.knowledgeLeader, 0o600);
+}
+function removeLeaderMetadata(token) {
+    try {
+        const current = JSON.parse(readFileSync(paths.knowledgeLeader, "utf8"));
+        if (current.leaderToken === token)
+            rmSync(paths.knowledgeLeader, { force: true });
+    }
+    catch { /* Another contender may already have replaced it. */ }
+}
+function broadcast(message) {
+    const line = `${JSON.stringify(message)}\n`;
+    for (const socket of standbySockets) {
+        if (socket.writable)
+            socket.write(line);
+    }
+}
+function scheduleElection(delay = Math.floor(Math.random() * 500)) {
+    if (!eligible || stopped || electionPending || leaderServer || standbySocket)
+        return;
+    electionPending = true;
+    const timer = setTimeout(() => {
+        electionPending = false;
+        attemptElection();
+    }, delay);
+    timer.unref();
+}
+function stepDown(reason) {
+    if (activeTask)
+        send({ type: "review_cancel", requestId: activeTask.requestId, reason });
+    activeTask = undefined;
+    const token = leaderToken;
+    leaderToken = "";
+    for (const socket of standbySockets)
+        socket.destroy();
+    standbySockets.clear();
+    const server = leaderServer;
+    leaderServer = undefined;
+    if (server) {
+        try {
+            server.close();
+        }
+        catch { /* The failing server may already be closed. */ }
+        if (process.platform !== "win32") {
+            try {
+                unlinkSync(coordinatorPath());
+            }
+            catch { /* Socket may already be gone. */ }
+        }
+    }
+    if (token)
+        removeLeaderMetadata(token);
+}
+function becomeStandby(socket) {
+    standbySocket = socket;
+    socket.unref();
+    send({ type: "leadership", state: "standby" });
+    let buffer = "";
+    socket.on("data", (chunk) => {
+        buffer += chunk.toString("utf8");
+        for (;;) {
+            const newline = buffer.indexOf("\n");
+            if (newline < 0)
+                break;
+            const line = buffer.slice(0, newline);
+            buffer = buffer.slice(newline + 1);
+            try {
+                const message = JSON.parse(line);
+                if (message.type === "generation_changed")
+                    send(message);
+            }
+            catch { /* Ignore malformed coordinator notifications. */ }
+        }
+    });
+    socket.once("close", () => {
+        if (standbySocket === socket)
+            standbySocket = undefined;
+        scheduleElection();
+    });
+}
+function cleanStaleSocketAndRetry() {
+    if (process.platform === "win32") {
+        scheduleElection(250);
+        return;
+    }
+    try {
+        mkdirSync(paths.knowledgeElectionLock, { mode: 0o700 });
+    }
+    catch {
+        scheduleElection(250);
+        return;
+    }
+    try {
+        if (existsSync(coordinatorPath()))
+            unlinkSync(coordinatorPath());
+    }
+    catch { /* Retrying listen will surface persistent failures. */ }
+    finally {
+        rmSync(paths.knowledgeElectionLock, { recursive: true, force: true });
+    }
+    scheduleElection();
+}
+function connectToLeader(probe = 0) {
+    if (!eligible || stopped || leaderServer || standbySocket)
+        return;
+    const socket = createConnection(coordinatorPath());
+    let connected = false;
+    socket.once("connect", () => { connected = true; becomeStandby(socket); });
+    socket.once("error", (error) => {
+        socket.destroy();
+        if (connected)
+            return;
+        if (error.code === "ECONNREFUSED" || error.code === "ENOENT") {
+            if (probe < 2) {
+                const timer = setTimeout(() => connectToLeader(probe + 1), 100);
+                timer.unref();
+            }
+            else
+                cleanStaleSocketAndRetry();
+        }
+        else {
+            send({ type: "review_failed", error: `Coordinator connection failed: ${error.message}` });
+            scheduleElection(1_000);
+        }
+    });
+}
+function becomeLeader(server) {
+    leaderServer = server;
+    leaderToken = randomUUID();
+    server.unref();
+    server.on("connection", (socket) => {
+        standbySockets.add(socket);
+        socket.unref();
+        socket.once("close", () => standbySockets.delete(socket));
+        const generationId = loadCurrentManifest(home).generationId;
+        if (generationId)
+            socket.write(`${JSON.stringify({ type: "generation_changed", generationId })}\n`);
+    });
+    server.on("error", (error) => {
+        send({ type: "review_failed", error: `Coordinator server failed: ${error.message}` });
+        stepDown("coordinator-server-failed");
+        scheduleElection();
+    });
+    writeLeaderMetadata();
+    send({ type: "leadership", state: "leader", leaderToken });
+    void scanForReview();
+}
+function attemptElection() {
+    if (!eligible || stopped || leaderServer || standbySocket)
+        return;
+    ensureKnowledgeDirectories(home);
+    const server = createServer();
+    const electionError = (error) => {
+        if (error.code === "EADDRINUSE")
+            connectToLeader();
+        else {
+            send({ type: "review_failed", error: `Coordinator election failed: ${error.message}` });
+            scheduleElection(1_000);
+        }
+    };
+    server.once("error", electionError);
+    server.listen(coordinatorPath(), () => {
+        server.off("error", electionError);
+        if (process.platform !== "win32")
+            chmodSync(coordinatorPath(), 0o600);
+        becomeLeader(server);
+    });
+}
+async function scanForReview() {
+    if (!leaderServer || !leaderToken || activeTask || !sessionsRoot || !eligible)
+        return;
+    try {
+        const manifest = loadCurrentManifest(home);
+        const task = findNextReviewTask(sessionsRoot, manifest.reviews);
+        if (!task)
+            return;
+        activeTask = task;
+        if (!task.delta) {
+            const result = commitKnowledgeReview([], task, leaderToken, home);
+            const generationId = loadCurrentManifest(home).generationId;
+            send({ type: "review_saved", requestId: task.requestId, generationId, result });
+            if (generationId) {
+                send({ type: "generation_changed", generationId });
+                broadcast({ type: "generation_changed", generationId });
+            }
+            activeTask = undefined;
+            queueMicrotask(() => void scanForReview());
+            return;
+        }
+        send({ type: "review_request", leaderToken, requestId: task.requestId, task });
+    }
+    catch (error) {
+        activeTask = undefined;
+        send({ type: "review_failed", error: error instanceof Error ? error.message : String(error) });
+    }
+}
+function handleReviewResult(message) {
+    const task = activeTask;
+    if (!task || message.requestId !== task.requestId || message.leaderToken !== leaderToken || !leaderServer)
+        return;
+    let committed = false;
+    try {
+        if (message.error || message.raw === undefined)
+            throw new Error(message.error || "Knowledge review returned no content");
+        const result = commitKnowledgeReview(parseCandidateEnvelope(message.raw), task, leaderToken, home);
+        const generationId = loadCurrentManifest(home).generationId;
+        send({ type: "review_saved", requestId: task.requestId, generationId, result });
+        if (generationId) {
+            send({ type: "generation_changed", generationId });
+            broadcast({ type: "generation_changed", generationId });
+        }
+        committed = true;
+    }
+    catch (error) {
+        send({ type: "review_failed", requestId: task.requestId, error: error instanceof Error ? error.message : String(error) });
+    }
+    finally {
+        activeTask = undefined;
+        if (committed)
+            queueMicrotask(() => void scanForReview());
+    }
+}
+const scanTimer = setInterval(() => { void scanForReview(); }, KNOWLEDGE_RUNTIME_DEFAULTS.review.intervalMs);
+scanTimer.unref();
+parentPort.on("message", (message) => {
+    if (message.type === "configure") {
+        sessionsRoot = message.sessionsRoot;
+        if (eligible === message.modelAvailable)
+            return;
+        eligible = message.modelAvailable;
+        if (!eligible) {
+            standbySocket?.destroy();
+            standbySocket = undefined;
+            stepDown("model-unavailable");
+            send({ type: "leadership", state: "ineligible" });
+        }
+        else
+            scheduleElection(0);
+        return;
+    }
+    if (message.type === "review_result") {
+        handleReviewResult(message);
+        return;
+    }
+    if (message.type === "scan_now") {
+        void scanForReview();
+        return;
+    }
+    stopped = true;
+    eligible = false;
+    clearInterval(scanTimer);
+    standbySocket?.destroy();
+    standbySocket = undefined;
+    stepDown("worker-stopped");
+    process.exit(0);
+});
+send({ type: "ready" });

package/.pi/dist/lib/knowledge/sanitize.js

@@ -0,0 +1,24 @@
+const PRIVATE_KEY_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/giu;
+const BEARER_RE = /\bBearer\s+[A-Za-z0-9._~+/=-]{12,}/giu;
+const JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/gu;
+const SECRET_ASSIGNMENT_RE = /\b(access[_-]?key|secret(?:[_-]?(?:access|key))?|password|passwd|token|credential|client[_-]?secret)\b(\s*[:=]\s*)[^\s,;]+/giu;
+const URL_CREDENTIAL_RE = /(https?:\/\/)[^\s/@:]+:[^\s/@]+@/giu;
+export function sanitizeKnowledgeText(value) {
+    return value
+        .replace(PRIVATE_KEY_RE, "<redacted-private-key>")
+        .replace(BEARER_RE, "Bearer <redacted>")
+        .replace(JWT_RE, "<redacted-jwt>")
+        .replace(SECRET_ASSIGNMENT_RE, (_match, key, separator) => `${key}${separator}<redacted>`)
+        .replace(URL_CREDENTIAL_RE, "$1<redacted>@")
+        .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/giu, "<runtime-uuid>")
+        .replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/gu, "<runtime-ip>")
+        .replace(/\b(vpc|subnet|secgroup|sg|vol|snap|img|inst|eip)-[0-9a-zA-Z]+/gu, "<$1-runtime-id>")
+        .replace(/\/(?:Users|home)\/[a-zA-Z0-9_-]+/gu, "~");
+}
+export function compactKnowledgeText(value, maxChars) {
+    return sanitizeKnowledgeText(value)
+        .replace(/\r\n/gu, "\n")
+        .replace(/[ \t]+$/gmu, "")
+        .trim()
+        .slice(0, maxChars);
+}

package/.pi/dist/lib/knowledge/session-scanner.js

@@ -0,0 +1,165 @@
+import { createHash, randomUUID } from "node:crypto";
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+import { buildContextEntries, parseSessionEntries, } from "@earendil-works/pi-coding-agent";
+import { KNOWLEDGE_RUNTIME_DEFAULTS } from "../runtime/defaults.js";
+import { knowledgeDeltaDigest } from "./extractor.js";
+import { sanitizeKnowledgeText } from "./sanitize.js";
+import { projectKnowledgeKey } from "./store.js";
+function hash(value, length = 64) {
+    return createHash("sha256").update(value).digest("hex").slice(0, length);
+}
+function textContent(content) {
+    if (typeof content === "string")
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content.map((item) => {
+        if (!item || typeof item !== "object")
+            return "";
+        const value = item;
+        return typeof value.text === "string" ? value.text : "";
+    }).filter(Boolean).join("\n");
+}
+function reviewPiece(entry, index) {
+    if (entry.type === "compaction" || entry.type === "branch_summary") {
+        return { index, priority: 1, category: "summary", text: `[summary]\n${sanitizeKnowledgeText(entry.summary).slice(0, 6_000)}` };
+    }
+    if (entry.type === "custom" && entry.customType.startsWith("hwcode-workflow")) {
+        return {
+            index, priority: 0, category: "workflow",
+            text: `[workflow_state:${entry.customType}]\n${sanitizeKnowledgeText(JSON.stringify(entry.data ?? {})).slice(0, 6_000)}`,
+        };
+    }
+    if (entry.type !== "message" || !entry.message || typeof entry.message !== "object")
+        return undefined;
+    const message = entry.message;
+    const role = typeof message.role === "string" ? message.role : "message";
+    const content = sanitizeKnowledgeText(textContent(message.content));
+    if (!content)
+        return undefined;
+    const priority = role === "user" ? 0 : role === "toolResult" ? 3 : 2;
+    const cap = role === "user" ? 8_000 : role === "toolResult" ? 1_500 : 4_000;
+    const category = role === "user" ? "user" : role === "toolResult" ? "tool" : "assistant";
+    return { index, priority, category, text: `[${role}]\n${content.slice(0, cap)}` };
+}
+function buildDelta(entries) {
+    const pieces = entries.map(reviewPiece).filter((piece) => Boolean(piece));
+    const selected = [];
+    const categoryRemaining = {
+        user: 14_000, workflow: 6_000, summary: 4_000, assistant: 8_000, tool: 4_000,
+    };
+    let remaining = KNOWLEDGE_RUNTIME_DEFAULTS.review.maxDeltaChars;
+    for (const piece of pieces.slice().sort((left, right) => left.priority - right.priority || left.index - right.index)) {
+        if (remaining <= 0)
+            break;
+        const text = piece.text.slice(0, Math.min(remaining, categoryRemaining[piece.category]));
+        if (text)
+            selected.push({ ...piece, text });
+        remaining -= text.length;
+        categoryRemaining[piece.category] -= text.length;
+    }
+    return selected.sort((left, right) => left.index - right.index).map((piece) => piece.text).join("\n\n");
+}
+function projectRootForSession(header, branch) {
+    for (const entry of branch.slice().reverse()) {
+        if (entry.type !== "custom" || entry.customType !== "hwcode-working-directory"
+            || !entry.data || typeof entry.data !== "object")
+            continue;
+        const cwd = entry.data.cwd;
+        if (typeof cwd === "string" && cwd)
+            return resolve(cwd);
+    }
+    return resolve(header.cwd);
+}
+export function discoverSessionFiles(root) {
+    const files = [];
+    const visit = (directory) => {
+        let entries;
+        try {
+            entries = readdirSync(directory, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const path = resolve(directory, entry.name);
+            if (entry.isDirectory())
+                visit(path);
+            else if (entry.isFile() && entry.name.endsWith(".jsonl"))
+                files.push(path);
+        }
+    };
+    visit(resolve(root));
+    return files.sort();
+}
+export function readReviewTask(sessionFile, cursor, now = Date.now()) {
+    let stat;
+    try {
+        stat = statSync(sessionFile);
+    }
+    catch {
+        return undefined;
+    }
+    if (!stat.isFile() || now - stat.mtimeMs < KNOWLEDGE_RUNTIME_DEFAULTS.review.idleMs)
+        return undefined;
+    let content;
+    try {
+        content = readFileSync(sessionFile, "utf8");
+    }
+    catch {
+        return undefined;
+    }
+    const lastNewline = content.lastIndexOf("\n");
+    if (lastNewline < 0)
+        return undefined;
+    const parsed = parseSessionEntries(content.slice(0, lastNewline + 1));
+    const header = parsed.find((entry) => entry.type === "session");
+    if (!header?.id || !header.cwd)
+        return undefined;
+    const allEntries = parsed.filter((entry) => entry.type !== "session");
+    const branch = buildContextEntries(allEntries);
+    if (branch.length === 0)
+        return undefined;
+    const cursorIndex = cursor?.lastReviewedEntryId
+        ? branch.findIndex((entry) => entry.id === cursor.lastReviewedEntryId)
+        : -1;
+    const pending = branch.slice(cursorIndex + 1);
+    if (pending.length === 0)
+        return undefined;
+    const delta = buildDelta(pending);
+    const firstEntryId = pending[0].id;
+    const lastEntryId = pending[pending.length - 1].id;
+    const deltaDigest = knowledgeDeltaDigest(delta || `${firstEntryId}\0${lastEntryId}`);
+    const sessionFileHash = hash(resolve(sessionFile), 16);
+    const sessionKey = hash(header.id, 24);
+    const reviewKey = hash(`${header.id}\0${firstEntryId}\0${lastEntryId}\0${deltaDigest}`);
+    const projectRoot = projectRootForSession(header, branch);
+    return {
+        requestId: randomUUID(), reviewKey, sessionId: header.id, sessionKey, sessionFileHash,
+        projectRoot, projectKey: projectKnowledgeKey(projectRoot),
+        firstEntryId, lastEntryId, delta, deltaDigest, fileSize: stat.size, fileMtimeMs: stat.mtimeMs,
+    };
+}
+export function findNextReviewTask(sessionsRoot, reviews, now = Date.now()) {
+    const files = discoverSessionFiles(sessionsRoot).map((path) => {
+        try {
+            return { path, mtimeMs: statSync(path).mtimeMs };
+        }
+        catch {
+            return undefined;
+        }
+    }).filter((item) => Boolean(item))
+        .sort((left, right) => left.mtimeMs - right.mtimeMs);
+    for (const file of files) {
+        const fileHash = hash(resolve(file.path), 16);
+        const cursor = Object.values(reviews).find((item) => item.sessionFileHash === fileHash);
+        let task = readReviewTask(file.path, cursor, now);
+        const stableCursor = task ? reviews[task.sessionKey] : undefined;
+        if (task && stableCursor && stableCursor !== cursor)
+            task = readReviewTask(file.path, stableCursor, now);
+        if (task)
+            return task;
+    }
+    return undefined;
+}