martin-loop 0.1.5 → 1.3.0

package/dist/vendor/core/memory/learning-pipeline.js

@@ -0,0 +1,391 @@
+/**
+ * learning-pipeline.ts
+ *
+ * Guarded promotion pipeline for self-improving memory heuristics.
+ *
+ * Flow: candidate → shadow test → holdout validation → approval gate → promoted
+ *
+ * A learning candidate NEVER becomes an active heuristic without passing:
+ *   1. Shadow tests (runs in parallel with existing logic, results compared)
+ *   2. Holdout validation (tested on a held-out set, not the training data)
+ *   3. Approval gate (auto-approved if confidence >= threshold, else requires manual sign-off)
+ *
+ * Fail-open: any pipeline error keeps the existing heuristic active untouched.
+ */
+import { appendFile, mkdir, readdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+export const DEFAULT_PROTECTED_LEARNING_PATHS = [
+    "packages/core/src/leash.ts",
+    "packages/core/src/leash/**",
+    "packages/core/src/policy.ts",
+    "packages/core/src/grounding.ts",
+    "packages/core/src/security/**",
+    "packages/core/src/attestation/**",
+    "packages/contracts/src/**",
+    "packages/mcp/src/**",
+    "packages/cli/src/index.ts",
+    "docs/release/**",
+    "docs/security/**",
+    "docs/handoffs/**",
+    ".planning/**",
+    "scripts/**"
+];
+export const DEFAULT_ACTIVE_LEARNING_HEURISTICS = {
+    schemaVersion: "1.0",
+    updatedAt: new Date(0).toISOString(),
+    recallScoring: {
+        failureClassWeight: 30,
+        interventionWeight: 14,
+        tokenOverlapWeight: 6,
+        completionWeight: 4
+    }
+};
+// ─── Shadow test harness ──────────────────────────────────────────────────────
+/**
+ * Runs the candidate heuristic in shadow mode alongside the existing logic.
+ * The shadow result is compared but never used to change actual behaviour.
+ *
+ * `existingScores` and `candidateScores` are parallel arrays — one per recall query.
+ * A case is "passed" when the candidate score >= existing score (does not regress).
+ */
+export function runShadowTest(candidateId, existingScores, candidateScores) {
+    if (existingScores.length === 0) {
+        return { candidateId, totalCases: 0, passedCases: 0, passRate: 1, passed: true };
+    }
+    const totalCases = existingScores.length;
+    let passedCases = 0;
+    for (let i = 0; i < totalCases; i++) {
+        const existing = existingScores[i] ?? 0;
+        const candidate = candidateScores[i] ?? 0;
+        // Pass when candidate is at least as good as existing (within 5% tolerance)
+        if (candidate >= existing * 0.95) {
+            passedCases++;
+        }
+    }
+    const passRate = passedCases / totalCases;
+    return { candidateId, totalCases, passedCases, passRate, passed: passRate >= 0.8 };
+}
+// ─── Holdout validation ───────────────────────────────────────────────────────
+/**
+ * Validates the candidate against a held-out set (data not used to build the candidate).
+ * The holdout set is scored by both the existing and candidate heuristic;
+ * the candidate passes if it meets or exceeds the existing on >= 80% of cases.
+ */
+export function runHoldoutValidation(candidateId, holdoutExistingScores, holdoutCandidateScores) {
+    if (holdoutExistingScores.length === 0) {
+        return {
+            candidateId,
+            holdoutSize: 0,
+            passedCases: 0,
+            passRate: 1,
+            passed: true
+        };
+    }
+    const holdoutSize = holdoutExistingScores.length;
+    let passedCases = 0;
+    for (let i = 0; i < holdoutSize; i++) {
+        const existing = holdoutExistingScores[i] ?? 0;
+        const candidate = holdoutCandidateScores[i] ?? 0;
+        if (candidate >= existing * 0.95) {
+            passedCases++;
+        }
+    }
+    const passRate = passedCases / holdoutSize;
+    return { candidateId, holdoutSize, passedCases, passRate, passed: passRate >= 0.8 };
+}
+// ─── Approval gate ────────────────────────────────────────────────────────────
+/**
+ * Determines whether a candidate can be promoted based on holdout results and
+ * the configured approval policy.
+ *
+ * Auto-approves when:
+ *   - holdout passed AND holdoutPassRate >= autoApproveThreshold
+ *   - AND requireManualApproval is false
+ *
+ * In all other cases, the candidate is held at "holdout_passed" status and
+ * awaits manual sign-off (future: webhook / dashboard approval flow).
+ */
+export function evaluateApprovalGate(holdout, config = { autoApproveThreshold: 0.85, requireManualApproval: false }) {
+    if (!holdout.passed) {
+        return {
+            approved: false,
+            reason: `Holdout failed: ${(holdout.passRate * 100).toFixed(1)}% pass rate (threshold 80%).`
+        };
+    }
+    if (config.requireManualApproval) {
+        return {
+            approved: false,
+            reason: "Manual approval required by policy. Candidate held at holdout_passed."
+        };
+    }
+    if (holdout.passRate >= config.autoApproveThreshold) {
+        return {
+            approved: true,
+            reason: `Auto-approved: ${(holdout.passRate * 100).toFixed(1)}% holdout pass rate >= ${(config.autoApproveThreshold * 100).toFixed(0)}% threshold.`
+        };
+    }
+    return {
+        approved: false,
+        reason: `Pass rate ${(holdout.passRate * 100).toFixed(1)}% below auto-approve threshold of ${(config.autoApproveThreshold * 100).toFixed(0)}%. Manual review required.`
+    };
+}
+// ─── Persistence ──────────────────────────────────────────────────────────────
+export async function persistCandidate(candidate, pipelineRoot) {
+    const dir = join(pipelineRoot, "learning-pipeline");
+    await mkdir(dir, { recursive: true });
+    const candidatePath = join(dir, `${candidate.candidateId}.json`);
+    await writeFile(candidatePath, JSON.stringify(candidate, null, 2), "utf8");
+    // Append to audit log
+    const auditEntry = JSON.stringify({
+        candidateId: candidate.candidateId,
+        status: candidate.status,
+        timestamp: new Date().toISOString()
+    });
+    await appendFile(join(dir, "pipeline-audit.jsonl"), `${auditEntry}\n`, "utf8");
+}
+export async function loadCandidate(candidateId, pipelineRoot) {
+    try {
+        const raw = await readFile(join(pipelineRoot, "learning-pipeline", `${candidateId}.json`), "utf8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function loadActiveLearningHeuristics(pipelineRoot) {
+    try {
+        const raw = await readFile(resolveActiveHeuristicsPath(pipelineRoot), "utf8");
+        const parsed = JSON.parse(raw);
+        const recall = isRecord(parsed.recallScoring)
+            ? parsed.recallScoring
+            : {};
+        return {
+            schemaVersion: "1.0",
+            updatedAt: typeof parsed.updatedAt === "string"
+                ? parsed.updatedAt
+                : DEFAULT_ACTIVE_LEARNING_HEURISTICS.updatedAt,
+            recallScoring: {
+                failureClassWeight: numberOrDefault(recall["failureClassWeight"], DEFAULT_ACTIVE_LEARNING_HEURISTICS.recallScoring.failureClassWeight),
+                interventionWeight: numberOrDefault(recall["interventionWeight"], DEFAULT_ACTIVE_LEARNING_HEURISTICS.recallScoring.interventionWeight),
+                tokenOverlapWeight: numberOrDefault(recall["tokenOverlapWeight"], DEFAULT_ACTIVE_LEARNING_HEURISTICS.recallScoring.tokenOverlapWeight),
+                completionWeight: numberOrDefault(recall["completionWeight"], DEFAULT_ACTIVE_LEARNING_HEURISTICS.recallScoring.completionWeight)
+            }
+        };
+    }
+    catch {
+        return DEFAULT_ACTIVE_LEARNING_HEURISTICS;
+    }
+}
+export async function summarizeLearningPipeline(pipelineRoot) {
+    const dir = join(pipelineRoot, "learning-pipeline");
+    let names = [];
+    try {
+        names = await readdir(dir);
+    }
+    catch {
+        return {
+            counts: emptySummaryCounts(),
+            protectedPaths: [...DEFAULT_PROTECTED_LEARNING_PATHS],
+            entries: []
+        };
+    }
+    const candidateFiles = names.filter((name) => name.endsWith(".json") && name !== "active-heuristics.json");
+    const entries = [];
+    const counts = emptySummaryCounts();
+    for (const name of candidateFiles) {
+        try {
+            const raw = await readFile(join(dir, name), "utf8");
+            const candidate = JSON.parse(raw);
+            incrementSummaryCount(counts, candidate.status);
+            entries.push({
+                candidateId: candidate.candidateId,
+                heuristicFamily: candidate.heuristicFamily,
+                status: candidate.status,
+                sourceRunId: candidate.sourceRunId,
+                createdAt: candidate.createdAt,
+                confidenceScore: candidate.provenance?.confidenceScore,
+                patternId: candidate.provenance?.patternId,
+                summary: candidate.description
+            });
+        }
+        catch {
+            continue;
+        }
+    }
+    return {
+        counts,
+        protectedPaths: [...DEFAULT_PROTECTED_LEARNING_PATHS],
+        entries: entries.sort((left, right) => right.createdAt.localeCompare(left.createdAt))
+    };
+}
+/**
+ * Runs the full guarded promotion pipeline:
+ * shadow → holdout → approval gate → persist result.
+ *
+ * Returns whether the candidate was promoted to "approved" status.
+ * Never throws — any error is caught and candidate is marked rejected.
+ */
+export async function runPromotionPipeline(input) {
+    const { candidate } = input;
+    try {
+        const scopeVerdict = validateCandidateScope(candidate);
+        if (!scopeVerdict.allowed) {
+            candidate.status = "blocked_protected_surface";
+            candidate.rejectedAt = new Date().toISOString();
+            candidate.rejectedReason = scopeVerdict.reason;
+            await persistCandidate(candidate, input.pipelineRoot);
+            return {
+                candidateId: candidate.candidateId,
+                promoted: false,
+                reason: scopeVerdict.reason
+            };
+        }
+        // Phase 1: Shadow test
+        const shadow = runShadowTest(candidate.candidateId, input.existingScores, input.candidateScores);
+        if (!shadow.passed) {
+            candidate.status = "shadow_failed";
+            candidate.shadowPassRate = shadow.passRate;
+            candidate.rejectedAt = new Date().toISOString();
+            candidate.rejectedReason = `Shadow test failed: ${(shadow.passRate * 100).toFixed(1)}% pass rate.`;
+            await persistCandidate(candidate, input.pipelineRoot);
+            return { candidateId: candidate.candidateId, promoted: false, reason: candidate.rejectedReason };
+        }
+        candidate.status = "shadow_passed";
+        candidate.shadowPassRate = shadow.passRate;
+        // Phase 2: Holdout validation
+        const holdout = runHoldoutValidation(candidate.candidateId, input.holdoutExistingScores, input.holdoutCandidateScores);
+        if (!holdout.passed) {
+            candidate.status = "holdout_failed";
+            candidate.holdoutPassRate = holdout.passRate;
+            candidate.rejectedAt = new Date().toISOString();
+            candidate.rejectedReason = `Holdout validation failed: ${(holdout.passRate * 100).toFixed(1)}% pass rate.`;
+            await persistCandidate(candidate, input.pipelineRoot);
+            return { candidateId: candidate.candidateId, promoted: false, reason: candidate.rejectedReason };
+        }
+        candidate.status = "holdout_passed";
+        candidate.holdoutPassRate = holdout.passRate;
+        // Phase 3: Approval gate
+        const gate = evaluateApprovalGate(holdout, input.approvalConfig);
+        if (gate.approved) {
+            candidate.approvedAt = new Date().toISOString();
+            candidate.status = "approved";
+            await applyCandidatePromotion(candidate, input.pipelineRoot);
+            candidate.status = "promoted";
+            await persistCandidate(candidate, input.pipelineRoot);
+            return { candidateId: candidate.candidateId, promoted: true, reason: gate.reason };
+        }
+        // Held for manual review — not rejected, not approved
+        candidate.status = "manual_review";
+        await persistCandidate(candidate, input.pipelineRoot);
+        return { candidateId: candidate.candidateId, promoted: false, reason: gate.reason };
+    }
+    catch (err) {
+        // Fail-open: never let pipeline errors affect the running system
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            candidateId: candidate.candidateId,
+            promoted: false,
+            reason: `Pipeline error (fail-open): ${message}`
+        };
+    }
+}
+function emptySummaryCounts() {
+    return {
+        proposed: 0,
+        shadowPassed: 0,
+        holdoutPassed: 0,
+        approved: 0,
+        promoted: 0,
+        rejected: 0,
+        blockedProtectedSurface: 0,
+        rolledBack: 0
+    };
+}
+function incrementSummaryCount(counts, status) {
+    switch (status) {
+        case "pending_shadow":
+            counts.proposed++;
+            break;
+        case "shadow_passed":
+            counts.shadowPassed++;
+            break;
+        case "holdout_passed":
+            counts.holdoutPassed++;
+            break;
+        case "approved":
+            counts.approved++;
+            break;
+        case "promoted":
+            counts.promoted++;
+            break;
+        case "blocked_protected_surface":
+            counts.blockedProtectedSurface++;
+            break;
+        case "rolled_back":
+            counts.rolledBack++;
+            break;
+        case "shadow_failed":
+        case "holdout_failed":
+        case "rejected":
+        case "manual_review":
+            counts.rejected++;
+            break;
+    }
+}
+function validateCandidateScope(candidate) {
+    const proposedPaths = candidate.proposedPaths ?? [];
+    if (proposedPaths.length === 0) {
+        return { allowed: true, reason: "No protected surfaces targeted." };
+    }
+    const blockedPath = proposedPaths.find((path) => DEFAULT_PROTECTED_LEARNING_PATHS.some((protectedPath) => matchesProtectedPath(path, protectedPath)));
+    if (!blockedPath) {
+        return { allowed: true, reason: "Candidate scope limited to allowed surfaces." };
+    }
+    return {
+        allowed: false,
+        reason: `Candidate targets protected surface: ${blockedPath}`
+    };
+}
+function matchesProtectedPath(candidatePath, protectedPath) {
+    const normalizedCandidate = normalizePath(candidatePath);
+    const normalizedProtected = normalizePath(protectedPath);
+    if (normalizedProtected.endsWith("/**")) {
+        const prefix = normalizedProtected.slice(0, -3);
+        return normalizedCandidate === prefix || normalizedCandidate.startsWith(`${prefix}/`);
+    }
+    return normalizedCandidate === normalizedProtected;
+}
+function normalizePath(value) {
+    return value.replace(/\\/g, "/").replace(/^\.\//, "").replace(/\/+$/u, "");
+}
+async function applyCandidatePromotion(candidate, pipelineRoot) {
+    if (candidate.heuristicFamily !== "recall_scoring") {
+        return;
+    }
+    const current = await loadActiveLearningHeuristics(pipelineRoot);
+    const weights = isRecord(candidate.payload["weights"])
+        ? candidate.payload["weights"]
+        : {};
+    const next = {
+        schemaVersion: "1.0",
+        updatedAt: new Date().toISOString(),
+        recallScoring: {
+            failureClassWeight: numberOrDefault(weights["failureClassWeight"], current.recallScoring.failureClassWeight),
+            interventionWeight: numberOrDefault(weights["interventionWeight"], current.recallScoring.interventionWeight),
+            tokenOverlapWeight: numberOrDefault(weights["tokenOverlapWeight"], current.recallScoring.tokenOverlapWeight),
+            completionWeight: numberOrDefault(weights["completionWeight"], current.recallScoring.completionWeight)
+        }
+    };
+    await mkdir(join(pipelineRoot, "learning-pipeline"), { recursive: true });
+    await writeFile(resolveActiveHeuristicsPath(pipelineRoot), JSON.stringify(next, null, 2), "utf8");
+}
+function resolveActiveHeuristicsPath(pipelineRoot) {
+    return join(pipelineRoot, "learning-pipeline", "active-heuristics.json");
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null;
+}
+function numberOrDefault(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) ? value : fallback;
+}
+//# sourceMappingURL=learning-pipeline.js.map

package/dist/vendor/core/memory/palace.d.ts

@@ -0,0 +1,84 @@
+import type { FailureClass, InterventionType, LoopRecord, LoopStatus, LoopLifecycleState } from "../../contracts/index.js";
+export interface MemoryRecallEntry {
+    drawerId: string;
+    roomKey: string;
+    lesson: string;
+    content: string;
+    sourceRunId: string;
+    sourceAttemptIndex?: number;
+    score: number;
+    reason: string;
+}
+export interface MemoryBrief {
+    wingKey: string;
+    recalledAt: string;
+    entries: MemoryRecallEntry[];
+}
+export interface LoopMemoryStore {
+    recall(input: {
+        workspaceId: string;
+        projectId: string;
+        objective: string;
+        focus: string;
+        previousAttempts: Array<{
+            failureClass?: FailureClass;
+            intervention?: InterventionType;
+            summary?: string;
+        }>;
+        limit?: number;
+        currentRunId?: string;
+        currentAttemptIndex?: number;
+    }): Promise<MemoryBrief | undefined>;
+    recordLoop(input: {
+        loop: LoopRecord;
+        status: LoopStatus;
+        lifecycleState: LoopLifecycleState;
+        reason?: string;
+    }): Promise<{
+        roomCount: number;
+        drawerCount: number;
+    }>;
+    recordPriorLearning(input: Record<string, unknown>): Promise<void>;
+}
+export interface PalaceMemoryRecall {
+    recallId: string;
+    wingKey: string;
+    roomKey: string;
+    drawerId: string;
+    runId: string;
+    attemptIndex: number;
+    recallScore: number;
+    recallReason: string;
+    recalledAt: string;
+}
+export interface PalaceMemoryRoom {
+    wingKey: string;
+    roomKey: string;
+    roomTitle: string;
+    roomType: "failure" | "outcome";
+    drawerCount: number;
+    lastRunId: string;
+    updatedAt: string;
+}
+export interface PalaceMemoryDrawer {
+    drawerId: string;
+    wingKey: string;
+    roomKey: string;
+    roomTitle: string;
+    workspaceId: string;
+    projectId: string;
+    sourceRunId: string;
+    sourceAttemptIndex?: number;
+    outcome: "failure" | "completion" | "exit";
+    failureClass?: FailureClass;
+    intervention?: InterventionType;
+    tags: string[];
+    tokens: string[];
+    lesson: string;
+    content: string;
+    createdAt: string;
+}
+export declare function resolveMemoryRoot(env?: NodeJS.ProcessEnv): string;
+export declare function createFileLoopMemoryStore(options?: {
+    memoryRoot?: string;
+}): LoopMemoryStore;