onto-mcp 0.3.2 → 0.4.1

package/dist/core-runtime/learning/promote/judgment-auditor.js

@@ -1,517 +0,0 @@
-/**
- * Phase 3 Promote — Judgment Auditor (Step 8b).
- *
- * Design authority:
- *   - learn-phase3-design-v4.md DD-13 (P-14 sequencing)
- *   - learn-phase3-design-v6.md DD-17 (obligation ledger lifecycle)
- *   - learn-phase3-design-v9.md DD-17 (carry-forward + expired_unattended ingress)
- *   - learn-phase3-design-v7.md DD-21 (AuditObligation encapsulation — this
- *     module transitions obligations exclusively through AuditObligation.transition())
- *   - .onto/processes/learn/promote.md Step 8 (judgment re-verification threshold)
- *
- * Responsibility:
- *   - DD-13 P-14 pre-step: determine eligible agents for judgment audit
- *     (count_threshold OR outstanding obligation).
- *   - For each eligible agent, LLM-evaluate the `[judgment]`-type items in
- *     their global learnings file against the current context.
- *   - Emit AuditOutcome per item (retain | modify | delete | audit_to_conflict_proposal).
- *   - Transition matching AuditObligations through their lifecycle
- *     (pending → in_progress → fulfilled | blocked | no_eligible_agents).
- *   - Return a compact AuditSummary consumable by PromoteReport assembly.
- *
- * Scope boundary:
- *   - Phase A only. No file mutation; audit outcomes are proposals that flow
- *     into PromoteReport for user approval before Phase B applies them.
- *   - ConflictProposal emission for `audit_to_conflict_proposal` outcomes is
- *     handled by the orchestrator (promoter.ts) — this module returns raw
- *     outcomes and leaves proposal assembly to the caller, which holds the
- *     origin/lineage context (DD-19).
- *
- * Failure model:
- *   - LLM unreachable → obligation transitions in_progress → blocked.
- *     Next promote will re-enter pending (audit-state.ts processCarryForward).
- *   - Malformed LLM JSON → blocked (treated the same as unreachable).
- *   - No eligible items for an obligation's affected_agents → in_progress →
- *     no_eligible_agents (terminal).
- */
-import { callLlm, hashPrompt } from "../shared/llm-caller.js";
-// ---------------------------------------------------------------------------
-// Policy
-// ---------------------------------------------------------------------------
-export const DEFAULT_AUDIT_POLICY = {
-    judgment_threshold: 10, // promote.md §8
-    obligation_max_carry_forward: 3,
-};
-// ---------------------------------------------------------------------------
-// Eligibility determination — DD-13 pre-step
-// ---------------------------------------------------------------------------
-/**
- * Count `[judgment]`-type items per agent across the given item pool.
- *
- * The caller supplies global_items from CollectionResult. Judgment audit runs
- * only against global items because audit is a re-verification of already-
- * promoted judgments — project-level items are the promote panel's domain.
- */
-function countJudgmentsByAgent(items) {
-    const counts = new Map();
-    for (const item of items) {
-        if (item.type !== "judgment")
-            continue;
-        counts.set(item.agent_id, (counts.get(item.agent_id) ?? 0) + 1);
-    }
-    return counts;
-}
-/**
- * DD-13 pre-step: resolve which agents are eligible for audit in this promote
- * session.
- *
- * Two trigger paths:
- *   1. count_threshold — an agent holds >= policy.judgment_threshold judgment
- *      items in its global file (promote.md §8).
- *   2. obligation — an active obligation whose `affected_agents` list names
- *      the agent (DD-17 carry-forward path).
- *
- * When both paths fire for the same agent the obligation path wins so the
- * audit explicitly fulfills the ledger entry.
- */
-export function determineAuditEligibility(globalItems, state, policy = DEFAULT_AUDIT_POLICY) {
-    const counts = countJudgmentsByAgent(globalItems);
-    const eligibility = [];
-    const handledAgents = new Set();
-    // Obligation-triggered eligibility first (higher priority)
-    for (const ob of state.obligations) {
-        if (ob.status !== "pending")
-            continue;
-        for (const agent of ob.affected_agents) {
-            if (handledAgents.has(agent))
-                continue;
-            handledAgents.add(agent);
-            eligibility.push({
-                agent_id: agent,
-                obligation_id: ob.obligation_id,
-                reason: `obligation ${ob.obligation_id}: ${ob.reason}`,
-                judgment_count: counts.get(agent) ?? 0,
-                trigger: "obligation",
-            });
-        }
-    }
-    // Count-threshold eligibility (only for agents not already covered)
-    for (const [agent, count] of counts) {
-        if (handledAgents.has(agent))
-            continue;
-        if (count < policy.judgment_threshold)
-            continue;
-        handledAgents.add(agent);
-        eligibility.push({
-            agent_id: agent,
-            obligation_id: null,
-            reason: `count_threshold: ${count} >= ${policy.judgment_threshold} judgment items`,
-            judgment_count: count,
-            trigger: "count_threshold",
-        });
-    }
-    return eligibility;
-}
-// ---------------------------------------------------------------------------
-// Prompt building
-// ---------------------------------------------------------------------------
-const AUDIT_SYSTEM_PROMPT = `You are re-verifying previously promoted [judgment]-type learnings for an agent. For each item, decide whether it is still contextually valid.
-
-Output ONE JSON object with an "outcomes" array. For each item output:
-
-{
-  "item_index": <0-based integer matching the input order>,
-  "decision": "retain" | "modify" | "delete" | "audit_to_conflict_proposal",
-  "reason": "<one-sentence justification>",
-  "modified_content": "<required only when decision == modify, otherwise null>"
-}
-
-Decision rules:
-  - "retain" — item remains valid as written; no action needed.
-  - "modify" — the core insight still holds but wording needs update; provide modified_content.
-  - "delete" — the judgment is no longer valid and should be retired.
-  - "audit_to_conflict_proposal" — the item conflicts with another global item;
-    a formal conflict proposal will be created for operator review.
-
-Coherence rules:
-  - Return exactly one outcome per input item, in order.
-  - modified_content MUST be set when decision is "modify", and MUST be null otherwise.
-
-Respond ONLY with valid JSON (no markdown fences):
-{"outcomes": [...]}
-`;
-function buildAuditUserPrompt(agentId, items) {
-    const itemBlock = items
-        .map((item, i) => `${i}. tags=[${item.applicability_tags.join(" ")}] role=${item.role ?? "<no-role>"}\n   content: ${item.content}`)
-        .join("\n");
-    return `Agent: ${agentId}
-Judgment items to re-verify: ${items.length}
-${itemBlock}
-
-Respond with {"outcomes":[...]}.`;
-}
-function parseAuditResponse(text) {
-    let cleaned = text.trim();
-    if (cleaned.startsWith("```")) {
-        cleaned = cleaned.replace(/^```(?:json)?\s*/, "").replace(/```\s*$/, "");
-    }
-    const parsed = JSON.parse(cleaned);
-    if (!parsed || !Array.isArray(parsed.outcomes)) {
-        throw new Error(`audit response missing "outcomes" array (got keys: ${Object.keys(parsed ?? {}).join(",")})`);
-    }
-    return { outcomes: parsed.outcomes };
-}
-const VALID_DECISIONS = [
-    "retain",
-    "modify",
-    "delete",
-    "audit_to_conflict_proposal",
-];
-function normalizeOutcomes(raw, items, agentId) {
-    const outcomes = [];
-    const failures = [];
-    const seen = new Set();
-    for (const r of raw) {
-        if (typeof r.item_index !== "number") {
-            failures.push(`item_index missing/non-numeric`);
-            continue;
-        }
-        const idx = r.item_index;
-        if (idx < 0 || idx >= items.length) {
-            failures.push(`item_index ${idx} out of range 0..${items.length - 1}`);
-            continue;
-        }
-        if (seen.has(idx)) {
-            failures.push(`item_index ${idx} duplicated`);
-            continue;
-        }
-        seen.add(idx);
-        const decision = r.decision;
-        if (!VALID_DECISIONS.includes(decision)) {
-            failures.push(`item ${idx}: invalid decision "${String(r.decision)}"`);
-            continue;
-        }
-        const outcome = {
-            agent_id: agentId,
-            item: items[idx],
-            decision,
-            reason: typeof r.reason === "string" ? r.reason : "",
-        };
-        if (decision === "modify") {
-            if (typeof r.modified_content !== "string" || !r.modified_content) {
-                failures.push(`item ${idx}: decision is modify but modified_content missing`);
-                continue;
-            }
-            outcome.modified_content = r.modified_content;
-        }
-        else if (r.modified_content !== undefined &&
-            r.modified_content !== null) {
-            failures.push(`item ${idx}: decision is ${decision} but modified_content supplied`);
-            continue;
-        }
-        outcomes.push(outcome);
-    }
-    for (let i = 0; i < items.length; i++) {
-        if (!seen.has(i)) {
-            failures.push(`item ${i}: no outcome returned`);
-        }
-    }
-    return { outcomes, failures };
-}
-/**
- * Maximum items per batched LLM call.
- *
- * Chunking keeps each prompt and response within a bounded size window and
- * lets per-chunk failures be isolated rather than failing the whole agent.
- */
-const AUDIT_BATCH_SIZE = 12;
-/**
- * Slice items into batches of `size` each. The constant AUDIT_BATCH_SIZE is
- * a positive number; the dead `size <= 0` defensive branch was removed in
- * the N-2 cleanup. If a future caller needs a configurable batch size and
- * passes 0, the for-loop's `i += 0` would infinite-loop — that's an
- * obviously broken contract that doesn't need silent special-casing.
- */
-function chunk(items, size) {
-    const chunks = [];
-    for (let i = 0; i < items.length; i += size) {
-        chunks.push(items.slice(i, i + size));
-    }
-    return chunks;
-}
-async function auditAgent(config) {
-    const judgmentItems = config.globalItems.filter((item) => item.agent_id === config.agentId && item.type === "judgment");
-    if (judgmentItems.length === 0) {
-        return {
-            kind: "no_eligible_items",
-            outcomes: [],
-            llm_calls: 0,
-            judgment_count_total: 0,
-            failed_chunks_count: 0,
-        };
-    }
-    // Chunk into batches so large agents (37+ items) don't blow the LLM
-    // timeout window. Each chunk gets its own LLM call with the same
-    // 1-retry-on-validation-error pattern. Per-chunk provider errors are
-    // isolated so a single transient failure doesn't lose the whole agent.
-    const batches = chunk(judgmentItems, AUDIT_BATCH_SIZE);
-    let llmCalls = 0;
-    const allOutcomes = [];
-    const failureReasons = [];
-    const tryOnce = async (batchItems, retryFeedback) => {
-        let userPrompt = buildAuditUserPrompt(config.agentId, batchItems);
-        if (retryFeedback && retryFeedback.length > 0) {
-            userPrompt +=
-                "\n\nPrevious attempt was rejected. Validator feedback:\n" +
-                    retryFeedback.map((f) => `  - ${f}`).join("\n") +
-                    "\nFix these issues and respond again.";
-        }
-        let llmText;
-        try {
-            const result = await callLlm(AUDIT_SYSTEM_PROMPT, userPrompt, {
-                max_tokens: config.maxTokens ?? 4096,
-                ...(config.modelId ? { model_id: config.modelId } : {}),
-            });
-            llmText = result.text;
-        }
-        catch (error) {
-            return {
-                kind: "provider_error",
-                outcomes: [],
-                failureReason: error instanceof Error ? error.message : String(error),
-            };
-        }
-        let parsed;
-        try {
-            parsed = parseAuditResponse(llmText);
-        }
-        catch (error) {
-            return {
-                kind: "blocked",
-                outcomes: [],
-                failureReason: `malformed JSON: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }
-        const normalized = normalizeOutcomes(parsed.outcomes, batchItems, config.agentId);
-        if (normalized.failures.length > 0) {
-            return {
-                kind: "blocked",
-                outcomes: normalized.outcomes,
-                failureReason: normalized.failures.join("; "),
-            };
-        }
-        return { kind: "success", outcomes: normalized.outcomes };
-    };
-    for (let i = 0; i < batches.length; i++) {
-        const batch = batches[i];
-        const first = await tryOnce(batch);
-        llmCalls += 1;
-        if (first.kind === "success") {
-            allOutcomes.push(...first.outcomes);
-            continue;
-        }
-        if (first.kind === "provider_error") {
-            failureReasons.push(`batch ${i + 1}/${batches.length} (${batch.length} items): ` +
-                `provider error: ${first.failureReason ?? "unknown"}`);
-            // Provider errors skip retry and skip the batch entirely.
-            continue;
-        }
-        // first.kind === "blocked" (validation error) — retry the batch
-        const second = await tryOnce(batch, first.failureReason !== undefined ? [first.failureReason] : undefined);
-        llmCalls += 1;
-        if (second.kind === "success") {
-            allOutcomes.push(...second.outcomes);
-            continue;
-        }
-        failureReasons.push(`batch ${i + 1}/${batches.length} (${batch.length} items): ` +
-            `${second.failureReason ?? first.failureReason ?? "unknown"}`);
-    }
-    const failedChunksCount = failureReasons.length;
-    if (failedChunksCount === 0) {
-        return {
-            kind: "success",
-            outcomes: allOutcomes,
-            llm_calls: llmCalls,
-            judgment_count_total: judgmentItems.length,
-            failed_chunks_count: 0,
-        };
-    }
-    if (allOutcomes.length === 0) {
-        return {
-            kind: "blocked",
-            outcomes: [],
-            llm_calls: llmCalls,
-            failure_reason: failureReasons.join(" | "),
-            judgment_count_total: judgmentItems.length,
-            failed_chunks_count: failedChunksCount,
-        };
-    }
-    // Some batches succeeded, some failed — partial success. The operator
-    // sees the partial outcomes plus the failure reasons for the missing
-    // batches in failed_agents.
-    return {
-        kind: "partial",
-        outcomes: allOutcomes,
-        llm_calls: llmCalls,
-        failure_reason: failureReasons.join(" | "),
-        judgment_count_total: judgmentItems.length,
-        failed_chunks_count: failedChunksCount,
-    };
-}
-/**
- * DD-13 + DD-17 entry point.
- *
- * Lifecycle contract:
- *   1. Determine eligibility from global_items and state.obligations.
- *   2. Mark each eligible obligation in_progress (via transition()).
- *   3. For each eligible agent, call the LLM and collect outcomes.
- *   4. Transition the obligation according to the result:
- *      - success → fulfilled
- *      - blocked (transient) → blocked
- *      - no_eligible_items → no_eligible_agents
- *   5. Emit ObligationProcessed entries so PromoteReport preserves the
- *      canonical pre/post transition record (DD-17 CONCISENESS-02).
- *
- * Mutation note: `state.obligations[*]` are AuditObligation class instances.
- * This function MUTATES their status via the class API — callers are
- * expected to saveAuditState() after Phase A completes so the ledger reflects
- * this pass's transitions.
- */
-export async function runJudgmentAudit(config) {
-    const policy = config.policy ?? DEFAULT_AUDIT_POLICY;
-    const eligibility = determineAuditEligibility(config.globalItems, config.state, policy);
-    // Index obligations by id for quick lookup.
-    const obligationById = new Map(config.state.obligations.map((o) => [o.obligation_id, o]));
-    const obligationsSeen = new Set();
-    const obligationsProcessed = [];
-    // Transition pending → in_progress for each obligation being picked up.
-    // We capture the starting status for the ObligationProcessed record.
-    const startingStatus = new Map();
-    for (const e of eligibility) {
-        if (!e.obligation_id)
-            continue;
-        if (obligationsSeen.has(e.obligation_id))
-            continue;
-        obligationsSeen.add(e.obligation_id);
-        const ob = obligationById.get(e.obligation_id);
-        if (!ob)
-            continue;
-        startingStatus.set(ob.obligation_id, ob.status);
-        if (ob.status === "pending") {
-            ob.transition("in_progress", `P-14 picked up in session ${config.sessionId}`, { session_id: config.sessionId });
-        }
-    }
-    const allOutcomes = [];
-    const auditedAgents = [];
-    const failedAgents = [];
-    let llmCalls = 0;
-    // Run audits per eligible agent.
-    for (const e of eligibility) {
-        const agentConfig = {
-            agentId: e.agent_id,
-            globalItems: config.globalItems,
-        };
-        if (config.maxTokens !== undefined) {
-            agentConfig.maxTokens = config.maxTokens;
-        }
-        if (config.modelId !== undefined) {
-            agentConfig.modelId = config.modelId;
-        }
-        const result = await auditAgent(agentConfig);
-        llmCalls += result.llm_calls;
-        if (result.kind === "success") {
-            auditedAgents.push(e.agent_id);
-            allOutcomes.push(...result.outcomes);
-            if (e.obligation_id) {
-                const ob = obligationById.get(e.obligation_id);
-                if (ob && ob.status === "in_progress") {
-                    ob.transition("fulfilled", `P-14 completed for ${e.agent_id}`, {
-                        session_id: config.sessionId,
-                    });
-                }
-            }
-            continue;
-        }
-        if (result.kind === "partial") {
-            // Partial success: some chunks succeeded, some failed. The agent is
-            // counted as audited (its outcomes are real) but also recorded in
-            // failed_agents so the operator sees the missing chunks. Obligation
-            // transitions to blocked because the audit isn't complete.
-            auditedAgents.push(e.agent_id);
-            allOutcomes.push(...result.outcomes);
-            failedAgents.push({
-                agent_id: e.agent_id,
-                reason: `partial: ${result.failure_reason ?? "unknown"}`,
-                judgment_count_total: result.judgment_count_total,
-                failed_chunks_count: result.failed_chunks_count,
-            });
-            if (e.obligation_id) {
-                const ob = obligationById.get(e.obligation_id);
-                if (ob && ob.status === "in_progress") {
-                    ob.transition("blocked", `P-14 partial failure: ${result.failure_reason ?? "unknown"}`, { session_id: config.sessionId });
-                }
-            }
-            continue;
-        }
-        if (result.kind === "no_eligible_items") {
-            if (e.obligation_id) {
-                const ob = obligationById.get(e.obligation_id);
-                if (ob && ob.status === "in_progress") {
-                    ob.transition("no_eligible_agents", `no judgment items found for ${e.agent_id}`, { session_id: config.sessionId });
-                }
-            }
-            continue;
-        }
-        // result.kind === "blocked" — fully failed, surface as failed_agent
-        failedAgents.push({
-            agent_id: e.agent_id,
-            reason: result.failure_reason ?? "unknown failure",
-            judgment_count_total: result.judgment_count_total,
-            failed_chunks_count: result.failed_chunks_count,
-        });
-        if (e.obligation_id) {
-            const ob = obligationById.get(e.obligation_id);
-            if (ob && ob.status === "in_progress") {
-                ob.transition("blocked", `P-14 transient failure: ${result.failure_reason ?? "unknown"}`, { session_id: config.sessionId });
-            }
-        }
-    }
-    // Build ObligationProcessed records from captured start/end status pairs.
-    for (const [obligationId, from] of startingStatus) {
-        const ob = obligationById.get(obligationId);
-        if (!ob)
-            continue;
-        obligationsProcessed.push({
-            obligation_id: obligationId,
-            transition: { from, to: ob.status },
-        });
-    }
-    const outcomesTally = {
-        retain: 0,
-        modify: 0,
-        delete: 0,
-        audit_to_conflict_proposal: 0,
-    };
-    for (const o of allOutcomes) {
-        outcomesTally[o.decision] += 1;
-    }
-    const summary = {
-        policy,
-        obligations_processed: obligationsProcessed,
-        eligibility,
-        execution: {
-            audited_agents: auditedAgents,
-            audited_items_count: allOutcomes.length,
-            llm_calls: llmCalls,
-        },
-        outcomes: outcomesTally,
-        failed_agents: failedAgents,
-    };
-    return { summary, outcomes: allOutcomes };
-}
-// ---------------------------------------------------------------------------
-// Prompt hash helper — exported for audit trail tests
-// ---------------------------------------------------------------------------
-export function hashAuditPrompt(agentId, items) {
-    return hashPrompt(AUDIT_SYSTEM_PROMPT + "\n" + buildAuditUserPrompt(agentId, items));
-}