@lsctech/polaris 0.5.6 → 0.5.8

package/dist/autoresearch/sol-evidence-normalizer.js

@@ -0,0 +1,337 @@
+"use strict";
+/**
+ * SOL evidence normalizer.
+ *
+ * Materializes raw metric events (SolMetricEvent[]) and source references
+ * (SolSourceRef[]) from a loaded SolEvidence record. This is the bridge
+ * between the evidence loader and scorecard generation.
+ *
+ * Design rules:
+ *   - Never throws on missing data — absent optional fields reduce confidence.
+ *   - Source refs track every artifact that contributed evidence (present or not).
+ *   - Provider startup failures are distinguished from worker execution failures:
+ *       startup failure = provider-exhausted or selected_provider=null in router evidence
+ *       worker execution failure = worker ran but status="failed"|"error"
+ *   - Router fallback + candidate/rejection context are preserved when present.
+ *   - Missing router or QC evidence sets availability flags; no metric events
+ *     are emitted for those categories.
+ *   - Confidence: "high" when all expected source refs are available,
+ *     "medium" when some are absent, "low" when most are absent.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeSolEvidence = normalizeSolEvidence;
+function makeSourceRef(kind, path, available, unavailableReason) {
+    return available
+        ? { kind, path, available: true }
+        : { kind, path, available: false, unavailable_reason: unavailableReason };
+}
+function computeConfidence(refs) {
+    if (refs.length === 0)
+        return "none";
+    const available = refs.filter((r) => r.available).length;
+    const ratio = available / refs.length;
+    if (ratio >= 0.8)
+        return "high";
+    if (ratio >= 0.4)
+        return "medium";
+    if (ratio > 0)
+        return "low";
+    return "none";
+}
+// ──────────────────────────────────────────────
+// Source ref builders
+// ──────────────────────────────────────────────
+function buildSourceRefs(evidence, paths) {
+    const refs = [];
+    // Run state
+    if (paths.runStatePath) {
+        refs.push(makeSourceRef("run-state", paths.runStatePath, evidence.run.status !== null));
+    }
+    else {
+        refs.push(makeSourceRef("run-state", ".taskchain_artifacts/polaris-run/current-state.json", false, "path not resolved"));
+    }
+    // Telemetry
+    if (paths.telemetryPath) {
+        const hasTelemetry = evidence.tokens.total_worker_heartbeats > 0 || evidence.foreman.escalation_events > 0;
+        refs.push(makeSourceRef("telemetry", paths.telemetryPath, hasTelemetry));
+    }
+    else {
+        refs.push(makeSourceRef("telemetry", ".taskchain_artifacts/polaris-run/runs/<run-id>/telemetry.jsonl", false, "path not resolved"));
+    }
+    // Cluster state — available when the path was resolved (regardless of QC status)
+    if (paths.clusterStatePath) {
+        refs.push(makeSourceRef("cluster-state", paths.clusterStatePath, true));
+    }
+    else {
+        refs.push(makeSourceRef("cluster-state", `.polaris/clusters/${evidence.cluster_id ?? "unknown"}/cluster-state.json`, false, evidence.cluster_id ? "not found" : "cluster id not resolved"));
+    }
+    // Result packets (one per child with a known path)
+    for (const child of evidence.children) {
+        // Match filenames that are exactly "{child_id}.json" or start with "{child_id}-"
+        // to handle commit-suffixed names (e.g. "POL-001-abc.json") while avoiding
+        // false positives when IDs share a prefix (e.g. "1" vs "10").
+        const packetPath = paths.resultPacketPaths.find((p) => {
+            const filename = p.split('/').pop() ?? '';
+            return filename === `${child.child_id}.json` || filename.startsWith(`${child.child_id}-`);
+        });
+        const path = packetPath ?? `.polaris/clusters/${evidence.cluster_id ?? "unknown"}/results/${child.child_id}.json`;
+        refs.push(makeSourceRef("result-packet", path, packetPath !== undefined));
+    }
+    // QC artifacts
+    if (evidence.qc.availability === "available") {
+        refs.push(makeSourceRef("qc-finding", paths.qcDir ?? `.polaris/clusters/${evidence.cluster_id ?? "unknown"}/qc`, true));
+    }
+    else if (evidence.qc.availability === "future") {
+        refs.push(makeSourceRef("qc-finding", `.polaris/clusters/${evidence.cluster_id ?? "unknown"}/qc`, false, "QC not yet run for this cluster"));
+    }
+    // "unavailable" (qc-disabled): no ref emitted
+    // Run report
+    if (paths.runReportPath) {
+        refs.push(makeSourceRef("run-report", paths.runReportPath, true));
+    }
+    else {
+        refs.push(makeSourceRef("run-report", `smartdocs/reports/sol/${evidence.run_id}-evaluation-report.md`, false, "path not resolved"));
+    }
+    return refs;
+}
+// ──────────────────────────────────────────────
+// Metric event materializers
+// ──────────────────────────────────────────────
+function materializeProviderStartupFailures(evidence) {
+    if (evidence.router.availability !== "available")
+        return [];
+    return evidence.router.decisions
+        .filter((d) => d.exhausted)
+        .map((d) => ({
+        category: "provider-startup-failure",
+        run_id: evidence.run_id,
+        child_id: d.child_id || undefined,
+        provider: d.selected_provider ?? d.providers_tried[d.providers_tried.length - 1] ?? "unknown",
+        failure_reason: d.exhausted_reason,
+        providers_tried: d.providers_tried,
+        all_providers_exhausted: d.exhausted,
+    }));
+}
+function materializeRouterFallbacks(evidence) {
+    if (evidence.router.availability !== "available")
+        return [];
+    return evidence.router.decisions
+        .filter((d) => d.fallback_used && !d.exhausted)
+        .map((d) => {
+        const originalProvider = d.providers_tried[0] ?? null;
+        const fallbackProvider = d.selected_provider;
+        // Child succeeded if it has a "done" completion (check worker evidence)
+        const childResult = evidence.children.find((c) => c.child_id === d.child_id);
+        const fallbackSucceeded = childResult?.status === "done";
+        return {
+            category: "router-fallback",
+            run_id: evidence.run_id,
+            child_id: d.child_id || undefined,
+            original_provider: originalProvider,
+            fallback_provider: fallbackProvider,
+            providers_tried: d.providers_tried,
+            fallback_succeeded: fallbackSucceeded ?? false,
+            rejection_reasons: d.rejection_reasons,
+        };
+    });
+}
+function materializeWorkerExecutionFailures(evidence) {
+    // Worker execution failures: worker ran (not a startup failure) but status failed/error
+    const exhaustedChildIds = new Set(evidence.router.availability === "available"
+        ? evidence.router.decisions.filter((d) => d.exhausted).map((d) => d.child_id)
+        : []);
+    return evidence.children
+        .filter((c) => {
+        // Must be a failed/error status (not "done" or "blocked")
+        if (c.status !== "failed" && c.status !== "error")
+            return false;
+        // Exclude children where the provider never started (startup failure)
+        if (exhaustedChildIds.has(c.child_id))
+            return false;
+        return true;
+    })
+        .map((c) => {
+        const validationRecord = evidence.validation.find((v) => v.child_id === c.child_id);
+        return {
+            category: "worker-execution-failure",
+            run_id: evidence.run_id,
+            child_id: c.child_id,
+            worker_status: c.status,
+            validation: validationRecord?.outcome ?? "unknown",
+            provider: c.provider,
+            error_message: validationRecord?.error_message ?? null,
+            out_of_scope_escalation: evidence.intervention.out_of_scope_count > 0,
+            escalation_count: c.escalation_count,
+        };
+    });
+}
+function materializeValidationFailures(evidence) {
+    return evidence.validation
+        .filter((v) => v.outcome === "failed")
+        .map((v) => {
+        const childResult = evidence.children.find((c) => c.child_id === v.child_id);
+        return {
+            category: "validation-failure",
+            run_id: evidence.run_id,
+            child_id: v.child_id,
+            worker_status: childResult?.status ?? "unknown",
+            failed_commands: [], // Commands not individually tracked in SolValidationEvidence
+            passed_commands: v.passed_commands,
+            error_message: v.error_message,
+        };
+    });
+}
+function materializeQcFindings(evidence) {
+    if (evidence.qc.availability !== "available")
+        return [];
+    const events = [];
+    // Blocking open findings by severity
+    const severities = ["critical", "high", "medium", "low", "info"];
+    for (const sev of severities) {
+        const count = evidence.qc.open_by_severity[sev];
+        for (let i = 0; i < count; i++) {
+            const isHighSeverity = sev === "critical" || sev === "high";
+            events.push({
+                category: "qc-finding",
+                run_id: evidence.run_id,
+                qc_provider: Object.keys(evidence.qc.provider_breakdown)[0] ?? "unknown",
+                severity: sev,
+                blocking: isHighSeverity && evidence.qc.blocks_delivery,
+                autofixed: false,
+                repaired: false,
+                waived: false,
+                unvalidated: false,
+                summary: `Open ${sev} QC finding`,
+                attribution_confidence: isHighSeverity ? "high" : "medium",
+            });
+        }
+    }
+    // Emit a single unvalidated aggregate event when unvalidated_findings > 0
+    if (evidence.qc.unvalidated_findings > 0) {
+        const noisyProvider = evidence.qc.noisy_providers[0] ?? "unknown";
+        events.push({
+            category: "qc-finding",
+            run_id: evidence.run_id,
+            qc_provider: noisyProvider,
+            severity: "info",
+            blocking: false,
+            autofixed: false,
+            repaired: false,
+            waived: false,
+            unvalidated: true,
+            summary: `${evidence.qc.unvalidated_findings} unvalidated finding(s) from provider ${noisyProvider}`,
+            attribution_confidence: "none",
+        });
+    }
+    return events;
+}
+function materializeInterventions(evidence) {
+    const events = [];
+    if (evidence.intervention.user_intervened) {
+        // Find the children that had user interventions for child_id attribution
+        const intervened = evidence.children.filter((c) => c.user_intervened === true);
+        if (intervened.length > 0) {
+            for (const child of intervened) {
+                events.push({
+                    category: "user-intervention",
+                    run_id: evidence.run_id,
+                    child_id: child.child_id,
+                    actor: "user",
+                    intervention_type: "commit",
+                    resolved: true,
+                });
+            }
+        }
+        else {
+            events.push({
+                category: "user-intervention",
+                run_id: evidence.run_id,
+                actor: "user",
+                intervention_type: "unspecified",
+                resolved: true,
+            });
+        }
+    }
+    if (evidence.intervention.foreman_intervened) {
+        const intervened = evidence.children.filter((c) => c.foreman_intervened === true);
+        if (intervened.length > 0) {
+            for (const child of intervened) {
+                events.push({
+                    category: "foreman-intervention",
+                    run_id: evidence.run_id,
+                    child_id: child.child_id,
+                    actor: "foreman",
+                    intervention_type: "commit",
+                    resolved: true,
+                });
+            }
+        }
+        else {
+            events.push({
+                category: "foreman-intervention",
+                run_id: evidence.run_id,
+                actor: "foreman",
+                intervention_type: "unspecified",
+                resolved: true,
+            });
+        }
+    }
+    if (evidence.intervention.state_repair_required) {
+        events.push({
+            category: "foreman-intervention",
+            run_id: evidence.run_id,
+            actor: "foreman",
+            intervention_type: "state-repair",
+            resolved: false,
+        });
+    }
+    if (evidence.intervention.out_of_scope_count > 0) {
+        events.push({
+            category: "user-intervention",
+            run_id: evidence.run_id,
+            actor: "user",
+            intervention_type: "out-of-scope",
+            resolved: evidence.intervention.blocked_event_count > 0,
+        });
+    }
+    return events;
+}
+// ──────────────────────────────────────────────
+// Main entry point
+// ──────────────────────────────────────────────
+/**
+ * Normalize a loaded SolEvidence record into raw metric events and source refs.
+ *
+ * Does not throw on missing data — absent optional fields mark source refs
+ * as unavailable and reduce confidence. Missing router or QC inputs emit no
+ * events for those categories.
+ *
+ * @param evidence — already-loaded SolEvidence (from aggregateSolEvidence)
+ * @param paths    — artifact path context for source ref generation
+ * @returns SolEvidenceNormalizationResult
+ */
+function normalizeSolEvidence(evidence, paths = {
+    runStatePath: null,
+    telemetryPath: null,
+    clusterStatePath: null,
+    resultPacketPaths: [],
+    qcDir: null,
+    runReportPath: null,
+}) {
+    const source_refs = buildSourceRefs(evidence, paths);
+    const events = [
+        ...materializeProviderStartupFailures(evidence),
+        ...materializeRouterFallbacks(evidence),
+        ...materializeWorkerExecutionFailures(evidence),
+        ...materializeValidationFailures(evidence),
+        ...materializeQcFindings(evidence),
+        ...materializeInterventions(evidence),
+    ];
+    return {
+        run_id: evidence.run_id,
+        normalized_at: new Date().toISOString(),
+        events,
+        source_refs,
+        evidence_confidence: computeConfidence(source_refs),
+    };
+}

package/dist/autoresearch/sol-recommendations.js

@@ -19,6 +19,10 @@ exports.generateRecommendations = generateRecommendations;
 exports.recommendationToProposal = recommendationToProposal;
 exports.recommendationsToProposals = recommendationsToProposals;
 exports.formatRecommendationsCli = formatRecommendationsCli;
+exports.generateQcRecommendations = generateQcRecommendations;
+exports.formatQcRecommendations = formatQcRecommendations;
+exports.scorecardToRecommendationSummary = scorecardToRecommendationSummary;
+exports.scorecardsToRecommendationSummaries = scorecardsToRecommendationSummaries;
 const sol_report_js_1 = require("./sol-report.js");
 // ──────────────────────────────────────────────
 // Constants and helpers
@@ -298,3 +302,211 @@ function formatRecommendationsCli(report) {
     }
     return lines.join("\n");
 }
+function makeQcRecommendation(id, category, actionType, affected, proposedAction, rationale, confidence, evidenceRunId) {
+    return {
+        id,
+        category,
+        action_type: actionType,
+        affected,
+        proposed_action: proposedAction,
+        confidence: clamp01(confidence),
+        evidence: {
+            group_key: `run_id=${evidenceRunId}`,
+            grouped_by: ["run_id"],
+            count: 1,
+            mean_composite: null,
+            min_composite: null,
+            max_composite: null,
+            mean_foreman_composite: null,
+            mean_worker_composite: null,
+            run_ids: [evidenceRunId],
+        },
+        rationale,
+    };
+}
+/**
+ * Generate QC repair-loop follow-up recommendations from a single run's
+ * SOL evidence. These are advisory signals for noisy providers, repeated
+ * repair failures, unresolved high-severity findings, and max-round exhaustion.
+ */
+function generateQcRecommendations(evidence) {
+    const recommendations = [];
+    const qc = evidence.qc;
+    const runId = evidence.run_id;
+    if (qc.availability !== "available") {
+        return {
+            generated_at: new Date().toISOString(),
+            evidence_run_id: runId,
+            recommendations,
+        };
+    }
+    for (const provider of qc.noisy_providers) {
+        const counts = qc.provider_breakdown[provider] ?? { total: 0, blocking: 0, unvalidated: 0 };
+        const confidence = counts.total > 0 ? counts.unvalidated / counts.total : 0;
+        recommendations.push(makeQcRecommendation(`qc-noisy-provider:${provider}`, "provider_policy", "analyze", { provider }, `Review QC provider '${provider}' for noisy/unvalidated findings; consider raising attribution confidence thresholds or disabling the provider for high-risk routes.`, `${counts.unvalidated}/${counts.total} findings from '${provider}' are unvalidated noise.`, confidence, runId));
+    }
+    if (qc.has_repair_failures) {
+        const failed = qc.repair_loop?.packets_failed ?? 0;
+        recommendations.push(makeQcRecommendation(`qc-repair-failure:${runId}`, "qc_follow_up", "implement", {}, "Repeated repair worker failures detected — inspect repair packet scope, validation commands, and Medic referral path.", `${failed} repair packet(s) failed; consider tightening acceptance criteria or escalating to Medic.`, 0.9, runId));
+    }
+    if (qc.unresolved_high_severity > 0) {
+        recommendations.push(makeQcRecommendation(`qc-unresolved-high-severity:${runId}`, "qc_follow_up", "analyze", {}, `${qc.unresolved_high_severity} unresolved critical/high QC findings remain — triage before delivery.`, `${qc.unresolved_high_severity} critical/high findings are still open after repair loop.`, Math.min(1, qc.unresolved_high_severity / 5), runId));
+    }
+    if (qc.max_round_exhausted) {
+        recommendations.push(makeQcRecommendation(`qc-max-rounds:${runId}`, "qc_follow_up", "analyze", {}, "QC repair loop exhausted max rounds — review provider noise, repair packet scope, and escalation policy.", `Repair loop reached round ${qc.repair_loop?.rounds_completed ?? 0}/${qc.repair_loop?.max_rounds ?? 0} without passing.`, 0.85, runId));
+    }
+    return {
+        generated_at: new Date().toISOString(),
+        evidence_run_id: runId,
+        recommendations,
+    };
+}
+function formatQcRecommendations(report) {
+    const lines = [];
+    lines.push("SOL QC Follow-Up Recommendations");
+    lines.push(`Generated: ${report.generated_at}`);
+    lines.push(`Run: ${report.evidence_run_id}`);
+    lines.push("");
+    if (report.recommendations.length === 0) {
+        lines.push("No QC follow-up signals detected.");
+        return lines.join("\n") + "\n";
+    }
+    for (const r of report.recommendations) {
+        lines.push(`[${r.action_type}] ${r.id}`);
+        lines.push(`  Category:    ${r.category}`);
+        lines.push(`  Affected:    ${fmtAffected(r.affected)}`);
+        lines.push(`  Confidence:  ${(r.confidence * 100).toFixed(1)}%`);
+        lines.push(`  Action:      ${r.proposed_action}`);
+        lines.push(`  Rationale:   ${r.rationale}`);
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+// ── Internal helpers ──
+function extractQptEvidence(subscores) {
+    const s = subscores.find((ss) => ss.dimension === "quality_per_token");
+    return { score: s?.score ?? null, detail: s?.detail ?? null };
+}
+function extractQcEvidence(subscores, rawMetrics) {
+    const qcSub = subscores.find((ss) => ss.dimension === "qc" || ss.dimension === "qc_repair_loop" || ss.dimension === "qc_result");
+    return {
+        qc_findings: rawMetrics.qc_total_findings,
+        blocking_findings: rawMetrics.qc_blocking_findings,
+        qc_repair_status: rawMetrics.qc_repair_loop_status,
+        qc_score: qcSub?.score ?? null,
+    };
+}
+function extractValidationEvidence(subscores, rawMetrics) {
+    const valSub = subscores.find((ss) => ss.dimension === "validation" ||
+        ss.dimension === "evidence_validation" ||
+        ss.dimension === "validation_result");
+    return {
+        validation_outcome: rawMetrics.validation_outcome,
+        passed_commands: [...(rawMetrics.passed_commands ?? [])],
+        validation_score: valSub?.score ?? null,
+    };
+}
+function deriveVerdict(scorecard, qcEvidence) {
+    const { subscores, aggregate_score, recommendation_inputs } = scorecard;
+    // Blocking QC findings always contradict.
+    if ((qcEvidence.blocking_findings ?? 0) > 0)
+        return "contradicted";
+    // confirmed_signal subscore is the strongest direct verdict signal.
+    const confirmed = subscores.find((s) => s.dimension === "confirmed_signal");
+    if (confirmed !== undefined && confirmed.score !== null) {
+        if (confirmed.score >= 0.9)
+            return "supported";
+        if (confirmed.score <= 0.1)
+            return "contradicted";
+    }
+    // Aggregate score determines verdict; intervention/router issues limit to inconclusive.
+    if (aggregate_score !== null) {
+        if (aggregate_score < 0.5)
+            return "contradicted";
+        if (aggregate_score >= 0.75 &&
+            !recommendation_inputs.intervention_detected &&
+            !recommendation_inputs.router_issue_detected) {
+            return "supported";
+        }
+    }
+    if (recommendation_inputs.intervention_detected || recommendation_inputs.router_issue_detected) {
+        return "inconclusive";
+    }
+    if (aggregate_score === null)
+        return "inconclusive";
+    if (aggregate_score >= 0.75)
+        return "supported";
+    if (aggregate_score < 0.6)
+        return "contradicted";
+    return "inconclusive";
+}
+function confidenceFromScorecard(scorecard) {
+    const { subscores, aggregate_score, recommendation_inputs } = scorecard;
+    const scoredCount = subscores.filter((s) => s.score !== null).length;
+    if (scoredCount === 0)
+        return 0.1; // minimal confidence without evidence
+    // Base: proportion of subscores with data.
+    const coverage = scoredCount / Math.max(subscores.length, 1);
+    // Small penalty when below threshold (more uncertainty in the recommendation).
+    const scorePenalty = aggregate_score !== null && recommendation_inputs.below_threshold ? 0.1 : 0;
+    return clamp01(coverage - scorePenalty);
+}
+function affectedFromScorecard(scorecard) {
+    const keys = scorecard.grouping_keys ?? {};
+    const affected = {};
+    if (keys.route)
+        affected.route = keys.route;
+    if (keys.task_type)
+        affected.task_type = keys.task_type;
+    if (keys.role)
+        affected.role = keys.role;
+    if (keys.provider)
+        affected.provider = keys.provider;
+    if (keys.model)
+        affected.model = keys.model;
+    // Subject-level key as fallback for provider/model scorecards.
+    if (!affected.provider && scorecard.subject === "provider")
+        affected.provider = scorecard.subject_key;
+    if (!affected.model && scorecard.subject === "model")
+        affected.model = scorecard.subject_key;
+    return affected;
+}
+/**
+ * Derive a `ScorecardRecommendationSummary` from a `SolScorecard`.
+ *
+ * This is a pure function: it does not read files, call APIs, or mutate
+ * the scorecard or any runtime state. Its output is advisory by default.
+ */
+function scorecardToRecommendationSummary(scorecard) {
+    const { subscores, raw_metrics, source_refs, recommendation_inputs } = scorecard;
+    const qc = extractQcEvidence(subscores, raw_metrics);
+    const validation = extractValidationEvidence(subscores, raw_metrics);
+    const quality_per_token = extractQptEvidence(subscores);
+    const verdict = deriveVerdict(scorecard, qc);
+    const confidence = confidenceFromScorecard(scorecard);
+    const affected = affectedFromScorecard(scorecard);
+    const clonedSourceRefs = source_refs.map((ref) => ({ ...ref }));
+    return {
+        scorecard_id: scorecard.scorecard_id,
+        subject: scorecard.subject,
+        subject_key: scorecard.subject_key,
+        affected,
+        verdict,
+        aggregate_score: scorecard.aggregate_score,
+        confidence,
+        low_scoring_dimensions: [...recommendation_inputs.low_scoring_dimensions],
+        quality_per_token,
+        qc,
+        validation,
+        source_refs: clonedSourceRefs,
+        intervention_detected: recommendation_inputs.intervention_detected,
+        router_issue_detected: recommendation_inputs.router_issue_detected,
+    };
+}
+/**
+ * Derive recommendation summaries from a set of scorecards.
+ * Returns all summaries; callers filter by verdict as needed.
+ */
+function scorecardsToRecommendationSummaries(scorecards) {
+    return scorecards.map(scorecardToRecommendationSummary);
+}