@lsctech/polaris 0.5.7 → 0.5.9

package/dist/autoresearch/sol-report-renderer.js

@@ -0,0 +1,282 @@
+"use strict";
+/**
+ * SOL SmartDocs Markdown report renderer.
+ *
+ * Renders human-readable SOL evaluation reports from a run-level
+ * evaluation record, the computed scorecard set, and optional QC
+ * follow-up recommendations.
+ *
+ * Reports include:
+ *   - Source references back to raw evidence
+ *   - Per-subject confidence and availability
+ *   - Skipped dimensions with reasons
+ *   - Window / grouping metadata
+ *   - Recommendation inputs for downstream routing advice
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderSolMarkdown = renderSolMarkdown;
+// ──────────────────────────────────────────────
+// Formatting helpers
+// ──────────────────────────────────────────────
+function fmtScore(v) {
+    return v !== null ? v.toFixed(4) : "N/A";
+}
+function fmtBoolean(v) {
+    if (v === null)
+        return "N/A";
+    return v ? "Yes" : "No";
+}
+function fmtList(items) {
+    if (!items || items.length === 0)
+        return "_None_";
+    return items.map((i) => `- ${i}`).join("\n");
+}
+function escapeCell(s) {
+    // Normalize line breaks so table cells remain valid Markdown.
+    return s.replace(/\r?\n/g, " ").replace(/\|/g, "\\|").trim();
+}
+// ──────────────────────────────────────────────
+// Subscore table
+// ──────────────────────────────────────────────
+function renderSubscoreTable(scorecard) {
+    const rows = scorecard.subscores.map((s) => {
+        const value = s.score !== null ? fmtScore(s.score) : "_skipped_";
+        const reason = s.skipped_reason ? ` (${s.skipped_reason})` : "";
+        const detail = s.detail ? ` ${s.detail}` : "";
+        return `| ${s.dimension} | ${value} | ${s.confidence} | ${escapeCell(reason + detail) || "—"} |`;
+    });
+    return [
+        "| Dimension | Score | Confidence | Notes |",
+        "|---|---|---|---|",
+        ...rows,
+    ].join("\n");
+}
+// ──────────────────────────────────────────────
+// Source references
+// ──────────────────────────────────────────────
+function renderSourceRefs(refs) {
+    const unique = new Map();
+    for (const ref of refs) {
+        unique.set(`${ref.kind}|${ref.path}`, ref);
+    }
+    const rows = Array.from(unique.values())
+        .sort((a, b) => a.path.localeCompare(b.path))
+        .map((r) => {
+        const status = r.available ? "available" : `missing${r.unavailable_reason ? `: ${r.unavailable_reason}` : ""}`;
+        return `| ${r.kind} | \`${r.path}\` | ${status} |`;
+    });
+    return [
+        "| Kind | Path | Status |",
+        "|---|---|---|",
+        ...rows,
+    ].join("\n");
+}
+// ──────────────────────────────────────────────
+// Recommendation inputs
+// ──────────────────────────────────────────────
+function renderRecommendationInputs(scorecard) {
+    const i = scorecard.recommendation_inputs;
+    const flags = [];
+    if (i.below_threshold)
+        flags.push("below threshold");
+    if (i.over_token_budget)
+        flags.push("over token budget");
+    if (i.intervention_detected)
+        flags.push("intervention detected");
+    if (i.router_issue_detected)
+        flags.push("router issue detected");
+    if (i.qc_issue_detected)
+        flags.push("QC issue detected");
+    const cells = [
+        scorecard.subject,
+        scorecard.subject_key,
+        flags.length > 0 ? flags.join(", ") : "—",
+        i.low_scoring_dimensions.join(", ") || "—",
+        i.skipped_dimensions.join(", ") || "—",
+    ];
+    return `| ${cells.map(escapeCell).join(" | ")} |`;
+}
+function renderRecommendationInputsHeader() {
+    return [
+        "| Subject | Key | Flags | Low dimensions | Skipped dimensions |",
+        "|---|---|---|---|---|",
+    ].join("\n");
+}
+// ──────────────────────────────────────────────
+// Scorecard detail section
+// ──────────────────────────────────────────────
+function renderScorecardDetail(scorecard) {
+    const lines = [];
+    lines.push(`### ${scorecard.subject}: ${scorecard.subject_key}`);
+    lines.push("");
+    lines.push(`- **Availability:** ${scorecard.availability}${scorecard.availability_reason ? ` — ${scorecard.availability_reason}` : ""}`);
+    lines.push(`- **Aggregate score:** ${fmtScore(scorecard.aggregate_score)} (${scorecard.aggregate_confidence})`);
+    lines.push(`- **Aggregate formula:** \`${scorecard.aggregate_formula_version}\``);
+    lines.push(`- **Window:** run_id=${scorecard.window.run_id ?? "n/a"}, cluster_id=${scorecard.window.cluster_id ?? "n/a"}, sample_count=${scorecard.window.sample_count ?? "n/a"}`);
+    const grouping = Object.entries(scorecard.grouping_keys)
+        .filter(([, v]) => v !== undefined)
+        .map(([k, v]) => `${k}=${v}`)
+        .join(", ");
+    lines.push(`- **Grouping keys:** ${grouping || "—"}`);
+    lines.push("");
+    lines.push(renderSubscoreTable(scorecard));
+    lines.push("");
+    lines.push("**Source references**");
+    lines.push("");
+    lines.push(renderSourceRefs(scorecard.source_refs));
+    lines.push("");
+    return lines.join("\n");
+}
+/**
+ * Render a complete SOL evaluation report as Markdown.
+ */
+function renderSolMarkdown(record, scorecardSet, qcRecommendations) {
+    const allScorecards = [
+        scorecardSet.foreman,
+        ...scorecardSet.workers,
+        ...scorecardSet.providers,
+        ...scorecardSet.models,
+        ...scorecardSet.routing,
+    ];
+    const skipped = allScorecards.flatMap((s) => s.subscores
+        .filter((sub) => sub.score === null && sub.skipped_reason)
+        .map((sub) => ({ subject: s.subject, key: s.subject_key, dimension: sub.dimension, reason: sub.skipped_reason })));
+    const lines = [];
+    lines.push(`# SOL Evaluation Report: ${record.run_id}`);
+    lines.push("");
+    lines.push(`**Generated:** ${record.generated_at}`);
+    lines.push(`**Scored at:** ${record.scored_at}`);
+    lines.push(`**Cluster:** ${record.cluster_id ?? "n/a"}`);
+    lines.push("");
+    // Run-level summary
+    lines.push("## Run summary");
+    lines.push("");
+    lines.push(`| Metric | Value |`);
+    lines.push(`|---|---|`);
+    lines.push(`| Run composite score | ${fmtScore(record.report.run_composite_score)} |`);
+    lines.push(`| Foreman composite | ${fmtScore(record.report.foreman.composite_score)} (${record.report.foreman.composite_confidence}) |`);
+    lines.push(`| Worker count | ${Object.keys(record.report.workers).length} |`);
+    lines.push(`| Scorecards produced | ${allScorecards.length} |`);
+    lines.push("");
+    // Scorecard index
+    lines.push("## Scorecard index");
+    lines.push("");
+    lines.push("| Subject | Key | Aggregate | Confidence | Availability |");
+    lines.push("|---|---|---|---|---|");
+    for (const s of allScorecards) {
+        lines.push(`| ${s.subject} | ${escapeCell(s.subject_key)} | ${fmtScore(s.aggregate_score)} | ${s.aggregate_confidence} | ${s.availability} |`);
+    }
+    lines.push("");
+    // Foreman
+    lines.push("## Foreman");
+    lines.push("");
+    lines.push(renderScorecardDetail(scorecardSet.foreman));
+    // Workers
+    if (scorecardSet.workers.length > 0) {
+        lines.push("## Workers");
+        lines.push("");
+        for (const s of scorecardSet.workers) {
+            lines.push(renderScorecardDetail(s));
+        }
+    }
+    // Providers
+    if (scorecardSet.providers.length > 0) {
+        lines.push("## Providers");
+        lines.push("");
+        for (const s of scorecardSet.providers) {
+            lines.push(renderScorecardDetail(s));
+        }
+    }
+    // Models
+    if (scorecardSet.models.length > 0) {
+        lines.push("## Models");
+        lines.push("");
+        for (const s of scorecardSet.models) {
+            lines.push(renderScorecardDetail(s));
+        }
+    }
+    // Routing
+    if (scorecardSet.routing.length > 0) {
+        lines.push("## Routing decisions");
+        lines.push("");
+        for (const s of scorecardSet.routing) {
+            lines.push(renderScorecardDetail(s));
+        }
+    }
+    // Token efficiency
+    lines.push("## Token efficiency");
+    lines.push("");
+    const qptScorecards = allScorecards.filter((s) => s.subscores.some((sub) => sub.dimension === "quality_per_token"));
+    if (qptScorecards.length === 0) {
+        lines.push("No quality-per-token subscores were computed (token evidence may be missing).");
+    }
+    else {
+        lines.push("| Subject | Key | Quality / token | Confidence |");
+        lines.push("|---|---|---|---|");
+        for (const s of qptScorecards) {
+            const qpt = s.subscores.find((sub) => sub.dimension === "quality_per_token");
+            lines.push(`| ${s.subject} | ${escapeCell(s.subject_key)} | ${fmtScore(qpt?.score ?? null)} | ${qpt?.confidence ?? "none"} |`);
+        }
+    }
+    lines.push("");
+    // QC outcome
+    lines.push("## QC outcome");
+    lines.push("");
+    const foremanQc = scorecardSet.foreman.subscores.find((s) => s.dimension === "qc_repair_loop");
+    if (!foremanQc) {
+        lines.push("No QC repair-loop subscore available.");
+    }
+    else if (foremanQc.score === null) {
+        lines.push(`QC repair-loop dimension skipped: ${foremanQc.skipped_reason ?? "no reason given"}.`);
+    }
+    else {
+        lines.push(`QC repair-loop score: **${fmtScore(foremanQc.score)}** (${foremanQc.confidence})${foremanQc.detail ? ` — ${escapeCell(foremanQc.detail)}` : ""}`);
+    }
+    lines.push("");
+    if (qcRecommendations && qcRecommendations.recommendations.length > 0) {
+        lines.push("### QC follow-up recommendations");
+        lines.push("");
+        for (const r of qcRecommendations.recommendations) {
+            lines.push(`- **[${r.action_type}] ${r.id}** — ${escapeCell(r.proposed_action)} (${(r.confidence * 100).toFixed(1)}% confidence)`);
+            lines.push(`  - Rationale: ${escapeCell(r.rationale)}`);
+        }
+        lines.push("");
+    }
+    // Recommendation inputs
+    lines.push("## Recommendation inputs");
+    lines.push("");
+    lines.push(renderRecommendationInputsHeader());
+    for (const s of allScorecards) {
+        lines.push(renderRecommendationInputs(s));
+    }
+    lines.push("");
+    // Skipped evidence
+    lines.push("## Skipped evidence");
+    lines.push("");
+    if (skipped.length === 0) {
+        lines.push("All dimensions were scored; no evidence was skipped.");
+    }
+    else {
+        lines.push("| Subject | Key | Dimension | Reason |");
+        lines.push("|---|---|---|---|");
+        for (const row of skipped) {
+            lines.push(`| ${row.subject} | ${escapeCell(row.key)} | ${row.dimension} | ${escapeCell(row.reason)} |`);
+        }
+    }
+    lines.push("");
+    // Source references (aggregated)
+    lines.push("## Source references");
+    lines.push("");
+    lines.push(renderSourceRefs(allScorecards.flatMap((s) => s.source_refs)));
+    lines.push("");
+    const markdown = lines.join("\n");
+    return {
+        markdown,
+        summary: {
+            run_id: record.run_id,
+            aggregate_score: record.report.run_composite_score,
+            scorecard_count: allScorecards.length,
+            skipped_dimensions: skipped.length,
+        },
+    };
+}

package/dist/autoresearch/sol-run-health-bridge.js

@@ -0,0 +1,246 @@
+"use strict";
+/**
+ * SOL → run-health bridge.
+ *
+ * Evaluates SOL score reports against operator-configured thresholds and,
+ * when policy enables it, appends run-health symptoms so that Medic can be
+ * engaged even when no worker explicitly wrote symptoms.
+ *
+ * Design rules:
+ *   - SOL is ADVISORY by default. No symptoms are created unless the operator
+ *     sets `sol.thresholds.enabled = true` AND at least one policy flag
+ *     (`createRunHealthReport` or `requireMedic`) is true.
+ *   - Symptoms reference the SOL evidence file as their source; they do NOT
+ *     mutate raw metric artifacts.
+ *   - Each threshold crossing produces exactly one symptom per evaluation.
+ *     Re-evaluating the same run does not duplicate symptoms because the
+ *     symptom id encodes the threshold name.
+ *   - Never throws — threshold evaluation must not block the run.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.detectSolThresholdCrossings = detectSolThresholdCrossings;
+exports.evaluateSolThresholds = evaluateSolThresholds;
+const index_js_1 = require("../run-health/index.js");
+// ── Defaults ─────────────────────────────────────────────────────────────────
+const DEFAULT_LOW_COMPOSITE_SCORE = 0.4;
+const DEFAULT_QC_REPAIR_LOOP_FAILURE_STATUSES = [
+    "max-rounds",
+    "medic-referral",
+    "all-providers-failed",
+];
+const DEFAULT_REPEATED_PROVIDER_FAILURES = 3;
+const DEFAULT_FOREMAN_INTERVENTION_COUNT = 2;
+const DEFAULT_STALE_WRONG_RUN_TELEMETRY = true;
+const DEFAULT_VALIDATION_FAILURES = 2;
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function makeSolSourceActor() {
+    return { role: "sol" };
+}
+function makeSolSymptom(crossing) {
+    return {
+        // Deterministic id: code only (no UUID) so repeated evaluations of the
+        // same run are idempotent — the same threshold crossing produces the
+        // same id, and a caller that deduplicates by id will skip the duplicate.
+        // ponytail: when run-health supports upsert-by-id, switch to that.
+        id: `sol:${crossing.code}`,
+        severity: solThresholdSeverity(crossing.code),
+        code: crossing.code,
+        message: crossing.message,
+        source_actor: makeSolSourceActor(),
+        evidence_refs: crossing.evidenceRefs,
+        occurred_at: new Date().toISOString(),
+    };
+}
+function solThresholdSeverity(code) {
+    switch (code) {
+        case "sol-qc-repair-loop-failure":
+        case "sol-repeated-provider-failures":
+            return "high";
+        case "sol-low-composite-score":
+        case "sol-high-foreman-intervention":
+        case "sol-validation-failures":
+            return "medium";
+        case "sol-stale-wrong-run-telemetry":
+            return "high";
+        default:
+            return "medium";
+    }
+}
+function upsertSolSymptom(runId, clusterId, symptom, repoRoot) {
+    const existing = (0, index_js_1.readRunHealthReport)(runId, repoRoot);
+    if (!existing) {
+        (0, index_js_1.createRunHealthReport)({
+            runId,
+            clusterId,
+            firstSymptom: symptom,
+            sourceActor: makeSolSourceActor(),
+            repoRoot,
+        });
+    }
+    else {
+        // Deduplicate: skip if a symptom with the same id already exists
+        if (existing.symptoms.some((s) => s.id === symptom.id)) {
+            return;
+        }
+        (0, index_js_1.appendSymptom)(runId, symptom, repoRoot);
+    }
+}
+// ── Threshold evaluation ──────────────────────────────────────────────────────
+/**
+ * Detect which thresholds are crossed given the SOL score report.
+ * Returns an array of crossing descriptors without any I/O side effects.
+ */
+function detectSolThresholdCrossings(scoreReport, thresholdsConfig, evidencePaths) {
+    const crossings = [];
+    const lowCompositeThreshold = thresholdsConfig.low_composite_score ?? DEFAULT_LOW_COMPOSITE_SCORE;
+    const qcFailureStatuses = thresholdsConfig.qc_repair_loop_failure_statuses ?? DEFAULT_QC_REPAIR_LOOP_FAILURE_STATUSES;
+    const providerFailureThreshold = thresholdsConfig.repeated_provider_failures ?? DEFAULT_REPEATED_PROVIDER_FAILURES;
+    const interventionThreshold = thresholdsConfig.foreman_intervention_count ?? DEFAULT_FOREMAN_INTERVENTION_COUNT;
+    const checkStaleWrongRun = thresholdsConfig.stale_wrong_run_telemetry ?? DEFAULT_STALE_WRONG_RUN_TELEMETRY;
+    const validationFailureThreshold = thresholdsConfig.validation_failures ?? DEFAULT_VALIDATION_FAILURES;
+    // 1. Low composite score
+    if (scoreReport.run_composite_score !== null &&
+        scoreReport.run_composite_score < lowCompositeThreshold) {
+        crossings.push({
+            code: "sol-low-composite-score",
+            message: `SOL run composite score ${scoreReport.run_composite_score.toFixed(4)} is below ` +
+                `configured threshold ${lowCompositeThreshold} — run health may be degraded.`,
+            evidenceRefs: evidencePaths,
+        });
+    }
+    // 2. QC repair-loop failure state
+    const foremanReport = scoreReport.foreman;
+    const qcRepairLoopDetail = foremanReport.qc_repair_loop?.detail ?? "";
+    for (const status of qcFailureStatuses) {
+        if (qcRepairLoopDetail.includes(`status=${status}`)) {
+            crossings.push({
+                code: "sol-qc-repair-loop-failure",
+                message: `SOL evidence shows QC repair loop reached failure status "${status}" — ` +
+                    `Medic consultation is recommended.`,
+                evidenceRefs: evidencePaths,
+            });
+            break; // One symptom per run for QC repair-loop failures
+        }
+    }
+    // Also detect via qc_repair_loop score == 0
+    if (foremanReport.qc_repair_loop?.score !== null &&
+        foremanReport.qc_repair_loop?.score === 0 &&
+        crossings.every((c) => c.code !== "sol-qc-repair-loop-failure")) {
+        crossings.push({
+            code: "sol-qc-repair-loop-failure",
+            message: `SOL evidence shows QC repair loop score 0.0 (worst outcome) — ` +
+                `Medic consultation is recommended.`,
+            evidenceRefs: evidencePaths,
+        });
+    }
+    // 3. Repeated provider failures — inferred from worker composite scores ≤ 0
+    //    and workers_failed count (available through foreman scores proxy).
+    const failedWorkers = Object.values(scoreReport.workers).filter((w) => w.composite_score !== null && w.composite_score <= 0);
+    if (failedWorkers.length >= providerFailureThreshold) {
+        crossings.push({
+            code: "sol-repeated-provider-failures",
+            message: `SOL evidence shows ${failedWorkers.length} worker(s) with zero composite score ` +
+                `(threshold: ${providerFailureThreshold}) — possible repeated provider failures.`,
+            evidenceRefs: evidencePaths,
+        });
+    }
+    // 4. High Foreman intervention count
+    // intervention.score: 0.0 = user_intervened, 0.5 = foreman_intervened
+    const interventionScore = foremanReport.intervention?.score;
+    if (interventionScore !== null && interventionScore !== undefined) {
+        // pre_analysis dimension uses escalation_events as a proxy
+        // intervention score < 0.5 means foreman_intervened; 0.0 means user_intervened
+        // We use the foreman.pre_analysis detail to extract escalation_events
+        const preAnalysisDetail = foremanReport.pre_analysis?.detail ?? "";
+        const escalationMatch = /escalation_events=(\d+)/.exec(preAnalysisDetail);
+        const escalationEvents = escalationMatch ? parseInt(escalationMatch[1], 10) : 0;
+        if (escalationEvents > interventionThreshold) {
+            crossings.push({
+                code: "sol-high-foreman-intervention",
+                message: `SOL evidence shows ${escalationEvents} escalation event(s) ` +
+                    `(threshold: ${interventionThreshold}) — possible Foreman intervention loop.`,
+                evidenceRefs: evidencePaths,
+            });
+        }
+    }
+    // 5. Stale / wrong-run telemetry
+    // Detected via foreman.dispatch and foreman.duration dimension details
+    if (checkStaleWrongRun) {
+        const dispatchDetail = foremanReport.dispatch?.detail ?? "";
+        const durationDetail = foremanReport.duration?.detail ?? "";
+        const hasRedispatch = /redispatched=\d+\//.exec(dispatchDetail);
+        const redispatchCount = hasRedispatch
+            ? parseInt(/redispatched=(\d+)/.exec(dispatchDetail)?.[1] ?? "0", 10)
+            : 0;
+        // dispatch_epoch >> continue_epoch suggests wrong-run telemetry
+        const dispatchEpochMatch = /dispatch_epoch=(\d+)/.exec(durationDetail);
+        const continueEpochMatch = /continue_epoch=(\d+)/.exec(durationDetail);
+        const dispatchEpoch = dispatchEpochMatch ? parseInt(dispatchEpochMatch[1], 10) : 0;
+        const continueEpoch = continueEpochMatch ? parseInt(continueEpochMatch[1], 10) : 0;
+        const epochOverhead = dispatchEpoch > 0 ? dispatchEpoch - (continueEpoch + 1) : 0;
+        if (redispatchCount >= providerFailureThreshold || epochOverhead >= 3) {
+            crossings.push({
+                code: "sol-stale-wrong-run-telemetry",
+                message: `SOL evidence suggests stale or wrong-run telemetry: ` +
+                    `redispatched=${redispatchCount}, epoch_overhead=${epochOverhead}. ` +
+                    `This pattern is consistent with the POL-509 wrong-run failure mode.`,
+                evidenceRefs: evidencePaths,
+            });
+        }
+    }
+    // 6. Validation failures
+    const validationFailedWorkers = Object.values(scoreReport.workers).filter((w) => w.validation?.score !== null && w.validation?.score === 0);
+    if (validationFailedWorkers.length >= validationFailureThreshold) {
+        crossings.push({
+            code: "sol-validation-failures",
+            message: `SOL evidence shows ${validationFailedWorkers.length} worker(s) with validation failures ` +
+                `(threshold: ${validationFailureThreshold}).`,
+            evidenceRefs: evidencePaths,
+        });
+    }
+    return crossings;
+}
+/**
+ * Evaluate SOL score thresholds and, when policy enables it, append
+ * run-health symptoms for each crossing.
+ *
+ * Advisory only by default. To activate run-health writes:
+ *   `sol.thresholds.enabled = true` AND one of:
+ *   `sol.thresholds.policy.createRunHealthReport = true`
+ *   `sol.thresholds.policy.requireMedic = true`
+ *
+ * Never throws — threshold evaluation must not block the run.
+ */
+function evaluateSolThresholds(params) {
+    const { runId, clusterId, scoreReport, evidencePaths = [], thresholdsConfig, repoRoot } = params;
+    const crossings = detectSolThresholdCrossings(scoreReport, thresholdsConfig, evidencePaths);
+    const policyEnabled = thresholdsConfig.enabled === true;
+    const createReport = thresholdsConfig.policy?.createRunHealthReport === true;
+    const requireMedic = thresholdsConfig.policy?.requireMedic === true;
+    const shouldWrite = policyEnabled && (createReport || requireMedic);
+    if (!shouldWrite || crossings.length === 0) {
+        return { crossings, symptomsAppended: 0, medicRequired: false };
+    }
+    let symptomsAppended = 0;
+    let medicDecisionWritten = false;
+    try {
+        for (const crossing of crossings) {
+            upsertSolSymptom(runId, clusterId, makeSolSymptom(crossing), repoRoot);
+            symptomsAppended++;
+        }
+        // If requireMedic is true, set medic_consult to pending so finalize gate fires.
+        if (requireMedic && crossings.length > 0) {
+            (0, index_js_1.markMedicDecision)(runId, { status: "pending" }, repoRoot);
+            medicDecisionWritten = true;
+        }
+    }
+    catch {
+        // Threshold evaluation must never block the run.
+        // ponytail: surface write errors to telemetry in a future pass
+    }
+    return {
+        crossings,
+        symptomsAppended,
+        medicRequired: medicDecisionWritten,
+    };
+}