@lsctech/polaris 0.5.7 → 0.5.9

package/dist/types/sol-metrics.js

@@ -0,0 +1,108 @@
+"use strict";
+/**
+ * SOL raw metric event contracts.
+ *
+ * Typed contracts for the raw metric events that the SOL scoring pipeline
+ * distinguishes when building evidence and scorecards. These types name the
+ * six metric categories defined in the POL-487 acceptance criteria:
+ *
+ *   1. Provider startup failure
+ *   2. Router fallback
+ *   3. Worker execution failure
+ *   4. Validation failure
+ *   5. QC findings
+ *   6. User / Foreman intervention
+ *
+ * These are read-model types — they normalize the raw telemetry events and
+ * result packet signals into a typed contract that scoring functions consume.
+ * Nothing in this file writes to artifacts or triggers side effects.
+ *
+ * Relationship to SolEvidence:
+ *   SolMetricEvent records are materialized by the evidence loader into the
+ *   SolEvidence structure. The SolEvidence fields (foreman, worker, router,
+ *   qc, intervention) already carry the aggregated versions; these typed
+ *   event contracts let callers reason about individual metric records and
+ *   construct per-scope scorecards.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isProviderStartupFailure = isProviderStartupFailure;
+exports.isRouterFallback = isRouterFallback;
+exports.isWorkerExecutionFailure = isWorkerExecutionFailure;
+exports.isValidationFailure = isValidationFailure;
+exports.isQcFinding = isQcFinding;
+exports.isIntervention = isIntervention;
+exports.summarizeMetricEvents = summarizeMetricEvents;
+// ──────────────────────────────────────────────
+// Type guards
+// ──────────────────────────────────────────────
+function isProviderStartupFailure(e) {
+    return e.category === "provider-startup-failure";
+}
+function isRouterFallback(e) {
+    return e.category === "router-fallback";
+}
+function isWorkerExecutionFailure(e) {
+    return e.category === "worker-execution-failure";
+}
+function isValidationFailure(e) {
+    return e.category === "validation-failure";
+}
+function isQcFinding(e) {
+    return e.category === "qc-finding";
+}
+function isIntervention(e) {
+    return e.category === "user-intervention" || e.category === "foreman-intervention";
+}
+/**
+ * Compute a SolMetricSummary from a list of metric events.
+ */
+function summarizeMetricEvents(events) {
+    let providerStartupFailures = 0;
+    let routerFallbacks = 0;
+    let routerFallbackSuccesses = 0;
+    let workerExecutionFailures = 0;
+    let validationFailures = 0;
+    let qcFindingsTotal = 0;
+    let qcFindingsBlocking = 0;
+    let qcFindingsUnvalidated = 0;
+    let userInterventions = 0;
+    let foremanInterventions = 0;
+    for (const e of events) {
+        if (isProviderStartupFailure(e))
+            providerStartupFailures++;
+        else if (isRouterFallback(e)) {
+            routerFallbacks++;
+            if (e.fallback_succeeded)
+                routerFallbackSuccesses++;
+        }
+        else if (isWorkerExecutionFailure(e))
+            workerExecutionFailures++;
+        else if (isValidationFailure(e))
+            validationFailures++;
+        else if (isQcFinding(e)) {
+            qcFindingsTotal++;
+            if (e.blocking)
+                qcFindingsBlocking++;
+            if (e.unvalidated)
+                qcFindingsUnvalidated++;
+        }
+        else if (isIntervention(e)) {
+            if (e.actor === "user")
+                userInterventions++;
+            else
+                foremanInterventions++;
+        }
+    }
+    return {
+        provider_startup_failures: providerStartupFailures,
+        router_fallbacks: routerFallbacks,
+        router_fallback_successes: routerFallbackSuccesses,
+        worker_execution_failures: workerExecutionFailures,
+        validation_failures: validationFailures,
+        qc_findings_total: qcFindingsTotal,
+        qc_findings_blocking: qcFindingsBlocking,
+        qc_findings_unvalidated: qcFindingsUnvalidated,
+        user_interventions: userInterventions,
+        foreman_interventions: foremanInterventions,
+    };
+}

package/dist/types/sol-scorecard.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * SOL scorecard schema.
+ *
+ * Defines durable, reproducible scorecards for SOL evaluation subjects
+ * (Foreman, workers, providers, models, routing decisions, token efficiency,
+ * QC outcomes, and user/Foreman intervention). Scorecards are distinct from
+ * the run-level SolScoreReport: they are per-scope, carry explicit formula
+ * versions, source references, and recommendation inputs.
+ *
+ * Architecture reference: sol-evaluation-report-architecture.md §Artifact classes
+ *
+ * Design rules:
+ *   - Every scorecard carries a formula_version so consumers can detect drift.
+ *   - source_refs link back to the immutable raw evidence that produced the scorecard.
+ *   - Missing evidence results in SolScorecardAvailability="skipped" with reasons.
+ *   - Scorecards are derived and reproducible; raw evidence always wins on conflict.
+ *   - Existing SolScoreReport (run-level evaluation) remains unchanged.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WORKER_QUALITY_PER_TOKEN_SPEC = exports.FOREMAN_QUALITY_PER_TOKEN_SPEC = exports.WORKER_TOKEN_EFFICIENCY_SPEC = exports.FOREMAN_TOKEN_EFFICIENCY_SPEC = exports.SOL_FORMULA_VERSIONS = void 0;
+exports.buildRecommendationInputs = buildRecommendationInputs;
+exports.computeAggregateScore = computeAggregateScore;
+exports.determineScorecardAvailability = determineScorecardAvailability;
+// ──────────────────────────────────────────────
+// Helpers: formula version constants
+// ──────────────────────────────────────────────
+/**
+ * Named formula version constants.
+ * Use these in subscore and aggregate formula_version fields.
+ */
+exports.SOL_FORMULA_VERSIONS = {
+    TOKEN_EFFICIENCY_V1: "token-efficiency/1.0",
+    QUALITY_PER_TOKEN_V1: "quality-per-token/1.0",
+    COMPOSITE_MEAN_V1: "composite-mean/1.0",
+    VALIDATION_BINARY_V1: "validation-binary/1.0",
+    DISPATCH_RATE_V1: "dispatch-rate/1.0",
+    INTERVENTION_BINARY_V1: "intervention-binary/1.0",
+    QC_REPAIR_LOOP_V1: "qc-repair-loop/1.0",
+    SCOPE_ADHERENCE_V1: "scope-adherence/1.0",
+    STARTUP_FAILURE_RATE_V1: "startup-failure-rate/1.0",
+    QUOTA_EXHAUSTION_RATE_V1: "quota-exhaustion-rate/1.0",
+    FALLBACK_FREQUENCY_V1: "fallback-frequency/1.0",
+    ROLE_SUITABILITY_V1: "role-suitability/1.0",
+    RUNTIME_PROXY_V1: "runtime-proxy/1.0",
+    QUALITY_OUTCOMES_V1: "quality-outcomes/1.0",
+    ROUTE_SELECTED_V1: "route-selected/1.0",
+    ROUTE_CANDIDATES_V1: "route-candidates/1.0",
+    ROUTE_FALLBACK_PATH_V1: "route-fallback-path/1.0",
+    ROUTE_OUTCOME_QUALITY_V1: "route-outcome-quality/1.0",
+    ROUTE_TOKEN_BURN_V1: "route-token-burn/1.0",
+    ROUTE_QC_RESULT_V1: "route-qc-result/1.0",
+    ROUTE_VALIDATION_RESULT_V1: "route-validation-result/1.0",
+    ROUTE_CONFIRMED_SIGNAL_V1: "route-confirmed-signal/1.0",
+};
+/**
+ * Token efficiency formula spec for the Foreman subject (v1.0).
+ * Budget = 150k tokens; 0.0 at 300k tokens.
+ */
+exports.FOREMAN_TOKEN_EFFICIENCY_SPEC = {
+    formula_version: "token-efficiency/1.0",
+    budget: 150_000,
+    max_penalized: 300_000,
+    source_event_type: "bootstrap-context-size",
+};
+/**
+ * Token efficiency formula spec for the Worker subject (v1.0).
+ * Budget = 200k tokens; 0.0 at 500k tokens.
+ */
+exports.WORKER_TOKEN_EFFICIENCY_SPEC = {
+    formula_version: "token-efficiency/1.0",
+    budget: 200_000,
+    max_penalized: 500_000,
+    source_event_type: "worker-heartbeat-tokens",
+};
+/**
+ * Quality-per-token formula spec for run-level (Foreman) subject (v1.0).
+ */
+exports.FOREMAN_QUALITY_PER_TOKEN_SPEC = {
+    formula_version: "quality-per-token/1.0",
+    budget_denominator: 150_000,
+    subject: "foreman",
+};
+/**
+ * Quality-per-token formula spec for per-child (Worker) subject (v1.0).
+ */
+exports.WORKER_QUALITY_PER_TOKEN_SPEC = {
+    formula_version: "quality-per-token/1.0",
+    budget_denominator: 200_000,
+    subject: "worker",
+};
+// ──────────────────────────────────────────────
+// Helpers: scorecard construction utilities
+// ──────────────────────────────────────────────
+/**
+ * Build SolRecommendationInputs from a list of subscores and raw metrics.
+ * Applies the 0.6 threshold for below_threshold detection.
+ */
+function buildRecommendationInputs(subscores, rawMetrics, aggregateScore) {
+    const THRESHOLD = 0.6;
+    const lowScoringDimensions = subscores
+        .filter((s) => s.score !== null && s.score < 0.5)
+        .map((s) => s.dimension);
+    const skippedDimensions = subscores
+        .filter((s) => s.score === null)
+        .map((s) => s.dimension);
+    return {
+        below_threshold: aggregateScore !== null && aggregateScore < THRESHOLD,
+        low_scoring_dimensions: lowScoringDimensions,
+        skipped_dimensions: skippedDimensions,
+        over_token_budget: (rawMetrics.max_bootstrap_tokens !== null &&
+            rawMetrics.max_bootstrap_tokens > exports.FOREMAN_TOKEN_EFFICIENCY_SPEC.budget) ||
+            (rawMetrics.worker_tokens_used !== null &&
+                rawMetrics.worker_tokens_used > exports.WORKER_TOKEN_EFFICIENCY_SPEC.budget),
+        intervention_detected: rawMetrics.user_intervened === true || rawMetrics.foreman_intervened === true,
+        router_issue_detected: rawMetrics.router_fallback_used === true || rawMetrics.router_exhausted === true,
+        qc_issue_detected: (rawMetrics.qc_blocking_findings !== null && rawMetrics.qc_blocking_findings > 0) ||
+            rawMetrics.qc_repair_loop_status === "all-providers-failed" ||
+            rawMetrics.qc_repair_loop_status === "operator-review",
+        notes: [],
+    };
+}
+/**
+ * Compute aggregate score from subscores (simple mean of non-null scores).
+ * Returns null when all subscores are skipped.
+ */
+function computeAggregateScore(subscores) {
+    const scored = subscores.filter((s) => s.score !== null);
+    if (scored.length === 0)
+        return null;
+    const mean = scored.reduce((sum, s) => sum + s.score, 0) / scored.length;
+    return Number(mean.toFixed(4));
+}
+/**
+ * Determine scorecard availability from subscores.
+ * "complete" when all subscores have scores; "partial" when some are skipped;
+ * "skipped" when all are skipped; "unavailable" when subscores is empty.
+ */
+function determineScorecardAvailability(subscores) {
+    if (subscores.length === 0)
+        return "unavailable";
+    const scored = subscores.filter((s) => s.score !== null).length;
+    if (scored === 0)
+        return "skipped";
+    if (scored < subscores.length)
+        return "partial";
+    return "complete";
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lsctech/polaris",
-  "version": "0.5.7",
+  "version": "0.5.9",
   "description": "Polaris — standalone taskchain orchestration framework for governed AI agent workflows",
   "bin": {
     "polaris": "dist/cli/index.js",