@qulib/core 0.7.0 → 0.9.0

package/dist/tools/scoring/confidence.js

@@ -0,0 +1,180 @@
+/**
+ * Release Confidence Aggregator — pure scorer.
+ *
+ * P3 — qulib Confidence Layer v1.
+ *
+ * Pure function: no I/O, no side effects. All I/O (CLI, MCP) lives in the wiring layer.
+ * Algorithm mirrors computeAutomationMaturity's denominator-renormalization math, generalized
+ * to operate over a heterogeneous evidence bundle.
+ *
+ * Score formula:
+ *   confidenceScore = round( Σ score_i * weight_i / Σ weight_i )
+ *   where i ∈ { applicable items with score !== null }
+ *
+ * Excluded from denominator: not_applicable | unknown | score === null items.
+ * Each excluded item is reported in contributions + narrated in honestyNotes.
+ *
+ * Verdict ladder (mirrors agent-summary.ts deriveGate, lifted to fused score):
+ *   any blocking item              → block
+ *   confidenceScore === null       → block  (nothing evaluable; honesty floor)
+ *   confidenceScore < failThreshold → hold
+ *   unknown on a requiredSource OR
+ *   confidenceScore < passThreshold → caution
+ *   else                           → ship
+ */
+import { ReleaseConfidenceSchema, ConfidencePolicySchema } from '../../schemas/confidence.schema.js';
+import { scoreLevel } from './levels.js';
+// ---------------------------------------------------------------------------
+// Default per-source weights (sum over the qulib-native set ≈ 0.90; renormalized at runtime)
+// Rationale grounded in §2.4 of the P3 spec.
+// ---------------------------------------------------------------------------
+const DEFAULT_WEIGHTS = {
+    'live-app-quality': 0.30,
+    'test-automation': 0.22,
+    'api-coverage': 0.15,
+    'accessibility': 0.13,
+    'crawl-coverage': 0.10,
+    'ci-results': 0.10,
+    // External sources reserved for P4 — zero weight until wired:
+    'deploy-metadata': 0.0,
+    'error-telemetry': 0.0,
+    'feature-flags': 0.0,
+    'doc-health': 0.0,
+    'human-approval': 0.0,
+    'agent-evidence': 0.0,
+};
+function resolvePolicy(p) {
+    const base = ConfidencePolicySchema.parse(p ?? {});
+    return {
+        passThreshold: base.passThreshold,
+        failThreshold: base.failThreshold,
+        maxListLength: base.maxListLength,
+        requiredSources: base.requiredSources,
+        weights: base.weights,
+    };
+}
+function resolveWeight(item, policyWeights) {
+    if (policyWeights && item.source in policyWeights) {
+        return policyWeights[item.source];
+    }
+    return item.weight > 0 ? item.weight : (DEFAULT_WEIGHTS[item.source] ?? 0.10);
+}
+function buildHonestyNote(item) {
+    const base = `'${item.source}' source`;
+    if (item.applicability === 'not_applicable') {
+        return `${base} is not applicable${item.reason ? ': ' + item.reason : ' for this subject'}.`;
+    }
+    if (item.applicability === 'unknown') {
+        return `${base} could not produce a reliable score${item.reason ? ': ' + item.reason : ''}.`;
+    }
+    if (item.score === null) {
+        return `${base} ran but returned a null score${item.reason ? ': ' + item.reason : ''}.`;
+    }
+    return `${base} has partial or degraded signal.`;
+}
+/**
+ * Compute the fused Release Confidence result from an evidence bundle.
+ *
+ * Pure function — deterministic over the same input.
+ */
+export function computeReleaseConfidence(input) {
+    const policy = resolvePolicy(input.policy);
+    const now = new Date().toISOString();
+    const limit = policy.maxListLength;
+    // Partition evidence into applicable (score !== null) vs excluded.
+    const applicable = input.evidence.filter((item) => (item.applicability ?? 'applicable') === 'applicable' &&
+        item.score !== null &&
+        !item.blocking);
+    const excluded = input.evidence.filter((item) => (item.applicability ?? 'applicable') !== 'applicable' ||
+        item.score === null);
+    // Blocking items are evaluated separately from the score.
+    const blockingItems = input.evidence.filter((item) => item.blocking);
+    // Compute weighted score over applicable set.
+    let confidenceScore = null;
+    const weightSum = applicable.reduce((s, item) => s + resolveWeight(item, policy.weights), 0);
+    if (weightSum > 0) {
+        const numerator = applicable.reduce((s, item) => s + (item.score ?? 0) * resolveWeight(item, policy.weights), 0);
+        confidenceScore = Math.round(numerator / weightSum);
+    }
+    // Build contributions (all evidence, not just applicable).
+    const contributions = input.evidence.map((item) => {
+        const w = resolveWeight(item, policy.weights);
+        const isApplicableNonNull = (item.applicability ?? 'applicable') === 'applicable' &&
+            item.score !== null &&
+            !item.blocking;
+        return {
+            source: item.source,
+            score: item.score,
+            weight: w,
+            effectiveWeight: isApplicableNonNull && weightSum > 0 ? w / weightSum : 0,
+            applicability: item.applicability ?? 'applicable',
+            blocking: item.blocking ?? false,
+        };
+    });
+    // Determine verdict.
+    let verdict = 'ship';
+    const blockers = [];
+    if (blockingItems.length > 0) {
+        verdict = 'block';
+        for (const b of blockingItems) {
+            blockers.push(`'${b.source}' is a hard blocker${b.reason ? ': ' + b.reason : ''}.`);
+        }
+    }
+    else if (confidenceScore === null) {
+        verdict = 'block';
+        blockers.push('No applicable evidence produced a score — nothing evaluable (honesty floor).');
+    }
+    else if (confidenceScore < policy.failThreshold) {
+        verdict = 'hold';
+    }
+    else {
+        // Check if any required source is 'unknown'.
+        const unknownRequired = input.evidence.filter((item) => policy.requiredSources.includes(item.source) &&
+            (item.applicability ?? 'applicable') === 'unknown');
+        if (unknownRequired.length > 0 || confidenceScore < policy.passThreshold) {
+            verdict = 'caution';
+        }
+    }
+    // Level / label from shared ladder.
+    const { level, label } = scoreLevel(confidenceScore ?? 0);
+    // Honesty notes — one per degraded/excluded source.
+    const honestyNotes = [];
+    for (const item of excluded) {
+        honestyNotes.push(buildHonestyNote(item));
+    }
+    // Also note any blocking items that aren't in the excluded set.
+    for (const item of blockingItems) {
+        if ((item.applicability ?? 'applicable') === 'applicable' && item.score !== null) {
+            honestyNotes.push(`'${item.source}' is a hard blocker${item.reason ? ': ' + item.reason : ''}.`);
+        }
+    }
+    // Top risks — merge evidence across sources, severity-sorted by position.
+    const allRisks = [
+        ...blockingItems.flatMap((item) => item.evidence),
+        ...input.evidence
+            .filter((item) => (item.applicability ?? 'applicable') === 'applicable')
+            .sort((a, b) => (a.score ?? 0) - (b.score ?? 0))
+            .flatMap((item) => item.evidence),
+    ];
+    const topRisks = [...new Set(allRisks)].slice(0, limit);
+    // Recommended next checks — merge and deduplicate.
+    const allRecs = input.evidence.flatMap((item) => item.recommendations ?? []);
+    const recommendedNextChecks = [...new Set(allRecs)].slice(0, limit);
+    const result = {
+        schemaVersion: 1,
+        computedAt: now,
+        subject: input.subject,
+        confidenceScore,
+        verdict,
+        level,
+        label,
+        contributions,
+        topRisks,
+        recommendedNextChecks,
+        honestyNotes: honestyNotes.slice(0, limit),
+        blockers,
+        scoreFormula: 'confidenceScore = round( Σ (score * weight) / Σ weight ) for applicable, non-null, non-blocking evidence only. ' +
+            'not_applicable, unknown, and null-score items are excluded from the denominator but reported in contributions and honestyNotes.',
+    };
+    return ReleaseConfidenceSchema.parse(result);
+}

package/dist/tools/scoring/levels.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Shared score-to-level ladder used by qulib scorers.
+ *
+ * Used by `computeAutomationMaturity` and `computeReleaseConfidence` so
+ * the L1–L5 numeric bands are consistent across all qulib scorers.
+ * Each scorer can provide its own label suffix if needed; this function
+ * provides the canonical L1–L5 numeric thresholds and default labels.
+ *
+ * L1 < 20 | L2 < 40 | L3 < 60 | L4 < 80 | L5 ≥ 80
+ */
+export declare function scoreLevel(overall: number): {
+    level: number;
+    label: string;
+};
+//# sourceMappingURL=levels.d.ts.map

package/dist/tools/scoring/levels.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"levels.d.ts","sourceRoot":"","sources":["../../../src/tools/scoring/levels.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAM5E"}

package/dist/tools/scoring/levels.js

@@ -0,0 +1,21 @@
+/**
+ * Shared score-to-level ladder used by qulib scorers.
+ *
+ * Used by `computeAutomationMaturity` and `computeReleaseConfidence` so
+ * the L1–L5 numeric bands are consistent across all qulib scorers.
+ * Each scorer can provide its own label suffix if needed; this function
+ * provides the canonical L1–L5 numeric thresholds and default labels.
+ *
+ * L1 < 20 | L2 < 40 | L3 < 60 | L4 < 80 | L5 ≥ 80
+ */
+export function scoreLevel(overall) {
+    if (overall < 20)
+        return { level: 1, label: 'L1 — nascent automation' };
+    if (overall < 40)
+        return { level: 2, label: 'L2 — emerging coverage' };
+    if (overall < 60)
+        return { level: 3, label: 'L3 — building maturity' };
+    if (overall < 80)
+        return { level: 4, label: 'L4 — strong automation' };
+    return { level: 5, label: 'L5 — advanced QA automation' };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@qulib/core",
-  "version": "0.7.0",
-  "description": "Qulib — analyze deployed web apps for honest quality gaps (CLI + programmatic API)",
+  "version": "0.9.0",
+  "description": "Qulib — release confidence for deployed web apps. Fuses live-app quality, automation maturity, and API coverage into a single ship/caution/hold/block verdict.",
   "license": "MIT",
   "author": "Tapesh Nagarwal",
   "homepage": "https://github.com/TapeshN/qulib#readme",
@@ -14,11 +14,13 @@
     "url": "https://github.com/TapeshN/qulib/issues"
   },
   "keywords": [
+    "release-confidence",
     "qa",
     "quality",
-    "accessibility",
+    "ship-verdict",
+    "automation-maturity",
     "gap-analysis",
-    "release-confidence",
+    "accessibility",
     "playwright",
     "mcp",
     "ai"
@@ -48,8 +50,12 @@
     "analyze": "tsx src/cli/index.ts analyze",
     "clean": "tsx src/cli/index.ts clean",
     "build": "tsc",
-    "test": "node --import tsx/esm --test src/llm/__tests__/cost-intelligence.test.ts src/llm/__tests__/context-builder.test.ts src/tools/scoring/__tests__/gaps.test.ts src/tools/auth/__tests__/gaps.test.ts src/tools/auth/__tests__/detect.test.ts src/tools/scoring/__tests__/automation-maturity.test.ts src/harness/__tests__/state-manager.test.ts src/telemetry/__tests__/redact-url.test.ts src/cli/__tests__/auth-login.test.ts src/cli/__tests__/cli-version.test.ts src/__tests__/agent-summary.test.ts src/__tests__/cli-agent-summary.test.ts src/__tests__/analyze.storage-state-invalid.test.ts src/__tests__/analyze.fixtures.test.ts",
+    "prepack": "npm run build",
+    "prepublishOnly": "npm run build",
+    "test": "node --import tsx/esm --test src/llm/__tests__/cost-intelligence.test.ts src/llm/__tests__/context-builder.test.ts src/tools/scoring/__tests__/gaps.test.ts src/tools/auth/__tests__/gaps.test.ts src/tools/auth/__tests__/detect.test.ts src/tools/scoring/__tests__/automation-maturity.test.ts src/tools/scoring/__tests__/api-coverage.test.ts src/tools/scoring/__tests__/automation-maturity-with-api.test.ts src/harness/__tests__/state-manager.test.ts src/telemetry/__tests__/redact-url.test.ts src/cli/__tests__/auth-login.test.ts src/cli/__tests__/cli-version.test.ts src/cli/__tests__/bin-shim.test.ts src/cli/__tests__/score-automation.test.ts src/cli/__tests__/scaffold.test.ts src/__tests__/agent-summary.test.ts src/__tests__/cli-agent-summary.test.ts src/__tests__/analyze.storage-state-invalid.test.ts src/__tests__/analyze.fixtures.test.ts src/adapters/__tests__/playwright-adapter.test.ts src/adapters/__tests__/api-adapter.test.ts src/adapters/__tests__/ci-results-adapter.test.ts src/adapters/__tests__/pr-metadata-adapter.test.ts src/adapters/__tests__/validate-specs.test.ts src/tools/repo/__tests__/api-surface.test.ts src/baseline/__tests__/baseline.test.ts evals/runner/__tests__/runner.test.ts evals/judge/__tests__/judge.test.ts src/tools/scoring/__tests__/confidence.test.ts src/tools/scoring/__tests__/confidence-from-qulib.test.ts src/tools/scoring/__tests__/confidence-views.test.ts src/cli/__tests__/confidence.test.ts src/__tests__/notquality-dogfood.test.ts src/cli/__tests__/default-config-fallback.test.ts",
     "test:integration": "node --import tsx/esm --test src/__tests__/analyze.integration.test.ts",
+    "eval": "node --import tsx/esm evals/runner/index.ts",
+    "eval:judge": "node --import tsx/esm evals/judge/eval-judge.ts",
     "smoke": "tsx src/cli/index.ts analyze --url https://example.com --ephemeral",
     "cost-doctor": "tsx src/cli/index.ts cost doctor"
   },
@@ -58,11 +64,13 @@
     "@playwright/test": "^1.44.0",
     "commander": "^12.1.0",
     "fast-glob": "^3.3.2",
+    "js-yaml": "^4.2.0",
+    "typescript": "^5.4.0",
     "zod": "^3.23.0"
   },
   "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
     "@types/node": "^20.0.0",
-    "tsx": "^4.11.0",
-    "typescript": "^5.4.0"
+    "tsx": "^4.11.0"
   }
 }