@ai-sdlc/orchestrator 0.6.0 → 0.10.0

package/dist/sa-scoring/feedback-store.js

@@ -0,0 +1,156 @@
+/**
+ * SA feedback store (RFC-0008 Addendum B §B.8).
+ *
+ * Wraps the `did_feedback_events` table and exposes precision
+ * aggregations used by the feedback flywheel (AISDLC-65 Cκ
+ * calibration, AISDLC-66 phase-weight auto-calibration).
+ *
+ * Signal semantics (§B.8.1):
+ *   accept    — admitted item was correctly scored (true-positive on positive path)
+ *   dismiss   — admitted item should NOT have been (false-positive)
+ *   escalate  — scored too low; should have been ranked higher (false-negative)
+ *   override  — HC_override bypass triggered; feedback auto-emitted
+ */
+// Directional-correctness threshold: score ≥ this counts as "high" for
+// dismiss-signal evaluation (a high-scored item that got dismissed was
+// over-scored, so directionally incorrect).
+const HIGH_SCORE_THRESHOLD = 0.5;
+// ── Store ────────────────────────────────────────────────────────────
+export class SAFeedbackStore {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    record(input) {
+        return this.store.recordDidFeedback({
+            didName: input.didName,
+            issueNumber: input.issueNumber,
+            dimension: input.dimension,
+            signal: input.signal,
+            principal: input.principal,
+            category: input.category,
+            structuralScore: input.structuralScore,
+            llmScore: input.llmScore,
+            compositeScore: input.compositeScore,
+            notes: input.notes,
+        });
+    }
+    list(window = {}) {
+        return this.store.getDidFeedbackEvents({
+            dimension: window.dimension,
+            since: window.since,
+            limit: 5000,
+        });
+    }
+    /**
+     * Directional correctness of the Layer 2 structural score.
+     *
+     *   signal=accept   + structural ≥ 0.5 → correct (high score → accepted)
+     *   signal=accept   + structural < 0.5 → incorrect (low score → accepted anyway, underconfident)
+     *   signal=dismiss  + structural ≥ 0.5 → incorrect (high score → dismissed, overconfident)
+     *   signal=dismiss  + structural < 0.5 → correct (low score → dismissed)
+     *   signal=escalate + structural < 0.5 → correct (low score → escalated, was underscored)
+     *   signal=escalate + structural ≥ 0.5 → incorrect (high score → escalated? unusual)
+     *   signal=override → EXCLUDED (bypass, not a judgement on structural)
+     */
+    structuralPrecision(window = {}) {
+        return this.computeDirectionalPrecision(window, (e) => e.structuralScore);
+    }
+    llmPrecision(window = {}) {
+        return this.computeDirectionalPrecision(window, (e) => e.llmScore);
+    }
+    /**
+     * Categories with the highest false-positive rates — i.e. where
+     * `dismiss` signals dominate `accept`. Phase 3 calibration
+     * (AISDLC-66) should prioritise tuning these.
+     */
+    highFalsePositiveCategories(window = {}, minSampleSize = 3) {
+        const events = this.list(window).filter((e) => e.category);
+        const grouped = new Map();
+        for (const e of events) {
+            const key = e.category;
+            const bucket = grouped.get(key) ?? { accept: 0, dismiss: 0 };
+            if (e.signal === 'accept')
+                bucket.accept++;
+            else if (e.signal === 'dismiss')
+                bucket.dismiss++;
+            grouped.set(key, bucket);
+        }
+        const rows = [];
+        for (const [category, counts] of grouped) {
+            const sampleSize = counts.accept + counts.dismiss;
+            if (sampleSize < minSampleSize)
+                continue;
+            rows.push({
+                category,
+                sampleSize,
+                falsePositiveCount: counts.dismiss,
+                falsePositiveRate: sampleSize === 0 ? 0 : counts.dismiss / sampleSize,
+            });
+        }
+        rows.sort((a, b) => b.falsePositiveRate - a.falsePositiveRate);
+        return rows;
+    }
+    computeDirectionalPrecision(window, scoreOf) {
+        const events = this.list(window);
+        let sampleSize = 0;
+        let correct = 0;
+        for (const e of events) {
+            if (e.signal === 'override')
+                continue;
+            const score = scoreOf(e);
+            if (score === undefined)
+                continue;
+            sampleSize++;
+            const high = score >= HIGH_SCORE_THRESHOLD;
+            const directionallyCorrect = (e.signal === 'accept' && high) ||
+                (e.signal === 'dismiss' && !high) ||
+                (e.signal === 'escalate' && !high);
+            if (directionallyCorrect)
+                correct++;
+        }
+        return {
+            sampleSize,
+            correct,
+            precision: sampleSize === 0 ? 0 : correct / sampleSize,
+        };
+    }
+}
+// ── Signal classification ────────────────────────────────────────────
+const SA_LABELS = Object.freeze({
+    'sa/accept': 'accept',
+    'sa/dismiss': 'dismiss',
+    'sa/escalate': 'escalate',
+});
+/**
+ * Map a GitHub label addition event to a feedback signal, or
+ * undefined if the label isn't one of our SA signals.
+ */
+export function classifyLabel(label) {
+    return SA_LABELS[label.toLowerCase()];
+}
+export const SA_FEEDBACK_LABELS = Object.freeze([
+    'sa/accept',
+    'sa/dismiss',
+    'sa/escalate',
+]);
+/**
+ * Emit an `override` feedback row when an admission score carries an
+ * `override` bypass. Safe no-op when `override` is undefined — callers
+ * can invoke unconditionally.
+ */
+export function recordOverrideFeedback(feedback, override, context) {
+    if (!override)
+        return;
+    feedback.record({
+        didName: context.didName,
+        issueNumber: context.issueNumber,
+        // Override is a bypass of the whole SA score; by convention we
+        // record against SA-1 (the dimension that gates admission).
+        dimension: 'SA-1',
+        signal: 'override',
+        principal: context.principal,
+        notes: override.reason ?? context.reason,
+    });
+}
+//# sourceMappingURL=feedback-store.js.map

package/dist/sa-scoring/index.d.ts

@@ -0,0 +1,71 @@
+/**
+ * SA scoring orchestration (RFC-0008 Addendum B §B.7.3).
+ *
+ * `scoreSoulAlignment(input, deps)` runs the three-layer SA scorer end
+ * to end with phase-aware short-circuits:
+ *
+ *   - Layer 1 hard gate → skips Layer 2/3, SA-1 forced to 0
+ *   - Phase 2a shadow  → all layers computed + persisted, but
+ *                         shadowMode=true signals caller to use
+ *                         label-based soulAlignment in ranking
+ *   - Phase 2b/2c/3    → sa1 replaces label-based soulAlignment
+ *
+ * Persists one `did_scoring_events` row per dimension so operators can
+ * query per-dimension precision by phase.
+ */
+import type { DesignIntentDocument, DesignSystemBinding } from '@ai-sdlc/reference';
+import type { StateStore } from '../state/store.js';
+import { type CompiledDid } from './did-compiler.js';
+import type { DepparseClient } from './depparse-client.js';
+import type { LLMClient } from './layer3-llm.js';
+import { type DeterministicScoringResult } from './layer1-deterministic.js';
+import { type DomainRelevanceResult, type PrincipleCoverageVector } from './layer2-structural.js';
+import { type LLMScoringResult } from './layer3-llm.js';
+import { type PhaseWeights, type SaPhase, type SoulAlignmentResult } from './composite.js';
+export interface ScoreSoulAlignmentInput {
+    issueText: string;
+    did: DesignIntentDocument;
+    dsb?: DesignSystemBinding;
+    phase: SaPhase;
+    calibratedWeights?: PhaseWeights;
+    /** Observed metrics keyed by signal metric name (for Layer 1). */
+    observedMetrics?: Record<string, number>;
+    /** Issue number for audit trail — used when persisting did_scoring_events. */
+    issueNumber?: number;
+}
+export interface ScoreSoulAlignmentDeps {
+    depparse: DepparseClient;
+    llm: LLMClient;
+    stateStore?: StateStore;
+    /** Pre-compiled DID (skip recompilation). */
+    compiledDid?: CompiledDid;
+}
+export interface SoulAlignmentScoringResult {
+    /** Final composite SA-1/SA-2 (respecting shadow mode). */
+    sa1: number;
+    sa2: number;
+    /** Full composite result including per-layer contributions. */
+    composite: SoulAlignmentResult;
+    /** Raw Layer 1 output (useful for preVerifiedSummary). */
+    layer1: DeterministicScoringResult;
+    /** Raw Layer 2 outputs. Always computed (even in shadow mode). */
+    layer2: {
+        domainRelevance: DomainRelevanceResult;
+        principleCoverage: PrincipleCoverageVector;
+    };
+    /** Raw Layer 3 output. Absent when Layer 1 hard-gated (skipped). */
+    layer3?: LLMScoringResult;
+    /** True when Phase 2a — caller must use label-based soulAlignment. */
+    shadowMode: boolean;
+    /** Audit — phase + weights used. */
+    phase: SaPhase;
+    weights: PhaseWeights;
+}
+export declare function scoreSoulAlignment(input: ScoreSoulAlignmentInput, deps: ScoreSoulAlignmentDeps): Promise<SoulAlignmentScoringResult>;
+/**
+ * Resolve the soulAlignment value to feed the admission composite
+ * based on phase + SA result. In shadow mode (2a), returns undefined
+ * so the caller falls back to the label-based heuristic.
+ */
+export declare function resolveSoulAlignmentOverride(result: SoulAlignmentScoringResult): number | undefined;
+//# sourceMappingURL=index.d.ts.map

package/dist/sa-scoring/index.js

@@ -0,0 +1,158 @@
+/**
+ * SA scoring orchestration (RFC-0008 Addendum B §B.7.3).
+ *
+ * `scoreSoulAlignment(input, deps)` runs the three-layer SA scorer end
+ * to end with phase-aware short-circuits:
+ *
+ *   - Layer 1 hard gate → skips Layer 2/3, SA-1 forced to 0
+ *   - Phase 2a shadow  → all layers computed + persisted, but
+ *                         shadowMode=true signals caller to use
+ *                         label-based soulAlignment in ranking
+ *   - Phase 2b/2c/3    → sa1 replaces label-based soulAlignment
+ *
+ * Persists one `did_scoring_events` row per dimension so operators can
+ * query per-dimension precision by phase.
+ */
+import { compileDid } from './did-compiler.js';
+import { runLayer1 } from './layer1-deterministic.js';
+import { computeDomainRelevance, computePrincipleCoverage, } from './layer2-structural.js';
+import { runLayer3 } from './layer3-llm.js';
+import { computeSoulAlignment, } from './composite.js';
+// ── Orchestration ───────────────────────────────────────────────────
+export async function scoreSoulAlignment(input, deps) {
+    const compiled = deps.compiledDid ?? compileDid(input.did);
+    // ── Layer 1 (deterministic) ────────────────────────────────────
+    const layer1 = await runLayer1({
+        issueText: input.issueText,
+        compiled,
+        depparse: deps.depparse,
+        observedMetrics: input.observedMetrics,
+    });
+    // ── Layer 2 (BM25 structural) ──────────────────────────────────
+    // Always compute, even in shadow mode, so precision tracking works.
+    const domainRelevance = computeDomainRelevance(input.issueText, compiled.bm25Corpus);
+    const principleCoverage = computePrincipleCoverage(input.issueText, compiled.principleCorpora);
+    // ── Layer 3 (LLM) ──────────────────────────────────────────────
+    // Skip Layer 3 if hard-gated — SA-1 = 0 short-circuits regardless
+    // of what the LLM might say. Matches §B.7.1 STOP condition.
+    let layer3;
+    if (!layer1.hardGated) {
+        layer3 = await runLayer3({
+            issueText: input.issueText,
+            did: input.did,
+            dsb: input.dsb,
+            preVerifiedSummary: layer1.preVerifiedSummary,
+            llm: deps.llm,
+        });
+    }
+    // ── DSB-derived computable inputs for SA-2 ────────────────────
+    const tokenCompliance = input.dsb?.status?.tokenCompliance?.currentCoverage ?? 0;
+    const catalogHealth = input.dsb?.status?.catalogHealth?.coveragePercent ?? 0;
+    // Count design anti-pattern hits for SA-2 conflict penalty.
+    const coreDesignAntiPatternCount = layer1.designAntiPatternHits.hits.filter((h) => h.identityClass === 'core').length;
+    const evolvingDesignAntiPatternCount = layer1.designAntiPatternHits.hits.filter((h) => h.identityClass === 'evolving').length;
+    // ── Composite ─────────────────────────────────────────────────
+    const composite = computeSoulAlignment({
+        phase: input.phase,
+        calibratedWeights: input.calibratedWeights,
+        sa1: {
+            hardGated: layer1.hardGated,
+            coreViolationCount: layer1.coreViolationCount,
+            evolvingViolationCount: layer1.evolvingViolationCount,
+            domainRelevance: domainRelevance.score,
+            domainIntent: layer3?.domainIntent ?? 0,
+            subtleConflicts: layer3?.subtleConflicts ?? [],
+        },
+        sa2: {
+            tokenCompliance,
+            catalogHealth,
+            principleCoverage: principleCoverage.overallCoverage,
+            principleAlignment: layer3?.principleAlignment ?? 0,
+            coreDesignAntiPatternCount,
+            evolvingDesignAntiPatternCount,
+            subtleDesignConflicts: layer3?.subtleDesignConflicts ?? [],
+        },
+    });
+    // ── Persist to did_scoring_events (one row per dimension) ─────
+    if (deps.stateStore && input.issueNumber !== undefined) {
+        persistScoringEvent(deps.stateStore, {
+            didName: input.did.metadata.name,
+            issueNumber: input.issueNumber,
+            dimension: 'SA-1',
+            phase: input.phase,
+            weights: composite.weights,
+            layer1,
+            layer2: domainRelevance,
+            layer3Sa1: layer3
+                ? {
+                    domainIntent: layer3.domainIntent,
+                    domainIntentConfidence: layer3.domainIntentConfidence,
+                    subtleConflicts: layer3.subtleConflicts,
+                }
+                : undefined,
+            compositeScore: composite.sa1.sa1,
+        });
+        persistScoringEvent(deps.stateStore, {
+            didName: input.did.metadata.name,
+            issueNumber: input.issueNumber,
+            dimension: 'SA-2',
+            phase: input.phase,
+            weights: composite.weights,
+            layer1,
+            layer2: principleCoverage,
+            layer3Sa2: layer3
+                ? {
+                    principleAlignment: layer3.principleAlignment,
+                    principleAlignmentConfidence: layer3.principleAlignmentConfidence,
+                    subtleDesignConflicts: layer3.subtleDesignConflicts,
+                }
+                : undefined,
+            compositeScore: composite.sa2.sa2,
+        });
+    }
+    return {
+        sa1: composite.sa1.sa1,
+        sa2: composite.sa2.sa2,
+        composite,
+        layer1,
+        layer2: { domainRelevance, principleCoverage },
+        layer3,
+        shadowMode: composite.shadowMode,
+        phase: input.phase,
+        weights: composite.weights,
+    };
+}
+function persistScoringEvent(store, input) {
+    const layer1Summary = {
+        hardGated: input.layer1.hardGated,
+        coreViolationCount: input.layer1.coreViolationCount,
+        evolvingViolationCount: input.layer1.evolvingViolationCount,
+        scopeOutOfScopeHits: input.layer1.scopeGate.outOfScopeHits.length,
+        constraintViolations: input.layer1.constraintViolations.violations.length,
+        antiPatternHits: input.layer1.antiPatternHits.hits.length + input.layer1.designAntiPatternHits.hits.length,
+    };
+    const layer3Payload = input.layer3Sa1 ?? input.layer3Sa2;
+    store.recordDidScoringEvent({
+        didName: input.didName,
+        issueNumber: input.issueNumber,
+        saDimension: input.dimension,
+        phase: input.phase,
+        layer1ResultJson: JSON.stringify(layer1Summary),
+        layer2ResultJson: JSON.stringify(input.layer2),
+        layer3ResultJson: layer3Payload ? JSON.stringify(layer3Payload) : undefined,
+        compositeScore: input.compositeScore,
+        phaseWeightsJson: JSON.stringify(input.weights),
+    });
+}
+// ── Admission-composite integration helpers ──────────────────────────
+/**
+ * Resolve the soulAlignment value to feed the admission composite
+ * based on phase + SA result. In shadow mode (2a), returns undefined
+ * so the caller falls back to the label-based heuristic.
+ */
+export function resolveSoulAlignmentOverride(result) {
+    if (result.shadowMode)
+        return undefined;
+    return result.sa1;
+}
+//# sourceMappingURL=index.js.map

package/dist/sa-scoring/layer1-deterministic.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Layer 1 — Deterministic SA scorer (RFC-0008 Addendum B §B.4).
+ *
+ * Consumes a `CompiledDid` + issue text + state store metrics and
+ * produces a fully-deterministic scoring result. Layer 1 never makes
+ * network or LLM calls *except* for the dep-parse sidecar (for
+ * requirement-construction detection).
+ *
+ * A `core` out-of-scope match is a **hard gate** — downstream SA-1
+ * scoring must treat `hardGated=true` as SA-1 = 0.0 regardless of
+ * Layer 2/3 output. Evolving matches surface as warnings but do not
+ * gate admission.
+ */
+import type { CompiledDid } from './did-compiler.js';
+import type { DepparseClient } from './depparse-client.js';
+export interface ScopeGateMatch {
+    label: string;
+    synonym?: string;
+    identityClass: 'core' | 'evolving';
+    matchedText: string;
+}
+export interface ScopeGateResult {
+    inScopeHits: ScopeGateMatch[];
+    outOfScopeHits: ScopeGateMatch[];
+    /** True when a core out-of-scope term matched (hard gate). */
+    hardGated: boolean;
+    /** Warnings for evolving out-of-scope hits (soft gate). */
+    warnings: string[];
+}
+export interface ConstraintViolation {
+    constraintId: string;
+    concept: string;
+    relationship: string;
+    pattern: string;
+    matchedText: string;
+    construction?: string;
+    depPath?: string[];
+    identityClass: 'core' | 'evolving';
+}
+export interface ConstraintViolationResult {
+    violations: ConstraintViolation[];
+    /** True when the depparse sidecar was unavailable (fail-soft skip). */
+    depparseSkipped: boolean;
+}
+export interface AntiPatternHit {
+    id: string;
+    label: string;
+    pattern: string;
+    matchedText: string;
+    identityClass: 'core' | 'evolving';
+    scope: 'product' | 'design-principle' | 'voice' | 'visual';
+    /** Present only for scope='design-principle'. */
+    principleId?: string;
+}
+export interface AntiPatternResult {
+    hits: AntiPatternHit[];
+}
+export type MeasurableSignalStatus = 'pass' | 'fail' | 'missing';
+export interface MeasurableSignalCheck {
+    id: string;
+    metric: string;
+    threshold: number;
+    operator: string;
+    observedValue?: number;
+    status: MeasurableSignalStatus;
+    identityClass: 'core' | 'evolving';
+}
+export interface MeasurableSignalResult {
+    checks: MeasurableSignalCheck[];
+    /** Count of signals with status='fail' on a core-classified signal. */
+    coreFailureCount: number;
+}
+export interface DeterministicScoringResult {
+    scopeGate: ScopeGateResult;
+    constraintViolations: ConstraintViolationResult;
+    antiPatternHits: AntiPatternResult;
+    designAntiPatternHits: AntiPatternResult;
+    measurableSignalChecks: MeasurableSignalResult;
+    /** Shortcut — true when the scope gate (core) fails. */
+    hardGated: boolean;
+    /** Total count of `core` violations across all sources. */
+    coreViolationCount: number;
+    /** Total count of `evolving` violations across all sources. */
+    evolvingViolationCount: number;
+    /** Human-readable pre-verified summary for Layer 3 prompt injection. */
+    preVerifiedSummary: string;
+}
+export declare function checkScopeGate(issueText: string, compiled: CompiledDid): ScopeGateResult;
+export declare function detectConstraintViolations(issueText: string, compiled: CompiledDid, client: DepparseClient): Promise<ConstraintViolationResult>;
+export declare function detectAntiPatterns(issueText: string, compiled: CompiledDid): {
+    product: AntiPatternResult;
+    design: AntiPatternResult;
+};
+export declare function checkMeasurableSignals(observed: Record<string, number>, compiled: CompiledDid): MeasurableSignalResult;
+export interface Layer1Input {
+    issueText: string;
+    compiled: CompiledDid;
+    depparse: DepparseClient;
+    /** Observed metric values keyed by signal metric name. */
+    observedMetrics?: Record<string, number>;
+}
+export declare function runLayer1(input: Layer1Input): Promise<DeterministicScoringResult>;
+interface SummaryInput {
+    scopeGate: ScopeGateResult;
+    constraintViolations: ConstraintViolationResult;
+    product: AntiPatternResult;
+    design: AntiPatternResult;
+    measurable: MeasurableSignalResult;
+    hardGated: boolean;
+    coreViolationCount: number;
+    evolvingViolationCount: number;
+}
+export declare function renderPreVerifiedSummary(input: SummaryInput): string;
+export {};
+//# sourceMappingURL=layer1-deterministic.d.ts.map