@ai-sdlc/orchestrator 0.5.0 → 0.9.0

package/dist/sa-scoring/exemplar-bank.d.ts

@@ -0,0 +1,78 @@
+/**
+ * SA Exemplar Bank loader (RFC-0008 §B.6.4, §B.10.2).
+ *
+ * Loads labeled SA-scoring exemplars from `.ai-sdlc/sa-exemplars.yaml`
+ * for Phase 2a shadow-mode precision tracking and Phase 2b progression
+ * gating. A Phase-2b-ready bank needs ≥ 5 exemplars including at least
+ * one true-positive AND one false-positive per dimension — without
+ * precision evidence on both sides, the structural layer can't be
+ * trusted to contribute to ranking.
+ */
+import type { SaDimension } from '../state/types.js';
+export type ExemplarType = 'true-positive' | 'false-positive' | 'true-negative' | 'false-negative';
+export interface ExemplarIssue {
+    title: string;
+    body: string;
+}
+export interface Layer1Expected {
+    hardGated?: boolean;
+    coreViolationCount?: number;
+    evolvingViolationCount?: number;
+    scopeGate?: Record<string, unknown>;
+    constraintViolations?: Record<string, unknown>;
+    antiPatternHits?: Record<string, unknown>;
+}
+export interface Layer2Expected {
+    domainRelevance?: number;
+    overallCoverage?: number;
+    [key: string]: unknown;
+}
+export interface Layer3Expected {
+    domainIntent?: number;
+    principleAlignment?: number;
+    reasoning?: string;
+    [key: string]: unknown;
+}
+export interface SaExemplar {
+    id: string;
+    dimension: SaDimension;
+    type: ExemplarType;
+    issue: ExemplarIssue;
+    layer1Expected?: Layer1Expected;
+    layer2Expected?: Layer2Expected;
+    layer3Expected?: Layer3Expected;
+    verdict: string;
+    principle?: string;
+    notes?: string;
+}
+export interface SaExemplarBank {
+    sa1: SaExemplar[];
+    sa2: SaExemplar[];
+}
+export declare const EMPTY_BANK: SaExemplarBank;
+/**
+ * Load and partition exemplars by dimension. Returns an empty bank
+ * when the file is missing — Phase 2a can run without any exemplars;
+ * `validatePhase2bExemplars` gates progression.
+ */
+export declare function loadExemplarBank(filePath: string): SaExemplarBank;
+export interface ExemplarGap {
+    dimension: SaDimension | 'overall';
+    reason: string;
+}
+export interface ExemplarReadinessResult {
+    ready: boolean;
+    gaps: ExemplarGap[];
+}
+export declare const MIN_TOTAL_EXEMPLARS = 5;
+export declare function validatePhase2bExemplars(bank: SaExemplarBank): ExemplarReadinessResult;
+export interface LayerPrecision {
+    truePositives: number;
+    falsePositives: number;
+    trueNegatives: number;
+    falseNegatives: number;
+    precision: number;
+    recall: number;
+}
+export declare function computeLayerPrecision(exemplars: readonly SaExemplar[]): LayerPrecision;
+//# sourceMappingURL=exemplar-bank.d.ts.map

package/dist/sa-scoring/exemplar-bank.js

@@ -0,0 +1,154 @@
+/**
+ * SA Exemplar Bank loader (RFC-0008 §B.6.4, §B.10.2).
+ *
+ * Loads labeled SA-scoring exemplars from `.ai-sdlc/sa-exemplars.yaml`
+ * for Phase 2a shadow-mode precision tracking and Phase 2b progression
+ * gating. A Phase-2b-ready bank needs ≥ 5 exemplars including at least
+ * one true-positive AND one false-positive per dimension — without
+ * precision evidence on both sides, the structural layer can't be
+ * trusted to contribute to ranking.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { parse as parseYaml } from 'yaml';
+// ── Loader ──────────────────────────────────────────────────────────
+export const EMPTY_BANK = Object.freeze({
+    sa1: [],
+    sa2: [],
+});
+/**
+ * Load and partition exemplars by dimension. Returns an empty bank
+ * when the file is missing — Phase 2a can run without any exemplars;
+ * `validatePhase2bExemplars` gates progression.
+ */
+export function loadExemplarBank(filePath) {
+    if (!existsSync(filePath))
+        return { sa1: [], sa2: [] };
+    const raw = readFileSync(filePath, 'utf-8');
+    let doc;
+    try {
+        doc = parseYaml(raw);
+    }
+    catch (err) {
+        throw new Error(`Failed to parse SA exemplar bank at ${filePath}: ${err.message}`);
+    }
+    if (!doc || typeof doc !== 'object' || !('exemplars' in doc)) {
+        throw new Error(`SA exemplar bank ${filePath} must contain a top-level "exemplars" array`);
+    }
+    const list = doc.exemplars;
+    if (!Array.isArray(list)) {
+        throw new Error(`SA exemplar bank ${filePath}: "exemplars" must be an array`);
+    }
+    const sa1 = [];
+    const sa2 = [];
+    for (const entry of list) {
+        const ex = validateExemplar(entry);
+        if (ex.dimension === 'SA-1')
+            sa1.push(ex);
+        else
+            sa2.push(ex);
+    }
+    return { sa1, sa2 };
+}
+function validateExemplar(entry) {
+    if (!entry || typeof entry !== 'object') {
+        throw new Error(`Exemplar must be an object`);
+    }
+    const e = entry;
+    const id = requireString(e, 'id');
+    const dimension = requireEnum(e, 'dimension', ['SA-1', 'SA-2']);
+    const type = requireEnum(e, 'type', [
+        'true-positive',
+        'false-positive',
+        'true-negative',
+        'false-negative',
+    ]);
+    const issueVal = e.issue;
+    if (!issueVal || typeof issueVal !== 'object') {
+        throw new Error(`Exemplar "${id}" missing required "issue" object`);
+    }
+    const issueObj = issueVal;
+    const issue = {
+        title: requireString(issueObj, 'title'),
+        body: requireString(issueObj, 'body'),
+    };
+    const verdict = requireString(e, 'verdict');
+    return {
+        id,
+        dimension,
+        type,
+        issue,
+        layer1Expected: e.layer1Expected,
+        layer2Expected: e.layer2Expected,
+        layer3Expected: e.layer3Expected,
+        verdict,
+        principle: typeof e.principle === 'string' ? e.principle : undefined,
+        notes: typeof e.notes === 'string' ? e.notes : undefined,
+    };
+}
+function requireString(obj, key) {
+    const v = obj[key];
+    if (typeof v !== 'string' || v.length === 0) {
+        throw new Error(`Field "${key}" must be a non-empty string`);
+    }
+    return v;
+}
+function requireEnum(obj, key, options) {
+    const v = obj[key];
+    if (typeof v !== 'string' || !options.includes(v)) {
+        throw new Error(`Field "${key}" must be one of: ${options.join(', ')}`);
+    }
+    return v;
+}
+export const MIN_TOTAL_EXEMPLARS = 5;
+export function validatePhase2bExemplars(bank) {
+    const gaps = [];
+    const total = bank.sa1.length + bank.sa2.length;
+    if (total < MIN_TOTAL_EXEMPLARS) {
+        gaps.push({
+            dimension: 'overall',
+            reason: `Need ≥${MIN_TOTAL_EXEMPLARS} exemplars total (have ${total})`,
+        });
+    }
+    for (const dimension of ['SA-1', 'SA-2']) {
+        const slice = dimension === 'SA-1' ? bank.sa1 : bank.sa2;
+        if (slice.length === 0) {
+            gaps.push({
+                dimension,
+                reason: `Need ≥1 exemplar for ${dimension}`,
+            });
+            continue;
+        }
+        const hasTP = slice.some((e) => e.type === 'true-positive');
+        const hasFP = slice.some((e) => e.type === 'false-positive');
+        if (!hasTP) {
+            gaps.push({
+                dimension,
+                reason: `${dimension} needs ≥1 true-positive exemplar`,
+            });
+        }
+        if (!hasFP) {
+            gaps.push({
+                dimension,
+                reason: `${dimension} needs ≥1 false-positive exemplar (regression tracking)`,
+            });
+        }
+    }
+    return { ready: gaps.length === 0, gaps };
+}
+export function computeLayerPrecision(exemplars) {
+    const tp = exemplars.filter((e) => e.type === 'true-positive').length;
+    const fp = exemplars.filter((e) => e.type === 'false-positive').length;
+    const tn = exemplars.filter((e) => e.type === 'true-negative').length;
+    const fn = exemplars.filter((e) => e.type === 'false-negative').length;
+    const precision = tp + fp === 0 ? 0 : tp / (tp + fp);
+    const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
+    return {
+        truePositives: tp,
+        falsePositives: fp,
+        trueNegatives: tn,
+        falseNegatives: fn,
+        precision,
+        recall,
+    };
+}
+//# sourceMappingURL=exemplar-bank.js.map

package/dist/sa-scoring/feedback-store.d.ts

@@ -0,0 +1,100 @@
+/**
+ * SA feedback store (RFC-0008 Addendum B §B.8).
+ *
+ * Wraps the `did_feedback_events` table and exposes precision
+ * aggregations used by the feedback flywheel (AISDLC-65 Cκ
+ * calibration, AISDLC-66 phase-weight auto-calibration).
+ *
+ * Signal semantics (§B.8.1):
+ *   accept    — admitted item was correctly scored (true-positive on positive path)
+ *   dismiss   — admitted item should NOT have been (false-positive)
+ *   escalate  — scored too low; should have been ranked higher (false-negative)
+ *   override  — HC_override bypass triggered; feedback auto-emitted
+ */
+import type { StateStore } from '../state/store.js';
+import type { DidFeedbackEventRecord, FeedbackSignal, SaDimension } from '../state/types.js';
+export interface RecordFeedbackInput {
+    didName: string;
+    issueNumber: number;
+    dimension: SaDimension;
+    signal: FeedbackSignal;
+    principal?: string;
+    /** Optional category label — e.g. product/design/engineering pillar, or custom. */
+    category?: string;
+    /** Layer 2 structural score at admission time (for precision tracking). */
+    structuralScore?: number;
+    /** Layer 3 LLM score at admission time. */
+    llmScore?: number;
+    /** Composite SA score. */
+    compositeScore?: number;
+    notes?: string;
+}
+export interface PrecisionWindow {
+    /** ISO timestamp — filter feedback events newer than this. */
+    since?: string;
+    /** Filter to one dimension. */
+    dimension?: SaDimension;
+}
+export interface PrecisionResult {
+    /** Sample size after filters. */
+    sampleSize: number;
+    /** Directionally-correct count (accept/escalate when score was high, dismiss when score was low). */
+    correct: number;
+    /** Precision = correct / sampleSize. 0 when sampleSize=0. */
+    precision: number;
+}
+export interface CategoryFalsePositive {
+    category: string;
+    sampleSize: number;
+    falsePositiveCount: number;
+    falsePositiveRate: number;
+}
+export declare class SAFeedbackStore {
+    private readonly store;
+    constructor(store: StateStore);
+    record(input: RecordFeedbackInput): number;
+    list(window?: PrecisionWindow): DidFeedbackEventRecord[];
+    /**
+     * Directional correctness of the Layer 2 structural score.
+     *
+     *   signal=accept   + structural ≥ 0.5 → correct (high score → accepted)
+     *   signal=accept   + structural < 0.5 → incorrect (low score → accepted anyway, underconfident)
+     *   signal=dismiss  + structural ≥ 0.5 → incorrect (high score → dismissed, overconfident)
+     *   signal=dismiss  + structural < 0.5 → correct (low score → dismissed)
+     *   signal=escalate + structural < 0.5 → correct (low score → escalated, was underscored)
+     *   signal=escalate + structural ≥ 0.5 → incorrect (high score → escalated? unusual)
+     *   signal=override → EXCLUDED (bypass, not a judgement on structural)
+     */
+    structuralPrecision(window?: PrecisionWindow): PrecisionResult;
+    llmPrecision(window?: PrecisionWindow): PrecisionResult;
+    /**
+     * Categories with the highest false-positive rates — i.e. where
+     * `dismiss` signals dominate `accept`. Phase 3 calibration
+     * (AISDLC-66) should prioritise tuning these.
+     */
+    highFalsePositiveCategories(window?: PrecisionWindow, minSampleSize?: number): CategoryFalsePositive[];
+    private computeDirectionalPrecision;
+}
+/**
+ * Map a GitHub label addition event to a feedback signal, or
+ * undefined if the label isn't one of our SA signals.
+ */
+export declare function classifyLabel(label: string): FeedbackSignal | undefined;
+export declare const SA_FEEDBACK_LABELS: readonly string[];
+export interface OverrideFeedbackInput {
+    didName: string;
+    issueNumber: number;
+    /** The override reason from `PriorityScore.override.reason`. */
+    reason?: string;
+    /** Principal who invoked the override, if known. */
+    principal?: string;
+}
+/**
+ * Emit an `override` feedback row when an admission score carries an
+ * `override` bypass. Safe no-op when `override` is undefined — callers
+ * can invoke unconditionally.
+ */
+export declare function recordOverrideFeedback(feedback: SAFeedbackStore, override: {
+    reason?: string;
+} | undefined, context: OverrideFeedbackInput): void;
+//# sourceMappingURL=feedback-store.d.ts.map

package/dist/sa-scoring/feedback-store.js

@@ -0,0 +1,156 @@
+/**
+ * SA feedback store (RFC-0008 Addendum B §B.8).
+ *
+ * Wraps the `did_feedback_events` table and exposes precision
+ * aggregations used by the feedback flywheel (AISDLC-65 Cκ
+ * calibration, AISDLC-66 phase-weight auto-calibration).
+ *
+ * Signal semantics (§B.8.1):
+ *   accept    — admitted item was correctly scored (true-positive on positive path)
+ *   dismiss   — admitted item should NOT have been (false-positive)
+ *   escalate  — scored too low; should have been ranked higher (false-negative)
+ *   override  — HC_override bypass triggered; feedback auto-emitted
+ */
+// Directional-correctness threshold: score ≥ this counts as "high" for
+// dismiss-signal evaluation (a high-scored item that got dismissed was
+// over-scored, so directionally incorrect).
+const HIGH_SCORE_THRESHOLD = 0.5;
+// ── Store ────────────────────────────────────────────────────────────
+export class SAFeedbackStore {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    record(input) {
+        return this.store.recordDidFeedback({
+            didName: input.didName,
+            issueNumber: input.issueNumber,
+            dimension: input.dimension,
+            signal: input.signal,
+            principal: input.principal,
+            category: input.category,
+            structuralScore: input.structuralScore,
+            llmScore: input.llmScore,
+            compositeScore: input.compositeScore,
+            notes: input.notes,
+        });
+    }
+    list(window = {}) {
+        return this.store.getDidFeedbackEvents({
+            dimension: window.dimension,
+            since: window.since,
+            limit: 5000,
+        });
+    }
+    /**
+     * Directional correctness of the Layer 2 structural score.
+     *
+     *   signal=accept   + structural ≥ 0.5 → correct (high score → accepted)
+     *   signal=accept   + structural < 0.5 → incorrect (low score → accepted anyway, underconfident)
+     *   signal=dismiss  + structural ≥ 0.5 → incorrect (high score → dismissed, overconfident)
+     *   signal=dismiss  + structural < 0.5 → correct (low score → dismissed)
+     *   signal=escalate + structural < 0.5 → correct (low score → escalated, was underscored)
+     *   signal=escalate + structural ≥ 0.5 → incorrect (high score → escalated? unusual)
+     *   signal=override → EXCLUDED (bypass, not a judgement on structural)
+     */
+    structuralPrecision(window = {}) {
+        return this.computeDirectionalPrecision(window, (e) => e.structuralScore);
+    }
+    llmPrecision(window = {}) {
+        return this.computeDirectionalPrecision(window, (e) => e.llmScore);
+    }
+    /**
+     * Categories with the highest false-positive rates — i.e. where
+     * `dismiss` signals dominate `accept`. Phase 3 calibration
+     * (AISDLC-66) should prioritise tuning these.
+     */
+    highFalsePositiveCategories(window = {}, minSampleSize = 3) {
+        const events = this.list(window).filter((e) => e.category);
+        const grouped = new Map();
+        for (const e of events) {
+            const key = e.category;
+            const bucket = grouped.get(key) ?? { accept: 0, dismiss: 0 };
+            if (e.signal === 'accept')
+                bucket.accept++;
+            else if (e.signal === 'dismiss')
+                bucket.dismiss++;
+            grouped.set(key, bucket);
+        }
+        const rows = [];
+        for (const [category, counts] of grouped) {
+            const sampleSize = counts.accept + counts.dismiss;
+            if (sampleSize < minSampleSize)
+                continue;
+            rows.push({
+                category,
+                sampleSize,
+                falsePositiveCount: counts.dismiss,
+                falsePositiveRate: sampleSize === 0 ? 0 : counts.dismiss / sampleSize,
+            });
+        }
+        rows.sort((a, b) => b.falsePositiveRate - a.falsePositiveRate);
+        return rows;
+    }
+    computeDirectionalPrecision(window, scoreOf) {
+        const events = this.list(window);
+        let sampleSize = 0;
+        let correct = 0;
+        for (const e of events) {
+            if (e.signal === 'override')
+                continue;
+            const score = scoreOf(e);
+            if (score === undefined)
+                continue;
+            sampleSize++;
+            const high = score >= HIGH_SCORE_THRESHOLD;
+            const directionallyCorrect = (e.signal === 'accept' && high) ||
+                (e.signal === 'dismiss' && !high) ||
+                (e.signal === 'escalate' && !high);
+            if (directionallyCorrect)
+                correct++;
+        }
+        return {
+            sampleSize,
+            correct,
+            precision: sampleSize === 0 ? 0 : correct / sampleSize,
+        };
+    }
+}
+// ── Signal classification ────────────────────────────────────────────
+const SA_LABELS = Object.freeze({
+    'sa/accept': 'accept',
+    'sa/dismiss': 'dismiss',
+    'sa/escalate': 'escalate',
+});
+/**
+ * Map a GitHub label addition event to a feedback signal, or
+ * undefined if the label isn't one of our SA signals.
+ */
+export function classifyLabel(label) {
+    return SA_LABELS[label.toLowerCase()];
+}
+export const SA_FEEDBACK_LABELS = Object.freeze([
+    'sa/accept',
+    'sa/dismiss',
+    'sa/escalate',
+]);
+/**
+ * Emit an `override` feedback row when an admission score carries an
+ * `override` bypass. Safe no-op when `override` is undefined — callers
+ * can invoke unconditionally.
+ */
+export function recordOverrideFeedback(feedback, override, context) {
+    if (!override)
+        return;
+    feedback.record({
+        didName: context.didName,
+        issueNumber: context.issueNumber,
+        // Override is a bypass of the whole SA score; by convention we
+        // record against SA-1 (the dimension that gates admission).
+        dimension: 'SA-1',
+        signal: 'override',
+        principal: context.principal,
+        notes: override.reason ?? context.reason,
+    });
+}
+//# sourceMappingURL=feedback-store.js.map

package/dist/sa-scoring/index.d.ts

@@ -0,0 +1,71 @@
+/**
+ * SA scoring orchestration (RFC-0008 Addendum B §B.7.3).
+ *
+ * `scoreSoulAlignment(input, deps)` runs the three-layer SA scorer end
+ * to end with phase-aware short-circuits:
+ *
+ *   - Layer 1 hard gate → skips Layer 2/3, SA-1 forced to 0
+ *   - Phase 2a shadow  → all layers computed + persisted, but
+ *                         shadowMode=true signals caller to use
+ *                         label-based soulAlignment in ranking
+ *   - Phase 2b/2c/3    → sa1 replaces label-based soulAlignment
+ *
+ * Persists one `did_scoring_events` row per dimension so operators can
+ * query per-dimension precision by phase.
+ */
+import type { DesignIntentDocument, DesignSystemBinding } from '@ai-sdlc/reference';
+import type { StateStore } from '../state/store.js';
+import { type CompiledDid } from './did-compiler.js';
+import type { DepparseClient } from './depparse-client.js';
+import type { LLMClient } from './layer3-llm.js';
+import { type DeterministicScoringResult } from './layer1-deterministic.js';
+import { type DomainRelevanceResult, type PrincipleCoverageVector } from './layer2-structural.js';
+import { type LLMScoringResult } from './layer3-llm.js';
+import { type PhaseWeights, type SaPhase, type SoulAlignmentResult } from './composite.js';
+export interface ScoreSoulAlignmentInput {
+    issueText: string;
+    did: DesignIntentDocument;
+    dsb?: DesignSystemBinding;
+    phase: SaPhase;
+    calibratedWeights?: PhaseWeights;
+    /** Observed metrics keyed by signal metric name (for Layer 1). */
+    observedMetrics?: Record<string, number>;
+    /** Issue number for audit trail — used when persisting did_scoring_events. */
+    issueNumber?: number;
+}
+export interface ScoreSoulAlignmentDeps {
+    depparse: DepparseClient;
+    llm: LLMClient;
+    stateStore?: StateStore;
+    /** Pre-compiled DID (skip recompilation). */
+    compiledDid?: CompiledDid;
+}
+export interface SoulAlignmentScoringResult {
+    /** Final composite SA-1/SA-2 (respecting shadow mode). */
+    sa1: number;
+    sa2: number;
+    /** Full composite result including per-layer contributions. */
+    composite: SoulAlignmentResult;
+    /** Raw Layer 1 output (useful for preVerifiedSummary). */
+    layer1: DeterministicScoringResult;
+    /** Raw Layer 2 outputs. Always computed (even in shadow mode). */
+    layer2: {
+        domainRelevance: DomainRelevanceResult;
+        principleCoverage: PrincipleCoverageVector;
+    };
+    /** Raw Layer 3 output. Absent when Layer 1 hard-gated (skipped). */
+    layer3?: LLMScoringResult;
+    /** True when Phase 2a — caller must use label-based soulAlignment. */
+    shadowMode: boolean;
+    /** Audit — phase + weights used. */
+    phase: SaPhase;
+    weights: PhaseWeights;
+}
+export declare function scoreSoulAlignment(input: ScoreSoulAlignmentInput, deps: ScoreSoulAlignmentDeps): Promise<SoulAlignmentScoringResult>;
+/**
+ * Resolve the soulAlignment value to feed the admission composite
+ * based on phase + SA result. In shadow mode (2a), returns undefined
+ * so the caller falls back to the label-based heuristic.
+ */
+export declare function resolveSoulAlignmentOverride(result: SoulAlignmentScoringResult): number | undefined;
+//# sourceMappingURL=index.d.ts.map