@ai-sdlc/orchestrator 0.6.0 → 0.10.0

package/dist/sa-scoring/drift-monitor.d.ts

@@ -0,0 +1,84 @@
+/**
+ * SA Drift Monitor (RFC-0008 Addendum B §B.9.2).
+ *
+ * Periodically inspects `did_scoring_events` and triggers a
+ * `SoulDriftDetected` event when SA-1 or SA-2 rolling statistics
+ * indicate the scorer is drifting. Separately reports structural vs
+ * LLM means so operators can tell whether the drift is coming from
+ * the product (DID review needed) or the LLM layer (exemplar bank
+ * recalibration needed).
+ *
+ * Trigger (per dimension):
+ *   mean < 0.4  OR  stddev > 0.15   — for 3 consecutive windows
+ *
+ * Hysteresis: caller supplies `lastTriggerAt`; the monitor won't fire
+ * again within `recoveryMs` (default 7d) of the previous emission.
+ */
+import type { DesignSystemBinding } from '@ai-sdlc/reference';
+import type { StateStore } from '../state/store.js';
+import type { DidScoringEventRecord, SaDimension } from '../state/types.js';
+export declare const DEFAULT_MEAN_THRESHOLD = 0.4;
+export declare const DEFAULT_STDDEV_THRESHOLD = 0.15;
+/** Number of consecutive windows that must violate to trigger. */
+export declare const DEFAULT_CONSECUTIVE_WINDOWS = 3;
+export declare const DEFAULT_WINDOW_DAYS = 30;
+export declare const DEFAULT_RECOVERY_MS: number;
+export interface WindowStats {
+    count: number;
+    mean: number;
+    stddev: number;
+    structuralMean: number;
+    llmMean: number;
+    deterministicFlags: number;
+    /** Inclusive window range — [start, end) in ms. */
+    windowStartMs: number;
+    windowEndMs: number;
+    /** True if this window violated the drift threshold. */
+    violates: boolean;
+}
+export type DriftTrend = 'increasing' | 'decreasing' | 'stable';
+export interface SoulDriftDetectedEvent {
+    type: 'SoulDriftDetected';
+    dimension: SaDimension;
+    rollingMean: number;
+    rollingStdDev: number;
+    sprintsInViolation: number;
+    trend: DriftTrend;
+    driftSource: {
+        deterministicFlags: number;
+        structuralScoreMean: number;
+        llmScoreMean: number;
+        note: string;
+    };
+    notifiedPrincipals: string[];
+    triggeredAt: string;
+}
+export declare function mean(values: readonly number[]): number;
+export declare function stddev(values: readonly number[]): number;
+export declare function computeWindowStats(events: readonly DidScoringEventRecord[], windowStartMs: number, windowEndMs: number, meanThreshold: number, stddevThreshold: number): WindowStats;
+export declare function computeTrend(windows: readonly WindowStats[]): DriftTrend;
+export declare function describeDriftSource(windows: readonly WindowStats[]): {
+    structuralScoreMean: number;
+    llmScoreMean: number;
+    note: string;
+    deterministicFlags: number;
+};
+export interface DriftDetectorDeps {
+    stateStore: StateStore;
+    /** Resolve the DSB that owns this DID (for notification principals). */
+    getBinding?: (dimension: SaDimension) => DesignSystemBinding | undefined;
+    /** Previous trigger timestamp (for hysteresis). */
+    getLastTriggerAt?: (dimension: SaDimension) => string | undefined;
+    /** Clock injection for tests. */
+    now?: () => number;
+    /** Configuration overrides. */
+    config?: Partial<{
+        meanThreshold: number;
+        stddevThreshold: number;
+        consecutiveWindows: number;
+        windowDays: number;
+        recoveryMs: number;
+    }>;
+}
+export declare function detectSoulDrift(dimension: SaDimension, deps: DriftDetectorDeps): SoulDriftDetectedEvent | undefined;
+//# sourceMappingURL=drift-monitor.d.ts.map

package/dist/sa-scoring/drift-monitor.js

@@ -0,0 +1,186 @@
+/**
+ * SA Drift Monitor (RFC-0008 Addendum B §B.9.2).
+ *
+ * Periodically inspects `did_scoring_events` and triggers a
+ * `SoulDriftDetected` event when SA-1 or SA-2 rolling statistics
+ * indicate the scorer is drifting. Separately reports structural vs
+ * LLM means so operators can tell whether the drift is coming from
+ * the product (DID review needed) or the LLM layer (exemplar bank
+ * recalibration needed).
+ *
+ * Trigger (per dimension):
+ *   mean < 0.4  OR  stddev > 0.15   — for 3 consecutive windows
+ *
+ * Hysteresis: caller supplies `lastTriggerAt`; the monitor won't fire
+ * again within `recoveryMs` (default 7d) of the previous emission.
+ */
+// ── Constants ───────────────────────────────────────────────────────
+export const DEFAULT_MEAN_THRESHOLD = 0.4;
+export const DEFAULT_STDDEV_THRESHOLD = 0.15;
+/** Number of consecutive windows that must violate to trigger. */
+export const DEFAULT_CONSECUTIVE_WINDOWS = 3;
+export const DEFAULT_WINDOW_DAYS = 30;
+export const DEFAULT_RECOVERY_MS = 7 * 24 * 60 * 60 * 1000;
+// ── Pure helpers ────────────────────────────────────────────────────
+export function mean(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((a, b) => a + b, 0) / values.length;
+}
+export function stddev(values) {
+    if (values.length === 0)
+        return 0;
+    const m = mean(values);
+    const variance = values.reduce((acc, v) => acc + (v - m) * (v - m), 0) / values.length;
+    return Math.sqrt(variance);
+}
+export function computeWindowStats(events, windowStartMs, windowEndMs, meanThreshold, stddevThreshold) {
+    const composites = [];
+    const structural = [];
+    const llm = [];
+    let deterministicFlags = 0;
+    for (const e of events) {
+        if (typeof e.compositeScore === 'number')
+            composites.push(e.compositeScore);
+        if (e.layer2ResultJson) {
+            try {
+                const parsed = JSON.parse(e.layer2ResultJson);
+                if (typeof parsed.score === 'number')
+                    structural.push(parsed.score);
+            }
+            catch {
+                // ignore malformed entry
+            }
+        }
+        if (e.layer3ResultJson) {
+            try {
+                const parsed = JSON.parse(e.layer3ResultJson);
+                const candidate = parsed.domainIntent ?? parsed.principleAlignment;
+                if (typeof candidate === 'number')
+                    llm.push(candidate);
+            }
+            catch {
+                // ignore malformed entry
+            }
+        }
+        if (e.layer1ResultJson) {
+            try {
+                const parsed = JSON.parse(e.layer1ResultJson);
+                if (parsed.hardGated)
+                    deterministicFlags++;
+            }
+            catch {
+                // ignore malformed entry
+            }
+        }
+    }
+    const m = mean(composites);
+    const s = stddev(composites);
+    return {
+        count: composites.length,
+        mean: m,
+        stddev: s,
+        structuralMean: mean(structural),
+        llmMean: mean(llm),
+        deterministicFlags,
+        windowStartMs,
+        windowEndMs,
+        violates: composites.length > 0 && (m < meanThreshold || s > stddevThreshold),
+    };
+}
+export function computeTrend(windows) {
+    if (windows.length < 2)
+        return 'stable';
+    // Windows ordered newest-first; trend is the direction of mean over time.
+    const newestMean = windows[0].mean;
+    const oldestMean = windows[windows.length - 1].mean;
+    const delta = newestMean - oldestMean;
+    if (Math.abs(delta) < 0.05)
+        return 'stable';
+    return delta > 0 ? 'increasing' : 'decreasing';
+}
+export function describeDriftSource(windows) {
+    const structural = mean(windows.map((w) => w.structuralMean));
+    const llm = mean(windows.map((w) => w.llmMean));
+    const flags = windows.reduce((acc, w) => acc + w.deterministicFlags, 0);
+    let note;
+    if (llm < structural - 0.15)
+        note = 'LLM-layer drift (consider recalibrating exemplar bank)';
+    else if (structural < llm - 0.15)
+        note = 'Structural drift (review DID corpus + compile artifacts)';
+    else if (flags > 0)
+        note = `Mixed drift with ${flags} hard-gated events (review DID scope)`;
+    else
+        note = 'Uniform drift across layers (product review recommended)';
+    return {
+        structuralScoreMean: structural,
+        llmScoreMean: llm,
+        note,
+        deterministicFlags: flags,
+    };
+}
+export function detectSoulDrift(dimension, deps) {
+    const cfg = deps.config ?? {};
+    const meanThreshold = cfg.meanThreshold ?? DEFAULT_MEAN_THRESHOLD;
+    const stddevThreshold = cfg.stddevThreshold ?? DEFAULT_STDDEV_THRESHOLD;
+    const consecutive = cfg.consecutiveWindows ?? DEFAULT_CONSECUTIVE_WINDOWS;
+    const windowDays = cfg.windowDays ?? DEFAULT_WINDOW_DAYS;
+    const recoveryMs = cfg.recoveryMs ?? DEFAULT_RECOVERY_MS;
+    const nowMs = (deps.now ?? (() => Date.now()))();
+    // Hysteresis
+    const lastTriggerAt = deps.getLastTriggerAt?.(dimension);
+    if (lastTriggerAt) {
+        const lastMs = Date.parse(lastTriggerAt);
+        if (!Number.isNaN(lastMs) && nowMs - lastMs < recoveryMs)
+            return undefined;
+    }
+    const windowMs = windowDays * 24 * 60 * 60 * 1000;
+    const earliestMs = nowMs - consecutive * windowMs;
+    // Fetch events for the combined range.
+    const since = new Date(earliestMs).toISOString();
+    const events = deps.stateStore.getDidScoringEvents({
+        saDimension: dimension,
+        since,
+        limit: 10_000,
+    });
+    // Partition into consecutive windows (newest → oldest).
+    const windows = [];
+    for (let i = 0; i < consecutive; i++) {
+        const endMs = nowMs - i * windowMs;
+        const startMs = endMs - windowMs;
+        const inWindow = events.filter((e) => {
+            const t = e.createdAt ? Date.parse(e.createdAt) : NaN;
+            return !Number.isNaN(t) && t >= startMs && t < endMs;
+        });
+        windows.push(computeWindowStats(inWindow, startMs, endMs, meanThreshold, stddevThreshold));
+    }
+    const allViolate = windows.every((w) => w.violates);
+    if (!allViolate)
+        return undefined;
+    // All windows violated — fire the event.
+    const compositeMeans = windows.map((w) => w.mean);
+    const compositeStddevs = windows.map((w) => w.stddev);
+    const rollingMean = mean(compositeMeans);
+    const rollingStdDev = mean(compositeStddevs);
+    const trend = computeTrend(windows);
+    const driftSource = describeDriftSource(windows);
+    const binding = deps.getBinding?.(dimension);
+    const principals = binding ? collectPrincipals(binding) : [];
+    return {
+        type: 'SoulDriftDetected',
+        dimension,
+        rollingMean,
+        rollingStdDev,
+        sprintsInViolation: consecutive,
+        trend,
+        driftSource,
+        notifiedPrincipals: principals,
+        triggeredAt: new Date(nowMs).toISOString(),
+    };
+}
+function collectPrincipals(binding) {
+    const design = binding.spec.stewardship.designAuthority.principals;
+    const engineering = binding.spec.stewardship.engineeringAuthority.principals;
+    return Array.from(new Set([...design, ...engineering]));
+}
+//# sourceMappingURL=drift-monitor.js.map

package/dist/sa-scoring/exemplar-bank.d.ts

@@ -0,0 +1,78 @@
+/**
+ * SA Exemplar Bank loader (RFC-0008 §B.6.4, §B.10.2).
+ *
+ * Loads labeled SA-scoring exemplars from `.ai-sdlc/sa-exemplars.yaml`
+ * for Phase 2a shadow-mode precision tracking and Phase 2b progression
+ * gating. A Phase-2b-ready bank needs ≥ 5 exemplars including at least
+ * one true-positive AND one false-positive per dimension — without
+ * precision evidence on both sides, the structural layer can't be
+ * trusted to contribute to ranking.
+ */
+import type { SaDimension } from '../state/types.js';
+export type ExemplarType = 'true-positive' | 'false-positive' | 'true-negative' | 'false-negative';
+export interface ExemplarIssue {
+    title: string;
+    body: string;
+}
+export interface Layer1Expected {
+    hardGated?: boolean;
+    coreViolationCount?: number;
+    evolvingViolationCount?: number;
+    scopeGate?: Record<string, unknown>;
+    constraintViolations?: Record<string, unknown>;
+    antiPatternHits?: Record<string, unknown>;
+}
+export interface Layer2Expected {
+    domainRelevance?: number;
+    overallCoverage?: number;
+    [key: string]: unknown;
+}
+export interface Layer3Expected {
+    domainIntent?: number;
+    principleAlignment?: number;
+    reasoning?: string;
+    [key: string]: unknown;
+}
+export interface SaExemplar {
+    id: string;
+    dimension: SaDimension;
+    type: ExemplarType;
+    issue: ExemplarIssue;
+    layer1Expected?: Layer1Expected;
+    layer2Expected?: Layer2Expected;
+    layer3Expected?: Layer3Expected;
+    verdict: string;
+    principle?: string;
+    notes?: string;
+}
+export interface SaExemplarBank {
+    sa1: SaExemplar[];
+    sa2: SaExemplar[];
+}
+export declare const EMPTY_BANK: SaExemplarBank;
+/**
+ * Load and partition exemplars by dimension. Returns an empty bank
+ * when the file is missing — Phase 2a can run without any exemplars;
+ * `validatePhase2bExemplars` gates progression.
+ */
+export declare function loadExemplarBank(filePath: string): SaExemplarBank;
+export interface ExemplarGap {
+    dimension: SaDimension | 'overall';
+    reason: string;
+}
+export interface ExemplarReadinessResult {
+    ready: boolean;
+    gaps: ExemplarGap[];
+}
+export declare const MIN_TOTAL_EXEMPLARS = 5;
+export declare function validatePhase2bExemplars(bank: SaExemplarBank): ExemplarReadinessResult;
+export interface LayerPrecision {
+    truePositives: number;
+    falsePositives: number;
+    trueNegatives: number;
+    falseNegatives: number;
+    precision: number;
+    recall: number;
+}
+export declare function computeLayerPrecision(exemplars: readonly SaExemplar[]): LayerPrecision;
+//# sourceMappingURL=exemplar-bank.d.ts.map

package/dist/sa-scoring/exemplar-bank.js

@@ -0,0 +1,154 @@
+/**
+ * SA Exemplar Bank loader (RFC-0008 §B.6.4, §B.10.2).
+ *
+ * Loads labeled SA-scoring exemplars from `.ai-sdlc/sa-exemplars.yaml`
+ * for Phase 2a shadow-mode precision tracking and Phase 2b progression
+ * gating. A Phase-2b-ready bank needs ≥ 5 exemplars including at least
+ * one true-positive AND one false-positive per dimension — without
+ * precision evidence on both sides, the structural layer can't be
+ * trusted to contribute to ranking.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import { parse as parseYaml } from 'yaml';
+// ── Loader ──────────────────────────────────────────────────────────
+export const EMPTY_BANK = Object.freeze({
+    sa1: [],
+    sa2: [],
+});
+/**
+ * Load and partition exemplars by dimension. Returns an empty bank
+ * when the file is missing — Phase 2a can run without any exemplars;
+ * `validatePhase2bExemplars` gates progression.
+ */
+export function loadExemplarBank(filePath) {
+    if (!existsSync(filePath))
+        return { sa1: [], sa2: [] };
+    const raw = readFileSync(filePath, 'utf-8');
+    let doc;
+    try {
+        doc = parseYaml(raw);
+    }
+    catch (err) {
+        throw new Error(`Failed to parse SA exemplar bank at ${filePath}: ${err.message}`);
+    }
+    if (!doc || typeof doc !== 'object' || !('exemplars' in doc)) {
+        throw new Error(`SA exemplar bank ${filePath} must contain a top-level "exemplars" array`);
+    }
+    const list = doc.exemplars;
+    if (!Array.isArray(list)) {
+        throw new Error(`SA exemplar bank ${filePath}: "exemplars" must be an array`);
+    }
+    const sa1 = [];
+    const sa2 = [];
+    for (const entry of list) {
+        const ex = validateExemplar(entry);
+        if (ex.dimension === 'SA-1')
+            sa1.push(ex);
+        else
+            sa2.push(ex);
+    }
+    return { sa1, sa2 };
+}
+function validateExemplar(entry) {
+    if (!entry || typeof entry !== 'object') {
+        throw new Error(`Exemplar must be an object`);
+    }
+    const e = entry;
+    const id = requireString(e, 'id');
+    const dimension = requireEnum(e, 'dimension', ['SA-1', 'SA-2']);
+    const type = requireEnum(e, 'type', [
+        'true-positive',
+        'false-positive',
+        'true-negative',
+        'false-negative',
+    ]);
+    const issueVal = e.issue;
+    if (!issueVal || typeof issueVal !== 'object') {
+        throw new Error(`Exemplar "${id}" missing required "issue" object`);
+    }
+    const issueObj = issueVal;
+    const issue = {
+        title: requireString(issueObj, 'title'),
+        body: requireString(issueObj, 'body'),
+    };
+    const verdict = requireString(e, 'verdict');
+    return {
+        id,
+        dimension,
+        type,
+        issue,
+        layer1Expected: e.layer1Expected,
+        layer2Expected: e.layer2Expected,
+        layer3Expected: e.layer3Expected,
+        verdict,
+        principle: typeof e.principle === 'string' ? e.principle : undefined,
+        notes: typeof e.notes === 'string' ? e.notes : undefined,
+    };
+}
+function requireString(obj, key) {
+    const v = obj[key];
+    if (typeof v !== 'string' || v.length === 0) {
+        throw new Error(`Field "${key}" must be a non-empty string`);
+    }
+    return v;
+}
+function requireEnum(obj, key, options) {
+    const v = obj[key];
+    if (typeof v !== 'string' || !options.includes(v)) {
+        throw new Error(`Field "${key}" must be one of: ${options.join(', ')}`);
+    }
+    return v;
+}
+export const MIN_TOTAL_EXEMPLARS = 5;
+export function validatePhase2bExemplars(bank) {
+    const gaps = [];
+    const total = bank.sa1.length + bank.sa2.length;
+    if (total < MIN_TOTAL_EXEMPLARS) {
+        gaps.push({
+            dimension: 'overall',
+            reason: `Need ≥${MIN_TOTAL_EXEMPLARS} exemplars total (have ${total})`,
+        });
+    }
+    for (const dimension of ['SA-1', 'SA-2']) {
+        const slice = dimension === 'SA-1' ? bank.sa1 : bank.sa2;
+        if (slice.length === 0) {
+            gaps.push({
+                dimension,
+                reason: `Need ≥1 exemplar for ${dimension}`,
+            });
+            continue;
+        }
+        const hasTP = slice.some((e) => e.type === 'true-positive');
+        const hasFP = slice.some((e) => e.type === 'false-positive');
+        if (!hasTP) {
+            gaps.push({
+                dimension,
+                reason: `${dimension} needs ≥1 true-positive exemplar`,
+            });
+        }
+        if (!hasFP) {
+            gaps.push({
+                dimension,
+                reason: `${dimension} needs ≥1 false-positive exemplar (regression tracking)`,
+            });
+        }
+    }
+    return { ready: gaps.length === 0, gaps };
+}
+export function computeLayerPrecision(exemplars) {
+    const tp = exemplars.filter((e) => e.type === 'true-positive').length;
+    const fp = exemplars.filter((e) => e.type === 'false-positive').length;
+    const tn = exemplars.filter((e) => e.type === 'true-negative').length;
+    const fn = exemplars.filter((e) => e.type === 'false-negative').length;
+    const precision = tp + fp === 0 ? 0 : tp / (tp + fp);
+    const recall = tp + fn === 0 ? 0 : tp / (tp + fn);
+    return {
+        truePositives: tp,
+        falsePositives: fp,
+        trueNegatives: tn,
+        falseNegatives: fn,
+        precision,
+        recall,
+    };
+}
+//# sourceMappingURL=exemplar-bank.js.map

package/dist/sa-scoring/feedback-store.d.ts

@@ -0,0 +1,100 @@
+/**
+ * SA feedback store (RFC-0008 Addendum B §B.8).
+ *
+ * Wraps the `did_feedback_events` table and exposes precision
+ * aggregations used by the feedback flywheel (AISDLC-65 Cκ
+ * calibration, AISDLC-66 phase-weight auto-calibration).
+ *
+ * Signal semantics (§B.8.1):
+ *   accept    — admitted item was correctly scored (true-positive on positive path)
+ *   dismiss   — admitted item should NOT have been (false-positive)
+ *   escalate  — scored too low; should have been ranked higher (false-negative)
+ *   override  — HC_override bypass triggered; feedback auto-emitted
+ */
+import type { StateStore } from '../state/store.js';
+import type { DidFeedbackEventRecord, FeedbackSignal, SaDimension } from '../state/types.js';
+export interface RecordFeedbackInput {
+    didName: string;
+    issueNumber: number;
+    dimension: SaDimension;
+    signal: FeedbackSignal;
+    principal?: string;
+    /** Optional category label — e.g. product/design/engineering pillar, or custom. */
+    category?: string;
+    /** Layer 2 structural score at admission time (for precision tracking). */
+    structuralScore?: number;
+    /** Layer 3 LLM score at admission time. */
+    llmScore?: number;
+    /** Composite SA score. */
+    compositeScore?: number;
+    notes?: string;
+}
+export interface PrecisionWindow {
+    /** ISO timestamp — filter feedback events newer than this. */
+    since?: string;
+    /** Filter to one dimension. */
+    dimension?: SaDimension;
+}
+export interface PrecisionResult {
+    /** Sample size after filters. */
+    sampleSize: number;
+    /** Directionally-correct count (accept/escalate when score was high, dismiss when score was low). */
+    correct: number;
+    /** Precision = correct / sampleSize. 0 when sampleSize=0. */
+    precision: number;
+}
+export interface CategoryFalsePositive {
+    category: string;
+    sampleSize: number;
+    falsePositiveCount: number;
+    falsePositiveRate: number;
+}
+export declare class SAFeedbackStore {
+    private readonly store;
+    constructor(store: StateStore);
+    record(input: RecordFeedbackInput): number;
+    list(window?: PrecisionWindow): DidFeedbackEventRecord[];
+    /**
+     * Directional correctness of the Layer 2 structural score.
+     *
+     *   signal=accept   + structural ≥ 0.5 → correct (high score → accepted)
+     *   signal=accept   + structural < 0.5 → incorrect (low score → accepted anyway, underconfident)
+     *   signal=dismiss  + structural ≥ 0.5 → incorrect (high score → dismissed, overconfident)
+     *   signal=dismiss  + structural < 0.5 → correct (low score → dismissed)
+     *   signal=escalate + structural < 0.5 → correct (low score → escalated, was underscored)
+     *   signal=escalate + structural ≥ 0.5 → incorrect (high score → escalated? unusual)
+     *   signal=override → EXCLUDED (bypass, not a judgement on structural)
+     */
+    structuralPrecision(window?: PrecisionWindow): PrecisionResult;
+    llmPrecision(window?: PrecisionWindow): PrecisionResult;
+    /**
+     * Categories with the highest false-positive rates — i.e. where
+     * `dismiss` signals dominate `accept`. Phase 3 calibration
+     * (AISDLC-66) should prioritise tuning these.
+     */
+    highFalsePositiveCategories(window?: PrecisionWindow, minSampleSize?: number): CategoryFalsePositive[];
+    private computeDirectionalPrecision;
+}
+/**
+ * Map a GitHub label addition event to a feedback signal, or
+ * undefined if the label isn't one of our SA signals.
+ */
+export declare function classifyLabel(label: string): FeedbackSignal | undefined;
+export declare const SA_FEEDBACK_LABELS: readonly string[];
+export interface OverrideFeedbackInput {
+    didName: string;
+    issueNumber: number;
+    /** The override reason from `PriorityScore.override.reason`. */
+    reason?: string;
+    /** Principal who invoked the override, if known. */
+    principal?: string;
+}
+/**
+ * Emit an `override` feedback row when an admission score carries an
+ * `override` bypass. Safe no-op when `override` is undefined — callers
+ * can invoke unconditionally.
+ */
+export declare function recordOverrideFeedback(feedback: SAFeedbackStore, override: {
+    reason?: string;
+} | undefined, context: OverrideFeedbackInput): void;
+//# sourceMappingURL=feedback-store.d.ts.map