@sun-asterisk/sungen 3.2.21-beta.1 → 3.2.21

package/src/harness/flow-contract.ts

@@ -1,229 +0,0 @@
-/**
- * Flow Contract — the declared boundary of a flow, and the sensors that verify
- * the suite against it. (#569, docs/spec/sungen-flow-quality-spec.md)
- *
- * A flow is the SMALLEST complete business action chain: one clear trigger ending
- * in ONE observable, valuable outcome. Nothing enforced that: `add-flow` asked only
- * "which screens, in order?", so a real example mixed three business goals (cart,
- * filter, product-detail) in one "flow", and the harness scored flows with screen
- * machinery — a registration flow was judged against the `form` page-type checklist
- * (coverage 0%), and every flow audit showed taxonomy=0% because flow phase ids
- * (FL-HP-001) don't even parse as categories.
- *
- * The contract is a declaration the QA owns (AI proposes at add-flow; a filled
- * contract is an INPUT to generation, never an output — same rule as
- * test-viewpoint.md). These sensors are deterministic checks against it:
- *
- *   - outcome proof   — the flow proves its own goal with an automated data assertion
- *   - scope creep     — scenarios that never touch the outcome are a second goal
- *   - phase coverage  — HP / ER / EH journey phases, the flow's coverage axis
- *   - handoff         — a cross-screen transition is followed by an assertion
- *   - stateful depth  — generalizes the cart-hardcoded regression dims to any
- *                       declared collection (order, application, submission …)
- */
-import * as fs from 'fs';
-import * as path from 'path';
-import { parse as parseYaml } from 'yaml';
-import { ScenarioInfo } from './parse';
-import { readTextFile } from './read-text';
-
-export interface FlowContract {
-  goal: string;
-  actor?: string;
-  trigger?: string;
-  precondition?: string;
-  outcome: { screen: string; assertion?: string };
-  value?: string;
-  /** Journey phases this flow declares. Default [HP, ER, EH]; UI is allowed but
-   *  never demanded (presentation is the balance axis's business, not coverage's). */
-  phases: string[];
-  /** The mutated collection (cart, order, application …) — enables regression dims. */
-  stateful?: string;
-  budgets?: Record<string, number>;
-}
-
-export interface FlowQualityResult {
-  hasContract: boolean;
-  contract?: FlowContract;
-  /** Parse/shape errors — a broken contract is reported, never silently ignored. */
-  errors: string[];
-  outcomeProven: boolean;
-  /** Manual-only proof: the goal is covered but not by automation. */
-  outcomeManualOnly: boolean;
-  /** Scenario names that never touch the outcome screen (guard/error phases excluded). */
-  offGoal: string[];
-  offGoalRatio: number;
-  /** Off-goal categories, for the split suggestion ("VP-FILTER-* looks like its own flow"). */
-  offGoalCategories: string[];
-  phases: { phase: string; covered: boolean; automated: boolean }[];
-  /** Covered-and-automated phases / declared phases (UI excluded) — the flow coverage axis. */
-  phaseRatio: number;
-  /** Cross-namespace transitions followed by an assertion / all transitions. */
-  handoffs: { total: number; asserted: number; ratio: number };
-}
-
-const DEFAULT_PHASES = ['HP', 'ER', 'EH'];
-
-export function flowContractPath(unitDir: string): string {
-  return path.join(unitDir, 'requirements', 'flow-contract.yaml');
-}
-
-/** Load + validate. Returns null when absent; a present-but-broken file returns errors. */
-export function loadFlowContract(unitDir: string): { contract: FlowContract | null; errors: string[] } {
-  const p = flowContractPath(unitDir);
-  if (!fs.existsSync(p)) return { contract: null, errors: [] };
-  let raw: Record<string, unknown>;
-  try {
-    raw = parseYaml(readTextFile(p)) as Record<string, unknown>;
-  } catch (e) {
-    return { contract: null, errors: [`flow-contract.yaml does not parse: ${(e as Error).message}`] };
-  }
-  if (!raw || typeof raw !== 'object') return { contract: null, errors: ['flow-contract.yaml is empty'] };
-  const errors: string[] = [];
-  if (!raw.goal || typeof raw.goal !== 'string') errors.push('missing `goal:` (Verb + outcome, e.g. "Place an order for a product added from home")');
-  const outcome = raw.outcome as { screen?: unknown; assertion?: unknown } | undefined;
-  if (!outcome || typeof outcome.screen !== 'string' || !outcome.screen.trim()) {
-    errors.push('missing `outcome.screen:` — the screen namespace that carries the final proof');
-  }
-  if (errors.length > 0) return { contract: null, errors };
-  const phases = Array.isArray(raw.phases) && raw.phases.length > 0
-    ? (raw.phases as unknown[]).map((x) => String(x).toUpperCase())
-    : DEFAULT_PHASES;
-  return {
-    contract: {
-      goal: String(raw.goal),
-      actor: raw.actor !== undefined ? String(raw.actor) : undefined,
-      trigger: raw.trigger !== undefined ? String(raw.trigger) : undefined,
-      precondition: raw.precondition !== undefined ? String(raw.precondition) : undefined,
-      outcome: { screen: String(outcome!.screen).toLowerCase(), assertion: outcome!.assertion !== undefined ? String(outcome!.assertion) : undefined },
-      value: raw.value !== undefined ? String(raw.value) : undefined,
-      phases,
-      stateful: raw.stateful !== undefined ? String(raw.stateful).toLowerCase() : undefined,
-      budgets: (raw.budgets && typeof raw.budgets === 'object') ? raw.budgets as Record<string, number> : undefined,
-    },
-    errors: [],
-  };
-}
-
-/** `[screen:element]` namespaces referenced by a scenario's steps, in step order. */
-function namespacesInOrder(s: ScenarioInfo): string[] {
-  const out: string[] = [];
-  for (const m of s.stepsText.matchAll(/\[([a-z0-9_.-]+):/g)) out.push(m[1]);
-  return out;
-}
-
-function touchesOutcome(s: ScenarioInfo, outcomeScreen: string): boolean {
-  return namespacesInOrder(s).includes(outcomeScreen);
-}
-
-/** Phase of a scenario: its declared phase token (FL-HP-001 / VP-FLOW-ER-02 / MS-EH-005)
- *  when present, else vocabulary detection. */
-export function phaseOf(s: ScenarioInfo, declared: string[]): string | null {
-  const id = (s.vpId ?? '').toUpperCase();
-  for (const ph of declared) {
-    if (new RegExp(`(^|-)${ph}(-|$)`).test(id)) return ph;
-  }
-  const hay = s.haystack;
-  if (declared.includes('EH') && /\b(direct access|without (a |the )?(submit|login)|browser back|refresh|expired|tamper|unauthoriz|redirect(ed)? (back )?to|guard)\b/.test(hay)) return 'EH';
-  if (declared.includes('ER') && /\b(invalid|error|required|validation|malformed|blocked|then correct|recover)\b/.test(hay)) return 'ER';
-  if (declared.includes('HP') && s.hasDataAssertion) return 'HP';
-  return null;
-}
-
-/**
- * Verify the suite against the contract. Deterministic; a flow without a contract
- * returns hasContract:false and neutral values (the audit reports the checklist).
- */
-export function flowQuality(unitDir: string, scenarios: ScenarioInfo[]): FlowQualityResult {
-  const { contract, errors } = loadFlowContract(unitDir);
-  const neutral: FlowQualityResult = {
-    hasContract: false, errors, outcomeProven: false, outcomeManualOnly: false,
-    offGoal: [], offGoalRatio: 0, offGoalCategories: [],
-    phases: [], phaseRatio: 1, handoffs: { total: 0, asserted: 0, ratio: 1 },
-  };
-  if (!contract) return neutral;
-
-  const outcomeScreen = contract.outcome.screen;
-
-  // --- Outcome proof: the flow proves its own goal, by automation -------------
-  const proofs = scenarios.filter((s) => touchesOutcome(s, outcomeScreen) && s.hasDataAssertion);
-  const outcomeProven = proofs.some((s) => !s.manual);
-  const outcomeManualOnly = !outcomeProven && proofs.length > 0;
-
-  // --- Scope creep: a scenario that never touches the outcome and is not a ----
-  // guard/error phase is evidence of a SECOND business goal in this flow.
-  const declaredPhases = contract.phases;
-  const offGoalScenarios = scenarios.filter((s) => {
-    if (touchesOutcome(s, outcomeScreen)) return false;
-    const ph = phaseOf(s, declaredPhases);
-    return ph !== 'EH' && ph !== 'ER';    // guards/error-recovery legitimately stop early
-  });
-  const offGoalRatio = scenarios.length ? offGoalScenarios.length / scenarios.length : 0;
-  const offGoalCategories = Array.from(new Set(
-    offGoalScenarios.map((s) => s.category ?? s.vpId?.replace(/-\d+.*$/, '') ?? '?')));
-
-  // --- Phase coverage: the flow's coverage axis (UI never demanded) -----------
-  const demanded = declaredPhases.filter((p) => p !== 'UI');
-  const phases = demanded.map((phase) => {
-    const inPhase = scenarios.filter((s) => phaseOf(s, declaredPhases) === phase);
-    // HP must additionally prove the outcome — a data assertion elsewhere is not the goal.
-    const relevant = phase === 'HP' ? inPhase.filter((s) => touchesOutcome(s, outcomeScreen)) : inPhase;
-    return {
-      phase,
-      covered: relevant.length > 0,
-      automated: relevant.some((s) => !s.manual),
-    };
-  });
-  const phaseRatio = demanded.length
-    ? phases.filter((p) => p.covered && p.automated).length / demanded.length
-    : 1;
-
-  // --- Handoff integrity: no blind tail after a cross-namespace transition. ---
-  // A transition counts as asserted when ANY assertion follows it — in the entered
-  // namespace or later. Demanding the assertion in the entered namespace itself
-  // flagged two legitimate shapes: a guard that asserts the REDIRECT target
-  // ("go to [Checkout] → see [Home] page"), and a passthrough click en route
-  // ("click [Cart:Checkout]" asserting on the next screen). What the sensor
-  // actually guards against is a flow that clicks through screens and ends blind.
-  let total = 0; let asserted = 0;
-  for (const s of scenarios) {
-    if (s.manual) continue;
-    const steps = s.steps ?? [];
-    let current: string | null = null;
-    for (let i = 0; i < steps.length; i++) {
-      const ns = (steps[i].text.match(/\[([A-Za-z0-9_.-]+):/) || [])[1]?.toLowerCase() ?? null;
-      if (!ns) continue;
-      if (current !== null && ns !== current) {
-        total++;
-        if (steps.slice(i).some((st) => st.bucket === 'then')) asserted++;
-      }
-      current = ns;
-    }
-  }
-  const handoffs = { total, asserted, ratio: total ? asserted / total : 1 };
-
-  return {
-    hasContract: true, contract, errors: [],
-    outcomeProven, outcomeManualOnly,
-    offGoal: offGoalScenarios.map((s) => s.name.slice(0, 80)),
-    offGoalRatio, offGoalCategories,
-    phases, phaseRatio, handoffs,
-  };
-}
-
-/**
- * Generalized stateful regression depth: the contract names the mutated collection,
- * so the three dims (count-proof · teardown · multi-source) stop being cart-only.
- */
-export function statefulDepthFor(collection: string, scenarios: ScenarioInfo[]): { countProof: boolean; teardown: boolean; multiSource: boolean; missing: string[]; ratio: number } {
-  const hay = scenarios.map((s) => s.haystack);
-  const any = (re: RegExp) => hay.some((h) => re.test(h));
-  const noun = collection.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-  const countProof = any(new RegExp(`\\b(quantity|qty|row count|count|number of|two (rows|lines|items)|\\d+ (rows|lines|items))\\b`)) ;
-  const teardown = any(/\b(remove|delete|clear|cancel|withdraw)(?:s|d|ed|ing|n)?\b/) && any(new RegExp(`\\b(empty|emptied|no items|zero|removed|cleared|cancelled|withdrawn|0 items)\\b|empty[- ]${noun}`));
-  const adds = hay.filter((h) => new RegExp(`\\b(add|submit|create|place).{0,40}${noun}|${noun}.{0,40}\\b(add|submit|create|place)`).test(h));
-  const multiSource = any(/\b(recommended|related|you may also|another source|both sources|second (list|source))\b/) && adds.length > 0;
-  const dims: Array<[string, boolean]> = [['count-proof', countProof], ['teardown', teardown], ['multi-source', multiSource]];
-  const missing = dims.filter(([, v]) => !v).map(([k]) => k);
-  return { countProof, teardown, multiSource, missing, ratio: (dims.length - missing.length) / dims.length };
-}

package/src/harness/perf.ts

@@ -1,112 +0,0 @@
-/**
- * Performance budgets — config + percentile math + the per-unit verdict. (#569)
- *
- * A flow's regression value includes "still fast enough": after a lib/framework
- * upgrade the main journeys must not only pass but hold their response-time
- * budget. Sungen had no perf concept at all — the Playwright JSON parser even
- * dropped the `duration` field Playwright already emits on every result.
- *
- * Scope discipline:
- *   - This is config + measurement + report over runs sungen already makes.
- *     Real load tests stay @manual:M8 → a dedicated tool.
- *   - The AUDIT never reads it: the quality score is documented as a pure
- *     function of the design artifacts ("reads no test-results, live page, or
- *     clock"). Perf reports where runs are already read — `sungen delivery`
- *     and the dashboard. Advisory: a blown budget never fails the design gate.
- *
- * Config: qa/perf.yaml
- *   percentile: p75          # default p75 — "≥75% of runs meet the budget"
- *   defaults:
- *     scenario_ms: 30000     # whole-scenario wall clock (Playwright duration)
- *     page_load_ms: 3000     # Phase B — needs per-transition runtime timing
- *     transition_ms: 2000    # Phase B
- *   units:
- *     place-order: { scenario_ms: 20000 }
- */
-import * as fs from 'fs';
-import * as path from 'path';
-import { parse as parseYaml } from 'yaml';
-import { readTextFile } from './read-text';
-
-export interface PerfConfig {
-  /** 0..100 — e.g. 75 for p75. */
-  percentile: number;
-  defaults: Record<string, number>;
-  units: Record<string, Record<string, number>>;
-}
-
-export interface PerfVerdict {
-  unit: string;
-  metric: string;            // 'scenario_ms' today; page_load_ms/transition_ms in Phase B
-  percentile: number;        // 75
-  budgetMs: number;
-  measuredMs: number;        // the pXX of the observed durations
-  samples: number;
-  pass: boolean;
-  /** Titles of the slowest offenders (only when failing), for the report. */
-  slowest: Array<{ title: string; ms: number }>;
-}
-
-export function perfConfigPath(projectRoot: string): string {
-  return path.join(projectRoot, 'qa', 'perf.yaml');
-}
-
-/** Absent file → null (perf reporting is opt-in; nothing changes until configured). */
-export function loadPerfConfig(projectRoot: string): PerfConfig | null {
-  const p = perfConfigPath(projectRoot);
-  if (!fs.existsSync(p)) return null;
-  let raw: Record<string, unknown>;
-  try { raw = parseYaml(readTextFile(p)) as Record<string, unknown>; } catch { return null; }
-  if (!raw || typeof raw !== 'object') return null;
-  const pctRaw = String(raw.percentile ?? 'p75').toLowerCase().replace(/^p/, '');
-  const percentile = Math.min(100, Math.max(1, Number(pctRaw) || 75));
-  const num = (o: unknown): Record<string, number> => {
-    const out: Record<string, number> = {};
-    if (o && typeof o === 'object') {
-      for (const [k, v] of Object.entries(o as Record<string, unknown>)) {
-        const n = Number(v);
-        if (Number.isFinite(n) && n > 0) out[k] = n;
-      }
-    }
-    return out;
-  };
-  const units: Record<string, Record<string, number>> = {};
-  if (raw.units && typeof raw.units === 'object') {
-    for (const [u, o] of Object.entries(raw.units as Record<string, unknown>)) units[u] = num(o);
-  }
-  return { percentile, defaults: num(raw.defaults), units };
-}
-
-/**
- * Nearest-rank percentile (ceil), the standard "≥pXX of samples meet the budget"
- * reading: p75 of [a…] is the value at ceil(0.75·n) in the sorted list. One
- * sample → that sample. Deterministic, no interpolation.
- */
-export function percentileOf(p: number, values: number[]): number {
-  if (values.length === 0) return 0;
-  const sorted = [...values].sort((a, b) => a - b);
-  const rank = Math.min(sorted.length, Math.max(1, Math.ceil((p / 100) * sorted.length)));
-  return sorted[rank - 1];
-}
-
-/** Budget for a metric on a unit: per-unit override, else defaults, else none. */
-export function budgetFor(config: PerfConfig, unit: string, metric: string): number | undefined {
-  return config.units[unit]?.[metric] ?? config.defaults[metric];
-}
-
-/**
- * The scenario_ms verdict for one unit's run. `durations` = per-test wall-clock ms
- * (a @cases scenario contributes one sample per row-test — each is a real run).
- */
-export function perfVerdict(
-  config: PerfConfig,
-  unit: string,
-  samples: Array<{ title: string; ms: number }>,
-): PerfVerdict | null {
-  const budgetMs = budgetFor(config, unit, 'scenario_ms');
-  if (budgetMs === undefined || samples.length === 0) return null;
-  const measuredMs = percentileOf(config.percentile, samples.map((s) => s.ms));
-  const pass = measuredMs <= budgetMs;
-  const slowest = pass ? [] : [...samples].sort((a, b) => b.ms - a.ms).slice(0, 3);
-  return { unit, metric: 'scenario_ms', percentile: config.percentile, budgetMs, measuredMs, samples: samples.length, pass, slowest };
-}