@ryuenn3123/agentic-senior-core 4.4.0 → 5.1.0

package/scripts/ui-design-judge.mjs

@@ -1,319 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-
-/**
- * ui-design-judge.mjs
- *
- * Advisory-default UI design contract judge.
- *
- * Repo-internal workflow audit; no user-facing runtime modes.
- * Stays advisory when no provider is configured or no auto-fail signal is found.
- * reviewRubric.genericityAutoFail escalates named drift signals and forbidden
- * patterns into blocking findings and blocking required actions when provider
- * evidence is available.
- *
- * Validation anchors for repo governance:
- * - Do not reward generic SaaS defaults or popular template patterns.
- * - UI design judge only evaluates changed UI surfaces.
- * - Structured design execution summary was supplied to semantic review.
- * - designExecutionSignalCount
- * - designExecutionPolicy
- * - designExecutionHandoff
- * - handoffReady
- * - structuredInspectionAvailable
- */
-
-import { collectChangedFiles, collectPullRequestDiff, isUiRelevantFilePath } from './ui-design-judge/git-input.mjs';
-import { buildSystemPrompt, buildUserMessage } from './ui-design-judge/prompting.mjs';
-import { selectAvailableProvider } from './ui-design-judge/providers.mjs';
-import { calibrateGenericityAssessment } from './ui-design-judge/rubric-calibration.mjs';
-import {
-  buildReport,
-  emitMachineReadableReport,
-  extractVerdictObject,
-  normalizeFindings,
-  normalizeGenericityAssessment,
-  normalizeRubricBreakdown,
-} from './ui-design-judge/reporting.mjs';
-import { loadDesignGuide, loadDesignIntent, summarizeDesignExecutionPolicy, summarizeReviewRubric } from './ui-design-judge/design-execution-summary.mjs';
-
-function applyGenericityAutoFail({
-  reviewRubricSummary,
-  calibration,
-  findings,
-  notes,
-}) {
-  const autoFailEnabled = reviewRubricSummary?.genericityAutoFail === true;
-  const namedGenericityDetected = Array.isArray(calibration?.matchedGenericitySignals)
-    && calibration.matchedGenericitySignals.length > 0;
-  const forbiddenPatternDetected = Array.isArray(calibration?.matchedForbiddenPatterns)
-    && calibration.matchedForbiddenPatterns.length > 0;
-  const shouldAutoFail = autoFailEnabled
-    && calibration?.calibratedStatus === 'generic'
-    && (namedGenericityDetected || forbiddenPatternDetected);
-
-  if (!shouldAutoFail) {
-    return {
-      findings,
-      notes,
-      autoFailTriggered: false,
-    };
-  }
-
-  const normalizedFindings = Array.isArray(findings) ? findings.map((finding) => ({ ...finding })) : [];
-  const updatedFindings = normalizedFindings.length > 0
-    ? normalizedFindings.map((finding) => ({
-      ...finding,
-      blockingRecommended: true,
-    }))
-    : [
-      {
-        area: 'design-contract',
-        severity: 'high',
-        problem: 'The UI matches named genericity drift signals that the contract marks as auto-fail.',
-        evidence: `Matched signals: ${[...(calibration.matchedGenericitySignals || []), ...(calibration.matchedForbiddenPatterns || [])].join(', ')}`,
-        requiredAction: 'Rebuild the affected UI surfaces from the contract and remove the named generic patterns instead of polishing them.',
-        blockingRecommended: true,
-      },
-    ];
-
-  return {
-    findings: updatedFindings,
-    notes: [
-      ...(Array.isArray(notes) ? notes : []),
-      'reviewRubric.genericityAutoFail triggered because named genericity drift signals or forbidden patterns were detected.',
-    ],
-    autoFailTriggered: true,
-  };
-}
-
-async function main() {
-  const changedFiles = collectChangedFiles();
-  const changedUiFiles = changedFiles.filter(isUiRelevantFilePath);
-  const rawDiff = collectPullRequestDiff();
-  const designIntentContent = loadDesignIntent();
-  const designGuideContent = loadDesignGuide();
-
-  if (!designGuideContent.trim()) {
-    emitMachineReadableReport(buildReport({
-      skipped: true,
-      skipReason: 'Design guide is missing or unreadable. Skipping UI design judge.',
-      contractPresent: false,
-      notes: ['docs/DESIGN.md is required for contract-aware UI judging.'],
-    }));
-    return;
-  }
-
-  if (changedUiFiles.length === 0) {
-    emitMachineReadableReport(buildReport({
-      skipped: true,
-      skipReason: 'No UI-relevant changed files detected.',
-      contractPresent: true,
-      summary: {
-        changedUiFileCount: 0,
-        alignmentScore: null,
-        driftCount: 0,
-        blockingCandidateCount: 0,
-        designExecutionSignalCount: 0,
-        genericityStatus: 'unclear',
-      },
-      notes: ['UI design judge only evaluates changed UI surfaces.'],
-    }));
-    return;
-  }
-
-  const designExecutionSummary = summarizeDesignExecutionPolicy(designIntentContent || {});
-  const reviewRubricSummary = summarizeReviewRubric(designIntentContent || {});
-
-  const systemPrompt = buildSystemPrompt();
-  const userMessage = buildUserMessage(
-    designIntentContent,
-    designGuideContent,
-    rawDiff,
-    changedUiFiles,
-    designExecutionSummary
-  );
-
-  const selectedProvider = selectAvailableProvider();
-  if (!selectedProvider) {
-    const calibration = calibrateGenericityAssessment({
-      reviewRubricSummary,
-      designExecutionSummary,
-      genericityAssessment: { status: 'unclear', reason: 'No provider review was run.' },
-      rubricBreakdown: [],
-      findings: [],
-      notes: [],
-      tasteVsFailureSeparated: null,
-    });
-
-    emitMachineReadableReport(buildReport({
-      provider: 'none',
-      contractPresent: true,
-      summary: {
-        changedUiFileCount: changedUiFiles.length,
-        alignmentScore: null,
-        driftCount: 0,
-        blockingCandidateCount: 0,
-        designExecutionSignalCount: designExecutionSummary.enabledCapabilities.length,
-        structuredInspectionAvailable: designExecutionSummary.structuredInspectionAvailable,
-        genericityStatus: calibration.calibratedStatus,
-      },
-      designExecution: designExecutionSummary,
-      rubric: {
-        expectedDimensions: reviewRubricSummary.dimensions.map((dimension) => dimension.key),
-        breakdown: [],
-        genericityAssessment: {
-          status: 'unclear',
-          reason: 'No provider review was run.',
-        },
-        tasteVsFailureSeparated: null,
-        calibration,
-      },
-      semanticJudge: {
-        attempted: false,
-        skipped: true,
-        skipReason: 'no-provider-configured',
-      },
-      notes: [
-        'No LLM provider configured. UI design judge skipped provider review and stayed advisory.',
-        ...designExecutionSummary.notes,
-      ],
-    }));
-    return;
-  }
-
-  let rawJudgeResponse;
-  try {
-    rawJudgeResponse = await selectedProvider.invokeProvider(systemPrompt, userMessage);
-  } catch (providerError) {
-    const calibration = calibrateGenericityAssessment({
-      reviewRubricSummary,
-      designExecutionSummary,
-      genericityAssessment: { status: 'unclear', reason: 'Provider review failed before rubric scoring completed.' },
-      rubricBreakdown: [],
-      findings: [],
-      notes: [],
-      tasteVsFailureSeparated: null,
-    });
-
-    const providerErrorMessage = providerError instanceof Error
-      ? providerError.message
-      : 'Unknown provider error';
-
-    emitMachineReadableReport(buildReport({
-      provider: selectedProvider.providerName,
-      contractPresent: true,
-      providerError: true,
-      summary: {
-        changedUiFileCount: changedUiFiles.length,
-        alignmentScore: null,
-        driftCount: 0,
-        blockingCandidateCount: 0,
-        designExecutionSignalCount: designExecutionSummary.enabledCapabilities.length,
-        structuredInspectionAvailable: designExecutionSummary.structuredInspectionAvailable,
-        genericityStatus: calibration.calibratedStatus,
-      },
-      designExecution: designExecutionSummary,
-      rubric: {
-        expectedDimensions: reviewRubricSummary.dimensions.map((dimension) => dimension.key),
-        breakdown: [],
-        genericityAssessment: {
-          status: 'unclear',
-          reason: 'Provider review failed before rubric scoring completed.',
-        },
-        tasteVsFailureSeparated: null,
-        calibration,
-      },
-      semanticJudge: {
-        attempted: true,
-        skipped: false,
-        skipReason: null,
-      },
-      notes: [`Provider call failed: ${providerErrorMessage}`, ...designExecutionSummary.notes],
-      passed: true,
-    }));
-    return;
-  }
-
-  const { verdict, malformed } = extractVerdictObject(rawJudgeResponse);
-  const findings = normalizeFindings(verdict?.findings);
-  const rubricBreakdown = normalizeRubricBreakdown(
-    verdict?.rubricBreakdown,
-    reviewRubricSummary.dimensions.map((dimension) => dimension.key)
-  );
-  const genericityAssessment = normalizeGenericityAssessment(verdict?.genericityAssessment);
-  const tasteVsFailureSeparated = typeof verdict?.tasteVsFailureSeparated === 'boolean'
-    ? verdict.tasteVsFailureSeparated
-    : null;
-  const alignmentScore = typeof verdict?.alignmentScore === 'number' ? verdict.alignmentScore : null;
-  const notes = Array.isArray(verdict?.notes)
-    ? verdict.notes.map((note) => String(note))
-    : [];
-  const calibration = calibrateGenericityAssessment({
-    reviewRubricSummary,
-    designExecutionSummary,
-    genericityAssessment,
-    rubricBreakdown,
-    findings,
-    notes,
-    tasteVsFailureSeparated,
-  });
-  const autoFailResolution = applyGenericityAutoFail({
-    reviewRubricSummary,
-    calibration,
-    findings,
-    notes,
-  });
-  const resolvedFindings = autoFailResolution.findings;
-  const resolvedNotes = autoFailResolution.notes;
-  const blockingCandidateCount = resolvedFindings.filter((finding) => finding.blockingRecommended || finding.severity === 'high').length;
-
-  emitMachineReadableReport(buildReport({
-    provider: selectedProvider.providerName,
-    contractPresent: true,
-    mode: autoFailResolution.autoFailTriggered ? 'blocking-recommended' : 'advisory',
-    advisoryOnly: !autoFailResolution.autoFailTriggered,
-    passed: !autoFailResolution.autoFailTriggered,
-    autoFailTriggered: autoFailResolution.autoFailTriggered,
-    malformedVerdict: malformed,
-    summary: {
-      changedUiFileCount: changedUiFiles.length,
-      alignmentScore,
-      driftCount: resolvedFindings.length,
-      blockingCandidateCount,
-      designExecutionSignalCount: designExecutionSummary.enabledCapabilities.length,
-      structuredInspectionAvailable: designExecutionSummary.structuredInspectionAvailable,
-      genericityStatus: calibration.calibratedStatus,
-    },
-    designExecution: designExecutionSummary,
-    rubric: {
-      expectedDimensions: reviewRubricSummary.dimensions.map((dimension) => dimension.key),
-      breakdown: rubricBreakdown,
-      genericityAssessment,
-      tasteVsFailureSeparated,
-      calibration,
-    },
-    semanticJudge: {
-      attempted: true,
-      skipped: false,
-      skipReason: null,
-    },
-    findings: resolvedFindings,
-    notes: malformed
-      ? ['LLM response was malformed. Advisory mode kept the audit non-blocking.', ...designExecutionSummary.notes]
-      : [...resolvedNotes, ...calibration.notes, ...designExecutionSummary.notes],
-  }));
-}
-
-main().catch((unexpectedError) => {
-  const errorMessage = unexpectedError instanceof Error
-    ? unexpectedError.message
-    : 'Unknown unexpected error';
-
-  emitMachineReadableReport(buildReport({
-    provider: 'none',
-    providerError: true,
-    passed: true,
-    notes: [`Unexpected ui-design-judge failure: ${errorMessage}`],
-  }));
-});

package/scripts/ui-rubric-calibration.mjs

@@ -1,35 +0,0 @@
-#!/usr/bin/env node
-// @ts-check
-
-import { readFileSync } from 'node:fs';
-import { dirname, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { buildRubricCalibrationReport } from './ui-design-judge/rubric-calibration.mjs';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const GOLDSET_PATH = resolve(__dirname, 'ui-design-judge', 'rubric-goldset.json');
-const REPORT_NAME = 'ui-rubric-calibration';
-
-function loadGoldset() {
-  return JSON.parse(readFileSync(GOLDSET_PATH, 'utf8'));
-}
-
-function main() {
-  const goldset = loadGoldset();
-  const report = buildRubricCalibrationReport({
-    cases: goldset.cases,
-    reviewRubricSummary: goldset.reviewRubric,
-  });
-  report.reportName = REPORT_NAME;
-  // Keep the report surface explicit so static validation can detect the machine-readable payload contract.
-  report.accuracyPercent = report.accuracyPercent;
-
-  console.log(JSON.stringify(report, null, 2));
-
-  if (!report.passed) {
-    process.exitCode = 1;
-  }
-}
-
-main();

package/scripts/validate/audits/cache-layer-contract.mjs

@@ -1,230 +0,0 @@
-#!/usr/bin/env node
-/**
- * audit-cache-layer-contract.mjs
- *
- * Phase 2 cache-layer integrity gate. Validates provider cache metadata,
- * fixture segmentation, and the emitted cache simulation JSON without calling
- * provider APIs.
- *
- * Boundary: this audit covers the TECHNICAL contract (provider matrix,
- * layer definitions, fixture segmentation, simulation result JSON shape).
- * For public-prose hygiene around caching saving figures see
- * audit-caching-scope-hygiene.mjs.
- */
-
-import { existsSync, readFileSync } from 'node:fs';
-import { dirname, join, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import {
-  CACHE_LAYER_DEFINITIONS,
-  CACHE_LAYER_IDS,
-  validateCacheLayerContract,
-} from '../../../benchmarks/token-usage/lib/cache-layer-contract.mjs';
-import {
-  CACHE_MATRIX_VERIFIED_AT,
-  PROVIDER_CACHE_MATRIX,
-  listProviderCacheEntries,
-} from '../../../benchmarks/token-usage/lib/provider-cache-matrix.mjs';
-import {
-  buildCacheLayeredScenarioPrompts,
-  loadFixtures,
-} from '../../../benchmarks/token-usage/runners/_shared.mjs';
-
-const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
-const REPOSITORY_ROOT = resolve(dirname(SCRIPT_FILE_PATH), '../../..');
-const DEFAULT_RESULT_PATH = join(REPOSITORY_ROOT, 'benchmarks', 'results', 'cache-phase-2-2026-05-16.json');
-const ARGS = new Set(process.argv.slice(2));
-const JSON_ONLY = ARGS.has('--json');
-
-const REQUIRED_PROVIDER_KEYS = [
-  'provider',
-  'sourceUrl',
-  'sourceType',
-  'verifiedAt',
-  'cacheMode',
-  'minimumCacheableTokens',
-  'costModel',
-];
-
-function addViolation(violations, kind, detail, context = {}) {
-  violations.push({ kind, detail, ...context });
-}
-
-function validateProviderMatrix(violations) {
-  for (const entry of listProviderCacheEntries()) {
-    for (const key of REQUIRED_PROVIDER_KEYS) {
-      if (!(key in entry)) {
-        addViolation(violations, 'provider-metadata.missing-key', `${entry.provider} missing ${key}`, { provider: entry.provider });
-      }
-    }
-
-    if (entry.sourceType === 'official-docs') {
-      if (typeof entry.sourceUrl !== 'string' || !entry.sourceUrl.startsWith('https://')) {
-        addViolation(violations, 'provider-metadata.invalid-source-url', `${entry.provider} official sourceUrl must be https`, { provider: entry.provider });
-      }
-      if (entry.verifiedAt !== CACHE_MATRIX_VERIFIED_AT) {
-        addViolation(violations, 'provider-metadata.invalid-verified-at', `${entry.provider} verifiedAt must be ${CACHE_MATRIX_VERIFIED_AT}`, { provider: entry.provider });
-      }
-    }
-
-    if (entry.provider === 'anthropic') {
-      const ttlOptions = entry.costModel?.ttlOptions;
-      if (ttlOptions?.['5m']?.writeMultiplier !== 1.25 || ttlOptions?.['5m']?.readMultiplier !== 0.1) {
-        addViolation(violations, 'provider-metadata.anthropic-5m-multiplier', 'Anthropic 5m cache multipliers drifted');
-      }
-      if (ttlOptions?.['1h']?.writeMultiplier !== 2.0 || ttlOptions?.['1h']?.readMultiplier !== 0.1) {
-        addViolation(violations, 'provider-metadata.anthropic-1h-multiplier', 'Anthropic 1h cache multipliers drifted');
-      }
-    }
-
-    if (['openai', 'gemini'].includes(entry.provider) && entry.costModel?.accurate !== false) {
-      addViolation(violations, 'provider-metadata.fake-universal-pricing', `${entry.provider} must not claim universal accurate pricing`, { provider: entry.provider });
-    }
-  }
-}
-
-function validateLayerDefinitions(violations) {
-  const definitionIds = Object.keys(CACHE_LAYER_DEFINITIONS);
-  const uniqueDefinitionIds = new Set(definitionIds);
-  if (definitionIds.length !== uniqueDefinitionIds.size) {
-    addViolation(violations, 'layer-definition.duplicate-id', 'Cache layer definition IDs must be unique');
-  }
-
-  const expectedIds = [
-    CACHE_LAYER_IDS.STATIC_PREFIX,
-    CACHE_LAYER_IDS.SEMI_STATIC_CONTEXT,
-    CACHE_LAYER_IDS.DYNAMIC_SUFFIX,
-  ];
-  if (JSON.stringify(definitionIds) !== JSON.stringify(expectedIds)) {
-    addViolation(violations, 'layer-definition.unexpected-order', `Expected ${expectedIds.join(', ')}, got ${definitionIds.join(', ')}`);
-  }
-}
-
-function validateFixtureSegmentation(violations) {
-  const fixtures = loadFixtures();
-  let auditedScenarioCount = 0;
-  for (const fixture of fixtures) {
-    const layered = buildCacheLayeredScenarioPrompts(fixture);
-    const scenarios = [
-      { name: 'always_included', contract: layered.alwaysIncluded },
-      { name: 'with_loaded_rules', contract: layered.withLoadedRules },
-    ];
-
-    for (const scenario of scenarios) {
-      auditedScenarioCount += 1;
-      try {
-        validateCacheLayerContract(scenario.contract);
-      } catch (error) {
-        addViolation(violations, 'fixture-segmentation.invalid-contract', error.message, {
-          fixture_id: fixture.id,
-          scenario: scenario.name,
-        });
-      }
-
-      const layerIds = scenario.contract.layers.map((layer) => layer.id);
-      if (new Set(layerIds).size !== layerIds.length) {
-        addViolation(violations, 'fixture-segmentation.duplicate-layer-id', 'Fixture contract contains duplicate layer IDs', {
-          fixture_id: fixture.id,
-          scenario: scenario.name,
-        });
-      }
-
-      for (const layer of [
-        scenario.contract.layer_1_static_prefix,
-        scenario.contract.layer_2_semi_static_context,
-      ]) {
-        if (layer.content.includes(fixture.user_message)) {
-          addViolation(violations, 'fixture-segmentation.dynamic-leak', `Fixture user message leaked into ${layer.id}`, {
-            fixture_id: fixture.id,
-            scenario: scenario.name,
-          });
-        }
-      }
-
-      if (scenario.contract.layer_3_dynamic_suffix.content.trim().length === 0) {
-        addViolation(violations, 'fixture-segmentation.missing-dynamic-layer', 'Layer 3 dynamic suffix is empty', {
-          fixture_id: fixture.id,
-          scenario: scenario.name,
-        });
-      }
-    }
-  }
-  return { fixtureCount: fixtures.length, auditedScenarioCount };
-}
-
-function validateSimulationResultJson(violations, resultPath = DEFAULT_RESULT_PATH) {
-  if (!existsSync(resultPath)) {
-    addViolation(violations, 'result-json.missing', `Missing cache simulation result: ${resultPath}`);
-    return { resultPath, resultCount: 0 };
-  }
-
-  const parsed = JSON.parse(readFileSync(resultPath, 'utf8'));
-  if (parsed.report_version !== '2.0.0') {
-    addViolation(violations, 'result-json.schema-version', `Unexpected report_version ${parsed.report_version}`);
-  }
-  if (!Array.isArray(parsed.results) || parsed.results.length === 0) {
-    addViolation(violations, 'result-json.results-empty', 'results must be a non-empty array');
-    return { resultPath, resultCount: 0 };
-  }
-
-  const expectedResultCount = parsed.fixture_count * parsed.provider_count * parsed.scenario_count;
-  if (parsed.results.length !== expectedResultCount) {
-    addViolation(violations, 'result-json.result-count', `Expected ${expectedResultCount} rows, got ${parsed.results.length}`);
-  }
-
-  for (const result of parsed.results) {
-    if (!result.token_counts || !result.economic_projection) {
-      addViolation(violations, 'result-json.missing-separation', 'Result must separate token_counts and economic_projection', {
-        fixture_id: result.fixture_id,
-        provider: result.provider,
-        scenario: result.scenario,
-      });
-      continue;
-    }
-    if (typeof result.token_counts.layer_3_dynamic_suffix !== 'number' || result.token_counts.layer_3_dynamic_suffix <= 0) {
-      addViolation(violations, 'result-json.missing-layer-3-tokens', 'Layer 3 token count must be present and positive', {
-        fixture_id: result.fixture_id,
-        provider: result.provider,
-        scenario: result.scenario,
-      });
-    }
-    if (result.economic_projection.accurate === true && (!result.source?.sourceUrl || !result.source?.verifiedAt)) {
-      addViolation(violations, 'result-json.accurate-without-source', 'Accurate projection requires official source metadata', {
-        fixture_id: result.fixture_id,
-        provider: result.provider,
-        scenario: result.scenario,
-      });
-    }
-    if (['openai', 'gemini'].includes(result.provider) && result.economic_projection.first_request_effective_tokens !== null) {
-      addViolation(violations, 'result-json.fake-openai-gemini-savings', `${result.provider} must not emit exact savings without pricing metadata`, {
-        fixture_id: result.fixture_id,
-        scenario: result.scenario,
-      });
-    }
-  }
-
-  return { resultPath, resultCount: parsed.results.length };
-}
-
-export function runCacheLayerContractAudit({ resultPath = DEFAULT_RESULT_PATH } = {}) {
-  const violations = [];
-  validateProviderMatrix(violations);
-  validateLayerDefinitions(violations);
-  const segmentationStats = validateFixtureSegmentation(violations);
-  const resultStats = validateSimulationResultJson(violations, resultPath);
-
-  return {
-    auditName: 'audit-cache-layer-contract',
-    reportVersion: '1.0.0',
-    generatedAt: new Date().toISOString(),
-    providerCount: Object.keys(PROVIDER_CACHE_MATRIX).length,
-    ...segmentationStats,
-    ...resultStats,
-    violationCount: violations.length,
-    violations,
-    passed: violations.length === 0,
-  };
-}
-
-