@ryuenn3123/agentic-senior-core 3.0.16 → 3.0.19

package/scripts/ui-design-judge/reporting.mjs

@@ -0,0 +1,181 @@
+// @ts-check
+
+import { detectCiProvider } from './git-input.mjs';
+
+function normalizeSeverity(rawSeverityValue) {
+  const normalizedSeverityValue = String(rawSeverityValue || '').trim().toLowerCase();
+
+  if (['critical', 'high', 'medium', 'low'].includes(normalizedSeverityValue)) {
+    return normalizedSeverityValue;
+  }
+
+  if (normalizedSeverityValue === 'major') {
+    return 'high';
+  }
+
+  if (normalizedSeverityValue === 'minor' || normalizedSeverityValue === 'info') {
+    return 'low';
+  }
+
+  return 'low';
+}
+
+export function extractVerdictObject(rawResponseText) {
+  const verdictMatch = rawResponseText.match(/JSON_VERDICT:\s*(\{[\s\S]*\})/i);
+  if (!verdictMatch) {
+    return { verdict: null, malformed: true };
+  }
+
+  try {
+    return {
+      verdict: JSON.parse(verdictMatch[1]),
+      malformed: false,
+    };
+  } catch {
+    return {
+      verdict: null,
+      malformed: true,
+    };
+  }
+}
+
+export function normalizeFindings(rawFindings) {
+  if (!Array.isArray(rawFindings)) {
+    return [];
+  }
+
+  return rawFindings.map((rawFinding) => ({
+    area: String(rawFinding?.area || 'general'),
+    severity: normalizeSeverity(rawFinding?.severity),
+    problem: String(rawFinding?.problem || 'No problem description provided.'),
+    evidence: String(rawFinding?.evidence || 'No evidence provided.'),
+    recommendation: String(rawFinding?.recommendation || 'No recommendation provided.'),
+    blockingRecommended: rawFinding?.blockingRecommended === true,
+  }));
+}
+
+function normalizeRubricVerdict(rawVerdictValue) {
+  const normalizedVerdictValue = String(rawVerdictValue || '').trim().toLowerCase();
+  if (['strong', 'acceptable', 'weak', 'unclear'].includes(normalizedVerdictValue)) {
+    return normalizedVerdictValue;
+  }
+  return 'unclear';
+}
+
+export function normalizeRubricBreakdown(rawRubricBreakdown, expectedDimensions = []) {
+  if (!Array.isArray(rawRubricBreakdown)) {
+    return [];
+  }
+
+  const expectedDimensionNames = Array.isArray(expectedDimensions)
+    ? expectedDimensions.map((dimension) => String(dimension || '').trim()).filter(Boolean)
+    : [];
+
+  return rawRubricBreakdown
+    .map((rawDimensionEntry) => ({
+      dimension: String(rawDimensionEntry?.dimension || '').trim(),
+      score: typeof rawDimensionEntry?.score === 'number' && Number.isFinite(rawDimensionEntry.score)
+        ? rawDimensionEntry.score
+        : null,
+      verdict: normalizeRubricVerdict(rawDimensionEntry?.verdict),
+      reason: String(rawDimensionEntry?.reason || 'No rubric reason provided.'),
+      blocking: rawDimensionEntry?.blocking === true,
+    }))
+    .filter((dimensionEntry) => {
+      if (!dimensionEntry.dimension) {
+        return false;
+      }
+      return expectedDimensionNames.length === 0 || expectedDimensionNames.includes(dimensionEntry.dimension);
+    });
+}
+
+export function normalizeGenericityAssessment(rawGenericityAssessment) {
+  const normalizedStatus = String(rawGenericityAssessment?.status || '').trim().toLowerCase();
+  return {
+    status: ['distinctive', 'mixed', 'generic', 'unclear'].includes(normalizedStatus)
+      ? normalizedStatus
+      : 'unclear',
+    reason: String(rawGenericityAssessment?.reason || 'No genericity assessment provided.'),
+  };
+}
+
+export function buildReport(partialReport) {
+  return {
+    generatedAt: new Date().toISOString(),
+    auditName: 'ui-design-judge',
+    schemaVersion: '1.2',
+    mode: 'advisory',
+    advisoryOnly: true,
+    passed: true,
+    skipped: false,
+    skipReason: null,
+    provider: 'none',
+    ciProvider: detectCiProvider(),
+    contractPresent: false,
+    summary: {
+      changedUiFileCount: 0,
+      alignmentScore: null,
+      driftCount: 0,
+      blockingCandidateCount: 0,
+      designExecutionSignalCount: 0,
+      genericityStatus: 'unclear',
+    },
+    designExecution: {
+      policyPresent: false,
+      representationStrategy: null,
+      contractReady: false,
+      screenshotDependencyForbidden: false,
+      repoEvidenceAvailable: false,
+      handoffPresent: false,
+      handoffVersion: null,
+      handoffReady: false,
+      handoffArtifactCount: 0,
+      presentHandoffArtifacts: [],
+      missingHandoffArtifacts: [],
+      repoEvidenceSummaryVersion: null,
+      requiredCapabilities: [],
+      enabledCapabilities: [],
+      missingCapabilities: [],
+      semanticReviewFocus: [],
+      notes: [],
+    },
+    rubric: {
+      expectedDimensions: [],
+      breakdown: [],
+      genericityAssessment: {
+        status: 'unclear',
+        reason: 'No genericity assessment provided.',
+      },
+      tasteVsFailureSeparated: null,
+      calibration: {
+        version: 'ui-rubric-calibration-v1',
+        providerStatus: 'unclear',
+        calibratedStatus: 'unclear',
+        statusChanged: false,
+        namedGenericityRequired: false,
+        matchedGenericitySignals: [],
+        matchedValidBoldSignals: [],
+        blockingFindingCount: 0,
+        contractFidelityWeak: false,
+        contractDriftDetected: false,
+        tasteVsFailureSeparated: null,
+        evidenceTextCount: 0,
+        notes: ['No rubric calibration was performed.'],
+      },
+    },
+    semanticJudge: {
+      attempted: false,
+      skipped: false,
+      skipReason: null,
+    },
+    malformedVerdict: false,
+    providerError: false,
+    findings: [],
+    notes: [],
+    ...partialReport,
+  };
+}
+
+export function emitMachineReadableReport(machineReportPayload) {
+  console.log(JSON.stringify(machineReportPayload, null, 2));
+}

package/scripts/ui-design-judge/rubric-calibration.mjs

@@ -0,0 +1,211 @@
+// @ts-check
+
+function normalizeForSignalMatch(rawValue) {
+  return String(rawValue || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .replace(/\s+/g, ' ');
+}
+
+function collectEvidenceTexts({
+  genericityAssessment,
+  rubricBreakdown,
+  findings,
+  notes,
+}) {
+  const textParts = [
+    genericityAssessment?.reason,
+    ...(Array.isArray(rubricBreakdown)
+      ? rubricBreakdown.flatMap((dimensionEntry) => [dimensionEntry?.dimension, dimensionEntry?.reason])
+      : []),
+    ...(Array.isArray(findings)
+      ? findings.flatMap((finding) => [
+        finding?.area,
+        finding?.problem,
+        finding?.evidence,
+        finding?.recommendation,
+      ])
+      : []),
+    ...(Array.isArray(notes) ? notes : []),
+  ];
+
+  return textParts
+    .map((textValue) => normalizeForSignalMatch(textValue))
+    .filter(Boolean);
+}
+
+export function collectMatchedSignals(signalList, evidenceTexts) {
+  const normalizedEvidenceText = Array.isArray(evidenceTexts)
+    ? evidenceTexts.join(' ')
+    : normalizeForSignalMatch(evidenceTexts);
+
+  if (!normalizedEvidenceText) {
+    return [];
+  }
+
+  return (Array.isArray(signalList) ? signalList : [])
+    .map((signalValue) => ({
+      raw: String(signalValue || '').trim(),
+      normalized: normalizeForSignalMatch(signalValue),
+    }))
+    .filter((signalEntry) => signalEntry.raw && signalEntry.normalized)
+    .filter((signalEntry) => normalizedEvidenceText.includes(signalEntry.normalized))
+    .map((signalEntry) => signalEntry.raw);
+}
+
+function detectContractDrift(rubricBreakdown, findings, designExecutionSummary) {
+  const contractFidelityEntry = Array.isArray(rubricBreakdown)
+    ? rubricBreakdown.find((dimensionEntry) => dimensionEntry?.dimension === 'contractFidelity')
+    : null;
+  const blockingFindingCount = Array.isArray(findings)
+    ? findings.filter((finding) => finding?.blockingRecommended === true || ['critical', 'high'].includes(String(finding?.severity || '').toLowerCase())).length
+    : 0;
+  const contractFidelityWeak = contractFidelityEntry
+    ? contractFidelityEntry.verdict === 'weak'
+      || contractFidelityEntry.verdict === 'unclear'
+      || (typeof contractFidelityEntry.score === 'number' && contractFidelityEntry.score < 70)
+    : false;
+  const contractReady = designExecutionSummary?.contractReady === true;
+
+  return {
+    blockingFindingCount,
+    contractFidelityWeak,
+    contractDriftDetected: !contractReady || contractFidelityWeak || blockingFindingCount > 0,
+  };
+}
+
+export function calibrateGenericityAssessment({
+  reviewRubricSummary,
+  designExecutionSummary,
+  genericityAssessment,
+  rubricBreakdown,
+  findings,
+  notes,
+  tasteVsFailureSeparated,
+}) {
+  const providerStatus = String(genericityAssessment?.status || 'unclear').trim().toLowerCase() || 'unclear';
+  const evidenceTexts = collectEvidenceTexts({
+    genericityAssessment,
+    rubricBreakdown,
+    findings,
+    notes,
+  });
+  const matchedGenericitySignals = collectMatchedSignals(reviewRubricSummary?.genericitySignals, evidenceTexts);
+  const matchedValidBoldSignals = collectMatchedSignals(reviewRubricSummary?.validBoldSignals, evidenceTexts);
+  const { blockingFindingCount, contractFidelityWeak, contractDriftDetected } = detectContractDrift(
+    rubricBreakdown,
+    findings,
+    designExecutionSummary
+  );
+  const namedGenericityRequired = reviewRubricSummary?.reportingRules?.mustExplainGenericity === true;
+  const calibrationNotes = [];
+  let calibratedStatus = providerStatus;
+
+  if (
+    namedGenericityRequired
+    && ['generic', 'mixed'].includes(providerStatus)
+    && matchedGenericitySignals.length === 0
+  ) {
+    calibratedStatus = 'unclear';
+    calibrationNotes.push('Genericity claim was not backed by any named drift signal.');
+  }
+
+  if (matchedGenericitySignals.length > 0 && matchedValidBoldSignals.length === 0) {
+    calibratedStatus = contractDriftDetected || matchedGenericitySignals.length >= 2
+      ? 'generic'
+      : 'mixed';
+    calibrationNotes.push('Named genericity drift signals dominate the review evidence.');
+  } else if (matchedValidBoldSignals.length > 0 && matchedGenericitySignals.length === 0) {
+    if (contractDriftDetected) {
+      calibratedStatus = 'mixed';
+      calibrationNotes.push('Authored signals are present, but contract drift prevents a distinctive verdict.');
+    } else if (matchedValidBoldSignals.length >= 2) {
+      calibratedStatus = 'distinctive';
+      calibrationNotes.push('Multiple valid bold signals were named without generic drift evidence.');
+    } else if (providerStatus === 'unclear') {
+      calibratedStatus = 'mixed';
+      calibrationNotes.push('One valid bold signal was named, but evidence is not strong enough for a distinctive verdict.');
+    }
+  } else if (matchedGenericitySignals.length > 0 && matchedValidBoldSignals.length > 0) {
+    calibratedStatus = contractDriftDetected ? 'mixed' : 'mixed';
+    calibrationNotes.push('The evidence contains both generic drift and legitimate authored moves.');
+  } else if (providerStatus === 'distinctive' && contractDriftDetected) {
+    calibratedStatus = 'mixed';
+    calibrationNotes.push('Distinctive tone does not override contract drift or blocking findings.');
+  }
+
+  if (tasteVsFailureSeparated === false && calibratedStatus === 'distinctive') {
+    calibratedStatus = 'mixed';
+    calibrationNotes.push('The review did not separate taste preference from real failure conditions.');
+  }
+
+  if (calibrationNotes.length === 0) {
+    calibrationNotes.push('Provider verdict stayed intact after rubric calibration.');
+  }
+
+  return {
+    version: 'ui-rubric-calibration-v1',
+    providerStatus,
+    calibratedStatus,
+    statusChanged: calibratedStatus !== providerStatus,
+    namedGenericityRequired,
+    matchedGenericitySignals,
+    matchedValidBoldSignals,
+    blockingFindingCount,
+    contractFidelityWeak,
+    contractDriftDetected,
+    tasteVsFailureSeparated,
+    evidenceTextCount: evidenceTexts.length,
+    notes: calibrationNotes,
+  };
+}
+
+export function buildRubricCalibrationReport({
+  cases,
+  reviewRubricSummary,
+}) {
+  const normalizedCases = Array.isArray(cases) ? cases : [];
+  const results = normalizedCases.map((caseEntry) => {
+    const calibration = calibrateGenericityAssessment({
+      reviewRubricSummary,
+      designExecutionSummary: caseEntry.designExecutionSummary,
+      genericityAssessment: caseEntry.genericityAssessment,
+      rubricBreakdown: caseEntry.rubricBreakdown,
+      findings: caseEntry.findings,
+      notes: caseEntry.notes,
+      tasteVsFailureSeparated: caseEntry.tasteVsFailureSeparated,
+    });
+    const expected = caseEntry.expected && typeof caseEntry.expected === 'object'
+      ? caseEntry.expected
+      : {};
+
+    const statusMatches = String(expected.calibratedStatus || '') === calibration.calibratedStatus;
+    const contractDriftMatches = typeof expected.contractDriftDetected === 'boolean'
+      ? expected.contractDriftDetected === calibration.contractDriftDetected
+      : true;
+
+    return {
+      id: String(caseEntry.id || 'unknown-case'),
+      label: String(caseEntry.label || ''),
+      passed: statusMatches && contractDriftMatches,
+      expected,
+      calibration,
+    };
+  });
+
+  const passedCaseCount = results.filter((resultEntry) => resultEntry.passed).length;
+  const totalCases = results.length;
+
+  return {
+    generatedAt: new Date().toISOString(),
+    reportName: 'ui-rubric-calibration',
+    schemaVersion: '1.0',
+    passed: passedCaseCount === totalCases,
+    failureCount: totalCases - passedCaseCount,
+    totalCases,
+    passedCaseCount,
+    accuracyPercent: totalCases === 0 ? 0 : Number(((passedCaseCount / totalCases) * 100).toFixed(1)),
+    results,
+  };
+}

package/scripts/ui-design-judge/rubric-goldset.json

@@ -0,0 +1,188 @@
+{
+  "version": "ui-rubric-goldset-v1",
+  "reviewRubric": {
+    "version": "ui-rubric-v1",
+    "genericitySignals": [
+      "safe-centered-hero-without-product-rationale",
+      "balanced-card-grid-without-priority-shift",
+      "default-framework-button-and-input-treatment",
+      "trend-gradient-without-structural-role",
+      "interchangeable-dashboard-chrome"
+    ],
+    "validBoldSignals": [
+      "one-clear-signature-move",
+      "project-specific-layout-tension",
+      "purposeful-motion-as-identity",
+      "distinct-typographic-hierarchy",
+      "non-template-task-priority"
+    ],
+    "reportingRules": {
+      "mustExplainGenericity": true,
+      "mustSeparateTasteFromFailure": true,
+      "contractFidelityOverridesPersonalTaste": true
+    }
+  },
+  "cases": [
+    {
+      "id": "distinctive-authored-valid",
+      "label": "Authored UI with clear valid bold signals and no contract drift",
+      "designExecutionSummary": {
+        "contractReady": true
+      },
+      "genericityAssessment": {
+        "status": "distinctive",
+        "reason": "The UI lands one clear signature move with project specific layout tension and non template task priority."
+      },
+      "rubricBreakdown": [
+        {
+          "dimension": "contractFidelity",
+          "score": 88,
+          "verdict": "strong",
+          "reason": "Contract fidelity stays intact while the layout uses non template task priority.",
+          "blocking": true
+        }
+      ],
+      "findings": [],
+      "notes": [
+        "The composition keeps one clear signature move instead of collapsing into default chrome."
+      ],
+      "tasteVsFailureSeparated": true,
+      "expected": {
+        "calibratedStatus": "distinctive",
+        "contractDriftDetected": false
+      }
+    },
+    {
+      "id": "bold-but-contract-drift",
+      "label": "Bold ideas exist but contract drift keeps the result mixed",
+      "designExecutionSummary": {
+        "contractReady": true
+      },
+      "genericityAssessment": {
+        "status": "distinctive",
+        "reason": "There is project specific layout tension and one clear signature move, but the mobile hierarchy drifted."
+      },
+      "rubricBreakdown": [
+        {
+          "dimension": "contractFidelity",
+          "score": 54,
+          "verdict": "weak",
+          "reason": "Contract fidelity weakened when the primary CTA moved below secondary content.",
+          "blocking": true
+        }
+      ],
+      "findings": [
+        {
+          "area": "responsive",
+          "severity": "high",
+          "problem": "Mobile layout no longer follows the contract.",
+          "evidence": "The primary action now drops below supporting proof.",
+          "recommendation": "Restore the intended task order.",
+          "blockingRecommended": true
+        }
+      ],
+      "notes": [
+        "The distinctive direction is real, but contract fidelity still drifted."
+      ],
+      "tasteVsFailureSeparated": true,
+      "expected": {
+        "calibratedStatus": "mixed",
+        "contractDriftDetected": true
+      }
+    },
+    {
+      "id": "generic-template-drift",
+      "label": "Generic template drift is named clearly and stays generic",
+      "designExecutionSummary": {
+        "contractReady": false
+      },
+      "genericityAssessment": {
+        "status": "mixed",
+        "reason": "The redesign fell back to balanced card grid without priority shift plus default framework button and input treatment."
+      },
+      "rubricBreakdown": [
+        {
+          "dimension": "contractFidelity",
+          "score": 49,
+          "verdict": "weak",
+          "reason": "The hierarchy collapsed into interchangeable dashboard chrome.",
+          "blocking": true
+        }
+      ],
+      "findings": [
+        {
+          "area": "layout",
+          "severity": "high",
+          "problem": "The page uses balanced card grid without priority shift.",
+          "evidence": "Every block carries equal weight and the CTA no longer leads.",
+          "recommendation": "Rebuild the page around one dominant task surface.",
+          "blockingRecommended": true
+        }
+      ],
+      "notes": [
+        "Interchangeable dashboard chrome now dominates the flow."
+      ],
+      "tasteVsFailureSeparated": true,
+      "expected": {
+        "calibratedStatus": "generic",
+        "contractDriftDetected": true
+      }
+    },
+    {
+      "id": "provider-overcalls-generic",
+      "label": "Provider called the work generic, but the evidence points to valid authored moves",
+      "designExecutionSummary": {
+        "contractReady": true
+      },
+      "genericityAssessment": {
+        "status": "generic",
+        "reason": "The work feels generic, although it carries one clear signature move and distinct typographic hierarchy."
+      },
+      "rubricBreakdown": [
+        {
+          "dimension": "contractFidelity",
+          "score": 82,
+          "verdict": "strong",
+          "reason": "Contract fidelity is strong and the task order remains deliberate.",
+          "blocking": true
+        }
+      ],
+      "findings": [],
+      "notes": [
+        "The surface uses one clear signature move with distinct typographic hierarchy and non template task priority."
+      ],
+      "tasteVsFailureSeparated": true,
+      "expected": {
+        "calibratedStatus": "distinctive",
+        "contractDriftDetected": false
+      }
+    },
+    {
+      "id": "unnamed-genericity-claim",
+      "label": "Genericity claim without named drift signal drops to unclear",
+      "designExecutionSummary": {
+        "contractReady": true
+      },
+      "genericityAssessment": {
+        "status": "generic",
+        "reason": "The layout still feels a little too safe and familiar."
+      },
+      "rubricBreakdown": [
+        {
+          "dimension": "contractFidelity",
+          "score": 78,
+          "verdict": "acceptable",
+          "reason": "The contract is mostly intact.",
+          "blocking": true
+        }
+      ],
+      "findings": [],
+      "notes": [],
+      "tasteVsFailureSeparated": true,
+      "expected": {
+        "calibratedStatus": "unclear",
+        "contractDriftDetected": false
+      }
+    }
+  ]
+}