@ryuenn3123/agentic-senior-core 1.8.0 → 1.8.1

package/scripts/benchmark-gate.mjs

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+/**
+ * benchmark-gate.mjs
+ *
+ * Anti-regression gate for benchmark quality signals.
+ * Fails when benchmark metrics drop below configured thresholds.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFileSync } from 'node:child_process';
+
+const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
+const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
+const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
+const BENCHMARK_THRESHOLD_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-thresholds.json');
+const DETECTION_BENCHMARK_PATH = join(REPOSITORY_ROOT, 'scripts', 'detection-benchmark.mjs');
+
+function readThresholdConfiguration() {
+  if (!existsSync(BENCHMARK_THRESHOLD_PATH)) {
+    return {
+      minimumTop1Accuracy: 0.9,
+      maximumManualCorrectionRate: 0.12,
+      maximumTop1AccuracyDrop: 0.02,
+      maximumManualCorrectionIncrease: 0.03,
+      previousReleaseBaseline: {
+        top1Accuracy: 0.9167,
+        manualCorrectionRate: 0.0833,
+      },
+    };
+  }
+
+  return JSON.parse(readFileSync(BENCHMARK_THRESHOLD_PATH, 'utf8'));
+}
+
+function runDetectionBenchmark() {
+  const benchmarkRawOutput = execFileSync('node', [DETECTION_BENCHMARK_PATH], {
+    cwd: REPOSITORY_ROOT,
+    encoding: 'utf8',
+  });
+
+  return JSON.parse(benchmarkRawOutput);
+}
+
+function buildCheckResult(checkName, passed, details) {
+  return {
+    checkName,
+    passed,
+    details,
+  };
+}
+
+function runBenchmarkGate() {
+  const thresholdConfiguration = readThresholdConfiguration();
+  const benchmarkResult = runDetectionBenchmark();
+  const benchmarkChecks = [];
+
+  const top1AccuracyPassed = benchmarkResult.top1Accuracy >= thresholdConfiguration.minimumTop1Accuracy;
+  benchmarkChecks.push(
+    buildCheckResult(
+      'minimum-top1-accuracy',
+      top1AccuracyPassed,
+      `top1Accuracy=${benchmarkResult.top1Accuracy} minimum=${thresholdConfiguration.minimumTop1Accuracy}`,
+    ),
+  );
+
+  const manualCorrectionPassed = benchmarkResult.manualCorrectionRate <= thresholdConfiguration.maximumManualCorrectionRate;
+  benchmarkChecks.push(
+    buildCheckResult(
+      'maximum-manual-correction-rate',
+      manualCorrectionPassed,
+      `manualCorrectionRate=${benchmarkResult.manualCorrectionRate} maximum=${thresholdConfiguration.maximumManualCorrectionRate}`,
+    ),
+  );
+
+  const previousReleaseBaseline = thresholdConfiguration.previousReleaseBaseline;
+  if (previousReleaseBaseline && typeof previousReleaseBaseline === 'object') {
+    const top1AccuracyDrop = Number((previousReleaseBaseline.top1Accuracy - benchmarkResult.top1Accuracy).toFixed(4));
+    const manualCorrectionIncrease = Number((benchmarkResult.manualCorrectionRate - previousReleaseBaseline.manualCorrectionRate).toFixed(4));
+
+    const top1AccuracyDropPassed = top1AccuracyDrop <= thresholdConfiguration.maximumTop1AccuracyDrop;
+    benchmarkChecks.push(
+      buildCheckResult(
+        'maximum-top1-accuracy-drop',
+        top1AccuracyDropPassed,
+        `drop=${top1AccuracyDrop} maximum=${thresholdConfiguration.maximumTop1AccuracyDrop}`,
+      ),
+    );
+
+    const manualCorrectionIncreasePassed = manualCorrectionIncrease <= thresholdConfiguration.maximumManualCorrectionIncrease;
+    benchmarkChecks.push(
+      buildCheckResult(
+        'maximum-manual-correction-increase',
+        manualCorrectionIncreasePassed,
+        `increase=${manualCorrectionIncrease} maximum=${thresholdConfiguration.maximumManualCorrectionIncrease}`,
+      ),
+    );
+  }
+
+  const failedCheckCount = benchmarkChecks.filter((benchmarkCheck) => !benchmarkCheck.passed).length;
+  const benchmarkGateReport = {
+    generatedAt: new Date().toISOString(),
+    gateName: 'benchmark-gate',
+    passed: failedCheckCount === 0,
+    failureCount: failedCheckCount,
+    benchmarkResult: {
+      fixtureCount: benchmarkResult.fixtureCount,
+      top1Accuracy: benchmarkResult.top1Accuracy,
+      manualCorrectionRate: benchmarkResult.manualCorrectionRate,
+    },
+    thresholds: thresholdConfiguration,
+    results: benchmarkChecks,
+  };
+
+  console.log(JSON.stringify(benchmarkGateReport, null, 2));
+  process.exit(benchmarkGateReport.passed ? 0 : 1);
+}
+
+runBenchmarkGate();

package/scripts/benchmark-intelligence.mjs

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+
+/**
+ * benchmark-intelligence.mjs
+ *
+ * Competitive intelligence cadence validator.
+ * Ensures benchmark watchlist coverage and review freshness.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
+const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
+const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
+const WATCHLIST_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-watchlist.json');
+const REVIEW_SLA_DAYS = 14;
+const REQUIRED_BENCHMARK_REPOSITORIES = new Set([
+  'sickn33/antigravity-awesome-skills',
+  'github/awesome-copilot',
+  'MiniMax-AI/skills',
+]);
+
+function parseDateOrNull(rawDateValue) {
+  if (typeof rawDateValue !== 'string') {
+    return null;
+  }
+
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(rawDateValue)) {
+    return null;
+  }
+
+  const parsedDate = new Date(`${rawDateValue}T00:00:00.000Z`);
+  return Number.isNaN(parsedDate.getTime()) ? null : parsedDate;
+}
+
+function calculateAgeInDays(referenceDate, targetDate) {
+  const ageInMilliseconds = referenceDate.getTime() - targetDate.getTime();
+  return Math.floor(ageInMilliseconds / (1000 * 60 * 60 * 24));
+}
+
+function loadWatchlistConfiguration() {
+  if (!existsSync(WATCHLIST_PATH)) {
+    return { repositories: [] };
+  }
+
+  return JSON.parse(readFileSync(WATCHLIST_PATH, 'utf8'));
+}
+
+function runIntelligenceValidation() {
+  const watchlistConfiguration = loadWatchlistConfiguration();
+  const watchlistEntries = Array.isArray(watchlistConfiguration.repositories)
+    ? watchlistConfiguration.repositories
+    : [];
+  const validationResults = [];
+  const currentDate = new Date();
+
+  const trackedRepositoryNames = new Set();
+  for (const watchlistEntry of watchlistEntries) {
+    trackedRepositoryNames.add(watchlistEntry.repository);
+  }
+
+  for (const requiredRepositoryName of REQUIRED_BENCHMARK_REPOSITORIES) {
+    const hasRequiredRepository = trackedRepositoryNames.has(requiredRepositoryName);
+    validationResults.push({
+      checkName: 'required-benchmark-repository',
+      repository: requiredRepositoryName,
+      passed: hasRequiredRepository,
+      details: hasRequiredRepository
+        ? `${requiredRepositoryName} is present in watchlist`
+        : `${requiredRepositoryName} is missing from watchlist`,
+    });
+  }
+
+  const watchlistReport = watchlistEntries.map((watchlistEntry) => {
+    const repositoryName = watchlistEntry.repository;
+    const repositoryOwner = watchlistEntry.owner;
+    const lastReviewedDate = parseDateOrNull(watchlistEntry.lastReviewedAt);
+
+    const hasOwner = typeof repositoryOwner === 'string' && repositoryOwner.trim().length > 0;
+    validationResults.push({
+      checkName: 'watchlist-owner-defined',
+      repository: repositoryName,
+      passed: hasOwner,
+      details: hasOwner ? `Owner ${repositoryOwner} is defined` : 'Owner is missing',
+    });
+
+    if (!lastReviewedDate) {
+      validationResults.push({
+        checkName: 'review-date-format',
+        repository: repositoryName,
+        passed: false,
+        details: `Invalid or missing lastReviewedAt: ${String(watchlistEntry.lastReviewedAt)}`,
+      });
+
+      return {
+        repository: repositoryName,
+        owner: repositoryOwner,
+        lastReviewedAt: watchlistEntry.lastReviewedAt,
+        ageInDays: null,
+        stale: true,
+      };
+    }
+
+    const reviewAgeInDays = calculateAgeInDays(currentDate, lastReviewedDate);
+    const reviewWithinSla = reviewAgeInDays <= REVIEW_SLA_DAYS;
+
+    validationResults.push({
+      checkName: 'review-sla-compliance',
+      repository: repositoryName,
+      passed: reviewWithinSla,
+      details: `ageInDays=${reviewAgeInDays} slaDays=${REVIEW_SLA_DAYS}`,
+    });
+
+    return {
+      repository: repositoryName,
+      owner: repositoryOwner,
+      lastReviewedAt: watchlistEntry.lastReviewedAt,
+      ageInDays: reviewAgeInDays,
+      stale: !reviewWithinSla,
+    };
+  });
+
+  const failedCheckCount = validationResults.filter((validationResult) => !validationResult.passed).length;
+  const intelligenceReport = {
+    generatedAt: new Date().toISOString(),
+    reportName: 'benchmark-intelligence',
+    passed: failedCheckCount === 0,
+    failureCount: failedCheckCount,
+    reviewSlaDays: REVIEW_SLA_DAYS,
+    watchlist: watchlistReport,
+    results: validationResults,
+  };
+
+  console.log(JSON.stringify(intelligenceReport, null, 2));
+  process.exit(intelligenceReport.passed ? 0 : 1);
+}
+
+runIntelligenceValidation();

package/scripts/skill-tier-policy.mjs

@@ -0,0 +1,76 @@
+export const SKILL_TIER_MINIMUMS = {
+  standard: { minWords: 60, minHeadings: 1, minChecklistItems: 0, minCodeBlocks: 0 },
+  advance: { minWords: 100, minHeadings: 2, minChecklistItems: 1, minCodeBlocks: 0 },
+  expert: { minWords: 130, minHeadings: 3, minChecklistItems: 1, minCodeBlocks: 0 },
+  above: { minWords: 240, minHeadings: 3, minChecklistItems: 1, minCodeBlocks: 1 },
+};
+
+export function countWords(markdownContent) {
+  return markdownContent
+    .replace(/```[\s\S]*?```/g, ' ')
+    .replace(/[^A-Za-z0-9_\-\s]/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean).length;
+}
+
+export function countMarkdownHeadings(markdownContent) {
+  const headingMatches = markdownContent.match(/^#{2,6}\s+/gm);
+  return headingMatches ? headingMatches.length : 0;
+}
+
+export function countChecklistItems(markdownContent) {
+  const checklistMatches = markdownContent.match(/^\s*[-*]\s+\[[ xX]\]\s+/gm);
+  return checklistMatches ? checklistMatches.length : 0;
+}
+
+export function countCodeBlocks(markdownContent) {
+  const fenceMatches = markdownContent.match(/```/g);
+  if (!fenceMatches) {
+    return 0;
+  }
+
+  return Math.floor(fenceMatches.length / 2);
+}
+
+export function extractSkillTier(markdownContent) {
+  const normalizedMarkdownContent = markdownContent.replace(/\*\*/g, '');
+  const tierMatch = normalizedMarkdownContent.match(/\bTier\s*:\s*`?(standard|advance|expert|above)`?\b/i);
+  return tierMatch ? tierMatch[1].toLowerCase() : null;
+}
+
+export function validateSkillTopicContent(markdownContent) {
+  const detectedTier = extractSkillTier(markdownContent);
+
+  if (!detectedTier) {
+    return { isValid: false, reason: 'missing-tier' };
+  }
+
+  const minimumRules = SKILL_TIER_MINIMUMS[detectedTier];
+  if (!minimumRules) {
+    return { isValid: false, reason: 'unsupported-tier', detectedTier };
+  }
+
+  const wordCount = countWords(markdownContent);
+  const headingCount = countMarkdownHeadings(markdownContent);
+  const checklistCount = countChecklistItems(markdownContent);
+  const codeBlockCount = countCodeBlocks(markdownContent);
+
+  if (wordCount < minimumRules.minWords) {
+    return { isValid: false, reason: 'word-count', detectedTier, wordCount, minimumRules };
+  }
+
+  if (headingCount < minimumRules.minHeadings) {
+    return { isValid: false, reason: 'heading-count', detectedTier, headingCount, minimumRules };
+  }
+
+  if (checklistCount < minimumRules.minChecklistItems) {
+    return { isValid: false, reason: 'checklist-count', detectedTier, checklistCount, minimumRules };
+  }
+
+  if (codeBlockCount < minimumRules.minCodeBlocks) {
+    return { isValid: false, reason: 'code-block-count', detectedTier, codeBlockCount, minimumRules };
+  }
+
+  return { isValid: true, detectedTier, wordCount, headingCount, checklistCount, codeBlockCount, minimumRules };
+}

package/scripts/validate.mjs

@@ -16,6 +16,7 @@
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { dirname, join, relative, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { validateSkillTopicContent } from './skill-tier-policy.mjs';
 
 const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
 const ROOT_DIR = resolve(dirname(SCRIPT_FILE_PATH), '..');
@@ -25,6 +26,7 @@ const CHANGELOG_PATH = join(ROOT_DIR, 'CHANGELOG.md');
 const README_PATH = join(ROOT_DIR, 'README.md');
 const POLICY_FILE_PATH = join(ROOT_DIR, '.agent-context', 'policies', 'llm-judge-threshold.json');
 const OVERRIDE_FILE_PATH = join(ROOT_DIR, '.agent-override.md');
+const SKILLS_DIR = join(AGENT_CONTEXT_DIR, 'skills');
 const GENERATED_RULE_FILES = ['.cursorrules', '.windsurfrules'];
 const ALLOWED_SEVERITIES = new Set(['critical', 'high', 'medium', 'low']);
 const OVERRIDE_WARNING_WINDOW_DAYS = 30;
@@ -101,6 +103,8 @@ async function validateRequiredFiles() {
     'scripts/validate.mjs',
     'scripts/llm-judge.mjs',
     'scripts/detection-benchmark.mjs',
+    'scripts/benchmark-gate.mjs',
+    'scripts/benchmark-intelligence.mjs',
     'scripts/frontend-usability-audit.mjs',
     'scripts/release-gate.mjs',
     'scripts/generate-sbom.mjs',
@@ -121,8 +125,13 @@ async function validateRequiredFiles() {
     'docs/v1.7-execution-playbook.md',
     'docs/v1.7-issue-breakdown.md',
     'docs/v1.8-operations-playbook.md',
+    'docs/v2-upgrade-playbook.md',
+    '.agent-context/state/benchmark-watchlist.json',
+    '.agent-context/state/skill-platform.json',
+    '.agent-context/skills/index.json',
     '.github/workflows/release-gate.yml',
     '.github/workflows/sbom-compliance.yml',
+    '.github/workflows/benchmark-intelligence.yml',
     'tests/cli-smoke.test.mjs',
     'tests/llm-judge.test.mjs',
     'tests/enterprise-ops.test.mjs',
@@ -204,10 +213,24 @@ async function validateRuleFiles() {
     'profiles/platform.md',
     'review-checklists/pr-checklist.md',
     'review-checklists/frontend-usability.md',
+    'review-checklists/frontend-skill-parity.md',
     'review-checklists/release-operations.md',
     'review-checklists/security-audit.md',
     'review-checklists/performance-audit.md',
     'review-checklists/architecture-review.md',
+    'skills/README.md',
+    'skills/frontend/README.md',
+    'skills/backend/README.md',
+    'skills/fullstack/README.md',
+    'skills/cli/README.md',
+    'skills/distribution/README.md',
+    'skills/review-quality/README.md',
+    'skills/frontend.md',
+    'skills/backend.md',
+    'skills/fullstack.md',
+    'skills/cli.md',
+    'skills/distribution.md',
+    'skills/review-quality.md',
     'state/architecture-map.md',
     'state/dependency-map.md',
   ];
@@ -230,6 +253,63 @@ async function validateRuleFiles() {
   }
 }
 
+async function validateSkillTierQuality() {
+  console.log('\nChecking skill tier quality...');
+
+  const skillMarkdownFiles = await collectFiles(SKILLS_DIR, (fileName) => fileName.endsWith('.md'));
+  const scopedSkillTopicFiles = skillMarkdownFiles.filter((skillFilePath) => {
+    if (skillFilePath.endsWith('README.md')) {
+      return false;
+    }
+
+    const relativeSkillPath = relative(SKILLS_DIR, skillFilePath);
+    return /[\\/]/.test(relativeSkillPath);
+  });
+
+  for (const skillTopicPath of scopedSkillTopicFiles) {
+    const skillTopicContent = await readTextFile(skillTopicPath);
+    const relativeSkillTopicPath = relative(ROOT_DIR, skillTopicPath);
+    const validationResult = validateSkillTopicContent(skillTopicContent);
+
+    if (!validationResult.isValid) {
+      if (validationResult.reason === 'missing-tier') {
+        fail(`${relativeSkillTopicPath} is missing explicit Tier metadata`);
+        continue;
+      }
+
+      if (validationResult.reason === 'unsupported-tier') {
+        fail(`${relativeSkillTopicPath} has unsupported tier: ${validationResult.detectedTier}`);
+        continue;
+      }
+
+      if (validationResult.reason === 'word-count') {
+        fail(`${relativeSkillTopicPath} tier ${validationResult.detectedTier} must include at least ${validationResult.minimumRules.minWords} words (found ${validationResult.wordCount})`);
+        continue;
+      }
+
+      if (validationResult.reason === 'heading-count') {
+        fail(`${relativeSkillTopicPath} tier ${validationResult.detectedTier} must include at least ${validationResult.minimumRules.minHeadings} section headings (found ${validationResult.headingCount})`);
+        continue;
+      }
+
+      if (validationResult.reason === 'checklist-count') {
+        fail(`${relativeSkillTopicPath} tier ${validationResult.detectedTier} must include at least ${validationResult.minimumRules.minChecklistItems} checklist item(s) (found ${validationResult.checklistCount})`);
+        continue;
+      }
+
+      if (validationResult.reason === 'code-block-count') {
+        fail(`${relativeSkillTopicPath} tier ${validationResult.detectedTier} must include at least ${validationResult.minimumRules.minCodeBlocks} code block(s) (found ${validationResult.codeBlockCount})`);
+        continue;
+      }
+
+      fail(`${relativeSkillTopicPath} failed tier validation`);
+      continue;
+    }
+
+    pass(`${relativeSkillTopicPath} tier ${validationResult.detectedTier} quality gate passed`);
+  }
+}
+
 function stripMarkdownCodeBlocks(markdownText) {
   return markdownText.replace(/```[\s\S]*?```/g, '');
 }
@@ -484,6 +564,7 @@ async function validateDocumentationFlow() {
     'npm run validate',
     'docs/faq.md',
     'docs/deep-dive.md',
+    'docs/v2-upgrade-playbook.md',
   ];
 
   for (const requiredReadmeSnippet of requiredReadmeSnippets) {
@@ -523,6 +604,7 @@ async function main() {
   await validateRequiredFiles();
   await validateMarkdownFiles();
   await validateRuleFiles();
+  await validateSkillTierQuality();
   await validateOverrideGovernance();
   await validateAgentsManifest();
   await validateCrossReferences();