forge-workflow 0.0.9 → 0.1.0-beta.2

package/scripts/test-ci-shard.js

@@ -0,0 +1,251 @@
+#!/usr/bin/env node
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const rootDir = path.join(__dirname, '..');
+const reportDir = path.join(rootDir, 'test-results');
+const PROFILE_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
+
+const DEFAULT_SMOKE_FILES = [
+  'test/cli-flags.test.js',
+  'test/commands/test.test.js',
+  'test/ci-workflow.test.js',
+  'test/detect-agent.test.js',
+  'test/scripts/test-runner.test.js',
+  'test/workflow-profiles.test.js',
+];
+
+function parseArgs(argv) {
+  const args = { label: 'unit', mode: 'shard' };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const current = argv[index];
+    const next = argv[index + 1];
+
+    if (current === '--mode') args.mode = next;
+    if (current === '--label') args.label = next;
+    if (current === '--shard-index') args.shardIndex = Number.parseInt(next, 10);
+    if (current === '--shard-total') args.shardTotal = Number.parseInt(next, 10);
+  }
+
+  return args;
+}
+
+function getUnitTestRoots() {
+  const roots = [path.join(rootDir, 'test')];
+  const packagesDir = path.join(rootDir, 'packages');
+  if (!fs.existsSync(packagesDir)) {
+    return roots.filter((dir) => fs.existsSync(dir));
+  }
+
+  for (const entry of fs.readdirSync(packagesDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const testRoot = path.join(packagesDir, entry.name, 'test');
+    if (fs.existsSync(testRoot)) {
+      roots.push(testRoot);
+    }
+  }
+
+  return roots;
+}
+
+function walkTests(dir) {
+  const results = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const absolute = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (absolute === path.join(rootDir, 'test', 'e2e')) continue;
+      results.push(...walkTests(absolute));
+      continue;
+    }
+
+    if (!entry.name.endsWith('.test.js') && !entry.name.endsWith('.spec.js')) continue;
+    results.push(path.relative(rootDir, absolute).replace(/\\/g, '/'));
+  }
+  return results;
+}
+
+function listAllUnitTests() {
+  return getUnitTestRoots()
+    .flatMap((dir) => walkTests(dir))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+function ensureFilesExist(files) {
+  return files.filter((file) => fs.existsSync(path.join(rootDir, file)));
+}
+
+function normalizePath(file) {
+  return (file || '').replace(/\\/g, '/').replace(/\/+/g, '/').replace(/^\.\//, '');
+}
+
+function selectModuloShard(files, shardIndex, shardTotal) {
+  if (!Number.isInteger(shardIndex) || !Number.isInteger(shardTotal) || shardTotal < 1) {
+    throw new Error(`Invalid shard args: index=${shardIndex} total=${shardTotal}`);
+  }
+  return files.filter((_, index) => index % shardTotal === shardIndex);
+}
+
+function listProfileFiles(dir) {
+  if (!fs.existsSync(dir)) return [];
+  return fs.readdirSync(dir)
+    .filter((entry) => entry.endsWith('.profile.json'))
+    .map((entry) => path.join(dir, entry));
+}
+
+function readNewestProfile(dir, { now = Date.now(), maxAgeMs = PROFILE_MAX_AGE_MS } = {}) {
+  const candidates = [];
+  for (const file of listProfileFiles(dir)) {
+    try {
+      const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+      const timestamp = Date.parse(parsed.timestamp || '');
+      if (!Number.isFinite(timestamp)) continue;
+      if ((now - timestamp) > maxAgeMs) continue;
+      candidates.push({ file, profile: parsed, timestamp });
+    } catch (_error) {
+      // Ignore invalid profile files and fall back to deterministic sharding.
+    }
+  }
+
+  candidates.sort((left, right) => right.timestamp - left.timestamp || left.file.localeCompare(right.file));
+  return candidates[0]?.profile || null;
+}
+
+function createDurationMap(profile) {
+  const durationMap = new Map();
+  const entries = Array.isArray(profile?.allFileDurations) && profile.allFileDurations.length > 0
+    ? profile.allFileDurations
+    : profile?.slowestFiles || [];
+  for (const entry of entries) {
+    if (!entry || typeof entry.file !== 'string') continue;
+    durationMap.set(normalizePath(entry.file), Number(entry.durationMs) || 0);
+  }
+  return durationMap;
+}
+
+function selectRuntimeBalancedShard(files, shardIndex, shardTotal, durationMap) {
+  if (!Number.isInteger(shardIndex) || !Number.isInteger(shardTotal) || shardTotal < 1) {
+    throw new Error(`Invalid shard args: index=${shardIndex} total=${shardTotal}`);
+  }
+
+  const shards = Array.from({ length: shardTotal }, (_, index) => ({
+    files: [],
+    index,
+    totalDurationMs: 0,
+  }));
+  const weightedFiles = files
+    .map((file) => ({
+      durationMs: durationMap.get(normalizePath(file)) || 0,
+      file,
+    }))
+    .sort((left, right) => right.durationMs - left.durationMs || left.file.localeCompare(right.file));
+
+  for (const entry of weightedFiles) {
+    const targetShard = shards
+      .slice()
+      .sort((left, right) => left.totalDurationMs - right.totalDurationMs
+        || left.files.length - right.files.length
+        || left.index - right.index)[0];
+    targetShard.files.push(entry.file);
+    targetShard.totalDurationMs += entry.durationMs;
+  }
+
+  return shards[shardIndex].files.sort((left, right) => left.localeCompare(right));
+}
+
+function selectShard(files, shardIndex, shardTotal, durationMap = new Map()) {
+  if (durationMap.size === 0) {
+    return selectModuloShard(files, shardIndex, shardTotal);
+  }
+  return selectRuntimeBalancedShard(files, shardIndex, shardTotal, durationMap);
+}
+
+function getShardPlan(args, { allUnitTests, durationMap }) {
+  if (args.mode === 'smoke') {
+    return {
+      files: ensureFilesExist(DEFAULT_SMOKE_FILES),
+      source: 'smoke',
+    };
+  }
+
+  const effectiveDurationMap = durationMap || createDurationMap(readNewestProfile(reportDir));
+  return {
+    files: selectShard(allUnitTests, args.shardIndex, args.shardTotal, effectiveDurationMap),
+    source: effectiveDurationMap && effectiveDurationMap.size > 0 ? 'runtime-balanced' : 'modulo',
+  };
+}
+
+function buildBunTestArgs({ junitPath, files }) {
+  return [
+    'test',
+    '--timeout',
+    '15000',
+    '--reporter=junit',
+    '--reporter-outfile',
+    junitPath,
+    ...files,
+  ];
+}
+
+function runTests(label, files) {
+  fs.mkdirSync(reportDir, { recursive: true });
+  const junitPath = path.join(reportDir, `${label}.xml`);
+  const bunCommand = process.env.BUN_EXE || 'bun';
+  const result = spawnSync(bunCommand, buildBunTestArgs({ junitPath, files }), {
+    cwd: rootDir,
+    stdio: 'inherit',
+    shell: process.platform === 'win32',
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+
+  return result.status ?? 1;
+}
+
+function main(argv = process.argv.slice(2)) {
+  const args = parseArgs(argv);
+  const allUnitTests = listAllUnitTests();
+  const plan = getShardPlan(args, { allUnitTests });
+  const files = plan.files;
+
+  if (files.length === 0) {
+    console.log(`No test files selected for ${args.label}`);
+    return 0;
+  }
+
+  console.log(`Selected ${files.length} test file(s) for ${args.label} via ${plan.source}`);
+  for (const file of files) {
+    console.log(` - ${file}`);
+  }
+
+  return runTests(args.label, files);
+}
+
+if (require.main === module) {
+  process.exit(main());
+}
+
+module.exports = {
+  DEFAULT_SMOKE_FILES,
+  PROFILE_MAX_AGE_MS,
+  buildBunTestArgs,
+  createDurationMap,
+  ensureFilesExist,
+  getShardPlan,
+  listProfileFiles,
+  main,
+  normalizePath,
+  parseArgs,
+  readNewestProfile,
+  runTests,
+  getUnitTestRoots,
+  listAllUnitTests,
+  selectModuloShard,
+  selectRuntimeBalancedShard,
+  selectShard,
+  walkTests,
+};

package/scripts/test-dashboard.js

@@ -4,35 +4,59 @@
  * Test Quality Dashboard — collects test metrics into a JSON summary.
  *
  * Usage:
- *   node scripts/test-dashboard.js           # Human-readable output
- *   node scripts/test-dashboard.js --json    # JSON output (for CI/tests)
+ *   node scripts/test-dashboard.js
+ *   node scripts/test-dashboard.js --json
+ *   node scripts/test-dashboard.js --profiles-dir test-results/profiles
  */
 
 const fs = require('node:fs');
 const path = require('node:path');
-const { execFileSync } = require('node:child_process');
 
 const rootDir = path.join(__dirname, '..');
 
+function parseArgs(argv) {
+  const args = {
+    json: argv.includes('--json'),
+    profilesDir: path.join(rootDir, 'test-results'),
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    if (argv[index] === '--profiles-dir') {
+      args.profilesDir = path.resolve(rootDir, argv[index + 1]);
+    }
+  }
+
+  return args;
+}
+
+function walk(dir, extension) {
+  if (!fs.existsSync(dir)) return [];
+  const results = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const absolute = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...walk(absolute, extension));
+      continue;
+    }
+    if (absolute.endsWith(extension)) {
+      results.push(absolute);
+    }
+  }
+  return results;
+}
+
 function getTestCount() {
-  // Count test cases by scanning test files for test() calls
-  // This avoids running the full test suite (which would be slow/recursive)
   const testDirs = [
     path.join(rootDir, 'test'),
     path.join(rootDir, 'test-env'),
-    path.join(rootDir, 'packages', 'skills', 'test')
+    path.join(rootDir, 'packages', 'skills', 'test'),
   ];
   let count = 0;
   for (const dir of testDirs) {
-    if (!fs.existsSync(dir)) continue;
-    const files = fs.readdirSync(dir, { recursive: true });
-    for (const file of files) {
-      const filePath = path.join(dir, String(file));
-      if (!filePath.endsWith('.test.js') && !filePath.endsWith('.spec.js')) continue;
-      if (!fs.statSync(filePath).isFile()) continue;
-      const content = fs.readFileSync(filePath, 'utf-8');
-      // Count test() and it() calls (excluding test.skip)
-      const testMatches = content.match(/\btest\s*\(/g) || [];
+    for (const file of walk(dir, '.js')) {
+      if (!file.endsWith('.test.js') && !file.endsWith('.spec.js')) continue;
+      const content = fs.readFileSync(file, 'utf-8');
+      const testMatches = content.match(/\btest(?:\.skip|\.todo|\.if|\.skipIf)?\s*\(/g) || [];
       const itMatches = content.match(/\bit\s*\(/g) || [];
       const skipMatches = content.match(/\btest\.skip\s*\(/g) || [];
       count += testMatches.length + itMatches.length - skipMatches.length;
@@ -46,24 +70,19 @@ function getCoverageThreshold() {
   return pkg.c8 ? pkg.c8.lines : 0;
 }
 
-function getEslintWarnings() {
-  // We enforce --max-warnings 0 in pre-push hooks (lefthook.yml),
-  // so this is always 0 when code passes CI. Report 0 to avoid
-  // slow ESLint execution during dashboard generation.
-  return 0;
-}
-
 function getSkippedTestCount() {
-  try {
-    const output = execFileSync('grep', ['-r', 'test.skip', 'test/', 'test-env/'], {
-      cwd: rootDir,
-      encoding: 'utf-8',
-      stdio: ['pipe', 'pipe', 'pipe']
-    });
-    return output.trim().split('\n').filter(Boolean).length;
-  } catch (_e) {
-    return 0;
+  const testDirs = [
+    path.join(rootDir, 'test'),
+    path.join(rootDir, 'test-env'),
+    path.join(rootDir, 'packages', 'skills', 'test'),
+  ];
+  let skipped = 0;
+  for (const file of testDirs.flatMap((dir) => walk(dir, '.js'))) {
+    if (!file.endsWith('.test.js') && !file.endsWith('.spec.js')) continue;
+    const content = fs.readFileSync(file, 'utf8');
+    skipped += (content.match(/\btest\.skip\s*\(/g) || []).length;
   }
+  return skipped;
 }
 
 function getMutationScore() {
@@ -71,37 +90,137 @@ function getMutationScore() {
   if (!fs.existsSync(reportPath)) return null;
   try {
     const report = JSON.parse(fs.readFileSync(reportPath, 'utf-8'));
-    if (report.schemaVersion) {
-      // Stryker v8+ JSON report format
-      const files = Object.values(report.files || {});
-      let killed = 0;
-      let total = 0;
-      for (const file of files) {
-        for (const mutant of (file.mutants || [])) {
-          total++;
-          if (mutant.status === 'Killed') killed++;
-        }
+    const files = Object.values(report.files || {});
+    let killed = 0;
+    let total = 0;
+    for (const file of files) {
+      for (const mutant of (file.mutants || [])) {
+        total += 1;
+        if (mutant.status === 'Killed') killed += 1;
       }
-      return total > 0 ? Math.round((killed / total) * 100) : null;
     }
+    return total > 0 ? Math.round((killed / total) * 100) : null;
+  } catch (_error) {
     return null;
-  } catch (_e) {
-    return null;
   }
 }
 
+function getProfileMetrics(profilesDir) {
+  const files = walk(profilesDir, '.json').filter((file) => file.endsWith('.profile.json'));
+  if (files.length === 0) {
+    return {
+      suiteDurationMs: 0,
+      slowestFiles: [],
+      timedOutFiles: [],
+      integrationSkipped: true,
+    };
+  }
+
+  const slowest = new Map();
+  const timedOut = new Set();
+  let suiteDurationMs = 0;
+  let integrationSkipped = false;
+
+  for (const file of files) {
+    let profile;
+    try {
+      profile = JSON.parse(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+      console.warn(`Skipping malformed profile ${path.relative(rootDir, file)}: ${error.message}`);
+      continue;
+    }
+
+    if (!profile || typeof profile !== 'object') {
+      console.warn(`Skipping invalid profile ${path.relative(rootDir, file)}: expected object`);
+      continue;
+    }
+
+    suiteDurationMs += profile.suiteDurationMs || 0;
+    integrationSkipped = integrationSkipped || Boolean(profile.integrationSkipped);
+
+    const entries = Array.isArray(profile.allFileDurations) && profile.allFileDurations.length > 0
+      ? profile.allFileDurations
+      : Array.isArray(profile.slowestFiles) ? profile.slowestFiles : [];
+    for (const entry of entries) {
+      if (!entry || typeof entry !== 'object' || typeof entry.file !== 'string') {
+        continue;
+      }
+      slowest.set(entry.file, Math.max(slowest.get(entry.file) || 0, entry.durationMs || 0));
+    }
+
+    for (const timedOutFile of Array.isArray(profile.timedOutFiles) ? profile.timedOutFiles : []) {
+      if (typeof timedOutFile !== 'string' || !timedOutFile) {
+        continue;
+      }
+      timedOut.add(timedOutFile);
+    }
+  }
+
+  return {
+    suiteDurationMs,
+    slowestFiles: Array.from(slowest.entries())
+      .map(([file, durationMs]) => ({ file, durationMs }))
+      .sort((left, right) => right.durationMs - left.durationMs)
+      .slice(0, 10),
+    timedOutFiles: Array.from(timedOut).sort((left, right) => left.localeCompare(right)),
+    integrationSkipped,
+  };
+}
+
+function getBenchmarkMetrics(profilesDir) {
+  const files = walk(profilesDir, '.json').filter((file) => path.basename(file) === 'benchmark-results.json');
+  if (files.length === 0) {
+    return {
+      groupCount: 0,
+      groups: [],
+      slowestGroup: null,
+      totalMedianMs: 0,
+    };
+  }
+
+  try {
+    const benchmark = JSON.parse(fs.readFileSync(files.sort((left, right) => right.localeCompare(left))[0], 'utf8'));
+    const groups = Array.isArray(benchmark.groups)
+      ? benchmark.groups.map((group) => ({
+        groupId: group.groupId,
+        groupLabel: group.groupLabel,
+        medianMs: group.medianMs || 0,
+      }))
+      : [];
+    return {
+      groupCount: groups.length,
+      groups,
+      slowestGroup: benchmark.slowestGroup || null,
+      totalMedianMs: benchmark.totalMedianMs || groups.reduce((sum, group) => sum + group.medianMs, 0),
+    };
+  } catch (_error) {
+    return {
+      groupCount: 0,
+      groups: [],
+      slowestGroup: null,
+      totalMedianMs: 0,
+    };
+  }
+}
+
+const args = parseArgs(process.argv.slice(2));
+const profileMetrics = getProfileMetrics(args.profilesDir);
+const benchmarkMetrics = getBenchmarkMetrics(args.profilesDir);
 const dashboard = {
+  benchmarks: benchmarkMetrics,
   testCount: getTestCount(),
   coverageThreshold: getCoverageThreshold(),
-  eslintWarnings: getEslintWarnings(),
+  eslintWarnings: 0,
   skippedTests: getSkippedTestCount(),
   mutationScore: getMutationScore(),
-  timestamp: new Date().toISOString()
+  suiteDurationMs: profileMetrics.suiteDurationMs,
+  slowestFiles: profileMetrics.slowestFiles,
+  timedOutFiles: profileMetrics.timedOutFiles,
+  integrationSkipped: profileMetrics.integrationSkipped,
+  timestamp: new Date().toISOString(),
 };
 
-const jsonMode = process.argv.includes('--json');
-
-if (jsonMode) {
+if (args.json) {
   process.stdout.write(JSON.stringify(dashboard));
 } else {
   console.log('\n  Test Quality Dashboard');
@@ -110,14 +229,31 @@ if (jsonMode) {
   console.log(`  Coverage threshold: ${dashboard.coverageThreshold}%`);
   console.log(`  ESLint warnings:    ${dashboard.eslintWarnings}`);
   console.log(`  Skipped tests:      ${dashboard.skippedTests}`);
-  console.log(`  Mutation score:     ${dashboard.mutationScore !== null ? dashboard.mutationScore + '%' : 'N/A (run test:mutation first)'}`);
+  console.log(`  Suite duration:     ${dashboard.suiteDurationMs}ms`);
+  console.log(`  Timed out files:    ${dashboard.timedOutFiles.length}`);
+  console.log(`  Integration skipped:${dashboard.integrationSkipped ? ' yes' : ' no'}`);
+  console.log(`  Mutation score:     ${dashboard.mutationScore !== null ? `${dashboard.mutationScore}%` : 'N/A (run test:mutation first)'}`);
+  if (dashboard.benchmarks.groupCount > 0) {
+    console.log(`  Benchmarks:         ${dashboard.benchmarks.groupCount} groups (${dashboard.benchmarks.totalMedianMs}ms median total)`);
+  }
   console.log(`  Generated:          ${dashboard.timestamp}`);
+  if (dashboard.slowestFiles.length > 0) {
+    console.log('\n  Slowest files:');
+    for (const entry of dashboard.slowestFiles) {
+      console.log(`   - ${entry.file}: ${entry.durationMs}ms`);
+    }
+  }
+  if (dashboard.benchmarks.groupCount > 0) {
+    console.log('\n  Benchmark medians:');
+    for (const entry of dashboard.benchmarks.groups) {
+      console.log(`   - ${entry.groupLabel}: ${entry.medianMs}ms`);
+    }
+  }
   console.log('');
 
-  // Write results file
   const outputPath = path.join(rootDir, 'test-dashboard.json');
   fs.writeFileSync(outputPath, JSON.stringify(dashboard, null, 2));
-  console.log(`  Results saved to: test-dashboard.json\n`);
+  console.log('  Results saved to: test-dashboard.json\n');
 }
 
 module.exports = dashboard;