forge-workflow 0.0.9 → 0.1.0-beta.2

package/scripts/behavioral-judge.sh

@@ -66,10 +66,25 @@ else
   INPUT="$(cat)"
 fi
 
-# Extract plan_output text if input is JSON, otherwise use raw input
-PLAN_TEXT="$INPUT"
-if command -v python3 >/dev/null 2>&1; then
-  EXTRACTED=$(echo "$INPUT" | python3 -c "
+extract_plan_text() {
+  if command -v node >/dev/null 2>&1; then
+    node -e "
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { input += chunk; });
+process.stdin.on('end', () => {
+  try {
+    const parsed = JSON.parse(input);
+    const value = parsed && parsed.plan_output;
+    if (value) process.stdout.write(String(value));
+  } catch (_) {}
+});
+" 2>/dev/null
+    return $?
+  fi
+
+  if command -v python3 >/dev/null 2>&1; then
+    python3 -c "
 import sys, json
 try:
     d = json.load(sys.stdin)
@@ -78,10 +93,16 @@ try:
         print(val)
 except:
     pass
-" 2>/dev/null) || true
-  if [ -n "$EXTRACTED" ]; then
-    PLAN_TEXT="$EXTRACTED"
+" 2>/dev/null
+    return $?
   fi
+}
+
+# Extract plan_output text if input is JSON, otherwise use raw input
+PLAN_TEXT="$INPUT"
+EXTRACTED=$(printf '%s' "$INPUT" | extract_plan_text) || true
+if [ -n "$EXTRACTED" ]; then
+  PLAN_TEXT="$EXTRACTED"
 fi
 
 # ─── Judge prompt ────────────────────────────────────────────────────────────
@@ -158,7 +179,22 @@ parse_scores() {
   local response="$1"
   # Extract the JSON object from the LLM response content
   local content
-  content=$(echo "$response" | python3 -c "
+  if command -v node >/dev/null 2>&1; then
+    content=$(printf '%s' "$response" | node -e "
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { input += chunk; });
+process.stdin.on('end', () => {
+  try {
+    const data = JSON.parse(input);
+    const choice = (data.choices || [])[0] || {};
+    const message = choice.message || {};
+    process.stdout.write(String(message.content || ''));
+  } catch (_) {}
+});
+" 2>/dev/null) || true
+  elif command -v python3 >/dev/null 2>&1; then
+    content=$(printf '%s' "$response" | python3 -c "
 import sys, json
 try:
     data = json.load(sys.stdin)
@@ -170,6 +206,9 @@ try:
 except Exception as e:
     pass
 " 2>/dev/null) || true
+  else
+    content=""
+  fi
 
   if [ -z "$content" ]; then
     echo ""
@@ -178,7 +217,37 @@ except Exception as e:
 
   # Parse scores from content JSON
   local scores
-  scores=$(echo "$content" | python3 -c "
+  if command -v node >/dev/null 2>&1; then
+    scores=$(printf '%s' "$content" | node -e "
+let text = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { text += chunk; });
+process.stdin.on('end', () => {
+  function emit(value) {
+    const security = Number.parseInt(value.security, 10);
+    const tdd = Number.parseInt(value.tdd, 10);
+    const design = Number.parseInt(value.design, 10);
+    const structural = Number.parseInt(value.structural, 10);
+    if ([security, tdd, design, structural].every(score => Number.isInteger(score) && score >= 0 && score <= 5)) {
+      console.log([security, tdd, design, structural].join(' '));
+      return true;
+    }
+    return false;
+  }
+  try {
+    if (emit(JSON.parse(text.trim()))) return;
+  } catch (_) {}
+  const match = text.match(/\{[^}]+\}/s);
+  if (match) {
+    try {
+      if (emit(JSON.parse(match[0]))) return;
+    } catch (_) {}
+  }
+  process.exit(1);
+});
+" 2>/dev/null) || { echo ""; return 1; }
+  elif command -v python3 >/dev/null 2>&1; then
+    scores=$(printf '%s' "$content" | python3 -c "
 import sys, json, re
 text = sys.stdin.read().strip()
 # Try direct JSON parse first
@@ -209,6 +278,10 @@ if m:
         pass
 sys.exit(1)
 " 2>/dev/null) || { echo ""; return 1; }
+  else
+    echo ""
+    return 1
+  fi
 
   echo "$scores"
 }
@@ -221,10 +294,23 @@ call_openrouter() {
 
   # Escape prompt for JSON
   local escaped_prompt
-  escaped_prompt=$(echo "$prompt" | python3 -c "
+  if command -v node >/dev/null 2>&1; then
+    escaped_prompt=$(printf '%s' "$prompt" | node -e "
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { input += chunk; });
+process.stdin.on('end', () => {
+  process.stdout.write(JSON.stringify(input));
+});
+" 2>/dev/null) || escaped_prompt='""'
+  elif command -v python3 >/dev/null 2>&1; then
+    escaped_prompt=$(printf '%s' "$prompt" | python3 -c "
 import sys, json
 print(json.dumps(sys.stdin.read()))
 " 2>/dev/null) || escaped_prompt='""'
+  else
+    escaped_prompt='""'
+  fi
 
   local body
   body=$(cat <<JSON
@@ -295,7 +381,22 @@ if [ "${BEHAVIORAL_JUDGE_TEST_MODE}" = "1" ]; then
   fi
 
   # Parse mock scores
-  scores=$(echo "$mock_scores" | python3 -c "
+  if command -v node >/dev/null 2>&1; then
+    scores=$(printf '%s' "$mock_scores" | node -e "
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => { input += chunk; });
+process.stdin.on('end', () => {
+  const scores = JSON.parse(input);
+  const security = Number.parseInt(scores.security ?? 3, 10);
+  const tdd = Number.parseInt(scores.tdd ?? 3, 10);
+  const design = Number.parseInt(scores.design ?? 3, 10);
+  const structural = Number.parseInt(scores.structural ?? 3, 10);
+  console.log([security, tdd, design, structural].join(' '));
+});
+" 2>/dev/null) || scores="3 3 3 3"
+  elif command -v python3 >/dev/null 2>&1; then
+    scores=$(printf '%s' "$mock_scores" | python3 -c "
 import sys, json
 d = json.loads(sys.stdin.read())
 s = int(d.get('security', 3))
@@ -304,6 +405,9 @@ de = int(d.get('design', 3))
 st = int(d.get('structural', 3))
 print(f'{s} {t} {de} {st}')
 " 2>/dev/null) || scores="3 3 3 3"
+  else
+    scores="3 3 3 3"
+  fi
 
   read -r sec tdd_s des str <<< "$scores"
   compute_result "$sec" "$tdd_s" "$des" "$str" "$judge_model" "$judge_calls"

package/scripts/benchmark.js

@@ -1,85 +1,371 @@
 #!/usr/bin/env node
 
-/**
- * Performance benchmark script for Forge CLI.
- * Measures CLI startup time, autoDetect, and detectFramework performance.
- *
- * Usage:
- *   node scripts/benchmark.js           # Human-readable output
- *   node scripts/benchmark.js --json    # JSON output (for CI/tests)
- */
-
-const { execFileSync } = require('node:child_process');
-const { performance } = require('node:perf_hooks');
-const path = require('node:path');
+const { spawnSync: defaultSpawnSync } = require('node:child_process');
 const fs = require('node:fs');
+const path = require('node:path');
+const { performance } = require('node:perf_hooks');
+
+const { buildProfile, parseJUnitFiles, walk } = require('./test-profile');
 
 const rootDir = path.join(__dirname, '..');
-const ITERATIONS = 3;
+const DEFAULT_OUTPUT = path.join(rootDir, 'test-results', 'benchmark-results.json');
+const DEFAULT_PROFILE_DIR = path.join(rootDir, 'test-results', 'benchmark-profiles');
+const DEFAULT_SAMPLES = 3;
+const DEFAULT_JUNIT_PATH = path.join(rootDir, 'test-results', 'test-results.xml');
 
-function benchmarkCLIStartup() {
-  const times = [];
-  for (let i = 0; i < ITERATIONS; i++) {
-    const start = performance.now();
-    execFileSync('node', [path.join(rootDir, 'bin', 'forge.js'), '--help'], {
-      cwd: rootDir,
-      encoding: 'utf-8',
-      timeout: 10000,
-      stdio: 'pipe'
-    });
-    times.push(performance.now() - start);
+const BENCHMARK_GROUPS = [
+  {
+    id: 'full-suite',
+    label: 'Whole suite',
+    command: ['bun', 'test'],
+  },
+  {
+    id: 'pre-push-runner',
+    label: 'Pre-push runner slice',
+    command: ['bun', 'test', 'test/scripts/test-runner.test.js'],
+  },
+  {
+    id: 'validation-core',
+    label: 'Validation core slice',
+    command: [
+      'bun',
+      'test',
+      'test/scripts/test-profile.test.js',
+      'test/scripts/test-ci-shard.test.js',
+      'test/scripts/test-runner.test.js',
+    ],
+  },
+  {
+    id: 'hotspot-shell',
+    label: 'Hotspot shell slice',
+    command: [
+      'bun',
+      'test',
+      'test/scripts/smart-status.conflicts.files.test.js',
+      'test/scripts/smart-status.conflicts.merge-tree.test.js',
+      'test/scripts/dep-guard.check-ripple.basic.test.js',
+    ],
+  },
+  {
+    id: 'validate',
+    label: 'Validate command',
+    command: ['bash', 'scripts/validate.sh'],
+  },
+];
+
+function parseArgs(argv) {
+  const args = {
+    groups: [],
+    json: false,
+    output: DEFAULT_OUTPUT,
+    profileDir: DEFAULT_PROFILE_DIR,
+    samples: DEFAULT_SAMPLES,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const current = argv[index];
+    const next = argv[index + 1];
+
+    if (current === '--json') args.json = true;
+    if (current === '--output') args.output = next;
+    if (current === '--profile-dir') args.profileDir = next;
+    if (current === '--samples') args.samples = Number.parseInt(next, 10) || DEFAULT_SAMPLES;
+    if (current === '--group') args.groups.push(next);
   }
-  const mean = times.reduce((a, b) => a + b, 0) / times.length;
-  return { name: 'CLI startup (--help)', mean: Math.round(mean), unit: 'ms', samples: ITERATIONS };
+
+  return args;
 }
 
-function benchmarkAutoDetect() {
-  const { autoDetect } = require(path.join(rootDir, 'lib', 'project-discovery.js'));
-  const times = [];
-  for (let i = 0; i < ITERATIONS; i++) {
-    const start = performance.now();
-    autoDetect(rootDir);
-    times.push(performance.now() - start);
+function resolveGroups(groupIds = []) {
+  if (groupIds.length === 0) {
+    return BENCHMARK_GROUPS.map((group) => ({ ...group, command: [...group.command] }));
+  }
+
+  const byId = new Map(BENCHMARK_GROUPS.map((group) => [group.id, group]));
+  const resolved = [];
+  for (const id of groupIds) {
+    const match = byId.get(id);
+    if (!match) {
+      throw new Error(`Unknown benchmark group: ${id}`);
+    }
+    resolved.push({ ...match, command: [...match.command] });
+  }
+  return resolved;
+}
+
+function roundMs(value) {
+  return Math.round(value);
+}
+
+function calculateMedian(values) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return roundMs((sorted[middle - 1] + sorted[middle]) / 2);
+  }
+  return sorted[middle];
+}
+
+function summarizeSamples(samplesMs) {
+  if (samplesMs.length === 0) {
+    return {
+      maxMs: 0,
+      meanMs: 0,
+      medianMs: 0,
+      minMs: 0,
+      samplesMs: [],
+    };
+  }
+  const total = samplesMs.reduce((sum, value) => sum + value, 0);
+  return {
+    maxMs: Math.max(...samplesMs),
+    meanMs: roundMs(total / samplesMs.length),
+    medianMs: calculateMedian(samplesMs),
+    minMs: Math.min(...samplesMs),
+    samplesMs,
+  };
+}
+
+function buildJUnitCommand(command, junitPath) {
+  const [binary, subcommand, ...rest] = command;
+  if (binary === 'bun' && subcommand === 'test') {
+    return [
+      binary,
+      subcommand,
+      ...rest,
+      '--reporter=junit',
+      '--reporter-outfile',
+      junitPath,
+    ];
+  }
+  return [...command];
+}
+
+function runCommand(command, options = {}, spawnSync = defaultSpawnSync) {
+  const [binary, ...args] = command;
+  const result = spawnSync(binary, args, {
+    cwd: rootDir,
+    encoding: 'utf8',
+    stdio: 'pipe',
+    ...options,
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+
+  if ((result.status ?? 1) !== 0) {
+    throw new Error(result.stderr || `Command failed: ${command.join(' ')}`);
+  }
+
+  return result;
+}
+
+function materializeJUnitFile(expectedPath) {
+  if (fs.existsSync(expectedPath)) {
+    return expectedPath;
   }
-  const mean = times.reduce((a, b) => a + b, 0) / times.length;
-  return { name: 'autoDetect()', mean: Math.round(mean), unit: 'ms', samples: ITERATIONS };
+
+  if (fs.existsSync(DEFAULT_JUNIT_PATH)) {
+    fs.mkdirSync(path.dirname(expectedPath), { recursive: true });
+    fs.copyFileSync(DEFAULT_JUNIT_PATH, expectedPath);
+    return expectedPath;
+  }
+
+  throw new Error(`Benchmark run did not produce JUnit output at ${expectedPath}`);
+}
+
+function removeMaterializedOutputs(expectedPath) {
+	const directory = path.dirname(expectedPath);
+	const extension = path.extname(expectedPath);
+	const basename = path.basename(expectedPath, extension);
+
+	if (!fs.existsSync(directory)) {
+		return;
+	}
+
+	for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
+		if (!entry.isFile() || !entry.name.endsWith(extension)) {
+			continue;
+		}
+
+		if (entry.name === `${basename}${extension}` || entry.name.startsWith(`${basename}.`)) {
+			fs.rmSync(path.join(directory, entry.name), { force: true });
+		}
+	}
+}
+
+function indexXmlFiles(directory) {
+	return new Map(
+		walk(directory, '.xml').map((filePath) => {
+			const stats = fs.statSync(filePath);
+			return [filePath, `${stats.size}:${stats.mtimeMs}`];
+		}),
+	);
+}
+
+function listChangedXmlFiles(directory, beforeIndex) {
+	return walk(directory, '.xml')
+		.filter((filePath) => {
+			const stats = fs.statSync(filePath);
+			const fingerprint = `${stats.size}:${stats.mtimeMs}`;
+			return beforeIndex.get(filePath) !== fingerprint;
+		})
+		.sort((left, right) => left.localeCompare(right));
 }
 
-function benchmarkDetectFramework() {
-  const { detectFramework } = require(path.join(rootDir, 'lib', 'project-discovery.js'));
-  const times = [];
-  for (let i = 0; i < ITERATIONS; i++) {
+function materializeJUnitFiles(expectedPath, sourceFiles) {
+	const uniqueSources = [];
+	const seen = new Set();
+
+	for (const sourceFile of sourceFiles) {
+		if (typeof sourceFile !== 'string' || seen.has(sourceFile) || !fs.existsSync(sourceFile)) {
+			continue;
+		}
+		seen.add(sourceFile);
+		uniqueSources.push(sourceFile);
+	}
+
+	if (uniqueSources.length === 0) {
+		return [materializeJUnitFile(expectedPath)];
+	}
+
+	return uniqueSources.map((sourceFile, index) => {
+		const destination = index === 0
+			? expectedPath
+			: expectedPath.replace(/\.xml$/, `.${index + 1}.xml`);
+
+		if (sourceFile === destination) {
+			return destination;
+		}
+
+		fs.mkdirSync(path.dirname(destination), { recursive: true });
+		fs.copyFileSync(sourceFile, destination);
+		return destination;
+	});
+}
+
+function buildGroupProfile(group, xmlFiles) {
+  const metrics = parseJUnitFiles(xmlFiles);
+  return buildProfile({
+    integrationSkipped: true,
+    label: `benchmark-${group.id}`,
+  }, metrics);
+}
+
+function runBenchmarkGroup(group, options = {}) {
+  const profileDir = path.resolve(rootDir, options.profileDir || DEFAULT_PROFILE_DIR);
+  const samples = options.samples || DEFAULT_SAMPLES;
+  const spawnSync = options.spawnSync || defaultSpawnSync;
+
+  fs.mkdirSync(profileDir, { recursive: true });
+
+  const xmlFiles = [];
+  const samplesMs = [];
+
+  for (let sampleIndex = 0; sampleIndex < samples; sampleIndex += 1) {
+    const junitPath = path.join(profileDir, `${group.id}.sample-${sampleIndex + 1}.xml`);
+    removeMaterializedOutputs(junitPath);
+    fs.rmSync(DEFAULT_JUNIT_PATH, { force: true });
+    const beforeXmlFiles = indexXmlFiles(path.join(rootDir, 'test-results'));
+    const command = buildJUnitCommand(group.command, junitPath);
     const start = performance.now();
-    detectFramework(rootDir);
-    times.push(performance.now() - start);
+    runCommand(command, {}, spawnSync);
+    samplesMs.push(roundMs(performance.now() - start));
+    const changedXmlFiles = listChangedXmlFiles(path.join(rootDir, 'test-results'), beforeXmlFiles);
+    xmlFiles.push(...materializeJUnitFiles(junitPath, [
+      fs.existsSync(junitPath) ? junitPath : null,
+      ...changedXmlFiles,
+    ]));
   }
-  const mean = times.reduce((a, b) => a + b, 0) / times.length;
-  return { name: 'detectFramework()', mean: Math.round(mean), unit: 'ms', samples: ITERATIONS };
+
+  const summary = summarizeSamples(samplesMs);
+  const profile = buildGroupProfile(group, xmlFiles);
+  const profilePath = path.join(profileDir, `${group.id}.profile.json`);
+  fs.writeFileSync(profilePath, JSON.stringify(profile, null, 2));
+
+  return {
+    command: group.command.join(' '),
+    groupId: group.id,
+    groupLabel: group.label,
+    profilePath: path.relative(rootDir, profilePath).replace(/\\/g, '/'),
+    samples,
+    ...summary,
+  };
 }
 
-const results = [
-  benchmarkCLIStartup(),
-  benchmarkAutoDetect(),
-  benchmarkDetectFramework()
-];
+function buildBenchmarkResults(groups, results, timestamp = new Date().toISOString()) {
+  const totalMedianMs = results.reduce((sum, result) => sum + result.medianMs, 0);
+  const slowestGroup = [...results]
+    .sort((left, right) => right.medianMs - left.medianMs || left.groupId.localeCompare(right.groupId))[0] || null;
 
-const jsonMode = process.argv.includes('--json');
+  return {
+    groups: results,
+    requestedGroups: groups.map((group) => group.id),
+    samples: results[0]?.samples || DEFAULT_SAMPLES,
+    slowestGroup: slowestGroup
+      ? { groupId: slowestGroup.groupId, groupLabel: slowestGroup.groupLabel, medianMs: slowestGroup.medianMs }
+      : null,
+    timestamp,
+    totalMedianMs,
+  };
+}
+
+function formatResultLine(result) {
+  return `  ${result.groupLabel}: median ${result.medianMs}ms (min ${result.minMs}ms, max ${result.maxMs}ms)`;
+}
+
+function main(argv = process.argv.slice(2), deps = {}) {
+  const args = parseArgs(argv);
+  const groups = resolveGroups(args.groups);
+  const results = groups.map((group) => runBenchmarkGroup(group, {
+    profileDir: args.profileDir,
+    samples: args.samples,
+    spawnSync: deps.spawnSync,
+  }));
 
-if (jsonMode) {
-  process.stdout.write(JSON.stringify(results));
-} else {
-  console.log('\n  Forge Performance Benchmarks');
-  console.log('  ===========================\n');
-  for (const r of results) {
-    const status = r.mean < 1000 ? 'PASS' : r.mean < 3000 ? 'WARN' : 'SLOW';
-    console.log(`  ${status}  ${r.name}: ${r.mean}${r.unit} (${r.samples} samples)`);
+  const payload = buildBenchmarkResults(groups, results);
+  const outputPath = path.resolve(rootDir, args.output || DEFAULT_OUTPUT);
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+  fs.writeFileSync(outputPath, JSON.stringify(payload, null, 2));
+
+  if (args.json) {
+    process.stdout.write(JSON.stringify(payload));
+    return payload;
+  }
+
+  console.log('\n  Forge Test Benchmark Baselines');
+  console.log('  =============================\n');
+  for (const result of results) {
+    console.log(formatResultLine(result));
   }
-  console.log('');
+  console.log(`\n  Total median: ${payload.totalMedianMs}ms`);
+  console.log(`  Results saved to: ${path.relative(rootDir, outputPath).replace(/\\/g, '/')}\n`);
+  return payload;
+}
 
-  // Write results file
-  const outputPath = path.join(rootDir, 'benchmark-results.json');
-  fs.writeFileSync(outputPath, JSON.stringify(results, null, 2));
-  console.log(`  Results saved to: benchmark-results.json\n`);
+if (require.main === module) {
+  main();
 }
 
-module.exports = results;
+module.exports = {
+  BENCHMARK_GROUPS,
+  DEFAULT_OUTPUT,
+  DEFAULT_PROFILE_DIR,
+  DEFAULT_SAMPLES,
+  buildBenchmarkResults,
+  buildGroupProfile,
+  buildJUnitCommand,
+  calculateMedian,
+  formatResultLine,
+  main,
+  parseArgs,
+  resolveGroups,
+  roundMs,
+  materializeJUnitFile,
+  materializeJUnitFiles,
+  runBenchmarkGroup,
+  runCommand,
+  summarizeSamples,
+};