@vibe-forge/core 0.7.5 → 0.8.3

package/src/controllers/benchmark/runner.ts

@@ -1,415 +0,0 @@
-import { readFile } from 'node:fs/promises'
-import process from 'node:process'
-
-import { run as runTask } from '#~/controllers/task/index.js'
-import { uuid } from '#~/utils/uuid.js'
-
-import { getBenchmarkCase, listBenchmarkCases } from './discover'
-import { writeBenchmarkResult } from './result-store'
-import type { BenchmarkRunCaseInput, BenchmarkRunCaseOutput, BenchmarkRunCategoryInput, BenchmarkRunCategoryOutput, BenchmarkRunEvent } from './types'
-import type { BenchmarkResult } from './schema'
-import { createCaseWorkspace, disposeCaseWorkspace } from './workspace'
-import { execCommand, execShellCommand, parseDiffFiles, summarizeText } from './utils'
-
-interface TaskExecutionResult {
-  sessionId: string
-  exitCode: number
-  stderr: string
-}
-
-interface JudgedResult {
-  status: BenchmarkResult['status']
-  finalScore: number
-  scores: BenchmarkResult['scores']
-  judgeSummary: string
-  issues: string[]
-  changedFiles: string[]
-}
-
-const emitRunEvent = (
-  input: BenchmarkRunCaseInput | BenchmarkRunCategoryInput,
-  event: Omit<BenchmarkRunEvent, 'timestamp'>
-) => {
-  input.onEvent?.({
-    ...event,
-    timestamp: new Date().toISOString()
-  })
-}
-
-const runAgentTask = async (
-  input: BenchmarkRunCaseInput,
-  taskDescription: string,
-  cwd: string,
-  timeoutSec: number
-): Promise<TaskExecutionResult> => {
-  const sessionId = uuid()
-
-  return new Promise((resolve, reject) => {
-    let sessionHandle: { kill: () => void } | undefined
-    let settled = false
-    let stderr = ''
-    let exitCode = -1
-    let timer: NodeJS.Timeout | undefined
-
-    const finish = (value: TaskExecutionResult | Error, isError = false) => {
-      if (settled) return
-      settled = true
-      if (timer != null) clearTimeout(timer)
-      if (isError) {
-        reject(value)
-        return
-      }
-      resolve(value as TaskExecutionResult)
-    }
-
-    void (async () => {
-      try {
-        const { session } = await runTask({
-          adapter: input.adapter,
-          cwd,
-          env: input.env
-        }, {
-          type: 'create',
-          runtime: input.runtime ?? 'cli',
-          sessionId,
-          model: input.model,
-          systemPrompt: input.systemPrompt,
-          permissionMode: input.permissionMode,
-          mode: 'stream',
-          description: taskDescription,
-          onEvent: (event) => {
-            if (event.type === 'exit') {
-              stderr = event.data.stderr ?? stderr
-              exitCode = event.data.exitCode ?? exitCode
-              finish({
-                sessionId,
-                exitCode,
-                stderr
-              })
-            }
-          }
-        })
-
-        sessionHandle = session
-        timer = setTimeout(() => {
-          sessionHandle?.kill()
-          finish({
-            sessionId,
-            exitCode: -1,
-            stderr: 'Task execution timed out'
-          })
-        }, timeoutSec * 1000)
-      } catch (error) {
-        finish(error as Error, true)
-      }
-    })()
-  })
-}
-
-const collectCandidatePatch = async (cwd: string) => {
-  await execCommand({
-    command: 'git',
-    args: ['add', '-N', '.'],
-    cwd
-  })
-  const diffResult = await execCommand({
-    command: 'git',
-    args: ['diff', '--binary', '--no-ext-diff', '--submodule=diff', '--', '.'],
-    cwd
-  })
-  if (diffResult.exitCode !== 0) {
-    throw new Error(diffResult.stderr || 'Failed to collect candidate patch')
-  }
-  return diffResult.stdout
-}
-
-const applyPatchFromFile = async (cwd: string, patchPath: string) => {
-  const result = await execCommand({
-    command: 'git',
-    args: ['apply', '--allow-empty', '--recount', '--whitespace=nowarn', '--binary', patchPath],
-    cwd
-  })
-  if (result.exitCode !== 0) {
-    throw new Error(result.stderr || result.stdout || `Failed to apply patch: ${patchPath}`)
-  }
-}
-
-const judgeResult = (params: {
-  testExitCode: number
-  candidatePatch: string
-  referencePatch: string
-  taskExitCode: number
-  taskStderr: string
-}) => {
-  const changedFiles = parseDiffFiles(params.candidatePatch)
-  const referenceFiles = parseDiffFiles(params.referencePatch)
-  const overlapCount = referenceFiles.filter(file => changedFiles.includes(file)).length
-  const referenceScore = referenceFiles.length === 0
-    ? (changedFiles.length > 0 ? 1 : 0)
-    : Number((overlapCount / referenceFiles.length).toFixed(2))
-  const testScore = params.testExitCode === 0 ? 1 : 0
-
-  let goalScore = testScore
-  const issues: string[] = []
-
-  if (changedFiles.length === 0) {
-    goalScore = 0
-    issues.push('Agent 未产出代码改动')
-  } else if (testScore === 0) {
-    goalScore = Math.max(0.2, Number((referenceScore * 0.5).toFixed(2)))
-    issues.push('验收测试未通过')
-  }
-
-  if (params.taskExitCode !== 0) {
-    issues.push(`任务执行异常退出：${params.taskExitCode}`)
-  }
-  if (params.taskStderr.trim() !== '') {
-    issues.push(`任务执行 stderr：${summarizeText(params.taskStderr, 240)}`)
-  }
-  if (referenceFiles.length > 0 && overlapCount === 0 && changedFiles.length > 0) {
-    issues.push('改动文件与参考实现无重叠，需要人工复核目标一致性')
-  }
-
-  const finalScore = Number((0.7 * testScore + 0.2 * goalScore + 0.1 * referenceScore).toFixed(2))
-  const status: BenchmarkResult['status'] =
-    testScore === 1 && goalScore >= 0.8
-      ? 'pass'
-      : finalScore > 0
-          ? 'partial'
-          : 'fail'
-
-  const judgeSummary = status === 'pass'
-    ? '验收测试通过，任务目标完成度良好。'
-    : status === 'partial'
-        ? '实现存在部分有效改动，但仍需进一步完善。'
-        : '当前实现未完成任务目标。'
-
-  return {
-    status,
-    finalScore,
-    scores: {
-      testScore,
-      goalScore,
-      referenceScore
-    },
-    judgeSummary,
-    issues,
-    changedFiles
-  } satisfies JudgedResult
-}
-
-export const runBenchmarkCase = async (input: BenchmarkRunCaseInput): Promise<BenchmarkRunCaseOutput> => {
-  const workspaceFolder = input.workspaceFolder ?? process.cwd()
-  const runId = input.runId ?? uuid()
-  const startedAt = Date.now()
-
-  emitRunEvent(input, {
-    runId,
-    category: input.category,
-    title: input.title,
-    scope: 'case',
-    phase: 'discover',
-    message: 'Loading benchmark case'
-  })
-
-  const caseItem = await getBenchmarkCase({
-    workspaceFolder,
-    category: input.category,
-    title: input.title
-  })
-
-  if (caseItem == null) {
-    throw new Error(`Benchmark case not found: ${input.category}/${input.title}`)
-  }
-
-  emitRunEvent(input, {
-    runId,
-    category: input.category,
-    title: input.title,
-    scope: 'case',
-    phase: 'workspace',
-    message: 'Preparing isolated benchmark workspace'
-  })
-
-  let workspaceState: Awaited<ReturnType<typeof createCaseWorkspace>> | null = null
-
-  try {
-    workspaceState = await createCaseWorkspace({
-      workspaceFolder,
-      category: input.category,
-      title: input.title,
-      runId,
-      baseCommit: caseItem.frontmatter.baseCommit,
-      setupCommand: caseItem.frontmatter.setupCommand,
-      timeoutSec: caseItem.frontmatter.timeoutSec
-    })
-
-    emitRunEvent(input, {
-      runId,
-      category: input.category,
-      title: input.title,
-      scope: 'case',
-      phase: 'task',
-      message: 'Running task agent'
-    })
-
-    const taskResult = await runAgentTask(
-      input,
-      caseItem.rfcBody,
-      workspaceState.caseWorkspacePath,
-      caseItem.frontmatter.timeoutSec
-    )
-    const candidatePatch = await collectCandidatePatch(workspaceState.caseWorkspacePath)
-
-    emitRunEvent(input, {
-      runId,
-      category: input.category,
-      title: input.title,
-      scope: 'case',
-      phase: 'verify',
-      message: 'Applying benchmark test patch'
-    })
-
-    await applyPatchFromFile(workspaceState.caseWorkspacePath, caseItem.patchTestPath)
-    const testResult = await execShellCommand({
-      command: caseItem.frontmatter.testCommand,
-      cwd: workspaceState.caseWorkspacePath,
-      timeoutMs: caseItem.frontmatter.timeoutSec * 1000,
-      env: input.env
-    })
-    const referencePatch = await readFile(caseItem.patchPath, 'utf-8')
-    const judged = judgeResult({
-      testExitCode: testResult.exitCode,
-      candidatePatch,
-      referencePatch,
-      taskExitCode: taskResult.exitCode,
-      taskStderr: taskResult.stderr
-    })
-
-    const result: BenchmarkResult = {
-      category: caseItem.category,
-      title: caseItem.title,
-      status: judged.status,
-      finalScore: judged.finalScore,
-      scores: judged.scores,
-      baseCommit: caseItem.frontmatter.baseCommit,
-      durationMs: Date.now() - startedAt,
-      setupCommand: caseItem.frontmatter.setupCommand,
-      testCommand: caseItem.frontmatter.testCommand,
-      testExitCode: testResult.exitCode,
-      judgeSummary: judged.judgeSummary,
-      issues: judged.issues,
-      timestamp: new Date().toISOString(),
-      runId,
-      taskSessionId: taskResult.sessionId,
-      taskExitCode: taskResult.exitCode,
-      changedFiles: judged.changedFiles,
-      categoryRunId: input.categoryRunId
-    }
-
-    await writeBenchmarkResult(workspaceFolder, result)
-
-    emitRunEvent(input, {
-      runId,
-      category: input.category,
-      title: input.title,
-      scope: 'case',
-      phase: 'result',
-      message: `Benchmark completed with status ${result.status}`
-    })
-
-    return {
-      runId,
-      result
-    }
-  } catch (error) {
-    const result: BenchmarkResult = {
-      category: caseItem.category,
-      title: caseItem.title,
-      status: 'fail',
-      finalScore: 0,
-      scores: {
-        testScore: 0,
-        goalScore: 0,
-        referenceScore: 0
-      },
-      baseCommit: caseItem.frontmatter.baseCommit,
-      durationMs: Date.now() - startedAt,
-      setupCommand: caseItem.frontmatter.setupCommand,
-      testCommand: caseItem.frontmatter.testCommand,
-      testExitCode: -1,
-      judgeSummary: 'Benchmark 运行失败。',
-      issues: [error instanceof Error ? error.message : String(error)],
-      timestamp: new Date().toISOString(),
-      runId,
-      categoryRunId: input.categoryRunId
-    }
-    await writeBenchmarkResult(workspaceFolder, result)
-    return {
-      runId,
-      result
-    }
-  } finally {
-    if (workspaceState != null) {
-      await disposeCaseWorkspace(workspaceState)
-    }
-  }
-}
-
-export const runBenchmarkCategory = async (input: BenchmarkRunCategoryInput): Promise<BenchmarkRunCategoryOutput> => {
-  const workspaceFolder = input.workspaceFolder ?? process.cwd()
-  const runId = input.runId ?? uuid()
-  const allCases = await listBenchmarkCases({
-    workspaceFolder,
-    category: input.category
-  })
-  const selectedCases = input.titles?.length
-    ? allCases.filter(item => input.titles?.includes(item.title))
-    : allCases
-
-  if (selectedCases.length === 0) {
-    throw new Error(`No benchmark cases found for category: ${input.category}`)
-  }
-
-  const concurrency = Math.max(1, Math.min(input.concurrency ?? 2, selectedCases.length))
-  const results: BenchmarkResult[] = []
-  let nextIndex = 0
-
-  const worker = async () => {
-    while (nextIndex < selectedCases.length) {
-      const currentIndex = nextIndex
-      nextIndex += 1
-      const item = selectedCases[currentIndex]
-      emitRunEvent(input, {
-        runId,
-        category: input.category,
-        title: item.title,
-        scope: 'category',
-        phase: 'discover',
-        message: `Scheduling case ${item.title}`
-      })
-      const output = await runBenchmarkCase({
-        workspaceFolder,
-        category: input.category,
-        title: item.title,
-        adapter: input.adapter,
-        model: input.model,
-        systemPrompt: input.systemPrompt,
-        permissionMode: input.permissionMode,
-        runtime: input.runtime,
-        env: input.env,
-        categoryRunId: runId,
-        onEvent: input.onEvent
-      })
-      results.push(output.result)
-    }
-  }
-
-  await Promise.all(Array.from({ length: concurrency }, () => worker()))
-
-  return {
-    runId,
-    category: input.category,
-    results: results.sort((a, b) => a.title.localeCompare(b.title))
-  }
-}

package/src/controllers/benchmark/schema.ts

@@ -1,60 +0,0 @@
-import z from 'zod'
-
-export const BenchmarkFrontmatterSchema = z.object({
-  title: z.string().trim().optional(),
-  description: z.string().trim().optional(),
-  baseCommit: z.string().trim().min(1),
-  setupCommand: z.string().trim().min(1),
-  testCommand: z.string().trim().min(1),
-  timeoutSec: z.number().int().positive().default(900)
-})
-
-export const BenchmarkStatusSchema = z.enum(['pass', 'partial', 'fail'])
-
-export const BenchmarkScoresSchema = z.object({
-  testScore: z.number().min(0).max(1),
-  goalScore: z.number().min(0).max(1),
-  referenceScore: z.number().min(0).max(1)
-})
-
-export const BenchmarkResultSchema = z.object({
-  category: z.string().trim().min(1),
-  title: z.string().trim().min(1),
-  status: BenchmarkStatusSchema,
-  finalScore: z.number().min(0).max(1),
-  scores: BenchmarkScoresSchema,
-  baseCommit: z.string().trim().min(1),
-  durationMs: z.number().int().nonnegative(),
-  setupCommand: z.string().trim().min(1),
-  testCommand: z.string().trim().min(1),
-  testExitCode: z.number().int(),
-  judgeSummary: z.string(),
-  issues: z.array(z.string()),
-  timestamp: z.string().trim().min(1),
-  runId: z.string().trim().min(1).optional(),
-  taskSessionId: z.string().trim().min(1).optional(),
-  taskExitCode: z.number().int().optional(),
-  changedFiles: z.array(z.string()).optional(),
-  categoryRunId: z.string().trim().min(1).optional()
-})
-
-export const BenchmarkRunSummarySchema = z.object({
-  runId: z.string().trim().min(1),
-  category: z.string().trim().min(1),
-  title: z.string().trim().min(1).optional(),
-  status: z.enum(['queued', 'running', 'completed', 'failed']),
-  startedAt: z.number().int().positive(),
-  finishedAt: z.number().int().positive().optional(),
-  completedCount: z.number().int().nonnegative().optional(),
-  totalCount: z.number().int().positive().optional(),
-  result: BenchmarkResultSchema.optional(),
-  results: z.array(BenchmarkResultSchema).optional(),
-  error: z.string().optional(),
-  lastMessage: z.string().optional()
-})
-
-export type BenchmarkFrontmatter = z.infer<typeof BenchmarkFrontmatterSchema>
-export type BenchmarkStatus = z.infer<typeof BenchmarkStatusSchema>
-export type BenchmarkScores = z.infer<typeof BenchmarkScoresSchema>
-export type BenchmarkResult = z.infer<typeof BenchmarkResultSchema>
-export type BenchmarkRunSummary = z.infer<typeof BenchmarkRunSummarySchema>

package/src/controllers/benchmark/types.ts

@@ -1,80 +0,0 @@
-import type { BenchmarkFrontmatter, BenchmarkResult } from './schema'
-
-export interface BenchmarkCase {
-  id: string
-  category: string
-  title: string
-  caseDir: string
-  rfcPath: string
-  patchPath: string
-  patchTestPath: string
-  rfcBody: string
-  rfcRaw: string
-  summary: string
-  frontmatter: BenchmarkFrontmatter
-  latestResult?: BenchmarkResult | null
-}
-
-export interface BenchmarkCategory {
-  category: string
-  caseCount: number
-  lastStatuses: Record<'pass' | 'partial' | 'fail', number>
-}
-
-export interface BenchmarkListOptions {
-  workspaceFolder?: string
-  category?: string
-}
-
-export interface BenchmarkCaseSelector {
-  workspaceFolder?: string
-  category: string
-  title: string
-}
-
-export type BenchmarkPermissionMode = 'default' | 'acceptEdits' | 'plan' | 'dontAsk' | 'bypassPermissions'
-
-export interface BenchmarkAgentOptions {
-  adapter?: string
-  model?: string
-  systemPrompt?: string
-  permissionMode?: BenchmarkPermissionMode
-  runtime?: 'cli' | 'server'
-  env?: Record<string, string | null | undefined>
-}
-
-export interface BenchmarkRunEvent {
-  runId: string
-  category: string
-  title?: string
-  scope: 'category' | 'case'
-  phase: 'discover' | 'workspace' | 'setup' | 'task' | 'verify' | 'result'
-  message: string
-  timestamp: string
-}
-
-export interface BenchmarkRunCaseInput extends BenchmarkCaseSelector, BenchmarkAgentOptions {
-  runId?: string
-  categoryRunId?: string
-  onEvent?: (event: BenchmarkRunEvent) => void
-}
-
-export interface BenchmarkRunCaseOutput {
-  runId: string
-  result: BenchmarkResult
-}
-
-export interface BenchmarkRunCategoryInput extends BenchmarkAgentOptions {
-  workspaceFolder?: string
-  category: string
-  titles?: string[]
-  concurrency?: number
-  runId?: string
-  onEvent?: (event: BenchmarkRunEvent) => void
-}
-
-export interface BenchmarkRunCategoryOutput {
-  runId: string
-  category: string
-  results: BenchmarkResult[]
-}

package/src/controllers/benchmark/utils.ts

@@ -1,144 +0,0 @@
-import { spawn } from 'node:child_process'
-import { access, mkdir, readFile, symlink } from 'node:fs/promises'
-import { constants } from 'node:fs'
-import { dirname, join } from 'node:path'
-import process from 'node:process'
-
-export interface CommandResult {
-  exitCode: number
-  stdout: string
-  stderr: string
-  durationMs: number
-}
-
-interface ExecCommandInput {
-  command: string
-  args?: string[]
-  cwd: string
-  env?: Record<string, string | null | undefined>
-  timeoutMs?: number
-  input?: string
-}
-
-export const pathExists = async (targetPath: string) => {
-  try {
-    await access(targetPath, constants.F_OK)
-    return true
-  } catch {
-    return false
-  }
-}
-
-export const readTextIfExists = async (targetPath: string) => {
-  if (!await pathExists(targetPath)) return null
-  return readFile(targetPath, 'utf-8')
-}
-
-export const ensureParentDir = async (targetPath: string) => {
-  await mkdir(dirname(targetPath), { recursive: true })
-}
-
-export const execCommand = async (input: ExecCommandInput): Promise<CommandResult> => {
-  const startedAt = Date.now()
-  const {
-    command,
-    args = [],
-    cwd,
-    env,
-    timeoutMs,
-    input: stdin
-  } = input
-
-  return new Promise((resolve, reject) => {
-    const child = spawn(command, args, {
-      cwd,
-      env: {
-        ...process.env,
-        ...env
-      },
-      stdio: 'pipe'
-    })
-
-    const stdout: string[] = []
-    const stderr: string[] = []
-    let settled = false
-
-    const finish = (value: CommandResult | Error, isError = false) => {
-      if (settled) return
-      settled = true
-      if (timer) clearTimeout(timer)
-      if (isError) {
-        reject(value)
-        return
-      }
-      resolve(value as CommandResult)
-    }
-
-    child.stdout.on('data', (chunk) => {
-      stdout.push(String(chunk))
-    })
-    child.stderr.on('data', (chunk) => {
-      stderr.push(String(chunk))
-    })
-    child.on('error', (error) => {
-      finish(error, true)
-    })
-    child.on('close', (code) => {
-      finish({
-        exitCode: code ?? -1,
-        stdout: stdout.join(''),
-        stderr: stderr.join(''),
-        durationMs: Date.now() - startedAt
-      })
-    })
-
-    const timer = timeoutMs != null
-      ? setTimeout(() => {
-          child.kill('SIGTERM')
-        }, timeoutMs)
-      : null
-
-    if (stdin != null && stdin !== '') {
-      child.stdin.write(stdin)
-    }
-    child.stdin.end()
-  })
-}
-
-export const execShellCommand = async (input: Omit<ExecCommandInput, 'command' | 'args'> & { command: string }) => {
-  const shell = process.env.SHELL || '/bin/sh'
-  return execCommand({
-    command: shell,
-    args: ['-lc', input.command],
-    cwd: input.cwd,
-    env: input.env,
-    timeoutMs: input.timeoutMs,
-    input: input.input
-  })
-}
-
-export const parseDiffFiles = (patch: string) => {
-  const files = new Set<string>()
-  const matcher = /^diff --git a\/(.+?) b\/(.+)$/gm
-  for (const match of patch.matchAll(matcher)) {
-    const [, oldPath, newPath] = match
-    files.add(newPath === '/dev/null' ? oldPath : newPath)
-  }
-  return [...files]
-}
-
-export const summarizeText = (value: string, limit = 1200) => {
-  const normalized = value.trim()
-  if (normalized.length <= limit) return normalized
-  return `${normalized.slice(0, limit)}...`
-}
-
-export const linkPreparedNodeModules = async (sourceDir: string, targetDir: string) => {
-  const candidates = ['node_modules']
-  for (const name of candidates) {
-    const sourcePath = join(sourceDir, name)
-    const targetPath = join(targetDir, name)
-    if (!await pathExists(sourcePath) || await pathExists(targetPath)) continue
-    await symlink(sourcePath, targetPath, process.platform === 'win32' ? 'junction' : 'dir')
-  }
-}