@citeark/agent 0.3.20

package/src/job.mjs

@@ -0,0 +1,496 @@
+import { appendFile } from "node:fs/promises";
+import { DELIVERY_INSTRUCTIONS } from "./deployment/handoff.mjs";
+import { hasResearchWorkspace, usesWorkspaceSummary } from "./reproduction/workspace-mode.mjs";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+import {
+  cp,
+  lstat,
+  mkdir,
+  readFile,
+  realpath,
+  readdir,
+  readlink,
+  writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+
+import { listEvidenceParsers } from "./evidence/registry.mjs";
+import { initializeResearchPlan, stageResearchPlanTools } from "./reproduction/plan.mjs";
+import { captureProcess } from "./process.mjs";
+import {
+  isCiteArkReconstructionRepository,
+  prepareReconstructionRepository,
+} from "./repository-origin.mjs";
+import {
+  CiteArkError,
+  createRunId,
+  createSessionId,
+  pathExists,
+  sha256File,
+  sha256Value,
+  writeJson,
+} from "./util.mjs";
+
+const SOURCE_ROOT = path.dirname(fileURLToPath(import.meta.url));
+const PACKAGE_ROOT = path.resolve(SOURCE_ROOT, "..");
+const SNAPSHOT_EXCLUDED_DIRECTORIES = new Set([
+  ".git",
+  ".cache",
+  ".mypy_cache",
+  ".pytest_cache",
+  ".venv",
+  "__pycache__",
+  "node_modules",
+]);
+
+export async function prepareRunBundle({
+  task,
+  taskDirectory,
+  runsDirectory,
+  runId = createRunId(),
+  imageOverride,
+  repositoryOverride,
+  paperOverride,
+}) {
+  const runDirectory = path.resolve(runsDirectory, runId);
+  if (await pathExists(runDirectory)) {
+    throw new CiteArkError(`运行目录已经存在：${runDirectory}`);
+  }
+
+  const directories = {
+    run: runDirectory,
+    input: path.join(runDirectory, "input"),
+    workspace: path.join(runDirectory, "workspace"),
+    repository: path.join(runDirectory, "workspace", "repository"),
+    output: path.join(runDirectory, "output"),
+    runtimeHome: path.join(runDirectory, "runtime-home"),
+    execution: path.join(runDirectory, "execution"),
+  };
+  await Promise.all(
+    Object.values(directories)
+      .filter(
+        (directory) =>
+          directory !== directories.run && directory !== directories.repository,
+      )
+      .map((directory) => mkdir(directory, { recursive: true })),
+  );
+  // Codex refuses to start when CODEX_HOME points at a path that has not been
+  // created yet. Keep every runtime's mutable state inside the mounted,
+  // per-run home so retries can resume without touching the host user profile.
+  await mkdir(path.join(directories.runtimeHome, "codex"), { recursive: true });
+
+  const runtimeTask = structuredClone(task);
+  if (imageOverride) runtimeTask.environment.image = imageOverride;
+  if (paperOverride?.path) runtimeTask.paper.path = path.resolve(paperOverride.path);
+  if (paperOverride?.preparedReading) runtimeTask.paper.preparedReading = paperOverride.preparedReading;
+
+  await writeJson(path.join(directories.input, "submitted-task.json"), task);
+  if (paperOverride?.digest) {
+    if (!(await pathExists(runtimeTask.paper.path))) {
+      throw new CiteArkError(`流水线论文快照不存在：${runtimeTask.paper.path}`);
+    }
+    const observedPaperDigest = `sha256:${await sha256File(runtimeTask.paper.path)}`;
+    if (observedPaperDigest !== paperOverride.digest) {
+      throw new CiteArkError(
+        `流水线论文快照摘要不一致：expected=${paperOverride.digest}, observed=${observedPaperDigest}`,
+      );
+    }
+  }
+  await copyPaper(runtimeTask, taskDirectory, directories.input);
+  await copyAssets(runtimeTask, taskDirectory, directories.input);
+  const contractRepository = structuredClone(runtimeTask.repository);
+  const repositoryTask = repositoryOverride
+    ? {
+        ...runtimeTask,
+        repository: { path: path.resolve(repositoryOverride.path) },
+      }
+    : runtimeTask;
+  let repositoryProvenance = await prepareRepository(
+    repositoryTask,
+    taskDirectory,
+    directories.repository,
+  );
+  if (repositoryOverride) {
+    const copiedSnapshot = await snapshotDirectory(directories.repository);
+    const copiedDigest = snapshotDigest(copiedSnapshot);
+    if (
+      repositoryOverride.digest &&
+      repositoryOverride.digest !== copiedDigest
+    ) {
+      throw new CiteArkError(
+        `流水线仓库快照摘要不一致：expected=${repositoryOverride.digest}, observed=${copiedDigest}`,
+      );
+    }
+    repositoryProvenance = {
+      sourceType: "pipeline-snapshot",
+      contractUrl: contractRepository.url,
+      commit: contractRepository.commit,
+      dirty: false,
+      snapshotDigest: copiedDigest,
+    };
+  }
+
+  runtimeTask.run = {
+    id: runId,
+    repositoryPath: "/job/workspace/repository",
+    outputPath: "/job/output",
+  };
+  runtimeTask.repository.path = "/job/workspace/repository";
+  delete runtimeTask.repository.url;
+
+  await writeJson(path.join(directories.input, "task.json"), runtimeTask);
+  await initializeResearchPlan(runtimeTask, directories.output);
+  if (hasResearchWorkspace(runtimeTask)) {
+    await stageResearchPlanTools(directories.input);
+  }
+  await cp(
+    path.join(PACKAGE_ROOT, "schemas", usesWorkspaceSummary(runtimeTask) ? "workspace-summary.schema.json" : "result.schema.json"),
+    path.join(directories.input, "result.schema.json"),
+  );
+  await cp(
+    path.join(PACKAGE_ROOT, "prompts", usesWorkspaceSummary(runtimeTask) ? "execute-workspace-simple.md" : hasResearchWorkspace(runtimeTask) ? "execute-workspace.md" : "execute-contract.md"),
+    path.join(directories.input, "agent-instructions.md"),
+  );
+  await cp(
+    path.join(PACKAGE_ROOT, "data", "dataset-source-registry.v1.json"),
+    path.join(directories.input, "dataset-source-registry.json"),
+  );
+  await writeJson(
+    path.join(directories.input, "parser-catalog.json"),
+    listEvidenceParsers(),
+  );
+
+  await appendFile(path.join(directories.input, "agent-instructions.md"), DELIVERY_INSTRUCTIONS);
+  const beforeSnapshot = await snapshotDirectory(directories.repository);
+  await writeJson(
+    path.join(directories.execution, "workspace-before.json"),
+    beforeSnapshot,
+  );
+
+  const sessionId = createSessionId();
+  const state = {
+    schemaVersion: "0.1",
+    runId,
+    sessionId,
+    status: "prepared",
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    taskName: task.name,
+    runtime: task.agent.runtime,
+    sandbox: "docker",
+    image: runtimeTask.environment.image,
+    repository: repositoryProvenance,
+  };
+  await writeJson(path.join(directories.run, "run.json"), state);
+
+  return {
+    runId,
+    sessionId,
+    task,
+    runtimeTask,
+    directories,
+    state,
+    beforeSnapshot,
+    repositoryProvenance,
+  };
+}
+
+export async function copyPaper(runtimeTask, taskDirectory, inputDirectory) {
+  if (runtimeTask.paper.path) {
+    const source = path.resolve(taskDirectory, runtimeTask.paper.path);
+    if (!(await pathExists(source))) {
+      throw new CiteArkError(`论文文件不存在：${source}`);
+    }
+    const paperDirectory = path.join(inputDirectory, "paper");
+    await mkdir(paperDirectory, { recursive: true });
+    const destination = path.join(paperDirectory, path.basename(source));
+    await cp(source, destination, { recursive: true, verbatimSymlinks: true });
+    runtimeTask.paper.path = `/job/input/paper/${path.basename(source)}`;
+    const reading = runtimeTask.paper.preparedReading;
+    if (reading?.status === "ready") {
+      if (createHash("sha256").update(await readFile(source)).digest("hex") !== reading.sourceDigest) throw new CiteArkError("论文阅读副本与原 PDF 身份不一致");
+      for (const file of reading.files) {
+        if (typeof file.path !== "string" || file.path.startsWith("/") || file.path.includes("\\") || file.path.split("/").some((part) => !part || part === "." || part === "..") || (!["paper.md", "source-map.json"].includes(file.path) && !file.path.startsWith("images/"))) throw new CiteArkError("论文阅读副本路径无效");
+        const content = await readFile(path.join(reading.directory, file.path));
+        if (content.length !== file.size || createHash("sha256").update(content).digest("hex") !== file.digest) throw new CiteArkError("论文阅读副本摘要不一致");
+        const target = path.join(inputDirectory, "paper-reading", file.path);
+        await mkdir(path.dirname(target), { recursive: true });
+        await writeFile(target, content);
+      }
+      runtimeTask.paper.markdownPath = "/job/input/paper-reading/paper.md";
+      runtimeTask.paper.sourceMapPath = "/job/input/paper-reading/source-map.json";
+      delete runtimeTask.paper.preparedReading;
+    }
+  }
+}
+
+export async function copyAssets(runtimeTask, taskDirectory, inputDirectory) {
+  const assetsDirectory = path.join(inputDirectory, "assets");
+  const copiedAssets = [];
+  for (const asset of runtimeTask.assets) {
+    const source = path.resolve(taskDirectory, asset.path);
+    if (!(await pathExists(source))) {
+      throw new CiteArkError(`任务资产不存在：${source}`);
+    }
+    const name = asset.name ?? path.basename(source);
+    if (name.includes("/") || name.includes("\\") || name === "." || name === "..") {
+      throw new CiteArkError(`资产 name 只能是单个文件名：${name}`);
+    }
+    await mkdir(assetsDirectory, { recursive: true });
+    await cp(source, path.join(assetsDirectory, name), {
+      recursive: true,
+      verbatimSymlinks: true,
+    });
+    copiedAssets.push({
+      ...asset,
+      path: `/job/input/assets/${name}`,
+      name,
+    });
+  }
+  runtimeTask.assets = copiedAssets;
+}
+
+export async function prepareRepository(runtimeTask, taskDirectory, destination) {
+  if (isCiteArkReconstructionRepository(runtimeTask.repository)) {
+    const commit = await prepareReconstructionRepository(destination);
+    return {
+      sourceType: "citeark-reconstruction",
+      submittedSource: runtimeTask.repository.url,
+      commit,
+      dirty: false,
+    };
+  }
+  if (runtimeTask.repository.path) {
+    const source = path.resolve(taskDirectory, runtimeTask.repository.path);
+    if (!(await pathExists(source))) {
+      throw new CiteArkError(`仓库目录不存在：${source}`);
+    }
+    await cp(source, destination, {
+      recursive: true,
+      force: true,
+      verbatimSymlinks: true,
+    });
+    const git = await inspectGitRepository(destination);
+    if (
+      runtimeTask.repository.commit &&
+      git.commit &&
+      !git.commit.startsWith(runtimeTask.repository.commit)
+    ) {
+      throw new CiteArkError(
+        `本地仓库 HEAD (${git.commit}) 与任务指定 commit (${runtimeTask.repository.commit}) 不一致`,
+      );
+    }
+    return {
+      sourceType: "path",
+      submittedSource: runtimeTask.repository.path,
+      commit: git.commit ?? runtimeTask.repository.commit ?? null,
+      dirty: git.dirty,
+    };
+  }
+
+  const clone = await captureProcess(
+    "git",
+    ["clone", "--config", "core.autocrlf=false", "--filter=blob:none", runtimeTask.repository.url, destination],
+    { cwd: taskDirectory },
+  );
+  if (clone.code !== 0) {
+    throw new CiteArkError(`克隆仓库失败：\n${clone.stderr.trim()}`);
+  }
+  if (runtimeTask.repository.commit) {
+    const checkout = await captureProcess(
+      "git",
+      ["checkout", "--detach", runtimeTask.repository.commit],
+      { cwd: destination },
+    );
+    if (checkout.code !== 0) {
+      throw new CiteArkError(`检出 commit 失败：\n${checkout.stderr.trim()}`);
+    }
+  }
+  const git = await inspectGitRepository(destination);
+  return {
+    sourceType: "url",
+    submittedSource: runtimeTask.repository.url,
+    commit: git.commit,
+    dirty: git.dirty,
+  };
+}
+
+export async function inspectGitRepository(repositoryDirectory) {
+  const result = await captureProcess(
+    "git",
+    ["rev-parse", "--is-inside-work-tree"],
+    { cwd: repositoryDirectory },
+  );
+  if (result.code !== 0 || result.stdout.trim() !== "true") {
+    return { commit: null, dirty: null };
+  }
+  const topLevel = await captureProcess(
+    "git",
+    ["rev-parse", "--show-toplevel"],
+    { cwd: repositoryDirectory },
+  );
+  const [actualRoot, requestedRoot] = await Promise.all([
+    topLevel.code === 0 ? realpath(topLevel.stdout.trim()).catch(() => null) : null,
+    realpath(repositoryDirectory).catch(() => null),
+  ]);
+  if (!actualRoot || !requestedRoot || actualRoot !== requestedRoot) {
+    return { commit: null, dirty: null };
+  }
+  const [commit, status] = await Promise.all([
+    captureProcess("git", ["rev-parse", "HEAD"], { cwd: repositoryDirectory }),
+    captureProcess("git", ["status", "--porcelain=v1"], {
+      cwd: repositoryDirectory,
+    }),
+  ]);
+  return {
+    commit: commit.code === 0 ? commit.stdout.trim() : null,
+    dirty: status.code === 0 ? status.stdout.trim() !== "" : null,
+  };
+}
+
+export async function snapshotDirectory(root) {
+  const entries = [];
+  await walk(root, "", entries);
+  entries.sort((left, right) => left.path.localeCompare(right.path));
+  return {
+    generatedAt: new Date().toISOString(),
+    excludedDirectories: [...SNAPSHOT_EXCLUDED_DIRECTORIES].sort(),
+    entries,
+  };
+
+  async function walk(directory, relativeDirectory, output) {
+    const children = await readdir(directory, { withFileTypes: true });
+    children.sort((left, right) => left.name.localeCompare(right.name));
+    for (const child of children) {
+      const relative = path.posix.join(
+        relativeDirectory.split(path.sep).join(path.posix.sep),
+        child.name,
+      );
+      const absolute = path.join(directory, child.name);
+      if (child.isDirectory()) {
+        if (SNAPSHOT_EXCLUDED_DIRECTORIES.has(child.name)) continue;
+        await walk(absolute, relative, output);
+      } else if (child.isSymbolicLink()) {
+        output.push({
+          path: relative,
+          type: "symlink",
+          target: await readlink(absolute),
+        });
+      } else if (child.isFile()) {
+        const stat = await lstat(absolute);
+        output.push({
+          path: relative,
+          type: "file",
+          size: stat.size,
+          sha256: await sha256File(absolute),
+        });
+      }
+    }
+  }
+}
+
+export function snapshotDigest(snapshot) {
+  return `sha256:${sha256Value({
+    excludedDirectories: snapshot.excludedDirectories,
+    entries: snapshot.entries,
+  })}`;
+}
+
+export function diffSnapshots(before, after) {
+  const beforeByPath = new Map(before.entries.map((entry) => [entry.path, entry]));
+  const afterByPath = new Map(after.entries.map((entry) => [entry.path, entry]));
+  const added = [];
+  const modified = [];
+  const deleted = [];
+
+  for (const [entryPath, current] of afterByPath) {
+    const previous = beforeByPath.get(entryPath);
+    if (!previous) {
+      added.push(current);
+    } else if (hashEntry(previous) !== hashEntry(current)) {
+      modified.push({ path: entryPath, before: previous, after: current });
+    }
+  }
+  for (const [entryPath, previous] of beforeByPath) {
+    if (!afterByPath.has(entryPath)) deleted.push(previous);
+  }
+  return { added, modified, deleted };
+}
+
+function hashEntry(entry) {
+  return createHash("sha256").update(JSON.stringify(entry)).digest("hex");
+}
+
+export async function captureFinalWorkspace(bundle) {
+  const after = await snapshotDirectory(bundle.directories.repository);
+  const changes = diffSnapshots(bundle.beforeSnapshot, after);
+  await writeJson(
+    path.join(bundle.directories.execution, "workspace-after.json"),
+    after,
+  );
+  await writeJson(
+    path.join(bundle.directories.execution, "workspace-changes.json"),
+    changes,
+  );
+  await captureGitDiff(bundle.directories.repository, bundle.directories.execution);
+  return { after, changes };
+}
+
+export async function ensureFinalWorkspaceCapture(bundle) {
+  // Local execution can change files between validation attempts. Only remote
+  // captures are authoritative over the coordinator's repository copy.
+  if ((bundle.sandboxProvider ?? bundle.state?.sandbox) === "docker") {
+    return await captureFinalWorkspace(bundle);
+  }
+  const required = [
+    "workspace-after.json",
+    "workspace-changes.json",
+    "repository.patch",
+    "untracked-files.txt",
+  ].map((filename) => path.join(bundle.directories.execution, filename));
+  if ((await Promise.all(required.map(pathExists))).every(Boolean)) {
+    return { preservedRemoteAudit: true };
+  }
+  return await captureFinalWorkspace(bundle);
+}
+
+async function captureGitDiff(repositoryDirectory, executionDirectory) {
+  const git = await inspectGitRepository(repositoryDirectory);
+  if (!git.commit) {
+    await writeFile(path.join(executionDirectory, "repository.patch"), "");
+    return;
+  }
+  const diff = await captureProcess(
+    "git",
+    ["diff", "--binary", "--no-ext-diff", "HEAD"],
+    { cwd: repositoryDirectory },
+  );
+  if (diff.code === 0) {
+    await writeFile(path.join(executionDirectory, "repository.patch"), diff.stdout);
+  }
+  const untracked = await captureProcess(
+    "git",
+    ["ls-files", "--others", "--exclude-standard"],
+    { cwd: repositoryDirectory },
+  );
+  if (untracked.code === 0) {
+    await writeFile(
+      path.join(executionDirectory, "untracked-files.txt"),
+      untracked.stdout,
+    );
+  }
+}
+
+export async function updateRunState(bundle, patch) {
+  bundle.state = {
+    ...bundle.state,
+    ...patch,
+    updatedAt: new Date().toISOString(),
+  };
+  await writeJson(path.join(bundle.directories.run, "run.json"), bundle.state);
+}
+
+export async function readAgentInstructions(task) {
+  return (await readFile(path.join(PACKAGE_ROOT, "prompts", usesWorkspaceSummary(task) ? "execute-workspace-simple.md" : hasResearchWorkspace(task) ? "execute-workspace.md" : "execute-contract.md"), "utf8")) + DELIVERY_INSTRUCTIONS;
+}