@citeark/agent 0.3.20

package/src/compute/gcp-batch-executor.mjs

@@ -0,0 +1,1215 @@
+import { hasSessionEvents } from '../terminal/events.mjs';
+import { lstat, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { saveRunRecovery, saveRemoteRecovery, restoreRemoteRecovery } from "./coordinator-recovery.mjs";
+import { GcpBatchClient } from "../gcp/batch-client.mjs";
+import { GoogleAccessTokenProvider } from "../gcp/auth.mjs";
+import { GcsStagingClient } from "../gcp/storage.mjs";
+import { updateRunState } from "../job.mjs";
+import { captureProcess } from "../process.mjs";
+import { createRemoteStageArchive } from "./stage-archive.mjs";
+import { agentInvocationPromptArgument, buildAgentInvocation } from "../runtime/index.mjs";
+import {
+  CODEX_FINAL_MESSAGE_PATH,
+  recoverJsonFromFinalMessage,
+} from "../runtime/final-output.mjs";
+import {
+  CiteArkError,
+  pathExists,
+  printableCommand,
+  readJson,
+  sha256File,
+  sha256Value,
+  writeJson,
+} from "../util.mjs";
+import {
+  buildGcpBatchJob,
+  remoteProviderSession,
+  resolveCheckpointObjects,
+  resolveGcpBatchConfig,
+} from "./gcp-batch-job.mjs";
+import { assertSuccessfulGcpBatchTerminal } from "./gcp-batch-failure.mjs";
+import {
+  createLockedBatchJob,
+  deleteBatchJob,
+  releaseBatchJobAccountLock,
+} from "./gcp-batch-lifecycle.mjs";
+import {
+  infrastructureResumePrompt,
+  summarizeInfrastructureAttempt,
+  targetRecoveryPrompt,
+} from "./checkpoint-protocol.mjs";
+import {
+  describeRemoteAgentFailure,
+  readRemoteAttempt,
+  reconcileRemoteAgentOutcome,
+} from "./remote-attempt.mjs";
+import { prepareReproduction } from "../reproduction/lifecycle.mjs";
+import { agentWorkloadMetadata, defineAgentWorkload } from "../workloads/definition.mjs";
+import { createAutonomousReproductionWorkload } from "../workloads/reproduction.mjs";
+
+export async function executeGcpBatchReproduction({
+  taskPath,
+  referenceMaterialsPath,
+  runsDirectory,
+  imageOverride,
+  repositoryOverride,
+  paperOverride,
+  computeDecision,
+  runId,
+  agentRuntime,
+  executorOptions = {},
+  workloadAdapter = null,
+  dryRun = false,
+  progress = () => {},
+}) {
+  const workload = defineAgentWorkload(
+    workloadAdapter ?? createAutonomousReproductionWorkload(),
+  );
+  const gcpOptions = executorOptions.gcpBatch ?? {};
+  const bundle = await prepareReproduction({
+    taskPath,
+    referenceMaterialsPath,
+    runsDirectory,
+    imageOverride,
+    repositoryOverride,
+    paperOverride,
+    computeDecision,
+    runId,
+    agentRuntime,
+    requireApi: false,
+    sandboxProvider: "gcp-batch",
+    validateScientificComputeDecision: workload.validateScientificComputeDecision,
+  });
+  if (bundle.restored && bundle.state.recoveryRemoteResult && !dryRun) {
+    const attempt = bundle.state.recoveryRemoteResult.attempt;
+    const completionPath = path.join(bundle.directories.run, workload.completionRelativePath);
+    if (workload.recoverFinalMessage !== false) await recoverJsonFromFinalMessage({
+      finalMessagePath: hostRuntimePath(bundle, CODEX_FINAL_MESSAGE_PATH), outputPath: completionPath });
+    const submitted = await workload.validate({ bundle, completionPath });
+    const remoteFailure = await describeRemoteAgentFailure(bundle, attempt);
+    const reconciled = reconcileRemoteAgentOutcome({ validation: submitted, remoteFailure });
+    if (reconciled.blockingFailure || reconciled.validation.issues.length) {
+      throw new CiteArkError(`恢复的结果仍需修复：${reconciled.blockingFailure?.message ?? reconciled.validation.issues.join("；")}`, {
+        failureCode: "recovery.result_validation", failureCategory: "protocol", retryable: false,
+      });
+    }
+    progress("复用已下载的远端结果，仅继续协调器验证，不创建 Batch");
+    return workload.finalize({ bundle, validation: reconciled.validation, attempts: recoveryAttempts(bundle),
+      timedOut: false, sandboxImageIdentity: bundle.state.sandboxImageIdentity ?? remoteImageIdentity(bundle.executionEnvironment.image),
+      cloudExecution: bundle.state.cloudExecution, checkpoint: bundle.state.durableRemoteCheckpoint });
+  }
+  await workload.prepare?.(bundle);
+  bundle.paperBudget = gcpOptions.paperBudget;
+  if (bundle.paperBudget) bundle.runtimeAgent = { ...bundle.runtimeAgent, paperBudgetEnforced: true };
+  await updateRunState(bundle, agentWorkloadMetadata(workload));
+  const config = resolveGcpBatchConfig({ bundle, options: gcpOptions });
+  const clients = createClients(config, gcpOptions.clients);
+  if (bundle.restored && !dryRun) {
+    const recovered = bundle.state.recoveryContinueAgent ? null : await resumeExistingRemoteResult({ bundle, workload, config, clients, gcpOptions, progress });
+    if (recovered) return recovered;
+    const restoredCheckpoint = await restoreRemoteRecovery({ bundle, storage: clients.storage, bucket: config.stagingBucket });
+    if (restoredCheckpoint) {
+      config.restoreCheckpointPrefix = restoredCheckpoint.checkpointPrefix;
+      config.restoreCheckpointBucket = restoredCheckpoint.bucket;
+    }
+    if (bundle.state.recoveryContinueAgent) config.reuseCompletedResultOnRestore = false;
+    bundle.state.recoveryGeneration = (bundle.state.recoveryGeneration ?? 0) + 1;
+    config.stagingPrefix += `-r${bundle.state.recoveryGeneration}`;
+  }
+  const instructions = await workload.instructions(bundle);
+  await mkdir(path.join(bundle.directories.execution, "prompts"), { recursive: true });
+
+  let validation = { result: null, issues: ["尚未运行 Agent"], protocolVerification: null };
+  const firstInvocation = await buildRemoteInvocation({
+    bundle,
+    attempt: 1,
+    validation,
+    instructions,
+    workloadAdapter: workload,
+  });
+  const firstResumeInvocation = await buildInfrastructureResumeInvocation({
+    bundle,
+    instructions,
+    targetRecovery: Boolean(config.restoreCheckpointPrefix),
+    workloadAdapter: workload,
+  });
+  const checkpointAttempt = 1;
+  const firstPreview = jobPreview({
+    bundle,
+    invocation: firstInvocation,
+    resumeInvocation: firstResumeInvocation,
+    attempt: 1,
+    config,
+    checkpointAttempt,
+    restoreCheckpointOnStart: Boolean(config.restoreCheckpointPrefix),
+    restoreCheckpointPrefix: config.restoreCheckpointPrefix,
+    reuseCompletedResultOnRestore: externalRestoreReuseCompletedResult(config),
+    workloadAdapter: workload,
+  });
+  await recordJobPreview(bundle, firstPreview, 1);
+  await writeFile(
+    path.join(bundle.directories.execution, "prompts", "attempt-1.txt"),
+    `${agentInvocationPromptArgument(firstInvocation)}\n`,
+  );
+  await writeFile(
+    path.join(bundle.directories.execution, "prompts", "attempt-1-resume.txt"),
+    `${agentInvocationPromptArgument(firstResumeInvocation)}\n`,
+  );
+  if (!dryRun) await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+  if (dryRun) {
+    await updateRunState(bundle, {
+      status: "prepared",
+      cloudExecution: publicCloudExecution(config, []),
+    });
+    return {
+      dryRun: true,
+      bundle,
+      invocation: {
+        printable: printableCommand("POST", [`https://batch.googleapis.com/v1/projects/${config.projectId}/locations/${config.region}/jobs?job_id=${firstPreview.jobId}`]),
+        executor: "gcp-batch",
+        jobId: firstPreview.jobId,
+        jobSpec: sanitizeJob(firstPreview.job),
+      },
+      jobSpec: firstPreview.job,
+      cloud: publicCloudExecution(config, []),
+    };
+  }
+
+  const attempts = [];
+  const jobs = [];
+  const stagedObjects = new Set();
+  const checkpointPrefixes = new Set();
+  const temporaryDirectory = await mkdtemp(path.join(os.tmpdir(), "citeark-gcp-batch-"));
+  let timedOut = false;
+  let successful = false;
+  let processingLeaseLost = false;
+  try {
+    await updateRunState(bundle, {
+      status: "running",
+      startedAt: new Date().toISOString(),
+      cloudExecution: publicCloudExecution(config, jobs),
+    });
+    const controllerValidationRetries = Number.isInteger(workload.controllerValidationRetries)
+      ? Math.max(0, workload.controllerValidationRetries)
+      : bundle.runtimeAgent.validationRetries;
+    const maxAttempts = 1 + controllerValidationRetries;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+      const invocation = attempt === 1
+        ? firstInvocation
+        : await buildRemoteInvocation({ bundle, attempt, validation, instructions, workloadAdapter: workload });
+      const resumeInvocation = attempt === 1
+        ? firstResumeInvocation
+        : await buildInfrastructureResumeInvocation({ bundle, instructions, workloadAdapter: workload });
+      if (attempt > 1) {
+        await writeFile(
+          path.join(bundle.directories.execution, "prompts", `attempt-${attempt}.txt`),
+          `${agentInvocationPromptArgument(invocation)}\n`,
+        );
+        await writeFile(
+          path.join(bundle.directories.execution, "prompts", `attempt-${attempt}-resume.txt`),
+          `${agentInvocationPromptArgument(resumeInvocation)}\n`,
+        );
+      }
+      const basePreview = attempt === 1 ? firstPreview : jobPreview({
+        bundle,
+        invocation,
+        resumeInvocation,
+        attempt,
+        config,
+        checkpointAttempt,
+        restoreCheckpointOnStart: true,
+        reuseCompletedResultOnRestore: false,
+        workloadAdapter: workload,
+      });
+      const stageObject = basePreview.stageObject;
+      const resultObject = basePreview.resultObject;
+      stagedObjects.add(stageObject);
+      stagedObjects.add(resultObject);
+      if (hasSessionEvents()) stagedObjects.add(`${resultObject}.trace.jsonl`);
+      stagedObjects.add(basePreview.checkpointStatus);
+      checkpointPrefixes.add(basePreview.checkpointPrefix);
+      const stageArchive = path.join(temporaryDirectory, basePreview.stageFilename);
+      const resultArchive = path.join(temporaryDirectory, basePreview.resultFilename);
+      await createStageArchive(bundle, stageArchive);
+      progress(`GCP Batch ${bundle.runId}: 上传第 ${attempt}/${maxAttempts} 次执行输入`);
+      await clients.storage.uploadFile({
+        bucket: config.stagingBucket,
+        object: stageObject,
+        filePath: stageArchive,
+      });
+
+      const startedAt = new Date().toISOString();
+      try {
+        await gcpOptions.assertActive?.();
+      } catch (error) {
+        if (error?.processingLeaseLost === true) processingLeaseLost = true;
+        throw error;
+      }
+      let preview = basePreview;
+      let record = null;
+      let terminal = null;
+      try {
+        ({ preview, record, terminal } = await provisionBatchJobAcrossLocations({
+          bundle,
+          invocation,
+          resumeInvocation,
+          attempt,
+          basePreview,
+          config,
+          checkpointAttempt,
+          restoreCheckpointOnStart: attempt > 1 || Boolean(config.restoreCheckpointPrefix),
+          restoreCheckpointPrefix: attempt === 1 ? config.restoreCheckpointPrefix : null,
+          reuseCompletedResultOnRestore: attempt === 1
+            && externalRestoreReuseCompletedResult(config),
+          workloadAdapter: workload,
+          clients,
+          jobs,
+          gcpOptions,
+          progress,
+          startedAt,
+        }));
+        record.state = terminal?.status?.state ?? record.state;
+        record.finishedAt = new Date().toISOString();
+        record.runDuration = terminal?.status?.runDuration ?? null;
+        record.statusEvents = summarizeStatusEvents(terminal?.status?.statusEvents);
+        await writeJson(path.join(bundle.directories.execution, `gcp-batch-result-a${attempt}.json`), sanitizeJob(terminal));
+        assertSuccessfulGcpBatchTerminal({
+          job: terminal,
+          jobId: preview.jobId,
+          workload: workload.workload,
+          infrastructureFailure: summarizeInfrastructureAttempt(terminal).latestFailure,
+        });
+        await clients.storage.downloadFile({
+          bucket: config.stagingBucket,
+          object: resultObject,
+          filePath: resultArchive,
+        });
+        await extractResultArchive(bundle, resultArchive);
+        const externalObjects = await restoreExternalPublicationObjects({
+          bundle,
+          storage: clients.storage,
+          bucket: config.stagingBucket,
+        });
+        for (const object of externalObjects) stagedObjects.add(object);
+        await updateRunState(bundle, { recoveryRemoteResult: await remoteResultReceipt(bundle, record, attempt) });
+        await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+        // The durable result is now on the coordinator. Release the scientific
+        // job before parsing, validation or packaging can spend more time.
+        if (!config.keepJob) await releaseCompletedBatchJob(clients, record);
+      } catch (error) {
+        if (error?.processingLeaseLost === true) {
+          processingLeaseLost = true;
+          throw error;
+        }
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        timedOut ||= error?.failureCategory === "timeout"
+          || /超时|执行时限|maximum runtime/i.test(errorMessage);
+        record = error?.batchRecord ?? record;
+        if (record) {
+          record.state = timedOut ? "TIMED_OUT" : record.state;
+          record.finishedAt ??= new Date().toISOString();
+          record.error ??= errorMessage;
+        }
+        throw new CiteArkError(`GCP Batch 第 ${attempt} 次执行没有返回可恢复的运行包：${record?.error ?? errorMessage}`, {
+          cause: error,
+          retryable: error?.retryable,
+          failureCode: error?.failureCode,
+          failureCategory: error?.failureCategory,
+          recoveryAction: error?.recoveryAction,
+        });
+      }
+
+      const remoteAttempt = await readRemoteAttempt(bundle, attempt);
+      attempts.push({
+        attempt,
+        startedAt,
+        finishedAt: record.finishedAt,
+        exitCode: remoteAttempt?.exitCode ?? (record.state === "SUCCEEDED" ? 0 : 1),
+        signal: null,
+        timedOut: remoteAttempt?.timedOut === true,
+        executor: "gcp-batch",
+        batchJob: record.name,
+        batchState: record.state,
+        infrastructureAttempt: record.infrastructureAttempt,
+        infrastructureAttemptLimit: record.infrastructureAttemptLimit,
+        provisioningAttempt: record.provisioningAttempt,
+        provisioningAttemptLimit: record.provisioningAttemptLimit,
+        provisioningLocation: record.provisioningLocation,
+        resumeCount: record.resumeCount,
+        interruptionCount: record.interruptionCount,
+        checkpoint: record.checkpoint,
+      });
+      const completionPath = path.join(
+        bundle.directories.run,
+        workload.completionRelativePath,
+      );
+      if (workload.recoverFinalMessage !== false) {
+        await recoverJsonFromFinalMessage({
+          finalMessagePath: hostRuntimePath(bundle, CODEX_FINAL_MESSAGE_PATH),
+          outputPath: completionPath,
+        });
+      }
+      const submittedValidation = await workload.validate({ bundle, completionPath });
+      const remoteFailure = await describeRemoteAgentFailure(bundle, attempt);
+      const reconciled = reconcileRemoteAgentOutcome({
+        validation: submittedValidation,
+        remoteFailure,
+      });
+      if (reconciled.blockingFailure) {
+        throw new CiteArkError(reconciled.blockingFailure.message, {
+          retryable: reconciled.blockingFailure.retryable,
+          failureCode: reconciled.blockingFailure.failureCode,
+          failureCategory: reconciled.blockingFailure.failureCategory,
+        });
+      }
+      validation = reconciled.validation;
+      if (validation.issues.length === 0) break;
+      if (attempt < maxAttempts) progress(`结果协议校验失败，将在同一远程会话继续：${validation.issues.join("；")}`);
+    }
+
+    await writeJson(path.join(bundle.directories.execution, "gcp-batch-jobs.json"), {
+      schemaVersion: "0.1",
+      executor: "gcp-batch",
+      ...publicCloudExecution(config, jobs),
+    });
+    await writeGcpInvocation(bundle, jobs, config);
+    if (workload.recover) {
+      validation = await workload.recover({
+        bundle,
+        validation,
+        attempts,
+        timedOut,
+      });
+    }
+    const cloudExecution = publicCloudExecution(config, jobs);
+    const result = await workload.finalize({
+      bundle,
+      validation,
+      attempts,
+      timedOut,
+      sandboxImageIdentity: remoteImageIdentity(bundle.executionEnvironment.image),
+      cloudExecution,
+      checkpoint: recoveryCheckpointFromJobs(config, jobs),
+    });
+    successful = workload.isSucceeded(validation.result, validation.issues);
+    await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+    return result;
+  } catch (error) {
+    const recoveryCheckpoint = recoveryCheckpointFromJobs(config, jobs);
+    if (recoveryCheckpoint && error && typeof error === "object") {
+      error.recoveryCheckpoint = recoveryCheckpoint;
+    }
+    if (bundle.state.status === "running") {
+      await updateRunState(bundle, {
+        status: "failed",
+        finishedAt: new Date().toISOString(),
+        error: error instanceof Error ? error.message : String(error),
+        cloudExecution: publicCloudExecution(config, jobs),
+      });
+    }
+    throw error;
+  } finally {
+    const hasRecoveryCheckpoint = jobs.some((record) =>
+      typeof record?.checkpoint?.sequence === "string",
+    );
+    if (!config.keepJob && (successful || processingLeaseLost || hasRecoveryCheckpoint || jobs.some((record) => record.state === "SUCCEEDED"))) {
+      for (const record of jobs) {
+        if (record.jobDeletionRequested === true) continue;
+        await deleteBatchJob({ batch: clients.batch, record });
+      }
+    }
+    for (const record of jobs) {
+      if (record.jobDeletionRequested || new Set(["SUCCEEDED", "FAILED", "CANCELLED"]).has(record.state)) {
+        await releaseReproductionAccountLock({ bundle, config, clients, record });
+      }
+    }
+    if (!config.keepStaging && (successful || processingLeaseLost) && stagedObjects.size > 0) {
+      for (const prefix of checkpointPrefixes) {
+        const objects = await listCheckpointObjects(
+          clients.storage,
+          config.stagingBucket,
+          prefix,
+        );
+        for (const object of objects) stagedObjects.add(object);
+      }
+      const cleanup = await Promise.allSettled([...stagedObjects].map((object) =>
+        clients.storage.deleteObject({ bucket: config.stagingBucket, object }),
+      ));
+      const deleted = cleanup.every((result) => result.status === "fulfilled");
+      for (const record of jobs) record.stagingObjectsDeleted = deleted;
+    }
+    await rm(temporaryDirectory, { recursive: true, force: true });
+  }
+}
+
+async function releaseCompletedBatchJob(clients, record) {
+  await deleteBatchJob({ batch: clients.batch, record });
+}
+
+export function assertGcpProvisioningViable(job, config) {
+  const capacityEvents = (job?.status?.statusEvents ?? [])
+    .map((event) => event?.description)
+    .filter((description) =>
+      typeof description === "string"
+      && /(?:RESOURCE_POOL_EXHAUSTED|not enough resources|capacity is unavailable|资源不足|容量不足)/i.test(description));
+  if (!capacityEvents.length) return;
+  const spot = config?.provisioningModel === "SPOT";
+  throw new CiteArkError(
+    `GCP Batch ${spot ? "Spot" : "Standard"} 容量不足：${capacityEvents.at(-1)}`,
+    {
+      retryable: true,
+      failureCode: spot
+        ? "gcp.spot_capacity_unavailable"
+        : "gcp.standard_capacity_unavailable",
+      failureCategory: "infrastructure",
+      recoveryAction: spot
+        ? "保持科研计划不变；先轮换同区域可用区，全部缺货后切换到兼容的 GCP Standard 规格"
+        : "保持科研计划与规格不变，轮换同一区域内尚未尝试的可用区",
+    },
+  );
+}
+
+async function provisionBatchJobAcrossLocations({
+  bundle,
+  invocation,
+  resumeInvocation,
+  attempt,
+  basePreview,
+  config,
+  checkpointAttempt,
+  restoreCheckpointOnStart,
+  restoreCheckpointPrefix,
+  reuseCompletedResultOnRestore,
+  workloadAdapter,
+  clients,
+  jobs,
+  gcpOptions,
+  progress,
+  startedAt,
+}) {
+  const locations = rotatedLocations(config.allowedLocations, bundle.runId);
+  const candidates = locations.length ? locations : [null];
+  for (const [index, location] of candidates.entries()) {
+    const provisioningAttempt = index + 1;
+    const candidateConfig = {
+      ...config,
+      allowedLocations: location ? [location] : [],
+    };
+    const preview = candidates.length === 1
+      ? basePreview
+      : jobPreview({
+        bundle,
+        invocation,
+        resumeInvocation,
+        attempt,
+        config: candidateConfig,
+        checkpointAttempt,
+        restoreCheckpointOnStart,
+        restoreCheckpointPrefix,
+        reuseCompletedResultOnRestore,
+        provisioningAttempt,
+        workloadAdapter,
+      });
+    await recordJobPreview(bundle, preview, attempt, provisioningAttempt);
+    const locationLabel = location ?? config.region;
+    progress(
+      `GCP Batch ${bundle.runId}: 创建 ${preview.jobId}（${config.provisioningModel} / ${config.machineType} / ${locationLabel}）`,
+    );
+    bundle.state.recoveryPendingJob = { jobId: preview.jobId, name: clients.batch.jobName(preview.jobId),
+      resultObject: preview.resultObject, checkpointPrefix: preview.checkpointPrefix, checkpointStatus: preview.checkpointStatus,
+      attempt, state: "CREATING", startedAt };
+    await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+    const created = await createLockedBatchJob({
+      storage: clients.storage,
+      batch: clients.batch,
+      bucket: config.stagingBucket,
+      accountId: bundle.runtimeAgent.codexAccountId,
+      owner: `${bundle.runId}-${attempt}`,
+      jobId: preview.jobId,
+      job: preview.job,
+    });
+    const jobName = created?.name ?? clients.batch.jobName(preview.jobId);
+    const record = {
+      attempt,
+      jobId: preview.jobId,
+      name: jobName,
+      state: "QUEUED",
+      startedAt: provisioningAttempt === 1 ? startedAt : new Date().toISOString(),
+      infrastructureAttempt: 1,
+      infrastructureAttemptLimit: 1 + config.infrastructureRetryCount,
+      provisioningAttempt,
+      provisioningAttemptLimit: candidates.length,
+      provisioningLocation: location,
+      executionAttempt: 1,
+      resumeCount: 0,
+      interruptionCount: 0,
+      latestFailure: null,
+      checkpoint: null,
+      checkpointPrefix: preview.checkpointPrefix,
+      checkpointStatus: preview.checkpointStatus,
+      resultObject: preview.resultObject,
+      codexAccountLockOwner: `${bundle.runId}-${attempt}`,
+    };
+    jobs.push(record);
+    await updateRunState(bundle, { cloudExecution: publicCloudExecution(config, jobs) });
+    await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+    try {
+      const terminal = await clients.batch.waitForTerminal(jobName, {
+        liveOutput: { storage: clients.storage, bucket: config.stagingBucket, object: `${record.resultObject}.trace.jsonl`, source: `${bundle.runId}:${attempt}`, runtime: bundle.runtimeAgent.runtime },
+        timeoutMs: config.waitTimeoutMinutes * 60_000,
+        pollIntervalMs: config.pollIntervalMs,
+        onState: async (state) => {
+          progress(`GCP Batch ${preview.jobId}: ${state}`);
+        },
+        onPoll: async (job, state) => {
+          await gcpOptions.assertActive?.();
+          const previousRecovery = `${record.state}:${record.checkpoint?.sequence ?? ""}`;
+          await updateCloudJobObservation({
+            bundle,
+            config: candidateConfig,
+            jobs,
+            record,
+            job,
+            state,
+            checkpointStatus: preview.checkpointStatus,
+            storage: clients.storage,
+            progress,
+          });
+          if (previousRecovery !== `${record.state}:${record.checkpoint?.sequence ?? ""}`) {
+            await saveRemoteRecovery({ bundle, storage: clients.storage, bucket: config.stagingBucket, record, persist: gcpOptions.onCheckpoint, recoveryBucket: gcpOptions.recoveryBucket, recoveryNamespace: gcpOptions.recoveryNamespace });
+          }
+          assertGcpProvisioningViable(job, candidateConfig);
+        },
+      });
+      // The final recovery checkpoint is written immediately before the
+      // supervisor publishes its result. A last status read after Batch has
+      // reached a terminal state closes the race where the previous poll saw
+      // the result but not the durable target-recovery checkpoint pointer.
+      await updateCloudJobObservation({
+        bundle,
+        config: candidateConfig,
+        jobs,
+        record,
+        job: terminal,
+        state: terminal?.status?.state ?? record.state,
+        checkpointStatus: preview.checkpointStatus,
+        storage: clients.storage,
+        progress,
+      });
+      await saveRemoteRecovery({ bundle, storage: clients.storage, bucket: config.stagingBucket, record, persist: gcpOptions.onCheckpoint, recoveryBucket: gcpOptions.recoveryBucket, recoveryNamespace: gcpOptions.recoveryNamespace });
+      await releaseReproductionAccountLock({ bundle, config, clients, record });
+      assertGcpProvisioningViable(terminal, candidateConfig);
+      return { preview, record, terminal };
+    } catch (error) {
+      if (error?.processingLeaseLost === true) {
+        error.batchRecord = record;
+        throw error;
+      }
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      record.finishedAt = new Date().toISOString();
+      record.error = errorMessage;
+      const capacityFailure = /gcp\.(?:spot|standard)_capacity_unavailable/.test(
+        error?.failureCode ?? "",
+      );
+      if (capacityFailure) {
+        await deleteCapacityFailedJob({ clients, jobName, record });
+        await releaseReproductionAccountLock({ bundle, config, clients, record });
+        if (provisioningAttempt < candidates.length) {
+          progress(
+            `GCP Batch ${preview.jobId}: ${locationLabel} 容量不足，继续尝试下一个可用区`,
+          );
+          continue;
+        }
+      }
+      error.batchRecord = record;
+      throw error;
+    }
+  }
+  throw new CiteArkError("GCP Batch 没有可尝试的运行位置", {
+    retryable: true,
+    failureCode: "gcp.location_unavailable",
+    failureCategory: "infrastructure",
+  });
+}
+
+async function releaseReproductionAccountLock({ bundle, config, clients, record }) {
+  await releaseBatchJobAccountLock({
+    storage: clients.storage,
+    bucket: config.stagingBucket,
+    accountId: bundle.runtimeAgent?.codexAccountId,
+    owner: record?.codexAccountLockOwner ?? `${bundle.runId}-${record?.attempt ?? 1}`,
+    jobName: record?.name,
+    record,
+  });
+}
+
+async function deleteCapacityFailedJob({ clients, jobName, record }) {
+  if (!record.name) record.name = jobName;
+  await deleteBatchJob({ batch: clients.batch, record });
+}
+
+function rotatedLocations(locations, seed) {
+  if (!Array.isArray(locations) || locations.length < 2) return [...(locations ?? [])];
+  let hash = 0;
+  for (const character of seed) hash = ((hash * 31) + character.charCodeAt(0)) >>> 0;
+  const offset = hash % locations.length;
+  return [...locations.slice(offset), ...locations.slice(0, offset)];
+}
+
+function hostRuntimePath(bundle, containerPath) {
+  return path.join(bundle.directories.run, containerPath.replace(/^\/job\//, ""));
+}
+
+async function buildRemoteInvocation({
+  bundle,
+  attempt,
+  validation,
+  instructions,
+  workloadAdapter,
+}) {
+  const invocation = await buildAgentInvocation({
+    bundle,
+    attempt,
+    prompt: workloadAdapter
+      ? workloadAdapter.prompt({ bundle, attempt, validation })
+      : attempt === 1
+        ? initialReproductionPrompt(bundle)
+        : retryReproductionPrompt(validation.issues, bundle.runtimeTask),
+    instructions,
+    providerSession: remoteProviderSession(bundle.runtimeAgent),
+  });
+  return workloadAdapter?.wrapInvocation
+    ? workloadAdapter.wrapInvocation({ bundle, attempt, invocation, validation })
+    : invocation;
+}
+
+async function buildInfrastructureResumeInvocation({
+  bundle,
+  instructions,
+  targetRecovery = false,
+  workloadAdapter,
+}) {
+  const invocation = await buildAgentInvocation({
+    bundle,
+    attempt: 2,
+    prompt: workloadAdapter
+      ? workloadAdapter.resumePrompt({ bundle, targetRecovery })
+      : targetRecovery
+        ? targetRecoveryPrompt(bundle)
+        : infrastructureResumePrompt(bundle),
+    instructions,
+    providerSession: remoteProviderSession(bundle.runtimeAgent),
+  });
+  return workloadAdapter?.wrapInvocation
+    ? workloadAdapter.wrapInvocation({ bundle, attempt: 2, invocation, targetRecovery })
+    : invocation;
+}
+
+function jobPreview({
+  bundle,
+  invocation,
+  resumeInvocation,
+  attempt,
+  config,
+  checkpointAttempt = attempt,
+  restoreCheckpointOnStart = false,
+  restoreCheckpointPrefix = null,
+  reuseCompletedResultOnRestore = true,
+  provisioningAttempt = 1,
+  workloadAdapter = null,
+}) {
+  const normalized = `${bundle.runId}-a${attempt}`
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9-]/g, "-")
+    .replaceAll(/^-+|-+$/g, "");
+  const suffix = `${bundle.state.recoveryGeneration ? `-r${bundle.state.recoveryGeneration}` : ""}${provisioningAttempt > 1 ? `-p${provisioningAttempt}` : ""}`;
+  const jobId = reproductionBatchJobId(normalized, suffix);
+  const stageFilename = `stage-a${attempt}.tar.gz`;
+  const resultFilename = `result-a${attempt}.tar.gz`;
+  // Keep temporary objects at the bucket root. This avoids relying on
+  // implicit directory emulation in GCSFuse, whose CLI changed in v3.
+  const objectPrefix = config.stagingPrefix.replaceAll(/[^a-zA-Z0-9_.-]/g, "-");
+  const stageObject = `${objectPrefix}-${stageFilename}`;
+  const resultObject = `${objectPrefix}-${resultFilename}`;
+  const externalObjectPrefix = `${objectPrefix}-result-a${attempt}-object`;
+  const checkpoint = resolveCheckpointObjects(config, checkpointAttempt);
+  return {
+    jobId,
+    stageFilename,
+    resultFilename,
+    stageObject,
+    resultObject,
+    externalObjectPrefix,
+    checkpointPrefix: checkpoint.prefix,
+    checkpointStatus: checkpoint.status,
+    job: buildGcpBatchJob({
+      bundle,
+      invocation,
+      resumeInvocation,
+      attempt,
+      config,
+      stageObject,
+      resultObject,
+      externalObjectPrefix,
+      checkpointPrefix: checkpoint.prefix,
+      checkpointStatus: checkpoint.status,
+      restoreCheckpointPrefix,
+      restoreCheckpointOnStart,
+      reuseCompletedResultOnRestore,
+      completionRelativePath: workloadAdapter?.completionRelativePath,
+      completionHasEvidence: workloadAdapter?.completionHasEvidence,
+      completionValidatorRelativePath: workloadAdapter?.completionValidatorRelativePath,
+      completionSuccessJq: workloadAdapter?.completionSuccessJq,
+      preserveRebuildableCache: workloadAdapter?.preserveRebuildableCache,
+      finalCheckpointKind: workloadAdapter?.finalCheckpointKind,
+      finalCheckpointReason: workloadAdapter?.finalCheckpointReason,
+      allowCpuHandoff: workloadAdapter?.allowCpuHandoff,
+      runtimeBudgetMinutes: workloadAdapter?.runtimeBudgetMinutes,
+      workload: workloadAdapter?.workload ?? "reproduction",
+    }),
+  };
+}
+
+export function reproductionBatchJobId(normalized, suffix = "") {
+  const limit = 63 - suffix.length;
+  const stem = normalized.length <= limit ? normalized
+    : `${normalized.slice(0, limit - 13)}-${sha256Value(normalized).slice(0, 12)}`;
+  return `${stem}${suffix}`;
+}
+
+async function recordJobPreview(bundle, preview, attempt, provisioningAttempt = 1) {
+  const suffix = provisioningAttempt > 1 ? `-p${provisioningAttempt}` : "";
+  await writeJson(path.join(bundle.directories.execution, `gcp-batch-job-a${attempt}${suffix}.json`), {
+    schemaVersion: "0.1",
+    executor: "gcp-batch",
+    jobId: preview.jobId,
+    checkpointPrefix: preview.checkpointPrefix,
+    checkpointStatus: preview.checkpointStatus,
+    job: sanitizeJob(preview.job),
+  });
+}
+
+function recoveryCheckpointFromJobs(config, jobs) {
+  const job = [...jobs].reverse().find((candidate) =>
+    typeof candidate?.checkpointPrefix === "string"
+    && typeof candidate?.checkpoint?.sequence === "string",
+  );
+  if (!job) return null;
+  return {
+    schemaVersion: "0.1",
+    provider: "gcp",
+    bucket: config.stagingBucket,
+    checkpointPrefix: job.checkpointPrefix,
+    checkpointStatus: job.checkpointStatus,
+    sequence: job.checkpoint.sequence,
+    kind: job.checkpoint.kind,
+    reason: job.checkpoint.reason,
+    object: job.checkpoint.object,
+    bytes: job.checkpoint.bytes,
+    sha256: job.checkpoint.sha256,
+    createdAt: job.checkpoint.createdAt,
+  };
+}
+
+function createClients(config, injected = {}) {
+  if (injected.batch && injected.storage) return injected;
+  const auth = injected.auth ?? new GoogleAccessTokenProvider(injected.authOptions);
+  return {
+    batch: injected.batch ?? new GcpBatchClient({
+      projectId: config.projectId,
+      region: config.region,
+      auth,
+      ...injected.batchOptions,
+    }),
+    storage: injected.storage ?? new GcsStagingClient({ auth, ...injected.storageOptions }),
+  };
+}
+
+async function updateCloudJobObservation({
+  bundle,
+  config,
+  jobs,
+  record,
+  job,
+  state,
+  checkpointStatus,
+  storage,
+  progress,
+}) {
+  const previousState = record.state;
+  const previousInfrastructureAttempt = record.infrastructureAttempt;
+  const previousExecutionAttempt = record.executionAttempt;
+  const previousCheckpointSequence = record.checkpoint?.sequence ?? null;
+  const previousActivityDigest = record.activity?.traceDigest ?? null;
+  const infrastructure = summarizeInfrastructureAttempt(job);
+  record.state = state;
+  record.runDuration = job?.status?.runDuration ?? record.runDuration ?? null;
+  record.statusEvents = summarizeStatusEvents(job?.status?.statusEvents);
+  record.infrastructureAttempt = infrastructure.infrastructureAttempt;
+  record.executionAttempt = infrastructure.executionAttempt;
+  record.resumeCount = infrastructure.resumeCount;
+  record.interruptionCount = infrastructure.interruptionCount;
+  record.latestFailure = infrastructure.latestFailure;
+
+  if (config.checkpointing.enabled && typeof storage.readJsonObject === "function") {
+    const checkpoint = await storage.readJsonObject({
+      bucket: config.stagingBucket,
+      object: checkpointStatus,
+      missing: null,
+    }).catch(() => null);
+    if (
+      checkpoint?.schemaVersion === "0.1"
+      && typeof checkpoint.sequence === "string"
+      && typeof checkpoint.object === "string"
+    ) {
+      const activity = checkpoint.activity;
+      const { activity: _privateActivity, ...checkpointRecord } = checkpoint;
+      record.checkpoint = checkpointRecord;
+      if (
+        typeof activity?.traceDigest === "string"
+        && Array.isArray(activity.traceTail)
+      ) {
+        const traceTail = activity.traceTail
+          .filter(isPlainObject)
+          .map(sanitizeLiveTraceRecord)
+          .filter(Boolean)
+          .slice(-24);
+        const activityChanged = activity.traceDigest !== previousActivityDigest;
+        record.activity = {
+          traceDigest: activity.traceDigest,
+          traceTail,
+          lastOutputAt: activityChanged
+            ? checkpoint.createdAt ?? new Date().toISOString()
+            : record.activity?.lastOutputAt ?? null,
+        };
+      }
+    }
+  }
+
+  const checkpointChanged = (record.checkpoint?.sequence ?? null) !== previousCheckpointSequence;
+  const activityChanged = (record.activity?.traceDigest ?? null) !== previousActivityDigest;
+  const infrastructureChanged = record.infrastructureAttempt !== previousInfrastructureAttempt;
+  const executionAttemptChanged = record.executionAttempt !== previousExecutionAttempt;
+  const stateChanged = state !== previousState;
+  if (!checkpointChanged && !activityChanged && !infrastructureChanged && !executionAttemptChanged && !stateChanged) return;
+
+  if (infrastructureChanged && record.infrastructureAttempt > 1) {
+    progress(
+      `GCP Batch ${record.jobId}: 检测到可恢复的基础设施故障${record.latestFailure?.exitCode ? `（退出码 ${record.latestFailure.exitCode}）` : ""}，进入第 ${record.infrastructureAttempt}/${record.infrastructureAttemptLimit} 次执行并从最近检查点恢复`,
+    );
+  } else if (executionAttemptChanged && record.latestFailure?.category === "timeout") {
+    progress(
+      `GCP Batch ${record.jobId}: 上一次执行达到不可变时限（退出码 ${record.latestFailure.exitCode}）；该超时不属于基础设施恢复`,
+    );
+  }
+  if (checkpointChanged) {
+    progress(
+      `GCP Batch ${record.jobId}: 已保存${record.checkpoint.kind === "full" ? "完整" : "关键"}检查点 ${record.checkpoint.sequence}`,
+    );
+  }
+  await updateRunState(bundle, { cloudExecution: publicCloudExecution(config, jobs) });
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function sanitizeLiveTraceRecord(value) {
+  if (!isPlainObject(value) || typeof value.type !== "string") return null;
+  const record = compactObject({
+    type: value.type,
+    timestamp: safeText(value.timestamp, 64),
+    recorded_at: safeText(value.recorded_at, 64),
+    created_at: safeText(value.created_at, 64),
+    attempt: safeNumber(value.attempt),
+    executor: safeText(value.executor, 64),
+    infrastructureRetry: safeNumber(value.infrastructureRetry),
+    restoredCheckpoint: safeNumber(value.restoredCheckpoint),
+    kind: safeText(value.kind, 48),
+    reason: safeText(value.reason, 120),
+    sequence: safeText(value.sequence, 120),
+    bytes: safeNumber(value.bytes),
+    exitCode: safeNumber(value.exitCode),
+    exit_code: safeNumber(value.exit_code),
+    timed_out: typeof value.timed_out === "boolean" ? value.timed_out : undefined,
+    resultSha256: safeDigest(value.resultSha256),
+    signal: safeText(value.signal, 32),
+    thread_id: safeText(value.thread_id, 120),
+    turn_id: safeText(value.turn_id, 120),
+    is_error: typeof value.is_error === "boolean" ? value.is_error : undefined,
+    tool_name: safeText(value.tool_name, 80),
+    elapsed_time_seconds: safeNumber(value.elapsed_time_seconds),
+  });
+  if (isPlainObject(value.item)) {
+    record.item = compactObject({
+      id: safeText(value.item.id, 120),
+      type: safeText(value.item.type, 80),
+      status: safeText(value.item.status, 48),
+      exit_code: safeNumber(value.item.exit_code),
+      command: safeText(value.item.command, 300),
+      text: safeText(value.item.text, 500),
+    });
+  }
+  return record;
+}
+
+function compactObject(value) {
+  return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
+}
+
+function safeText(value, maximumLength) {
+  if (typeof value !== "string") return undefined;
+  const redacted = value
+    .replaceAll(/sk-[A-Za-z0-9_-]{12,}/g, "<redacted-secret>")
+    .replaceAll(/((?:api[_-]?key|token|password|secret)\s*[=:]\s*)[^\s,;]+/gi, "$1<redacted>")
+    .replaceAll(/[\r\n\t]+/g, " ")
+    .trim();
+  return redacted.length > maximumLength
+    ? `${redacted.slice(0, maximumLength - 1)}…`
+    : redacted;
+}
+
+function safeNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}
+
+function safeDigest(value) {
+  return typeof value === "string" && /^[a-f0-9]{64}$/i.test(value) ? value : undefined;
+}
+
+async function listCheckpointObjects(storage, bucket, prefix) {
+  if (typeof storage.listObjects !== "function") return [];
+  return await storage.listObjects({ bucket, prefix: `${prefix}-` }).catch(() => []);
+}
+
+async function createStageArchive(bundle, destination) {
+  await createRemoteStageArchive({
+    runDirectory: bundle.directories.run,
+    destination,
+    label: "GCP",
+    paperBudget: bundle.paperBudget,
+  });
+}
+
+async function extractResultArchive(bundle, source) {
+  const result = await captureProcess("tar", ["-xzf", source, "-C", bundle.directories.run]);
+  if (result.code !== 0) throw new CiteArkError(`解压 GCP 运行结果失败：\n${result.stderr.trim()}`);
+}
+
+export async function restoreExternalPublicationObjects({ bundle, storage, bucket }) {
+  const manifestPath = path.join(bundle.directories.execution, "result-export.json");
+  if (!(await pathExists(manifestPath))) return [];
+  const manifest = await readJson(manifestPath);
+  const descriptors = Array.isArray(manifest.externalObjects)
+    ? manifest.externalObjects
+    : [];
+  const restored = [];
+  for (const descriptor of descriptors) {
+    const relative = descriptor?.path;
+    const object = descriptor?.object;
+    const digest = descriptor?.sha256;
+    const size = descriptor?.size;
+    if (
+      typeof relative !== "string"
+      || typeof object !== "string"
+      || path.posix.basename(object) !== object
+      || !/^[a-f0-9]{64}$/.test(String(digest))
+      || !Number.isSafeInteger(size)
+      || size <= 0
+    ) {
+      throw new CiteArkError("远程发布清单包含无效的大对象描述");
+    }
+    const destination = path.resolve(bundle.directories.run, relative);
+    if (!destination.startsWith(`${path.resolve(bundle.directories.run)}${path.sep}`)) {
+      throw new CiteArkError(`远程发布大对象路径逃逸：${relative}`);
+    }
+    await mkdir(path.dirname(destination), { recursive: true });
+    await storage.downloadFile({ bucket, object, filePath: destination });
+    const [observedDigest, stat] = await Promise.all([
+      sha256File(destination),
+      lstat(destination),
+    ]);
+    if (!stat.isFile() || stat.size !== size || observedDigest !== digest) {
+      throw new CiteArkError(`远程发布大对象身份校验失败：${relative}`);
+    }
+    restored.push(object);
+  }
+  return restored;
+}
+
+async function writeGcpInvocation(bundle, jobs, config) {
+  await writeJson(path.join(bundle.directories.execution, "invocation.json"), {
+    schemaVersion: "0.1",
+    sandbox: "gcp-batch",
+    credentials: {
+      agent: bundle.runtimeAgent.runtime,
+      protocol: bundle.runtimeAgent.api.protocol,
+      source: "gcp-secret-manager",
+      mode: "secret-version-reference",
+      injectedIntoSandbox: true,
+      valueRecorded: false,
+    },
+    networkEgress: { mode: "direct", allowedHosts: [new URL(bundle.runtimeAgent.api.baseUrl).hostname] },
+    cloud: publicCloudExecution(config, jobs),
+    invocations: jobs.map((job) => ({ attempt: job.attempt, batchJob: job.name, state: job.state })),
+  });
+}
+
+function publicCloudExecution(config, jobs) {
+  const runSeconds = jobs.reduce((total, job) => total + durationSeconds(job.runDuration), 0);
+  const billableFloorSeconds = jobs.reduce((total, job) => {
+    const seconds = durationSeconds(job.runDuration);
+    return total + (seconds > 0 ? Math.max(60, seconds) : 0);
+  }, 0);
+  return {
+    provider: "gcp",
+    service: "batch",
+    projectId: config.projectId,
+    region: config.region,
+    machineType: config.machineType,
+    provisioningModel: config.provisioningModel,
+    stagingBucket: config.stagingBucket,
+    stagingPrefix: config.stagingPrefix,
+    checkpointing: {
+      ...config.checkpointing,
+      storage: "gcs",
+      durabilityBoundary: "checkpoint-object-closed",
+    },
+    temporaryResources: {
+      cleanupPolicy: {
+        deleteBatchJobs: !config.keepJob,
+        deleteStagingObjects: !config.keepStaging,
+      },
+      batchJobDeletionRequested: config.keepJob ? false : jobs.length ? jobs.every((job) => job.jobDeletionRequested === true) : null,
+      stagingObjectsDeleted: config.keepStaging ? false : jobs.length ? jobs.every((job) => job.stagingObjectsDeleted === true) : null,
+    },
+    credentialInjection: {
+      mode: "runtime-secret-manager-access",
+      valueRecorded: false,
+      batchEnvironmentContainsSecretValue: false,
+    },
+    computeUsage: {
+      jobRunSeconds: runSeconds,
+      billableFloorSeconds,
+      estimatedCostUsd: typeof config.hourlyUsd === "number"
+        ? Number(((billableFloorSeconds / 3600) * config.hourlyUsd).toFixed(6))
+        : null,
+      pricingSource: typeof config.hourlyUsd === "number" ? "catalog-rate-x-observed-duration" : "unavailable",
+      invoiceReconciled: false,
+    },
+    jobs: structuredClone(jobs),
+  };
+}
+
+function externalRestoreReuseCompletedResult(config) {
+  return config.restoreCheckpointPrefix
+    ? config.reuseCompletedResultOnRestore === true
+    : true;
+}
+
+function sanitizeJob(value) {
+  const clone = structuredClone(value);
+  for (const group of clone?.taskGroups ?? []) {
+    const variables = group?.taskSpec?.environment?.variables;
+    if (variables?.CITEARK_PROVIDER_SECRET_RESOURCE) {
+      variables.CITEARK_PROVIDER_SECRET_RESOURCE = "<secret-manager-version>";
+    }
+    if (variables?.CITEARK_PROVIDER_CREDENTIAL_TOKEN) {
+      variables.CITEARK_PROVIDER_CREDENTIAL_TOKEN = "<short-lived-credential-token>";
+    }
+    const secrets = group?.taskSpec?.environment?.secretVariables;
+    if (secrets) {
+      for (const key of Object.keys(secrets)) secrets[key] = "<secret-manager-version>";
+    }
+  }
+  return clone;
+}
+
+function summarizeStatusEvents(events = []) {
+  return events.slice(-20).map((event) => ({
+    type: event.type ?? null,
+    description: event.description ?? null,
+    eventTime: event.eventTime ?? null,
+  }));
+}
+
+function remoteImageIdentity(image) {
+  const digest = image.match(/@sha256:[0-9a-f]{64}$/)?.[0]?.slice(1) ?? null;
+  return { id: digest, repositoryDigests: digest ? [image] : [] };
+}
+
+function durationSeconds(value) {
+  const matched = typeof value === "string" ? value.match(/^(\d+(?:\.\d+)?)s$/) : null;
+  return matched ? Number(matched[1]) : 0;
+}
+
+async function resumeExistingRemoteResult({ bundle, workload, config, clients, gcpOptions, progress }) {
+  const previous = bundle.state.cloudExecution?.jobs?.at(-1) ?? bundle.state.recoveryPendingJob;
+  if (!previous?.name || typeof clients.batch.findJob !== "function") return null;
+  const cloud = await clients.batch.findJob(previous.name);
+  if (!cloud) return null;
+  progress(`重新连接已有计算任务 ${previous.jobId}，不创建重复任务`);
+  const jobs = [previous];
+  const terminal = await clients.batch.waitForTerminal(previous.name, {
+    liveOutput: { storage: clients.storage, bucket: config.stagingBucket, object: `${previous.resultObject}.trace.jsonl`, source: bundle.runId, runtime: bundle.runtimeAgent.runtime },
+    timeoutMs: config.waitTimeoutMinutes * 60_000, pollIntervalMs: config.pollIntervalMs,
+    onPoll: async (job, state) => {
+      await gcpOptions.assertActive?.();
+      const before = `${previous.state}:${previous.checkpoint?.sequence ?? ""}`;
+      await updateCloudJobObservation({ bundle, config, jobs, record: previous, job, state,
+        checkpointStatus: previous.checkpointStatus, storage: clients.storage, progress });
+      if (before !== `${previous.state}:${previous.checkpoint?.sequence ?? ""}`) {
+        await saveRemoteRecovery({ bundle, storage: clients.storage, bucket: config.stagingBucket, record: previous, persist: gcpOptions.onCheckpoint, recoveryBucket: gcpOptions.recoveryBucket, recoveryNamespace: gcpOptions.recoveryNamespace });
+      }
+    },
+  });
+  previous.state = terminal?.status?.state ?? previous.state;
+  await releaseReproductionAccountLock({ bundle, config, clients, record: previous });
+  if (terminal?.status?.state !== "SUCCEEDED") {
+    if (bundle.state.durableRemoteCheckpoint) {
+      await clients.batch.deleteJob(previous.name);
+      return null;
+    }
+    assertSuccessfulGcpBatchTerminal({ job: terminal, jobId: previous.jobId, workload: workload.workload });
+  }
+  if (!previous.resultObject) throw new CiteArkError("恢复记录缺少远端结果对象，禁止重复执行");
+  const temporary = await mkdtemp(path.join(os.tmpdir(), "citeark-resume-result-"));
+  try {
+    const archive = path.join(temporary, "result.tar.gz");
+    await clients.storage.downloadFile({ bucket: config.stagingBucket, object: previous.resultObject, filePath: archive });
+    await extractResultArchive(bundle, archive);
+    await restoreExternalPublicationObjects({ bundle, storage: clients.storage, bucket: config.stagingBucket });
+    await updateRunState(bundle, { recoveryRemoteResult: await remoteResultReceipt(bundle, { ...previous, state: "SUCCEEDED", finishedAt: new Date().toISOString() }, previous.attempt) });
+    await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+    const completionPath = path.join(bundle.directories.run, workload.completionRelativePath);
+    if (workload.recoverFinalMessage !== false) await recoverJsonFromFinalMessage({
+      finalMessagePath: hostRuntimePath(bundle, CODEX_FINAL_MESSAGE_PATH), outputPath: completionPath });
+    const validation = await workload.validate({ bundle, completionPath });
+    const remoteFailure = await describeRemoteAgentFailure(bundle, previous.attempt);
+    const reconciled = reconcileRemoteAgentOutcome({ validation, remoteFailure });
+    if (reconciled.blockingFailure || validation.issues.length) throw new CiteArkError(`已恢复结果，待修复验证：${reconciled.blockingFailure?.message ?? validation.issues.join("；")}`, { failureCode: "recovery.result_validation", retryable: false });
+    const result = await workload.finalize({ bundle, validation: reconciled.validation, attempts: recoveryAttempts(bundle), timedOut: false,
+      sandboxImageIdentity: bundle.state.sandboxImageIdentity ?? remoteImageIdentity(bundle.executionEnvironment.image), cloudExecution: bundle.state.cloudExecution, checkpoint: bundle.state.durableRemoteCheckpoint });
+    await saveRunRecovery(bundle, gcpOptions.onCheckpoint);
+    await clients.batch.deleteJob(previous.name);
+    return result;
+  } finally { await rm(temporary, { recursive: true, force: true }); }
+}
+
+
+async function remoteResultReceipt(bundle, record, attempt) {
+  const remote = await readRemoteAttempt(bundle, attempt);
+  return { attempt, jobName: record.name, downloadedAt: new Date().toISOString(), attemptRecord: {
+    attempt, startedAt: record.startedAt, finishedAt: record.finishedAt,
+    exitCode: remote?.exitCode ?? (record.state === "SUCCEEDED" ? 0 : 1), signal: null,
+    timedOut: remote?.timedOut === true, executor: "gcp-batch", batchJob: record.name, batchState: record.state,
+    infrastructureAttempt: record.infrastructureAttempt, resumeCount: record.resumeCount, checkpoint: record.checkpoint,
+  } };
+}
+function recoveryAttempts(bundle) {
+  return bundle.state.attempts?.length ? bundle.state.attempts
+    : bundle.state.recoveryRemoteResult?.attemptRecord ? [bundle.state.recoveryRemoteResult.attemptRecord] : [];
+}