@citeark/agent 0.3.20

package/src/compute/autodl-pro-remote.mjs

@@ -0,0 +1,810 @@
+import { hasSessionEvents, createRuntimeStream } from '../terminal/events.mjs';
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+
+import { buildCheckpointedWorkerScript } from "./gcp-checkpoint-worker.mjs";
+import {
+  resolveAutoDlProviderEnvironment,
+  shellEnvironmentFile,
+} from "./autodl-pro-job.mjs";
+import {
+  autoDlRoutingProxySource,
+  buildAutoDlNetworkRunnerScript,
+  resolveAutoDlManagedNetwork,
+} from "./autodl-pro-network.mjs";
+import { createAutoDlGcsStageTransfer } from "./autodl-pro-staging.mjs";
+import { captureProcess } from "../process.mjs";
+import { createRemoteStageArchive } from "./stage-archive.mjs";
+import {
+  CiteArkError,
+  pathExists,
+  readJson,
+  sha256File,
+  shellQuote,
+} from "../util.mjs";
+
+export async function executeAutoDlRemoteAttempt({
+  bundle,
+  lease,
+  config,
+  invocation,
+  resumeInvocation,
+  attempt,
+  stageArchive,
+  resultArchive,
+  checkpointPolicy,
+  restoreCheckpointOnStart = false,
+  reuseCompletedResultOnRestore = true,
+  completionRelativePath = "output/result.json",
+  completionHasEvidence = true,
+  secretManager,
+  stageTransfer,
+  stageClients,
+  assertActive = () => {},
+  progress = () => {},
+  onCheckpoint = () => {},
+}) {
+  const safeRunId = safeIdentifier(bundle.runId);
+  const remoteDirectory = `/root/citeark/${safeRunId}`;
+  const remoteStage = `${remoteDirectory}/stage-a${attempt}.tar.gz`;
+  const remoteResult = `${remoteDirectory}/result-a${attempt}.tar.gz`;
+  const externalPrefix = `${remoteDirectory}/result-a${attempt}-object`;
+  const checkpointPrefix = `${remoteDirectory}/checkpoint-a1`;
+  const checkpointStatus = `${checkpointPrefix}-status.json`;
+  const remoteScript = `${remoteDirectory}/worker-a${attempt}.sh`;
+  const remoteEnvironment = `${remoteDirectory}/environment-a${attempt}.sh`;
+  const remoteNetworkRunner = `${remoteDirectory}/network-runner-a${attempt}.sh`;
+  const remoteRouter = `${remoteDirectory}/network-router-a${attempt}.mjs`;
+  const remoteTunnelKey = `${remoteDirectory}/liguse-tunnel-a${attempt}`;
+  const remoteKnownHosts = `${remoteDirectory}/liguse-known-hosts-a${attempt}`;
+  const remoteProviderRelay = `${remoteDirectory}/provider-pro-relay-a${attempt}.mjs`;
+  const providerRelay = bundle.runtimeAgent.reasoningMode === "pro";
+  const environment = await resolveAutoDlProviderEnvironment({
+    agent: bundle.runtimeAgent,
+    providerSecret: config.providerSecret,
+    credentialSource: config.credentialSource,
+    allowRemoteModelKey: config.allowRemoteModelKey,
+    secretManager,
+  });
+  const managedNetwork = await resolveAutoDlManagedNetwork({ config, secretManager });
+  const workerScript = buildCheckpointedWorkerScript({
+    runId: bundle.runId,
+    executor: "autodl-pro",
+    initialInvocation: invocation,
+    resumeInvocation,
+    attempt,
+    stagePath: remoteStage,
+    resultPath: remoteResult,
+    streamTrace: hasSessionEvents(),
+    checkpointPrefixPath: checkpointPrefix,
+    checkpointStatusPath: checkpointStatus,
+    externalObjectPrefixPath: externalPrefix,
+    restoreCheckpointPrefixPath: checkpointPrefix,
+    checkpointPolicy,
+    completionRelativePath,
+    completionHasEvidence,
+    providerRelay,
+    providerRelayPath: remoteProviderRelay,
+    restoreCheckpointOnStart,
+    reuseCompletedResultOnRestore,
+  });
+
+  await assertActive();
+  await assertRemotePath(remoteDirectory);
+  const prepared = await lease.ssh.exec(
+    `rm -rf /job && install -d -m 0711 /job ${shellQuote(remoteDirectory)}`,
+    { timeoutMs: 60_000 },
+  );
+  if (prepared.code !== 0) {
+    throw new CiteArkError(`无法准备 AutoDL 远程目录：${prepared.stderr.trim()}`);
+  }
+  const stageUpload = {
+    finalPath: remoteStage,
+    uploadPath: `${remoteStage}.uploading`,
+    localPath: stageArchive,
+    byteSize: (await lstat(stageArchive)).size,
+    digest: await sha256File(stageArchive),
+  };
+  progress(
+    `AutoDL ${bundle.runId}: 准备第 ${attempt} 次运行输入（${formatMiB(stageUpload.byteSize)} MiB）`,
+  );
+  const resolvedStageTransfer = stageTransfer ?? createAutoDlGcsStageTransfer({
+    config,
+    clients: stageClients,
+  });
+  const stageTransferRecord = await transferAutoDlStageArchive({
+    ssh: lease.ssh,
+    transfer: resolvedStageTransfer,
+    runId: bundle.runId,
+    attempt,
+    stageUpload,
+    progress,
+  });
+  const controlUploads = [
+    textUpload(remoteScript, `${workerScript}\n`, 0o700),
+    ...(providerRelay ? [textUpload(
+      remoteProviderRelay,
+      await readFile(
+        new URL("../../docker/claude-code/codex-pro-relay.mjs", import.meta.url),
+        "utf8",
+      ),
+      0o700,
+    )] : []),
+    ...(managedNetwork ? [
+      textUpload(remoteRouter, await autoDlRoutingProxySource(), 0o700),
+      textUpload(remoteTunnelKey, `${managedNetwork.privateKey}\n`, 0o600),
+      textUpload(remoteKnownHosts, `${managedNetwork.knownHosts}\n`, 0o600),
+      textUpload(
+        remoteNetworkRunner,
+        buildAutoDlNetworkRunnerScript({
+          workerPath: remoteScript,
+          routerPath: remoteRouter,
+          keyPath: remoteTunnelKey,
+          knownHostsPath: remoteKnownHosts,
+          turboScript: config.networkTurboScript,
+          tunnel: managedNetwork,
+          providerBaseUrl: bundle.runtimeAgent?.api?.baseUrl,
+        }),
+        0o700,
+      ),
+    ] : []),
+  ];
+  const controlUploadStartedAt = Date.now();
+  await uploadVerifiedFiles(lease.ssh, controlUploads, {
+    onRetry: (retry, error) => progress(
+      `AutoDL ${bundle.runId}: 控制文件上传连接中断（${safeMessage(error)}），准备第 ${retry} 次重试`,
+    ),
+    onResume: (upload) => progress(
+      `AutoDL ${bundle.runId}: 已确认并复用 ${path.posix.basename(upload.finalPath)}`,
+    ),
+  });
+  progress(
+    `AutoDL ${bundle.runId}: ${controlUploads.length} 个控制文件上传并校验完成（${formatSeconds(
+      (Date.now() - controlUploadStartedAt) / 1000,
+    )}）`,
+  );
+  progress(`AutoDL ${bundle.runId}: 第 ${attempt} 次运行输入准备并校验完成`);
+
+  const startedAt = new Date().toISOString();
+  let checkpointInFlight = false;
+  let lastCheckpointSequence = null;
+  const executionAbort = new AbortController();
+  const checkpointTimer = setInterval(async () => {
+    if (checkpointInFlight) return;
+    checkpointInFlight = true;
+    try {
+      await assertActive();
+      const checkpoint = await readRemoteJson(lease.ssh, checkpointStatus);
+      if (
+        checkpoint?.schemaVersion === "0.1"
+        && typeof checkpoint.sequence === "string"
+        && checkpoint.sequence !== lastCheckpointSequence
+      ) {
+        lastCheckpointSequence = checkpoint.sequence;
+        await onCheckpoint(checkpoint);
+        progress(`AutoDL ${bundle.runId}: 已保存${checkpoint.kind === "full" ? "完整" : "关键"}检查点 ${checkpoint.sequence}`);
+      }
+    } catch (error) {
+      if (error?.processingLeaseLost === true) executionAbort.abort(error);
+      // A checkpoint is optional until the remote worker has created it.
+    } finally {
+      checkpointInFlight = false;
+    }
+  }, Math.max(5_000, checkpointPolicy.criticalIntervalSeconds * 1000));
+  checkpointTimer.unref();
+
+  const liveStream = hasSessionEvents() ? createRuntimeStream({ source: `${bundle.runId}:${attempt}`, runtime: bundle.runtimeAgent.runtime }) : null;
+  let remoteExecution;
+  let environmentUploadStarted = false;
+  try {
+    environmentUploadStarted = true;
+    const environmentUpload = textUpload(
+      remoteEnvironment,
+      shellEnvironmentFile(environment),
+      0o600,
+    );
+    await uploadVerifiedFiles(lease.ssh, [environmentUpload], {
+      onRetry: (retry, error) => progress(
+        `AutoDL ${bundle.runId}: 环境文件上传连接中断（${safeMessage(error)}），准备第 ${retry} 次重试`,
+      ),
+    });
+    await assertActive();
+    progress(`AutoDL ${bundle.runId}: Agent 已在 ${lease.instanceUuid} 启动`);
+    const seconds = Math.ceil(bundle.executionEnvironment.timeoutMinutes * 60);
+    const executionScript = managedNetwork ? remoteNetworkRunner : remoteScript;
+    remoteExecution = await lease.ssh.exec(
+      [
+        "set -a",
+        `. ${shellQuote(remoteEnvironment)}`,
+        "set +a",
+        `timeout --signal=TERM --kill-after=30s ${seconds}s bash ${shellQuote(executionScript)}`,
+      ].join("; "),
+      {
+        timeoutMs: (seconds + 90) * 1000,
+        signal: executionAbort.signal,
+        onStdout: chunk => liveStream?.write(chunk),
+        onStderr: (text) => {
+          if (liveStream) { liveStream.stderr(text); return; }
+          const line = text.replaceAll(/\s+/g, " ").trim();
+          if (line) progress(`AutoDL ${bundle.runId}: ${line.slice(0, 400)}`);
+        },
+      },
+    );
+  } finally {
+    liveStream?.end();
+    clearInterval(checkpointTimer);
+    if (environmentUploadStarted) {
+      await removeRemoteNetworkFiles(lease.ssh, [
+        `${remoteEnvironment}.uploading`,
+        remoteEnvironment,
+        remoteTunnelKey,
+        remoteKnownHosts,
+        remoteNetworkRunner,
+        remoteRouter,
+        remoteProviderRelay,
+      ]);
+    }
+  }
+  const finishedAt = new Date().toISOString();
+  const timedOut = remoteExecution.timedOut || remoteExecution.code === 124;
+  if (!(await lease.ssh.exists(remoteResult))) {
+    const detail = safeRemoteFailureDetail(remoteExecution.stderr);
+    throw new CiteArkError(
+      `AutoDL 远程运行没有返回结果包（退出码 ${remoteExecution.code}${timedOut ? "，已超时" : ""}）${detail ? `：${detail}` : ""}`,
+    );
+  }
+  await lease.ssh.getFile(remoteResult, resultArchive);
+  await extractAutoDlResultArchive(bundle, resultArchive);
+  const externalObjects = await restoreAutoDlExternalObjects({
+    bundle,
+    ssh: lease.ssh,
+    remoteDirectory,
+    maximumExternalBytes: config.maximumExternalBytes,
+    progress,
+  });
+  const durationSeconds = Math.max(
+    0,
+    (Date.parse(finishedAt) - Date.parse(startedAt)) / 1000,
+  );
+  return {
+    record: {
+      attempt,
+      instanceUuid: lease.instanceUuid,
+      state: remoteExecution.code === 0 ? "SUCCEEDED" : timedOut ? "TIMED_OUT" : "FAILED",
+      startedAt,
+      finishedAt,
+      durationSeconds,
+      exitCode: remoteExecution.code,
+      signal: remoteExecution.signal,
+      timedOut,
+      checkpoint: lastCheckpointSequence,
+      stageTransfer: stageTransferRecord,
+    },
+    externalObjects,
+  };
+}
+
+export async function transferAutoDlStageArchive({
+  ssh,
+  transfer,
+  runId,
+  attempt,
+  stageUpload,
+  progress = () => {},
+  now = () => Date.now(),
+}) {
+  let fallbackReason = null;
+  if (transfer) {
+    let published = null;
+    try {
+      const publishStartedAt = now();
+      published = await transfer.publish({
+        runId,
+        attempt,
+        filePath: stageUpload.localPath,
+        digest: stageUpload.digest,
+        byteSize: stageUpload.byteSize,
+      });
+      const publishSeconds = elapsedSeconds(publishStartedAt, now());
+      progress(
+        `AutoDL ${runId}: 输入包已暂存 GCS 并签名（${formatMiB(stageUpload.byteSize)} MiB，${formatSeconds(publishSeconds)}）`,
+      );
+      const downloadStartedAt = now();
+      await downloadVerifiedAutoDlStage({
+        ssh,
+        url: published.url,
+        stageUpload,
+        onRetry: (nextAttempt, detail) => progress(
+          `AutoDL ${runId}: GCS 分块下载中断（${detail}），保留完整分块并进行第 ${nextAttempt} 次续传`,
+        ),
+      });
+      const downloadSeconds = elapsedSeconds(downloadStartedAt, now());
+      const bytesPerSecond = transferRate(stageUpload.byteSize, downloadSeconds);
+      progress(
+        `AutoDL ${runId}: GCS 直连下载并校验完成（${formatSeconds(downloadSeconds)}，${formatRate(bytesPerSecond)}）`,
+      );
+      return {
+        mode: "gcs-signed-url",
+        byteSize: stageUpload.byteSize,
+        publishSeconds,
+        transferSeconds: downloadSeconds,
+        bytesPerSecond,
+        fallbackReason: null,
+      };
+    } catch (error) {
+      fallbackReason = safeMessage(error);
+      progress(
+        `AutoDL ${runId}: GCS 直连输入失败（${fallbackReason}），回退到校验 SFTP`,
+      );
+    } finally {
+      if (published?.remove) {
+        await published.remove().catch((error) => progress(
+          `AutoDL ${runId}: GCS 临时输入清理失败，将由 bucket 生命周期回收（${safeMessage(error)}）`,
+        ));
+      }
+    }
+  }
+
+  const uploadStartedAt = now();
+  await uploadVerifiedFiles(ssh, [stageUpload], {
+    onRetry: (retry, error) => progress(
+      `AutoDL ${runId}: SFTP 输入上传连接中断（${safeMessage(error)}），准备第 ${retry} 次重试`,
+    ),
+    onResume: () => progress(`AutoDL ${runId}: 已确认并复用完整 SFTP 输入包`),
+  });
+  const transferSeconds = elapsedSeconds(uploadStartedAt, now());
+  const bytesPerSecond = transferRate(stageUpload.byteSize, transferSeconds);
+  progress(
+    `AutoDL ${runId}: SFTP 输入上传并校验完成（${formatSeconds(transferSeconds)}，${formatRate(bytesPerSecond)}）`,
+  );
+  return {
+    mode: "sftp",
+    byteSize: stageUpload.byteSize,
+    publishSeconds: null,
+    transferSeconds,
+    bytesPerSecond,
+    fallbackReason,
+  };
+}
+
+export async function downloadVerifiedAutoDlStage({
+  ssh,
+  url,
+  stageUpload,
+  attempts = 3,
+  onRetry = () => {},
+}) {
+  if (typeof url !== "string" || !url.startsWith("https://storage.googleapis.com/")) {
+    throw new TypeError("AutoDL GCS 输入 URL 无效");
+  }
+  if (!/^[a-f0-9]{64}$/.test(String(stageUpload?.digest ?? ""))) {
+    throw new TypeError("AutoDL GCS 输入摘要无效");
+  }
+  if (!Number.isSafeInteger(stageUpload?.byteSize) || stageUpload.byteSize <= 0) {
+    throw new TypeError("AutoDL GCS 输入大小无效");
+  }
+  const maximumSeconds = autoDlStageDownloadTimeoutSeconds(stageUpload.byteSize);
+  const chunkBytes = autoDlStageDownloadChunkBytes(stageUpload.byteSize);
+  const partsDirectory = `${stageUpload.uploadPath}.parts`;
+  const expectedLine = `${stageUpload.digest}  ${stageUpload.uploadPath}`;
+  const downloadScript = [
+    "set -eu",
+    `url=${shellQuote(url)}`,
+    `destination=${shellQuote(stageUpload.uploadPath)}`,
+    `parts_directory=${shellQuote(partsDirectory)}`,
+    `byte_size=${stageUpload.byteSize}`,
+    `chunk_bytes=${chunkBytes}`,
+    "parallelism=4",
+    "rm -f -- \"$destination\"",
+    "mkdir -p -- \"$parts_directory\"",
+    "index=0",
+    "start=0",
+    "while [ \"$start\" -lt \"$byte_size\" ]; do",
+    "  pids=''",
+    "  launched=0",
+    "  while [ \"$launched\" -lt \"$parallelism\" ] && [ \"$start\" -lt \"$byte_size\" ]; do",
+    "    end=$((start + chunk_bytes - 1))",
+    "    if [ \"$end\" -ge \"$byte_size\" ]; then end=$((byte_size - 1)); fi",
+    "    part=$(printf '%08d' \"$index\")",
+    "    part_path=\"$parts_directory/part-$part\"",
+    "    partial_path=\"$part_path.partial\"",
+    "    expected_size=$((end - start + 1))",
+    "    if [ -f \"$part_path\" ] && [ \"$(stat -c '%s' -- \"$part_path\")\" = \"$expected_size\" ]; then",
+    "      index=$((index + 1))",
+    "      start=$((end + 1))",
+    "      launched=$((launched + 1))",
+    "      continue",
+    "    fi",
+    "    rm -f -- \"$part_path\" \"$partial_path\"",
+    "    (",
+    [
+      "      env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u all_proxy",
+      "curl --noproxy '*' --fail --silent --show-error --location",
+      "--retry 3 --retry-all-errors --retry-delay 2 --connect-timeout 20",
+      "--speed-limit 4096 --speed-time 45",
+      `--max-time ${maximumSeconds}`,
+      "--range \"$start-$end\"",
+      "--output \"$partial_path\" \"$url\"",
+    ].join(" "),
+    "      test \"$(stat -c '%s' -- \"$partial_path\")\" = \"$expected_size\"",
+    "      mv -f -- \"$partial_path\" \"$part_path\"",
+    "    ) &",
+    "    pids=\"$pids $!\"",
+    "    index=$((index + 1))",
+    "    start=$((end + 1))",
+    "    launched=$((launched + 1))",
+    "  done",
+    "  failed=0",
+    "  for pid in $pids; do wait \"$pid\" || failed=1; done",
+    "  test \"$failed\" -eq 0",
+    "done",
+    "cat \"$parts_directory\"/part-* > \"$destination\"",
+    `test "$(stat -c '%s' -- \"$destination\")" = ${shellQuote(String(stageUpload.byteSize))}`,
+    `printf '%s\\n' ${shellQuote(expectedLine)} | sha256sum --check --status -`,
+    `mv -f -- \"$destination\" ${shellQuote(stageUpload.finalPath)}`,
+    "rm -rf -- \"$parts_directory\"",
+  ].join("\n");
+  let lastDetail = "未知错误";
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      const result = await ssh.exec(
+        `timeout --signal=TERM --kill-after=15s ${maximumSeconds}s bash -c ${shellQuote(downloadScript)}`,
+        { timeoutMs: (maximumSeconds + 30) * 1000 },
+      );
+      if (result.code === 0) return;
+      lastDetail = safeRemoteFailureDetail(result.stderr) || `退出码 ${result.code}`;
+    } catch (error) {
+      lastDetail = safeMessage(error);
+    }
+    if (attempt < attempts) onRetry(attempt + 1, lastDetail);
+  }
+  await ssh.exec(
+    `rm -rf -- ${shellQuote(stageUpload.uploadPath)} ${shellQuote(partsDirectory)}`,
+    { timeoutMs: 30_000 },
+  ).catch(() => undefined);
+  throw new CiteArkError(`AutoDL 无代理 GCS 下载或身份校验失败：${lastDetail}`);
+}
+
+export function autoDlStageDownloadTimeoutSeconds(byteSize) {
+  const size = Number.isSafeInteger(byteSize) && byteSize > 0 ? byteSize : 1;
+  const minimumBytesPerSecond = 64 * 1024;
+  return Math.min(
+    30 * 60,
+    Math.max(5 * 60, Math.ceil(size / minimumBytesPerSecond) + 2 * 60),
+  );
+}
+
+export function autoDlStageDownloadChunkBytes(byteSize) {
+  const size = Number.isSafeInteger(byteSize) && byteSize > 0 ? byteSize : 1;
+  if (size <= 64 * 1024 ** 2) return 1024 ** 2;
+  if (size <= 512 * 1024 ** 2) return 4 * 1024 ** 2;
+  return 16 * 1024 ** 2;
+}
+
+export async function createAutoDlStageArchive(bundle, destination) {
+  await createRemoteStageArchive({
+    runDirectory: bundle.directories.run,
+    destination,
+    label: "AutoDL",
+  });
+}
+
+export async function extractAutoDlResultArchive(bundle, source) {
+  const result = await captureProcess(
+    "tar",
+    ["-xzf", source, "-C", bundle.directories.run],
+  );
+  if (result.code !== 0) {
+    throw new CiteArkError(`解压 AutoDL 运行结果失败：\n${result.stderr.trim()}`);
+  }
+}
+
+export async function restoreAutoDlExternalObjects({
+  bundle,
+  ssh,
+  remoteDirectory,
+  maximumExternalBytes = 8 * 1024 ** 3,
+  progress = () => {},
+}) {
+  const manifestPath = path.join(bundle.directories.execution, "result-export.json");
+  if (!(await pathExists(manifestPath))) return [];
+  const manifest = await readJson(manifestPath);
+  const descriptors = Array.isArray(manifest.externalObjects)
+    ? manifest.externalObjects
+    : [];
+  const normalized = descriptors.map((descriptor) => {
+    const relative = descriptor?.path;
+    const object = descriptor?.object;
+    const digest = descriptor?.sha256;
+    const size = descriptor?.size;
+    if (
+      typeof relative !== "string"
+      || typeof object !== "string"
+      || path.posix.basename(object) !== object
+      || !/^[a-f0-9]{64}$/.test(String(digest))
+      || !Number.isSafeInteger(size)
+      || size <= 0
+    ) {
+      throw new CiteArkError("AutoDL 发布清单包含无效的大对象描述");
+    }
+    return { relative, object, digest: String(digest), size };
+  });
+  const totalBytes = normalized.reduce((total, descriptor) => total + descriptor.size, 0);
+  if (!Number.isSafeInteger(totalBytes) || totalBytes > maximumExternalBytes) {
+    throw new CiteArkError(
+      `AutoDL 外部证据合计 ${formatMiB(totalBytes)} MiB，超过协调器本地预算 ${formatMiB(maximumExternalBytes)} MiB`,
+    );
+  }
+  const restored = [];
+  for (const descriptor of normalized) {
+    const { relative, object, digest, size } = descriptor;
+    const destination = path.resolve(bundle.directories.run, relative);
+    if (!destination.startsWith(`${path.resolve(bundle.directories.run)}${path.sep}`)) {
+      throw new CiteArkError(`AutoDL 发布大对象路径逃逸：${relative}`);
+    }
+    await mkdir(path.dirname(destination), { recursive: true });
+    await downloadVerifiedExternalObject({
+      ssh,
+      remotePath: `${remoteDirectory}/${object}`,
+      destination,
+      relative,
+      digest,
+      size,
+      progress,
+    });
+    restored.push(object);
+  }
+  return restored;
+}
+
+export function autoDlExternalObjectTransferTimeoutMs(byteSize) {
+  const size = Number.isSafeInteger(byteSize) && byteSize > 0 ? byteSize : 1;
+  const minimumTransferBytesPerSecond = 64 * 1024;
+  const verificationMarginMs = 5 * 60_000;
+  return Math.min(
+    4 * 60 * 60_000,
+    Math.max(
+      30 * 60_000,
+      Math.ceil((size / minimumTransferBytesPerSecond) * 1000)
+        + verificationMarginMs,
+    ),
+  );
+}
+
+async function downloadVerifiedExternalObject({
+  ssh,
+  remotePath,
+  destination,
+  relative,
+  digest,
+  size,
+  progress,
+  attempts = 3,
+}) {
+  const timeoutMs = autoDlExternalObjectTransferTimeoutMs(size);
+  let lastError;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    let lastReportedDecile = -1;
+    try {
+      await rm(destination, { force: true });
+      progress(
+        `AutoDL 下载外部证据 ${relative}（${formatMiB(size)} MiB，第 ${attempt}/${attempts} 次）`,
+      );
+      await ssh.getFile(remotePath, destination, {
+        timeoutMs,
+        onProgress: ({ transferredBytes, totalBytes }) => {
+          const total = Number(totalBytes) > 0 ? Number(totalBytes) : size;
+          const decile = Math.min(
+            10,
+            Math.floor((Number(transferredBytes) / total) * 10),
+          );
+          if (decile <= lastReportedDecile) return;
+          lastReportedDecile = decile;
+          progress(`AutoDL 外部证据 ${relative} 下载 ${decile * 10}%`);
+        },
+      });
+      const [observedDigest, stat] = await Promise.all([
+        sha256File(destination),
+        lstat(destination),
+      ]);
+      if (!stat.isFile() || stat.size !== size || observedDigest !== digest) {
+        throw new CiteArkError(`AutoDL 发布大对象身份校验失败：${relative}`);
+      }
+      progress(`AutoDL 外部证据 ${relative} 下载并校验完成`);
+      return;
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) {
+        progress(
+          `AutoDL 外部证据 ${relative} 下载中断，将重新连接并进行第 ${attempt + 1}/${attempts} 次尝试`,
+        );
+      }
+    }
+  }
+  await rm(destination, { force: true });
+  throw new CiteArkError(
+    `AutoDL 外部证据下载重试后仍失败：${relative}`,
+    { cause: lastError },
+  );
+}
+
+function formatMiB(byteSize) {
+  return (byteSize / (1024 * 1024)).toFixed(1);
+}
+
+function elapsedSeconds(startedAt, finishedAt) {
+  return Math.max(0, (Number(finishedAt) - Number(startedAt)) / 1000);
+}
+
+function transferRate(byteSize, seconds) {
+  return seconds > 0 ? Math.round(byteSize / seconds) : null;
+}
+
+function formatRate(bytesPerSecond) {
+  if (!Number.isFinite(bytesPerSecond) || bytesPerSecond <= 0) return "速度未知";
+  if (bytesPerSecond >= 1024 * 1024) {
+    return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MiB/s`;
+  }
+  return `${(bytesPerSecond / 1024).toFixed(1)} KiB/s`;
+}
+
+function formatSeconds(seconds) {
+  return `${Number(seconds).toFixed(1)} 秒`;
+}
+
+async function readRemoteJson(ssh, remotePath) {
+  const bytes = await ssh.readFile(remotePath, { maximumBytes: 2 * 1024 * 1024 });
+  return JSON.parse(bytes.toString("utf8"));
+}
+
+async function assertRemotePath(value) {
+  if (!/^\/root\/citeark\/[a-z0-9-]{1,80}$/.test(value)) {
+    throw new TypeError(`AutoDL 远程目录无效：${value}`);
+  }
+}
+
+function textUpload(finalPath, content, mode) {
+  const bytes = Buffer.from(content, "utf8");
+  if (!bytes.length) throw new CiteArkError(`AutoDL 上传内容为空：${path.posix.basename(finalPath)}`);
+  return {
+    finalPath,
+    uploadPath: `${finalPath}.uploading`,
+    content,
+    mode,
+    byteSize: bytes.length,
+    digest: createHash("sha256").update(bytes).digest("hex"),
+  };
+}
+
+export async function assertRemoteUploads(ssh, uploads) {
+  if (!uploads.length) return;
+  const paths = uploads.map((upload) => shellQuote(upload.uploadPath));
+  const result = await ssh.exec(
+    [
+      `sha256sum -- ${paths.join(" ")}`,
+      `stat -c '%s %a' -- ${paths.join(" ")}`,
+    ].join(" && "),
+    { timeoutMs: 60_000 },
+  );
+  if (result.code !== 0) {
+    throw new CiteArkError("AutoDL 上传完整性检查无法读取全部暂存文件");
+  }
+  const lines = result.stdout.trim().split(/\r?\n/);
+  if (lines.length !== uploads.length * 2) {
+    throw new CiteArkError("AutoDL 上传完整性检查返回的文件数量不一致");
+  }
+  const digests = new Map(lines.slice(0, uploads.length).map((line) => {
+    const matched = line.match(/^([0-9a-f]{64})\s+(.+)$/);
+    if (!matched) throw new CiteArkError("AutoDL 上传摘要输出无效");
+    return [matched[2], matched[1]];
+  }));
+  const stats = lines.slice(uploads.length).map((line) => {
+    const matched = line.match(/^(\d+) (\d+)$/);
+    if (!matched) {
+      throw new CiteArkError("AutoDL 上传文件状态输出无效");
+    }
+    return { size: Number(matched[1]), mode: matched[2] };
+  });
+  for (const [index, upload] of uploads.entries()) {
+    const remote = stats[index];
+    if (
+      !remote
+      || remote.size !== upload.byteSize
+      || digests.get(upload.uploadPath) !== upload.digest
+      || (upload.mode && remote.mode !== upload.mode.toString(8))
+    ) {
+      throw new CiteArkError(
+        `AutoDL 上传完整性检查失败：${path.posix.basename(upload.finalPath)}`,
+      );
+    }
+  }
+}
+
+export async function uploadVerifiedFiles(ssh, uploads, {
+  attempts = 5,
+  retryDelayMs = 5_000,
+  onRetry = () => {},
+  onResume = () => {},
+} = {}) {
+  let lastError;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      for (const upload of uploads) {
+        if (attempt > 1 && await remoteUploadIsVerified(ssh, upload)) {
+          onResume(upload);
+          continue;
+        }
+        if (upload.localPath) {
+          await ssh.putFile(upload.localPath, upload.uploadPath);
+        } else {
+          await ssh.writeFile(upload.uploadPath, upload.content, { mode: upload.mode });
+        }
+      }
+      await assertRemoteUploads(ssh, uploads);
+      await promoteRemoteUploads(ssh, uploads);
+      return;
+    } catch (error) {
+      lastError = error;
+      if (attempt >= attempts) break;
+      onRetry(attempt + 1, error);
+      if (retryDelayMs > 0) {
+        const backoffMs = Math.min(retryDelayMs * (2 ** (attempt - 1)), 30_000);
+        await new Promise((resolve) => setTimeout(resolve, backoffMs));
+      }
+    }
+  }
+  await removeRemoteNetworkFiles(ssh, [
+    ...uploads.flatMap((upload) => [upload.uploadPath, upload.finalPath]),
+  ]);
+  throw lastError;
+}
+
+async function remoteUploadIsVerified(ssh, upload) {
+  try {
+    await assertRemoteUploads(ssh, [upload]);
+    return true;
+  } catch (error) {
+    if (String(error?.message ?? "").startsWith("AutoDL 上传")) return false;
+    throw error;
+  }
+}
+
+async function promoteRemoteUploads(ssh, uploads) {
+  const command = uploads
+    .map((upload) => `mv -f -- ${shellQuote(upload.uploadPath)} ${shellQuote(upload.finalPath)}`)
+    .join(" && ");
+  const result = await ssh.exec(command, { timeoutMs: 60_000 });
+  if (result.code !== 0) {
+    throw new CiteArkError("AutoDL 已校验上传文件无法提交到运行目录");
+  }
+}
+
+function safeIdentifier(value) {
+  return String(value ?? "run")
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9-]/g, "-")
+    .replaceAll(/^-+|-+$/g, "")
+    .slice(0, 80) || "run";
+}
+
+function safeRemoteFailureDetail(value) {
+  return String(value ?? "")
+    .replaceAll(/(password|token|secret|authorization|private[_ -]?key)\s*[=:]\s*\S+/gi, "$1=<redacted>")
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .at(-1)
+    ?.slice(0, 500) ?? "";
+}
+
+function safeMessage(error) {
+  return safeRemoteFailureDetail(error instanceof Error ? error.message : String(error))
+    || "未知错误";
+}
+
+async function removeRemoteNetworkFiles(ssh, paths) {
+  await ssh.exec(["rm -f", ...paths.map(shellQuote)].join(" "), {
+    timeoutMs: 30_000,
+  }).catch(() => undefined);
+}