@citeark/agent 0.3.20

package/src/compute/autodl-pro-job.mjs

@@ -0,0 +1,843 @@
+import { AutoDlApiError, AutoDlProClient } from "../autodl/client.mjs";
+import { AutoDlSshClient, parseAutoDlSshConnection } from "../autodl/ssh.mjs";
+import { GcpSecretManagerClient } from "../gcp/secret-manager.mjs";
+import { remoteProviderEnvironment } from "./gcp-batch-job.mjs";
+import { CiteArkError, shellQuote } from "../util.mjs";
+
+export function resolveAutoDlProConfig({ bundle, options = {}, requireToken = true }) {
+  const profile = bundle.computeDecision?.selectedProfile?.executorConfig ?? {};
+  const computeProfileId = first(
+    options.computeProfileId,
+    bundle.computeDecision?.selectedProfile?.id,
+  );
+  const apiToken = first(options.apiToken, process.env.AUTODL_API_TOKEN);
+  const legacyImageUuid = first(
+    options.imageUuid,
+    profile.imageUuid,
+    process.env.AUTODL_IMAGE_UUID,
+  );
+  const legacyDataCenterList = stringList(
+    options.dataCenterList
+      ?? profile.dataCenterList
+      ?? commaList(process.env.AUTODL_DATA_CENTER_LIST),
+  );
+  const regionImageCandidates = normalizeRegionImageCandidates(
+    options.regionImageCandidates
+      ?? profile.regionImageCandidates
+      ?? parseAutoDlRegionImagePairs(process.env.AUTODL_REGION_IMAGE_PAIRS),
+    {
+      imageUuid: legacyImageUuid,
+      dataCenterList: legacyDataCenterList,
+    },
+  );
+  const primaryRegionImage = regionImageCandidates?.[0] ?? null;
+  const imageUuid = primaryRegionImage?.imageUuid ?? legacyImageUuid;
+  const dataCenterList = primaryRegionImage?.dataCenterList ?? legacyDataCenterList;
+  const gpuSpecUuid = first(
+    options.forceGpuSpecOverride ? options.gpuSpecUuid : undefined,
+    profile.gpuSpecUuid,
+    options.gpuSpecUuid,
+    process.env.AUTODL_GPU_SPEC_UUID,
+  );
+  const gpuSpecFallbackUuids = uniqueStrings(stringList(
+    (options.forceGpuSpecOverride
+      ? options.gpuSpecFallbackUuids
+      : profile.gpuSpecFallbackUuids ?? options.gpuSpecFallbackUuids)
+      ?? commaList(process.env.AUTODL_GPU_SPEC_FALLBACK_UUIDS),
+  ) ?? []).filter((value) => value !== gpuSpecUuid);
+  const providerSecret = first(
+    options.providerSecret,
+    process.env.CITEARK_GCP_PROVIDER_SECRET,
+  );
+  const liguseSgTunnelSecretResource = first(
+    options.liguseSgTunnelSecretResource,
+    process.env.CITEARK_LIGUSE_SG_TUNNEL_SECRET,
+  );
+  const credentialSource = options.credentialSource ?? profile.credentialSource ?? 'gcp-secret';
+  const issues = [];
+  if (!['gcp-secret', 'task-key'].includes(credentialSource)) issues.push('credentialSource must be gcp-secret or task-key');
+  if (credentialSource === 'task-key' && options.allowRemoteModelKey !== true) issues.push('Sending a model key to an AutoDL task requires --allow-remote-model-key');
+  if (credentialSource === 'task-key' && !bundle.runtimeAgent?.api?.apiKey) issues.push('The selected model configuration has no API key');
+  if (requireToken && !apiToken) issues.push("缺少 AUTODL_API_TOKEN");
+  if (!imageUuid) issues.push("缺少 AutoDL 私有镜像 UUID");
+  if (!gpuSpecUuid) issues.push("缺少 AutoDL GPU 规格 UUID");
+  if (credentialSource === 'gcp-secret' && !providerSecret) issues.push("Provide a Secret Manager reference or explicitly configure task-key credentials");
+  if (issues.length) {
+    throw new TypeError(`AutoDL Pro 执行参数无效：\n- ${issues.join("\n- ")}`);
+  }
+  const hourlyCny = optionalNumber(
+    options.hourlyCny ?? bundle.computeDecision?.selectedProfile?.hourlyCny,
+    "hourlyCny",
+  );
+  const maxHourlyCny = optionalNumber(
+    options.maxHourlyCny ?? profile.maxHourlyCny ?? process.env.AUTODL_MAX_HOURLY_CNY,
+    "maxHourlyCny",
+  );
+  const stagingBucket = first(
+    options.stagingBucket,
+    profile.stagingBucket,
+    process.env.CITEARK_GCP_STAGING_BUCKET,
+  );
+  const stageSigningServiceAccount = first(
+    options.stageSigningServiceAccount,
+    profile.stageSigningServiceAccount,
+    process.env.AUTODL_STAGE_SIGNING_SERVICE_ACCOUNT,
+    process.env.CITEARK_GCP_BATCH_SERVICE_ACCOUNT,
+  );
+  const waitTimeoutMinutes = positiveInteger(
+    options.waitTimeoutMinutes ?? profile.waitTimeoutMinutes,
+    Math.ceil(bundle.executionEnvironment.timeoutMinutes + 30),
+    "waitTimeoutMinutes",
+  );
+  const startupTimeoutMinutes = positiveInteger(
+    options.startupTimeoutMinutes ?? profile.startupTimeoutMinutes,
+    15,
+    "startupTimeoutMinutes",
+  );
+  const startupMaximumTimeoutMinutes = Math.max(
+    startupTimeoutMinutes,
+    positiveInteger(
+      options.startupMaximumTimeoutMinutes
+        ?? profile.startupMaximumTimeoutMinutes
+        ?? process.env.AUTODL_STARTUP_MAXIMUM_TIMEOUT_MINUTES,
+      Math.max(180, startupTimeoutMinutes),
+      "startupMaximumTimeoutMinutes",
+    ),
+  );
+  return {
+    apiToken,
+    apiBaseUrl: first(options.apiBaseUrl, profile.apiBaseUrl) ?? "https://api.autodl.com",
+    imageUuid,
+    regionImageCandidates,
+    computeProfileId,
+    gpuSpecUuid,
+    gpuSpecFallbackUuids,
+    providerSecret,
+    credentialSource,
+    allowRemoteModelKey: options.allowRemoteModelKey === true,
+    liguseSgTunnelSecretResource,
+    networkTurboScript: first(
+      options.networkTurboScript,
+      process.env.AUTODL_NETWORK_TURBO_SCRIPT,
+    ) ?? "/etc/network_turbo",
+    dataCenterList,
+    gpuCount: positiveInteger(options.gpuCount ?? profile.gpuCount, 1, "gpuCount"),
+    cudaVersionFrom: positiveInteger(
+      options.cudaVersionFrom ?? profile.cudaVersionFrom,
+      118,
+      "cudaVersionFrom",
+    ),
+    expandSystemDiskGb: nonNegativeInteger(
+      options.expandSystemDiskGb ?? profile.expandSystemDiskGb,
+      0,
+      "expandSystemDiskGb",
+    ),
+    waitTimeoutMinutes,
+    startupTimeoutMinutes,
+    startupMaximumTimeoutMinutes,
+    remoteInfrastructureRetries: nonNegativeInteger(
+      options.remoteInfrastructureRetries
+        ?? profile.remoteInfrastructureRetries
+        ?? process.env.AUTODL_REMOTE_INFRASTRUCTURE_RETRIES,
+      2,
+      "remoteInfrastructureRetries",
+    ),
+    pollIntervalMs: positiveInteger(
+      options.pollIntervalMs ?? profile.pollIntervalMs,
+      5_000,
+      "pollIntervalMs",
+    ),
+    hourlyUsd: bundle.computeDecision?.selectedProfile?.hourlyUsd ?? null,
+    hourlyCny,
+    maxHourlyCny,
+    stagingBucket,
+    stageSigningServiceAccount,
+    stageSignedUrlExpiresSeconds: positiveInteger(
+      options.stageSignedUrlExpiresSeconds
+        ?? profile.stageSignedUrlExpiresSeconds
+        ?? process.env.AUTODL_STAGE_SIGNED_URL_EXPIRES_SECONDS,
+      30 * 60,
+      "stageSignedUrlExpiresSeconds",
+    ),
+    maximumLifetimeMinutes: positiveInteger(
+      options.maximumLifetimeMinutes
+        ?? profile.maximumLifetimeMinutes
+        ?? process.env.AUTODL_MAXIMUM_LIFETIME_MINUTES,
+      waitTimeoutMinutes + 60,
+      "maximumLifetimeMinutes",
+    ),
+    lifecycleHeartbeatIntervalMs: positiveInteger(
+      options.lifecycleHeartbeatIntervalMs
+        ?? process.env.AUTODL_LIFECYCLE_HEARTBEAT_INTERVAL_MS,
+      2 * 60_000,
+      "lifecycleHeartbeatIntervalMs",
+    ),
+    maximumExternalBytes: positiveInteger(
+      options.maximumExternalBytes
+        ?? profile.maximumExternalBytes
+        ?? process.env.AUTODL_MAXIMUM_EXTERNAL_BYTES,
+      8 * 1024 ** 3,
+      "maximumExternalBytes",
+    ),
+    instanceNamePrefix: first(options.instanceNamePrefix, profile.instanceNamePrefix)
+      ?? "citeark",
+    lifecycle: options.lifecycle ?? null,
+    keepInstance: options.keepInstance ?? profile.keepInstance ?? false,
+    keepFailedInstance: options.keepFailedInstance ?? profile.keepFailedInstance ?? false,
+    checkpointing: {
+      enabled: options.checkpointEnabled !== false && profile.checkpointEnabled !== false,
+      criticalIntervalSeconds: positiveInteger(
+        options.criticalCheckpointIntervalSeconds
+          ?? profile.criticalCheckpointIntervalSeconds,
+        20,
+        "criticalCheckpointIntervalSeconds",
+      ),
+      fullIntervalSeconds: positiveInteger(
+        options.fullCheckpointIntervalSeconds ?? profile.fullCheckpointIntervalSeconds,
+        300,
+        "fullCheckpointIntervalSeconds",
+      ),
+      retention: positiveInteger(
+        options.checkpointRetention ?? profile.checkpointRetention,
+        3,
+        "checkpointRetention",
+      ),
+    },
+  };
+}
+
+export function autoDlInstanceRequest(config, runId) {
+  const maximumLifetimeMinutes = positiveInteger(
+    config.maximumLifetimeMinutes,
+    config.waitTimeoutMinutes + 60,
+    "maximumLifetimeMinutes",
+  );
+  return {
+    dataCenterList: config.dataCenterList,
+    gpuCount: config.gpuCount,
+    expandSystemDiskGb: config.expandSystemDiskGb,
+    gpuSpecUuid: config.gpuSpecUuid,
+    imageUuid: config.imageUuid,
+    cudaVersionFrom: config.cudaVersionFrom,
+    instanceName: `${safeIdentifier(config.instanceNamePrefix)}-${safeIdentifier(runId)}`.slice(0, 80),
+    startCommand: `exec timeout --signal=TERM --kill-after=60s ${maximumLifetimeMinutes}m sleep infinity`,
+  };
+}
+
+export async function acquireAutoDlProLease({
+  config,
+  runId,
+  clients = {},
+  progress = () => {},
+}) {
+  const api = clients.api ?? new AutoDlProClient({
+    token: config.apiToken,
+    baseUrl: config.apiBaseUrl,
+  });
+  let request;
+  let instanceUuid;
+  let instanceCreatedAtMs;
+  let selectedConfig;
+  const instanceCandidates = autoDlInstanceCandidates(config, runId);
+  let lastInventoryError = null;
+  for (const [candidateIndex, candidateConfig] of instanceCandidates.entries()) {
+    const candidateRequest = autoDlInstanceRequest(candidateConfig, runId);
+    try {
+      instanceUuid = await api.createInstance(candidateRequest);
+      instanceCreatedAtMs = Date.now();
+      selectedConfig = candidateConfig;
+      request = candidateRequest;
+      break;
+    } catch (error) {
+      if (!isInventoryError(error)) throw error;
+      lastInventoryError = error;
+      const nextCandidate = instanceCandidates[candidateIndex + 1];
+      if (nextCandidate) {
+        progress(
+          `AutoDL ${candidateConfig.dataCenterList.join("/")} 的 ${candidateConfig.gpuSpecUuid} 暂无库存，尝试 ${nextCandidate.dataCenterList.join("/")} 的 ${nextCandidate.gpuSpecUuid}`,
+        );
+      }
+    }
+  }
+  if (!instanceUuid || !request || !selectedConfig) {
+    throw lastInventoryError ?? new CiteArkError("AutoDL 没有可用的算力规格");
+  }
+  let ssh = null;
+  let snapshot = null;
+  let released = false;
+  try {
+    await invokeLifecycle(selectedConfig.lifecycle, "created", {
+      instanceUuid,
+      runId,
+      request,
+      maximumLifetimeSeconds: selectedConfig.maximumLifetimeMinutes * 60,
+    });
+    progress(`AutoDL ${runId}: 已创建实例 ${instanceUuid}，等待 GPU 节点启动`);
+    await waitForAutoDlStartup({ api, instanceUuid, config: selectedConfig, progress });
+    const deadline = Date.now() + selectedConfig.startupTimeoutMinutes * 60_000;
+    let connection;
+    while (Date.now() < deadline) {
+      snapshot = await api.getInstance(instanceUuid);
+      try {
+        connection = parseAutoDlSshConnection(snapshot);
+        break;
+      } catch {
+        await sleep(selectedConfig.pollIntervalMs);
+      }
+    }
+    if (!connection) throw new CiteArkError(`AutoDL 实例 ${instanceUuid} 未返回 SSH 信息`);
+    const observedHourlyCny = autoDlHourlyCny(snapshot);
+    if (
+      typeof selectedConfig.maxHourlyCny === "number"
+      && typeof observedHourlyCny === "number"
+      && observedHourlyCny > selectedConfig.maxHourlyCny
+    ) {
+      throw new CiteArkError(
+        `AutoDL 实例价格 ¥${observedHourlyCny.toFixed(3)}/时超过预算上限 ¥${selectedConfig.maxHourlyCny.toFixed(3)}/时`,
+      );
+    }
+    ssh = clients.sshFactory
+      ? clients.sshFactory(connection)
+      : new AutoDlSshClient(connection);
+    await retry(
+      () => ssh.connect(),
+      { attempts: 8, delayMs: selectedConfig.pollIntervalMs },
+    );
+    const readiness = await ssh.exec(
+      "command -v bash node python3 jq tar git timeout ssh curl uv sha256sum >/dev/null && id node >/dev/null && test -x /opt/citeark/venv/bin/python && test -r /opt/citeark/runtime-manifest.json",
+      { timeoutMs: 60_000 },
+    );
+    if (readiness.code !== 0) {
+      throw new CiteArkError(
+        "AutoDL 私有镜像缺少 CiteArk Runner 或科研基线依赖：需要 bash、node、python3、jq、tar、git、timeout、ssh、curl、uv、sha256sum、node 用户、/opt/citeark/venv 和运行时清单",
+      );
+    }
+    const lease = new AutoDlProLease({
+      api,
+      ssh,
+      config: selectedConfig,
+      instanceUuid,
+      snapshot,
+      observedHourlyCny,
+      request,
+      runId,
+      progress,
+      createdAtMs: instanceCreatedAtMs,
+    });
+    await lease.startLifecycle();
+    return lease;
+  } catch (error) {
+    ssh?.close();
+    let cleanupError = null;
+    await powerOffAndMaybeRelease({
+      api,
+      instanceUuid,
+      config: selectedConfig,
+      release: !config.keepFailedInstance,
+      progress,
+    }).then((value) => { released = value.released; }).catch((releaseError) => {
+      cleanupError = releaseError;
+    });
+    await invokeLifecycle(selectedConfig.lifecycle, "finished", {
+      instanceUuid,
+      runId,
+      released,
+      error: cleanupError instanceof Error
+        ? cleanupError.message
+        : error instanceof Error ? error.message : String(error),
+    }).catch(() => undefined);
+    if (released) progress(`AutoDL ${instanceUuid}: 启动失败，实例已释放`);
+    throw error;
+  }
+}
+
+/**
+ * Build the complete region × compatible-GPU candidate list. Regions have
+ * equal priority: a stable run-id hash rotates the first region, then each GPU
+ * specification is tried across every region before moving to the next one.
+ */
+export function autoDlInstanceCandidates(config, runId) {
+  const regions = configuredRegionImageCandidates(config);
+  const gpuSpecUuids = uniqueStrings([
+    config.gpuSpecUuid,
+    ...(config.gpuSpecFallbackUuids ?? []),
+  ]);
+  const offset = stableRegionOffset(runId, regions.length);
+  const orderedRegions = regions.map((_, index) =>
+    regions[(index + offset) % regions.length]
+  );
+  return gpuSpecUuids.flatMap((gpuSpecUuid) =>
+    orderedRegions.map((regionImage) => ({
+      ...config,
+      imageUuid: regionImage.imageUuid,
+      dataCenterList: regionImage.dataCenterList,
+      gpuSpecUuid,
+    }))
+  );
+}
+
+function isInventoryError(error) {
+  return error instanceof AutoDlApiError
+    && /库存|stock|capacity/i.test(error.message);
+}
+
+export class AutoDlProLease {
+  constructor({
+    api,
+    ssh,
+    config,
+    instanceUuid,
+    snapshot,
+    observedHourlyCny,
+    request,
+    runId,
+    progress,
+    createdAtMs = Date.now(),
+  }) {
+    this.api = api;
+    this.ssh = ssh;
+    this.config = config;
+    this.instanceUuid = instanceUuid;
+    this.snapshot = snapshot;
+    this.observedHourlyCny = observedHourlyCny;
+    this.request = request;
+    this.runId = runId;
+    this.progress = progress;
+    this.closed = false;
+    this.released = false;
+    this.lifecycleHeartbeat = null;
+    this.createdAtMs = createdAtMs;
+  }
+
+  async startLifecycle() {
+    await this.heartbeatLifecycle();
+    if (!this.config.lifecycle) return;
+    this.lifecycleHeartbeat = setInterval(() => {
+      this.heartbeatLifecycle().catch((error) => {
+        this.progress(
+          `AutoDL ${this.instanceUuid}: 持久化租约心跳失败：${error instanceof Error ? error.message : String(error)}`,
+        );
+      });
+    }, this.config.lifecycleHeartbeatIntervalMs);
+    this.lifecycleHeartbeat.unref?.();
+  }
+
+  async heartbeatLifecycle() {
+    await invokeLifecycle(this.config.lifecycle, "heartbeat", {
+      instanceUuid: this.instanceUuid,
+      runId: this.runId,
+      observedHourlyCny: this.observedHourlyCny,
+    });
+  }
+
+  publicRecord() {
+    return {
+      instanceUuid: this.instanceUuid,
+      region: text(this.snapshot?.region_name) ?? text(this.snapshot?.region_sign),
+      gpu: text(this.snapshot?.snapshot_gpu_alias_name),
+      observedHourlyCny: this.observedHourlyCny,
+      imageUuid: this.config.imageUuid,
+      gpuSpecUuid: this.config.gpuSpecUuid,
+      hostKeySha256: this.ssh.fingerprint ?? null,
+    };
+  }
+
+  async isHealthy() {
+    if (this.closed) return false;
+    try {
+      const status = String(await this.api.getStatus(this.instanceUuid)).trim().toLowerCase();
+      if (status !== "running") return false;
+      const readiness = await this.ssh.exec("true", { timeoutMs: 30_000 });
+      return readiness.code === 0;
+    } catch {
+      return false;
+    }
+  }
+
+  async cleanTargetWorkspace(runId) {
+    if (this.closed) throw new CiteArkError(`AutoDL 实例 ${this.instanceUuid} 已关闭`);
+    const remoteDirectory = `/root/citeark/${safeIdentifier(runId)}`;
+    const cleanup = await this.ssh.exec(
+      `rm -rf -- /job ${shellQuote(remoteDirectory)}`,
+      { timeoutMs: 60_000 },
+    );
+    if (cleanup.code !== 0) {
+      throw new CiteArkError(
+        `AutoDL 目标工作区清理失败：${cleanup.stderr.trim() || `退出码 ${cleanup.code}`}`,
+      );
+    }
+  }
+
+  async close({ successful = false } = {}) {
+    if (this.closed) return { released: this.released };
+    this.closed = true;
+    if (this.lifecycleHeartbeat) clearInterval(this.lifecycleHeartbeat);
+    this.ssh.close();
+    if (this.config.keepInstance) return { released: false };
+    const release = successful || !this.config.keepFailedInstance;
+    try {
+      const cleanup = await powerOffAndMaybeRelease({
+        api: this.api,
+        instanceUuid: this.instanceUuid,
+        config: this.config,
+        release,
+        progress: this.progress,
+      });
+      this.released = cleanup.released;
+      await invokeLifecycle(this.config.lifecycle, "finished", {
+        instanceUuid: this.instanceUuid,
+        runId: this.runId,
+        released: cleanup.released,
+      });
+      return cleanup;
+    } catch (error) {
+      await invokeLifecycle(this.config.lifecycle, "finished", {
+        instanceUuid: this.instanceUuid,
+        runId: this.runId,
+        released: false,
+        error: error instanceof Error ? error.message : String(error),
+      }).catch(() => undefined);
+      throw error;
+    }
+  }
+}
+
+export class AutoDlProLeasePool {
+  constructor({ acquireLease = acquireAutoDlProLease, progress = () => {} } = {}) {
+    this.acquireLease = acquireLease;
+    this.progress = progress;
+    this.lease = null;
+    this.borrowed = false;
+  }
+
+  async acquire({ config, runId, clients, progress = this.progress }) {
+    if (this.borrowed) throw new CiteArkError("AutoDL 共享租约仍被另一个目标占用");
+    if (this.lease) {
+      const compatible = canReuseAutoDlProLease(this.lease, config);
+      const healthy = compatible && await this.lease.isHealthy();
+      if (compatible && healthy) {
+        this.borrowed = true;
+        await this.lease.heartbeatLifecycle();
+        progress(`AutoDL ${runId}: 复用已运行实例 ${this.lease.instanceUuid}`);
+        return this.lease;
+      }
+      progress(
+        compatible
+          ? `AutoDL ${this.lease.instanceUuid}: 健康检查失败，先释放再重建实例`
+          : `AutoDL ${this.lease.instanceUuid}: 与下一目标资源不兼容，先释放再创建实例`,
+      );
+      await this.#closeCurrent();
+    }
+    this.lease = await this.acquireLease({ config, runId, clients, progress });
+    this.borrowed = true;
+    return this.lease;
+  }
+
+  async release(lease, { reusable = true, runId } = {}) {
+    if (lease !== this.lease) {
+      return await lease.close({ successful: true });
+    }
+    try {
+      if (runId) await lease.cleanTargetWorkspace(runId);
+    } catch (error) {
+      reusable = false;
+      this.progress(
+        `AutoDL ${lease.instanceUuid}: 目标隔离清理失败，将释放实例：${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+    this.borrowed = false;
+    if (!reusable) return await this.#closeCurrent();
+    return { released: false };
+  }
+
+  async close() {
+    this.borrowed = false;
+    await this.#closeCurrent();
+  }
+
+  async #closeCurrent() {
+    const lease = this.lease;
+    this.lease = null;
+    if (lease) return await lease.close({ successful: true });
+    return { released: false };
+  }
+}
+
+export function canReuseAutoDlProLease(lease, config, { now = Date.now() } = {}) {
+  if (!lease || lease.closed) return false;
+  const current = lease.config ?? {};
+  const allowedGpuSpecs = new Set([
+    config.gpuSpecUuid,
+    ...(config.gpuSpecFallbackUuids ?? []),
+  ]);
+  const requiredLifetimeMs = (config.waitTimeoutMinutes + 5) * 60_000;
+  const maximumLifetimeMs = current.maximumLifetimeMinutes * 60_000;
+  const ageMs = Math.max(0, now - lease.createdAtMs);
+  const regionImageCompatible = configuredRegionImageCandidates(config).some(
+    (candidate) =>
+      candidate.imageUuid === current.imageUuid
+      && sameStringSet(candidate.dataCenterList, current.dataCenterList),
+  );
+  return current.computeProfileId === config.computeProfileId
+    && current.apiBaseUrl === config.apiBaseUrl
+    && regionImageCompatible
+    && allowedGpuSpecs.has(current.gpuSpecUuid)
+    && current.gpuCount === config.gpuCount
+    && current.cudaVersionFrom === config.cudaVersionFrom
+    && current.expandSystemDiskGb === config.expandSystemDiskGb
+    && current.providerSecret === config.providerSecret
+    && current.liguseSgTunnelSecretResource === config.liguseSgTunnelSecretResource
+    && (typeof config.maxHourlyCny !== "number"
+      || typeof lease.observedHourlyCny !== "number"
+      || lease.observedHourlyCny <= config.maxHourlyCny)
+    && ageMs + requiredLifetimeMs < maximumLifetimeMs;
+}
+
+async function waitForAutoDlStartup({ api, instanceUuid, config, progress }) {
+  const softTimeoutMs = config.startupTimeoutMinutes * 60_000;
+  const maximumTimeoutMs = config.startupMaximumTimeoutMinutes * 60_000;
+  const wait = async (timeoutMs) => await api.waitForStatus(instanceUuid, "running", {
+    timeoutMs,
+    pollIntervalMs: config.pollIntervalMs,
+    onStatus: (status) => progress(`AutoDL ${instanceUuid}: ${status}`),
+  });
+  try {
+    await wait(softTimeoutMs);
+    return;
+  } catch (error) {
+    if (
+      !(error instanceof AutoDlApiError)
+      || !error.retryable
+      || !/等待.*超时|wait.*timeout/i.test(error.message)
+      || maximumTimeoutMs <= softTimeoutMs
+    ) {
+      throw error;
+    }
+    const status = String(await api.getStatus(instanceUuid)).trim().toLowerCase();
+    if (status === "running") return;
+    if (!new Set(["creating", "sys_volume_creating", "starting", "pending"]).has(status)) {
+      throw error;
+    }
+    progress(
+      `AutoDL ${instanceUuid}: ${config.startupTimeoutMinutes} 分钟后仍处于 ${status}，镜像或系统盘仍在准备；继续等待，硬上限 ${config.startupMaximumTimeoutMinutes} 分钟`,
+    );
+    await wait(maximumTimeoutMs - softTimeoutMs);
+  }
+}
+
+async function invokeLifecycle(lifecycle, event, payload) {
+  const callback = lifecycle?.[event];
+  if (typeof callback === "function") await callback(payload);
+}
+
+export async function resolveAutoDlProviderEnvironment({
+  agent,
+  providerSecret,
+  secretManager,
+  credentialSource = 'gcp-secret',
+  allowRemoteModelKey = false,
+}) {
+  if (credentialSource === 'task-key' && !allowRemoteModelKey) throw Error('Remote model credential transfer was not authorized for this task.');
+  const descriptor = remoteProviderEnvironment(agent, providerSecret);
+  const resolver = secretManager ?? new GcpSecretManagerClient();
+  const environment = { ...descriptor.variables };
+  const credentialName = environment.CITEARK_PROVIDER_SECRET_ENV_NAME;
+  if (!/^(ANTHROPIC_API_KEY|OPENAI_API_KEY|DEEPSEEK_API_KEY|CITEARK_API_KEY)$/.test(
+    String(credentialName ?? ""),
+  )) {
+    throw new CiteArkError("AutoDL 模型密钥环境变量名无效");
+  }
+  if (credentialSource === 'task-key') {
+    if (!agent.api?.apiKey) throw Error('The selected model configuration has no API key');
+    environment[credentialName] = agent.api.apiKey;
+    delete environment.CITEARK_PROVIDER_SECRET_RESOURCE;
+  } else environment[credentialName] = await resolver.accessVersion(providerSecret);
+  for (const [name, resource] of Object.entries(descriptor.secretVariables ?? {})) {
+    environment[name] = await resolver.accessVersion(resource);
+  }
+  return environment;
+}
+
+export function shellEnvironmentFile(environment) {
+  const lines = ["# Generated for one AutoDL lease; never persist or publish this file."];
+  for (const [name, value] of Object.entries(environment)) {
+    if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) {
+      throw new TypeError(`AutoDL 环境变量名无效：${name}`);
+    }
+    lines.push(`export ${name}=${shellQuote(String(value))}`);
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+export function autoDlHourlyCny(snapshot) {
+  const raw = Number(snapshot?.payg_price);
+  if (!Number.isFinite(raw) || raw < 0) return null;
+  return Number.isInteger(raw) ? raw / 1000 : raw;
+}
+
+async function powerOffAndMaybeRelease({ api, instanceUuid, config, release, progress }) {
+  await api.powerOff(instanceUuid).catch((error) => {
+    if (!/stopp|关机|released|释放/i.test(String(error?.message ?? ""))) throw error;
+  });
+  await api.waitForStatus(instanceUuid, ["stopped", "shutdown", "shutoff"], {
+    timeoutMs: 5 * 60_000,
+    pollIntervalMs: config.pollIntervalMs,
+    onStatus: (status) => progress(`AutoDL ${instanceUuid}: ${status}`),
+  }).catch(() => undefined);
+  if (!release) {
+    progress(`AutoDL ${instanceUuid}: 已关机并保留失败现场`);
+    return { released: false };
+  }
+  await api.release(instanceUuid);
+  progress(`AutoDL ${instanceUuid}: 已关机并释放`);
+  return { released: true };
+}
+
+async function retry(operation, { attempts, delayMs }) {
+  let lastError;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await operation();
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) await sleep(delayMs);
+    }
+  }
+  throw lastError;
+}
+
+function safeIdentifier(value) {
+  return String(value ?? "run")
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9-]/g, "-")
+    .replaceAll(/^-+|-+$/g, "")
+    .slice(0, 60) || "run";
+}
+
+function first(...values) {
+  return values.find((value) => typeof value === "string" && value.trim())?.trim() ?? null;
+}
+
+function stringList(value) {
+  if (value === undefined || value === null || value.length === 0) return undefined;
+  if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim())) {
+    throw new TypeError("AutoDL dataCenterList 必须是字符串数组");
+  }
+  return value.map((item) => item.trim());
+}
+
+function uniqueStrings(values) {
+  return [...new Set(values)];
+}
+
+function sameStringSet(left, right) {
+  const a = [...new Set(left ?? [])].sort();
+  const b = [...new Set(right ?? [])].sort();
+  return a.length === b.length && a.every((value, index) => value === b[index]);
+}
+
+function commaList(value) {
+  return typeof value === "string"
+    ? value.split(/[;,]/).map((item) => item.trim()).filter(Boolean)
+    : undefined;
+}
+
+export function parseAutoDlRegionImagePairs(value) {
+  if (value === undefined || value === null || value === "") return undefined;
+  if (typeof value !== "string") {
+    throw new TypeError("AUTODL_REGION_IMAGE_PAIRS 必须是分号分隔的 地区=镜像UUID 列表");
+  }
+  return value.split(";").map((entry) => {
+    const separator = entry.indexOf("=");
+    const dataCenter = separator >= 0 ? entry.slice(0, separator).trim() : "";
+    const imageUuid = separator >= 0 ? entry.slice(separator + 1).trim() : "";
+    if (!dataCenter || !imageUuid) {
+      throw new TypeError(
+        "AUTODL_REGION_IMAGE_PAIRS 必须使用 地区=镜像UUID;地区=镜像UUID 格式",
+      );
+    }
+    return { dataCenterList: [dataCenter], imageUuid };
+  });
+}
+
+function normalizeRegionImageCandidates(value, fallback) {
+  if (value === undefined || value === null) {
+    return fallback.imageUuid
+      ? [{
+          dataCenterList: fallback.dataCenterList,
+          imageUuid: fallback.imageUuid,
+        }]
+      : undefined;
+  }
+  if (!Array.isArray(value) || value.length === 0) {
+    throw new TypeError("AutoDL regionImageCandidates 必须是非空数组");
+  }
+  const normalized = value.map((candidate) => {
+    if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) {
+      throw new TypeError("AutoDL regionImageCandidates 项无效");
+    }
+    const imageUuid = first(candidate.imageUuid);
+    const dataCenterList = stringList(
+      candidate.dataCenterList
+        ?? (typeof candidate.dataCenter === "string" ? [candidate.dataCenter] : undefined),
+    );
+    if (!imageUuid || !dataCenterList?.length) {
+      throw new TypeError("AutoDL 地区镜像项必须同时包含地区和镜像 UUID");
+    }
+    return { dataCenterList, imageUuid };
+  });
+  const regions = normalized.flatMap((candidate) => candidate.dataCenterList);
+  if (new Set(regions).size !== regions.length) {
+    throw new TypeError("AutoDL 地区镜像列表不能重复地区");
+  }
+  return normalized;
+}
+
+function configuredRegionImageCandidates(config) {
+  return config.regionImageCandidates?.length
+    ? config.regionImageCandidates
+    : [{ dataCenterList: config.dataCenterList, imageUuid: config.imageUuid }];
+}
+
+function stableRegionOffset(runId, regionCount) {
+  if (regionCount <= 1) return 0;
+  let hash = 2_166_136_261;
+  for (const character of String(runId ?? "")) {
+    hash ^= character.codePointAt(0);
+    hash = Math.imul(hash, 16_777_619);
+  }
+  return (hash >>> 0) % regionCount;
+}
+
+function positiveInteger(value, fallback, field) {
+  const numeric = value === undefined || value === null || value === "" ? fallback : Number(value);
+  if (!Number.isInteger(numeric) || numeric <= 0) {
+    throw new TypeError(`AutoDL ${field} 必须是正整数`);
+  }
+  return numeric;
+}
+
+function nonNegativeInteger(value, fallback, field) {
+  const numeric = value === undefined || value === null || value === "" ? fallback : Number(value);
+  if (!Number.isInteger(numeric) || numeric < 0) {
+    throw new TypeError(`AutoDL ${field} 必须是非负整数`);
+  }
+  return numeric;
+}
+
+function optionalNumber(value, field) {
+  if (value === undefined || value === null || value === "") return null;
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric) || numeric < 0) {
+    throw new TypeError(`AutoDL ${field} 必须是非负数`);
+  }
+  return numeric;
+}
+
+function text(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function sleep(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}