@citeark/agent 0.3.20

package/src/assets/lifecycle.mjs

@@ -0,0 +1,213 @@
+import { cp, mkdir } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { buildRunnerAudit } from "../execution/runner-audit.mjs";
+import { ensureFinalWorkspaceCapture, updateRunState } from "../job.mjs";
+import { readAgentUsage } from "../telemetry/usage.mjs";
+import { CiteArkError, pathExists, readJson, writeJson } from "../util.mjs";
+import { defineAgentWorkload } from "../workloads/definition.mjs";
+import { validateAssetPreparationManifest } from "./manifest.mjs";
+import { initialAssetPreparationPrompt, resumeAssetPreparationPrompt } from "./prompts.mjs";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+
+export function createAssetPreparationWorkload(plan) {
+  return defineAgentWorkload({
+    workload: "asset-preparation",
+    agentBoundary: {
+      mode: "bounded",
+      decisionRights: [
+        "select_files_within_the_immutable_asset_plan",
+        "install_or_replace_task_local_dependencies",
+        "resume_and_verify_declared_public_transports",
+        "debug_asset_loaders_and_environment_checks",
+      ],
+      immutableConstraints: [
+        "asset_identity_and_required_file_scope",
+        "no_scientific_experiment_execution",
+        "allocated_compute_time_and_storage_ceiling",
+        "manifest_and_audit_protocol",
+      ],
+    },
+    validateScientificComputeDecision: false,
+    controllerValidationRetries: 0,
+    completionRelativePath: "output/asset-preparation-manifest.json",
+    completionHasEvidence: false,
+    completionValidatorRelativePath: "input/validate-asset-preparation.py",
+    completionSuccessJq: '.kind == "citeark.asset-preparation-manifest" and (.manifestDigest | startswith("sha256:"))',
+    preserveRebuildableCache: true,
+    finalCheckpointKind: "full",
+    finalCheckpointReason: "asset-preparation",
+    recoverFinalMessage: false,
+    async prepare(bundle) {
+      await writeJson(path.join(bundle.directories.input, "asset-preparation-plan.json"), plan);
+      await Promise.all([
+        cp(
+          path.join(PACKAGE_ROOT, "scripts", "validate-asset-preparation.py"),
+          path.join(bundle.directories.input, "validate-asset-preparation.py"),
+        ),
+        cp(
+          path.join(PACKAGE_ROOT, "scripts", "deterministic-asset-preparation.py"),
+          path.join(bundle.directories.input, "deterministic-asset-preparation.py"),
+        ),
+        cp(
+          path.join(PACKAGE_ROOT, "scripts", "run-asset-preparation.sh"),
+          path.join(bundle.directories.input, "run-asset-preparation.sh"),
+        ),
+      ]);
+      await updateRunState(bundle, {
+        workload: "asset-preparation",
+        assetPreparation: { planDigest: plan.planDigest, status: "prepared" },
+      });
+    },
+    wrapInvocation({ invocation }) {
+      return {
+        command: "/bin/bash",
+        args: [
+          "/job/input/run-asset-preparation.sh",
+          "/job/input/deterministic-asset-preparation.py",
+          invocation.command,
+          ...(Array.isArray(invocation.args) ? invocation.args : []),
+        ],
+        prompt: invocation.prompt,
+        promptArgument: invocation.promptArgument,
+      };
+    },
+    instructions: async () => [
+      "You are CiteArk's dedicated asset-preparation Agent.",
+      "Only prepare and verify the immutable asset plan. Do not execute the full scientific experiment and do not change its scope.",
+      "Preserve complete operational evidence. The runner independently validates the manifest before it can be reused.",
+    ].join("\n"),
+    prompt({ bundle, attempt, validation }) {
+      if (attempt === 1) return initialAssetPreparationPrompt(bundle);
+      return [
+        resumeAssetPreparationPrompt(bundle),
+        "The previous manifest failed deterministic validation:",
+        ...validation.issues.map((issue) => `- ${issue}`),
+        "Repair only the failed acquisition, environment or evidence records; retain valid downloads and logs.",
+      ].join("\n");
+    },
+    resumePrompt({ bundle }) {
+      return resumeAssetPreparationPrompt(bundle);
+    },
+    async validate({ bundle, completionPath }) {
+      if (!(await pathExists(completionPath))) {
+        return { result: null, issues: ["缺少资产准备清单"], protocolVerification: null };
+      }
+      try {
+        const manifest = await readJson(completionPath);
+        return {
+          result: manifest,
+          issues: validateAssetPreparationManifest(plan, manifest),
+          protocolVerification: null,
+        };
+      } catch (error) {
+        return {
+          result: null,
+          issues: [error instanceof Error ? error.message : String(error)],
+          protocolVerification: null,
+        };
+      }
+    },
+    isSucceeded(result, issues) {
+      return issues.length === 0 && result?.kind === "citeark.asset-preparation-manifest";
+    },
+    async finalize({
+      bundle,
+      validation,
+      attempts,
+      timedOut,
+      sandboxImageIdentity,
+      cloudExecution,
+      checkpoint,
+    }) {
+      await ensureFinalWorkspaceCapture(bundle);
+      const usage = await readAgentUsage(path.join(bundle.directories.execution, "trace.jsonl"), {
+        runtimeHome: bundle.directories.runtimeHome,
+        model: bundle.runtimeAgent.model,
+      });
+      const valid = validation.issues.length === 0 && !timedOut;
+      await updateRunState(bundle, {
+        status: valid ? "completed" : timedOut ? "timed_out" : "invalid_result",
+        finishedAt: new Date().toISOString(),
+        attempts,
+        validationIssues: validation.issues,
+        sandboxImageIdentity,
+        cloudExecution,
+        usage,
+        assetPreparation: {
+          planDigest: plan.planDigest,
+          manifestDigest: validation.result?.manifestDigest ?? null,
+          status: valid ? "completed" : "failed",
+          checkpoint,
+        },
+      });
+      const runnerAudit = await buildRunnerAudit({
+        runDirectory: bundle.directories.run,
+        runtime: bundle.runtimeAgent.runtime,
+        sandbox: "gcp-batch",
+      });
+      if (!valid) {
+        const error = new CiteArkError(`资产准备未通过确定性校验：\n- ${validation.issues.join("\n- ")}`, {
+          exitCode: 2,
+          retryable: !timedOut,
+          failureCode: timedOut
+            ? "asset_preparation.runtime_budget_exhausted"
+            : "asset_preparation.manifest_invalid",
+          failureCategory: timedOut ? "timeout" : "preflight",
+        });
+        if (checkpoint) error.recoveryCheckpoint = checkpoint;
+        throw error;
+      }
+      if (!checkpoint || checkpoint.kind !== "full") {
+        throw new CiteArkError("资产准备完成但没有留下可供 GPU 复用的完整检查点", {
+          retryable: true,
+          failureCode: "asset_preparation.checkpoint_missing",
+          failureCategory: "publication",
+        });
+      }
+      return {
+        dryRun: false,
+        bundle,
+        manifest: validation.result,
+        validation,
+        attempts,
+        runnerAudit,
+        checkpoint,
+        preparationMode: validation.result?.preparationMode === "deterministic"
+          ? "deterministic"
+          : "agent",
+      };
+    },
+  });
+}
+
+export async function recordAssetCacheHit({ runsDirectory, runId, plan, manifest, cache }) {
+  const runDirectory = path.resolve(runsDirectory, runId);
+  await Promise.all([
+    mkdir(path.join(runDirectory, "input"), { recursive: true }),
+    mkdir(path.join(runDirectory, "output"), { recursive: true }),
+    mkdir(path.join(runDirectory, "execution"), { recursive: true }),
+  ]);
+  await Promise.all([
+    writeJson(path.join(runDirectory, "input", "asset-preparation-plan.json"), plan),
+    writeJson(path.join(runDirectory, "output", "asset-preparation-manifest.json"), manifest),
+    writeJson(path.join(runDirectory, "execution", "asset-cache.json"), cache),
+    writeJson(path.join(runDirectory, "run.json"), {
+      schemaVersion: "0.1",
+      runId,
+      workload: "asset-preparation",
+      status: "completed",
+      cacheHit: true,
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+      assetPreparation: {
+        planDigest: plan.planDigest,
+        manifestDigest: manifest.manifestDigest,
+        checkpoint: cache.checkpoint,
+      },
+    }),
+  ]);
+  return { run: runDirectory };
+}

package/src/assets/manifest.mjs

@@ -0,0 +1,242 @@
+import { sha256Value } from "../util.mjs";
+import { validateSelectedSource } from "./source-policy.mjs";
+
+export function validateAssetPreparationManifest(plan, manifest) {
+  const issues = [];
+  if (manifest?.schemaVersion !== "0.1") issues.push("manifest.schemaVersion 必须是 0.1");
+  if (manifest?.kind !== "citeark.asset-preparation-manifest") {
+    issues.push("manifest.kind 无效");
+  }
+  if (manifest?.planDigest !== plan?.planDigest) issues.push("manifest 与资产计划摘要不一致");
+  const records = Array.isArray(manifest?.assets) ? manifest.assets : [];
+  const byId = new Map(records.map((record) => [record?.assetId, record]));
+  if (byId.size !== records.length) issues.push("资产清单包含重复 assetId");
+  const plannedAssetIds = new Set((plan?.assets ?? []).map((asset) => asset.id));
+  for (const record of records) {
+    if (!plannedAssetIds.has(record?.assetId)) issues.push(`资产清单包含计划外记录：${record?.assetId ?? "unknown"}`);
+  }
+  for (const asset of Array.isArray(plan?.assets) ? plan.assets : []) {
+    const record = byId.get(asset.id);
+    if (!record) {
+      issues.push(`缺少资产准备记录：${asset.id}`);
+      continue;
+    }
+    if (record.localPath !== asset.localPath) {
+      issues.push(`资产 ${asset.id} 的本地绑定路径发生变化`);
+    }
+    validateAssetIdentity(asset, record, issues);
+    const accepted = asset.decision.action === "metadata"
+      ? new Set(["metadata_verified", "prepared"])
+      : asset.decision.action === "stream"
+        ? new Set(["stream_ready", "prepared"])
+        : asset.decision.action === "inspect" && asset.supportsStreaming
+          ? new Set(["prepared", "stream_ready"])
+        : asset.decision.action === "blocked"
+          ? new Set(["blocked"])
+          : new Set(["prepared"]);
+    if (!accepted.has(record.status)) {
+      issues.push(`资产 ${asset.id} 状态 ${record.status ?? "missing"} 不满足计划 ${asset.decision.action}`);
+    }
+    if (new Set(["prepared", "stream_ready"]).has(record.status)) {
+      validateAcquisitionRecord(asset, record.acquisition, issues, record.sourceLocator);
+      validatePreparedRecord(asset, record, issues);
+    }
+  }
+  const downloadedBytes = records.reduce(
+    (sum, record) => sum + (
+      Number.isSafeInteger(record?.acquisition?.downloadBytes)
+        ? record.acquisition.downloadBytes
+        : 0
+    ),
+    0,
+  );
+  if (downloadedBytes > (plan?.limits?.totalDownloadBytes ?? Number.POSITIVE_INFINITY)) {
+    issues.push("资产准备总下载字节数超过任务批准上限");
+  }
+  const preparedBytes = records.reduce(
+    (sum, record) => sum + (Number.isSafeInteger(record?.totalBytes) ? record.totalBytes : 0),
+    0,
+  );
+  if (preparedBytes > (plan?.limits?.totalExpandedBytes ?? Number.POSITIVE_INFINITY)) {
+    issues.push("资产准备总落盘字节数超过任务批准上限");
+  }
+  const checks = Array.isArray(manifest?.checks) ? manifest.checks : [];
+  const checksById = new Map(checks.map((record) => [record?.checkId, record]));
+  if (checksById.size !== checks.length) issues.push("资产清单包含重复 checkId");
+  const plannedCheckIds = new Set((plan?.checks ?? []).map((check) => check.id));
+  for (const record of checks) {
+    if (!plannedCheckIds.has(record?.checkId)) issues.push(`资产清单包含计划外检查：${record?.checkId ?? "unknown"}`);
+  }
+  for (const check of Array.isArray(plan?.checks) ? plan.checks : []) {
+    const record = checksById.get(check.id);
+    if (!record) {
+      issues.push(`缺少资产准备检查记录：${check.id}`);
+      continue;
+    }
+    if (record.kind !== check.kind) issues.push(`检查 ${check.id} 的类型发生变化`);
+    if (record.status !== "passed") issues.push(`检查 ${check.id} 未通过真实预检`);
+    if (!Array.isArray(record.commands) || record.commands.length === 0) {
+      issues.push(`检查 ${check.id} 没有记录实际命令`);
+    }
+    if (!Array.isArray(record.evidence) || record.evidence.length === 0) {
+      issues.push(`检查 ${check.id} 没有可核验输出`);
+    } else {
+      for (const evidence of record.evidence) validateEvidenceDescriptor(check.id, evidence, issues);
+    }
+  }
+  validateRuntimeEnvironment(manifest?.runtimeEnvironment, issues);
+  const body = { ...manifest };
+  delete body.manifestDigest;
+  const expected = `sha256:${sha256Value(body)}`;
+  if (manifest?.manifestDigest !== expected) issues.push("manifestDigest 与清单内容不一致");
+  return issues;
+}
+
+function validateAssetIdentity(asset, record, issues) {
+  if (record.identity !== asset.identity) issues.push(`资产 ${asset.id} 的科学身份不一致`);
+  if (asset.revision && record.revision !== asset.revision) {
+    issues.push(`资产 ${asset.id} 没有固定到计划 revision`);
+  }
+  const sources = new Set((asset.sourceLocators ?? []).map((item) => item.url));
+  if (sources.size && !sources.has(record.sourceLocator)) {
+    issues.push(`资产 ${asset.id} 的来源不在计划允许列表中`);
+  }
+  const verification = record.verification;
+  if (verification?.status !== "passed") {
+    issues.push(`资产 ${asset.id} 没有通过身份校验`);
+    return;
+  }
+  if (!Array.isArray(verification.commands) || verification.commands.length === 0) {
+    issues.push(`资产 ${asset.id} 没有记录身份校验命令`);
+  }
+  if (!Array.isArray(verification.evidence) || verification.evidence.length === 0) {
+    issues.push(`资产 ${asset.id} 没有身份校验证据`);
+  } else {
+    for (const evidence of verification.evidence) {
+      validateEvidenceDescriptor(`asset:${asset.id}`, evidence, issues);
+    }
+  }
+}
+
+function validateEvidenceDescriptor(checkId, value, issues) {
+  if (!safeRelativePath(String(value?.path ?? ""))) {
+    issues.push(`检查 ${checkId} 包含不安全证据路径`);
+  }
+  if (!Number.isSafeInteger(value?.bytes) || value.bytes < 0) {
+    issues.push(`检查 ${checkId} 的证据大小无效`);
+  }
+  if (!/^sha256:[0-9a-f]{64}$/.test(String(value?.digest))) {
+    issues.push(`检查 ${checkId} 的证据摘要无效`);
+  }
+}
+
+function validateAcquisitionRecord(asset, value, issues, sourceLocator) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    issues.push(`资产 ${asset.id} 缺少传输审计记录`);
+    return;
+  }
+  if (!Number.isSafeInteger(value.downloadBytes) || value.downloadBytes < 0) {
+    issues.push(`资产 ${asset.id} 的实际下载字节数无效`);
+  } else if (value.downloadBytes > asset.maximumDownloadBytes) {
+    issues.push(`资产 ${asset.id} 超过批准的下载上限`);
+  }
+  if (!Array.isArray(value.commands) || value.commands.length === 0) {
+    issues.push(`资产 ${asset.id} 没有记录实际传输命令`);
+  }
+  if (!Array.isArray(value.evidence) || value.evidence.length === 0) {
+    issues.push(`资产 ${asset.id} 没有传输证据`);
+  } else {
+    for (const evidence of value.evidence) {
+      validateEvidenceDescriptor(`acquisition:${asset.id}`, evidence, issues);
+    }
+  }
+  issues.push(...validateSelectedSource(asset, {
+    ...value,
+    sourceLocator,
+  }));
+}
+
+function validateRuntimeEnvironment(value, issues) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    issues.push("manifest 缺少任务隔离运行环境记录");
+    return;
+  }
+  if (value.path !== "/job/runtime-venv") {
+    issues.push("任务依赖环境必须绑定到 /job/runtime-venv");
+  }
+  if (typeof value.pythonVersion !== "string" || !value.pythonVersion.trim()) {
+    issues.push("任务依赖环境缺少 Python 版本");
+  }
+  if (!/^sha256:[0-9a-f]{64}$/.test(String(value.lockDigest))) {
+    issues.push("任务依赖环境缺少有效锁定摘要");
+  }
+  if (!/^sha256:[0-9a-f]{64}$/.test(String(value.packageInventoryDigest))) {
+    issues.push("任务依赖环境缺少包清单摘要");
+  }
+}
+
+export function finalizeAssetPreparationManifest(value) {
+  const body = {
+    schemaVersion: "0.1",
+    kind: "citeark.asset-preparation-manifest",
+    ...value,
+  };
+  delete body.manifestDigest;
+  return { ...body, manifestDigest: `sha256:${sha256Value(body)}` };
+}
+
+function validatePreparedRecord(asset, record, issues) {
+  if (!Array.isArray(record.files) || record.files.length === 0) {
+    issues.push(`已准备资产 ${asset.id} 没有文件清单`);
+    return;
+  }
+  let totalBytes = 0;
+  const paths = new Set();
+  for (const file of record.files) {
+    if (typeof file?.path !== "string" || !safeRelativePath(file.path)) {
+      issues.push(`资产 ${asset.id} 包含不安全文件路径`);
+    } else if (paths.has(file.path)) {
+      issues.push(`资产 ${asset.id} 文件路径重复：${file.path}`);
+    } else {
+      paths.add(file.path);
+    }
+    if (!Number.isSafeInteger(file?.bytes) || file.bytes < 0) {
+      issues.push(`资产 ${asset.id} 文件大小无效：${file?.path ?? "unknown"}`);
+    } else {
+      totalBytes += file.bytes;
+    }
+    if (!/^sha256:[0-9a-f]{64}$/.test(String(file?.digest))) {
+      issues.push(`资产 ${asset.id} 文件摘要无效：${file?.path ?? "unknown"}`);
+    }
+  }
+  if (record.totalBytes !== totalBytes) issues.push(`资产 ${asset.id} 总字节数与文件清单不一致`);
+  if (totalBytes > asset.maximumExpandedBytes) issues.push(`资产 ${asset.id} 超过批准的落盘上限`);
+  // requiredPaths is a backward-compatible prefetch selection hint. Older
+  // Research Plans may list alternative checkpoint formats as separate
+  // patterns (for example safetensors and pytorch_model.bin). A cache manifest
+  // records what was actually fetched; scientific format sufficiency is owned
+  // by the complete reproduction Agent and later evidence validation.
+  for (const pattern of asset.excludedPaths ?? []) {
+    if ([...paths].some((candidate) => globMatches(pattern, candidate))) {
+      issues.push(`资产 ${asset.id} 包含排除文件模式：${pattern}`);
+    }
+  }
+}
+
+function globMatches(pattern, value) {
+  const source = pattern
+    .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+    .replaceAll("**", "\u0000")
+    .replaceAll("*", "[^/]*")
+    .replaceAll("?", "[^/]")
+    .replaceAll("\u0000", ".*");
+  return new RegExp(`^${source}$`).test(value);
+}
+
+function safeRelativePath(value) {
+  return value.length > 0
+    && !value.startsWith("/")
+    && !value.includes("\\")
+    && !value.split("/").includes("..")
+    && !/[\u0000-\u001f]/.test(value);
+}

package/src/assets/opportunistic-preparation.mjs

@@ -0,0 +1,81 @@
+import { assetPreparationPlanFailures } from "./plan.mjs";
+
+/**
+ * Decide whether the cache optimizer is worth starting. This decision never
+ * changes the immutable experiment and never decides whether the autonomous
+ * reproduction Agent may run.
+ */
+export function decideOpportunisticAssetPreparation({ plan, admission, executor }) {
+  const constraints = assetPreparationPlanFailures(plan).filter((failure) =>
+    String(failure?.reasonCode ?? "").startsWith("asset.")
+    && !String(failure.reasonCode).startsWith("asset.requirements_"));
+  if (constraints.length) {
+    return {
+      action: "defer",
+      reasonCode: constraints[0].reasonCode,
+      message: `预下载已跳过：${constraints.map((failure) => failure.message).join("；")}最终复现 Agent 仍可自行处理。`,
+      constraints,
+    };
+  }
+
+  const summary = admission?.summary ?? admission ?? {};
+  const unavailable = count(
+    summary.transportUnavailableAssetCount
+      ?? summary.terminalCount
+      ?? summary.blockedCount,
+  );
+  const retryable = count(
+    summary.transportRetryableAssetCount
+      ?? summary.retryableCount,
+  );
+  if (unavailable > 0 || retryable > 0) {
+    return {
+      action: "defer",
+      reasonCode: unavailable > 0
+        ? "asset.prefetch_transport_unavailable"
+        : "asset.prefetch_transport_transient",
+      message: unavailable > 0
+        ? "部分原始链接当前不可下载；已跳过预下载并把链接记录交给最终复现 Agent 自主处理。"
+        : "部分链接暂时异常；本次不创建新的 Worker 重试，已交给最终复现 Agent 自主处理。",
+      constraints: [],
+    };
+  }
+
+  if (executor !== "gcp-batch") {
+    return {
+      action: "defer",
+      reasonCode: "asset.prefetch_checkpoint_unsupported",
+      message: `执行器 ${executor} 不支持独立缓存检查点；资料由最终复现 Agent 在执行环境内准备。`,
+      constraints: [],
+    };
+  }
+
+  return {
+    action: "attempt",
+    reasonCode: "asset.prefetch_candidate",
+    message: "公开传输可用且预算允许，尝试在低成本环境中预先准备可复用缓存。",
+    constraints: [],
+  };
+}
+
+export function assetPreparationFallback(error) {
+  const raw = error instanceof Error ? error.message : String(error);
+  const detail = raw.replaceAll(/\s+/g, " ").trim().slice(0, 360);
+  return {
+    state: "deferred",
+    preparationMode: "reproduction-agent-fallback",
+    reasonCode: error?.failureCode ?? "asset.prefetch_failed",
+    message: detail
+      ? `资料预下载未完成（${detail}）。这只是缓存优化失败；最终复现 Agent 将从原始链接继续。`
+      : "资料预下载未完成；最终复现 Agent 将从原始链接继续。",
+  };
+}
+
+export function mustAbortAfterAssetPreparationFailure(error) {
+  return error?.processingLeaseLost === true || error?.cancelled === true;
+}
+
+function count(value) {
+  const numeric = Number(value);
+  return Number.isSafeInteger(numeric) && numeric >= 0 ? numeric : 0;
+}