@citeark/agent 0.3.20

package/src/adapters/paper2agent.mjs

@@ -0,0 +1,107 @@
+import { lstat, mkdtemp, readFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import { assembleResearchCompilationCap } from "../cap/v2/compilation-artifact.mjs";
+import { captureProcess } from "../process.mjs";
+import { CiteArkError, pathExists } from "../util.mjs";
+
+/** Import one current Paper2Agent delivery without interpreting it as scientific truth. */
+export async function assemblePaper2AgentCompilationCap({
+  directory,
+  inventory,
+  deliveryPath,
+  paper2AgentVersion,
+  runId,
+  createdAt,
+  signingKeyPath,
+}) {
+  const delivery = await readPaper2AgentDelivery(deliveryPath);
+  const built = await assembleResearchCompilationCap({
+    directory,
+    inventory,
+    compiler: {
+      id: "paper2agent",
+      name: "Paper2Agent",
+      version: paper2AgentVersion ?? "current-skill-output",
+      capabilities: delivery.capabilities,
+      identity: { deliveryKind: delivery.kind },
+    },
+    runId,
+    createdAt,
+    signingKeyPath,
+    nativeOutputBytes: delivery.bytes,
+    nativeOutputMediaType: delivery.mediaType,
+  });
+  return { ...built, delivery };
+}
+
+export async function readPaper2AgentDelivery(deliveryPath) {
+  const source = path.resolve(deliveryPath);
+  let sourceStat;
+  try {
+    sourceStat = await lstat(source);
+  } catch {
+    throw new CiteArkError(`Paper2Agent 交付物不存在：${source}`);
+  }
+
+  if (sourceStat.isFile()) {
+    if (path.extname(source).toLowerCase() !== ".zip") {
+      throw new CiteArkError("Paper2Agent 文件交付物必须是 Paper2MCP ZIP");
+    }
+    return {
+      source,
+      kind: "paper2mcp-zip",
+      capabilities: ["mcp-tools"],
+      mediaType: "application/zip",
+      bytes: await readFile(source),
+    };
+  }
+
+  if (!sourceStat.isDirectory()) {
+    throw new CiteArkError("Paper2Agent 交付物必须是目录或 ZIP 文件");
+  }
+
+  const kind = await classifyDirectory(source);
+  const capabilities = kind === "combined-agent"
+    ? ["mcp-tools", "paper-skill"]
+    : kind === "paper-skill"
+      ? ["paper-skill"]
+      : ["mcp-tools"];
+  const temporary = await mkdtemp(path.join(os.tmpdir(), "citeark-paper2agent-"));
+  try {
+    const archivePath = path.join(temporary, "paper2agent-output.tar.gz");
+    const packed = await captureProcess(
+      "tar",
+      ["-czf", archivePath, "--exclude=.DS_Store", "--exclude=._*", path.basename(source)],
+      { cwd: path.dirname(source), env: { ...process.env, COPYFILE_DISABLE: "1" } },
+    );
+    if (packed.code !== 0) {
+      throw new CiteArkError(`无法封装 Paper2Agent 交付目录：${packed.stderr.trim()}`);
+    }
+    return {
+      source,
+      kind,
+      capabilities,
+      mediaType: "application/gzip",
+      bytes: await readFile(archivePath),
+    };
+  } finally {
+    await rm(temporary, { recursive: true, force: true });
+  }
+}
+
+async function classifyDirectory(source) {
+  if (
+    await pathExists(path.join(source, "skill"))
+    && await pathExists(path.join(source, "mcp"))
+  ) return "combined-agent";
+  if (
+    await pathExists(path.join(source, "SKILL.md"))
+    && await pathExists(path.join(source, "references", "paper.md"))
+  ) return "paper-skill";
+  if (await pathExists(path.join(source, "USAGE.md"))) return "paper2mcp-directory";
+  throw new CiteArkError(
+    "无法识别 Paper2Agent 交付目录；应为组合 agent、paper skill 或解压后的 Paper2MCP 目录",
+  );
+}

package/src/assets/cache.mjs

@@ -0,0 +1,159 @@
+import { validateAssetPreparationManifest } from "./manifest.mjs";
+import { sha256Value } from "../util.mjs";
+
+export class GcsAssetCache {
+  constructor({ storage, bucket, sourceBucket = bucket, runtimeImage }) {
+    this.storage = storage;
+    this.bucket = bucket;
+    this.sourceBucket = sourceBucket;
+    this.runtimeImage = runtimeImage;
+  }
+
+  async lookup(plan) {
+    const key = cacheCompatibilityKey(plan, this.runtimeImage);
+    const object = planIndexObject(key);
+    const index = await this.storage.readJsonObject({
+      bucket: this.bucket,
+      object,
+      missing: null,
+    });
+    if (!validIndex(index, plan, this.runtimeImage)) {
+      return { hit: false, key, indexObject: object, reason: "missing_or_incompatible" };
+    }
+    const manifestIssues = validateAssetPreparationManifest(plan, index.manifest);
+    if (manifestIssues.length) {
+      return { hit: false, key, indexObject: object, reason: "manifest_invalid", issues: manifestIssues };
+    }
+    const objects = await this.storage.listObjects({
+      bucket: this.bucket,
+      prefix: `${index.checkpoint.checkpointPrefix}-full-`,
+    });
+    if (!objects.some((name) => name === index.checkpoint.object)) {
+      return { hit: false, key, indexObject: object, reason: "checkpoint_missing" };
+    }
+    const checkpointStatus = await this.storage.readJsonObject({
+      bucket: this.bucket,
+      object: index.checkpoint.checkpointStatus,
+      missing: null,
+    });
+    if (!validCheckpointStatus(checkpointStatus, index.checkpoint)) {
+      return { hit: false, key, indexObject: object, reason: "checkpoint_status_invalid" };
+    }
+    return {
+      hit: true,
+      key,
+      indexObject: object,
+      manifest: index.manifest,
+      checkpoint: index.checkpoint,
+      createdAt: index.createdAt,
+    };
+  }
+
+  async publish(plan, manifest, checkpoint) {
+    const issues = validateAssetPreparationManifest(plan, manifest);
+    if (issues.length) throw new TypeError(`拒绝缓存无效资产清单：${issues.join("；")}`);
+    if (
+      !checkpoint?.object
+      || checkpoint?.kind !== "full"
+      || !/^sha256:[0-9a-f]{64}$/.test(String(checkpoint?.sha256))
+    ) {
+      throw new TypeError("拒绝缓存没有完整检查点的资产准备结果");
+    }
+    const manifestHex = manifest.manifestDigest.slice("sha256:".length);
+    const canonicalPrefix = `citeark-asset-cache-manifest-${manifestHex}`;
+    const canonicalObject = `${canonicalPrefix}-full-${checkpoint.sequence}.tar.gz`;
+    if (checkpoint.object !== canonicalObject) {
+      await this.storage.copyObject({
+        sourceBucket: this.sourceBucket,
+        sourceObject: checkpoint.object,
+        destinationBucket: this.bucket,
+        destinationObject: canonicalObject,
+      });
+    }
+    const canonicalCheckpoint = {
+      ...checkpoint,
+      bucket: this.bucket,
+      checkpointPrefix: canonicalPrefix,
+      checkpointStatus: `${canonicalPrefix}-status.json`,
+      object: canonicalObject,
+      reason: "asset-preparation",
+    };
+    await this.storage.writeJsonObject({
+      bucket: this.bucket,
+      object: canonicalCheckpoint.checkpointStatus,
+      value: {
+        schemaVersion: "0.1",
+        sequence: canonicalCheckpoint.sequence,
+        kind: "full",
+        reason: "asset-preparation",
+        object: canonicalObject,
+        createdAt: canonicalCheckpoint.createdAt,
+        bytes: canonicalCheckpoint.bytes,
+        sha256: canonicalCheckpoint.sha256,
+      },
+    });
+    const key = cacheCompatibilityKey(plan, this.runtimeImage);
+    const index = {
+      schemaVersion: "0.1",
+      kind: "citeark.asset-cache-index",
+      key,
+      planDigest: plan.planDigest,
+      manifestDigest: manifest.manifestDigest,
+      runtimeImage: this.runtimeImage,
+      checkpoint: canonicalCheckpoint,
+      manifest,
+      createdAt: new Date().toISOString(),
+    };
+    await Promise.all([
+      this.storage.writeJsonObject({
+        bucket: this.bucket,
+        object: manifestIndexObject(manifestHex),
+        value: index,
+      }),
+      this.storage.writeJsonObject({
+        bucket: this.bucket,
+        object: planIndexObject(key),
+        value: index,
+      }),
+    ]);
+    return index;
+  }
+}
+
+export function cacheCompatibilityKey(plan, runtimeImage) {
+  return sha256Value({
+    schemaVersion: "0.1",
+    planDigest: plan?.planDigest ?? null,
+    runtimeImage: runtimeImage ?? null,
+  });
+}
+
+function validIndex(value, plan, runtimeImage) {
+  return value?.schemaVersion === "0.1"
+    && value?.kind === "citeark.asset-cache-index"
+    && value?.planDigest === plan?.planDigest
+    && value?.runtimeImage === runtimeImage
+    && value?.checkpoint?.kind === "full"
+    && value?.checkpoint?.reason === "asset-preparation"
+    && typeof value?.checkpoint?.checkpointPrefix === "string"
+    && typeof value?.checkpoint?.object === "string"
+    && /^sha256:[0-9a-f]{64}$/.test(String(value?.checkpoint?.sha256));
+}
+
+function validCheckpointStatus(value, checkpoint) {
+  return value?.schemaVersion === "0.1"
+    && value?.sequence === checkpoint.sequence
+    && value?.kind === "full"
+    && value?.reason === "asset-preparation"
+    && value?.object === checkpoint.object
+    && value?.bytes === checkpoint.bytes
+    && value?.sha256 === checkpoint.sha256;
+}
+
+function planIndexObject(key) {
+  return `citeark-asset-cache-plan-${key}.json`;
+}
+
+function manifestIndexObject(manifestHex) {
+  return `citeark-asset-cache-manifest-${manifestHex}.json`;
+}

package/src/assets/compute.mjs

@@ -0,0 +1,98 @@
+import { CiteArkError, sha256Value } from "../util.mjs";
+
+export function selectAssetPreparationProfile(catalog, reproductionDecision, { profileOverride } = {}) {
+  const reproductionConfig = reproductionDecision?.selectedProfile?.executorConfig ?? {};
+  const runtimeImage = reproductionDecision?.environment?.image;
+  const requiredProvisioningModel = reproductionDecision?.policy?.requiredProvisioningModel;
+  const candidates = (Array.isArray(catalog?.profiles) ? catalog.profiles : [])
+    .filter((profile) => profile?.available !== false)
+    .filter((profile) => profile?.executor === "gcp-batch" && profile?.accelerator === "cpu")
+    .filter((profile) => !profileOverride || profile.id === profileOverride)
+    .filter((profile) => sameControlPlane(profile, reproductionConfig))
+    .filter((profile) => !runtimeImage || profile?.image === runtimeImage)
+    .filter((profile) => !requiredProvisioningModel
+      || profile.executorConfig?.provisioningModel === requiredProvisioningModel)
+    .sort((left, right) => {
+      return (left.hourlyUsd ?? Number.POSITIVE_INFINITY)
+        - (right.hourlyUsd ?? Number.POSITIVE_INFINITY);
+    });
+  if (!candidates.length) {
+    throw new CiteArkError(`${profileOverride ? `指定的资产 profile ${profileOverride}` : "资产准备"} 需要可用的 GCP Batch CPU profile${runtimeImage ? "，并与复现任务使用同一镜像和 GCS 控制面" : ""}，但计算目录中没有兼容项`, {
+      retryable: false,
+      failureCode: "asset_preparation.cpu_profile_missing",
+      failureCategory: "configuration",
+    });
+  }
+  return candidates[0];
+}
+
+export function createAssetPreparationComputeDecision({
+  profile,
+  contract,
+  experiment,
+  plan,
+  catalog,
+  researchRuntimeMinutes,
+  decidedAt = new Date().toISOString(),
+}) {
+  const researchPreparation = researchRuntimeMinutes !== undefined;
+  if (researchPreparation && (!Number.isFinite(researchRuntimeMinutes) || researchRuntimeMinutes <= 0)) {
+    throw new CiteArkError("CPU 研究准备需要有限的累计科研时限");
+  }
+  // Full workspace preparation includes rereading, planning and dependencies.
+  // The legacy quick preflight limit must not truncate this research phase.
+  const timeoutMinutes = researchPreparation
+    ? Math.min(researchRuntimeMinutes, profile.timeoutMinutes ?? researchRuntimeMinutes, 360)
+    : Math.max(15, Math.min(plan?.maxMinutes ?? 120, profile.timeoutMinutes ?? plan?.maxMinutes ?? 120));
+  const expectedHours = timeoutMinutes / 60;
+  const body = {
+    schemaVersion: "0.1",
+    kind: "citeark.operational-compute-decision",
+    workload: researchPreparation ? "research-preparation" : "asset-preparation",
+    experimentId: experiment?.id ?? null,
+    experimentVersionId: experiment?.versionId ?? null,
+    contractDigest: contract?.contractDigest ?? null,
+    planDigest: plan?.planDigest ?? null,
+    requirement: {
+      classification: "cpu-only",
+      accelerator: "none",
+      cpuCores: Math.min(profile.cpuCores ?? 4, 8),
+      memoryGb: Math.min(profile.memoryGb ?? 16, 32),
+      preemptionSafe: true,
+    },
+    selectedProfile: structuredClone(profile),
+    environment: {
+      image: profile.image,
+      timeoutMinutes,
+      cpus: Math.min(profile.cpuCores ?? 4, 8),
+      memoryGb: Math.min(profile.memoryGb ?? 16, 32),
+      shmGb: Math.max(1, Math.min(profile.shmGb ?? 2, 4)),
+      gpu: "none",
+      networkAccess: true,
+    },
+    estimate: {
+      durationMinutes: timeoutMinutes,
+      costUsd: typeof profile.hourlyUsd === "number"
+        ? Number((expectedHours * profile.hourlyUsd).toFixed(6))
+        : null,
+      pricingKnown: typeof profile.hourlyUsd === "number",
+    },
+    catalogDigest: `sha256:${sha256Value(catalog)}`,
+    rationale: researchPreparation ? "在 CPU 延续研究工作区，时限取累计科研上限与 CPU 阶段上限，不受旧短预检限制。" : "在 GPU 分配前，用同一 GCS 控制面的低成本 CPU 工作负载准备并验证资产与任务隔离依赖环境。",
+    decidedAt,
+    scheduler: {
+      name: "citeark-asset-preparation-scheduler",
+      version: "0.1.0",
+      strategy: "same-control-plane-cheapest-cpu",
+    },
+  };
+  return { ...body, decisionDigest: `sha256:${sha256Value(body)}` };
+}
+
+function sameControlPlane(profile, config) {
+  const candidate = profile?.executorConfig ?? {};
+  if (!config.stagingBucket) return true;
+  return candidate.stagingBucket === config.stagingBucket
+    && candidate.projectId === config.projectId
+    && candidate.region === config.region;
+}

package/src/assets/executor.mjs

@@ -0,0 +1,145 @@
+import { GoogleAccessTokenProvider } from "../gcp/auth.mjs";
+import { GcsStagingClient } from "../gcp/storage.mjs";
+import { executeGcpBatchReproduction } from "../compute/gcp-batch-executor.mjs";
+import { CiteArkError } from "../util.mjs";
+import { GcsAssetCache } from "./cache.mjs";
+import { createAssetPreparationWorkload, recordAssetCacheHit } from "./lifecycle.mjs";
+
+export async function executeGcpAssetPreparation({
+  taskPath,
+  runsDirectory,
+  repositoryOverride,
+  paperOverride,
+  computeDecision,
+  assetPlan,
+  runId,
+  agentRuntime,
+  executorOptions = {},
+  dryRun = false,
+  progress = () => {},
+}) {
+  const gcpOptions = executorOptions.gcpBatch ?? {};
+  const {
+    restoreCheckpointPrefix: _reproductionCheckpoint,
+    restoreCheckpointBucket: _reproductionCheckpointBucket,
+    assetPreparationRestoreCheckpointPrefix,
+    assetPreparationRestoreCheckpointBucket,
+    ...assetGcpOptions
+  } = gcpOptions;
+  if (assetPreparationRestoreCheckpointPrefix) {
+    assetGcpOptions.restoreCheckpointPrefix = assetPreparationRestoreCheckpointPrefix;
+    if (assetPreparationRestoreCheckpointBucket) {
+      assetGcpOptions.restoreCheckpointBucket = assetPreparationRestoreCheckpointBucket;
+    }
+  }
+  const profile = computeDecision?.selectedProfile?.executorConfig ?? {};
+  const stagingBucket = gcpOptions.stagingBucket
+    ?? profile.stagingBucket
+    ?? process.env.CITEARK_GCP_STAGING_BUCKET;
+  const cacheBucket = gcpOptions.assetCacheBucket
+    ?? process.env.CITEARK_GCP_ASSET_CACHE_BUCKET
+    ?? stagingBucket;
+  if (!stagingBucket || !cacheBucket) throw new CiteArkError("资产准备缺少 GCS 暂存或缓存 bucket");
+  const clients = resolveAssetPreparationClients(
+    gcpOptions,
+    gcpOptions.assetPreparationClients ?? gcpOptions.clients,
+  );
+  const runtimeImage = computeDecision?.environment?.image;
+  const cache = new GcsAssetCache({
+    storage: clients.storage,
+    bucket: cacheBucket,
+    sourceBucket: stagingBucket,
+    runtimeImage,
+  });
+
+  if (!dryRun && gcpOptions.disableAssetCache !== true) {
+    const cached = await cache.lookup(assetPlan);
+    if (cached.hit) {
+      progress(`资产准备命中已验证缓存 ${cached.manifest.manifestDigest}`);
+      const directories = await recordAssetCacheHit({
+        runsDirectory,
+        runId,
+        plan: assetPlan,
+        manifest: cached.manifest,
+        cache: cached,
+      });
+      return {
+        dryRun: false,
+        cacheHit: true,
+        directories,
+        manifest: cached.manifest,
+        checkpoint: cached.checkpoint,
+        usage: null,
+      };
+    }
+    progress(`资产准备缓存未命中（${cached.reason}），启动低成本 CPU 工作负载`);
+  }
+
+  const execution = await executeGcpBatchReproduction({
+    taskPath,
+    runsDirectory,
+    repositoryOverride,
+    paperOverride,
+    computeDecision,
+    runId,
+    agentRuntime,
+    executorOptions: {
+      ...executorOptions,
+      gcpBatch: {
+        ...assetGcpOptions,
+        clients,
+        keepStaging: true,
+      },
+    },
+    workloadAdapter: createAssetPreparationWorkload(assetPlan),
+    dryRun,
+    progress,
+  });
+  if (dryRun) return execution;
+
+  const index = await cache.publish(assetPlan, execution.manifest, execution.checkpoint);
+  progress(`资产准备已按内容清单缓存 ${execution.manifest.manifestDigest}`);
+  await cleanupEphemeralPreparationObjects({
+    storage: clients.storage,
+    bucket: stagingBucket,
+    checkpoint: execution.checkpoint,
+  });
+  return {
+    ...execution,
+    cacheHit: false,
+    checkpoint: index.checkpoint,
+    cache: {
+      key: index.key,
+      planDigest: index.planDigest,
+      manifestDigest: index.manifestDigest,
+      checkpoint: index.checkpoint,
+    },
+  };
+}
+
+export function resolveAssetPreparationClients(options = {}, injected = {}) {
+  const configured = {
+    authOptions: options.authOptions,
+    batchOptions: options.batchOptions,
+    storageOptions: options.storageOptions,
+    ...(injected ?? {}),
+  };
+  if (configured.storage) return configured;
+  const auth = configured.auth ?? new GoogleAccessTokenProvider(configured.authOptions);
+  return {
+    ...configured,
+    auth,
+    storage: new GcsStagingClient({
+      auth,
+      ...(configured.storageOptions ?? {}),
+    }),
+  };
+}
+
+async function cleanupEphemeralPreparationObjects({ storage, bucket, checkpoint }) {
+  const prefix = checkpoint?.checkpointPrefix;
+  if (!prefix || prefix.startsWith("citeark-asset-cache-manifest-")) return;
+  const runPrefix = prefix.replace(/-checkpoint-a\d+$/, "");
+  const objects = await storage.listObjects({ bucket, prefix: runPrefix }).catch(() => []);
+  await Promise.allSettled(objects.map((object) => storage.deleteObject({ bucket, object })));
+}