@citeark/agent 0.3.20

package/src/assets/plan.mjs

@@ -0,0 +1,411 @@
+import path from "node:path";
+
+import { sha256Value } from "../util.mjs";
+import {
+  assetRequirementCoverageFailures,
+  evaluateAssetRequirementCoverage,
+  normalizeAssetRequirements,
+} from "./requirements.mjs";
+import { CURRENT_PREFLIGHT_SCHEMA_VERSION } from "../contracts/preflight-schema.mjs";
+import { rankedAssetLocators } from "./source-policy.mjs";
+
+export const ASSET_SCIENTIFIC_ROLES = new Set([
+  "execution_required",
+  "identity_only",
+  "provenance_only",
+  "optional",
+]);
+
+export const ASSET_ACQUISITION_MODES = new Set([
+  "auto",
+  "metadata",
+  "selective",
+  "full",
+  "stream",
+]);
+
+export const DEFAULT_ASSET_LIMITS = Object.freeze({
+  totalDownloadBytes: 64 * 1024 ** 3,
+  totalExpandedBytes: 256 * 1024 ** 3,
+  datasetDownloadBytes: 32 * 1024 ** 3,
+  modelDownloadBytes: 32 * 1024 ** 3,
+  repositoryDownloadBytes: 8 * 1024 ** 3,
+  otherDownloadBytes: 8 * 1024 ** 3,
+  expandedToDownloadRatio: 4,
+  minimumFreeBytesAfterPreparation: 20 * 1024 ** 3,
+});
+
+export function createAssetPreparationPlan({
+  experiment,
+  datasetRegistry,
+  limits = {},
+  now = () => new Date(),
+}) {
+  const policy = normalizeLimits(limits);
+  const registry = registryMap(datasetRegistry);
+  const declaredAssets = Array.isArray(experiment?.protocol?.preflight?.assets)
+    ? experiment.protocol.preflight.assets
+    : [];
+  const assets = declaredAssets.map((asset) => planAsset({
+    asset,
+    experimentId: experiment?.id,
+    registry: registry.get(asset?.registryDatasetId),
+    policy,
+  }));
+  const checks = assetPreparationChecks(experiment?.protocol?.preflight?.checks);
+  const requirements = normalizeAssetRequirements(
+    experiment?.protocol?.preflight?.requirements,
+  );
+  const scientificCompleteness = evaluateAssetRequirementCoverage({
+    requirements: experiment?.protocol?.preflight?.requirements,
+    assets: declaredAssets,
+    checks: experiment?.protocol?.preflight?.checks,
+  });
+  const expectedDownloadBytes = assets.reduce(
+    (sum, asset) => sum + (
+      new Set(["prepare", "inspect"]).has(asset.decision.action)
+        ? asset.expectedDownloadBytes ?? 0
+        : 0
+    ),
+    0,
+  );
+  const expectedExpandedBytes = assets.reduce(
+    (sum, asset) => sum + (
+      new Set(["prepare", "inspect"]).has(asset.decision.action)
+        ? asset.expectedExpandedBytes ?? asset.expectedDownloadBytes ?? 0
+        : 0
+    ),
+    0,
+  );
+  const failures = experiment?.protocol?.preflight?.schemaVersion === CURRENT_PREFLIGHT_SCHEMA_VERSION
+    ? [...assetRequirementCoverageFailures(scientificCompleteness)]
+    : [];
+  if (expectedDownloadBytes > policy.totalDownloadBytes) {
+    failures.push({
+      assetId: null,
+      reasonCode: "asset.total_download_budget_exceeded",
+      message: `预计总下载 ${expectedDownloadBytes} 字节，超过任务总上限 ${policy.totalDownloadBytes} 字节。`,
+    });
+  }
+  if (expectedExpandedBytes > policy.totalExpandedBytes) {
+    failures.push({
+      assetId: null,
+      reasonCode: "asset.total_expanded_budget_exceeded",
+      message: `预计落盘 ${expectedExpandedBytes} 字节，超过任务落盘上限 ${policy.totalExpandedBytes} 字节。`,
+    });
+  }
+  const immutable = {
+    schemaVersion: "0.1",
+    policyVersion: "asset-preparation-v2",
+    experimentId: experiment?.id ?? null,
+    maxMinutes: boundedPositiveInteger(experiment?.protocol?.preflight?.maxMinutes, 120, 360),
+    assets,
+    checks,
+    requirements,
+    scientificCompleteness,
+    limits: policy,
+    failures,
+  };
+  const plan = {
+    ...immutable,
+    planDigest: `sha256:${sha256Value(immutable)}`,
+    createdAt: now().toISOString(),
+    summary: {
+      assetCount: assets.length,
+      executionRequiredCount: assets.filter(
+        (asset) => asset.scientificRole === "execution_required",
+      ).length,
+      metadataOnlyCount: assets.filter((asset) => asset.decision.action === "metadata").length,
+      inspectCount: assets.filter((asset) => asset.decision.action === "inspect").length,
+      prepareCount: assets.filter((asset) => asset.decision.action === "prepare").length,
+      streamCount: assets.filter((asset) => asset.decision.action === "stream").length,
+      blockedCount: assets.filter((asset) => asset.decision.action === "blocked").length,
+      expectedDownloadBytes,
+      expectedExpandedBytes,
+      expectedSizeUnknownCount: assets.filter(
+        (asset) => asset.expectedDownloadBytes === null,
+      ).length,
+    },
+  };
+  return plan;
+}
+
+export function assetPreparationPlanFailures(plan) {
+  return [
+    ...(Array.isArray(plan?.failures) ? plan.failures : []),
+    ...(Array.isArray(plan?.assets) ? plan.assets : [])
+    .filter((asset) => asset?.decision?.action === "blocked")
+    .map((asset) => ({
+      assetId: asset.id,
+      reasonCode: asset.decision.reasonCode,
+      message: asset.decision.message,
+    })),
+  ];
+}
+
+function assetPreparationChecks(value) {
+  const kinds = new Set(["runtime", "dependencies", "dataset_loader"]);
+  return (Array.isArray(value) ? value : []).flatMap((check) => {
+    if (!check || typeof check !== "object" || !kinds.has(check.kind)) return [];
+    const id = text(check.id);
+    const successCriterion = text(check.successCriterion);
+    if (!id || !successCriterion) return [];
+    return [{
+      id,
+      kind: check.kind,
+      ...(text(check.command) ? { command: check.command.trim() } : {}),
+      successCriterion,
+    }];
+  });
+}
+
+function planAsset({ asset, experimentId, registry, policy }) {
+  const required = asset?.required === true;
+  const scientificRole = ASSET_SCIENTIFIC_ROLES.has(asset?.scientificRole)
+    ? asset.scientificRole
+    : required
+      ? "execution_required"
+      : "optional";
+  const acquisition = asset?.acquisition && typeof asset.acquisition === "object"
+    ? asset.acquisition
+    : {};
+  const mode = ASSET_ACQUISITION_MODES.has(acquisition.mode)
+    ? acquisition.mode
+    : "auto";
+  const kind = String(asset?.kind ?? "other");
+  const expectedDownloadBytes = nonNegativeSafeInteger(
+    acquisition.expectedDownloadBytes ?? registry?.candidateTransport?.archiveBytes,
+  );
+  const expectedExpandedBytes = nonNegativeSafeInteger(acquisition.expectedExpandedBytes);
+  const maximumDownloadBytes = positiveSafeInteger(
+    acquisition.maximumDownloadBytes,
+    defaultMaximumBytes(kind, policy),
+  );
+  const defaultMaximumExpandedBytes = Math.max(
+    maximumDownloadBytes,
+    maximumDownloadBytes * policy.expandedToDownloadRatio,
+  );
+  const maximumExpandedBytes = positiveSafeInteger(
+    acquisition.maximumExpandedBytes,
+    defaultMaximumExpandedBytes,
+  );
+  const supportsStreaming = acquisition.supportsStreaming === true;
+  const decision = acquisitionDecision({
+    required,
+    scientificRole,
+    mode,
+    expectedDownloadBytes,
+    expectedExpandedBytes,
+    maximumDownloadBytes,
+    maximumExpandedBytes,
+    supportsStreaming,
+    policy,
+  });
+  return {
+    id: String(asset?.id ?? "asset"),
+    experimentId: experimentId ?? null,
+    kind,
+    availability: String(asset?.availability ?? "unknown"),
+    identity: String(asset?.identity ?? asset?.id ?? "unknown"),
+    required,
+    scientificRole,
+    acquisitionMode: mode,
+    revision: text(acquisition.revision),
+    localPath: assetLocalPath(kind, asset?.id ?? "asset"),
+    sourceLocators: assetLocators(asset, registry),
+    sourceEvidence: normalizeSourceEvidence(asset?.sourceEvidence),
+    requiredPaths: normalizePatterns(acquisition.requiredPaths),
+    excludedPaths: normalizePatterns(acquisition.excludedPaths),
+    expectedDownloadBytes,
+    expectedExpandedBytes,
+    maximumDownloadBytes,
+    maximumExpandedBytes,
+    supportsStreaming,
+    verification: String(asset?.verification ?? "Verify identity before use."),
+    decision,
+  };
+}
+
+function acquisitionDecision({
+  required,
+  scientificRole,
+  mode,
+  expectedDownloadBytes,
+  expectedExpandedBytes,
+  maximumDownloadBytes,
+  maximumExpandedBytes,
+  supportsStreaming,
+  policy,
+}) {
+  if (!required || scientificRole !== "execution_required" || mode === "metadata") {
+    return {
+      action: "metadata",
+      reasonCode: "asset.content_not_required",
+      message: "该资产只需要来源或身份元数据，不下载正文。",
+    };
+  }
+  if (mode === "stream") {
+    return supportsStreaming
+      ? {
+          action: "stream",
+          reasonCode: "asset.streaming_declared",
+          message: "资产按不可变清单流式读取，不预先完整落盘。",
+        }
+      : {
+          action: "blocked",
+          reasonCode: "asset.streaming_not_supported",
+          message: "计划要求流式读取，但没有声明可验证的流式协议。",
+        };
+  }
+  if (expectedDownloadBytes === null) {
+    return {
+      action: "inspect",
+      reasonCode: "asset.size_inventory_required",
+      message: "先读取远端文件清单与大小，再决定是否下载。",
+    };
+  }
+  if (expectedDownloadBytes > maximumDownloadBytes) {
+    return supportsStreaming
+      ? {
+          action: "stream",
+          reasonCode: "asset.size_requires_streaming",
+          message: "完整下载超过单资产预算，改用声明且可验证的流式协议。",
+        }
+      : {
+          action: "blocked",
+          reasonCode: "asset.download_budget_exceeded",
+          message: `预计下载 ${expectedDownloadBytes} 字节，超过单资产上限 ${maximumDownloadBytes} 字节。`,
+        };
+  }
+  if (expectedExpandedBytes !== null && expectedExpandedBytes > maximumExpandedBytes) {
+    return {
+      action: "blocked",
+      reasonCode: "asset.expanded_size_budget_exceeded",
+      message: `预计展开后 ${expectedExpandedBytes} 字节，超过展开空间上限 ${maximumExpandedBytes} 字节。`,
+    };
+  }
+  return {
+    action: "prepare",
+    reasonCode: mode === "selective"
+      ? "asset.selective_download_ready"
+      : "asset.download_ready",
+    message: mode === "selective"
+      ? "只准备计划明确需要的文件。"
+      : "资产可在廉价准备阶段完整获取。",
+  };
+}
+
+function assetLocators(asset, registry) {
+  return rankedAssetLocators(asset, registry);
+}
+
+function normalizeSourceEvidence(value) {
+  return (Array.isArray(value) ? value : []).flatMap((entry) => {
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) return [];
+    const source = text(entry.source);
+    const locator = text(entry.locator);
+    if (!source || !locator) return [];
+    return [{
+      source,
+      locator,
+      ...(text(entry.path) ? { path: entry.path } : {}),
+      ...(text(entry.commit) ? { commit: entry.commit } : {}),
+    }];
+  });
+}
+
+function normalizePatterns(value) {
+  return [...new Set((Array.isArray(value) ? value : [])
+    .map(text)
+    .filter((pattern) => pattern && safeAssetPattern(pattern)))];
+}
+
+function safeAssetPattern(value) {
+  const normalized = String(value).replaceAll("\\", "/");
+  return !path.posix.isAbsolute(normalized)
+    && !normalized.split("/").includes("..")
+    && !/[\u0000-\u001f]/.test(normalized);
+}
+
+function normalizeLimits(value) {
+  return {
+    totalDownloadBytes: positiveSafeInteger(
+      value.totalDownloadBytes,
+      DEFAULT_ASSET_LIMITS.totalDownloadBytes,
+    ),
+    totalExpandedBytes: positiveSafeInteger(
+      value.totalExpandedBytes,
+      DEFAULT_ASSET_LIMITS.totalExpandedBytes,
+    ),
+    datasetDownloadBytes: positiveSafeInteger(
+      value.datasetDownloadBytes,
+      DEFAULT_ASSET_LIMITS.datasetDownloadBytes,
+    ),
+    modelDownloadBytes: positiveSafeInteger(
+      value.modelDownloadBytes,
+      DEFAULT_ASSET_LIMITS.modelDownloadBytes,
+    ),
+    repositoryDownloadBytes: positiveSafeInteger(
+      value.repositoryDownloadBytes,
+      DEFAULT_ASSET_LIMITS.repositoryDownloadBytes,
+    ),
+    otherDownloadBytes: positiveSafeInteger(
+      value.otherDownloadBytes,
+      DEFAULT_ASSET_LIMITS.otherDownloadBytes,
+    ),
+    expandedToDownloadRatio: positiveSafeInteger(
+      value.expandedToDownloadRatio,
+      DEFAULT_ASSET_LIMITS.expandedToDownloadRatio,
+    ),
+    minimumFreeBytesAfterPreparation: positiveSafeInteger(
+      value.minimumFreeBytesAfterPreparation,
+      DEFAULT_ASSET_LIMITS.minimumFreeBytesAfterPreparation,
+    ),
+  };
+}
+
+function defaultMaximumBytes(kind, limits) {
+  if (kind === "dataset") return limits.datasetDownloadBytes;
+  if (kind === "model" || kind === "checkpoint") return limits.modelDownloadBytes;
+  if (kind === "repository") return limits.repositoryDownloadBytes;
+  return limits.otherDownloadBytes;
+}
+
+function registryMap(value) {
+  return new Map((Array.isArray(value?.datasets) ? value.datasets : [])
+    .filter((entry) => typeof entry?.id === "string")
+    .map((entry) => [entry.id, entry]));
+}
+
+export function assetLocalPath(kind, id) {
+  return `/job/assets/${safeSegment(kind)}/${safeSegment(id)}`;
+}
+
+function safeSegment(value) {
+  return String(value ?? "asset")
+    .toLowerCase()
+    .replaceAll(/[^a-z0-9_.-]+/g, "-")
+    .replaceAll(/^-+|-+$/g, "")
+    .slice(0, 120) || "asset";
+}
+
+function text(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function nonNegativeSafeInteger(value) {
+  const number = Number(value);
+  return Number.isSafeInteger(number) && number >= 0 ? number : null;
+}
+
+function positiveSafeInteger(value, fallback) {
+  const number = Number(value);
+  return Number.isSafeInteger(number) && number > 0 ? number : fallback;
+}
+
+function boundedPositiveInteger(value, fallback, maximum) {
+  const number = Number(value);
+  return Number.isSafeInteger(number) && number > 0
+    ? Math.min(number, maximum)
+    : fallback;
+}

package/src/assets/prompts.mjs

@@ -0,0 +1,29 @@
+export function initialAssetPreparationPrompt(bundle) {
+  return [
+    `Run ID: ${bundle.runId}`,
+    "This is the dedicated asset-preparation phase. Do not run the full scientific experiment.",
+    "Read /job/input/asset-preparation-plan.json and /job/input/task.json. The plan is immutable.",
+    "For metadata actions, record source observations without downloading content. For selective actions, treat requiredPaths as backward-compatible download-selection hints: fetch the useful matching subset within budget, record unmatched hints, and never require every listed alternative format to exist. Always exclude excludedPaths. For inspect actions, first obtain a remote file inventory and sizes, then stay within both per-asset and total limits. For stream actions, prepare a bounded streaming descriptor and a small representative loader probe; never silently turn streaming into a full download.",
+    "Put durable asset bytes only at each plan localPath under /job/assets. Put package caches only under /job/runtime-cache. Modify dependencies only inside /job/runtime-venv; the base image is a download cache, not a version restriction.",
+    "Use resumable and retrying transfers. Preserve every URL, redirect target, status, attempt, byte count and checksum in /job/output/logs/asset-preparation. Never log credentials or signed query parameters.",
+    "Try sourceLocators in rank order. Use a lower-priority mirror only after recording each higher-priority source as failed. A mirror marked fallbackRequiresDigestMatch is forbidden unless its planned expectedDigest matches acquisition.sourceDigest. Record acquisition.sourceDigest and attemptedSources in the manifest.",
+    "The deterministic downloader already ran before this Agent. Inspect /job/output/logs/asset-preparation/deterministic and retain any valid partial downloads; repair only the ambiguity, dependency conflict, loader failure, or unsupported transport that caused the handoff.",
+    "For dependencies, inspect repository metadata and actual import errors. You may upgrade, downgrade or replace packages when evidence requires it. Record the final Python version and package inventory; do not modify the global image environment.",
+    "Run every plan check. A dataset_loader check must invoke the real evaluator or dataset loader, resolve the declared configuration and split, read representative rows, and verify schema/identity. An import-only check or web-page reachability check is not sufficient.",
+    "Never download a historical training corpus merely to evaluate a published checkpoint. If an item is unnecessary, unavailable, or over budget, record that preparation observation and leave it to the complete reproduction Agent; this cache phase must not reduce the experiment scope or decide scientific executability.",
+    "Write /job/output/asset-preparation-manifest.json using schemaVersion 0.1 and kind citeark.asset-preparation-manifest. Include planDigest; one record per planned asset with assetId, identity, revision when planned, sourceLocator, localPath, status, verification {status: passed, commands, evidence}, and for prepared/stream_ready records acquisition {downloadBytes, commands, evidence}; and one record per planned check with checkId, kind, status: passed, commands and evidence paths relative to /job/output. Every evidence path is relative to /job/output. Evidence may initially be a relative-path string or {path}; the supervisor replaces it with a canonical {path, bytes, digest} descriptor, inventories prepared files, and records the task-local environment. Do not invent manifestDigest yourself.",
+    "Before stopping, run `python3 /job/input/validate-asset-preparation.py /job/input/asset-preparation-plan.json /job/output/asset-preparation-manifest.json`. Repair any reported factual path or status error, rerun it, and stop only after it prints status passed. The runner repeats this deterministic validation independently.",
+    "Stop after the manifest and logs are complete.",
+  ].join("\n");
+}
+
+export function resumeAssetPreparationPrompt(bundle) {
+  return [
+    `Resume asset preparation for ${bundle.runId} from the restored checkpoint.`,
+    "Inspect /job/input/asset-preparation-plan.json, the existing /job/assets, /job/runtime-cache, /job/runtime-venv, /job/execution/completion-validation.log and prior logs.",
+    "The plan is immutable. In particular, keep every assetId, sourceLocator and localPath exactly as planned; never move a prepared asset merely to make the manifest easier to write.",
+    "Continue partial downloads with content-range/resume support. Do not restart valid completed transfers or repeat already-passed loader checks unless their inputs or environment changed.",
+    "Finish /job/output/asset-preparation-manifest.json and preserve all earlier attempt evidence. Evidence paths are relative to /job/output; the supervisor fills their byte counts and digests.",
+    "Before stopping, run `python3 /job/input/validate-asset-preparation.py /job/input/asset-preparation-plan.json /job/output/asset-preparation-manifest.json`, repair any reported factual error, and stop only after it prints status passed.",
+  ].join("\n");
+}