@citeark/agent 0.3.20

package/src/assets/public-asset-probe.mjs

@@ -0,0 +1,525 @@
+import { lookup } from "node:dns/promises";
+import net from "node:net";
+
+const TERMINAL_HTTP_STATUSES = new Set([401, 403, 404, 410]);
+const RANGE_FALLBACK_STATUSES = new Set([403, 404, 405, 501]);
+const REDIRECT_HTTP_STATUSES = new Set([301, 302, 303, 307, 308]);
+const USER_AGENT = "CiteArk-Reproduction-Admission/1.0";
+const DEFAULT_MAX_ATTEMPTS = 3;
+const DEFAULT_RETRY_BASE_DELAY_MS = 500;
+const DEFAULT_RETRY_MAX_DELAY_MS = 60_000;
+const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429]);
+const SENSITIVE_QUERY_KEY = /(auth|credential|key|secret|signature|token|x-amz)/i;
+const MAX_RECORDED_HUGGING_FACE_FILES = 20_000;
+
+class UnsafeAssetHostError extends Error {}
+
+/** Probe only public, unauthenticated transport feasibility. */
+export async function probePublicAssetUrl(value, {
+  fetchImpl = fetch,
+  timeoutMs = 12_000,
+  resolveHost = lookup,
+  maxAttempts = DEFAULT_MAX_ATTEMPTS,
+  retryBaseDelayMs = DEFAULT_RETRY_BASE_DELAY_MS,
+  retryMaxDelayMs = DEFAULT_RETRY_MAX_DELAY_MS,
+  sleepImpl = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)),
+  randomImpl = Math.random,
+  observe,
+  now = () => new Date(),
+} = {}) {
+  const boundedAttempts = positiveInteger(maxAttempts, DEFAULT_MAX_ATTEMPTS, 1, 8);
+  const attempts = [];
+  let result;
+  for (let attempt = 1; attempt <= boundedAttempts; attempt += 1) {
+    const startedAt = now();
+    const operations = [];
+    result = await probePublicAssetUrlOnce(value, {
+      fetchImpl,
+      timeoutMs,
+      resolveHost,
+      operations,
+      now,
+    });
+    const finishedAt = now();
+    const attemptRecord = {
+      attempt,
+      maxAttempts: boundedAttempts,
+      startedAt: startedAt.toISOString(),
+      finishedAt: finishedAt.toISOString(),
+      durationMs: Math.max(0, finishedAt.getTime() - startedAt.getTime()),
+      status: result.status,
+      ...(result.httpStatus === undefined ? {} : { httpStatus: result.httpStatus }),
+      ...(result.reasonCode ? { reasonCode: result.reasonCode } : {}),
+      ...(result.retryAfterMs === undefined ? {} : { retryAfterMs: result.retryAfterMs }),
+      ...(result.detail ? { detail: result.detail } : {}),
+      operations,
+    };
+    attempts.push(attemptRecord);
+    emitProbeObservation(observe, result, attemptRecord);
+    if (result.status !== "retryable" || attempt === boundedAttempts) break;
+    await sleepImpl(retryDelayMs({
+      result,
+      attempt,
+      retryBaseDelayMs,
+      retryMaxDelayMs,
+      randomImpl,
+    }));
+  }
+  return {
+    ...result,
+    observedAt: now().toISOString(),
+    attemptCount: attempts.length,
+    attempts,
+  };
+}
+
+async function probePublicAssetUrlOnce(value, {
+  fetchImpl,
+  timeoutMs,
+  resolveHost,
+  operations,
+  now,
+}) {
+  let current;
+  try {
+    current = safePublicUrl(value);
+  } catch (error) {
+    return terminal(value, "admission.asset_url_unsafe", error);
+  }
+
+  const huggingFace = huggingFaceApiUrl(current);
+  if (huggingFace) {
+    return await probeHuggingFaceRepository(huggingFace, {
+      originalUrl: current,
+      fetchImpl,
+      timeoutMs,
+      resolveHost,
+      operations,
+      now,
+    });
+  }
+
+  for (let redirect = 0; redirect <= 5; redirect += 1) {
+    let response;
+    try {
+      await assertPublicResolution(current, resolveHost, { operations, now });
+      response = await tracedFetch(current, {
+        method: "HEAD",
+        redirect: "manual",
+        signal: AbortSignal.timeout(timeoutMs),
+        headers: { "user-agent": USER_AGENT },
+      }, { fetchImpl, operations, now });
+    } catch (error) {
+      return networkOrSafetyFailure(current, error);
+    }
+    if (REDIRECT_HTTP_STATUSES.has(response.status)) {
+      const redirected = safeRedirect(current, response);
+      if (redirected.observation) return redirected.observation;
+      current = redirected.url;
+      continue;
+    }
+    if (response.ok) return reachable(current, response);
+    if (RANGE_FALLBACK_STATUSES.has(response.status)) {
+      return await rangeProbe(current, { fetchImpl, timeoutMs, operations, now });
+    }
+    return httpFailure(current, response);
+  }
+  return terminal(current, "admission.asset_redirect_limit");
+}
+
+async function probeHuggingFaceRepository(apiUrl, {
+  originalUrl,
+  fetchImpl,
+  timeoutMs,
+  resolveHost,
+  operations,
+  now,
+}) {
+  let response;
+  try {
+    await assertPublicResolution(apiUrl, resolveHost, { operations, now });
+    response = await tracedFetch(apiUrl, {
+      method: "GET",
+      redirect: "manual",
+      signal: AbortSignal.timeout(timeoutMs),
+      headers: { accept: "application/json", "user-agent": USER_AGENT },
+    }, { fetchImpl, operations, now });
+  } catch (error) {
+    return networkOrSafetyFailure(originalUrl, error);
+  }
+  if (!response.ok) {
+    if (response?.body) await response.body.cancel().catch(() => {});
+    return httpFailure(originalUrl, response);
+  }
+  let metadata;
+  try {
+    metadata = await response.json();
+  } catch (error) {
+    return {
+      url: loggableUrl(originalUrl),
+      status: "retryable",
+      reasonCode: "admission.asset_metadata_invalid",
+      detail: error instanceof Error ? error.message : String(error),
+    };
+  }
+  if (metadata?.private === true || Boolean(metadata?.gated)) {
+    return {
+      url: loggableUrl(originalUrl),
+      probeUrl: loggableUrl(apiUrl),
+      status: "terminal",
+      httpStatus: response.status,
+      reasonCode: "admission.asset_requires_auth_or_acceptance",
+    };
+  }
+  const siblings = Array.isArray(metadata?.siblings) ? metadata.siblings : [];
+  const inventory = siblings.map((item) => ({
+    path: String(item?.rfilename ?? ""),
+    bytes: nonNegativeInteger(item?.size ?? item?.lfs?.size),
+    ...(typeof item?.lfs?.sha256 === "string" ? { sha256: item.lfs.sha256 } : {}),
+  }));
+  const unknownSizeCount = inventory.filter((item) => item.bytes === null).length;
+  return {
+    url: loggableUrl(originalUrl),
+    probeUrl: loggableUrl(apiUrl),
+    status: "reachable",
+    httpStatus: response.status,
+    contentType: response.headers.get("content-type"),
+    inventory: {
+      provider: "hugging-face",
+      revision: typeof metadata?.sha === "string" ? metadata.sha : null,
+      fileCount: inventory.length,
+      totalBytes: unknownSizeCount === 0
+        ? inventory.reduce((sum, item) => sum + item.bytes, 0)
+        : null,
+      unknownSizeCount,
+      truncated: inventory.length > MAX_RECORDED_HUGGING_FACE_FILES,
+      files: inventory.slice(0, MAX_RECORDED_HUGGING_FACE_FILES),
+    },
+  };
+}
+
+function huggingFaceApiUrl(url) {
+  if (url.hostname.toLowerCase() !== "huggingface.co") return null;
+  const parts = url.pathname.split("/").filter(Boolean);
+  if (parts[0] === "api") return null;
+  const dataset = parts[0] === "datasets";
+  const offset = dataset ? 1 : 0;
+  if (!parts[offset] || !parts[offset + 1]) return null;
+  const kind = dataset ? "datasets" : "models";
+  const api = new URL(`/api/${kind}/${encodeURIComponent(parts[offset])}/${encodeURIComponent(parts[offset + 1])}`, url.origin);
+  api.searchParams.set("blobs", "true");
+  return api;
+}
+
+async function rangeProbe(url, { fetchImpl, timeoutMs, operations, now }) {
+  let response;
+  try {
+    response = await tracedFetch(url, {
+      method: "GET",
+      redirect: "manual",
+      signal: AbortSignal.timeout(timeoutMs),
+      headers: { range: "bytes=0-0", "user-agent": USER_AGENT },
+    }, { fetchImpl, operations, now });
+  } catch (error) {
+    return networkOrSafetyFailure(url, error);
+  }
+  if (response?.body) await response.body.cancel().catch(() => {});
+  if (response.ok || response.status === 206) return reachable(url, response);
+  return httpFailure(url, response);
+}
+
+async function assertPublicResolution(url, resolveHost, { operations, now }) {
+  if (net.isIP(url.hostname)) return;
+  const startedAt = now();
+  let addresses;
+  try {
+    addresses = await resolveHost(url.hostname, { all: true, verbatim: true });
+  } catch (error) {
+    const finishedAt = now();
+    operations.push({
+      kind: "dns",
+      hostname: url.hostname,
+      status: "failed",
+      startedAt: startedAt.toISOString(),
+      finishedAt: finishedAt.toISOString(),
+      durationMs: Math.max(0, finishedAt.getTime() - startedAt.getTime()),
+      detail: error instanceof Error ? error.message : String(error),
+    });
+    throw error;
+  }
+  const finishedAt = now();
+  operations.push({
+    kind: "dns",
+    hostname: url.hostname,
+    status: "succeeded",
+    addressCount: Array.isArray(addresses) ? addresses.length : 0,
+    addresses: Array.isArray(addresses)
+      ? addresses.map((item) => ({ address: item?.address, family: item?.family }))
+      : [],
+    startedAt: startedAt.toISOString(),
+    finishedAt: finishedAt.toISOString(),
+    durationMs: Math.max(0, finishedAt.getTime() - startedAt.getTime()),
+  });
+  if (!Array.isArray(addresses) || addresses.length === 0) {
+    throw new Error("资产主机没有可解析地址");
+  }
+  if (addresses.some((item) => privateIp(item?.address))) {
+    throw new UnsafeAssetHostError("资产主机解析到私有或保留地址");
+  }
+}
+
+async function tracedFetch(url, init, { fetchImpl, operations, now }) {
+  const startedAt = now();
+  try {
+    const response = await fetchImpl(url, init);
+    const finishedAt = now();
+    operations.push({
+      kind: "http",
+      method: init?.method ?? "GET",
+      url: loggableUrl(url),
+      status: "completed",
+      httpStatus: response.status,
+      ...retryAfter(response),
+      startedAt: startedAt.toISOString(),
+      finishedAt: finishedAt.toISOString(),
+      durationMs: Math.max(0, finishedAt.getTime() - startedAt.getTime()),
+    });
+    return response;
+  } catch (error) {
+    const finishedAt = now();
+    operations.push({
+      kind: "http",
+      method: init?.method ?? "GET",
+      url: loggableUrl(url),
+      status: "failed",
+      detail: error instanceof Error ? error.message : String(error),
+      startedAt: startedAt.toISOString(),
+      finishedAt: finishedAt.toISOString(),
+      durationMs: Math.max(0, finishedAt.getTime() - startedAt.getTime()),
+    });
+    throw error;
+  }
+}
+
+function safeRedirect(current, response) {
+  const location = response.headers.get("location");
+  if (!location) {
+    return { observation: terminal(current, "admission.asset_redirect_missing_location", null, response.status) };
+  }
+  try {
+    return { url: safePublicUrl(new URL(location, current).toString()) };
+  } catch (error) {
+    return { observation: terminal(current, "admission.asset_redirect_unsafe", error, response.status) };
+  }
+}
+
+function safePublicUrl(value) {
+  const url = new URL(value);
+  if (!new Set(["http:", "https:"]).has(url.protocol)) throw new Error("资产 URL 必须使用 HTTP(S)");
+  if (url.username || url.password) throw new Error("资产 URL 不能内嵌凭据");
+  if (url.port && !new Set(["80", "443"]).has(url.port)) throw new Error("资产 URL 端口不在允许范围");
+  const hostname = url.hostname.toLowerCase().replace(/\.$/, "");
+  if (!hostname || hostname === "localhost" || hostname.endsWith(".local") || hostname.endsWith(".internal")) {
+    throw new Error("资产 URL 不能指向本地或内部主机");
+  }
+  if (net.isIP(hostname) && privateIp(hostname)) throw new Error("资产 URL 不能指向私有地址");
+  return url;
+}
+
+function privateIp(hostname) {
+  if (net.isIPv4(hostname)) {
+    const [a, b] = hostname.split(".").map(Number);
+    return a === 0 || a === 10 || a === 127 || (a === 169 && b === 254)
+      || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168)
+      || a >= 224;
+  }
+  const normalized = String(hostname ?? "").toLowerCase();
+  return normalized === "::" || normalized === "::1" || normalized.startsWith("fc")
+    || normalized.startsWith("fd") || normalized.startsWith("fe8")
+    || normalized.startsWith("fe9") || normalized.startsWith("fea")
+    || normalized.startsWith("feb");
+}
+
+function reachable(url, response) {
+  return {
+    url: loggableUrl(url),
+    status: "reachable",
+    httpStatus: response.status,
+    contentLength: nonNegativeHeader(response.headers.get("content-length")),
+    contentType: response.headers.get("content-type"),
+  };
+}
+
+function httpFailure(url, response) {
+  const status = Number(response?.status);
+  const isRetryable = RETRYABLE_HTTP_STATUSES.has(status) || status >= 500;
+  const isTerminal = TERMINAL_HTTP_STATUSES.has(status)
+    || (status >= 400 && status < 500 && !isRetryable)
+    || (status >= 300 && status < 400);
+  return {
+    url: loggableUrl(url),
+    status: isTerminal ? "terminal" : "retryable",
+    httpStatus: status,
+    reasonCode: isTerminal
+      ? "admission.asset_transport_unavailable"
+      : "admission.asset_transport_transient",
+    ...retryAfter(response),
+  };
+}
+
+function networkOrSafetyFailure(url, error) {
+  const unsafe = error instanceof UnsafeAssetHostError;
+  return {
+    url: loggableUrl(url),
+    status: unsafe ? "terminal" : "retryable",
+    reasonCode: unsafe ? "admission.asset_url_unsafe" : "admission.asset_probe_network_error",
+    detail: error instanceof Error ? error.message : String(error),
+  };
+}
+
+function terminal(url, reasonCode, error, httpStatus) {
+  return {
+    url: loggableUrl(url),
+    status: "terminal",
+    ...(httpStatus === undefined ? {} : { httpStatus }),
+    reasonCode,
+    ...(error ? { detail: error instanceof Error ? error.message : String(error) } : {}),
+  };
+}
+
+function retryAfter(response) {
+  const value = response?.headers?.get?.("retry-after");
+  if (!value) return {};
+  const seconds = Number(value);
+  if (Number.isFinite(seconds) && seconds >= 0) {
+    return { retryAfterMs: Math.round(seconds * 1000) };
+  }
+  const timestamp = Date.parse(value);
+  if (!Number.isFinite(timestamp)) return {};
+  return { retryAfterMs: Math.max(0, timestamp - Date.now()) };
+}
+
+function retryDelayMs({
+  result,
+  attempt,
+  retryBaseDelayMs,
+  retryMaxDelayMs,
+  randomImpl,
+}) {
+  const base = Math.max(0, Number(retryBaseDelayMs) || 0) * 2 ** Math.max(0, attempt - 1);
+  const jitter = base * (0.75 + Math.max(0, Math.min(1, Number(randomImpl()) || 0)) * 0.5);
+  const requested = Math.max(jitter, Number(result?.retryAfterMs) || 0);
+  return Math.round(Math.min(Math.max(0, Number(retryMaxDelayMs) || 0), requested));
+}
+
+function emitProbeObservation(observe, result, attempt) {
+  if (typeof observe !== "function") return;
+  try {
+    for (const operation of attempt.operations) observe({
+      schemaVersion: "1.0",
+      kind: "external_request",
+      component: "public_asset_probe",
+      operation: operation.kind === "dns" ? "resolve_asset_host" : "request_public_asset",
+      service: operation.kind === "dns"
+        ? operation.hostname
+        : serviceName(operation.url),
+      request: operation.kind === "dns" ? {
+        attempt: attempt.attempt,
+        maxAttempts: attempt.maxAttempts,
+        method: "DNS_LOOKUP",
+        hostname: operation.hostname,
+      } : {
+        attempt: attempt.attempt,
+        maxAttempts: attempt.maxAttempts,
+        method: operation.method,
+        url: operation.url,
+      },
+      response: {
+        ...(operation.httpStatus === undefined ? {} : { status: operation.httpStatus }),
+        ...(operation.retryAfterMs === undefined ? {} : { retryAfterMs: operation.retryAfterMs }),
+        ...(operation.addressCount === undefined ? {} : { addressCount: operation.addressCount }),
+      },
+      outcome: {
+        status: operation.status === "failed" ? "failed" : "succeeded",
+        retryable: operation.status === "failed" && attempt.status === "retryable",
+        ...(operation.status === "failed" && attempt.reasonCode
+          ? { reasonCode: attempt.reasonCode }
+          : {}),
+      },
+      timing: {
+        startedAt: operation.startedAt,
+        finishedAt: operation.finishedAt,
+        durationMs: operation.durationMs,
+      },
+      details: {
+        finalProbeStatus: attempt.status,
+        ...(operation.addresses ? { addresses: operation.addresses } : {}),
+        ...(operation.detail ? { error: operation.detail } : {}),
+      },
+    });
+    observe({
+      schemaVersion: "1.0",
+      kind: "decision",
+      component: "public_asset_probe",
+      operation: "classify_probe_attempt",
+      service: serviceName(result?.probeUrl ?? result?.url),
+      outcome: {
+        status: attempt.status === "reachable" ? "succeeded" : "failed",
+        retryable: attempt.status === "retryable",
+        ...(attempt.reasonCode ? { reasonCode: attempt.reasonCode } : {}),
+      },
+      timing: {
+        startedAt: attempt.startedAt,
+        finishedAt: attempt.finishedAt,
+        durationMs: attempt.durationMs,
+      },
+      details: {
+        attempt: attempt.attempt,
+        maxAttempts: attempt.maxAttempts,
+        url: result?.url,
+        operationCount: attempt.operations.length,
+        ...(attempt.httpStatus === undefined ? {} : { httpStatus: attempt.httpStatus }),
+        ...(attempt.retryAfterMs === undefined ? {} : { retryAfterMs: attempt.retryAfterMs }),
+        ...(attempt.detail ? { error: attempt.detail } : {}),
+      },
+    });
+  } catch {
+    // Observability must never turn a successful transport probe into a failure.
+  }
+}
+
+function serviceName(value) {
+  try {
+    return new URL(String(value)).hostname;
+  } catch {
+    return "unknown";
+  }
+}
+
+function loggableUrl(value) {
+  try {
+    const url = value instanceof URL ? new URL(value) : new URL(String(value));
+    for (const key of [...url.searchParams.keys()]) {
+      if (SENSITIVE_QUERY_KEY.test(key)) url.searchParams.set(key, "[REDACTED]");
+    }
+    return url.toString();
+  } catch {
+    return String(value);
+  }
+}
+
+function positiveInteger(value, fallback, minimum, maximum) {
+  const number = Number(value);
+  if (!Number.isInteger(number)) return fallback;
+  return Math.max(minimum, Math.min(maximum, number));
+}
+
+function nonNegativeHeader(value) {
+  const number = Number(value);
+  return Number.isFinite(number) && number >= 0 ? number : null;
+}
+
+function nonNegativeInteger(value) {
+  const number = Number(value);
+  return Number.isSafeInteger(number) && number >= 0 ? number : null;
+}

package/src/assets/qualification.mjs

@@ -0,0 +1,119 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+  assessAutomaticReproductionAdmission,
+  assertNoTransientAdmissionFailures,
+} from "./reproduction-admission.mjs";
+import { loadResearchStructure } from "../research/structure.mjs";
+import { CiteArkError, readJson, writeJson } from "../util.mjs";
+import { validateAssetPreparationManifest } from "./manifest.mjs";
+import { assetPreparationPlanFailures, createAssetPreparationPlan } from "./plan.mjs";
+
+const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
+
+export const ASSET_QUALIFICATION_BOUNDARY = "asset_preparation";
+
+export async function loadAssetQualificationContext({
+  researchPath,
+  claimId,
+  experimentId,
+  datasetRegistryPath,
+}) {
+  const [research, datasetRegistry] = await Promise.all([
+    loadResearchStructure(researchPath),
+    readJson(datasetRegistryPath ?? path.join(PACKAGE_ROOT, "data", "dataset-source-registry.v1.json")),
+  ]);
+  const { claim, experiment } = selectAssetQualificationTarget({
+    research,
+    claimId,
+    experimentId,
+  });
+  return { research, datasetRegistry, claim, experiment };
+}
+
+export function selectAssetQualificationTarget({ research, claimId, experimentId }) {
+  const claims = Array.isArray(research?.claims) ? research.claims : [];
+  const experiments = Array.isArray(research?.experiments) ? research.experiments : [];
+  const experiment = experimentId
+    ? experiments.find((item) => item.id === experimentId)
+    : experiments.find((item) => Array.isArray(item.claimIds) && item.claimIds.some((id) => (
+        !claimId || id === claimId
+      )));
+  if (!experiment) throw new CiteArkError(`找不到可做资产验收的 experiment：${experimentId ?? "未指定"}`);
+  const claim = claimId
+    ? claims.find((item) => item.id === claimId)
+    : claims.find((item) => experiment.claimIds?.includes(item.id));
+  if (!claim) throw new CiteArkError(`实验 ${experiment.id} 没有可验收的 claim`);
+  if (!experiment.claimIds?.includes(claim.id)) {
+    throw new CiteArkError(`claim ${claim.id} 与 experiment ${experiment.id} 没有绑定`);
+  }
+  return { claim, experiment };
+}
+
+export function planAssetQualification({ experiment, datasetRegistry, limits, now }) {
+  const plan = createAssetPreparationPlan({ experiment, datasetRegistry, limits, now });
+  return {
+    plan,
+    failures: assetPreparationPlanFailures(plan),
+  };
+}
+
+export async function probeAssetQualification({
+  research,
+  claim,
+  experiment,
+  datasetRegistry,
+  outputPath,
+  ...probeOptions
+}) {
+  const admission = await assessAutomaticReproductionAdmission({
+    research,
+    targets: [{ claimId: claim.id, experimentId: experiment.id }],
+    datasetRegistry,
+    ...probeOptions,
+  });
+  if (outputPath) await writeJson(outputPath, admission);
+  return admission;
+}
+
+export function assertAssetQualificationAdmission(admission) {
+  assertNoTransientAdmissionFailures(admission);
+  if (admission?.blockedTargets?.length) {
+    const reasonCodes = admission.blockedTargets
+      .map((target) => target.reasonCode)
+      .filter(Boolean);
+    throw new CiteArkError(
+      `${admission.blockedTargets.length} 个目标未通过公开资产准入，未启动资产 Agent`,
+      {
+        exitCode: 2,
+        retryable: false,
+        failureCode: reasonCodes[0] ?? "admission.asset_qualification_blocked",
+        failureCategory: "admission",
+      },
+    );
+  }
+  if (admission?.admittedTargets?.length !== admission?.summary?.targetCount) {
+    throw new CiteArkError("资产准入结果不完整，未启动资产 Agent", {
+      exitCode: 2,
+      retryable: false,
+      failureCode: "admission.asset_qualification_incomplete",
+      failureCategory: "admission",
+    });
+  }
+  return admission;
+}
+
+export async function verifyAssetQualification({ planPath, manifestPath }) {
+  const [plan, manifest] = await Promise.all([
+    readJson(planPath),
+    readJson(manifestPath),
+  ]);
+  const issues = validateAssetPreparationManifest(plan, manifest);
+  return {
+    valid: issues.length === 0,
+    issues,
+    plan,
+    manifest,
+  };
+}