@citeark/agent 0.3.20

package/src/assets/source-grounding.mjs

@@ -0,0 +1,341 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+const MAX_SOURCE_TEXT_BYTES = 8 * 1024 * 1024;
+
+/**
+ * Mechanically bind public asset locators to the immutable sources available
+ * to Research Compiler. This is deliberately narrower than semantic identity:
+ * the preparation worker still has to verify bytes, format and the real loader.
+ */
+export async function assetSourceGroundingIssues(research, context = {}) {
+  const issues = [];
+  let fixedPaperText;
+  let fixedPaperIssue;
+  const readFixedPaper = async () => {
+    if (fixedPaperText !== undefined || fixedPaperIssue !== undefined) {
+      return { text: fixedPaperText, issue: fixedPaperIssue };
+    }
+    try {
+      fixedPaperText = await fixedPaperSourceText(context);
+      return { text: fixedPaperText };
+    } catch (error) {
+      fixedPaperIssue = error instanceof Error ? error.message : String(error);
+      return { issue: fixedPaperIssue };
+    }
+  };
+  const experiments = Array.isArray(research?.experiments) ? research.experiments : [];
+  for (const experiment of experiments) {
+    const preflight = experiment?.protocol?.preflight;
+    if (!new Set(["0.2", "0.3"]).has(preflight?.schemaVersion)) continue;
+    for (const asset of Array.isArray(preflight.assets) ? preflight.assets : []) {
+      if (asset?.availability !== "public_unverified") continue;
+      const label = `experiment ${experiment.id} asset ${asset?.id ?? "unknown"}`;
+      const evidence = Array.isArray(asset?.sourceEvidence) ? asset.sourceEvidence : [];
+      const groundedUrls = new Set();
+      let registryGrounded = false;
+      for (const entry of evidence) {
+        if (entry?.source === "repository") {
+          const result = await validateRepositoryEvidence(
+            entry,
+            context,
+            Array.isArray(asset?.sourceUrls) ? asset.sourceUrls : [],
+          );
+          if (result.issue) issues.push(`${label}: ${result.issue}`);
+          for (const url of result.urls ?? []) groundedUrls.add(comparableUrl(url));
+        } else if (entry?.source === "dataset_registry") {
+          const result = validateRegistryEvidence(entry, asset, context.datasetRegistry);
+          if (result.issue) issues.push(`${label}: ${result.issue}`);
+          registryGrounded ||= result.grounded;
+          for (const url of result.urls ?? []) groundedUrls.add(comparableUrl(url));
+        } else if (entry?.source === "paper") {
+          const result = await validatePaperEvidence(entry, context, readFixedPaper);
+          if (result.issue) issues.push(`${label}: ${result.issue}`);
+          if (result.url) groundedUrls.add(comparableUrl(result.url));
+        }
+      }
+      for (const url of Array.isArray(asset?.sourceUrls) ? asset.sourceUrls : []) {
+        if (!groundedUrls.has(comparableUrl(url))) {
+          issues.push(`${label}: 公开链接没有绑定到固定论文、固定仓库原文或数据注册表：${url}`);
+          continue;
+        }
+        for (const issue of assetTransportCompatibilityIssues(asset, url)) {
+          issues.push(`${label}: ${issue}`);
+        }
+      }
+      if (asset?.registryDatasetId && !registryGrounded) {
+        issues.push(`${label}: registryDatasetId 没有绑定到数据注册表记录`);
+      }
+    }
+  }
+  return [...new Set(issues)];
+}
+
+async function validatePaperEvidence(entry, context, readFixedPaper) {
+  if (!context.paperDigest) return { issue: "paper 来源没有绑定到固定论文摘要" };
+  const locator = typeof entry?.locator === "string" ? entry.locator.trim() : "";
+  if (!/^https?:\/\//i.test(locator)) {
+    return { issue: "paper 来源的 locator 必须是论文中出现的完整 HTTP(S) 链接" };
+  }
+  const paper = await readFixedPaper();
+  if (paper.issue) return { issue: `无法核验固定论文中的链接：${paper.issue}` };
+  if (!containsPossiblyWrappedUrl(paper.text, locator)) {
+    return { issue: "固定论文快照没有逐字包含定位链接" };
+  }
+  return { url: locator };
+}
+
+async function fixedPaperSourceText(context) {
+  if (typeof context.paperText === "string") {
+    return context.paperText.slice(0, MAX_SOURCE_TEXT_BYTES);
+  }
+  if (typeof context.paperPath !== "string" || !context.paperPath) {
+    throw new Error("缺少固定论文文件路径");
+  }
+  if (path.extname(context.paperPath).toLowerCase() === ".pdf") {
+    const result = await execFileAsync(
+      "pdftotext",
+      ["-nopgbrk", context.paperPath, "-"],
+      { encoding: "utf8", maxBuffer: MAX_SOURCE_TEXT_BYTES },
+    );
+    return result.stdout.slice(0, MAX_SOURCE_TEXT_BYTES);
+  }
+  return (await readFile(context.paperPath, "utf8")).slice(0, MAX_SOURCE_TEXT_BYTES);
+}
+
+function containsPossiblyWrappedUrl(text, url) {
+  if (text.includes(url)) return true;
+  const pattern = [...url]
+    .map((character) => character.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+    .join("\\s*");
+  return new RegExp(pattern, "i").test(text);
+}
+
+async function validateRepositoryEvidence(entry, context, assetSourceUrls = []) {
+  if (!context.repositoryIdentity?.available || !context.repositoryRoot) {
+    return { issue: "repository 来源不可用" };
+  }
+  if (entry.commit !== context.repositoryIdentity.commit) {
+    return { issue: "repository 来源没有绑定到固定 commit" };
+  }
+  if (!safeRelativePath(entry.path)) {
+    return { issue: "repository 来源路径无效" };
+  }
+  const root = path.resolve(context.repositoryRoot);
+  const absolute = path.resolve(root, entry.path);
+  if (!absolute.startsWith(`${root}${path.sep}`)) return { issue: "repository 来源路径逃逸" };
+  let contents;
+  try {
+    contents = await readFile(absolute, "utf8");
+  } catch {
+    return { issue: `repository 来源文件不存在：${entry.path}` };
+  }
+  if (contents.length > 4 * 1024 * 1024) return { issue: `repository 来源文件过大，无法作为链接证据：${entry.path}` };
+  const locator = typeof entry.locator === "string" ? entry.locator.trim() : "";
+  const resolver = entry.resolver ?? "exact_url";
+  if (resolver === "exact_url") {
+    if (/^https?:\/\//i.test(locator) && containsPossiblyWrappedUrl(contents, locator)) {
+      return { urls: [locator] };
+    }
+    const locatorMatchesFixedFile = fixedRepositoryFileLocator(entry, context, locator);
+    const candidates = assetSourceUrls.filter((url) =>
+      repositoryContentsGroundAssetUrl(contents, url));
+    if (
+      candidates.length
+      && (locatorMatchesFixedFile || candidates.some((url) => comparableUrl(url) === comparableUrl(locator)))
+    ) {
+      return { urls: candidates };
+    }
+    return { issue: `repository 来源文件无法逐字或确定性组合出资产定位链接：${entry.path}` };
+  }
+  if (resolver === "append_path") {
+    const suffix = typeof entry.suffix === "string" ? entry.suffix.trim() : "";
+    if (!/^https?:\/\//i.test(locator) || !containsPossiblyWrappedUrl(contents, locator)) {
+      return { issue: `repository 来源文件没有逐字包含 URL 基址：${entry.path}` };
+    }
+    if (!safeUrlPathSuffix(suffix) || !contents.includes(suffix)) {
+      return { issue: `repository 来源文件没有逐字包含安全的资产路径后缀：${entry.path}` };
+    }
+    return { urls: [joinPublicUrl(locator, suffix)] };
+  }
+  if (resolver === "huggingface_model" || resolver === "huggingface_dataset") {
+    if (!safeRepositoryIdentifier(locator) || !contents.includes(locator)) {
+      return { issue: `repository 来源文件没有逐字包含 Hugging Face 仓库标识：${entry.path}` };
+    }
+    const prefix = resolver === "huggingface_dataset"
+      ? "https://huggingface.co/datasets/"
+      : "https://huggingface.co/";
+    return { urls: [`${prefix}${locator}`] };
+  }
+  return { issue: `repository 来源 resolver 无效：${resolver}` };
+}
+
+/**
+ * Reject transports that are reachable but cannot represent the declared
+ * asset. Full byte, format, split and loader verification remains in the
+ * separate asset-preparation stage.
+ */
+export function assetTransportCompatibilityIssues(asset, value) {
+  let url;
+  try {
+    url = new URL(String(value));
+  } catch {
+    return ["资产传输地址不是有效 URL"];
+  }
+  const kind = String(asset?.kind ?? "other");
+  const identity = `${asset?.id ?? ""} ${asset?.identity ?? ""}`;
+  const hostname = url.hostname.toLowerCase();
+  const parts = url.pathname.split("/").filter(Boolean);
+
+  if (hostname === "huggingface.co") {
+    const dataset = parts[0] === "datasets";
+    const offset = dataset ? 1 : 0;
+    const repositoryId = parts[offset] && parts[offset + 1]
+      ? `${parts[offset]}/${parts[offset + 1]}`
+      : "";
+    if (kind === "dataset" && !dataset) {
+      return ["dataset 资产必须指向 Hugging Face dataset，而不是 model 仓库"];
+    }
+    if (new Set(["checkpoint", "model"]).has(kind) && dataset) {
+      return [`${kind} 资产不能指向 Hugging Face dataset 仓库`];
+    }
+    if (repositoryId && !transportIdentityMatches(identity, repositoryId)) {
+      return [`Hugging Face 仓库 ${repositoryId} 与资产身份不匹配`];
+    }
+  }
+
+  if (hostname === "github.com" && parts.length === 2) {
+    const repositoryId = `${parts[0]}/${parts[1].replace(/\.git$/i, "")}`;
+    if (new Set(["checkpoint", "model"]).has(kind)) {
+      return [`${kind} 资产不能把 GitHub 源码仓库根地址当作模型字节来源`];
+    }
+    if (!transportIdentityMatches(identity, repositoryId)) {
+      return [`GitHub 仓库 ${repositoryId} 与资产身份不匹配`];
+    }
+  }
+
+  return [];
+}
+
+function validateRegistryEvidence(entry, asset, registry) {
+  const record = (Array.isArray(registry?.datasets) ? registry.datasets : [])
+    .find((candidate) => candidate?.id === asset?.registryDatasetId);
+  if (!record) return { issue: `数据注册表不存在 ${asset?.registryDatasetId ?? "对应记录"}`, grounded: false };
+  const urls = new Set([
+    record?.candidateTransport?.url,
+    ...(Array.isArray(record?.candidateTransport?.mirrors) ? record.candidateTransport.mirrors : []),
+  ].filter(Boolean));
+  const locator = typeof entry.locator === "string" ? entry.locator.trim() : "";
+  if (locator !== record.id && !urls.has(locator)) {
+    return { issue: "dataset_registry locator 与固定注册表记录不一致", grounded: false };
+  }
+  return {
+    grounded: true,
+    urls: locator === record.id ? [...urls] : [locator],
+  };
+}
+
+function repositoryContentsGroundAssetUrl(contents, value) {
+  if (typeof value !== "string" || !/^https?:\/\//i.test(value)) return false;
+  if (containsPossiblyWrappedUrl(contents, value)) return true;
+  let url;
+  try {
+    url = new URL(value);
+  } catch {
+    return false;
+  }
+  if (url.hostname.toLowerCase() !== "huggingface.co") return false;
+  const parts = url.pathname.split("/").filter(Boolean);
+  const offset = parts[0] === "datasets" ? 1 : 0;
+  const owner = parts[offset];
+  const name = parts[offset + 1];
+  if (!owner || !name) return false;
+  if (contents.includes(`${owner}/${name}`)) return true;
+  const base = parts[0] === "datasets"
+    ? `https://huggingface.co/datasets/${owner}`
+    : `https://huggingface.co/${owner}`;
+  return containsPossiblyWrappedUrl(contents, base) && contents.includes(name);
+}
+
+function fixedRepositoryFileLocator(entry, context, value) {
+  if (typeof value !== "string" || !/^https?:\/\//i.test(value)) return false;
+  let locator;
+  let repository;
+  try {
+    locator = new URL(value);
+    repository = new URL(context.repositoryIdentity?.url);
+  } catch {
+    return false;
+  }
+  if (locator.hostname.toLowerCase() !== "github.com" || repository.hostname.toLowerCase() !== "github.com") {
+    return false;
+  }
+  const repositoryParts = repository.pathname.replace(/\.git$/i, "").split("/").filter(Boolean);
+  const locatorParts = locator.pathname.split("/").filter(Boolean);
+  if (repositoryParts.length !== 2 || locatorParts.length < 5) return false;
+  const [owner, name] = repositoryParts;
+  const [locatorOwner, locatorName, marker, commit, ...fileParts] = locatorParts;
+  return owner === locatorOwner
+    && name === locatorName
+    && marker === "blob"
+    && commit === context.repositoryIdentity?.commit
+    && fileParts.join("/") === entry.path;
+}
+
+function safeRelativePath(value) {
+  return typeof value === "string"
+    && value.length > 0
+    && !path.posix.isAbsolute(value.replaceAll("\\", "/"))
+    && !value.replaceAll("\\", "/").split("/").includes("..")
+    && !/[\u0000-\u001f]/.test(value);
+}
+
+function safeRepositoryIdentifier(value) {
+  return typeof value === "string"
+    && /^[a-zA-Z0-9][a-zA-Z0-9._-]*\/[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(value);
+}
+
+function safeUrlPathSuffix(value) {
+  return typeof value === "string"
+    && value.length > 0
+    && value.length <= 500
+    && !value.startsWith("/")
+    && !value.split("/").includes("..")
+    && !/[?#\\\u0000-\u001f]/.test(value);
+}
+
+function joinPublicUrl(base, suffix) {
+  const normalizedBase = base.endsWith("/") ? base : `${base}/`;
+  return new URL(suffix, normalizedBase).toString();
+}
+
+function comparableUrl(value) {
+  try {
+    const url = new URL(String(value));
+    url.hash = "";
+    return url.toString().replace(/\/$/, "");
+  } catch {
+    return String(value);
+  }
+}
+
+function transportIdentityMatches(identity, repositoryId) {
+  const normalizedIdentity = normalizeIdentity(identity);
+  const [owner, name] = repositoryId.split("/");
+  const normalizedRepository = normalizeIdentity(repositoryId);
+  const normalizedName = normalizeIdentity(name);
+  if (normalizedRepository.length >= 5 && normalizedIdentity.includes(normalizedRepository)) return true;
+  if (normalizedName.length >= 4 && normalizedIdentity.includes(normalizedName)) return true;
+  const normalizedOwner = normalizeIdentity(owner);
+  return normalizedOwner.length >= 6
+    && normalizedName.length >= 3
+    && normalizedIdentity.includes(normalizedOwner)
+    && normalizedIdentity.includes(normalizedName);
+}
+
+function normalizeIdentity(value) {
+  return String(value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "");
+}

package/src/assets/source-policy.mjs

@@ -0,0 +1,118 @@
+const SOURCE_PRIORITIES = Object.freeze({
+  author_repository: 10,
+  paper_explicit: 20,
+  primary_archive: 30,
+  official_registry: 40,
+  research_plan: 50,
+  verified_mirror: 60,
+  unverified_mirror: 70,
+});
+
+export function rankedAssetLocators(asset, registry) {
+  const evidence = new Map((Array.isArray(asset?.sourceEvidence) ? asset.sourceEvidence : [])
+    .filter((entry) => typeof entry?.locator === "string")
+    .map((entry) => [entry.locator, entry.source]));
+  const candidates = [];
+  for (const url of Array.isArray(asset?.sourceUrls) ? asset.sourceUrls : []) {
+    const source = evidence.get(url);
+    candidates.push(locator(url, source === "repository"
+      ? "author_repository"
+      : source === "paper"
+        ? "paper_explicit"
+        : "research_plan"));
+  }
+  const primary = registry?.bridgeEvidence?.primaryUrl;
+  if (typeof primary === "string") {
+    candidates.push(locator(primary, "primary_archive", {
+      expectedDigest: digest(registry?.bridgeEvidence?.primaryArchiveSha256),
+    }));
+  }
+  const transport = registry?.candidateTransport;
+  if (typeof transport?.url === "string") {
+    candidates.push(locator(transport.url, "official_registry", {
+      expectedDigest: digest(transport.archiveSha256),
+      equivalence: registry?.confidence ?? null,
+    }));
+  }
+  for (const mirror of Array.isArray(transport?.mirrors) ? transport.mirrors : []) {
+    const value = typeof mirror === "string" ? { url: mirror } : mirror;
+    candidates.push(locator(value?.url, value?.sha256 ? "verified_mirror" : "unverified_mirror", {
+      expectedDigest: digest(value?.sha256),
+      fallbackRequiresDigestMatch: true,
+    }));
+  }
+  const deduplicated = new Map();
+  for (const candidate of candidates) {
+    if (!safeHttpUrl(candidate.url)) continue;
+    const previous = deduplicated.get(candidate.url);
+    if (
+      !previous
+      || candidate.priority < previous.priority
+      || (!previous.expectedDigest && candidate.expectedDigest)
+    ) {
+      deduplicated.set(candidate.url, {
+        ...previous,
+        ...candidate,
+        expectedDigest: candidate.expectedDigest ?? previous?.expectedDigest ?? null,
+        equivalence: candidate.equivalence ?? previous?.equivalence ?? null,
+        fallbackRequiresDigestMatch:
+          candidate.fallbackRequiresDigestMatch === true
+          || previous?.fallbackRequiresDigestMatch === true,
+      });
+    }
+  }
+  return [...deduplicated.values()]
+    .sort((left, right) => left.priority - right.priority)
+    .map((candidate, index) => ({ ...candidate, rank: index + 1 }));
+}
+
+export function validateSelectedSource(asset, acquisition) {
+  const issues = [];
+  const locators = Array.isArray(asset?.sourceLocators) ? asset.sourceLocators : [];
+  const selected = locators.find((candidate) => candidate.url === acquisition?.sourceLocator);
+  if (!selected) return issues;
+  if (selected.expectedDigest && acquisition?.sourceDigest !== selected.expectedDigest) {
+    issues.push(`资产 ${asset.id} 的来源摘要与计划不一致`);
+  }
+  if (selected.fallbackRequiresDigestMatch && !selected.expectedDigest) {
+    issues.push(`资产 ${asset.id} 使用镜像但计划没有可核验的等价摘要`);
+  }
+  const earlier = locators.filter((candidate) => candidate.rank < selected.rank);
+  if (earlier.length) {
+    const attempts = new Map((Array.isArray(acquisition?.attemptedSources)
+      ? acquisition.attemptedSources
+      : []).map((attempt) => [attempt?.url, attempt]));
+    for (const candidate of earlier) {
+      const attempt = attempts.get(candidate.url);
+      if (!attempt || attempt.status !== "failed") {
+        issues.push(`资产 ${asset.id} 使用后备来源前没有记录更高优先级来源失败：${candidate.url}`);
+      }
+    }
+  }
+  return issues;
+}
+
+function locator(url, sourceClass, extra = {}) {
+  return {
+    url,
+    source: sourceClass,
+    sourceClass,
+    priority: SOURCE_PRIORITIES[sourceClass],
+    ...extra,
+  };
+}
+
+function digest(value) {
+  if (typeof value !== "string" || !value) return null;
+  const normalized = value.startsWith("sha256:") ? value : `sha256:${value}`;
+  return /^sha256:[0-9a-f]{64}$/.test(normalized) ? normalized : null;
+}
+
+function safeHttpUrl(value) {
+  try {
+    const url = new URL(value);
+    return new Set(["http:", "https:"]).has(url.protocol) && !url.username && !url.password;
+  } catch {
+    return false;
+  }
+}