@citeark/agent 0.3.20

package/src/research/continuation-selection.mjs

@@ -0,0 +1,26 @@
+/** Explicit re-execution is distinct from a failed historical Execution. */
+export function continuationRepairTargets(continuation) {
+  const requested = Array.isArray(continuation?.requestedTargets) ? continuation.requestedTargets : [];
+  const candidates = requested.length ? requested : continuation?.failedTargets ?? [];
+  return candidates.filter(validTarget);
+}
+
+export function selectContinuationTargets(targets, continuation) {
+  const requested = Array.isArray(continuation?.requestedTargets) ? continuation.requestedTargets : [];
+  if (!requested.length) return targets;
+  const keys = new Set(requested.map(targetKey));
+  const selected = targets.filter((target) => keys.has(targetKey(target)));
+  if (selected.length !== keys.size) throw new Error('Requested continuation targets are missing from the immutable Research Plan');
+  return selected;
+}
+
+export function priorCompletedContinuationTargets(continuation) {
+  const requested = new Set((continuation?.requestedTargets ?? []).map(targetKey));
+  return (continuation?.completedTargets ?? []).filter((target) => !requested.has(targetKey(target)));
+}
+
+function validTarget(value) {
+  return value && typeof value.claimId === 'string' && value.claimId.trim()
+    && typeof value.experimentId === 'string' && value.experimentId.trim();
+}
+function targetKey(value) { return `${value?.claimId}\u0000${value?.experimentId}`; }

package/src/research/execution-graph-context.mjs

@@ -0,0 +1,43 @@
+import { sha256Value } from '../util.mjs';
+
+export function executionGraphContext(research, experiment) {
+  if (!research.researchObjects?.length && !experiment.steps?.length) return null;
+  const definitions = new Map((research.experiments ?? []).flatMap(item => item.steps ?? []).map(step => [step.id, step]));
+  const included = new Set(), steps = [];
+  const includeStep = step => {
+    if (!step || included.has(step.id)) return;
+    included.add(step.id);
+    for (const id of step.dependsOn ?? []) includeStep(definitions.get(id));
+    steps.push(structuredClone(step));
+  };
+  for (const step of experiment.steps ?? []) includeStep(step);
+  const ids = new Set([...(experiment.inputIds ?? []), ...steps.flatMap(step => [...(step.inputIds ?? []), ...(step.outputIds ?? [])])]);
+  const objects = new Map((research.researchObjects ?? []).map(object => [object.id, object]));
+  const include = id => {
+    for (const parent of objects.get(id)?.derivedFromIds ?? []) if (!ids.has(parent)) { ids.add(parent); include(parent); }
+  };
+  for (const id of [...ids]) include(id);
+  return { objects: [...ids].map(id => structuredClone(objects.get(id))), steps, inputIds: [...(experiment.inputIds ?? [])] };
+}
+
+export function mergeExecutionGraphContexts(contracts) {
+  const contexts = contracts.map(contract => contract.researchGraph).filter(Boolean);
+  if (!contexts.length) return null;
+  const objects = new Map(), steps = new Map(), inputs = new Set();
+  for (const context of contexts) {
+    for (const object of context.objects) {
+      if (objects.has(object.id) && sha256Value(objects.get(object.id)) !== sha256Value(object)) throw Error(`Conflicting research object ${object.id}`);
+      objects.set(object.id, object);
+    }
+    for (const step of context.steps) {
+      const { claimIds, ...definition } = step;
+      if (steps.has(step.id)) {
+        const { claimIds: priorClaims, ...prior } = steps.get(step.id);
+        if (sha256Value(prior) !== sha256Value(definition)) throw Error(`Conflicting scientific step ${step.id}`);
+        steps.set(step.id, { ...definition, claimIds: [...new Set([...(priorClaims ?? []), ...(claimIds ?? [])])] });
+      } else steps.set(step.id, structuredClone(step));
+    }
+    for (const id of context.inputIds) inputs.add(id);
+  }
+  return { objects: [...objects.values()], steps: [...steps.values()], inputIds: [...inputs] };
+}

package/src/research/experiment-importance.mjs

@@ -0,0 +1,15 @@
+/** Editorial planning metadata, never admission, budget or scientific evidence. */
+export function normalizeExperimentImportance(value, experimentIds) {
+  const seen = new Set();
+  return (Array.isArray(value) ? value : []).flatMap((item) => {
+    if (!item || !experimentIds.has(item.experimentId) || seen.has(item.experimentId)
+      || ![1, 2, 3].includes(item.stars) || typeof item.rationale !== "string"
+      || !item.rationale.trim()) return [];
+    seen.add(item.experimentId);
+    return [{ experimentId: item.experimentId, stars: item.stars,
+      rationale: item.rationale.trim(),
+      ...(typeof item.headline === "string" && item.headline.trim()
+        ? { headline: item.headline.trim().slice(0, 180) } : {}),
+    }];
+  });
+}

package/src/research/inventory-handoff.mjs

@@ -0,0 +1,104 @@
+import { mergeResearchObjects } from './research-objects.mjs';
+import path from "node:path";
+import { CiteArkError, readJson, writeJson, sha256Value } from "../util.mjs";
+import { mergeAgentUsage } from "../telemetry/usage.mjs";
+import { normalizeResearchStructure } from "./structure.mjs";
+import { revisedInventoryClaims } from "./inventory-revisions.mjs";
+
+export function mergeInventoryPreparation(inventory, draft) {
+  const source = normalizeResearchStructure(inventory);
+  if (source.compilationStage !== "inventory") throw new CiteArkError("Reproduction preparation requires a source inventory");
+  if (draft.reading !== undefined || draft.researchRelations !== undefined || draft.claims !== undefined || draft.hypotheses !== undefined || draft.work !== undefined || draft.sources !== undefined)
+    throw new CiteArkError("Reproduction preparation must return experiments and dispositions only; source content is host-owned");
+  if (!Array.isArray(draft.dispositions) || !Array.isArray(draft.experiments)) throw new CiteArkError("Preparation requires experiments and dispositions arrays");
+  const claims = normalizeResearchStructure({ ...source,
+    claims: revisedInventoryClaims(source, draft.inventoryRevisions).map((claim) => ({ ...claim, reproduction: { status: "deferred", reason: "Provisional source interpretation awaiting preparation." } })),
+  }).claims;
+  const ids = new Set(claims.map((claim) => claim.id));
+  const decisions = new Map();
+  for (const item of draft.dispositions) {
+    if (!ids.has(item?.claimId) || decisions.has(item.claimId) || !item.reproduction || typeof item.reproduction !== "object")
+      throw new CiteArkError("Invalid or duplicate inventory disposition");
+    decisions.set(item.claimId, item.reproduction);
+  }
+  for (const id of ids) if (!decisions.has(id)) decisions.set(id, {
+    status: "deferred", reason: "Initial preparation has not resolved this question; retain it for subsequent investigation and coverage assessment.",
+  });
+  const result = { ...structuredClone(source),
+    claims: claims.map((claim) => ({ ...claim, reproduction: structuredClone(decisions.get(claim.id)) })),
+    experiments: structuredClone(draft.experiments),
+    ...((source.researchObjects !== undefined || draft.researchObjects !== undefined)
+      ? { researchObjects: mergeResearchObjects(source.researchObjects, draft.researchObjects) } : {}),
+    ...(draft.experimentSelection && typeof draft.experimentSelection === "object"
+      ? { experimentSelection: structuredClone(draft.experimentSelection) }
+      : {}),
+    sourceInventory: { digest: `sha256:${sha256Value(source)}`, research: structuredClone(source),
+      ...(draft.inventoryRevisions?.length ? { revisions: structuredClone(draft.inventoryRevisions) } : {}) },
+  };
+  delete result.compilationStage;
+  delete result.coverage;
+  delete result.lineage;
+  delete result.provenance.compilerSourceReview;
+  return result;
+}
+
+/** Prepare using the original fixed sources and the caller's existing transport.
+ * Merely writes a task. The caller owns model execution, budget and checkpoints.
+ */
+export async function writeReproductionPreparationTask({ compilation, sourceTaskPath, outputPath, computeContext }) {
+  const inventory = normalizeResearchStructure(compilation.research);
+  if (inventory.compilationStage !== "inventory") throw new CiteArkError("Expected source inventory");
+  const original = await readJson(sourceTaskPath);
+  const runtimePaper = compilation.bundle.runtimeTask.paper;
+  if (!runtimePaper.path?.startsWith("/job/input/")) throw new CiteArkError("Preparation requires the fixed local paper snapshot");
+  const paperPath = path.join(compilation.bundle.directories.input, runtimePaper.path.slice("/job/input/".length));
+  const inventoryPath = path.join(path.dirname(outputPath), "source-inventory.json");
+  await writeJson(inventoryPath, inventory);
+  const repository = compilation.bundle.repositoryIdentity?.available
+    ? { path: compilation.bundle.directories.repositorySnapshot } : undefined;
+  const paper = { ...runtimePaper, path: paperPath,
+    ...(runtimePaper.reading?.status === "ready" ? { preparedReading: { ...runtimePaper.reading,
+      directory: path.join(compilation.bundle.directories.input, "paper-reading") } } : {}),
+  };
+  delete paper.url;
+  delete paper.markdownPath;
+  delete paper.sourceMapPath;
+  const task = { ...original, name: `${original.name}-reproduction-preparation`, compilationStage: "execution_plan",
+    paper, ...(computeContext ? { computeContext } : {}), ...(repository ? { repository } : {}), sourceInventory: { path: inventoryPath, digest: `sha256:${sha256Value(inventory)}` } };
+  await writeJson(outputPath, task);
+  return outputPath;
+}
+
+export async function prepareInventoryForExecution({ compilation, sourceTaskPath, outputDirectory, execute, computeContext, progress = () => {} }) {
+  if (compilation.research?.compilationStage !== "inventory") return compilation;
+  progress("论文研究清单已完成；复现准备开始读取清单与固定来源，选择验证路线和资源。");
+  const taskPath = await writeReproductionPreparationTask({ compilation, sourceTaskPath,
+    outputPath: path.join(outputDirectory, "prepare-reproduction-task.json"), computeContext });
+  const preparation = await execute({ taskPath, runId: `${compilation.bundle.runId}-prepare` });
+  return { ...preparation,
+    inventoryCompilation: { runId: compilation.bundle.runId, outputPath: path.join(compilation.bundle.directories.output, "research.normalized.json"),
+      startedAt: compilation.bundle.state.startedAt, finishedAt: compilation.bundle.state.finishedAt, usage: compilation.bundle.state.usage },
+    totalUsage: mergeAgentUsage(compilation.bundle.state.usage, preparation.bundle.state.usage),
+  };
+}
+
+/** Standalone inventories can enter the pipeline with their original sources. */
+export async function writePreparationTaskFromInventory({ inventoryPath, paperPath, repositoryPath, outputPath, computeContext }) {
+  const inventory = normalizeResearchStructure(await readJson(inventoryPath));
+  if (inventory.compilationStage !== "inventory") throw new CiteArkError("Expected a research inventory");
+  const paperUri = inventory.sources.find((source) => source.kind === "paper")?.uri;
+  const paper = paperPath ? { path: path.resolve(paperPath) }
+    : /^https?:\/\//.test(paperUri ?? "") ? { url: paperUri } : null;
+  if (!paper) throw new CiteArkError("研究清单需要原论文文件；请为复现准备提供 --paper");
+  const identity = inventory.provenance.sourceIdentity?.repository;
+  const repository = identity?.available ? repositoryPath ? { path: path.resolve(repositoryPath) }
+    : /^https?:\/\//.test(identity.url ?? "") ? { url: identity.url, commit: identity.commit } : null : undefined;
+  if (identity?.available && !repository) throw new CiteArkError("研究清单绑定本地仓库快照；请提供相同内容的 --repository");
+  const task = { schemaVersion: "0.1", name: `${inventory.work.id}-reproduction-preparation`, compilationStage: "execution_plan",
+    reproductionScope: inventory.reproductionScope, paper, ...(repository ? { repository } : {}),
+    ...(computeContext ? { computeContext } : {}),
+    licensePolicy: { executable: inventory.license.executionAllowed, paperLicense: inventory.license.paper, codeLicense: inventory.license.code },
+    sourceInventory: { path: path.resolve(inventoryPath), digest: `sha256:${sha256Value(inventory)}` } };
+  await writeJson(outputPath, task);
+  return outputPath;
+}

package/src/research/inventory-revisions.mjs

@@ -0,0 +1,32 @@
+import { CiteArkError } from "../util.mjs";
+
+export const sourceClaim = ({ reproduction, versionId, ...claim }) => claim;
+
+/** The fixed paper is the source; a model's reading of it is revisable.
+ * Retain the initial extraction and all identities so corrections remain visible.
+ */
+export function revisedInventoryClaims(inventory, revisions = []) {
+  if (!Array.isArray(revisions)) throw new CiteArkError("inventoryRevisions must be an array");
+  const claims = inventory.claims.map((claim) => structuredClone(sourceClaim(claim)));
+  const seen = new Set();
+  for (const revision of revisions) {
+    const { claimId, claim, reason, sources } = revision ?? {};
+    if (!claimId || claim?.id !== claimId || seen.has(claimId)
+      || typeof reason !== "string" || !reason.trim()
+      || !Array.isArray(sources) || !sources.length
+      || sources.some((source) => !inventory.sources.some((item) => item.id === source?.sourceId)
+        || typeof source.locator !== "string" || !source.locator.trim())) {
+      throw new CiteArkError("Inventory revision requires a unique claim ID, corrected claim, reason and fixed-source locators");
+    }
+    seen.add(claimId);
+    const index = claims.findIndex((item) => item.id === claimId);
+    if (index >= 0) {
+      const ids = new Set((claim.reportedMeasurements ?? []).map((item) => item.id));
+      if ((claims[index].reportedMeasurements ?? []).some((item) => !ids.has(item.id))) {
+        throw new CiteArkError("Inventory correction must retain existing measurement IDs and disclose mistaken interpretations");
+      }
+      claims[index] = structuredClone(sourceClaim(claim));
+    } else claims.push(structuredClone(sourceClaim(claim)));
+  }
+  return claims;
+}

package/src/research/mineru-local.mjs

@@ -0,0 +1,73 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { researchSignal } from '../workspace/control.mjs';
+
+export const MINERU_RUNTIME_DIRECTORY = fileURLToPath(new URL('../../runtime/mineru/', import.meta.url));
+export const mineruHome = () => path.resolve(process.env.CITEARK_MINERU_HOME ?? path.join(os.homedir(), '.citeark', 'mineru-4.0.10'));
+export const mineruPython = () => process.env.CITEARK_MINERU_PYTHON ?? path.join(mineruHome(), 'venv', process.platform === 'win32' ? 'Scripts/python.exe' : 'bin/python');
+
+async function run(command, args, { env = process.env, timeoutMs = 3_600_000, signal } = {}) {
+  signal?.throwIfAborted();
+  return await new Promise((resolve, reject) => {
+    const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, detached: process.platform !== 'win32' });
+    // Keep progress visible without allowing large buffered native-model logs.
+    child.stderr.on('data', chunk => process.stderr.write(chunk));
+    child.stdout.on('data', chunk => process.stderr.write(chunk));
+    let timedOut = false;
+    const stop = () => {
+      if (process.platform === 'win32') child.kill('SIGKILL');
+      else { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* Process already exited. */ } }
+    };
+    const timer = setTimeout(() => { timedOut = true; stop(); }, timeoutMs);
+    signal?.addEventListener('abort', stop, { once: true });
+    const cleanup = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', stop);
+    };
+    child.once('error', error => { cleanup(); reject(error); });
+    child.once('close', code => {
+      cleanup();
+      if (signal?.aborted) { reject(signal.reason); return; }
+      code === 0 ? resolve() : reject(new Error(timedOut ? 'MinerU conversion timed out' : 'Local MinerU conversion failed'));
+    });
+  });
+}
+
+export async function setupMineru({ python = process.platform === 'win32' ? 'python' : 'python3.12', source = 'huggingface', runImpl = run } = {}) {
+  if (!['huggingface', 'modelscope'].includes(source)) throw Error('Model source must be huggingface or modelscope');
+  const home = mineruHome();
+  await mkdir(home, { recursive: true });
+  const config = path.join(home, 'citeark-config.json');
+  await writeFile(config, '{}');
+  const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('MINERU_') && key !== 'HF_HUB_OFFLINE'));
+  Object.assign(env, { MINERU_HOME: home, MINERU_CONFIG: config, HF_HUB_DISABLE_XET: '1', DO_NOT_TRACK: '1',
+    HF_HOME: path.join(home, 'hf-cache'), HF_HUB_CACHE: path.join(home, 'hf-cache', 'hub') });
+  if (!process.env.CITEARK_MINERU_PYTHON) await runImpl(python, ['-m', 'venv', path.join(home, 'venv')], { env });
+  const executable = mineruPython();
+  await runImpl(executable, ['-m', 'pip', 'install', '-r', path.join(MINERU_RUNTIME_DIRECTORY, 'requirements.txt')], { env });
+  const kit = path.join(path.dirname(executable), process.platform === 'win32' ? 'mineru-kit.exe' : 'mineru-kit');
+  const backends = ['--tier', 'standard', '--small-backend', 'onnx', '--vlm-engine', 'llama-cpp'];
+  await runImpl(kit, ['models', 'download', ...backends, '--source', source], { env });
+  await runImpl(kit, ['models', 'verify', ...backends], { env });
+  return { status: 'ready', home, python: executable, mineruVersion: '4.0.10', gpu: 0 };
+}
+
+export async function convertPaperWithMinerU({ bytes, sourceDigest, onProgress = async () => {}, runImpl = run, signal = researchSignal() }) {
+  signal?.throwIfAborted();
+  const python = mineruPython();
+  try { await access(python); } catch { throw Error('Local MinerU is not installed; run citeark paper setup --python python3.12'); }
+  const directory = await mkdtemp(path.join(os.tmpdir(), 'citeark-mineru-'));
+  try {
+    const input = path.join(directory, 'paper.pdf'), output = path.join(directory, 'paper.zip');
+    await writeFile(input, bytes);
+    await onProgress({ stage: 'local_cpu_parse', providerState: 'running' });
+    await runImpl(python, [path.join(MINERU_RUNTIME_DIRECTORY, 'parse.py'), '--input', input, '--output', output,
+      '--source-digest', sourceDigest, '--threads', process.env.CITEARK_MINERU_THREADS ?? '4'], { signal });
+    signal?.throwIfAborted();
+    await onProgress({ providerState: 'done' });
+    return { archive: await readFile(output), batchId: 'local-cpu' };
+  } finally { await rm(directory, { recursive: true, force: true }); }
+}

package/src/research/paper-command.mjs

@@ -0,0 +1,19 @@
+import path from 'node:path';
+import { setupMineru } from './mineru-local.mjs';
+import { preparePaperMarkdown } from './paper-markdown.mjs';
+
+export async function paperCommand([command, ...rest], parseOptions) {
+  if (command === 'setup') {
+    const options = parseOptions(rest, { '--python': { type: 'string' }, '--source': { type: 'string', default: 'huggingface' } });
+    console.log(JSON.stringify(await setupMineru({ python: options['--python'], source: options['--source'] }), null, 2));
+    return;
+  }
+  if (command === 'parse') {
+    const options = parseOptions(rest, { '--input': { type: 'string', required: true }, '--output': { type: 'string', required: true }, '--cache': { type: 'string' } });
+    const outputDirectory = path.resolve(options['--output']);
+    console.log(JSON.stringify(await preparePaperMarkdown({ paperPath: path.resolve(options['--input']), outputDirectory,
+      cacheDirectory: path.resolve(options['--cache'] ?? path.join(path.dirname(outputDirectory), '.paper-markdown-cache')) }), null, 2));
+    return;
+  }
+  throw Error('Usage: citeark paper setup [--python python3.12] [--source huggingface|modelscope] | citeark paper parse --input paper.pdf --output ./paper-reading');
+}

package/src/research/paper-markdown.mjs

@@ -0,0 +1,180 @@
+import { createHash, randomUUID } from "node:crypto";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { appendFile, cp, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { convertPaperWithMinerU } from "./mineru-local.mjs";
+export { convertPaperWithMinerU } from "./mineru-local.mjs";
+import { mappedMineru4Markdown } from "./paper-source-map.mjs";
+import { researchSignal } from "../workspace/control.mjs";
+
+const exec = promisify(execFile);
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+export const PAPER_MARKDOWN_CONFIG = Object.freeze({ version: 3, provider: "mineru-local", mineruVersion: "4.0.10", llamaVersion: "0.1.2", docvortexVersion: "0.5.8", onnxVersion: "1.30.0", tier: "standard", smallBackend: "onnx", vlmEngine: "llama-cpp", device: "cpu", ocrMode: "auto", imageAnalysis: true });
+const safePath = (name) => typeof name === "string" && name.length > 0 && !name.startsWith("/") && !name.includes("\\") && !name.split("/").some((part) => !part || part === "." || part === "..");
+
+const networkCodes = new Set(["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EAI_AGAIN", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "ABORT_ERR"]);
+const controlledFailureCodes = new Map([
+  ["Paper conversion response exceeds size limit", "response_too_large"],
+  ["Local MinerU conversion failed", "mineru_local_process_failed"],
+  ["Local MinerU is not installed; run citeark paper setup --python python3.12", "mineru_runtime_missing"],
+  ["MinerU conversion failed", "mineru_conversion_failed"],
+  ["MinerU conversion timed out", "mineru_conversion_timed_out"],
+  ["Invalid MinerU archive paths", "mineru_archive_paths_invalid"],
+  ["MinerU archive must contain exactly one markdown.md", "mineru_markdown_missing"],
+  ["MinerU reading assets exceed size limit", "mineru_reading_assets_too_large"],
+  ["MinerU archive must contain one content list for page mapping", "mineru_content_list_missing"],
+  ["Mapped paper Markdown exceeds reading limit", "mineru_mapped_markdown_too_large"],
+  ["MinerU Markdown is empty", "mineru_markdown_empty"],
+  ["MinerU Markdown references an unavailable image", "mineru_image_reference_missing"],
+]);
+
+function safeFailure(error, stage) {
+  const code = error?.cause?.code ?? error?.code;
+  const message = error instanceof Error ? error.message : "";
+  const failureCode = controlledFailureCodes.get(message)
+    ?? (error instanceof SyntaxError
+      ? (stage === "unpack" ? "mineru_archive_json_invalid" : "mineru_result_json_invalid")
+      : undefined)
+    ?? (networkCodes.has(code) ? "mineru_network_error" : undefined)
+    ?? (stage === "unpack" ? "mineru_archive_unpack_failed" : "paper_markdown_preparation_failed");
+  const timeout = error?.name === "TimeoutError"
+    || error?.cause?.name === "TimeoutError"
+    || code === "ABORT_ERR";
+  return {
+    failureCode,
+    ...(networkCodes.has(code) ? { networkCode: code } : {}),
+    ...(timeout ? { timeout: true } : {}),
+  };
+}
+
+function logSafeFailure(failure) {
+  console.error(JSON.stringify({
+    severity: "ERROR",
+    event: "paper_markdown.failed",
+    provider: "mineru",
+    ...failure,
+  }));
+}
+
+/** Extract just reading assets. Original ZIP, including raw JSON, stays backend-only. */
+export async function unpackPaperMarkdown(archivePath, outputDirectory) {
+  const { stdout } = await exec("unzip", ["-Z1", archivePath], { maxBuffer: 2_000_000 });
+  const members = stdout.trim().split(/\r?\n/).filter((name) => !name.endsWith("/"));
+  if (members.length > 20_000 || members.some((name) => !safePath(name)) || new Set(members).size !== members.length) throw new Error("Invalid MinerU archive paths");
+  const markdowns = members.filter((name) => path.posix.basename(name) === "markdown.md");
+  if (markdowns.length !== 1) throw new Error("MinerU archive must contain exactly one markdown.md");
+  const root = path.posix.dirname(markdowns[0]);
+  const prefix = root === "." ? "" : `${root}/`;
+  const selected = members.filter((name) => name === markdowns[0] || (name.startsWith(`${prefix}images/`) && /\.(png|jpe?g|webp)$/i.test(name)));
+  let total = 0;
+  const files = [];
+  for (const member of selected) {
+    // -p writes bytes only, so even ZIP symlinks cannot create filesystem links.
+    const { stdout: bytes } = await exec("unzip", ["-p", archivePath, member], { encoding: "buffer", maxBuffer: member === markdowns[0] ? 2_000_000 : 20_000_000 });
+    total += bytes.length;
+    if (total > 160_000_000) throw new Error("MinerU reading assets exceed size limit");
+    const relative = member === markdowns[0] ? "paper.md" : member.slice(prefix.length);
+    await mkdir(path.dirname(path.join(outputDirectory, relative)), { recursive: true });
+    await writeFile(path.join(outputDirectory, relative), bytes);
+    files.push({ path: relative, digest: digest(bytes), size: bytes.length });
+  }
+  const contentLists = members.filter((name) => name.startsWith(prefix) && name === `${prefix}structured_content.json`);
+  if (contentLists.length !== 1) throw new Error("MinerU archive must contain one content list for page mapping");
+  const { stdout: rawBlocks } = await exec("unzip", ["-p", archivePath, contentLists[0]], { maxBuffer: 20_000_000 });
+  const mapped = mappedMineru4Markdown(JSON.parse(rawBlocks));
+  if (Buffer.byteLength(mapped.markdown) > 2_000_000) throw new Error("Mapped paper Markdown exceeds reading limit");
+  for (const [relative, content] of [["paper.md", mapped.markdown], ["source-map.json", JSON.stringify(mapped.sourceMap)]]) {
+    await writeFile(path.join(outputDirectory, relative), content);
+    const oldIndex = files.findIndex((file) => file.path === relative);
+    if (oldIndex >= 0) files.splice(oldIndex, 1);
+    files.push({ path: relative, digest: digest(content), size: Buffer.byteLength(content) });
+  }
+  const markdown = mapped.markdown;
+  if (!markdown.trim()) throw new Error("MinerU Markdown is empty");
+  const imagePaths = new Set(files.map((file) => file.path));
+  for (const match of markdown.matchAll(/!\[[^\]]*\]\(<?([^\s)>]+)>?(?:\s+[^)]*)?\)|<img\b[^>]*\bsrc=["']([^"']+)["']/g)) {
+    const reference = (match[1] ?? match[2]).replace(/^\.\//, "");
+    if (!safePath(reference) || !imagePaths.has(reference)) throw new Error("MinerU Markdown references an unavailable image");
+  }
+  return files;
+}
+
+async function readCache(directory, sourceDigest, configDigest) {
+  try {
+    const manifest = JSON.parse(await readFile(path.join(directory, "manifest.json"), "utf8"));
+    if (manifest.sourceDigest !== sourceDigest || manifest.configDigest !== configDigest || !manifest.files?.some((file) => file.path === "paper.md")) return null;
+    for (const file of manifest.files) {
+      if (!safePath(file.path) || (!["paper.md", "source-map.json"].includes(file.path) && !file.path.startsWith("images/"))) return null;
+      const bytes = await readFile(path.join(directory, "reading", file.path));
+      if (bytes.length !== file.size || digest(bytes) !== file.digest) return null;
+    }
+    return manifest;
+  } catch { return null; }
+}
+
+export async function preparePaperMarkdown({ paperPath, outputDirectory, cacheDirectory, convert = convertPaperWithMinerU, unpack = unpackPaperMarkdown, logFailure = logSafeFailure, signal = researchSignal() }) {
+  signal?.throwIfAborted();
+  const bytes = await readFile(paperPath);
+  if (bytes.subarray(0, 5).toString() !== "%PDF-") return { status: "not_pdf" };
+  const sourceDigest = digest(bytes);
+  const configDigest = digest(JSON.stringify(PAPER_MARKDOWN_CONFIG));
+  const cache = path.join(cacheDirectory, `${sourceDigest}-${configDigest}`);
+  let manifest = await readCache(cache, sourceDigest, configDigest);
+  const cacheHit = Boolean(manifest);
+  if (!manifest) {
+    if (bytes.length > 80_000_000) throw new Error("Paper exceeds conversion input size limit");
+    await mkdir(cacheDirectory, { recursive: true });
+    // Remove an invalid entry before conversion so it cannot block atomic publication.
+    await rm(cache, { recursive: true, force: true });
+    const attempt = `${cache}.attempt-${randomUUID()}`;
+    await mkdir(path.join(attempt, "reading"), { recursive: true });
+    let diagnostic = { stage: "convert" };
+    const onProgress = async (event) => {
+      if (event.stage) {
+        const { batchId } = diagnostic;
+        diagnostic = { ...(batchId ? { batchId } : {}) };
+      }
+      diagnostic = { ...diagnostic, ...event, recordedAt: new Date().toISOString() };
+      await appendFile(path.join(attempt, "progress.jsonl"), JSON.stringify(diagnostic) + "\n");
+    };
+    try {
+      const converted = await convert({ bytes, sourceDigest, onProgress, signal });
+      signal?.throwIfAborted();
+      await onProgress({ stage: "archive_write", batchId: converted.batchId });
+      await writeFile(path.join(attempt, "source.zip"), converted.archive);
+      await onProgress({ stage: "unpack" });
+      const files = await unpack(path.join(attempt, "source.zip"), path.join(attempt, "reading"));
+      manifest = { sourceDigest, configDigest, config: PAPER_MARKDOWN_CONFIG, batchId: converted.batchId, files, archiveDigest: digest(converted.archive) };
+      await onProgress({ stage: "cache_publish" });
+      await writeFile(path.join(attempt, "manifest.json"), JSON.stringify(manifest, null, 2));
+      try { await rename(attempt, cache); }
+      catch (error) {
+        // A concurrent conversion may have completed first. Never overwrite it.
+        const winner = await readCache(cache, sourceDigest, configDigest);
+        if (!winner) throw error;
+        manifest = winner;
+      }
+    } catch (error) {
+      if (signal?.aborted) throw signal.reason;
+      // Provider response bodies and signed URLs must not appear in persistent diagnostics.
+      const failure = {
+        status: "failed",
+        sourceDigest,
+        configDigest,
+        ...diagnostic,
+        ...safeFailure(error, diagnostic.stage),
+      };
+      await writeFile(path.join(attempt, "failure.json"), JSON.stringify(failure));
+      try { await logFailure(failure); } catch { /* Logging must not mask the conversion failure. */ }
+      throw new Error(error.message?.startsWith("Local MinerU is not installed;") ? error.message : "Paper Markdown preparation failed; conversion attempt retained in backend cache");
+    }
+  }
+  await mkdir(outputDirectory, { recursive: true });
+  for (const file of manifest.files) {
+    const destination = path.join(outputDirectory, file.path);
+    await mkdir(path.dirname(destination), { recursive: true });
+    await cp(path.join(cache, "reading", file.path), destination);
+  }
+  return { status: "ready", sourceDigest, configDigest, cacheHit, files: manifest.files };
+}

package/src/research/paper-source-map.mjs

@@ -0,0 +1,69 @@
+/** The v4 public format has normalized boxes and nested captions/footnotes. */
+export function mappedMineru4Markdown(document) {
+  if (!Array.isArray(document?.pages) || !document.pages.length || document.is_full_document !== true)
+    throw Error('Missing complete MinerU document pages');
+  const entries = [];
+  let markdown = '# Paper reading copy\n\nRead Markdown first. Markers name physical PDF pages starting at 1. For uncertain text, equations, captions or values, inspect that page in the original PDF; a caption may continue on an adjacent page.\n\n';
+  const location = (bbox) => Array.isArray(bbox) && bbox.length === 4
+    && bbox.every(value => Number.isFinite(value) && value >= 0 && value <= 1)
+    && bbox[0] <= bbox[2] && bbox[1] <= bbox[3];
+  function append(block, page, parentId) {
+    if (!location(block.bbox) || typeof block.type !== 'string' || typeof block.content !== 'string')
+      throw Error('MinerU block lacks valid content or page location');
+    const id = `p${page}-b${entries.length + 1}`;
+    const parts = [];
+    if (block.image_source) parts.push(`![](${block.image_source})`);
+    if (block.content) {
+      const content = block.type === 'equation' ? `$$\n${block.content}\n$$`
+        : ['doc_title', 'paragraph_title'].includes(block.type)
+          ? `${'#'.repeat(Math.min(6, Math.max(1, Number(block.level) || 1)))} ${block.content}` : block.content;
+      parts.push(content);
+    }
+    const startLine = markdown.split('\n').length;
+    markdown += `<!-- PDF page ${page} | block ${id} -->\n${parts.join('\n\n') || '[No text extracted; inspect original PDF page.]'}\n\n`;
+    entries.push({ id, page, bbox: block.bbox, type: block.type, ...(parentId ? { parentId } : {}), startLine, endLine: markdown.split('\n').length - 2 });
+    for (const [field, type] of [['captions', 'caption'], ['footnotes', 'footnote']]) {
+      if (block[field] !== undefined && !Array.isArray(block[field])) throw Error('Invalid MinerU nested content');
+      for (const child of block[field] ?? []) append({ ...child, type }, page, id);
+    }
+  }
+  for (const [index, page] of document.pages.entries()) {
+    if (page.page_idx !== index || !Array.isArray(page.blocks)) throw Error('MinerU pages are missing or unordered');
+    if (!page.blocks.length) markdown += `<!-- PDF page ${index + 1} | no extracted blocks -->\n\n`;
+    for (const block of page.blocks) append(block, index + 1);
+  }
+  return { markdown, sourceMap: { version: 2, pageNumbering: 'physical_pdf_one_based',
+    bboxConvention: 'normalized_0_1', pageCount: document.pages.length, blocks: entries } };
+}
+
+/** Legacy serializer retained for previously saved archive inspection. */
+export function mappedPaperMarkdown(blocks) {
+  if (!Array.isArray(blocks) || !blocks.length) throw new Error("Missing MinerU content blocks");
+  const entries = [];
+  let markdown = "# Paper reading copy\n\nRead Markdown first. Markers name physical PDF pages starting at 1, not printed page labels. If unclear, open that page in the original PDF. Boxes are navigation hints; a merged paragraph or caption may continue onto an adjacent page.\n\n";
+  for (const [index, block] of blocks.entries()) {
+    if (!Number.isInteger(block.page_idx) || block.page_idx < 0 || !Array.isArray(block.bbox) || block.bbox.length !== 4 || !block.bbox.every(Number.isFinite)) throw new Error("MinerU block lacks a valid page location");
+    const page = block.page_idx + 1;
+    const id = `p${page}-b${index + 1}`;
+    const parts = [];
+    if (typeof block.text === "string") {
+      const heading = block.type === "text" && Number.isInteger(block.text_level) ? `${"#".repeat(Math.min(6, Math.max(1, block.text_level)))} ` : "";
+      parts.push(heading + block.text);
+    } else if (block.type === "list" && Array.isArray(block.list_items) && block.list_items.every((item) => typeof item === "string")) {
+      parts.push(...block.list_items);
+    } else if (["table", "image", "chart", "code"].includes(block.type)) {
+      if (block.img_path) parts.push(`![](${block.img_path})`);
+      for (const suffix of ["body", "caption", "footnote"]) {
+        const value = block[`${block.type}_${suffix}`];
+        if (typeof value === "string") parts.push(value);
+        else if (Array.isArray(value) && value.every((item) => typeof item === "string")) parts.push(...value);
+      }
+      if (typeof block.content === "string" && block.content) parts.push(block.content);
+    } else throw new Error(`Unsupported MinerU block type: ${block.type}`);
+    const content = parts.filter(Boolean).join("\n\n") || "[No text extracted for this block; consult the marked PDF page if needed.]";
+    const startLine = markdown.split("\n").length;
+    markdown += `<!-- PDF page ${page} | block ${id} -->\n${content}\n\n`;
+    entries.push({ id, page, bbox: block.bbox, type: block.type, startLine, endLine: markdown.split("\n").length - 2 });
+  }
+  return { markdown, sourceMap: { version: 1, pageNumbering: "physical_pdf_one_based", bboxConvention: "mineru_content_list_0_1000", blocks: entries } };
+}