@luffysolution/pi-scholar 0.1.0

package/src/mineru.ts

@@ -0,0 +1,281 @@
+import { createHash, randomUUID } from "node:crypto";
+import { stat } from "node:fs/promises";
+import { readFileBounded, readResponseBytes } from "./io.js";
+export { readResponseBytes } from "./io.js";
+import type { MinerUInfo } from "./model.js";
+
+const API_BASE = "https://mineru.net";
+const MAX_PDF_BYTES = 200 * 1024 * 1024;
+const MAX_ARCHIVE_BYTES = 512 * 1024 * 1024;
+const MAX_HTTP_ATTEMPTS = 4;
+
+export interface MinerUOptions {
+  token: string;
+  /** Overall polling window. Individual HTTP attempts are capped at 60 seconds. */
+  timeoutMs?: number;
+  initialPollMs?: number;
+  maxPollMs?: number;
+  maxAttempts?: number;
+  language?: string;
+  enableFormula?: boolean;
+  enableTable?: boolean;
+  isOcr?: boolean;
+  modelVersion?: string;
+}
+
+export interface MinerURun {
+  archive: Uint8Array;
+  info: MinerUInfo;
+  pdfSha256: string;
+}
+
+export type Progress = (stage: string) => void;
+type Sleeper = (ms: number, signal?: AbortSignal) => Promise<void>;
+
+const delay: Sleeper = (ms, signal) => new Promise<void>((resolve, reject) => {
+  if (signal?.aborted) {
+    reject(signal.reason ?? new Error("MinerU operation cancelled"));
+    return;
+  }
+  const onAbort = () => {
+    clearTimeout(timer);
+    reject(signal?.reason ?? new Error("MinerU operation cancelled"));
+  };
+  const timer = setTimeout(() => {
+    signal?.removeEventListener("abort", onAbort);
+    resolve();
+  }, ms);
+  signal?.addEventListener("abort", onAbort, { once: true });
+});
+
+const stringOrNull = (value: unknown): string | null =>
+  typeof value === "string" && value.length > 0 ? value : null;
+
+function secretSafe(error: unknown, secrets: string[] = []): string {
+  let message = error instanceof Error ? error.message : String(error);
+  message = message
+    .replace(/https?:\/\/\S+/g, "[redacted URL]")
+    .replace(/Bearer\s+\S+/gi, "Bearer [redacted]");
+  for (const secret of secrets) {
+    if (secret) message = message.replaceAll(secret, "[redacted]");
+  }
+  return message;
+}
+
+function secureRemoteUrl(value: string, label: string): string {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`${label} was malformed`);
+  }
+  if (url.protocol !== "https:" || url.username || url.password) {
+    throw new Error(`${label} must be an HTTPS URL without user info`);
+  }
+  return url.toString();
+}
+
+export class MinerUClient {
+  constructor(
+    private readonly options: MinerUOptions,
+    private readonly fetcher: typeof fetch = fetch,
+    private readonly sleeper: Sleeper = delay,
+  ) {
+    if (!options.token) throw new Error("MinerU credentials missing: set MINERU_API_TOKEN");
+  }
+
+  private remaining(deadline?: number): number | undefined {
+    return deadline === undefined ? undefined : Math.max(0, deadline - Date.now());
+  }
+
+  private async wait(requestedMs: number, signal?: AbortSignal, deadline?: number): Promise<boolean> {
+    const remaining = this.remaining(deadline);
+    if (remaining !== undefined && remaining <= 0) return false;
+    const maxDelay = Math.max(1, this.options.maxPollMs ?? 15_000);
+    const bounded = Math.min(Math.max(0, requestedMs), maxDelay, remaining ?? Number.POSITIVE_INFINITY);
+    if (bounded <= 0) return false;
+    await this.sleeper(bounded, signal);
+    return true;
+  }
+
+  private async control(
+    pathname: string,
+    init: RequestInit,
+    signal?: AbortSignal,
+    deadline?: number,
+  ): Promise<Record<string, unknown>> {
+    for (let attempt = 0; attempt < MAX_HTTP_ATTEMPTS; attempt += 1) {
+      const response = await this.retry(
+        `${API_BASE}${pathname}`,
+        {
+          ...init,
+          headers: {
+            Authorization: `Bearer ${this.options.token}`,
+            ...(init.body ? { "Content-Type": "application/json" } : {}),
+            ...init.headers,
+          },
+        },
+        signal,
+        "MinerU API",
+        deadline,
+      );
+      const json = JSON.parse(Buffer.from(await readResponseBytes(response, 2 * 1024 * 1024, signal)).toString("utf8")) as Record<string, unknown>;
+      if (!json || typeof json !== "object" || Array.isArray(json)) throw new Error("MinerU API returned malformed JSON");
+      if (json.code === 0) return json;
+      if (json.code !== -10001 || attempt === MAX_HTTP_ATTEMPTS - 1) {
+        throw new Error(
+          `MinerU API rejected request (code ${String(json.code)}): ${secretSafe(json.msg ?? "unknown error", [this.options.token])}`,
+        );
+      }
+      const waited = await this.wait(1000 * 2 ** attempt, signal, deadline);
+      if (!waited) break;
+    }
+    throw new Error("MinerU API timed out during bounded retries");
+  }
+
+  private async retry(
+    url: string,
+    init: RequestInit,
+    signal: AbortSignal | undefined,
+    stage: string,
+    deadline?: number,
+  ): Promise<Response> {
+    let last: unknown;
+    for (let attempt = 0; attempt < MAX_HTTP_ATTEMPTS; attempt += 1) {
+      const remaining = this.remaining(deadline);
+      if (remaining !== undefined && remaining <= 0) break;
+      const requestTimeoutMs = Math.min(
+        Math.max(this.options.timeoutMs ?? 600_000, 1_000),
+        60_000,
+        remaining ?? Number.POSITIVE_INFINITY,
+      );
+      try {
+        const timeout = AbortSignal.timeout(Math.max(1, requestTimeoutMs));
+        const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
+        const response = await this.fetcher(url, { ...init, redirect: "error", signal: requestSignal });
+        if (response.ok) return response;
+        await response.body?.cancel().catch(() => undefined);
+        if (response.status < 500 && response.status !== 429) {
+          throw new Error(`${stage} rejected (${response.status})`);
+        }
+        last = new Error(`${stage} transient failure (${response.status})`);
+        if (attempt < MAX_HTTP_ATTEMPTS - 1) {
+          const retryAfterSeconds = Number(response.headers.get("retry-after"));
+          const requestedDelay = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0
+            ? retryAfterSeconds * 1000
+            : 1000 * 2 ** attempt;
+          if (!await this.wait(requestedDelay, signal, deadline)) break;
+        }
+      } catch (error) {
+        if (signal?.aborted) throw signal.reason ?? new Error("MinerU operation cancelled");
+        if (error instanceof Error && /rejected \(4\d\d\)/.test(error.message)) throw error;
+        last = error;
+        if (attempt < MAX_HTTP_ATTEMPTS - 1 && !await this.wait(1000 * 2 ** attempt, signal, deadline)) break;
+      }
+    }
+    if (deadline !== undefined && Date.now() >= deadline) throw new Error(`${stage} timed out`);
+    throw new Error(`${stage} failed after retries: ${secretSafe(last, [this.options.token])}`);
+  }
+
+  async extract(pdfPath: string, signal?: AbortSignal, progress?: Progress): Promise<MinerURun> {
+    const info = await stat(pdfPath);
+    if (!info.isFile()) throw new Error("Selected Zotero PDF is not a file");
+    if (info.size > MAX_PDF_BYTES) throw new Error("Selected PDF exceeds MinerU's 200 MB limit");
+
+    signal?.throwIfAborted();
+    const bytes = await readFileBounded(pdfPath, MAX_PDF_BYTES, signal);
+    if (bytes.subarray(0, 5).toString() !== "%PDF-") {
+      throw new Error("Selected Zotero attachment is not a readable PDF");
+    }
+    const hash = createHash("sha256").update(bytes).digest("hex");
+    const name = pdfPath.replace(/\\/g, "/").split("/").pop() ?? "paper.pdf";
+    const dataId = randomUUID();
+    const requestBody = {
+      files: [{ name, data_id: dataId, is_ocr: this.options.isOcr ?? false }],
+      model_version: this.options.modelVersion ?? "vlm",
+      enable_formula: this.options.enableFormula ?? true,
+      enable_table: this.options.enableTable ?? true,
+      language: this.options.language ?? "en",
+    };
+
+    progress?.("requesting signed upload");
+    const creation = await this.control(
+      "/api/v4/file-urls/batch",
+      { method: "POST", body: JSON.stringify(requestBody) },
+      signal,
+    );
+    const data = creation.data as Record<string, unknown> | undefined;
+    const batchId = String(data?.batch_id ?? "");
+    const signed = (data?.file_urls as unknown[] | undefined)?.[0];
+    if (!batchId || typeof signed !== "string") throw new Error("MinerU signed-upload response was malformed");
+
+    progress?.("uploading PDF");
+    await this.retry(
+      secureRemoteUrl(signed, "MinerU signed-upload URL"),
+      { method: "PUT", body: bytes as unknown as BodyInit },
+      signal,
+      "MinerU signed upload",
+    );
+
+    progress?.("waiting for extraction");
+    const deadline = Date.now() + (this.options.timeoutMs ?? 600_000);
+    const maxAttempts = this.options.maxAttempts ?? 120;
+    let interval = this.options.initialPollMs ?? 3000;
+    let completed: Record<string, unknown> | null = null;
+    for (let attempt = 0; attempt < maxAttempts && Date.now() < deadline; attempt += 1) {
+      const polled = await this.control(
+        `/api/v4/extract-results/batch/${encodeURIComponent(batchId)}`,
+        { method: "GET" },
+        signal,
+        deadline,
+      );
+      const rows = ((polled.data as Record<string, unknown> | undefined)?.extract_result as unknown[] | undefined) ?? [];
+      const row = (rows.find((value) => String((value as Record<string, unknown>).data_id ?? "") === dataId)
+        ?? rows[0]) as Record<string, unknown> | undefined;
+      if (!row) throw new Error("MinerU poll response did not contain the submitted file");
+      const state = String(row.state);
+      if (state === "failed") {
+        throw new Error(`MinerU parse failed: ${secretSafe(row.err_msg ?? "unknown service error", [this.options.token])}`);
+      }
+      if (state === "done") {
+        completed = row;
+        break;
+      }
+      if (attempt + 1 >= maxAttempts) break;
+      if (!await this.wait(interval, signal, deadline)) break;
+      interval = Math.min(Math.ceil(interval * 1.5), this.options.maxPollMs ?? 15_000);
+    }
+
+    if (!completed) throw new Error("MinerU polling timed out before extraction completed");
+    const archiveUrl = completed.full_zip_url;
+    if (typeof archiveUrl !== "string") throw new Error("MinerU completed without a result archive");
+
+    progress?.("downloading result");
+    const archiveResponse = await this.retry(
+      secureRemoteUrl(archiveUrl, "MinerU result archive URL"),
+      { method: "GET" },
+      signal,
+      "MinerU result download",
+    );
+    const archive = await readResponseBytes(archiveResponse, MAX_ARCHIVE_BYTES, signal);
+
+    return {
+      archive,
+      pdfSha256: hash,
+      info: {
+        batchId,
+        state: "done",
+        fileName: String(completed.file_name ?? name),
+        dataId: stringOrNull(completed.data_id),
+        modelVersion: stringOrNull(completed.model_version ?? requestBody.model_version),
+        parserVersion: stringOrNull(completed.version),
+        options: {
+          language: requestBody.language,
+          enableFormula: requestBody.enable_formula,
+          enableTable: requestBody.enable_table,
+          isOcr: requestBody.files[0].is_ocr,
+        },
+      },
+    };
+  }
+}

package/src/model.ts

@@ -0,0 +1,36 @@
+export interface Creator { creatorType: string; firstName: string | null; lastName: string | null; name: string | null }
+export interface CollectionRef { key: string; name: string | null }
+export interface PaperNote { key: string; version: number; parentItem: string; note: string }
+export interface PaperAnnotation {
+  key: string; version: number; parentAttachment: string; text: string | null; comment: string | null;
+  color: string | null; pageLabel: string | null; sortIndex: string | null; position: unknown | null; tags: string[];
+}
+export type IndexedText = { status: "available"; content: string; indexedPages?: number; totalPages?: number } | { status: "unavailable" };
+export interface PaperAttachment {
+  key: string; version: number; parentItem: string; title: string; filename: string | null; contentType: string | null;
+  linkMode: string | null; path: string | null; md5: string | null; mtime: number | null; selected: boolean;
+  localPath: string | null; indexedText: IndexedText; annotations: PaperAnnotation[];
+}
+export interface BibliographicEnrichment {
+  zoteroKey: string; zoteroVersion: number; match: "normalized-title-first-author"; filledFields: string[];
+  /** Complete donor-item data retained in the metadata sidecar for provenance. */
+  metadata: Record<string, unknown>;
+}
+export interface Paper {
+  zoteroKey: string; zoteroVersion: number; itemType: string; title: string; creators: Creator[];
+  /** Complete selected parent-item data, retained in the metadata sidecar. */
+  metadata: Record<string, unknown>;
+  /** Optional exact local duplicate used only to fill missing bibliographic fields. */
+  bibliographicEnrichment?: BibliographicEnrichment;
+  date: string | null; year: string | null; doi: string | null; isbn: string | null; issn: string | null;
+  publicationTitle: string | null; volume: string | null; issue: string | null; pages: string | null;
+  url: string | null; abstract: string | null; tags: string[]; collections: CollectionRef[];
+  notes: PaperNote[]; annotations: PaperAnnotation[]; attachments: PaperAttachment[]; selectedPdf: PaperAttachment | null;
+}
+export interface MinerUInfo { batchId: string; state: "done"; fileName: string; dataId: string | null; modelVersion: string | null; parserVersion: string | null; options: Record<string, unknown> }
+export interface PublishedPaper { markdownPath: string; assetsDirectory: string; pdfSha256: string; mineru: MinerUInfo; parsedAt: string }
+
+export function scholarlyIdentity(paper: Pick<Paper, "doi" | "title" | "year">): string {
+  if (paper.doi) return `doi:${paper.doi.trim().toLowerCase().replace(/^https?:\/\/(dx\.)?doi\.org\//, "")}`;
+  return `title-year:${paper.title.normalize("NFKC").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim()}:${paper.year ?? ""}`;
+}

package/src/output.ts

@@ -0,0 +1,233 @@
+import path from "node:path";
+import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import AdmZip from "adm-zip";
+import YAML from "yaml";
+import type { Paper, PublishedPaper } from "./model.js";
+
+const INVALID=/[<>:"/\\|?*\u0000-\u001f]/g;
+const RESERVED=/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i;
+const IMAGE_EXT=/^\.(png|jpe?g|gif|webp|bmp|svg)$/;
+const MAX_ENTRIES=10_000;
+const MAX_EXPANDED=1024*1024*1024;
+
+function component(value:string,fallback:string):string {
+  let s=value.normalize("NFC").replace(/<[^>]*>/g,"").replace(/\+/g," ").replace(INVALID," ").replace(/\s+/g," ").trim().replace(/[. ]+$/g,"");
+  if(!s||s==="."||s==="..")s=fallback;
+  if(RESERVED.test(s))s=`_${s}`;
+  return s;
+}
+function utf8Limit(value:string,max:number):string {
+  const points=Array.from(value);
+  while(Buffer.byteLength(points.join(""),"utf8")>max)points.pop();
+  return points.join("").replace(/[. ]+$/g,"")||"Untitled";
+}
+export function paperStem(paper: Pick<Paper,"creators"|"year"|"title">,separator="+"):string {
+  const first=paper.creators.find(c=>c.creatorType==="author");
+  const author=component(first?.lastName ?? first?.name ?? "UnknownAuthor","UnknownAuthor");
+  const year=component(paper.year ?? "UnknownYear","UnknownYear");
+  const title=component(paper.title,"Untitled");
+  return utf8Limit([author,year,title].join(separator),220);
+}
+
+export interface NormalizedArchive { body:string; assets:{name:string;bytes:Buffer}[] }
+function safeEntry(name:string):string {
+  const n=name.replace(/\\/g,"/");
+  if(!n||n.startsWith("/")||/^[A-Za-z]:/.test(n)||n.split("/").includes("..")||n.includes("\0"))throw new Error(`Unsafe MinerU archive entry: ${name}`);
+  const normalized=path.posix.normalize(n).replace(/^\.\//,"");
+  if(!normalized||normalized==="."||normalized.startsWith("../"))throw new Error(`Unsafe MinerU archive entry: ${name}`);
+  return normalized;
+}
+
+interface CentralEntry { name:string; expanded:number; mode:number }
+function inspectCentralDirectory(bytes:Buffer):CentralEntry[] {
+  let eocd=-1;
+  for(let i=bytes.length-22;i>=Math.max(0,bytes.length-65_557);i--){if(bytes.readUInt32LE(i)===0x06054b50){eocd=i;break;}}
+  if(eocd<0)throw new Error("MinerU result archive is corrupt");
+  const count=bytes.readUInt16LE(eocd+10);
+  const offset=bytes.readUInt32LE(eocd+16);
+  if(count>MAX_ENTRIES)throw new Error(`MinerU archive exceeds ${MAX_ENTRIES} entries`);
+  const entries:CentralEntry[]=[];
+  let cursor=offset;
+  for(let i=0;i<count;i++){
+    if(cursor+46>bytes.length||bytes.readUInt32LE(cursor)!==0x02014b50)throw new Error("MinerU result archive has a malformed directory");
+    const flags=bytes.readUInt16LE(cursor+8);
+    if(flags&1)throw new Error("MinerU archive contains an encrypted entry");
+    const expanded=bytes.readUInt32LE(cursor+24);
+    const nameLength=bytes.readUInt16LE(cursor+28),extraLength=bytes.readUInt16LE(cursor+30),commentLength=bytes.readUInt16LE(cursor+32);
+    const end=cursor+46+nameLength+extraLength+commentLength;
+    if(end>bytes.length)throw new Error("MinerU result archive has a malformed entry");
+    const nameBytes=bytes.subarray(cursor+46,cursor+46+nameLength);
+    const name=(flags&0x800?nameBytes.toString("utf8"):nameBytes.toString("latin1"));
+    safeEntry(name);
+    entries.push({name,expanded,mode:(bytes.readUInt32LE(cursor+38)>>>16)&0xf000});
+    cursor=end;
+  }
+  return entries;
+}
+
+export function normalizeArchive(bytes:Uint8Array,assetPrefix:string,signal?:AbortSignal,assetFilePrefix="figure"):NormalizedArchive {
+  signal?.throwIfAborted();
+  const buffer=Buffer.from(bytes);
+  const central=inspectCentralDirectory(buffer);
+  let declared=0;
+  for(const entry of central){declared+=entry.expanded;if(declared>MAX_EXPANDED)throw new Error("MinerU archive expands beyond 1 GB");if(entry.mode===0xa000)throw new Error("MinerU archive contains a symbolic link");}
+  let zip:AdmZip;
+  try{zip=new AdmZip(buffer);}catch{throw new Error("MinerU result archive is corrupt");}
+  const entries=zip.getEntries();
+  if(entries.length!==central.length)throw new Error("MinerU result archive directory is inconsistent");
+  let expanded=0;
+  const files=new Map<string,Buffer>();
+  for(let i=0;i<entries.length;i++){
+    signal?.throwIfAborted();
+    const entry=entries[i]!,declaredEntry=central[i]!;
+    const name=safeEntry(declaredEntry.name);
+    if(entry.isDirectory)continue;
+    const data=entry.getData();
+    expanded+=data.length;
+    if(expanded>MAX_EXPANDED)throw new Error("MinerU archive expands beyond 1 GB");
+    if(files.has(name))throw new Error("MinerU archive contains duplicate entries");
+    files.set(name,data);
+  }
+  const candidates=[...files.keys()].filter(n=>path.posix.basename(n)==="full.md").sort((a,b)=>a.split("/").length-b.split("/").length||a.localeCompare(b));
+  const md=candidates[0] ?? [...files.keys()].filter(n=>/\.md$/i.test(n)).sort()[0];
+  if(!md)throw new Error("MinerU result archive contains no Markdown");
+  let body=files.get(md)!.toString("utf8").replace(/^\uFEFF/,"").replace(/\r\n?/g,"\n");
+  if(body.startsWith("---\n")){const end=body.indexOf("\n---\n",4);if(end>=0)body=body.slice(end+5);}
+  const mdDir=path.posix.dirname(md);
+  const assets:{name:string;bytes:Buffer}[]=[];
+  const mapped=new Map<string,string>();
+  const resolve=(raw:string):string=>{
+    signal?.throwIfAborted();
+    let clean:string;
+    try{clean=decodeURIComponent(raw.trim().replace(/^<|>$/g,"").split(/[?#]/,1)[0]!);}catch{throw new Error(`MinerU Markdown contains an invalid image path: ${raw}`);}
+    if(/^(?:[a-z][a-z\d+.-]*:|\/|\\|#)/i.test(clean))throw new Error("MinerU Markdown contains a non-local or absolute image reference");
+    const source=safeEntry(path.posix.join(mdDir,clean));
+    const data=files.get(source);
+    if(!data)throw new Error(`MinerU Markdown references missing asset: ${clean}`);
+    let name=mapped.get(source);
+    if(!name){const ext=path.posix.extname(source).toLowerCase();if(!IMAGE_EXT.test(ext))throw new Error(`Unsupported MinerU image type: ${ext}`);name=`${assetFilePrefix}-${String(assets.length+1).padStart(2,"0")}${ext}`;mapped.set(source,name);assets.push({name,bytes:data});}
+    return `${assetPrefix}/${name}`;
+  };
+  body=body.replace(/!\[([^\]]*)\]\(\s*(<[^>]+>|[^\s)]+)(?:\s+["'][^"']*["'])?\s*\)/g,(_m,alt,dest)=>`![${alt}](<${resolve(dest)}>)`);
+  const imageRefs=new Set<string>();
+  body.replace(/!\[[^\]]*\]\[([^\]]+)\]/g,(_m,id)=>{imageRefs.add(String(id).toLowerCase());return _m;});
+  body=body.replace(/^\[([^\]]+)\]:\s*(<[^>]+>|\S+)(.*)$/gm,(m,id,dest,tail)=>imageRefs.has(String(id).toLowerCase())?`[${id}]: <${resolve(dest)}>${tail}`:m);
+  body=body.replace(/<img\b([^>]*?)\bsrc=["']([^"']+)["']([^>]*)>/gi,(_m,before,src,after)=>{const alt=(`${before} ${after}`.match(/\balt=["']([^"']*)["']/i)?.[1]??"").replace(/[&<>"']/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]!));return `<img src="${resolve(src)}" alt="${alt}"/>`;});
+  return {body:body.trim()+"\n",assets};
+}
+
+function plainText(value:string):string{return value.replace(/^\/?jats:title>\s*/i,"").replace(/<[^>]*>/g,"").replace(/&amp;/gi,"&").replace(/&lt;/gi,"<").replace(/&gt;/gi,">").replace(/&quot;/gi,'"').replace(/&#(?:39|x27);/gi,"'").replace(/\s+/g," ").trim();}
+function authorNames(paper:Paper):string{return paper.creators.filter(creator=>creator.creatorType==="author").map(creator=>creator.name??[creator.firstName,creator.lastName].filter(Boolean).join(" ")).filter(Boolean).join("; ");}
+export function obsidianTags(tags:string[],spaceReplacement:"-"|"_"="-"):string[]{return [...new Set(tags.map(value=>{let tag=value.normalize("NFKC").trim().replace(/^#+/,"").replace(/\s+/g,spaceReplacement).replace(/[^\p{L}\p{N}_\/-]+/gu,spaceReplacement);const repeated=spaceReplacement==="-"?/-+/g:/_+/g;tag=tag.replace(repeated,spaceReplacement).split("/").map(part=>part.replace(/^[-_]+|[-_]+$/g,"")).filter(Boolean).join("/");if(/^\d+$/.test(tag))tag=`tag-${tag}`;return tag;}).filter(Boolean))];}
+function safeAttachments(paper:Paper):unknown[]{return paper.attachments.map(({localPath:_localPath,path:_path,...attachment})=>({...attachment,indexedText:attachment.indexedText.status==="available"?{...attachment.indexedText,content:undefined}:attachment.indexedText}));}
+
+/** Compact, human-facing properties. Verbose provenance lives in metadata.json. */
+export function frontmatter(paper:Paper,published:Omit<PublishedPaper,"markdownPath"|"assetsDirectory">,tagSpaceReplacement:"-"|"_"="-"):Record<string,unknown>{
+  const authors=authorNames(paper);
+  return {
+    schema_version:1,
+    title:plainText(paper.title),
+    ...(authors?{authors}:{}),
+    ...(paper.year?{year:paper.year}:{}),
+    ...(paper.date&&paper.date!==paper.year?{date:paper.date}:{}),
+    ...(paper.doi?{doi:paper.doi}:{}),
+    ...(paper.isbn?{isbn:paper.isbn}:{}),
+    ...(paper.issn?{issn:paper.issn}:{}),
+    ...(paper.publicationTitle?{journal:paper.publicationTitle}:{}),
+    ...(paper.volume?{volume:paper.volume}:{}),
+    ...(paper.issue?{issue:paper.issue}:{}),
+    ...(paper.pages?{pages:paper.pages}:{}),
+    ...(paper.url?{url:paper.url}:{}),
+    ...(paper.tags.length?{tags:obsidianTags(paper.tags,tagSpaceReplacement)}:{}),
+    zotero:`zotero://select/library/items/${paper.zoteroKey}`,
+    ...(paper.selectedPdf?{zotero_pdf:`zotero://open-pdf/library/items/${paper.selectedPdf.key}`}:{ }),
+    parsed_at:published.parsedAt,
+  };
+}
+
+/** Complete, path-safe provenance kept out of note-property UIs. */
+export function metadataSidecar(paper:Paper,published:Omit<PublishedPaper,"markdownPath"|"assetsDirectory">):Record<string,unknown>{
+  return {
+    schema_version:1,
+    bibliographic:{title:plainText(paper.title),authors:paper.creators,date:paper.date,year:paper.year,doi:paper.doi,isbn:paper.isbn,issn:paper.issn,publication_title:paper.publicationTitle,volume:paper.volume,issue:paper.issue,pages:paper.pages,url:paper.url,abstract:paper.abstract,tags:paper.tags,collections:paper.collections},
+    zotero:{selected_key:paper.zoteroKey,selected_version:paper.zoteroVersion,selected_metadata:paper.metadata,...(paper.bibliographicEnrichment?{enrichment:{source_key:paper.bibliographicEnrichment.zoteroKey,source_version:paper.bibliographicEnrichment.zoteroVersion,match:paper.bibliographicEnrichment.match,filled_fields:paper.bibliographicEnrichment.filledFields,source_metadata:paper.bibliographicEnrichment.metadata}}:{})},
+    notes:paper.notes,
+    annotations:paper.annotations,
+    attachments:safeAttachments(paper),
+    selected_attachment_key:paper.selectedPdf?.key??null,
+    pdf:{sha256:published.pdfSha256},
+    mineru:published.mineru,
+    parsed_at:published.parsedAt,
+  };
+}
+export function composeMarkdown(meta:Record<string,unknown>,body:string):string{return `---\n${YAML.stringify(meta,{lineWidth:0}).trimEnd()}\n---\n\n${body.replace(/^---\n[\s\S]*?\n---\n/,"")}`;}
+async function existingOwner(file:string):Promise<string|null>{try{const text=await readFile(file,"utf8");if(!text.startsWith("---\n"))return null;const end=text.indexOf("\n---\n",4);if(end<0)return null;const value=YAML.parse(text.slice(4,end));if(typeof value?.zotero_key==="string")return value.zotero_key;if(typeof value?.zotero==="string")return value.zotero.match(/\/items\/([A-Z0-9]{8})$/)?.[1]??null;return null;}catch{return null;}}
+async function exists(p:string):Promise<boolean>{try{await readFile(p);return true;}catch{try{return (await readdir(p)).length>=0;}catch{return false;}}}
+async function priorStem(root:string,zoteroKey:string):Promise<string|null>{for(const name of (await readdir(root)).filter(n=>n.toLowerCase().endsWith(".md")).sort()){if(await existingOwner(path.join(root,name))===zoteroKey)return name.slice(0,-3);}return null;}
+
+async function allocateStem(root:string,base:string,zoteroKey:string,assetsSuffix:string):Promise<string>{
+  const prior=await priorStem(root,zoteroKey);
+  if(prior)return prior;
+  for(let index=1;;index+=1){
+    const stem=index===1?base:`${base} (${index})`;
+    const markdownPath=path.join(root,`${stem}.md`);
+    const assetsPath=path.join(root,`${stem}${assetsSuffix}`);
+    const owner=await existingOwner(markdownPath);
+    if(owner===zoteroKey)return stem;
+    if(!await exists(markdownPath)&&!await exists(assetsPath))return stem;
+  }
+}
+
+export interface OutputNamingOptions { filenameSeparator?:string; assetsSuffix?:string; metadataFileName?:string; tagSpaceReplacement?:"-"|"_" }
+export async function publishPaper(outputRoot:string,paper:Paper,normalized:NormalizedArchive,data:Omit<PublishedPaper,"markdownPath"|"assetsDirectory">,signal?:AbortSignal,naming:OutputNamingOptions={}):Promise<PublishedPaper>{
+  signal?.throwIfAborted();
+  await mkdir(outputRoot,{recursive:true});
+  const resolved=path.resolve(outputRoot);
+  const locks=path.join(resolved,".pi-scholar-locks");
+  await mkdir(locks,{recursive:true});
+  const filenameSeparator=naming.filenameSeparator??"+",assetsSuffix=naming.assetsSuffix??".assets",metadataFileName=naming.metadataFileName??"metadata.json";
+  const base=paperStem(paper,filenameSeparator);
+  // Serialize name allocation and publication for the output root so two distinct
+  // papers with the same sanitized basename cannot race into the same paths.
+  const lock=path.join(locks,"publish.lock");
+  try{await mkdir(lock);}catch{throw new Error("Another parse publication is already in progress for this output directory");}
+  let stage:string|null=null;
+  try{
+    signal?.throwIfAborted();
+    const stem=await allocateStem(resolved,base,paper.zoteroKey,assetsSuffix);
+    const mdPath=path.join(resolved,`${stem}.md`),assetDir=path.join(resolved,`${stem}${assetsSuffix}`);
+    if(path.dirname(mdPath)!==resolved||path.dirname(assetDir)!==resolved)throw new Error("Unsafe output path");
+    stage=path.join(resolved,`.pi-scholar-${randomUUID()}`);
+    const stageAssets=path.join(stage,`${stem}${assetsSuffix}`);
+    await mkdir(stageAssets,{recursive:true});
+    for(const asset of normalized.assets){signal?.throwIfAborted();await writeFile(path.join(stageAssets,asset.name),asset.bytes);}
+    signal?.throwIfAborted();
+    // Keep a true relative path. Angle-bracket Markdown destinations are emitted
+    // during normalization so spaces remain readable and work in Obsidian.
+    const assetUrl=`./${stem}${assetsSuffix}`;
+    await writeFile(path.join(stageAssets,metadataFileName),JSON.stringify(metadataSidecar(paper,data),null,2)+"\n","utf8");
+    const markdown=composeMarkdown(frontmatter(paper,data,naming.tagSpaceReplacement),normalized.body.replaceAll("__ASSET_PREFIX__",assetUrl));
+    await writeFile(path.join(stage,`${stem}.md`),markdown,"utf8");
+    signal?.throwIfAborted();
+    const backup=path.join(resolved,`.pi-scholar-backup-${randomUUID()}`);
+    await mkdir(backup);
+    try{
+      if(await exists(mdPath))await rename(mdPath,path.join(backup,`${stem}.md`));
+      signal?.throwIfAborted();
+      if(await exists(assetDir))await rename(assetDir,path.join(backup,`${stem}${assetsSuffix}`));
+      signal?.throwIfAborted();
+      await rename(stageAssets,assetDir);
+      signal?.throwIfAborted();
+      await rename(path.join(stage,`${stem}.md`),mdPath);
+      signal?.throwIfAborted();
+      await rm(backup,{recursive:true,force:true});
+    }catch(error){
+      await rm(mdPath,{force:true});await rm(assetDir,{recursive:true,force:true});
+      if(await exists(path.join(backup,`${stem}.md`)))await rename(path.join(backup,`${stem}.md`),mdPath);
+      if(await exists(path.join(backup,`${stem}${assetsSuffix}`)))await rename(path.join(backup,`${stem}${assetsSuffix}`),assetDir);
+      throw error;
+    }finally{await rm(backup,{recursive:true,force:true});}
+    return{markdownPath:mdPath,assetsDirectory:assetDir,...data};
+  }finally{if(stage)await rm(stage,{recursive:true,force:true});await rm(lock,{recursive:true,force:true});}
+}

package/src/tools.ts

@@ -0,0 +1,33 @@
+import { Type } from "typebox";
+import { StringEnum } from "@earendil-works/pi-ai";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { loadConfig, type ScholarConfig } from "./config.js";
+import { ZoteroClient } from "./zotero.js";
+import { MinerUClient } from "./mineru.js";
+import { normalizeArchive, publishPaper } from "./output.js";
+
+const LIMIT_BYTES=50*1024,LIMIT_LINES=2000;
+export function truncate(text: string): { text: string; truncated: boolean; bytes: number; lines: number } {
+  const bytes = Buffer.byteLength(text), lines = text.split("\n").length;
+  if (bytes <= LIMIT_BYTES && lines <= LIMIT_LINES) return { text, truncated: false, bytes, lines };
+  // Reserve marker space and count its lines inside the advertised total limit.
+  const source = text.split("\n");
+  const output: string[] = [];
+  let shownBytes = 0;
+  for (const line of source.slice(0, LIMIT_LINES)) {
+    const size = Buffer.byteLength(line) + (output.length ? 1 : 0);
+    if (shownBytes + size > LIMIT_BYTES - 160) break;
+    output.push(line); shownBytes += size;
+  }
+  const shown = output.length;
+  return { text: output.join("\n") + `\n\n[Truncated: displayed ${shown}/${lines} lines and ${shownBytes}/${bytes} bytes. Refine the query or read a specific item.]`, truncated: true, bytes, lines };
+}
+const configFor = (ctx: ExtensionContext): ScholarConfig => loadConfig(process.env, ctx.cwd, ctx.isProjectTrusted?.() === true);
+const zoteroFor = (c: ScholarConfig): ZoteroClient => new ZoteroClient({ baseUrl: c.zoteroBaseUrl, dataDir: c.zoteroDataDir, timeoutMs: c.zoteroTimeoutMs, maxItems: c.zoteroMaxItems });
+const result=(value:unknown,details:Record<string,unknown>={})=>{const t=truncate(JSON.stringify(value,null,2));return{content:[{type:"text" as const,text:t.text}],details:{...details,truncated:t.truncated,totalBytes:t.bytes,totalLines:t.lines}};};
+export function registerScholarTools(pi:ExtensionAPI):void{
+  pi.registerTool({name:"zotero_collections",label:"Zotero Collections",description:"Read-only list, metadata read, or item listing for Zotero Desktop collections. GET requests to loopback only. Output is limited to 50KB/2000 lines.",promptSnippet:"List or read local Zotero collections and their items",promptGuidelines:["Use zotero_collections to inspect local Zotero collection names, keys, metadata, and top-level items without modifying Zotero."],parameters:Type.Object({action:StringEnum(["list","read","items"] as const),key:Type.Optional(Type.String({pattern:"^[A-Z0-9]{8}$"})),limit:Type.Optional(Type.Integer({minimum:1,maximum:5000}))}),async execute(_id,p,signal,_onUpdate,ctx){const c=configFor(ctx);const z=zoteroFor(c);if(p.action==="read"){if(!p.key)throw new Error("zotero_collections read requires key");return result(await z.getCollection(p.key,signal));}if(p.action==="items"){if(!p.key)throw new Error("zotero_collections items requires key");return result(await z.listCollectionItems(p.key,p.limit,signal));}return result(await z.listCollections(p.limit,signal));}});
+  pi.registerTool({name:"zotero_search",label:"Zotero Search",description:"Search top-level items in the read-only Zotero Desktop Local API, optionally in a collection/type. Output is limited to 50KB/2000 lines.",promptSnippet:"Search the local Zotero library",promptGuidelines:["Use zotero_search to match online papers locally, preferring DOI and then normalized title/year."],parameters:Type.Object({query:Type.String({minLength:1,maxLength:1000}),collectionKey:Type.Optional(Type.String({pattern:"^[A-Z0-9]{8}$"})),itemType:Type.Optional(Type.String({minLength:1,maxLength:100})),limit:Type.Optional(Type.Integer({minimum:1,maximum:500}))}),async execute(_id,p,signal,_onUpdate,ctx){const c=configFor(ctx);const z=zoteroFor(c);return result(await z.searchItems(p.query,{...(p.collectionKey?{collectionKey:p.collectionKey}:{}),...(p.itemType?{itemType:p.itemType}:{}),...(p.limit?{limit:p.limit}:{})},signal));}});
+  pi.registerTool({name:"zotero_item",label:"Zotero Item",description:"Read one raw Zotero item or its complete normalized Paper aggregate including notes, attachments, annotations, indexed-text availability and PDF location. Read-only; output limited to 50KB/2000 lines.",promptSnippet:"Read complete Zotero paper metadata and child content",promptGuidelines:["Use zotero_item with aggregate mode before analyzing or parsing a local paper."],parameters:Type.Object({key:Type.String({pattern:"^[A-Z0-9]{8}$"}),mode:StringEnum(["item","aggregate"] as const),attachmentKey:Type.Optional(Type.String({pattern:"^[A-Z0-9]{8}$"}))}),async execute(_id,p,signal,_onUpdate,ctx){const c=configFor(ctx);const z=zoteroFor(c);return result(p.mode==="item"?await z.getItem(p.key,signal):await z.getPaper(p.key,p.attachmentKey,signal));}});
+  pi.registerTool({name:"pi_scholar_parse",label:"Parse Zotero PDF with MinerU",description:"Uploads one local Zotero PDF to MinerU's official API, polls, downloads safely, and atomically saves plain Markdown/assets. Network use may consume quota. Returns only a bounded path/hash/task summary; never secrets.",promptSnippet:"Parse a Zotero PDF into structured local Markdown and assets",promptGuidelines:["Use pi_scholar_parse only when structured full text or figures are needed and a current hash-matching parse is unavailable."],parameters:Type.Object({key:Type.String({pattern:"^[A-Z0-9]{8}$"}),attachmentKey:Type.Optional(Type.String({pattern:"^[A-Z0-9]{8}$"}))}),async execute(_id,p,signal,onUpdate,ctx){const c=configFor(ctx);if(!c.mineruToken)throw new Error("MinerU credentials missing: set MINERU_API_TOKEN and reload Pi");const z=zoteroFor(c);const paper=await z.getPaper(p.key,p.attachmentKey,signal);if(!paper.selectedPdf)throw new Error("Zotero item has no PDF attachment");if(!paper.selectedPdf.localPath)throw new Error("Selected Zotero PDF file location cannot be resolved; check Zotero file availability or ZOTERO_DATA_DIR");const mineru=new MinerUClient({token:c.mineruToken,timeoutMs:c.mineruTimeoutMs,initialPollMs:c.mineruPollInitialMs,maxPollMs:c.mineruPollMaxMs,maxAttempts:c.mineruMaxAttempts,language:c.mineruLanguage,enableFormula:c.mineruEnableFormula,enableTable:c.mineruEnableTable,isOcr:c.mineruIsOcr,modelVersion:c.mineruModelVersion});const run=await mineru.extract(paper.selectedPdf.localPath,signal,stage=>onUpdate?.({content:[{type:"text",text:`MinerU: ${stage}`}],details:{stage}}));const normalized=normalizeArchive(run.archive,"__ASSET_PREFIX__",signal,c.assetFilePrefix);const parsedAt=new Date().toISOString();const published=await publishPaper(c.outputDir,paper,normalized,{pdfSha256:run.pdfSha256,mineru:run.info,parsedAt},signal,{filenameSeparator:c.filenameSeparator,assetsSuffix:c.assetsSuffix,metadataFileName:c.metadataFileName,tagSpaceReplacement:c.tagSpaceReplacement});return result({status:"done",zoteroKey:paper.zoteroKey,selectedAttachmentKey:paper.selectedPdf.key,...published});}});
+}