@ncukondo/reference-manager 0.29.5 → 0.30.0

package/dist/chunks/{index-6Aild0Uk.js → index-DoOmAYKd.js}

@@ -9,11 +9,11 @@ import "@citation-js/plugin-bibtex";
 import "@citation-js/plugin-ris";
 import { z } from "zod";
 import * as path from "node:path";
-import path__default, { extname, join, dirname } from "node:path";
+import path__default, { extname, join, dirname, basename } from "node:path";
 import "@citation-js/plugin-csl";
-import fs__default, { stat, rename, copyFile, unlink, rm, readFile, mkdir, writeFile, mkdtemp, access, rmdir } from "node:fs/promises";
+import fs__default, { stat, rename, copyFile, unlink, rm, readFile, mkdir, writeFile, access, mkdtemp, rmdir } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import "node:child_process";
+import { exec, execFile } from "node:child_process";
 import "node:crypto";
 function slugifyLabel(label) {
   return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
@@ -31,16 +31,16 @@ function parseFilename(filename) {
   }
   const ext = path__default.extname(filename);
   const extWithoutDot = ext.startsWith(".") ? ext.slice(1) : ext;
-  const basename = ext ? filename.slice(0, -ext.length) : filename;
-  const firstHyphenIndex = basename.indexOf("-");
+  const basename2 = ext ? filename.slice(0, -ext.length) : filename;
+  const firstHyphenIndex = basename2.indexOf("-");
   if (firstHyphenIndex === -1) {
     return {
-      role: basename,
+      role: basename2,
       ext: extWithoutDot
     };
   }
-  const role = basename.slice(0, firstHyphenIndex);
-  const label = basename.slice(firstHyphenIndex + 1);
+  const role = basename2.slice(0, firstHyphenIndex);
+  const label = basename2.slice(firstHyphenIndex + 1);
   if (label) {
     return {
       role,
@@ -10313,6 +10313,393 @@ async function fulltextDiscover(library, options) {
   }
   return discoverResult;
 }
+function shellQuote(value) {
+  return `'${value.replaceAll("'", "'\\''")}'`;
+}
+function expandTemplate(template, vars) {
+  return template.replaceAll("{input_dir}", shellQuote(dirname(vars.input))).replaceAll("{input_name}", shellQuote(basename(vars.input))).replaceAll("{output_name}", shellQuote(basename(vars.output))).replaceAll("{input}", shellQuote(vars.input)).replaceAll("{output}", shellQuote(vars.output));
+}
+async function isCommandAvailable(command) {
+  const lookupCmd = process.platform === "win32" ? "where" : "which";
+  return new Promise((resolve) => {
+    execFile(lookupCmd, [command], (err) => {
+      resolve(err === null);
+    });
+  });
+}
+async function runCheckCommand(checkCommand) {
+  return new Promise((resolve) => {
+    exec(checkCommand, (err) => {
+      resolve(err === null);
+    });
+  });
+}
+class CustomPdfConverter {
+  name;
+  config;
+  constructor(name, config) {
+    this.name = name;
+    this.config = config;
+  }
+  async isAvailable() {
+    const checkCmd = this.getCheckCommand();
+    if (checkCmd) {
+      return runCheckCommand(checkCmd);
+    }
+    const command = this.getCommand();
+    const binary = command.split(/\s+/)[0] ?? command;
+    return isCommandAvailable(binary);
+  }
+  async convert(pdfPath, outputPath) {
+    const command = this.getCommand();
+    const expanded = expandTemplate(command, { input: pdfPath, output: outputPath });
+    const outputMode = this.config.outputMode ?? "file";
+    const timeoutMs = (this.config.timeout ?? 300) * 1e3;
+    try {
+      const { stdout } = await this.execCommand(expanded, timeoutMs);
+      if (outputMode === "stdout") {
+        await writeFile(outputPath, stdout, "utf-8");
+        return { success: true, outputPath };
+      }
+      try {
+        await access(outputPath);
+      } catch {
+        return {
+          success: false,
+          error: `Output file was not created: ${outputPath}`,
+          code: "output-not-created"
+        };
+      }
+      return { success: true, outputPath };
+    } catch (err) {
+      return this.handleExecError(err);
+    }
+  }
+  getCommand() {
+    if (process.platform === "win32" && this.config.commandWindows) {
+      return this.config.commandWindows;
+    }
+    return this.config.command;
+  }
+  getCheckCommand() {
+    if (process.platform === "win32" && this.config.checkCommandWindows) {
+      return this.config.checkCommandWindows;
+    }
+    return this.config.checkCommand;
+  }
+  execCommand(command, timeoutMs) {
+    return new Promise((resolve, reject) => {
+      exec(command, { timeout: timeoutMs }, (err, stdout, stderr) => {
+        if (err) {
+          reject(Object.assign(err, { stderr: stderr || err.stderr }));
+        } else {
+          resolve({ stdout, stderr });
+        }
+      });
+    });
+  }
+  handleExecError(err) {
+    const error = err;
+    if (error.killed || error.signal === "SIGTERM") {
+      return {
+        success: false,
+        error: `PDF conversion timed out after ${this.config.timeout ?? 300} seconds`,
+        code: "timeout"
+      };
+    }
+    return {
+      success: false,
+      error: error.message ?? "Conversion failed",
+      code: "conversion-failed",
+      ...error.stderr ? { stderr: error.stderr } : {}
+    };
+  }
+}
+const BUILTIN_CONVERTER_NAMES = ["marker", "docling", "mineru", "pymupdf"];
+const BUILTIN_CONVERTER_INFO = {
+  marker: {
+    install: "pip install marker-pdf",
+    description: "GPU recommended, best quality"
+  },
+  docling: {
+    install: "pip install docling",
+    description: "CPU OK, good tables"
+  },
+  mineru: {
+    install: "pip install mineru[all]",
+    description: "GPU recommended, fastest"
+  },
+  pymupdf: {
+    install: "pip install pymupdf4llm",
+    description: "CPU only, lightweight"
+  }
+};
+const BUILTIN_CONFIGS = {
+  marker: {
+    command: "marker_single {input} --output_dir {input_dir}",
+    checkCommand: "marker_single --help"
+  },
+  docling: {
+    command: "docling --from pdf --to md --output {input_dir} {input}",
+    checkCommand: "docling --help"
+  },
+  mineru: {
+    command: "mineru -p {input} -o {input_dir} -m auto",
+    checkCommand: "mineru --help"
+  },
+  pymupdf: {
+    command: `python3 -c "import pymupdf4llm, pathlib; md=pymupdf4llm.to_markdown('{input}'); pathlib.Path('{output}').write_text(md)"`,
+    checkCommand: 'python3 -c "import pymupdf4llm"'
+  }
+};
+function getBuiltinConverter(name) {
+  if (!isBuiltinName(name)) return void 0;
+  return new CustomPdfConverter(name, BUILTIN_CONFIGS[name]);
+}
+function isBuiltinName(name) {
+  return BUILTIN_CONVERTER_NAMES.includes(name);
+}
+async function resolveConverter(name, options) {
+  if (name === "auto") {
+    return resolveAuto(options);
+  }
+  return resolveExplicit(name, options);
+}
+async function resolveAuto(options) {
+  const { priority, customConverters } = options;
+  for (const converterName of priority) {
+    const converter = getConverterByName(converterName, customConverters);
+    if (!converter) continue;
+    if (await converter.isAvailable()) {
+      return { success: true, converter };
+    }
+  }
+  return {
+    success: false,
+    code: "no-converter",
+    error: "No PDF converter found",
+    hints: buildNoConverterHints(priority)
+  };
+}
+async function resolveExplicit(name, options) {
+  const converter = getConverterByName(name, options.customConverters);
+  if (!converter) {
+    return {
+      success: false,
+      code: "not-installed",
+      error: `PDF converter '${name}' not found`
+    };
+  }
+  if (!await converter.isAvailable()) {
+    const isBuiltin = BUILTIN_CONVERTER_NAMES.includes(name);
+    const installHint = isBuiltin ? BUILTIN_CONVERTER_INFO[name].install : void 0;
+    const result = {
+      success: false,
+      code: "not-installed",
+      error: `PDF converter '${name}' is not installed`
+    };
+    if (installHint) {
+      result.hints = `Install with: ${installHint}`;
+    }
+    return result;
+  }
+  return { success: true, converter };
+}
+function getConverterByName(name, customConverters) {
+  const customConfig = customConverters[name];
+  if (customConfig) {
+    return new CustomPdfConverter(name, customConfig);
+  }
+  return getBuiltinConverter(name);
+}
+function buildNoConverterHints(checked) {
+  const lines = [
+    `Checked: ${checked.join(", ")} (none available)`,
+    "",
+    "Install one of the following:",
+    ""
+  ];
+  for (const name of BUILTIN_CONVERTER_NAMES) {
+    const info = BUILTIN_CONVERTER_INFO[name];
+    lines.push(`  ${name.padEnd(9)} ${info.install.padEnd(28)} (${info.description})`);
+  }
+  lines.push("", "Or configure a custom converter in your config file.");
+  return lines.join("\n");
+}
+function findXmlFile(item) {
+  const attachments = item.custom?.attachments;
+  if (!attachments?.files) return void 0;
+  const xmlFile = attachments.files.find(
+    (f) => f.role === "fulltext" && f.filename.endsWith(".xml")
+  );
+  return xmlFile?.filename;
+}
+function findPdfFile(item) {
+  const attachments = item.custom?.attachments;
+  if (!attachments?.files) return void 0;
+  const pdfFile = attachments.files.find(
+    (f) => f.role === "fulltext" && f.filename.endsWith(".pdf")
+  );
+  return pdfFile?.filename;
+}
+function getFilePath(item, filename, fulltextDirectory) {
+  const attachments = item.custom?.attachments;
+  const directory = attachments?.directory ?? "";
+  return join(fulltextDirectory, directory, filename);
+}
+async function fulltextConvert(library, options) {
+  const { identifier, idType = "id", from } = options;
+  const item = await library.find(identifier, { idType });
+  if (!item) {
+    return { success: false, error: `Reference '${identifier}' not found` };
+  }
+  const format = resolveFormat(item, from);
+  if (!format.success) {
+    return format.result;
+  }
+  if (format.type === "xml") {
+    return convertXml(library, item, options);
+  }
+  return convertPdf(library, item, options);
+}
+function resolveFormat(item, from) {
+  const hasXml = findXmlFile(item) !== void 0;
+  const hasPdf = findPdfFile(item) !== void 0;
+  if (from === "xml") {
+    if (!hasXml) {
+      return {
+        success: false,
+        result: {
+          success: false,
+          error: `No PMC XML file attached to '${item.id}'`
+        }
+      };
+    }
+    return { success: true, type: "xml" };
+  }
+  if (from === "pdf") {
+    if (!hasPdf) {
+      return {
+        success: false,
+        result: {
+          success: false,
+          error: `No PDF file attached to '${item.id}'`,
+          code: "no-pdf",
+          hints: [
+            "This reference has no fulltext PDF. You can:",
+            `  1. Download OA fulltext:  ref fulltext fetch ${item.id}`,
+            `  2. Attach a local PDF:    ref fulltext attach ${item.id} /path/to/paper.pdf`
+          ].join("\n")
+        }
+      };
+    }
+    return { success: true, type: "pdf" };
+  }
+  if (hasXml) return { success: true, type: "xml" };
+  if (hasPdf) return { success: true, type: "pdf" };
+  return {
+    success: false,
+    result: {
+      success: false,
+      error: `No PMC XML file attached to '${item.id}'`
+    }
+  };
+}
+async function convertXml(library, item, options) {
+  const { identifier, idType = "id", fulltextDirectory } = options;
+  const xmlFilename = findXmlFile(item);
+  if (!xmlFilename) {
+    return { success: false, error: `No PMC XML file attached to '${identifier}'` };
+  }
+  const xmlPath = getFilePath(item, xmlFilename, fulltextDirectory);
+  try {
+    await access(xmlPath);
+  } catch {
+    return { success: false, error: `XML file not found on disk: ${xmlPath}` };
+  }
+  const mdPath = xmlPath.replace(/\.xml$/, ".md");
+  const convertResult2 = await convertPmcXmlToMarkdown(xmlPath, mdPath);
+  if (!convertResult2.success) {
+    return {
+      success: false,
+      error: `Failed to convert PMC XML to Markdown: ${convertResult2.error ?? "unknown error"}`
+    };
+  }
+  const attachResult = await fulltextAttach(library, {
+    identifier,
+    idType,
+    filePath: mdPath,
+    type: "markdown",
+    force: true,
+    move: true,
+    fulltextDirectory
+  });
+  if (!attachResult.success) {
+    return { success: false, error: attachResult.error ?? "Attach failed" };
+  }
+  return {
+    success: true,
+    filename: attachResult.filename ?? "fulltext.md"
+  };
+}
+async function convertPdf(library, item, options) {
+  const { identifier, idType = "id", fulltextDirectory, fulltextConfig } = options;
+  const converterName = options.converter ?? fulltextConfig?.pdfConverter ?? "auto";
+  const pdfFilename = findPdfFile(item);
+  if (!pdfFilename) {
+    return { success: false, error: `No PDF file attached to '${identifier}'`, code: "no-pdf" };
+  }
+  const pdfPath = getFilePath(item, pdfFilename, fulltextDirectory);
+  try {
+    await access(pdfPath);
+  } catch {
+    return { success: false, error: `PDF file not found on disk: ${pdfPath}` };
+  }
+  const resolveResult = await resolveConverter(converterName, {
+    priority: fulltextConfig?.pdfConverterPriority ?? ["marker", "docling", "mineru", "pymupdf"],
+    customConverters: fulltextConfig?.converters ?? {}
+  });
+  if (!resolveResult.success) {
+    const result = {
+      success: false,
+      error: resolveResult.error,
+      code: resolveResult.code
+    };
+    if (resolveResult.hints) {
+      result.hints = resolveResult.hints;
+    }
+    return result;
+  }
+  const mdPath = pdfPath.replace(/\.pdf$/, ".md");
+  const pdfResult = await resolveResult.converter.convert(pdfPath, mdPath);
+  if (!pdfResult.success) {
+    const result = {
+      success: false,
+      error: `Failed to convert PDF to Markdown using ${resolveResult.converter.name}: ${pdfResult.error}`,
+      code: pdfResult.code
+    };
+    if (pdfResult.stderr) {
+      result.stderr = pdfResult.stderr;
+    }
+    return result;
+  }
+  const attachResult = await fulltextAttach(library, {
+    identifier,
+    idType,
+    filePath: mdPath,
+    type: "markdown",
+    force: true,
+    move: true,
+    fulltextDirectory
+  });
+  if (!attachResult.success) {
+    return { success: false, error: attachResult.error ?? "Attach failed" };
+  }
+  return {
+    success: true,
+    filename: attachResult.filename ?? "fulltext.md"
+  };
+}
 function extractPmcidFromLocations(locations) {
   for (const loc of locations) {
     if (loc.source !== "pmc") continue;
@@ -10533,7 +10920,8 @@ async function fulltextFetch(library, options) {
     identifier,
     idType,
     fulltextDirectory,
-    force: force ?? false
+    force: force ?? false,
+    fulltextConfig
   };
   try {
     const result = await downloadAndAttach(
@@ -10573,6 +10961,19 @@ function buildDownloadError(locations, identifier, attempts) {
   }
   return { success: false, error: `Failed to download fulltext for ${identifier}`, hint };
 }
+async function tryMarkdownConversion(locations, pmcid, tempDir, ctx, attachedFiles, attempts) {
+  if (pmcid) {
+    const mdAttached = await tryDownloadPmcXmlAndConvert(pmcid, tempDir, ctx, attempts);
+    if (mdAttached) return "pmc";
+  }
+  const arxivResult = await tryArxivHtmlFromLocations(locations, tempDir, ctx, attempts);
+  if (arxivResult.attached) return arxivResult.source;
+  if (attachedFiles.includes("pdf") && ctx.fulltextConfig.preferredType === "markdown") {
+    const converted = await tryPdfToMarkdownConvert(ctx);
+    if (converted) return "pdf-convert";
+  }
+  return void 0;
+}
 async function downloadAndAttach(locations, pmcid, tempDir, ctx, referenceId, identifier) {
   const attachedFiles = [];
   let usedSource = "";
@@ -10582,19 +10983,17 @@ async function downloadAndAttach(locations, pmcid, tempDir, ctx, referenceId, id
     attachedFiles.push("pdf");
     usedSource = pdfResult.source;
   }
-  if (pmcid) {
-    const mdAttached = await tryDownloadPmcXmlAndConvert(pmcid, tempDir, ctx, attempts);
-    if (mdAttached) {
-      attachedFiles.push("markdown");
-      if (!usedSource) usedSource = "pmc";
-    }
-  }
-  if (!attachedFiles.includes("markdown")) {
-    const arxivResult = await tryArxivHtmlFromLocations(locations, tempDir, ctx, attempts);
-    if (arxivResult.attached) {
-      attachedFiles.push("markdown");
-      if (!usedSource) usedSource = arxivResult.source;
-    }
+  const mdSource = await tryMarkdownConversion(
+    locations,
+    pmcid,
+    tempDir,
+    ctx,
+    attachedFiles,
+    attempts
+  );
+  if (mdSource) {
+    attachedFiles.push("markdown");
+    if (!usedSource) usedSource = mdSource;
   }
   if (attachedFiles.length > 0) {
     return { success: true, referenceId, source: usedSource, attachedFiles };
@@ -10604,64 +11003,24 @@ async function downloadAndAttach(locations, pmcid, tempDir, ctx, referenceId, id
     attempts: attempts.length > 0 ? attempts : void 0
   };
 }
-const fetch$1 = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
-  __proto__: null,
-  fulltextFetch
-}, Symbol.toStringTag, { value: "Module" }));
-function findXmlFile(item) {
-  const attachments = item.custom?.attachments;
-  if (!attachments?.files) return void 0;
-  const xmlFile = attachments.files.find(
-    (f) => f.role === "fulltext" && f.filename.endsWith(".xml")
-  );
-  return xmlFile?.filename;
-}
-function getXmlPath(item, xmlFilename, fulltextDirectory) {
-  const attachments = item.custom?.attachments;
-  const directory = attachments?.directory ?? "";
-  return join(fulltextDirectory, directory, xmlFilename);
-}
-async function fulltextConvert(library, options) {
-  const { identifier, idType = "id", fulltextDirectory } = options;
-  const item = await library.find(identifier, { idType });
-  if (!item) {
-    return { success: false, error: `Reference '${identifier}' not found` };
-  }
-  const xmlFilename = findXmlFile(item);
-  if (!xmlFilename) {
-    return { success: false, error: `No PMC XML file attached to '${identifier}'` };
-  }
-  const xmlPath = getXmlPath(item, xmlFilename, fulltextDirectory);
+async function tryPdfToMarkdownConvert(ctx) {
   try {
-    await access(xmlPath);
+    const result = await fulltextConvert(ctx.library, {
+      identifier: ctx.identifier,
+      idType: ctx.idType,
+      fulltextDirectory: ctx.fulltextDirectory,
+      from: "pdf",
+      fulltextConfig: ctx.fulltextConfig
+    });
+    return result.success;
   } catch {
-    return { success: false, error: `XML file not found on disk: ${xmlPath}` };
-  }
-  const mdPath = xmlPath.replace(/\.xml$/, ".md");
-  const convertResult2 = await convertPmcXmlToMarkdown(xmlPath, mdPath);
-  if (!convertResult2.success) {
-    return {
-      success: false,
-      error: `Failed to convert PMC XML to Markdown: ${convertResult2.error ?? "unknown error"}`
-    };
-  }
-  const attachResult = await fulltextAttach(library, {
-    identifier,
-    idType,
-    filePath: mdPath,
-    type: "markdown",
-    force: true,
-    move: true,
-    fulltextDirectory
-  });
-  if (!attachResult.success) {
-    return { success: false, error: attachResult.error ?? "Attach failed" };
+    return false;
   }
-  return {
-    success: true,
-    filename: attachResult.filename ?? "fulltext.md"
-  };
 }
+const fetch$1 = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
+  __proto__: null,
+  fulltextFetch
+}, Symbol.toStringTag, { value: "Module" }));
 function formatFirstAuthor(item) {
   if (!item.author || item.author.length === 0) {
     return "Unknown";
@@ -12421,7 +12780,7 @@ function createAddRoute(library, config) {
 }
 const CHECK_CONCURRENCY = 5;
 async function checkReferences(library, options) {
-  const { checkReference } = await import("./checker-hg3aQ_DJ.js");
+  const { checkReference } = await import("./checker-DhHnmgq0.js");
   const save = options.save !== false;
   const skipDays = options.skipDays ?? 7;
   const items = await resolveItems(library, options);
@@ -12940,13 +13299,24 @@ function createReferencesRoute(library, config) {
     });
     route.post("/uuid/:uuid/fulltext/convert", async (c) => {
       const uuid = c.req.param("uuid");
+      const body = await c.req.json().catch(() => ({}));
+      const validFrom = ["xml", "pdf", void 0];
+      if (body.from !== void 0 && !validFrom.includes(body.from)) {
+        return c.json({ error: `Invalid 'from' value: must be 'xml', 'pdf', or omitted` }, 400);
+      }
+      if (body.converter !== void 0 && typeof body.converter !== "string") {
+        return c.json({ error: `Invalid 'converter' value: must be a string or omitted` }, 400);
+      }
       const result = await fulltextConvert(library, {
         identifier: uuid,
         idType: "uuid",
-        fulltextDirectory: config.attachments.directory
+        fulltextDirectory: config.attachments.directory,
+        from: body.from,
+        converter: body.converter,
+        fulltextConfig: config.fulltext
       });
       if (!result.success) {
-        return c.json({ error: result.error }, 400);
+        return c.json({ error: result.error, code: result.code, hints: result.hints }, 400);
       }
       return c.json(result);
     });
@@ -13100,4 +13470,4 @@ export {
   fetcher as y,
   add as z
 };
-//# sourceMappingURL=index-6Aild0Uk.js.map
+//# sourceMappingURL=index-DoOmAYKd.js.map