docrev 0.10.2 → 0.11.0

package/lib/rate-limiter.ts

@@ -7,6 +7,21 @@ export interface RateLimiterOptions {
   maxDelay?: number;
   maxRetries?: number;
   backoffFactor?: number;
+  /** Per-request timeout in ms; a stalled connection aborts instead of hanging. */
+  requestTimeout?: number;
+}
+
+/**
+ * Parse an HTTP `Retry-After` header, which may be a delay in seconds or an
+ * HTTP-date. Returns the delay in milliseconds, or null when unparseable.
+ */
+export function parseRetryAfter(value: string | null, now: number = Date.now()): number | null {
+  if (!value) return null;
+  const seconds = Number(value);
+  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
+  const date = Date.parse(value);
+  if (!Number.isNaN(date)) return Math.max(0, date - now);
+  return null;
 }
 
 export class RateLimiter {
@@ -14,6 +29,7 @@ export class RateLimiter {
   private maxDelay: number;
   private maxRetries: number;
   private backoffFactor: number;
+  private requestTimeout: number;
   private lastRequestTime: number;
   private currentDelay: number;
   private consecutiveErrors: number;
@@ -23,6 +39,7 @@ export class RateLimiter {
     this.maxDelay = options.maxDelay || 30000;    // Max delay after backoff (ms)
     this.maxRetries = options.maxRetries || 3;    // Max retry attempts
     this.backoffFactor = options.backoffFactor || 2;
+    this.requestTimeout = options.requestTimeout || 15000; // Abort a stalled request
     this.lastRequestTime = 0;
     this.currentDelay = this.minDelay;
     this.consecutiveErrors = 0;
@@ -54,17 +71,27 @@ export class RateLimiter {
 
   async fetchWithRetry(url: string, options: RequestInit = {}): Promise<Response> {
     let lastError: Error | undefined;
+    const callerSignal = options.signal ?? undefined;
 
     for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
       await this.wait();
 
+      // Bound each attempt so a half-open connection cannot hang the run
+      // forever; merge the caller's signal so an external cancel still works.
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(new Error('Request timed out')), this.requestTimeout);
+      const onCallerAbort = () => controller.abort((callerSignal as AbortSignal).reason);
+      if (callerSignal) {
+        if (callerSignal.aborted) controller.abort(callerSignal.reason);
+        else callerSignal.addEventListener('abort', onCallerAbort, { once: true });
+      }
+
       try {
-        const response = await fetch(url, options);
+        const response = await fetch(url, { ...options, signal: controller.signal });
 
         if (response.status === 429) {
-          // Rate limited - back off
-          const retryAfter = response.headers.get('Retry-After');
-          const delay = retryAfter ? parseInt(retryAfter, 10) * 1000 : this.currentDelay * 2;
+          // Rate limited - back off. Retry-After may be seconds or an HTTP-date.
+          const delay = parseRetryAfter(response.headers.get('Retry-After')) ?? this.currentDelay * 2;
           this.currentDelay = Math.min(this.maxDelay, delay);
           if (!this.onError(429)) break;
           continue;
@@ -79,8 +106,14 @@ export class RateLimiter {
         this.onSuccess();
         return response;
       } catch (err) {
+        // A caller-initiated abort is intentional cancellation, not a failure
+        // to retry around.
+        if (callerSignal?.aborted) throw err;
         lastError = err as Error;
         if (!this.onError(0)) break;
+      } finally {
+        clearTimeout(timer);
+        if (callerSignal) callerSignal.removeEventListener('abort', onCallerAbort);
       }
     }
 

package/lib/types.ts

@@ -218,7 +218,13 @@ export interface BibEntry {
 
 export interface DoiCheckResult {
   valid: boolean;
-  source?: 'crossref' | 'datacite';
+  source?: 'crossref' | 'datacite' | 'doi.org';
+  /**
+   * True when the registry could not be reached (network error, timeout, or
+   * 5xx) — the DOI's validity is unknown, distinct from a definitive "not
+   * found". Callers must not report an unreachable DOI as invalid.
+   */
+  unreachable?: boolean;
   metadata?: {
     title: string;
     authors: string[];
@@ -258,6 +264,8 @@ export interface BibCheckResult {
   entries: Array<BibEntry & { status: string; message?: string; metadata?: object }>;
   valid: number;
   invalid: number;
+  /** DOIs whose registry could not be reached (network/timeout) — not invalid. */
+  unreachable: number;
   missing: number;
   skipped: number;
 }

package/lib/word-extraction.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { exec } from 'child_process';
 import { promisify } from 'util';
+import { buildDocTextModel, buildCommentAnchorModel, extractComments, openDocx, readPartText } from './ooxml.js';
 
 const execAsync = promisify(exec);
 
@@ -99,130 +100,19 @@ export interface ExtractFromWordResult {
  * Extract comments directly from Word docx comments.xml
  */
 export async function extractWordComments(docxPath: string): Promise<WordComment[]> {
-  const AdmZip = (await import('adm-zip')).default;
-  const { parseStringPromise } = await import('xml2js');
-
-  const comments: WordComment[] = [];
-
-  // Validate file exists
   if (!fs.existsSync(docxPath)) {
     throw new Error(`File not found: ${docxPath}`);
   }
 
-  try {
-    let zip;
-    try {
-      zip = new AdmZip(docxPath);
-    } catch (err: any) {
-      throw new Error(`Invalid Word document (not a valid .docx file): ${err.message}`);
-    }
-
-    const commentsEntry = zip.getEntry('word/comments.xml');
-
-    if (!commentsEntry) {
-      return comments;
-    }
-
-    let commentsXml;
-    try {
-      commentsXml = commentsEntry.getData().toString('utf8');
-    } catch (err: any) {
-      throw new Error(`Failed to read comments from document: ${err.message}`);
-    }
-
-    const parsed = await parseStringPromise(commentsXml, { explicitArray: false });
-
-    const commentsRoot = parsed['w:comments'];
-    if (!commentsRoot || !commentsRoot['w:comment']) {
-      return comments;
-    }
-
-    // Ensure it's an array
-    const commentNodes = Array.isArray(commentsRoot['w:comment'])
-      ? commentsRoot['w:comment']
-      : [commentsRoot['w:comment']];
-
-    // Map every paraId that lives inside a comment back to that comment's id.
-    // Word's commentsExtended.xml expresses threading via w15:paraIdParent,
-    // which references the parent's first <w:p>. Replies use a secondary
-    // (often-empty) <w:p>, so each comment may contribute multiple paraIds.
-    const paraIdToCommentId = new Map<string, string>();
-
-    for (const comment of commentNodes) {
-      const id = comment.$?.['w:id'] || '';
-      const author = comment.$?.['w:author'] || 'Unknown';
-      const date = comment.$?.['w:date'] || '';
-
-      // Extract text from nested w:p/w:r/w:t elements and record paraIds.
-      let text = '';
-      const extractText = (node: any): void => {
-        if (!node) return;
-        if (typeof node === 'string') {
-          text += node;
-          return;
-        }
-        if (node['w:t']) {
-          const t = node['w:t'];
-          text += typeof t === 'string' ? t : (t._ || t);
-        }
-        if (node['w:r']) {
-          const runs = Array.isArray(node['w:r']) ? node['w:r'] : [node['w:r']];
-          runs.forEach(extractText);
-        }
-        if (node['w:p']) {
-          const paras = Array.isArray(node['w:p']) ? node['w:p'] : [node['w:p']];
-          for (const para of paras) {
-            const paraId = para?.$?.['w14:paraId'];
-            if (paraId && id) paraIdToCommentId.set(paraId, id);
-            extractText(para);
-          }
-        }
-      };
-      extractText(comment);
-
-      comments.push({ id, author, date: date.slice(0, 10), text: text.trim() });
-    }
-
-    // Resolve parent links from commentsExtended.xml. Missing entry just
-    // means the docx has no threading metadata (e.g. legacy/non-Word source).
-    const extendedEntry = zip.getEntry('word/commentsExtended.xml');
-    if (extendedEntry && paraIdToCommentId.size > 0) {
-      let extendedXml = '';
-      try {
-        extendedXml = extendedEntry.getData().toString('utf8');
-      } catch {
-        // Unreadable threading metadata is non-fatal; skip parent linking.
-      }
-      if (extendedXml) {
-        const parentByCommentId = new Map<string, string>();
-        const exPattern = /<w15:commentEx\b([^>]*?)\/>/g;
-        let m: RegExpExecArray | null;
-        while ((m = exPattern.exec(extendedXml)) !== null) {
-          const attrs = m[1] ?? '';
-          const paraIdMatch = attrs.match(/w15:paraId="([^"]+)"/);
-          const parentMatch = attrs.match(/w15:paraIdParent="([^"]+)"/);
-          if (!paraIdMatch || !parentMatch) continue;
-          const childCommentId = paraIdToCommentId.get(paraIdMatch[1]);
-          const parentCommentId = paraIdToCommentId.get(parentMatch[1]);
-          if (childCommentId && parentCommentId && childCommentId !== parentCommentId) {
-            parentByCommentId.set(childCommentId, parentCommentId);
-          }
-        }
-        for (const c of comments) {
-          const parent = parentByCommentId.get(c.id);
-          if (parent) c.parentId = parent;
-        }
-      }
-    }
-  } catch (err: any) {
-    // Re-throw with more context if it's already an Error we created
-    if (err.message.includes('Invalid Word document') || err.message.includes('File not found')) {
-      throw err;
-    }
-    throw new Error(`Error extracting comments from ${path.basename(docxPath)}: ${err.message}`);
-  }
-
-  return comments;
+  const zip = openDocx(docxPath);
+  // Word truncates the stored date to its day for display; keep that contract.
+  return extractComments(zip).map((c) => ({
+    id: c.id,
+    author: c.author,
+    date: c.date.slice(0, 10),
+    text: c.text,
+    parentId: c.parentId,
+  }));
 }
 
 /**
@@ -231,159 +121,41 @@ export async function extractWordComments(docxPath: string): Promise<WordComment
  * Also returns fullDocText for section boundary matching
  */
 export async function extractCommentAnchors(docxPath: string): Promise<CommentAnchorsResult> {
-  const AdmZip = (await import('adm-zip')).default;
   const anchors = new Map<string, CommentAnchorData>();
-  let fullDocText = '';
-
-  try {
-    const zip = new AdmZip(docxPath);
-    const docEntry = zip.getEntry('word/document.xml');
-
-    if (!docEntry) {
-      return { anchors, fullDocText };
-    }
-
-    const docXml = docEntry.getData().toString('utf8');
-
-    // ========================================
-    // STEP 1: Build text position mapping
-    // ========================================
-    const textNodePattern = /<w:t[^>]*>([^<]*)<\/w:t>/g;
-    const textNodes: TextNode[] = [];
-    let textPosition = 0;
-    let nodeMatch;
-
-    while ((nodeMatch = textNodePattern.exec(docXml)) !== null) {
-      const rawText = nodeMatch[1] ?? '';
-      const decodedText = decodeXmlEntities(rawText);
-      textNodes.push({
-        xmlStart: nodeMatch.index,
-        xmlEnd: nodeMatch.index + nodeMatch[0].length,
-        textStart: textPosition,
-        textEnd: textPosition + decodedText.length,
-        text: decodedText
-      });
-      textPosition += decodedText.length;
-    }
-
-    fullDocText = textNodes.map(n => n.text).join('');
 
-    // Helper: convert XML position to text position
-    function xmlPosToTextPos(xmlPos: number): number {
-      for (let i = 0; i < textNodes.length; i++) {
-        const node = textNodes[i];
-        if (!node) continue;
-        if (xmlPos >= node.xmlStart && xmlPos < node.xmlEnd) {
-          return node.textStart;
-        }
-        if (xmlPos < node.xmlStart) {
-          return node.textStart;
-        }
-      }
-      const lastNode = textNodes[textNodes.length - 1];
-      return lastNode ? lastNode.textEnd : 0;
-    }
-
-    // Helper: extract context before a position
-    function getContextBefore(position: number, maxLength: number = 150): string {
-      const beforeText = fullDocText.slice(Math.max(0, position - maxLength), position);
-      const sentenceStart = beforeText.search(/[.!?]\s+[A-Z][^.!?]*$/);
-      return sentenceStart >= 0
-        ? beforeText.slice(sentenceStart + 2).trim()
-        : beforeText.slice(-80).trim();
-    }
-
-    // Helper: extract context after a position
-    function getContextAfter(position: number, maxLength: number = 150): string {
-      const afterText = fullDocText.slice(position, position + maxLength);
-      const sentenceEnd = afterText.search(/[.!?]\s/);
-      return sentenceEnd >= 0
-        ? afterText.slice(0, sentenceEnd + 1).trim()
-        : afterText.slice(0, 80).trim();
-    }
-
-    // ========================================
-    // STEP 2: Collect all start/end markers separately
-    // ========================================
-    const startPattern = /<w:commentRangeStart[^>]*w:id="(\d+)"[^>]*\/?>/g;
-    const endPattern = /<w:commentRangeEnd[^>]*w:id="(\d+)"[^>]*\/?>/g;
-
-    const starts = new Map<string, number>();  // id -> position after start tag
-    const ends = new Map<string, number>();    // id -> position before end tag
-
-    let match;
-    while ((match = startPattern.exec(docXml)) !== null) {
-      const id = match[1];
-      if (!starts.has(id)) {
-        starts.set(id, match.index + match[0].length);
-      }
-    }
-
-    while ((match = endPattern.exec(docXml)) !== null) {
-      const id = match[1];
-      if (!ends.has(id)) {
-        ends.set(id, match.index);
-      }
-    }
-
-    // ========================================
-    // STEP 3: Process each comment range by ID
-    // ========================================
-    for (const [id, startXmlPos] of starts) {
-      const endXmlPos = ends.get(id);
-
-      // Missing end marker - skip with warning
-      if (endXmlPos === undefined) {
-        console.warn(`Comment ${id}: missing end marker`);
-        continue;
-      }
+  const zip = openDocx(docxPath);
+  const { fullDocText, comments } = buildCommentAnchorModel(zip);
+  if (!fullDocText && comments.length === 0) {
+    return { anchors, fullDocText: '' };
+  }
 
-      // Calculate text position
-      const docPosition = xmlPosToTextPos(startXmlPos);
-
-      // Handle empty or inverted ranges
-      if (endXmlPos <= startXmlPos) {
-        anchors.set(id, {
-          anchor: '',
-          before: getContextBefore(docPosition),
-          after: getContextAfter(docPosition),
-          docPosition,
-          docLength: fullDocText.length,
-          isEmpty: true
-        });
-        continue;
-      }
+  // Context surrounding the anchor, taken from the same plain-text coordinate
+  // system as docPosition so the placement engine can compare like with like.
+  function getContextBefore(position: number, maxLength: number = 150): string {
+    const beforeText = fullDocText.slice(Math.max(0, position - maxLength), position);
+    const sentenceStart = beforeText.search(/[.!?]\s+[A-Z][^.!?]*$/);
+    return sentenceStart >= 0
+      ? beforeText.slice(sentenceStart + 2).trim()
+      : beforeText.slice(-80).trim();
+  }
 
-      // Extract XML segment between markers
-      const segment = docXml.slice(startXmlPos, endXmlPos);
+  function getContextAfter(position: number, maxLength: number = 150): string {
+    const afterText = fullDocText.slice(position, position + maxLength);
+    const sentenceEnd = afterText.search(/[.!?]\s/);
+    return sentenceEnd >= 0
+      ? afterText.slice(0, sentenceEnd + 1).trim()
+      : afterText.slice(0, 80).trim();
+  }
 
-      // Extract text from w:t (regular) AND w:delText (deleted text in track changes)
-      const textInRangePattern = /<w:t[^>]*>([^<]*)<\/w:t>|<w:delText[^>]*>([^<]*)<\/w:delText>/g;
-      let anchorText = '';
-      let tm;
-      while ((tm = textInRangePattern.exec(segment)) !== null) {
-        anchorText += tm[1] || tm[2] || '';
-      }
-      anchorText = decodeXmlEntities(anchorText);
-
-      // Get context
-      const anchorLength = anchorText.length;
-      const before = getContextBefore(docPosition);
-      const after = getContextAfter(docPosition + anchorLength);
-
-      // ALWAYS add entry (even if anchor is empty)
-      anchors.set(id, {
-        anchor: anchorText.trim(),
-        before,
-        after,
-        docPosition,
-        docLength: fullDocText.length,
-        isEmpty: !anchorText.trim()
-      });
-    }
-  } catch (err: any) {
-    console.error('Error extracting comment anchors:', err.message);
-    return { anchors, fullDocText: '' };
+  for (const range of comments) {
+    anchors.set(range.id, {
+      anchor: range.anchor,
+      before: getContextBefore(range.start),
+      after: getContextAfter(range.end),
+      docPosition: range.start,
+      docLength: fullDocText.length,
+      isEmpty: range.isEmpty,
+    });
   }
 
   return { anchors, fullDocText };
@@ -402,73 +174,20 @@ export async function extractCommentAnchors(docxPath: string): Promise<CommentAn
  * concatenated.
  */
 export async function extractHeadings(docxPath: string): Promise<DocxHeading[]> {
-  const AdmZip = (await import('adm-zip')).default;
-
   if (!fs.existsSync(docxPath)) {
     throw new Error(`File not found: ${docxPath}`);
   }
 
-  const zip = new AdmZip(docxPath);
-  const docEntry = zip.getEntry('word/document.xml');
-  if (!docEntry) return [];
-  const xml = docEntry.getData().toString('utf8');
-
-  // Build the same xml-pos → text-pos mapping that extractCommentAnchors does
-  const textNodePattern = /<w:t[^>]*>([^<]*)<\/w:t>/g;
-  const nodes: Array<{ xmlStart: number; xmlEnd: number; textStart: number; textEnd: number }> = [];
-  let textPos = 0;
-  let m;
-  while ((m = textNodePattern.exec(xml)) !== null) {
-    const decoded = decodeXmlEntities(m[1] ?? '');
-    nodes.push({
-      xmlStart: m.index,
-      xmlEnd: m.index + m[0].length,
-      textStart: textPos,
-      textEnd: textPos + decoded.length,
-    });
-    textPos += decoded.length;
-  }
-
-  function xmlToTextPos(xmlPos: number): number {
-    for (const n of nodes) {
-      if (xmlPos >= n.xmlStart && xmlPos < n.xmlEnd) return n.textStart;
-      if (xmlPos < n.xmlStart) return n.textStart;
-    }
-    return nodes.length ? nodes[nodes.length - 1].textEnd : 0;
-  }
-
-  const headings: DocxHeading[] = [];
-  const paraPattern = /<w:p\b[^>]*>([\s\S]*?)<\/w:p>/g;
-  let pm;
-  while ((pm = paraPattern.exec(xml)) !== null) {
-    const inner = pm[1];
-    const styleMatch = inner.match(/<w:pStyle[^>]*w:val="([^"]+)"/);
-    if (!styleMatch) continue;
-    const style = styleMatch[1];
-    if (!/heading/i.test(style)) continue;
-
-    // Concatenate text runs; include w:delText so a heading inside a tracked
-    // deletion is still surfaced (verifying anchors against an original draft)
-    const textInRange = /<w:t[^>]*>([^<]*)<\/w:t>|<w:delText[^>]*>([^<]*)<\/w:delText>/g;
-    let txt = '';
-    let tm;
-    while ((tm = textInRange.exec(inner)) !== null) {
-      txt += decodeXmlEntities(tm[1] || tm[2] || '');
-    }
-    const trimmed = txt.trim();
-    if (!trimmed) continue;
-
-    const levelMatch = style.match(/(\d+)/);
-    const level = levelMatch ? parseInt(levelMatch[1], 10) : 0;
-    headings.push({
-      style,
-      level,
-      text: trimmed,
-      docPosition: xmlToTextPos(pm.index),
-    });
-  }
+  const zip = openDocx(docxPath);
+  const docXml = readPartText(zip, 'word/document.xml');
+  if (docXml === null) return [];
 
-  return headings;
+  return buildDocTextModel(docXml).headings.map((h) => ({
+    style: h.style,
+    level: h.level,
+    text: h.text,
+    docPosition: h.position,
+  }));
 }
 
 /**