dsh-book-reader 0.2.0 → 0.3.0

package/lib/core/pdf.js

@@ -6,7 +6,9 @@
  *   - a small LRU cache of parsed PDF.js documents (path + mtime + size),
  *   - metadata (page labels, detected printed-page offset, resolved outline),
  *   - whitespace-normalised page text,
- *   - literal full-text search with physical *and* printed page numbers.
+ *   - full-text search with physical *and* printed page numbers.  Search is
+ *     literal-ish on NFKC-normalised text, with compact (space-insensitive)
+ *     and all-terms fallbacks, body pages ranked before front matter.
  *
  * Design constraints that shaped this file:
  *
@@ -850,19 +852,438 @@ export async function getPageText(path, pdfPage) {
   })
 }
 
+// ---------------------------------------------------------------------------
+// Query cleanup + searchable text layout
+// ---------------------------------------------------------------------------
+
+/** Longest accepted query after wrapper/whitespace cleanup. */
+export const MAX_SEARCH_QUERY_LENGTH = 1000
+
+/** At most this many front-matter hits are returned, after body hits. */
+export const MAX_FRONT_MATTER_HITS = 3
+
+/**
+ * Whole-query wrappers a model commonly adds.  Only a pair surrounding the
+ * entire query is stripped, so `don't` or an inner quote is left alone.
+ */
+const QUERY_WRAPPERS = [
+  ['"', '"'],
+  ["'", "'"],
+  ['“', '”'],
+  ['‘', '’'],
+  ['`', '`'],
+  ['$', '$'],
+]
+
+/** Unicode dash variants deleted for hyphen-insensitive matching. */
+const DASH_CHARACTERS = new Set(['-', '\u2010', '\u2011', '\u2012', '\u2013', '\u2014', '\u2015', '\u2212'])
+
+const SOFT_HYPHEN = '\u00ad'
+const WHITESPACE_RE = /\s/u
+
+/** Term-AND fallback: ignore one-character noise, examine at most 8 words. */
+const MAX_TERM_TOKENS = 8
+
+/** @param {string} value */
+function isWhitespace(value) {
+  return WHITESPACE_RE.test(value)
+}
+
+/**
+ * Trim a query, strip wrapping quotes/backticks/`$…$`, collapse whitespace,
+ * and reject empty or over-long results.  This is deliberately done before
+ * opening the PDF:
+ *
+ *   "eigenvalue"  -> eigenvalue
+ *   `L(V)`        -> L(V)
+ *   $T \in L(V)$  -> T \in L(V)
+ *
+ * @param {string} raw
+ * @returns {string}
+ */
+export function cleanSearchQuery(raw) {
+  if (typeof raw !== 'string') {
+    throw new TypeError('searchPdf: query must be a non-empty string')
+  }
+
+  let query = raw.trim()
+  let stripped = true
+  while (stripped && query.length >= 2) {
+    stripped = false
+    for (const [open, close] of QUERY_WRAPPERS) {
+      if (query.startsWith(open) && query.endsWith(close)) {
+        query = query.slice(open.length, query.length - close.length).trim()
+        stripped = true
+        break
+      }
+    }
+  }
+
+  query = query.replace(/\s+/gu, ' ').trim()
+  if (query === '') {
+    throw new TypeError('searchPdf: query must be a non-empty string')
+  }
+  if (query.length > MAX_SEARCH_QUERY_LENGTH) {
+    const error = new RangeError(
+      `searchPdf: query is longer than ${MAX_SEARCH_QUERY_LENGTH} characters`,
+    )
+    error.code = 'PDF_SEARCH_QUERY_TOO_LONG'
+    throw error
+  }
+  return query
+}
+
+/** Read one code point (a surrogate pair stays together). */
+function readCodePointAt(text, index) {
+  const code = text.charCodeAt(index)
+  if (code >= 0xd800 && code <= 0xdbff && index + 1 < text.length) {
+    const next = text.charCodeAt(index + 1)
+    if (next >= 0xdc00 && next <= 0xdfff) {
+      return { ch: text.slice(index, index + 2), width: 2 }
+    }
+  }
+  return { ch: text[index], width: 1 }
+}
+
+/**
+ * Build NFKC-normalised text plus a `normalizedIndex -> originalIndex` map.
+ *
+ * Normalisation rules (matching-only; snippets still come from the original):
+ *   - NFKC per code point: `ℒ (𝑉)` -> `L (V)`, math italics -> ASCII,
+ *     `fi` -> `fi`, full-width letters/digits -> ASCII.
+ *   - soft hyphens (U+00AD) are removed;
+ *   - every dash (U+002D, U+2010–U+2015, U+2212) is removed.  This makes
+ *     matching hyphen-insensitive.  When a dash is followed by whitespace --
+ *     which is what a line-break hyphen looks like after extraction -- the
+ *     whitespace is removed too, so `com- plex` becomes `complex`.
+ *   - whitespace runs collapse to one space; leading/trailing space is
+ *     dropped.
+ *
+ * The map covers every UTF-16 unit of the result, so callers can cut snippets
+ * from the original text.
+ *
+ * @param {string} original already whitespace-collapsed source text
+ * @returns {{text: string, map: number[]}}
+ */
+function buildNormalizedText(original) {
+  const chars = []
+  const map = []
+  let pendingSpaceAt = -1
+  let index = 0
+
+  while (index < original.length) {
+    const { ch, width } = readCodePointAt(original, index)
+    if (ch === SOFT_HYPHEN) {
+      index += width
+      continue
+    }
+
+    // ASCII is NFKC-stable; skip the costlier normalisation call for it.
+    const folded = ch.charCodeAt(0) < 0x80 ? ch : ch.normalize('NFKC')
+
+    if (DASH_CHARACTERS.has(folded)) {
+      index += width
+      // Line-break hyphenation: join the two halves by dropping the
+      // whitespace that follows the dash as well.
+      while (index < original.length) {
+        const next = readCodePointAt(original, index)
+        if (!isWhitespace(next.ch)) break
+        index += next.width
+      }
+      continue
+    }
+
+    if (isWhitespace(folded)) {
+      if (chars.length > 0 && pendingSpaceAt < 0) pendingSpaceAt = index
+      index += width
+      continue
+    }
+
+    if (pendingSpaceAt >= 0) {
+      chars.push(' ')
+      map.push(pendingSpaceAt)
+      pendingSpaceAt = -1
+    }
+    // A code point may expand (`fi` -> `fi`); every produced unit maps back to
+    // the same source code point.
+    for (let offset = 0; offset < folded.length; offset += 1) {
+      chars.push(folded[offset])
+      map.push(index)
+    }
+    index += width
+  }
+
+  return { text: chars.join(''), map }
+}
+
+/**
+ * Normalise text for matching (NFKC, dash-insensitive, single-spaced).
+ * Exported for tests; searchPdf uses the same rules.
+ *
+ * @param {string} raw
+ * @returns {string}
+ */
+export function normalizeSearchText(raw) {
+  const original = String(raw ?? '')
+    .replace(/\s+/gu, ' ')
+    .trim()
+  return buildNormalizedText(original).text
+}
+
+/**
+ * One page's matching views.  `original` is exactly what getPageText()
+ * returns, so `indexOnPage` and snippets use the host-facing text.
+ *
+ * @typedef {Object} SearchLayout
+ * @property {string} original
+ * @property {string} normalized
+ * @property {number[]} normalizedToOriginal
+ * @property {string} compact
+ * @property {number[]} compactToNormalized
+ * @property {string} normalizedLower
+ * @property {number[]} lowerToNormalized
+ */
+
+/**
+ * @param {string} rawText
+ * @returns {SearchLayout}
+ */
+export function buildSearchLayout(rawText) {
+  const original = String(rawText ?? '')
+    .replace(/\s+/gu, ' ')
+    .trim()
+  const { text: normalized, map: normalizedToOriginal } = buildNormalizedText(original)
+
+  const compactChars = []
+  const compactToNormalized = []
+  for (let index = 0; index < normalized.length; index += 1) {
+    if (normalized[index] === ' ') continue
+    compactChars.push(normalized[index])
+    compactToNormalized.push(index)
+  }
+
+  // Case-folded view for the all-terms pass, with its own index map so a
+  // snippet can still be traced back to `original`.
+  const lowerChars = []
+  const lowerToNormalized = []
+  for (let index = 0; index < normalized.length; ) {
+    const { ch, width } = readCodePointAt(normalized, index)
+    const lower = ch.toLowerCase()
+    for (let offset = 0; offset < lower.length; offset += 1) {
+      lowerChars.push(lower[offset])
+      lowerToNormalized.push(index)
+    }
+    index += width
+  }
+
+  return {
+    original,
+    normalized,
+    normalizedToOriginal,
+    compact: compactChars.join(''),
+    compactToNormalized,
+    normalizedLower: lowerChars.join(''),
+    lowerToNormalized,
+  }
+}
+
+/**
+ * Prepare both the user-facing (cleaned) query and its matchable form.
+ *
+ * @param {string} raw
+ * @returns {{query: string, needle: string}}
+ */
+export function prepareSearchQuery(raw) {
+  const query = cleanSearchQuery(raw)
+  const needle = normalizeSearchText(query)
+  if (needle === '') {
+    // e.g. a query made only of dashes, which normalisation deletes.
+    throw new TypeError('searchPdf: query must be a non-empty string')
+  }
+  return { query, needle }
+}
+
+// ---------------------------------------------------------------------------
+// Matching helpers
+// ---------------------------------------------------------------------------
+
 function escapeRegExp(value) {
   return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 }
 
 function makeSnippet(text, index, matchLength, contextChars) {
-  const start = Math.max(0, index - contextChars)
-  const end = Math.min(text.length, index + matchLength + contextChars)
+  let start = Math.max(0, index - contextChars)
+  let end = Math.min(text.length, index + matchLength + contextChars)
+  // Do not cut a surrogate pair in half: a math-italic glyph sits next to
+  // almost every hit in a formula-heavy book, and half a pair renders as U+FFFD.
+  if (start > 0 && start < text.length) {
+    const code = text.charCodeAt(start)
+    if (code >= 0xdc00 && code <= 0xdfff && isHighSurrogate(text.charCodeAt(start - 1))) start -= 1
+  }
+  if (end > 0 && end < text.length) {
+    const code = text.charCodeAt(end - 1)
+    if (isHighSurrogate(code) && isLowSurrogate(text.charCodeAt(end))) end += 1
+  }
   let snippet = text.slice(start, end).trim()
   if (start > 0) snippet = `…${snippet}`
   if (end < text.length) snippet = `${snippet}…`
   return snippet
 }
 
+function isHighSurrogate(code) {
+  return code >= 0xd800 && code <= 0xdbff
+}
+
+function isLowSurrogate(code) {
+  return code >= 0xdc00 && code <= 0xdfff
+}
+
+/** @param {string} needle @param {boolean} ignoreCase */
+function makeSearchPattern(needle, ignoreCase) {
+  return new RegExp(escapeRegExp(needle), ignoreCase ? 'giu' : 'gu')
+}
+
+/** Yield every match; guards a future zero-length needle against a spin. */
+function* iterateMatches(pattern, text) {
+  pattern.lastIndex = 0
+  let match
+  while ((match = pattern.exec(text)) !== null) {
+    yield match
+    if (match[0].length === 0) pattern.lastIndex += 1
+  }
+}
+
+/** Map a [start, end) range in `normalized` back to the original text. */
+function normalizedRangeToOriginal(layout, startIndex, endIndex) {
+  const start = layout.normalizedToOriginal[startIndex] ?? 0
+  const end =
+    endIndex < layout.normalized.length
+      ? layout.normalizedToOriginal[endIndex] ?? layout.original.length
+      : layout.original.length
+  return { start, length: Math.max(0, end - start) }
+}
+
+/** Map a [start, end) range in `compact` back to the original text. */
+function compactRangeToOriginal(layout, startIndex, endIndex) {
+  const normalizedStart = layout.compactToNormalized[startIndex] ?? 0
+  const normalizedEnd =
+    endIndex < layout.compact.length
+      ? layout.compactToNormalized[endIndex] ?? layout.normalized.length
+      : layout.normalized.length
+  return normalizedRangeToOriginal(layout, normalizedStart, normalizedEnd)
+}
+
+/** Map a [start, end) range in `normalizedLower` back to the original text. */
+function lowerRangeToOriginal(layout, startIndex, endIndex) {
+  const normalizedStart = layout.lowerToNormalized[startIndex] ?? 0
+  const normalizedEnd =
+    endIndex < layout.normalizedLower.length
+      ? layout.lowerToNormalized[endIndex] ?? layout.normalized.length
+      : layout.normalized.length
+  return normalizedRangeToOriginal(layout, normalizedStart, normalizedEnd)
+}
+
+function makeSearchHit(layout, pdfPage, printed, range, contextChars) {
+  return {
+    pdf: pdfPage,
+    printed,
+    indexOnPage: range.start,
+    snippet: makeSnippet(layout.original, range.start, range.length, contextChars),
+  }
+}
+
+/**
+ * True when `printed` is a body-page number.  Roman numerals / missing labels
+ * (front matter) return false.
+ *
+ * @param {number|string|null} printed
+ * @returns {boolean}
+ */
+export function isBodyPage(printed) {
+  return typeof printed === 'number' && Number.isFinite(printed)
+}
+
+/**
+ * Body hits first (page/offset order), then up to `frontMatterLimit` front
+ * matter hits, filling no more than `limit` slots in total.
+ *
+ * @param {Array<{pdf:number, printed:number|string|null, indexOnPage?:number}>} hits
+ * @param {{limit?: number, frontMatterLimit?: number}} [options]
+ * @returns {Array<Object>}
+ */
+export function rankSearchHits(hits, { limit = 200, frontMatterLimit = MAX_FRONT_MATTER_HITS } = {}) {
+  if (!Array.isArray(hits) || hits.length === 0 || limit < 1) return []
+  const byPosition = (a, b) =>
+    (Number(a.pdf) || 0) - (Number(b.pdf) || 0) ||
+    (Number(a.indexOnPage) || 0) - (Number(b.indexOnPage) || 0)
+  const body = hits.filter((hit) => isBodyPage(hit.printed)).sort(byPosition)
+  const front = hits.filter((hit) => !isBodyPage(hit.printed)).sort(byPosition)
+  const selected = body.slice(0, limit)
+  const room = Math.max(0, limit - selected.length)
+  return [...selected, ...front.slice(0, Math.min(frontMatterLimit, room))]
+}
+
+/**
+ * Split a normalised query into term-AND tokens.  One-character noise is
+ * dropped and at most MAX_TERM_TOKENS words are examined.
+ */
+function buildSearchTerms(needle, ignoreCase) {
+  const tokens = needle
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((token) => token.length >= 2)
+    .slice(0, MAX_TERM_TOKENS)
+  return ignoreCase ? tokens.map((token) => token.toLowerCase()) : tokens
+}
+
+/** Earliest position in a page where every term occurs, or null. */
+function findTermsSpan(layout, terms, ignoreCase) {
+  if (terms.length === 0) return null
+  const haystack = ignoreCase ? layout.normalizedLower : layout.normalized
+  let span = null
+  for (const term of terms) {
+    const index = haystack.indexOf(term)
+    if (index < 0) return null
+    if (span === null || index < span.start) span = { start: index, length: term.length }
+  }
+  return span
+}
+
+/**
+ * Collect ranked hits for one matching pass without keeping the whole book in
+ * memory.  Pages may be noted even after the hit caps are full, because
+ * `pages` must count every matched page.
+ */
+function createSearchPass(limit, frontMatterLimit = MAX_FRONT_MATTER_HITS) {
+  const bodyHits = []
+  const frontHits = []
+  let pages = 0
+  let countedPdfPage = -1
+
+  return {
+    get pages() {
+      return pages
+    },
+    notePage(pdfPage) {
+      if (countedPdfPage !== pdfPage) {
+        countedPdfPage = pdfPage
+        pages += 1
+      }
+    },
+    canAdd(printed) {
+      return isBodyPage(printed) ? bodyHits.length < limit : frontHits.length < frontMatterLimit
+    },
+    add(hit, printed) {
+      if (isBodyPage(printed)) bodyHits.push(hit)
+      else frontHits.push(hit)
+    },
+    finish() {
+      // Body hits were added first; slice applies the overall limit while the
+      // front-matter cap above keeps at most three from the tail.
+      return { hits: [...bodyHits, ...frontHits].slice(0, limit), pages }
+    },
+  }
+}
+
 function normalizeSearchOptions(options) {
   const opts = options && typeof options === 'object' ? options : {}
   const ignoreCase = opts.ignoreCase !== false
@@ -879,37 +1300,91 @@ function normalizeSearchOptions(options) {
     )
   }
 
-  return { ignoreCase, limit, contextChars }
+  // Optional 1-based physical page filter. `null` means the whole document.
+  let pdfPages = null
+  if (opts.pdfPages !== undefined && opts.pdfPages !== null) {
+    if (!Array.isArray(opts.pdfPages)) {
+      throw new RangeError('searchPdf: pdfPages must be an array of 1-based page numbers')
+    }
+    const unique = []
+    for (const page of opts.pdfPages) {
+      if (!Number.isInteger(page) || page < 1) {
+        throw new RangeError(
+          `searchPdf: pdfPages must contain positive integers (got ${JSON.stringify(page)})`,
+        )
+      }
+      if (!unique.includes(page)) unique.push(page)
+    }
+    if (unique.length === 0) throw new RangeError('searchPdf: pdfPages must not be empty')
+    pdfPages = unique.sort((a, b) => a - b)
+  }
+
+  return { ignoreCase, limit, contextChars, pdfPages }
 }
 
 /**
- * Literal full-text search, one page at a time so the event loop is not
- * blocked for the whole book.  Returns physical and printed page numbers.
+ * Whole-document text search, one extraction pass per page.
+ *
+ * Matching is tried in this order, and the first pass with any hit decides the
+ * result (and `mode`):
+ *
+ *   1. `phrase`  -- literal search over NFKC-normalised page text, with
+ *                   wrapping quotes and dashes ignored and case-insensitive by
+ *                   default.  Hyphenation is normalised on both sides, so
+ *                   `complex` finds a page containing `com- plex`.
+ *   2. `compact` -- only when no phrase matched: all spaces are removed from
+ *                   both needle and page, so `L(V)` finds `ℒ (𝑉)` and
+ *                   `T∈L(V)` finds `𝑆 , 𝑇 ∈ ℒ (𝑉)`.
+ *   3. `terms`   -- only when neither matched: every letter/digit token of the
+ *                   query must occur somewhere on the page (AND), at most one
+ *                   hit per page.
+ *
+ * Hits are ranked body pages first, then front matter (at most
+ * MAX_FRONT_MATTER_HITS front hits).  `pages` counts every matched page in the
+ * active pass before ranking/limit.
  *
  * `indexOnPage` is a UTF-16 offset into the same whitespace-normalised text
- * that getPageText() returns.
+ * that getPageText() returns (the original, not the normalised search view),
+ * and snippets are cut from that original text.
  *
  * @param {string} path
  * @param {string} query
  * @param {{ignoreCase?: boolean, limit?: number, contextChars?: number}} [options]
  * @returns {Promise<{hits: Array<{pdf:number, printed:number|string|null,
- *   indexOnPage:number, snippet:string}>, pages:number}>}
+ *   indexOnPage:number, snippet:string}>, pages:number,
+ *   mode: 'phrase'|'compact'|'terms'}>}
  */
 export async function searchPdf(path, query, options) {
-  if (typeof query !== 'string' || query.trim() === '') {
-    throw new TypeError('searchPdf: query must be a non-empty string')
-  }
+  const { needle } = prepareSearchQuery(query)
+  const { ignoreCase, limit, contextChars, pdfPages } = normalizeSearchOptions(options)
+
+  const phrasePattern = makeSearchPattern(needle, ignoreCase)
+  const compactNeedle = needle.replace(/ /gu, '')
+  const compactPattern =
+    compactNeedle === '' ? null : makeSearchPattern(compactNeedle, ignoreCase)
+  const terms = buildSearchTerms(needle, ignoreCase)
 
-  const needle = normalizePageText(query)
-  const { ignoreCase, limit, contextChars } = normalizeSearchOptions(options)
-  const pattern = new RegExp(escapeRegExp(needle), ignoreCase ? 'gi' : 'g')
+  const phrasePass = createSearchPass(limit)
+  const compactPass = createSearchPass(limit)
+  const termsPass = createSearchPass(limit)
 
   return withEntry(path, async (entry) => {
     const meta = await ensureMetadata(entry)
-    const hits = []
-    let pages = 0
+    if (pdfPages !== null) {
+      for (const page of pdfPages) {
+        if (page > entry.doc.numPages) {
+          const error = new RangeError(
+            `PDF page must be an integer between 1 and ${entry.doc.numPages} (got ${page})`,
+          )
+          error.code = 'PDF_PAGE_OUT_OF_RANGE'
+          throw error
+        }
+      }
+    }
+    const scanCount = pdfPages === null ? entry.doc.numPages : pdfPages.length
 
-    for (let pdfPage = 1; pdfPage <= entry.doc.numPages; pdfPage += 1) {
+    for (let scanIndex = 0; scanIndex < scanCount; scanIndex += 1) {
+      const pdfPage = pdfPages === null ? scanIndex + 1 : pdfPages[scanIndex]
       let text
       try {
         text = await extractPageText(entry, pdfPage)
@@ -924,29 +1399,83 @@ export async function searchPdf(path, query, options) {
         throw wrapped
       }
 
-      pattern.lastIndex = 0
-      let match
-      let pageCounted = false
-      while ((match = pattern.exec(text)) !== null) {
-        if (!pageCounted) {
-          pages += 1
-          pageCounted = true
+      const layout = buildSearchLayout(text)
+      const printed = printedForPdf(meta, pdfPage)
+
+      // 1. Phrase (always evaluated; it wins over any fallback).
+      let pageMatched = false
+      for (const match of iterateMatches(phrasePattern, layout.normalized)) {
+        if (!pageMatched) {
+          phrasePass.notePage(pdfPage)
+          pageMatched = true
         }
-        if (hits.length < limit) {
-          hits.push({
-            pdf: pdfPage,
-            printed: printedForPdf(meta, pdfPage),
-            indexOnPage: match.index,
-            snippet: makeSnippet(text, match.index, match[0].length, contextChars),
-          })
+        if (!phrasePass.canAdd(printed)) break
+        phrasePass.add(
+          makeSearchHit(
+            layout,
+            pdfPage,
+            printed,
+            normalizedRangeToOriginal(layout, match.index, match.index + match[0].length),
+            contextChars,
+          ),
+          printed,
+        )
+      }
+
+      // Once any phrase hit exists, the fallbacks can no longer win.  This
+      // also keeps the common case to ~one regex pass per page.
+      if (phrasePass.pages > 0) continue
+
+      // 2. Compact (space-insensitive).
+      if (compactPattern !== null) {
+        pageMatched = false
+        for (const match of iterateMatches(compactPattern, layout.compact)) {
+          if (!pageMatched) {
+            compactPass.notePage(pdfPage)
+            pageMatched = true
+          }
+          if (!compactPass.canAdd(printed)) break
+          compactPass.add(
+            makeSearchHit(
+              layout,
+              pdfPage,
+              printed,
+              compactRangeToOriginal(layout, match.index, match.index + match[0].length),
+              contextChars,
+            ),
+            printed,
+          )
         }
-        // A zero-length match is impossible (needle is non-empty), but guard
-        // anyway so a future change cannot spin forever.
-        if (match[0].length === 0) pattern.lastIndex += 1
+      }
+
+      if (compactPass.pages > 0) continue
+
+      // 3. All-terms AND fallback.
+      const span = findTermsSpan(layout, terms, ignoreCase)
+      if (span !== null) {
+        termsPass.notePage(pdfPage)
+        if (termsPass.canAdd(printed)) {
+          const range = ignoreCase
+            ? lowerRangeToOriginal(layout, span.start, span.start + span.length)
+            : normalizedRangeToOriginal(layout, span.start, span.start + span.length)
+          termsPass.add(makeSearchHit(layout, pdfPage, printed, range, contextChars), printed)
+        }
+      }
+    }
+
+    let mode = 'phrase'
+    let active = phrasePass.finish()
+    if (phrasePass.pages === 0) {
+      if (compactPass.pages > 0) {
+        mode = 'compact'
+        active = compactPass.finish()
+      } else if (termsPass.pages > 0) {
+        mode = 'terms'
+        active = termsPass.finish()
       }
     }
 
-    return { hits, pages }
+    return { hits: active.hits, pages: active.pages, mode }
   })
 }