@boxpdf/html-writer 0.1.9 → 0.1.10

package/dist/index.d.cts

@@ -1,5 +1,13 @@
 import { ExtractedPage } from '@boxpdf/reader';
 
+interface SemanticDocumentStats {
+    pagesProcessed: number;
+    peakBufferedPages: number;
+    peakBufferedLines: number;
+    mergedTables: number;
+    suppressedFurniture: number;
+}
+
 type HtmlLayout = "positioned" | "flow";
 type HtmlProfile = "visual" | "semantic";
 type HtmlWrite = (chunk: string) => void | Promise<void>;
@@ -12,9 +20,13 @@ interface HtmlWriterOptions {
     language?: string;
     includeDocument?: boolean;
     includeStyles?: boolean;
+    /** Maximum extracted page models retained for document-level semantic decisions. */
+    semanticLookaheadPages?: number;
+    /** Receives bounded-buffer and document-inference statistics after semantic output completes. */
+    onSemanticStats?: (stats: Readonly<SemanticDocumentStats>) => void;
 }
 declare function writeHtmlDocument(pages: AsyncIterable<ExtractedPage> | Iterable<ExtractedPage>, write: HtmlWrite, options?: HtmlWriterOptions): Promise<void>;
 declare function writePage(page: ExtractedPage, write: HtmlWrite, options?: HtmlWriterOptions): Promise<void>;
 declare function pageToHtml(page: ExtractedPage, options?: HtmlWriterOptions): Promise<string>;
 
-export { type HtmlLayout, type HtmlProfile, type HtmlWrite, type HtmlWriterOptions, pageToHtml, writeHtmlDocument, writePage };
+export { type HtmlLayout, type HtmlProfile, type HtmlWrite, type HtmlWriterOptions, type SemanticDocumentStats, pageToHtml, writeHtmlDocument, writePage };

package/dist/index.d.ts

@@ -1,5 +1,13 @@
 import { ExtractedPage } from '@boxpdf/reader';
 
+interface SemanticDocumentStats {
+    pagesProcessed: number;
+    peakBufferedPages: number;
+    peakBufferedLines: number;
+    mergedTables: number;
+    suppressedFurniture: number;
+}
+
 type HtmlLayout = "positioned" | "flow";
 type HtmlProfile = "visual" | "semantic";
 type HtmlWrite = (chunk: string) => void | Promise<void>;
@@ -12,9 +20,13 @@ interface HtmlWriterOptions {
     language?: string;
     includeDocument?: boolean;
     includeStyles?: boolean;
+    /** Maximum extracted page models retained for document-level semantic decisions. */
+    semanticLookaheadPages?: number;
+    /** Receives bounded-buffer and document-inference statistics after semantic output completes. */
+    onSemanticStats?: (stats: Readonly<SemanticDocumentStats>) => void;
 }
 declare function writeHtmlDocument(pages: AsyncIterable<ExtractedPage> | Iterable<ExtractedPage>, write: HtmlWrite, options?: HtmlWriterOptions): Promise<void>;
 declare function writePage(page: ExtractedPage, write: HtmlWrite, options?: HtmlWriterOptions): Promise<void>;
 declare function pageToHtml(page: ExtractedPage, options?: HtmlWriterOptions): Promise<string>;
 
-export { type HtmlLayout, type HtmlProfile, type HtmlWrite, type HtmlWriterOptions, pageToHtml, writeHtmlDocument, writePage };
+export { type HtmlLayout, type HtmlProfile, type HtmlWrite, type HtmlWriterOptions, type SemanticDocumentStats, pageToHtml, writeHtmlDocument, writePage };

package/dist/index.js

@@ -1,5 +1,163 @@
 // src/index.ts
-import { structurePage, tableToHtml } from "@boxpdf/reader/structure";
+import { structurePage as structurePage2, tableToHtml } from "@boxpdf/reader/structure";
+
+// src/semantic-document.ts
+import {
+  structurePage,
+  tableToRows
+} from "@boxpdf/reader/structure";
+async function writeSemanticDocument(pages, write, lookaheadPages) {
+  const stats = {
+    pagesProcessed: 0,
+    peakBufferedPages: 0,
+    peakBufferedLines: 0,
+    mergedTables: 0,
+    suppressedFurniture: 0
+  };
+  const buffer = [];
+  const seenFurniture = /* @__PURE__ */ new Set();
+  const sectionLevels = [];
+  let activeTable;
+  await write('<article class="pdf-semantic-document">');
+  const closeTable = async () => {
+    if (!activeTable) return;
+    await write("</table>");
+    activeTable = void 0;
+  };
+  const closeSections = async (minimumLevel = 0) => {
+    while ((sectionLevels.at(-1) ?? -1) >= minimumLevel && sectionLevels.length > 0) {
+      await write("</section>");
+      sectionLevels.pop();
+    }
+  };
+  const emitPage = async (page, future) => {
+    const futureFurniture = new Set(future.flatMap((candidate) => marginSignatures(candidate)));
+    const repeatedFurniture = /* @__PURE__ */ new Set([...seenFurniture, ...futureFurniture]);
+    for (const block of page.structured.blocks) {
+      if (isRepeatedFurniture(block, page, repeatedFurniture)) {
+        stats.suppressedFurniture += 1;
+        continue;
+      }
+      if (block.type === "table") {
+        const rows = tableToRows(block.table);
+        if (activeTable && tablesContinue(activeTable.table, block.table, page.width)) {
+          const continuationRows = sameRow(activeTable.header, rows[0]) ? rows.slice(1) : rows;
+          for (const row of continuationRows) await write(tableRow(row, false));
+          activeTable.table = block.table;
+          stats.mergedTables += 1;
+          continue;
+        }
+        await closeTable();
+        const header = tableHeader(rows);
+        await write("<table>");
+        for (const [index, row] of rows.entries())
+          await write(tableRow(row, Boolean(header && index === 0)));
+        activeTable = { table: block.table, header };
+        continue;
+      }
+      await closeTable();
+      if (block.type === "heading") {
+        await closeSections(block.level);
+        await write(
+          `<section data-level="${block.level}"><h${block.level}>${escapeHtml(block.text)}</h${block.level}>`
+        );
+        sectionLevels.push(block.level);
+        continue;
+      }
+      await write(semanticBlockHtml(block));
+    }
+    for (const signature of marginSignatures(page)) seenFurniture.add(signature);
+  };
+  for await (const page of pages) {
+    const structured = structurePage(page);
+    buffer.push({ width: page.width, height: page.height, structured });
+    stats.pagesProcessed += 1;
+    stats.peakBufferedPages = Math.max(stats.peakBufferedPages, buffer.length);
+    stats.peakBufferedLines = Math.max(
+      stats.peakBufferedLines,
+      buffer.reduce((total, item) => total + item.structured.lines.length, 0)
+    );
+    if (buffer.length >= lookaheadPages) {
+      const ready = buffer.shift();
+      if (ready) await emitPage(ready, buffer);
+    }
+  }
+  while (buffer.length > 0) {
+    const ready = buffer.shift();
+    if (ready) await emitPage(ready, buffer);
+  }
+  await closeTable();
+  await closeSections();
+  await write("</article>");
+  return stats;
+}
+function marginSignatures(page) {
+  return page.structured.blocks.flatMap(
+    (block) => blockLines(block).filter((line) => isMarginLine(line.bounds.y, line.bounds.height, page.height)).map((line) => furnitureSignature(line.text))
+  );
+}
+function isRepeatedFurniture(block, page, futureFurniture) {
+  const lines = blockLines(block);
+  return lines.length > 0 && lines.every(
+    (line) => isMarginLine(line.bounds.y, line.bounds.height, page.height) && futureFurniture.has(furnitureSignature(line.text))
+  );
+}
+function blockLines(block) {
+  return block.type === "list" ? block.items.flatMap((item) => item.lines) : block.lines;
+}
+function isMarginLine(y, height, pageHeight) {
+  return y + height <= pageHeight * 0.12 || y >= pageHeight * 0.88;
+}
+function furnitureSignature(value) {
+  return value.toLocaleLowerCase("en").replace(/\d+/g, "#").replace(/\s+/g, " ").trim();
+}
+function tablesContinue(previous, next, pageWidth) {
+  if (previous.columns.length !== next.columns.length || previous.columns.length < 2) return false;
+  return previous.columns.every(
+    (column, index) => Math.abs(column - (next.columns[index] ?? Number.POSITIVE_INFINITY)) / pageWidth <= 0.04
+  );
+}
+function tableHeader(rows) {
+  const first = rows[0];
+  if (!first) return void 0;
+  return first.some((value) => /^(?:item|description|feature|qty|unit|amount|total)$/i.test(value)) ? first : void 0;
+}
+function sameRow(left, right) {
+  return Boolean(
+    left && right && left.length === right.length && left.every((value, index) => value === right[index])
+  );
+}
+function tableRow(row, header) {
+  const cell = header ? "th" : "td";
+  return `<tr>${row.map((value) => `<${cell}>${escapeHtml(value)}</${cell}>`).join("")}</tr>`;
+}
+function semanticBlockHtml(block) {
+  if (block.type === "heading")
+    return `<h${block.level}>${escapeHtml(block.text)}</h${block.level}>`;
+  if (block.type === "paragraph") return `<p>${escapeHtml(block.text)}</p>`;
+  if (block.type === "definitionList") {
+    return `<dl>${block.entries.map(
+      (entry) => `<div><dt>${escapeHtml(entry.term)}</dt><dd>${escapeHtml(entry.description)}</dd></div>`
+    ).join("")}</dl>`;
+  }
+  if (block.type === "cardList") {
+    return `<div class="pdf-semantic-cards">${block.items.map(
+      (item) => `<article><h3>${escapeHtml(item.title)}</h3>${item.details.map((detail) => `<p>${escapeHtml(detail)}</p>`).join("")}</article>`
+    ).join("")}</div>`;
+  }
+  if (block.type === "sectionGroup") {
+    return `<div class="pdf-semantic-sections">${block.items.map(
+      (item) => `<section><h3>${escapeHtml(item.label)}</h3>${item.content.map((content) => `<p>${escapeHtml(content)}</p>`).join("")}</section>`
+    ).join("")}</div>`;
+  }
+  const tag = block.ordered ? "ol" : "ul";
+  return `<${tag}>${block.items.map((item) => `<li>${escapeHtml(item.text)}</li>`).join("")}</${tag}>`;
+}
+function escapeHtml(value) {
+  return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+
+// src/index.ts
 var styles = `.pdf-document{margin:0 auto}.pdf-page{box-sizing:border-box;margin:1rem auto;background:#fff;color:#000}.pdf-page--visual,.pdf-page--positioned{position:relative;overflow:hidden}.pdf-page-content{position:absolute;transform-origin:0 0}.pdf-span{position:absolute;white-space:pre;transform-origin:left bottom;unicode-bidi:isolate}.pdf-span[data-direction=ttb]{writing-mode:vertical-rl}.pdf-page--semantic,.pdf-page--flow{max-width:60rem;padding:1rem}.pdf-page--semantic p,.pdf-page--flow p{white-space:pre-wrap;unicode-bidi:plaintext}.pdf-page table{border-collapse:collapse}.pdf-page td{padding:.15rem .4rem;vertical-align:top}`;
 async function writeHtmlDocument(pages, write, options = {}) {
   const includeDocument = options.includeDocument ?? true;
@@ -9,15 +167,28 @@ async function writeHtmlDocument(pages, write, options = {}) {
       ` lang="${escapeAttribute(options.language ?? "en")}"><head><meta charset="utf-8">`
     );
     await write('<meta name="viewport" content="width=device-width,initial-scale=1">');
-    await write(`<title>${escapeHtml(options.title ?? "PDF document")}</title>`);
+    await write(`<title>${escapeHtml2(options.title ?? "PDF document")}</title>`);
     if (options.includeStyles ?? true) await write(`<style>${styles}</style>`);
     await write("</head><body>");
   }
   await write('<main class="pdf-document">');
-  for await (const page of pages) await writePage(page, write, options);
+  if (resolveProfile(options) === "semantic") {
+    const lookahead = semanticLookahead(options.semanticLookaheadPages);
+    const stats = await writeSemanticDocument(pages, write, lookahead);
+    options.onSemanticStats?.(stats);
+  } else {
+    for await (const page of pages) await writePage(page, write, options);
+  }
   await write("</main>");
   if (includeDocument) await write("</body></html>");
 }
+function semanticLookahead(value) {
+  const lookahead = value ?? 4;
+  if (!Number.isSafeInteger(lookahead) || lookahead < 1 || lookahead > 16) {
+    throw new RangeError("semanticLookaheadPages must be an integer between 1 and 16");
+  }
+  return lookahead;
+}
 async function writePage(page, write, options = {}) {
   if (resolveProfile(options) === "semantic") await writeFlowPage(page, write);
   else await writePositionedPage(page, write, options);
@@ -57,8 +228,12 @@ async function writePositionedPage(page, write, options) {
   await write(
     `<svg class="pdf-visual-text" xmlns="http://www.w3.org/2000/svg" width="${number(page.width)}pt" height="${number(page.height)}pt" viewBox="0 0 ${number(page.width)} ${number(page.height)}">`
   );
+  const clipDefinitions = imageClipDefinitions(page.images ?? [], page.number, page.height) + pathClipDefinitions(page.paths ?? [], page.number);
+  if (clipDefinitions) await write(`<defs>${clipDefinitions}</defs>`);
   if (reflectedOverlay) {
-    for (const image of page.images ?? []) await write(visualImage(image, page.height));
+    for (const [index, image] of (page.images ?? []).entries()) {
+      await write(visualImage(image, page.height, page.number, index));
+    }
   }
   for (const fill of page.fills ?? []) {
     const points = fill.points.map(([x, y]) => `${number(x)},${number(page.height - y)}`).join(" ");
@@ -69,7 +244,7 @@ async function writePositionedPage(page, write, options) {
   }
   if (page.paths?.length) {
     await write(`<g transform="translate(0 ${number(page.height)}) scale(1 -1)">`);
-    for (const path of page.paths) {
+    for (const [pathIndex, path] of page.paths.entries()) {
       if (!isSvgPath(path.d)) continue;
       const fill = isCssHexColor(path.fill) ? path.fill : "none";
       const stroke = isCssHexColor(path.stroke) ? path.stroke : "none";
@@ -81,14 +256,18 @@ async function writePositionedPage(page, write, options) {
       const dashoffset = Number.isFinite(path.strokeDashoffset) ? ` stroke-dashoffset="${number(path.strokeDashoffset ?? 0)}"` : "";
       const linecap = path.strokeLinecap ? ` stroke-linecap="${path.strokeLinecap}"` : "";
       const linejoin = path.strokeLinejoin ? ` stroke-linejoin="${path.strokeLinejoin}"` : "";
-      await write(
-        `<path d="${path.d}" fill="${fill}" stroke="${stroke}"${strokeWidth}${fillOpacity}${strokeOpacity}${dasharray}${dashoffset}${linecap}${linejoin}${fillRule}/>`
-      );
+      let output = `<path d="${path.d}" fill="${fill}" stroke="${stroke}"${strokeWidth}${fillOpacity}${strokeOpacity}${dasharray}${dashoffset}${linecap}${linejoin}${fillRule}/>`;
+      for (let index = (path.clips?.length ?? 0) - 1; index >= 0; index -= 1) {
+        output = `<g clip-path="url(#${pathClipId(page.number, pathIndex, index)})">${output}</g>`;
+      }
+      await write(output);
     }
     await write("</g>");
   }
   if (!reflectedOverlay) {
-    for (const image of page.images ?? []) await write(visualImage(image, page.height));
+    for (const [index, image] of (page.images ?? []).entries()) {
+      await write(visualImage(image, page.height, page.number, index));
+    }
   }
   for (const span of visualSpans) {
     if (!usesPositionedSpan(span)) {
@@ -109,13 +288,41 @@ function usesReflectedVisualOverlay(page, spans) {
     (span) => span.transform !== void 0 && Math.abs(span.transform[0] + 1) < 1e-6 && Math.abs(span.transform[1]) < 1e-6 && Math.abs(span.transform[2]) < 1e-6 && Math.abs(span.transform[3] - 1) < 1e-6
   );
 }
-function visualImage(image, pageHeight) {
+function visualImage(image, pageHeight, pageNumber, imageIndex) {
   const [a, b, c, d, e, f] = image.transform;
   const transform = [a, -b, -c, d, c + e, pageHeight - d - f].map(number).join(" ");
   const opacity = isUnitInterval(image.opacity) ? ` opacity="${number(image.opacity)}"` : "";
   const mime = image.format === "jpeg" ? "image/jpeg" : "image/bmp";
   const data = image.format === "jpeg" ? image.data : rgbBmp(image);
-  return `<image width="1" height="1" preserveAspectRatio="none" transform="matrix(${transform})" href="data:${mime};base64,${base64(data)}"${opacity}/>`;
+  let output = `<image width="1" height="1" preserveAspectRatio="none" transform="matrix(${transform})" href="data:${mime};base64,${base64(data)}"${opacity}/>`;
+  for (let index = (image.clips?.length ?? 0) - 1; index >= 0; index -= 1) {
+    output = `<g clip-path="url(#${imageClipId(pageNumber, imageIndex, index)})">${output}</g>`;
+  }
+  return output;
+}
+function imageClipDefinitions(images, pageNumber, pageHeight) {
+  return images.flatMap(
+    (image, imageIndex) => (image.clips ?? []).map((clip, clipIndex) => {
+      if (!isSvgPath(clip.d)) return "";
+      const fillRule = clip.fillRule ? ` clip-rule="${clip.fillRule}"` : "";
+      return `<clipPath id="${imageClipId(pageNumber, imageIndex, clipIndex)}" clipPathUnits="userSpaceOnUse"><path d="${clip.d}" transform="translate(0 ${number(pageHeight)}) scale(1 -1)"${fillRule}/></clipPath>`;
+    })
+  ).join("");
+}
+function imageClipId(pageNumber, imageIndex, clipIndex) {
+  return `boxpdf-clip-${pageNumber}-${imageIndex}-${clipIndex}`;
+}
+function pathClipDefinitions(paths, pageNumber) {
+  return paths.flatMap(
+    (path, pathIndex) => (path.clips ?? []).map((clip, clipIndex) => {
+      if (!isSvgPath(clip.d)) return "";
+      const fillRule = clip.fillRule ? ` clip-rule="${clip.fillRule}"` : "";
+      return `<clipPath id="${pathClipId(pageNumber, pathIndex, clipIndex)}" clipPathUnits="userSpaceOnUse"><path d="${clip.d}"${fillRule}/></clipPath>`;
+    })
+  ).join("");
+}
+function pathClipId(pageNumber, pathIndex, clipIndex) {
+  return `boxpdf-path-clip-${pageNumber}-${pathIndex}-${clipIndex}`;
 }
 function rgbBmp(image) {
   const stride = Math.ceil(image.width * 3 / 4) * 4;
@@ -169,28 +376,51 @@ function positionedSpan(span, fontAliases) {
       span.fontAssetId ? fontAliases.get(span.fontAssetId) : void 0
     )
   ].join(";");
-  return `<span class="pdf-span"${direction} style="${style}">${escapeHtml(span.text)}</span>`;
+  return `<span class="pdf-span"${direction} style="${style}">${escapeHtml2(span.text)}</span>`;
 }
 async function writeFlowPage(page, write) {
-  const structured = structurePage(page);
-  const tables = [...structured.tables].sort((left, right) => right.bounds.y - left.bounds.y);
-  const emittedTables = /* @__PURE__ */ new Set();
+  const structured = structurePage2(page);
   await write(
     `<section class="pdf-page pdf-page--semantic pdf-page--flow" data-page="${page.number}">`
   );
-  for (const line of structured.lines) {
-    const table = tables.find((candidate) => containsY(candidate, line.bounds.y));
-    if (table) {
-      if (!emittedTables.has(table)) {
-        await write(tableToHtml(table));
-        emittedTables.add(table);
+  for (const block of structured.blocks) {
+    if (block.type === "table") await write(tableToHtml(block.table));
+    else if (block.type === "heading") {
+      await write(`<h${block.level}>${escapeHtml2(block.text)}</h${block.level}>`);
+    } else if (block.type === "paragraph") {
+      await write(
+        `<p${directionAttribute(block.lines.flatMap((line) => line.spans))}>${escapeHtml2(block.text)}</p>`
+      );
+    } else if (block.type === "definitionList") {
+      await write("<dl>");
+      for (const entry of block.entries) {
+        await write(
+          `<div><dt>${escapeHtml2(entry.term)}</dt><dd>${escapeHtml2(entry.description)}</dd></div>`
+        );
       }
-      continue;
+      await write("</dl>");
+    } else if (block.type === "cardList") {
+      await write('<div class="pdf-semantic-cards">');
+      for (const item of block.items) {
+        await write(`<article><h3>${escapeHtml2(item.title)}</h3>`);
+        for (const detail of item.details) await write(`<p>${escapeHtml2(detail)}</p>`);
+        await write("</article>");
+      }
+      await write("</div>");
+    } else if (block.type === "sectionGroup") {
+      await write('<div class="pdf-semantic-sections">');
+      for (const item of block.items) {
+        await write(`<section><h3>${escapeHtml2(item.label)}</h3>`);
+        for (const content of item.content) await write(`<p>${escapeHtml2(content)}</p>`);
+        await write("</section>");
+      }
+      await write("</div>");
+    } else {
+      const tag = block.ordered ? "ol" : "ul";
+      await write(`<${tag}>`);
+      for (const item of block.items) await write(`<li>${escapeHtml2(item.text)}</li>`);
+      await write(`</${tag}>`);
     }
-    await write(`<p${directionAttribute(line.spans)}>${escapeHtml(line.text)}</p>`);
-  }
-  for (const table of tables) {
-    if (!emittedTables.has(table)) await write(tableToHtml(table));
   }
   await write("</section>");
 }
@@ -232,7 +462,7 @@ function visualText(span, pageHeight, fontAliases, counterRotateReflectedText =
   const anchorX = span.bounds.x + basisX * rtlOffset;
   const anchorY = pageHeight - span.bounds.y + basisY * rtlOffset;
   const position = transformed ? ` x="0" y="0" transform="matrix(${transform?.map(number).join(" ")} ${number(anchorX)} ${number(anchorY)})"` : ` x="${number(anchorX)}" y="${number(anchorY)}"`;
-  return `<text${direction}${position} font-size="${number(span.fontSize)}"${textLength}${style ? ` style="${style}"` : ""}>${escapeHtml(span.text)}</text>`;
+  return `<text${direction}${position} font-size="${number(span.fontSize)}"${textLength}${style ? ` style="${style}"` : ""}>${escapeHtml2(span.text)}</text>`;
 }
 function isAdobeCjkFont(fontFamily) {
   return /^Adobe(?:Heiti|Song|Kaiti|Ming|Gothic|Mincho)Std-/i.test(fontFamily ?? "");
@@ -345,16 +575,13 @@ function directionAttribute(spans) {
   if (vertical > rtl && vertical * 2 >= spans.length) return ' data-direction="ttb"';
   return rtl * 2 >= spans.length && spans.length > 0 ? ' dir="rtl"' : "";
 }
-function containsY(table, y) {
-  return y >= table.bounds.y && y <= table.bounds.y + table.bounds.height;
-}
 function number(value) {
   return Number.isFinite(value) ? String(Math.round(value * 1e3) / 1e3) : "0";
 }
 function escapeAttribute(value) {
-  return escapeHtml(value).replaceAll("`", "&#96;");
+  return escapeHtml2(value).replaceAll("`", "&#96;");
 }
-function escapeHtml(value) {
+function escapeHtml2(value) {
   return [...value].map((character) => {
     const codePoint = character.codePointAt(0) ?? 0;
     if (codePoint === 13) return "\n";