extract-pdf 0.1.20 → 0.1.22

package/README.md

@@ -1,74 +1,74 @@
-# extract-pdf
-
-Converts a PDF (URL or `ArrayBuffer`) into clean HTML with structural tagging — headings, lists, footnotes, code blocks, bold/italic, and Table of Contents entries. Works in Node.js, Cloudflare Workers, and browser environments via [pdfjs-serverless](https://github.com/johannschopplich/pdfjs-serverless).
-
-## Install 
-
-```sh
-bun add extract-pdf
-```
-
-## Usage
-
-```ts
-import { convertPDFToHTML } from "extract-pdf";
-
-const { html, title, author } = await convertPDFToHTML(
-  "https://example.com/paper.pdf",
-);
-// or pass an ArrayBuffer from fs.readFile / fetch
-const { html } = await convertPDFToHTML(buffer, { addPageNumbers: true });
-```
-
-### Options
-
-| Option           | Default | Description                                                                                |
-| ---------------- | ------- | ------------------------------------------------------------------------------------------ |
-| `addPageNumbers` | `false` | Inserts `[n]` markers at each page boundary                                                |
-| `addCitation`    | `true`  | Reads PDF metadata and first-page heading to populate `title`/`author` in the return value |
-
-### Return value
-
-```ts
-{ html: string, title?: string, author?: string, format: "pdf" }
-```
-
-## Pipeline
-
-The conversion runs a sequential chain of transformations on a `ParseResult` (pages → items):
-
-```
-Raw pdfjs text spans
-  → CalculateGlobalStats   — font heights, distances, format map
-  → CompactLines           — merge spans on the same y-line into LineItems
-  → RemoveRepetitiveElements — strip recurring page headers/footers
-  → VerticalToHorizontal   — rotate vertical character runs
-  → DetectTOC              — identify Table of Contents pages, link headings
-  → DetectHeaders          — classify items as H1–H6 by font height
-  → DetectListItems        — detect bullet/numbered list items
-  → GatherBlocks           — group adjacent same-type lines into blocks
-  → DetectCodeQuoteBlocks  — mark indented blocks as CODE
-  → DetectListLevels       — add indentation for nested list levels
-  → ToTextBlocks           — flatten blocks to { category, text } pairs
-  → ToHTML                 — render pairs as <p>, <h1>–<h6>, <ul>, <code>
-```
-
-## Folder structure
-
-```
-src/
-  pdf-to-html.ts          — main entry point (convertPDFToHTML)
-  models/                 — data classes: Page, ParseResult, TextItem,
-  │                         LineItem, LineItemBlock, Word, BlockType, …
-  transforms/
-  │  base/                — abstract Transformation, ToLineItem*, ToLineItemBlock*
-  │  line-item/           — per-line-item transformations
-  │  block/               — per-block transformations
-  │  calculate-global-stats.ts
-  │  to-text-blocks.ts
-  │  to-html.ts
-  utils/
-     string-functions.ts
-     page-item-functions.ts
-     page-number-functions.ts
-```
+# extract-pdf
+
+Converts a PDF (URL or `ArrayBuffer`) into clean HTML with structural tagging — headings, lists, footnotes, code blocks, bold/italic, and Table of Contents entries. Works in Node.js, Cloudflare Workers, and browser environments via [pdfjs-serverless](https://github.com/johannschopplich/pdfjs-serverless).
+
+## Install 
+
+```sh
+bun add extract-pdf
+```
+
+## Usage
+
+```ts
+import { convertPDFToHTML } from "extract-pdf";
+
+const { html, title, author } = await convertPDFToHTML(
+  "https://example.com/paper.pdf",
+);
+// or pass an ArrayBuffer from fs.readFile / fetch
+const { html } = await convertPDFToHTML(buffer, { addPageNumbers: true });
+```
+
+### Options
+
+| Option           | Default | Description                                                                                |
+| ---------------- | ------- | ------------------------------------------------------------------------------------------ |
+| `addPageNumbers` | `false` | Inserts `[n]` markers at each page boundary                                                |
+| `addCitation`    | `true`  | Reads PDF metadata and first-page heading to populate `title`/`author` in the return value |
+
+### Return value
+
+```ts
+{ html: string, title?: string, author?: string, format: "pdf" }
+```
+
+## Pipeline
+
+The conversion runs a sequential chain of transformations on a `ParseResult` (pages → items):
+
+```
+Raw pdfjs text spans
+  → CalculateGlobalStats   — font heights, distances, format map
+  → CompactLines           — merge spans on the same y-line into LineItems
+  → RemoveRepetitiveElements — strip recurring page headers/footers
+  → VerticalToHorizontal   — rotate vertical character runs
+  → DetectTOC              — identify Table of Contents pages, link headings
+  → DetectHeaders          — classify items as H1–H6 by font height
+  → DetectListItems        — detect bullet/numbered list items
+  → GatherBlocks           — group adjacent same-type lines into blocks
+  → DetectCodeQuoteBlocks  — mark indented blocks as CODE
+  → DetectListLevels       — add indentation for nested list levels
+  → ToTextBlocks           — flatten blocks to { category, text } pairs
+  → ToHTML                 — render pairs as <p>, <h1>–<h6>, <ul>, <code>
+```
+
+## Folder structure
+
+```
+src/
+  pdf-to-html.ts          — main entry point (convertPDFToHTML)
+  models/                 — data classes: Page, ParseResult, TextItem,
+  │                         LineItem, LineItemBlock, Word, BlockType, …
+  transforms/
+  │  base/                — abstract Transformation, ToLineItem*, ToLineItemBlock*
+  │  line-item/           — per-line-item transformations
+  │  block/               — per-block transformations
+  │  calculate-global-stats.ts
+  │  to-text-blocks.ts
+  │  to-html.ts
+  utils/
+     string-functions.ts
+     page-item-functions.ts
+     page-number-functions.ts
+```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-pdf",
-  "version": "0.1.20",
+  "version": "0.1.22",
   "description": "Convert a PDF (URL or ArrayBuffer) into clean HTML with structural tagging — headings, lists, footnotes, code blocks, bold/italic. Works in Node.js, Cloudflare Workers, and browser environments.",
   "author": "vtempest <grokthiscontact@gmail.com>",
   "license": "rights.institute/PROSPER",

package/src/models/annotation.ts

@@ -1,41 +1,41 @@
-/**
- * @description Diff-annotation markers used to tag document items during pipeline
- * transformations. Each `Annotation` carries a human-readable `category` and a
- * display `color`. Pre-built singletons (`ADDED_ANNOTATION`, `REMOVED_ANNOTATION`,
- * `UNCHANGED_ANNOTATION`, `DETECTED_ANNOTATION`, `MODIFIED_ANNOTATION`) are
- * attached to page items so debug views can highlight what each stage changed.
- */
-export default class Annotation {
-  category: string;
-  color: string;
-
-  constructor(options: { category: string; color: string }) {
-    this.category = options.category;
-    this.color = options.color;
-  }
-}
-
-export const ADDED_ANNOTATION = new Annotation({
-  category: "Added",
-  color: "green",
-});
-
-export const REMOVED_ANNOTATION = new Annotation({
-  category: "Removed",
-  color: "red",
-});
-
-export const UNCHANGED_ANNOTATION = new Annotation({
-  category: "Unchanged",
-  color: "brown",
-});
-
-export const DETECTED_ANNOTATION = new Annotation({
-  category: "Detected",
-  color: "green",
-});
-
-export const MODIFIED_ANNOTATION = new Annotation({
-  category: "Modified",
-  color: "green",
-});
+/**
+ * @description Diff-annotation markers used to tag document items during pipeline
+ * transformations. Each `Annotation` carries a human-readable `category` and a
+ * display `color`. Pre-built singletons (`ADDED_ANNOTATION`, `REMOVED_ANNOTATION`,
+ * `UNCHANGED_ANNOTATION`, `DETECTED_ANNOTATION`, `MODIFIED_ANNOTATION`) are
+ * attached to page items so debug views can highlight what each stage changed.
+ */
+export default class Annotation {
+  category: string;
+  color: string;
+
+  constructor(options: { category: string; color: string }) {
+    this.category = options.category;
+    this.color = options.color;
+  }
+}
+
+export const ADDED_ANNOTATION = new Annotation({
+  category: "Added",
+  color: "green",
+});
+
+export const REMOVED_ANNOTATION = new Annotation({
+  category: "Removed",
+  color: "red",
+});
+
+export const UNCHANGED_ANNOTATION = new Annotation({
+  category: "Unchanged",
+  color: "brown",
+});
+
+export const DETECTED_ANNOTATION = new Annotation({
+  category: "Detected",
+  color: "green",
+});
+
+export const MODIFIED_ANNOTATION = new Annotation({
+  category: "Modified",
+  color: "green",
+});

package/src/models/block-type.ts

@@ -1,203 +1,203 @@
-/**
- * @description Enum-like registry of semantic document block types (H1–H6, TOC,
- * FOOTNOTES, CODE, LIST, PARAGRAPH). Each entry carries a `toText()` serializer
- * that converts a `LineItemBlock` to HTML. Helper methods `isHeadline`,
- * `blockToText`, and `headlineByLevel` are mixed in on the registry object.
- * The inner `linesToText` function handles inline bold/italic/link/footnote
- * formatting by tracking open `WordFormat` spans across words and lines.
- */
-
-const BlockTypes: Record<string, any> = {
-  H1: {
-    name: "H1",
-    headline: true,
-    headlineLevel: 1,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h1>" + linesToText(block.items, true) + "</h1>";
-    },
-  },
-  H2: {
-    name: "H2",
-    headline: true,
-    headlineLevel: 2,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h2>" + linesToText(block.items, true) + "</h2>";
-    },
-  },
-  H3: {
-    name: "H3",
-    headline: true,
-    headlineLevel: 3,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h3>" + linesToText(block.items, true) + "</h3>";
-    },
-  },
-  H4: {
-    name: "H4",
-    headline: true,
-    headlineLevel: 4,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h4>" + linesToText(block.items, true) + "</h4>";
-    },
-  },
-  H5: {
-    name: "H5",
-    headline: true,
-    headlineLevel: 5,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h5>" + linesToText(block.items, true) + "</h5>";
-    },
-  },
-  H6: {
-    name: "H6",
-    headline: true,
-    headlineLevel: 6,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<h6>" + linesToText(block.items, true) + "</h6>";
-    },
-  },
-  TOC: {
-    name: "TOC",
-    mergeToBlock: true,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return linesToText(block.items, true);
-    },
-  },
-  FOOTNOTES: {
-    name: "FOOTNOTES",
-    mergeToBlock: true,
-    mergeFollowingNonTypedItems: true,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<p>" + linesToText(block.items, false) + "</p>";
-    },
-  },
-  CODE: {
-    name: "CODE",
-    mergeToBlock: true,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<code>" + linesToText(block.items, true) + "</code>";
-    },
-  },
-  LIST: {
-    name: "LIST",
-    mergeToBlock: false,
-    mergeFollowingNonTypedItemsWithSmallDistance: true,
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<ul>\n" + linesToText(block.items, false) + "</ul>";
-    },
-  },
-  PARAGRAPH: {
-    name: "PARAGRAPH",
-    toText(block /*: LineItemBlock */) /*: string */ {
-      return "<p>" + linesToText(block.items, false) + "</p>";
-    },
-  },
-};
-
-function firstFormat(lineItem: { words: Array<{ format: any }> }): any {
-  if (lineItem.words.length === 0) {
-    return null;
-  }
-  return lineItem.words[0].format;
-}
-
-function isPunctationCharacter(string: string): boolean {
-  if (string.length !== 1) {
-    return false;
-  }
-  return string[0] === "." || string[0] === "!" || string[0] === "?";
-}
-
-function linesToText(lineItems: any[], disableInlineFormats: boolean): string {
-  var text = "";
-  var openFormat;
-
-  const closeFormat = () => {
-    text += openFormat.endSymbol;
-    openFormat = null;
-  };
-
-  lineItems.forEach((line, lineIndex) => {
-    if (!line) return;
-
-    line.words.forEach((word, i) => {
-      const wordType = word.type;
-      const wordFormat = word.format;
-      if (openFormat && (!wordFormat || wordFormat !== openFormat)) {
-        closeFormat();
-      }
-
-      if (
-        i > 0 &&
-        !(wordType && wordType.attachWithoutWhitespace) &&
-        !isPunctationCharacter(word.string)
-      ) {
-        text += " ";
-      }
-
-      if (wordFormat && !openFormat && !disableInlineFormats) {
-        openFormat = wordFormat;
-        text += openFormat.startSymbol;
-      }
-
-      if (wordType && (!disableInlineFormats || wordType.plainTextFormat)) {
-        text += wordType.toText(word.string);
-      } else {
-        text += word.string;
-      }
-    });
-    if (
-      openFormat &&
-      (lineIndex === lineItems.length - 1 ||
-        firstFormat(lineItems[lineIndex + 1]) !== openFormat)
-    ) {
-      closeFormat();
-    }
-    text += "\n";
-  });
-  return text;
-}
-
-// Make the object immutable
-// Object.freeze(BlockTypes);
-// Object.values(BlockTypes).forEach(value => Object.freeze(value));
-
-BlockTypes.enumValueOf = function enumValueOf(name: string) {
-  return BlockTypes[name];
-};
-
-BlockTypes.isHeadline = function isHeadline(
-  type /*: typeof BlockTypes[keyof typeof BlockTypes] */,
-) /*: boolean */ {
-  return type && type.name.length === 2 && type.name[0] === "H";
-};
-
-BlockTypes.blockToText = function blockToText(
-  block /*: LineItemBlock */,
-) /*: string */ {
-  if (!block.type) {
-    return linesToText(block.items, false);
-  }
-  return block.type.toText(block);
-};
-
-BlockTypes.headlineByLevel = function headlineByLevel(level) {
-  if (level === 1) return BlockTypes.H1;
-  if (level === 2) return BlockTypes.H2;
-  if (level === 3) return BlockTypes.H3;
-  if (level === 4) return BlockTypes.H4;
-  if (level === 5) return BlockTypes.H5;
-
-  // if level is >= 6, just use BlockType H6
-  if (level > 6) {
-    // eslint-disable-next-line no-console
-    console.warn(
-      "Unsupported headline level: " +
-        level +
-        " (supported are 1-6), defaulting to level 6",
-    );
-  }
-  return BlockTypes.H6;
-};
-
-export default BlockTypes;
+/**
+ * @description Enum-like registry of semantic document block types (H1–H6, TOC,
+ * FOOTNOTES, CODE, LIST, PARAGRAPH). Each entry carries a `toText()` serializer
+ * that converts a `LineItemBlock` to HTML. Helper methods `isHeadline`,
+ * `blockToText`, and `headlineByLevel` are mixed in on the registry object.
+ * The inner `linesToText` function handles inline bold/italic/link/footnote
+ * formatting by tracking open `WordFormat` spans across words and lines.
+ */
+
+const BlockTypes: Record<string, any> = {
+  H1: {
+    name: "H1",
+    headline: true,
+    headlineLevel: 1,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h1>" + linesToText(block.items, true) + "</h1>";
+    },
+  },
+  H2: {
+    name: "H2",
+    headline: true,
+    headlineLevel: 2,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h2>" + linesToText(block.items, true) + "</h2>";
+    },
+  },
+  H3: {
+    name: "H3",
+    headline: true,
+    headlineLevel: 3,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h3>" + linesToText(block.items, true) + "</h3>";
+    },
+  },
+  H4: {
+    name: "H4",
+    headline: true,
+    headlineLevel: 4,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h4>" + linesToText(block.items, true) + "</h4>";
+    },
+  },
+  H5: {
+    name: "H5",
+    headline: true,
+    headlineLevel: 5,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h5>" + linesToText(block.items, true) + "</h5>";
+    },
+  },
+  H6: {
+    name: "H6",
+    headline: true,
+    headlineLevel: 6,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<h6>" + linesToText(block.items, true) + "</h6>";
+    },
+  },
+  TOC: {
+    name: "TOC",
+    mergeToBlock: true,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return linesToText(block.items, true);
+    },
+  },
+  FOOTNOTES: {
+    name: "FOOTNOTES",
+    mergeToBlock: true,
+    mergeFollowingNonTypedItems: true,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<p>" + linesToText(block.items, false) + "</p>";
+    },
+  },
+  CODE: {
+    name: "CODE",
+    mergeToBlock: true,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<code>" + linesToText(block.items, true) + "</code>";
+    },
+  },
+  LIST: {
+    name: "LIST",
+    mergeToBlock: false,
+    mergeFollowingNonTypedItemsWithSmallDistance: true,
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<ul>\n" + linesToText(block.items, false) + "</ul>";
+    },
+  },
+  PARAGRAPH: {
+    name: "PARAGRAPH",
+    toText(block /*: LineItemBlock */) /*: string */ {
+      return "<p>" + linesToText(block.items, false) + "</p>";
+    },
+  },
+};
+
+function firstFormat(lineItem: { words: Array<{ format: any }> }): any {
+  if (lineItem.words.length === 0) {
+    return null;
+  }
+  return lineItem.words[0].format;
+}
+
+function isPunctationCharacter(string: string): boolean {
+  if (string.length !== 1) {
+    return false;
+  }
+  return string[0] === "." || string[0] === "!" || string[0] === "?";
+}
+
+function linesToText(lineItems: any[], disableInlineFormats: boolean): string {
+  var text = "";
+  var openFormat;
+
+  const closeFormat = () => {
+    text += openFormat.endSymbol;
+    openFormat = null;
+  };
+
+  lineItems.forEach((line, lineIndex) => {
+    if (!line) return;
+
+    line.words.forEach((word, i) => {
+      const wordType = word.type;
+      const wordFormat = word.format;
+      if (openFormat && (!wordFormat || wordFormat !== openFormat)) {
+        closeFormat();
+      }
+
+      if (
+        i > 0 &&
+        !(wordType && wordType.attachWithoutWhitespace) &&
+        !isPunctationCharacter(word.string)
+      ) {
+        text += " ";
+      }
+
+      if (wordFormat && !openFormat && !disableInlineFormats) {
+        openFormat = wordFormat;
+        text += openFormat.startSymbol;
+      }
+
+      if (wordType && (!disableInlineFormats || wordType.plainTextFormat)) {
+        text += wordType.toText(word.string);
+      } else {
+        text += word.string;
+      }
+    });
+    if (
+      openFormat &&
+      (lineIndex === lineItems.length - 1 ||
+        firstFormat(lineItems[lineIndex + 1]) !== openFormat)
+    ) {
+      closeFormat();
+    }
+    text += "\n";
+  });
+  return text;
+}
+
+// Make the object immutable
+// Object.freeze(BlockTypes);
+// Object.values(BlockTypes).forEach(value => Object.freeze(value));
+
+BlockTypes.enumValueOf = function enumValueOf(name: string) {
+  return BlockTypes[name];
+};
+
+BlockTypes.isHeadline = function isHeadline(
+  type /*: typeof BlockTypes[keyof typeof BlockTypes] */,
+) /*: boolean */ {
+  return type && type.name.length === 2 && type.name[0] === "H";
+};
+
+BlockTypes.blockToText = function blockToText(
+  block /*: LineItemBlock */,
+) /*: string */ {
+  if (!block.type) {
+    return linesToText(block.items, false);
+  }
+  return block.type.toText(block);
+};
+
+BlockTypes.headlineByLevel = function headlineByLevel(level) {
+  if (level === 1) return BlockTypes.H1;
+  if (level === 2) return BlockTypes.H2;
+  if (level === 3) return BlockTypes.H3;
+  if (level === 4) return BlockTypes.H4;
+  if (level === 5) return BlockTypes.H5;
+
+  // if level is >= 6, just use BlockType H6
+  if (level > 6) {
+    // eslint-disable-next-line no-console
+    console.warn(
+      "Unsupported headline level: " +
+        level +
+        " (supported are 1-6), defaulting to level 6",
+    );
+  }
+  return BlockTypes.H6;
+};
+
+export default BlockTypes;