extract-pdf 0.1.0 → 0.1.2

package/src/models/text-item.ts

@@ -0,0 +1,50 @@
+/**
+ * @description A single raw text span extracted from a PDF page, carrying spatial
+ * coordinates (x, y, width, height), the text string, the font name, and optional
+ * inline format markers (lineFormat, unopenedFormat, unclosedFormat) used by the
+ * line-compaction pipeline to track bold/italic spans that cross item boundaries.
+ */
+import PageItem, { BlockTypeEntry } from "./page-item";
+import Annotation from "./annotation";
+import ParsedElements from "./parsed-elements";
+import { WordFormatEntry } from "./word";
+
+export default class TextItem extends PageItem {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  text: string;
+  font: string;
+  lineFormat: WordFormatEntry | null;
+  unopenedFormat: WordFormatEntry | null;
+  unclosedFormat: WordFormatEntry | null;
+
+  constructor(options: {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    text: string;
+    font: string;
+    lineFormat?: WordFormatEntry | null;
+    unopenedFormat?: WordFormatEntry | null;
+    unclosedFormat?: WordFormatEntry | null;
+    type?: BlockTypeEntry | null;
+    annotation?: Annotation | null;
+    parsedElements?: ParsedElements | null;
+  }) {
+    super(options);
+    this.x = options.x;
+    this.y = options.y;
+    this.width = options.width;
+    this.height = options.height;
+    this.text = options.text;
+    this.font = options.font;
+
+    this.lineFormat = options.lineFormat ?? null;
+    this.unopenedFormat = options.unopenedFormat ?? null;
+    this.unclosedFormat = options.unclosedFormat ?? null;
+  }
+}
+

package/src/models/word.ts

@@ -0,0 +1,31 @@
+/**
+ * @description Minimal token model for a single word within a `LineItem`.
+ * Carries the text `string`, an optional `type` (`WordType`: LINK,
+ * FOOTNOTE_LINK, FOOTNOTE) for semantic rendering, and an optional `format`
+ * (`WordFormat`: BOLD, OBLIQUE, BOLD_OBLIQUE) for inline HTML styling.
+ */
+
+export interface WordFormatEntry {
+  name: string;
+  startSymbol: string;
+  endSymbol: string;
+}
+
+export interface WordTypeEntry {
+  name: string;
+  attachWithoutWhitespace?: boolean;
+  plainTextFormat?: boolean;
+  toText(string: string): string;
+}
+
+export default class Word {
+  string: string;
+  type: WordTypeEntry | null;
+  format: WordFormatEntry | null;
+
+  constructor(options: { string: string; type?: WordTypeEntry | null; format?: WordFormatEntry | null }) {
+    this.string = options.string;
+    this.type = options.type ?? null;
+    this.format = options.format ?? null;
+  }
+}

package/src/pdf-to-html.ts

@@ -0,0 +1,225 @@
+/**
+ * @fileoverview High-fidelity PDF-to-HTML conversion pipeline.
+ * Extracts structural elements (headers, lists, code blocks) and handles page-level metadata.
+ */
+import {
+  findPageNumbers,
+  findFirstPage,
+  removePageNumber,
+} from "./utils/page-number-functions";
+import TextItem from "./models/text-item";
+import Page from "./models/page";
+
+/**
+ * Fetch wrapper for grabbing binary content
+ */
+async function grab(url: string, options: { responseType?: string; timeout?: number } = {}) {
+  const timeout = options.timeout ? options.timeout * 1000 : 10000;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+    });
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+
+    if (options.responseType === "arraybuffer") {
+      return await response.arrayBuffer();
+    }
+    return await response.text();
+  } catch (error) {
+    clearTimeout(timeoutId);
+    throw error;
+  }
+}
+
+import CalculateGlobalStats from "./transforms/calculate-global-stats";
+import CompactLines from "./transforms/line-item/compact-lines";
+import RemoveRepetitiveElements from "./transforms/line-item/remove-repetitive-elements";
+import VerticalToHorizontal from "./transforms/line-item/vertical-to-horizontal";
+import DetectTOC from "./transforms/line-item/detect-toc";
+import DetectListItems from "./transforms/line-item/detect-list-items";
+import DetectHeaders from "./transforms/line-item/detect-headers";
+
+import GatherBlocks from "./transforms/block/gather-blocks";
+import DetectCodeQuoteBlocks from "./transforms/block/detect-code-quote-blocks";
+import DetectListLevels from "./transforms/block/detect-list-levels";
+import ToTextBlocks from "./transforms/to-text-blocks";
+import ToHTML from "./transforms/to-html";
+import ParseResult from "./models/parse-result";
+
+/**
+ * Extracts formatted text from PDF with parsing of linebreaks ,
+ * page headers, footnotes, and section headings. Supports fonts, links, bold,
+ * italics, lists, headings, headers, footnotes, and Table of Contents,
+ * Quotes, and Code Blocks, . Removes repeated headers, links footnote anchors to the footnote,
+ *  and preserves number of the PDF page with invisible I element.
+ *
+ * This function uses [pdfjs-serverless](https://github.com/johannschopplich/pdfjs-serverless)
+ * to work in more environments than PDF.js-based tools:
+ * Cloudflare workers, serverless, node.js, and front-end only.
+ * @param {string} pdfURLOrBuffer - URL to a PDF file or buffer from fs.readFile
+ * @param {Object} [options]
+ * @param {boolean} options.addPageNumbers default=false - Adds  #  to end of each page
+ * @param {boolean} options.removePageHeaders default=true - Removes repeated headers found on each page
+ * @returns {string|Object} HTML formatted text
+ * @category Extract
+ * @author [vtempest (2025)](https://github.com/vtempest),
+ * [pdf-to-markdown (2017)](https://github.com/jzillmann/pdf-to-markdown/tree/master),
+ * [pdf.js (2012-)](https://github.com/mozilla/pdf.js/releases),
+ */
+export async function convertPDFToHTML(
+  pdfURLOrBuffer: any,
+  options: { addPageNumbers?: boolean; addCitation?: boolean } = {},
+) {
+  // try {
+  var { addPageNumbers = false, addCitation = true } = options;
+
+  // pass in databuffer or download all pdf data
+  // and convert to array buffer
+  var buffer =
+    typeof pdfURLOrBuffer === "string"
+      ? await grab(pdfURLOrBuffer, {
+          responseType: "arraybuffer",
+          timeout: 10,
+        })
+      : pdfURLOrBuffer;
+
+  let pdfDocument;
+  try {
+    let { resolvePDFJS } = await import("https://cdn.jsdelivr.net/npm/pdfjs-serverless@1.1.0/+esm" as any);
+    
+    const { getDocument } = await resolvePDFJS();
+    pdfDocument = await getDocument({
+      data: new Uint8Array(buffer),
+      useSystemFonts: true,
+      verbosity: 0,
+    }).promise;
+  } catch (e: any) {
+    return { error: e.message };
+  }
+
+  const pages = [...Array(pdfDocument.numPages).keys()].map(
+    (index) => new Page({ index }),
+  );
+
+  let pageIndexNumMap = {};
+  let firstPage;
+  for (let j = 1; j <= pdfDocument.numPages; j++) {
+    const page = await pdfDocument.getPage(j);
+    const textContent = await page.getTextContent();
+
+    if (Object.keys(pageIndexNumMap).length < 10) {
+      pageIndexNumMap = findPageNumbers(
+        pageIndexNumMap,
+        page.pageNumber - 1,
+        textContent.items,
+      );
+    } else {
+      firstPage = findFirstPage(pageIndexNumMap);
+      break;
+    }
+  }
+
+  let pageNum = firstPage ? firstPage.pageNum : 0;
+  for (let j = 1; j <= pdfDocument.numPages; j++) {
+    const page = await pdfDocument.getPage(j);
+
+    // Trigger the font retrieval for the page
+    await page.getOperatorList();
+
+    const scale = 1.0;
+    const viewport = page.getViewport({ scale });
+    let textContent = await page.getTextContent();
+    if (firstPage && (page as any).pageIndex >= firstPage.pageIndex) {
+      textContent = removePageNumber(textContent as any, pageNum) as any;
+      pageNum++;
+    }
+    const textItems = (textContent.items as any[]).map((item: any) => {
+      const tx = [1, 0, 0, 1, 0, 0];
+      for (let i = 0; i < 6; i++) {
+        tx[i] += item.transform[i] * viewport.transform[i % 2 ? 3 : 0];
+        if (i % 2) {
+          tx[i + 1] += item.transform[i] * viewport.transform[1];
+        }
+      }
+
+      const fontHeight = Math.sqrt(tx[2] * tx[2] + tx[3] * tx[3]);
+      const dividedHeight = item.height / fontHeight;
+      return new TextItem({
+        x: Math.round(item.transform[4]),
+        y: Math.round(item.transform[5]),
+        width: Math.round(item.width),
+        height: Math.round(dividedHeight <= 1 ? item.height : dividedHeight),
+        text: item.str,
+        font: item.fontName,
+      });
+    });
+    pages[page.pageNumber - 1].items = textItems;
+  }
+
+  var parseResult = new ParseResult({ pages });
+
+  let lastTransformation: (typeof transformations)[number] | undefined,
+    transformations = [
+      new CalculateGlobalStats(),
+      new CompactLines(),
+      new RemoveRepetitiveElements(),
+      new VerticalToHorizontal(),
+      new DetectTOC(),
+      new DetectHeaders(),
+      new DetectListItems(),
+
+      new GatherBlocks(),
+      new DetectCodeQuoteBlocks(),
+      new DetectListLevels(),
+
+      new ToTextBlocks(),
+      new ToHTML(),
+    ];
+
+  transformations?.forEach((transformation) => {
+    if (lastTransformation) {
+      parseResult = lastTransformation.completeTransform(parseResult);
+    }
+    parseResult = transformation.transform(parseResult);
+    lastTransformation = transformation;
+  });
+
+  var html = parseResult.pages.reduce((acc, page, pageNumber) => {
+    return (
+      acc +
+      `<p id="page-${pageNumber + 1}">${
+        addPageNumbers ? ` [${pageNumber + 1}] ` : ""
+      }${page.items.join('</p><p id="page-' + pageNumber + '">')}</p>`
+    );
+  }, "");
+
+  if (addCitation) {
+    // Get metadata
+    // avoid using date as it is unreliable sand generally file mod date
+    var metadata = await pdfDocument.getMetadata();
+    var { Author: author, Title: title } = metadata.info as any;
+    // date =
+    //   date.slice(2, 6) + "-" + date.slice(6, 8) + "-" + date.slice(8, 10);
+    // date = date ? new Date(date)?.toISOString().split("T")[0] : null;
+
+    //look for date in first page
+    // date = chrono
+    //   .parseDate(content.slice(0, 400))
+    //   ?.toISOString()
+    //   .split("T")[0];
+    // //  || date;
+
+    title = html.slice(0, 400).match(/<h[0-9]>(.*?)<\/h[0-9]>/)?.[1] || title;
+  }
+
+  return { author, title, html, format: "pdf" };
+}
+
+

package/src/transforms/base/to-line-item-block-transform.ts

@@ -0,0 +1,29 @@
+/**
+ * @description Abstract base for pipeline stages that produce `LineItemBlock`-typed
+ * page items (GatherBlocks, DetectCodeQuoteBlocks, DetectListLevels). Provides
+ * `completeTransform` which strips `REMOVED_ANNOTATION` items and clears all
+ * remaining annotations before passing the result to the next stage.
+ */
+import Transformation from './transformation'
+import LineItemBlock from '../../models/line-item-block'
+import ParseResult from '../../models/parse-result'
+import { REMOVED_ANNOTATION } from '../../models/annotation'
+
+// Abstract class for transformations producing LineItemBlock(s) to be shown in the LineItemBlockPageView
+export default class ToLineItemBlockTransformation extends Transformation {
+  constructor(name: string) {
+    super(name, LineItemBlock.name)
+    if (this.constructor === ToLineItemBlockTransformation) {
+      throw new TypeError('Can not construct abstract class.')
+    }
+  }
+
+  completeTransform(parseResult: ParseResult): ParseResult {
+    parseResult.messages = []
+    parseResult.pages.forEach(page => {
+      page.items = page.items.filter(item => !item.annotation || item.annotation !== REMOVED_ANNOTATION)
+      page.items.forEach(item => (item.annotation = null))
+    })
+    return parseResult
+  }
+}

package/src/transforms/base/to-line-item-transform.ts

@@ -0,0 +1,29 @@
+/**
+ * @description Abstract base for pipeline stages that produce `LineItem`-typed
+ * page items (CompactLines, RemoveRepetitiveElements, VerticalToHorizontal,
+ * DetectTOC, DetectHeaders, DetectListItems). Provides `completeTransform` which
+ * strips `REMOVED_ANNOTATION` items and clears annotations before the next stage.
+ */
+import Transformation from './transformation'
+import LineItem from '../../models/line-item'
+import ParseResult from '../../models/parse-result'
+import { REMOVED_ANNOTATION } from '../../models/annotation'
+
+// Abstract class for transformations producing LineItem(s) to be shown in the LineItemPageView
+export default class ToLineItemTransformation extends Transformation {
+  constructor(name: string) {
+    super(name, LineItem.name)
+    if (this.constructor === ToLineItemTransformation) {
+      throw new TypeError('Can not construct abstract class.')
+    }
+  }
+
+  completeTransform(parseResult: ParseResult): ParseResult {
+    parseResult.messages = []
+    parseResult.pages.forEach(page => {
+      page.items = page.items.filter(item => !item.annotation || item.annotation !== REMOVED_ANNOTATION)
+      page.items.forEach(item => (item.annotation = null))
+    })
+    return parseResult
+  }
+}

package/src/transforms/base/to-text-item-transform.ts

@@ -0,0 +1,28 @@
+/**
+ * @description Abstract base for pipeline stages that operate on raw `TextItem`
+ * page items (the earliest stages, currently only `CalculateGlobalStats`). Provides
+ * the standard `completeTransform` cleanup that removes `REMOVED_ANNOTATION` items
+ * and clears annotations before passing control to the next transformation.
+ */
+import Transformation from './transformation'
+import TextItem from '../../models/text-item'
+import ParseResult from '../../models/parse-result'
+import { REMOVED_ANNOTATION } from '../../models/annotation'
+
+export default class ToTextItemTransformation extends Transformation {
+  constructor(name: string) {
+    super(name, TextItem.name)
+    if (this.constructor === ToTextItemTransformation) {
+      throw new TypeError('Can not construct abstract class.')
+    }
+  }
+
+  completeTransform(parseResult: ParseResult): ParseResult {
+    parseResult.messages = []
+    parseResult.pages.forEach(page => {
+      page.items = page.items.filter(item => !item.annotation || item.annotation !== REMOVED_ANNOTATION)
+      page.items.forEach(item => (item.annotation = null))
+    })
+    return parseResult
+  }
+}

package/src/transforms/base/transformation.ts

@@ -0,0 +1,35 @@
+/**
+ * @description Abstract root class for all PDF pipeline stages. Enforces that
+ * subclasses implement `transform(parseResult)` and stores `name` (for debug
+ * display) and `itemType` (the class name of the items each stage produces,
+ * used for view routing). `completeTransform` is a no-op by default; subclasses
+ * override it to flush deferred mutations after the stage's debug view is rendered.
+ */
+import ParseResult from "../../models/parse-result";
+
+// A transformation from an PdfPage to an PdfPage
+export default class Transformation {
+  name: string;
+  itemType: string;
+
+  constructor(name: string, itemType: string) {
+    if (this.constructor === Transformation) {
+      throw new TypeError("Can not construct abstract class.");
+    }
+    if (this.transform === Transformation.prototype.transform) {
+      throw new TypeError("Please implement abstract method 'transform()'.");
+    }
+    this.name = name;
+    this.itemType = itemType;
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    throw new TypeError("Do not call abstract method foo from child.");
+  }
+
+  completeTransform(parseResult: ParseResult): ParseResult {
+    parseResult.messages = [];
+    return parseResult;
+  }
+}
+

package/src/transforms/block/detect-code-quote-blocks.ts

@@ -0,0 +1,57 @@
+/**
+ * @description Block-level transformation that marks blocks as `BlockType.CODE`
+ * when all constituent lines are indented beyond the page's leftmost x position
+ * and the text height matches body-text height. Heuristically distinguishes
+ * indented code/blockquote sections from regular paragraphs without relying on
+ * font metadata.
+ */
+
+import ToLineItemBlockTransformation from '../base/to-line-item-block-transform'
+import ParseResult from '../../models/parse-result'
+import { DETECTED_ANNOTATION } from '../../models/annotation'
+import BlockType from '../../models/block-type'
+import { minXFromBlocks } from '../../utils/page-item-functions'
+
+// Detect items which are code/quote blocks
+export default class DetectCodeQuoteBlocks extends ToLineItemBlockTransformation {
+  constructor () {
+    super('$1')
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const mostUsedHeight = parseResult.globals.mostUsedHeight ?? 0
+    var foundCodeItems = 0
+    parseResult.pages.forEach(page => {
+      var minX = minXFromBlocks(page.items)
+      page.items.forEach(block => {
+        if (!block.type && looksLikeCodeBlock(minX, block.items, mostUsedHeight)) {
+          block.annotation = DETECTED_ANNOTATION
+          block.type = BlockType.CODE
+          foundCodeItems++
+        }
+      })
+    })
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        'Detected ' + foundCodeItems + ' code/quote items.',
+      ],
+    })
+  }
+}
+
+function looksLikeCodeBlock(minX: number, items: any[], mostUsedHeight: number): boolean {
+  if (items.length === 0) {
+    return false
+  }
+  if (items.length === 1) {
+    return items[0].x > minX && items[0].height <= mostUsedHeight + 1
+  }
+  for (var item of items) {
+    if (item.x === minX) {
+      return false
+    }
+  }
+  return true
+}

package/src/transforms/block/detect-list-levels.ts

@@ -0,0 +1,64 @@
+/**
+ * @description Block-level transformation that adds hierarchical indentation to
+ * nested list items within `BlockType.LIST` blocks. Tracks nesting level by
+ * comparing each item's x position to the previous item's — deeper x means a
+ * new sub-level — and prepends spaces proportional to the level so the HTML
+ * output reflects the original visual hierarchy.
+ */
+import ToLineItemBlockTransformation from '../base/to-line-item-block-transform'
+import ParseResult from '../../models/parse-result'
+import Word from '../../models/word'
+import { MODIFIED_ANNOTATION, UNCHANGED_ANNOTATION } from '../../models/annotation'
+import BlockType from '../../models/block-type'
+
+// Cares for proper sub-item spacing/leveling
+export default class DetectListLevels extends ToLineItemBlockTransformation {
+  constructor () {
+    super('Level Lists')
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    var listBlocks = 0
+    var modifiedBlocks = 0
+    parseResult.pages.forEach(page => {
+      page.items.filter(block => block.type === BlockType.LIST).forEach(listBlock => {
+        var lastItemX: number | undefined
+        var currentLevel = 0
+        const xByLevel: Record<number, number> = {}
+        var modifiedBlock = false
+        listBlock.items.forEach(item => {
+          const isListItem = true
+          if (lastItemX && isListItem) {
+            if (item.x > lastItemX) {
+              currentLevel++
+              xByLevel[item.x] = currentLevel
+            } else if (item.x < lastItemX) {
+              currentLevel = xByLevel[item.x]
+            }
+          } else {
+            xByLevel[item.x] = 0
+          }
+          if (currentLevel > 0) {
+            item.words = [
+              new Word({ string: ' '.repeat(currentLevel * 3) }),
+            ].concat(item.words)
+            modifiedBlock = true
+          }
+          lastItemX = item.x
+        })
+        listBlocks++
+        if (modifiedBlock) {
+          modifiedBlocks++
+          listBlock.annotation = MODIFIED_ANNOTATION
+        } else {
+          listBlock.annotation = UNCHANGED_ANNOTATION
+        }
+      })
+    })
+
+    return new ParseResult({
+      ...parseResult,
+      messages: ['Modified ' + modifiedBlocks + ' / ' + listBlocks + ' list blocks.'],
+    })
+  }
+}

package/src/transforms/block/gather-blocks.ts

@@ -0,0 +1,113 @@
+/**
+ * @description Block-level transformation that merges consecutive `LineItem`s into
+ * `LineItemBlock`s. Flushes the current block when the item type changes or the
+ * vertical gap exceeds `mostUsedDistance`, while respecting per-type merge flags
+ * (`mergeToBlock`, `mergeFollowingNonTypedItems`,
+ * `mergeFollowingNonTypedItemsWithSmallDistance`) to handle lists and footnotes correctly.
+ */
+
+import ToLineItemBlockTransformation from "../base/to-line-item-block-transform";
+import ParseResult from "../../models/parse-result";
+import LineItemBlock from "../../models/line-item-block";
+import { DETECTED_ANNOTATION } from "../../models/annotation";
+import { minXFromPageItems } from "../../utils/page-item-functions";
+
+// Gathers lines to blocks
+export default class GatherBlocks extends ToLineItemBlockTransformation {
+  constructor() {
+    super("Gather Blocks");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const { mostUsedDistance } = parseResult.globals;
+    var createdBlocks = 0;
+    var lineItemCount = 0;
+    parseResult.pages.map((page) => {
+      lineItemCount += page.items.length;
+      const blocks = [];
+      var stashedBlock = new LineItemBlock({});
+      const flushStashedItems = () => {
+        if (stashedBlock.items.length > 1) {
+          stashedBlock.annotation = DETECTED_ANNOTATION;
+        }
+
+        blocks.push(stashedBlock);
+        stashedBlock = new LineItemBlock({});
+        createdBlocks++;
+      };
+
+      var minX = minXFromPageItems(page.items);
+      page.items.forEach((item) => {
+        if (
+          stashedBlock.items.length > 0 &&
+          shouldFlushBlock(stashedBlock, item, minX, mostUsedDistance)
+        ) {
+          flushStashedItems();
+        }
+        stashedBlock.addItem(item);
+      });
+      if (stashedBlock.items.length > 0) {
+        flushStashedItems();
+      }
+      page.items = blocks;
+    });
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        "Gathered " +
+          createdBlocks +
+          " blocks out of " +
+          lineItemCount +
+          " line items",
+      ],
+    });
+  }
+}
+
+function shouldFlushBlock(stashedBlock: any, item: any, minX: number, mostUsedDistance: number): boolean {
+  if (
+    stashedBlock.type &&
+    stashedBlock.type.mergeFollowingNonTypedItems &&
+    !item.type
+  ) {
+    return false;
+  }
+  const lastItem = stashedBlock.items[stashedBlock.items.length - 1];
+  const hasBigDistance = bigDistance(lastItem, item, minX, mostUsedDistance);
+  if (
+    stashedBlock.type &&
+    stashedBlock.type.mergeFollowingNonTypedItemsWithSmallDistance &&
+    !item.type &&
+    !hasBigDistance
+  ) {
+    return false;
+  }
+  if (item.type !== stashedBlock.type) {
+    return true;
+  }
+  if (item.type) {
+    return !item.type.mergeToBlock;
+  } else {
+    return hasBigDistance;
+  }
+}
+
+function bigDistance(lastItem: any, item: any, minX: number, mostUsedDistance: number): boolean {
+  const distance = lastItem.y - item.y;
+  if (distance < 0 - mostUsedDistance / 2) {
+    // distance is negative - and not only a bit
+    return true;
+  }
+  var allowedDisctance = mostUsedDistance + 1;
+  if (lastItem.x > minX && item.x > minX) {
+    // intended elements like lists often have greater spacing
+    allowedDisctance = mostUsedDistance + mostUsedDistance / 2;
+  }
+  if (distance > allowedDisctance) {
+    return true;
+  }
+  return false;
+}
+
+