extract-pdf 0.1.0 → 0.1.2

package/src/models/headline-finder.ts

@@ -0,0 +1,53 @@
+/**
+ * @description Incrementally matches a target headline string across consecutive
+ * `LineItem`s. Characters are normalised (uppercase, whitespace/dot stripped)
+ * before comparison so multi-line or split titles are found correctly. Returns
+ * the array of matching items once the full headline has been consumed, or `null`
+ * if the sequence breaks. Used by `DetectTOC` to locate heading text on content pages.
+ */
+import LineItem from "./line-item";
+import { normalizedCharCodeArray } from "../utils/string-functions";
+
+export default class HeadlineFinder {
+  headlineCharCodes: number[];
+  stackedLineItems: LineItem[];
+  stackedChars: number;
+
+  constructor(options: { headline: string }) {
+    this.headlineCharCodes = normalizedCharCodeArray(options.headline);
+    this.stackedLineItems = [];
+    this.stackedChars = 0;
+  }
+
+  consume(lineItem: LineItem): LineItem[] | null {
+    const normalizedCharCodes = normalizedCharCodeArray(lineItem.text());
+    const matchAll = this.matchAll(normalizedCharCodes);
+    if (matchAll) {
+      this.stackedLineItems.push(lineItem);
+      this.stackedChars += normalizedCharCodes.length;
+      if (this.stackedChars === this.headlineCharCodes.length) {
+        return this.stackedLineItems;
+      }
+    } else {
+      if (this.stackedChars > 0) {
+        this.stackedChars = 0;
+        this.stackedLineItems = [];
+        this.consume(lineItem);
+      }
+    }
+    return null;
+  }
+
+  matchAll(normalizedCharCodes: number[]): boolean {
+    for (var i = 0; i < normalizedCharCodes.length; i++) {
+      const headlineChar = this.headlineCharCodes[this.stackedChars + i];
+      const textItemChar = normalizedCharCodes[i];
+      if (textItemChar !== headlineChar) {
+        return false;
+      }
+    }
+    return true;
+  }
+}
+
+

package/src/models/line-converter.ts

@@ -0,0 +1,224 @@
+/**
+ * @description Converts an array of spatially-grouped TextItems (one line) into a single
+ * LineItem by detecting inline formatting (bold, italic), footnote superscripts, footnote
+ * anchors, and hyperlinks. WordFormat carries HTML open/close symbols; WordType carries
+ * rendering helpers for links and footnotes. The inner WordDetectionStream extends
+ * StashingStream to buffer consecutive same-format items before flushing them as Word nodes.
+ */
+import TextItem from "./text-item";
+import Word, { WordFormatEntry, WordTypeEntry } from "./word";
+import LineItem from "./line-item";
+import StashingStream from "./stashing-stream";
+import ParsedElements from "./parsed-elements";
+import { isNumber, isListItemCharacter } from "../utils/string-functions";
+import { sortByX } from "../utils/page-item-functions";
+
+export const WordFormat: Record<string, WordFormatEntry> = {
+  BOLD: {
+    name: "BOLD",
+    startSymbol: "<strong>",
+    endSymbol: "</strong>",
+  },
+
+  OBLIQUE: {
+    name: "OBLIQUE",
+    startSymbol: "<em>",
+    endSymbol: "</em>",
+  },
+
+  BOLD_OBLIQUE: {
+    name: "BOLD_OBLIQUE",
+    startSymbol: "<strong><em>",
+    endSymbol: "</em></strong>",
+  },
+};
+
+export const WordType: Record<string, WordTypeEntry> = {
+  LINK: {
+    name: "LINK",
+    toText(string: string) {
+      return `<a href="${string}">${string}</a>`;
+    },
+  },
+
+  FOOTNOTE_LINK: {
+    name: "FOOTNOTE_LINK",
+    attachWithoutWhitespace: true,
+    plainTextFormat: true,
+    toText(string: string) {
+      return `<sup><a href="#${string}">${string}</a></sup>`;
+    },
+  },
+
+  FOOTNOTE: {
+    name: "FOOTNOTE",
+    toText(string: string) {
+      return `<p id="${string}">^${string}</p>`;
+    },
+  },
+};
+
+export default class LineConverter {
+  fontToFormats: Map<string, string>;
+
+  constructor(fontToFormats: Map<string, string>) {
+    this.fontToFormats = fontToFormats;
+  }
+
+  compact(textItems: TextItem[]): LineItem {
+    sortByX(textItems);
+
+    const wordStream = new WordDetectionStream(this.fontToFormats);
+    wordStream.consumeAll(textItems.map((item) => new TextItem({ ...item })));
+    const words = wordStream.complete();
+
+    var maxHeight = 0;
+    var widthSum = 0;
+    textItems.forEach((item) => {
+      maxHeight = Math.max(maxHeight, item.height);
+      widthSum += item.width;
+    });
+    return new LineItem({
+      x: textItems[0].x,
+      y: textItems[0].y,
+      height: maxHeight,
+      width: widthSum,
+      words: words,
+      parsedElements: new ParsedElements({
+        footnoteLinks: wordStream.footnoteLinks,
+        footnotes: wordStream.footnotes,
+        containLinks: wordStream.containLinks,
+        formattedWords: wordStream.formattedWords,
+      }),
+    });
+  }
+}
+
+class WordDetectionStream extends StashingStream {
+  fontToFormats: Map<string, string>;
+  footnoteLinks: number[];
+  footnotes: string[];
+  formattedWords: number;
+  containLinks: boolean;
+  stashedNumber: boolean;
+  firstY?: number;
+  currentItem: TextItem | null;
+
+  constructor(fontToFormats: Map<string, string>) {
+    super();
+    this.fontToFormats = fontToFormats;
+    this.footnoteLinks = [];
+    this.footnotes = [];
+    this.formattedWords = 0;
+    this.containLinks = false;
+    this.stashedNumber = false;
+    this.currentItem = null;
+  }
+
+  shouldStash(item: any): boolean {
+    if (!this.firstY) {
+      this.firstY = item.y;
+    }
+    this.currentItem = item;
+    return true;
+  }
+
+  onPushOnStash(item: any): void {
+    this.stashedNumber = isNumber(item.text.trim());
+  }
+
+  doMatchesStash(lastItem: any, item: any): boolean {
+    const lastItemFormat = this.fontToFormats.get(lastItem.font);
+    const itemFormat = this.fontToFormats.get(item.font);
+    if (lastItemFormat !== itemFormat) {
+      return false;
+    }
+    const itemIsANumber = isNumber(item.text.trim());
+    return this.stashedNumber === itemIsANumber;
+  }
+
+  doFlushStash(stash: any[], results: any[]): void {
+    if (this.stashedNumber) {
+      const joinedNumber = stash
+        .map((item: any) => item.text)
+        .join("")
+        .trim();
+      if (stash[0].y > this.firstY) {
+        results.push(
+          new Word({
+            string: `${joinedNumber}`,
+            type: WordType.FOOTNOTE_LINK,
+          }),
+        );
+        this.footnoteLinks.push(parseInt(joinedNumber));
+      } else if (this.currentItem && this.currentItem.y < stash[0].y) {
+        results.push(
+          new Word({
+            string: `${joinedNumber}`,
+            type: WordType.FOOTNOTE,
+          }),
+        );
+        this.footnotes.push(joinedNumber);
+      } else {
+        this.copyStashItemsAsText(stash, results);
+      }
+    } else {
+      this.copyStashItemsAsText(stash, results);
+    }
+  }
+
+  copyStashItemsAsText(stash: any[], results: any[]): void {
+    const format = this.fontToFormats.get(stash[0].font);
+    results.push(...this.itemsToWords(stash, format));
+  }
+
+  itemsToWords(items: TextItem[], formatName: string | null): Word[] {
+    const combinedText = combineText(items);
+    const words = combinedText.split(" ");
+    const format = formatName ? WordFormat[formatName] : null;
+    return words
+      .filter((w) => w.trim().length > 0)
+      .map((word) => {
+        var type: WordTypeEntry | null = null;
+        if (word.startsWith("http:")) {
+          this.containLinks = true;
+          type = WordType.LINK;
+        } else if (word.startsWith("www.")) {
+          this.containLinks = true;
+          word = `http://${word}`;
+          type = WordType.LINK;
+        }
+
+        if (format) {
+          this.formattedWords++;
+        }
+        return new Word({ string: word, type, format });
+      });
+  }
+}
+
+function combineText(textItems: TextItem[]): string {
+  var text = "";
+  var lastItem: TextItem | null = null;
+  textItems.forEach((textItem) => {
+    var textToAdd = textItem.text;
+    if (!text.endsWith(" ") && !textToAdd.startsWith(" ")) {
+      if (lastItem) {
+        const xDistance = textItem.x - lastItem.x - lastItem.width;
+        if (xDistance > 5) {
+          text += " ";
+        }
+      } else {
+        if (isListItemCharacter(textItem.text)) {
+          textToAdd += " ";
+        }
+      }
+    }
+    text += textToAdd;
+    lastItem = textItem;
+  });
+  return text;
+}
+
+
+

package/src/models/line-item-block.ts

@@ -0,0 +1,51 @@
+/**
+ * @description A semantically coherent group of `LineItem`s that share the same
+ * block type (paragraph, list, heading, etc.). Extends `PageItem`. `addItem`
+ * enforces type consistency across all lines, merges `ParsedElements` metadata
+ * from each line into the block, and strips the type tag from individual lines
+ * so only the block-level type is authoritative.
+ */
+import PageItem, { BlockTypeEntry } from "./page-item";
+import Annotation from "./annotation";
+import ParsedElements from "./parsed-elements";
+import LineItem from "./line-item";
+
+// A block of LineItem[] within a Page
+export default class LineItemBlock extends PageItem {
+  items: LineItem[];
+
+  constructor(options: {
+    items?: LineItem[];
+    type?: BlockTypeEntry | null;
+    annotation?: Annotation | null;
+    parsedElements?: ParsedElements | null;
+  }) {
+    super(options);
+    this.items = [];
+    if (options.items) {
+      options.items.forEach((item) => this.addItem(item));
+    }
+  }
+
+  addItem(item: LineItem): void {
+    if (this.type && item.type && this.type !== item.type) {
+      throw new Error(
+        `Adding item of type ${item.type.name} to block of type ${this.type.name}`,
+      );
+    }
+    if (!this.type) {
+      this.type = item.type;
+    }
+    if (item.parsedElements) {
+      if (this.parsedElements) {
+        this.parsedElements.add(item.parsedElements);
+      } else {
+        this.parsedElements = item.parsedElements;
+      }
+    }
+    const copiedItem = new LineItem({ ...item });
+    copiedItem.type = null;
+    this.items.push(copiedItem);
+  }
+}
+

package/src/models/line-item.ts

@@ -0,0 +1,59 @@
+/**
+ * @description A single typographic line on a PDF page. Extends `PageItem` with
+ * spatial coordinates (x, y, width, height) and a `words` array of `Word` tokens.
+ * Can be constructed from a pre-built `words` array or from a raw `text` string,
+ * which is split on spaces and wrapped into `Word` instances automatically.
+ */
+import PageItem, { BlockTypeEntry } from "./page-item";
+import Annotation from "./annotation";
+import ParsedElements from "./parsed-elements";
+import Word from "./word";
+
+// A line within a page
+export default class LineItem extends PageItem {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  words: Word[];
+
+  constructor(options: {
+    x?: number;
+    y?: number;
+    width?: number;
+    height?: number;
+    words?: Word[];
+    text?: string;
+    type?: BlockTypeEntry | null;
+    annotation?: Annotation | null;
+    parsedElements?: ParsedElements | null;
+    font?: string;
+  }) {
+    super(options);
+    this.x = options.x ?? 0;
+    this.y = options.y ?? 0;
+    this.width = options.width ?? 0;
+    this.height = options.height ?? 0;
+    this.words = options.words || [];
+    if (options.text && !options.words) {
+      this.words = options.text
+        .split(" ")
+        .filter((string) => string.trim().length > 0)
+        .map(
+          (wordAsString) =>
+            new Word({
+              string: wordAsString,
+            }),
+        );
+    }
+  }
+
+  text(): string {
+    return this.wordStrings().join(" ");
+  }
+
+  wordStrings(): string[] {
+    return this.words.map((word) => word.string);
+  }
+}
+

package/src/models/metadata.ts

@@ -0,0 +1,29 @@
+/**
+ * @description Normalised PDF document metadata (title, author, creator, producer).
+ * Handles both the modern XMP `metadata` API and the legacy `info` dictionary
+ * returned by pdfjs, so callers always receive a consistent shape regardless of
+ * which metadata format the PDF uses.
+ */
+// Metadata of the PDF document
+export default class Metadata {
+  title?: string;
+  creator?: string;
+  producer?: string;
+  author?: string;
+
+  constructor(originalMetadata: {
+    metadata?: { get(key: string): string | undefined };
+    info?: { Title?: string; Author?: string; Creator?: string; Producer?: string };
+  }) {
+    if (originalMetadata.metadata) {
+      this.title = originalMetadata.metadata.get("dc:title");
+      this.creator = originalMetadata.metadata.get("xap:creatortool");
+      this.producer = originalMetadata.metadata.get("pdf:producer");
+    } else {
+      this.title = originalMetadata.info?.Title;
+      this.author = originalMetadata.info?.Author;
+      this.creator = originalMetadata.info?.Creator;
+      this.producer = originalMetadata.info?.Producer;
+    }
+  }
+}

package/src/models/page-item.ts

@@ -0,0 +1,36 @@
+/**
+ * @description Abstract base class for all page-level document nodes (`TextItem`,
+ * `LineItem`, `LineItemBlock`). Carries the shared fields `type` (semantic block
+ * category from `BlockType`), `annotation` (diff marker for debug rendering), and
+ * `parsedElements` (inline footnote/link metadata). Throws `TypeError` if
+ * instantiated directly.
+ */
+import Annotation from "./annotation";
+import ParsedElements from "./parsed-elements";
+
+export interface BlockTypeEntry {
+  name: string;
+  headline?: boolean;
+  headlineLevel?: number;
+  mergeToBlock?: boolean;
+  mergeFollowingNonTypedItems?: boolean;
+  mergeFollowingNonTypedItemsWithSmallDistance?: boolean;
+  toText?(block: any): string;
+}
+
+// A abstract PageItem class, can be TextItem, LineItem or LineItemBlock
+export default class PageItem {
+  type: BlockTypeEntry | null;
+  annotation: Annotation | null;
+  parsedElements: ParsedElements | null;
+
+  constructor(options: { type?: BlockTypeEntry | null; annotation?: Annotation | null; parsedElements?: ParsedElements | null }) {
+    if (this.constructor === PageItem) {
+      throw new TypeError("Can not construct abstract class.");
+    }
+    this.type = options.type ?? null;
+    this.annotation = options.annotation ?? null;
+    this.parsedElements = options.parsedElements ?? null;
+  }
+}
+

package/src/models/page.ts

@@ -0,0 +1,16 @@
+/**
+ * @description Simple container for a single PDF page. Holds a 0-based `index`
+ * and an `items` array whose element type evolves through the transformation
+ * pipeline: raw `TextItem[]` → compacted `LineItem[]` → grouped `LineItemBlock[]`
+ * → final serialized text objects.
+ */
+// A page which holds PageItems displayable via PdfPageView
+export default class Page {
+  index: number;
+  items: any[];
+
+  constructor(options: { index: number; items?: any[] }) {
+    this.index = options.index;
+    this.items = options.items || [];
+  }
+}

package/src/models/parse-result.ts

@@ -0,0 +1,32 @@
+/**
+ * @description Immutable value object passed between every pipeline stage.
+ * `pages` holds the document's page array (items mutate in type each stage),
+ * `globals` holds cross-page statistics computed by `CalculateGlobalStats`
+ * (most-used height, font, line distance, font-format map), and `messages`
+ * carries diagnostic strings emitted by each transformation for debugging.
+ */
+import Page from "./page";
+
+export interface ParseGlobals {
+  mostUsedHeight?: number;
+  mostUsedFont?: string;
+  mostUsedDistance?: number;
+  maxHeight?: number;
+  maxHeightFont?: string;
+  fontToFormats?: Map<string, string>;
+  tocPages?: number[];
+  headlineTypeToHeightRange?: Record<string, { min: number; max: number }>;
+}
+
+// The result of a PDF parse respectively a Transformation
+export default class ParseResult {
+  pages: Page[];
+  globals: ParseGlobals;
+  messages: string[];
+
+  constructor(options: { pages?: Page[]; globals?: ParseGlobals; messages?: string[] }) {
+    this.pages = options.pages || [];
+    this.globals = options.globals || {};
+    this.messages = options.messages || [];
+  }
+}

package/src/models/parsed-elements.ts

@@ -0,0 +1,29 @@
+/**
+ * @description Aggregated inline-element metadata for a `LineItem` or
+ * `LineItemBlock`. Tracks detected footnote link numbers, footnote definitions,
+ * whether any hyperlinks are present, and a count of bold/italic formatted words.
+ * The `add()` method merges a child item's `ParsedElements` into the parent
+ * block's running totals as lines are gathered into blocks.
+ */
+export default class ParsedElements {
+  footnoteLinks: number[];
+  footnotes: string[];
+  containLinks: boolean;
+  formattedWords: number;
+
+  constructor(options: { footnoteLinks?: number[]; footnotes?: string[]; containLinks?: boolean; formattedWords?: number }) {
+    this.footnoteLinks = options.footnoteLinks || [];
+    this.footnotes = options.footnotes || [];
+    this.containLinks = options.containLinks ?? false;
+    this.formattedWords = options.formattedWords ?? 0;
+  }
+
+  add(parsedElements: ParsedElements): void {
+    this.footnoteLinks = this.footnoteLinks.concat(
+      parsedElements.footnoteLinks,
+    );
+    this.footnotes = this.footnotes.concat(parsedElements.footnotes);
+    this.containLinks = this.containLinks || parsedElements.containLinks;
+    this.formattedWords += parsedElements.formattedWords;
+  }
+}

package/src/models/stashing-stream.ts

@@ -0,0 +1,86 @@
+/**
+ * @description Abstract stream processor that buffers ("stashes") consecutive
+ * items matching a predicate before flushing them as a group. Subclasses
+ * implement `shouldStash`, `doMatchesStash`, and `doFlushStash` to define what
+ * constitutes a run. Used by `WordDetectionStream` in `LineConverter` (grouping
+ * same-format text spans) and `VerticalsStream` in `VerticalToHorizontal`
+ * (merging single-character vertical lines into one horizontal line).
+ */
+// Abstract stream which allows stash items temporarily
+export default class StashingStream {
+  results: any[];
+  stash: any[];
+
+  constructor() {
+    if (this.constructor === StashingStream) {
+      throw new TypeError("Can not construct abstract class.");
+    }
+    this.results = [];
+    this.stash = [];
+  }
+
+  consumeAll(items: any[]): void {
+    items.forEach((item) => this.consume(item));
+  }
+
+  consume(item: any): void {
+    if (this.shouldStash(item)) {
+      if (!this.matchesStash(item)) {
+        this.flushStash();
+      }
+      this.pushOnStash(item);
+    } else {
+      if (this.stash.length > 0) {
+        this.flushStash();
+      }
+      this.results.push(item);
+    }
+  }
+
+  pushOnStash(item: any): void {
+    this.onPushOnStash(item);
+    this.stash.push(item);
+  }
+
+  complete(): any[] {
+    if (this.stash.length > 0) {
+      this.flushStash();
+    }
+    return this.results;
+  }
+
+  matchesStash(item: any): boolean {
+    if (this.stash.length === 0) {
+      return true;
+    }
+    const lastItem = this.stash[this.stash.length - 1];
+    return this.doMatchesStash(lastItem, item);
+  }
+
+  flushStash(): void {
+    if (this.stash.length > 0) {
+      this.doFlushStash(this.stash, this.results);
+      this.stash = [];
+    }
+  }
+
+  onPushOnStash(item: any): void {
+    // sub-classes may override
+  }
+
+  shouldStash(item: any): boolean {
+    throw new TypeError(" Do not call abstract method foo from child." + item);
+  }
+
+  doMatchesStash(lastItem: any, item: any): boolean {
+    throw new TypeError(
+      " Do not call abstract method foo from child." + lastItem + item,
+    );
+  }
+
+  doFlushStash(stash: any[], results: any[]): void {
+    throw new TypeError(
+      " Do not call abstract method foo from child." + stash + results,
+    );
+  }
+}

package/src/models/text-item-line-grouper.ts

@@ -0,0 +1,41 @@
+/**
+ * @description Groups a flat array of `TextItem`s into per-line buckets by
+ * clustering items whose y-coordinates differ by less than half of
+ * `mostUsedDistance`. Each bucket is then sorted by x so items read left-to-right,
+ * which is critical for footnote detection and word ordering in `CompactLines`.
+ */
+
+import TextItem from "./text-item";
+import { sortByX } from "../utils/page-item-functions";
+
+// Groups all text items which are on the same y line
+export default class TextItemLineGrouper {
+  mostUsedDistance: number;
+
+  constructor(options: { mostUsedDistance?: number }) {
+    this.mostUsedDistance = options.mostUsedDistance || 12;
+  }
+
+  group(textItems: TextItem[]): TextItem[][] {
+    const lines: TextItem[][] = [];
+    var currentLine: TextItem[] = [];
+    textItems.forEach((item) => {
+      if (
+        currentLine.length > 0 &&
+        Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2
+      ) {
+        lines.push(currentLine);
+        currentLine = [];
+      }
+      currentLine.push(item);
+    });
+    lines.push(currentLine);
+
+    lines.forEach((lineItems) => {
+      sortByX(lineItems);
+    });
+    return lines;
+  }
+}
+
+