extract-pdf 0.1.0 → 0.1.1

package/src/transforms/calculate-global-stats.ts

@@ -0,0 +1,132 @@
+/**
+ * @description First-pass transformation that scans all `TextItem`s to compute
+ * document-wide statistics stored in `parseResult.globals`: most-frequent text
+ * height (`mostUsedHeight`), most-frequent font, most-frequent line distance,
+ * maximum height, and a font-to-format map (BOLD/OBLIQUE/BOLD_OBLIQUE) derived
+ * from font names. Also deep-copies every page item so subsequent transforms
+ * never mutate the original parsed data.
+ */
+import ToTextItemTransformation from "./base/to-text-item-transform";
+import ParseResult from "../models/parse-result";
+import { WordFormat } from "../models/line-converter";
+
+export default class CalculateGlobalStats extends ToTextItemTransformation {
+  fontMap: Map<string, { name: string }> | undefined;
+
+  constructor(fontMap?: Map<string, { name: string }>) {
+    super("Calculate Global Stats");
+    this.fontMap = fontMap;
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const heightToOccurrence: Record<string, number> = {};
+    const fontToOccurrence: Record<string, number> = {};
+    var maxHeight = 0;
+    var maxHeightFont: string | undefined;
+    parseResult.pages.forEach((page) => {
+      page.items.forEach((item) => {
+        if (!item.height) return;
+        heightToOccurrence[item.height] = heightToOccurrence[item.height]
+          ? heightToOccurrence[item.height] + 1
+          : 1;
+        fontToOccurrence[item.font] = fontToOccurrence[item.font]
+          ? fontToOccurrence[item.font] + 1
+          : 1;
+        if (item.height > maxHeight) {
+          maxHeight = item.height;
+          maxHeightFont = item.font;
+        }
+      });
+    });
+    const mostUsedHeight = parseInt(getMostUsedKey(heightToOccurrence));
+    const mostUsedFont = getMostUsedKey(fontToOccurrence);
+
+    const distanceToOccurrence: Record<string, number> = {};
+    parseResult.pages.forEach((page) => {
+      var lastItemOfMostUsedHeight: any;
+      page.items.forEach((item) => {
+        if (item.height === mostUsedHeight && item.text.trim().length > 0) {
+          if (
+            lastItemOfMostUsedHeight &&
+            item.y !== lastItemOfMostUsedHeight.y
+          ) {
+            const distance = lastItemOfMostUsedHeight.y - item.y;
+            if (distance > 0) {
+              distanceToOccurrence[distance] = distanceToOccurrence[distance]
+                ? distanceToOccurrence[distance] + 1
+                : 1;
+            }
+          }
+          lastItemOfMostUsedHeight = item;
+        } else {
+          lastItemOfMostUsedHeight = null;
+        }
+      });
+    });
+    const mostUsedDistance = parseInt(getMostUsedKey(distanceToOccurrence));
+    const fontIdToName: string[] = [];
+    const fontToFormats = new Map<string, string>();
+    this.fontMap?.forEach(function (value, key) {
+      fontIdToName.push(key + " = " + value.name);
+      const fontName = value.name.toLowerCase();
+      var format: { name: string } | null = null;
+      if (key === mostUsedFont) {
+        format = null;
+      } else if (
+        fontName.includes("bold") &&
+        (fontName.includes("oblique") || fontName.includes("italic"))
+      ) {
+        format = WordFormat.BOLD_OBLIQUE;
+      } else if (fontName.includes("bold")) {
+        format = WordFormat.BOLD;
+      } else if (fontName.includes("oblique") || fontName.includes("italic")) {
+        format = WordFormat.OBLIQUE;
+      } else if (fontName === maxHeightFont) {
+        format = WordFormat.BOLD;
+      }
+      if (format) {
+        fontToFormats.set(key, format.name);
+      }
+    });
+    fontIdToName.sort();
+
+    const newPages = parseResult.pages.map((page) => {
+      return {
+        ...page,
+        items: page.items.map((textItem) => ({ ...textItem })),
+      };
+    });
+    return new ParseResult({
+      ...parseResult,
+      pages: newPages,
+      globals: {
+        mostUsedHeight,
+        mostUsedFont,
+        mostUsedDistance,
+        maxHeight,
+        maxHeightFont,
+        fontToFormats,
+      },
+      messages: [
+        "Items per height: " + JSON.stringify(heightToOccurrence),
+        "Items per font: " + JSON.stringify(fontToOccurrence),
+        "Items per distance: " + JSON.stringify(distanceToOccurrence),
+        "Fonts:" + JSON.stringify(fontIdToName),
+      ],
+    });
+  }
+}
+
+function getMostUsedKey(keyToOccurrence: Record<string, number>): string {
+  var maxOccurence = 0;
+  var maxKey = "";
+  Object.keys(keyToOccurrence).map((element) => {
+    if (!maxKey || keyToOccurrence[element] > maxOccurence) {
+      maxOccurence = keyToOccurrence[element];
+      maxKey = element;
+    }
+  });
+  return maxKey;
+}
+
+

package/src/transforms/line-item/compact-lines.ts

@@ -0,0 +1,92 @@
+/**
+ * @description Line-item transformation that collapses `TextItem`s sharing the
+ * same y-coordinate into single `LineItem`s via `TextItemLineGrouper` (line
+ * bucketing) and `LineConverter` (word/format detection). Simultaneously detects
+ * and records footnote links, footnote definitions, hyperlinks, and bold/italic
+ * word counts, tagging items as ADDED/REMOVED for debug rendering.
+ */
+import ToLineItemTransformation from "../base/to-line-item-transform";
+import ParseResult from "../../models/parse-result";
+import LineItem from "../../models/line-item";
+import TextItemLineGrouper from "../../models/text-item-line-grouper";
+import LineConverter from "../../models/line-converter";
+import BlockType from "../../models/block-type";
+import { REMOVED_ANNOTATION, ADDED_ANNOTATION } from "../../models/annotation";
+
+// gathers text items on the same y line to one line item
+export default class CompactLines extends ToLineItemTransformation {
+  constructor() {
+    super("Compact To Lines");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const { mostUsedDistance, fontToFormats } = parseResult.globals;
+    const foundFootnotes = [];
+    const foundFootnoteLinks = [];
+    var linkCount = 0;
+    var formattedWords = 0;
+
+    const lineGrouper = new TextItemLineGrouper({
+      mostUsedDistance: mostUsedDistance,
+    });
+    const lineCompactor = new LineConverter(fontToFormats);
+
+    parseResult.pages.forEach((page) => {
+      if (page.items.length > 0) {
+        const lineItems = [];
+        const textItemsGroupedByLine = lineGrouper.group(page.items);
+        textItemsGroupedByLine.forEach((lineTextItems) => {
+          const lineItem = lineCompactor.compact(lineTextItems);
+          if (lineTextItems.length > 1) {
+            lineItem.annotation = ADDED_ANNOTATION;
+            lineTextItems.forEach((item) => {
+              item.annotation = REMOVED_ANNOTATION;
+              lineItems.push(
+                new LineItem({
+                  ...item,
+                }),
+              );
+            });
+          }
+          if (lineItem.words.length === 0) {
+            lineItem.annotation = REMOVED_ANNOTATION;
+          }
+          lineItems.push(lineItem);
+
+          if (lineItem.parsedElements.formattedWords) {
+            formattedWords += lineItem.parsedElements.formattedWords;
+          }
+          if (lineItem.parsedElements.containLinks) {
+            linkCount++;
+          }
+          if (lineItem.parsedElements.footnoteLinks.length > 0) {
+            const footnoteLinks = lineItem.parsedElements.footnoteLinks.map(
+              (footnoteLink) => ({ footnoteLink, page: page.index + 1 }),
+            );
+            foundFootnoteLinks.push.apply(foundFootnoteLinks, footnoteLinks);
+          }
+          if (lineItem.parsedElements.footnotes.length > 0) {
+            lineItem.type = BlockType.FOOTNOTES;
+            const footnotes = lineItem.parsedElements.footnotes.map(
+              (footnote) => ({ footnote, page: page.index + 1 }),
+            );
+            foundFootnotes.push.apply(foundFootnotes, footnotes);
+          }
+        });
+        page.items = lineItems;
+      }
+    });
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        "Detected " + formattedWords + " formatted words",
+        "Found " + linkCount + " links",
+        "Detected " + foundFootnoteLinks.length + " footnotes links",
+        "Detected " + foundFootnotes.length + " footnotes",
+      ],
+    });
+  }
+}
+
+

package/src/transforms/line-item/detect-headers.ts

@@ -0,0 +1,173 @@
+/**
+ * @description Line-item transformation that assigns H1–H6 block types based on
+ * font height relative to the document's body height. Handles three cases: title
+ * pages (items at the document's maximum height), TOC-derived headline height
+ * ranges when a table of contents was found, and a fallback that ranks all
+ * above-average heights largest-first and maps them to heading levels. Also
+ * detects all-uppercase body-font lines as the next lower heading level.
+ */
+import ToLineItemTransformation from "../base/to-line-item-transform";
+import ParseResult from "../../models/parse-result";
+import { DETECTED_ANNOTATION } from "../../models/annotation";
+import BlockType from "../../models/block-type";
+import { isListItem } from "../../utils/string-functions";
+
+// Detect headlines based on heights
+export default class DetectHeaders extends ToLineItemTransformation {
+  constructor() {
+    super("Detect Headers");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const {
+      tocPages,
+      headlineTypeToHeightRange,
+      mostUsedHeight,
+      mostUsedDistance,
+      mostUsedFont,
+      maxHeight,
+    } = parseResult.globals;
+    const hasToc = tocPages.length > 0;
+    var detectedHeaders = 0;
+
+    // Handle title pages
+    const pagesWithMaxHeight = findPagesWithMaxHeight(
+      parseResult.pages,
+      maxHeight,
+    );
+    const min2ndLevelHeaderHeigthOnMaxPage =
+      mostUsedHeight + (maxHeight - mostUsedHeight) / 4;
+    pagesWithMaxHeight.forEach((titlePage) => {
+      titlePage.items.forEach((item) => {
+        const height = item.height;
+        if (!item.type && height > min2ndLevelHeaderHeigthOnMaxPage) {
+          if (height === maxHeight) {
+            item.type = BlockType.H1;
+          } else {
+            item.type = BlockType.H2;
+          }
+          item.annotation = DETECTED_ANNOTATION;
+          detectedHeaders++;
+        }
+      });
+    });
+
+    if (hasToc) {
+      // Use existing headline heights to find additional headlines
+      const headlineTypes = Object.keys(headlineTypeToHeightRange);
+      headlineTypes.forEach((headlineType) => {
+        var range = headlineTypeToHeightRange[headlineType];
+        if (range.max > mostUsedHeight) {
+          // use only very clear headlines, only use max
+          parseResult.pages.forEach((page) => {
+            page.items.forEach((item) => {
+              if (!item.type && item.height === range.max) {
+                item.annotation = DETECTED_ANNOTATION;
+                item.type = BlockType.enumValueOf(headlineType);
+                detectedHeaders++;
+              }
+            });
+          });
+        }
+      });
+    } else {
+      // Categorize headlines by the text heights
+      const heights = [];
+      var lastHeight;
+      parseResult.pages.forEach((page) => {
+        page.items.forEach((item) => {
+          if (
+            !item.type &&
+            item.height > mostUsedHeight &&
+            !isListItem(item.text())
+          ) {
+            if (
+              !heights.includes(item.height) &&
+              (!lastHeight || lastHeight > item.height)
+            ) {
+              heights.push(item.height);
+            }
+          }
+        });
+      });
+      heights.sort((a, b) => b - a);
+
+      heights.forEach((height, i) => {
+        const headlineLevel = i + 2;
+        if (headlineLevel <= 6) {
+          const headlineType = BlockType.headlineByLevel(2 + i);
+          parseResult.pages.forEach((page) => {
+            page.items.forEach((item) => {
+              if (
+                !item.type &&
+                item.height === height &&
+                !isListItem(item.text())
+              ) {
+                detectedHeaders++;
+                item.annotation = DETECTED_ANNOTATION;
+                item.type = headlineType;
+              }
+            });
+          });
+        }
+      });
+    }
+
+    // find headlines which have paragraph height
+    var smallesHeadlineLevel = 1;
+    parseResult.pages.forEach((page) => {
+      page.items.forEach((item) => {
+        if (item.type && item.type.headline) {
+          smallesHeadlineLevel = Math.max(
+            smallesHeadlineLevel,
+            item.type.headlineLevel,
+          );
+        }
+      });
+    });
+    if (smallesHeadlineLevel < 6) {
+      const nextHeadlineType = BlockType.headlineByLevel(
+        smallesHeadlineLevel + 1,
+      );
+      parseResult.pages.forEach((page) => {
+        var lastItem;
+        page.items.forEach((item) => {
+          if (
+            !item.type &&
+            item.height === mostUsedHeight &&
+            item.font !== mostUsedFont &&
+            (!lastItem ||
+              lastItem.y < item.y ||
+              (lastItem.type && lastItem.type.headline) ||
+              lastItem.y - item.y > mostUsedDistance * 2) &&
+            item.text() === item.text().toUpperCase()
+          ) {
+            detectedHeaders++;
+            item.annotation = DETECTED_ANNOTATION;
+            item.type = nextHeadlineType;
+          }
+          lastItem = item;
+        });
+      });
+    }
+
+    return new ParseResult({
+      ...parseResult,
+      messages: ["Detected " + detectedHeaders + " headlines."],
+    });
+  }
+}
+
+function findPagesWithMaxHeight(pages: any[], maxHeight: number): Set<any> {
+  const maxHeaderPagesSet = new Set<any>();
+  pages.forEach((page) => {
+    page.items.forEach((item) => {
+      if (!item.type && item.height === maxHeight) {
+        maxHeaderPagesSet.add(page);
+      }
+    });
+  });
+  return maxHeaderPagesSet;
+}
+
+

package/src/transforms/line-item/detect-list-items.ts

@@ -0,0 +1,68 @@
+/**
+ * @description Line-item transformation that identifies list items starting with
+ * bullet characters (-, •, –) or numbered patterns (`1. text`) and assigns
+ * `BlockType.LIST`. Non-dash bullet characters are normalised to `-` by marking
+ * the original line REMOVED and inserting a replacement `LineItem` (ADDED),
+ * preserving the debug diff trail.
+ */
+
+import ToLineItemTransformation from '../base/to-line-item-transform'
+import ParseResult from '../../models/parse-result'
+import LineItem from '../../models/line-item'
+import Word from '../../models/word'
+import { REMOVED_ANNOTATION, ADDED_ANNOTATION, DETECTED_ANNOTATION } from '../../models/annotation'
+import BlockType from '../../models/block-type'
+import { isListItemCharacter, isNumberedListItem } from '../../utils/string-functions'
+
+// Detect items starting with -, \u2022, etc...
+export default class DetectListItems extends ToLineItemTransformation {
+  constructor () {
+    super('Detect List Items')
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    var foundListItems = 0
+    var foundNumberedItems = 0
+    parseResult.pages.forEach(page => {
+      const newItems = []
+      page.items.forEach(item => {
+        newItems.push(item)
+        if (!item.type) {
+          var text = item.text()
+          if (isListItemCharacter(item.words[0].string)) {
+            foundListItems++
+            if (item.words[0].string === '-') {
+              item.annotation = DETECTED_ANNOTATION
+              item.type = BlockType.LIST
+            } else {
+              item.annotation = REMOVED_ANNOTATION
+              const newWords = item.words.map(word => new Word({
+                ...word,
+              }))
+              newWords[0].string = '-'
+              newItems.push(new LineItem({
+                ...item,
+                words: newWords,
+                annotation: ADDED_ANNOTATION,
+                type: BlockType.LIST,
+              }))
+            }
+          } else if (isNumberedListItem(text)) { // TODO check that starts with 1 (kala chakra)
+            foundNumberedItems++
+            item.annotation = DETECTED_ANNOTATION
+            item.type = BlockType.LIST
+          }
+        }
+      })
+      page.items = newItems
+    })
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        'Detected ' + foundListItems + ' plain list items.',
+        'Detected ' + foundNumberedItems + ' numbered list items.',
+      ],
+    })
+  }
+}