extract-pdf 0.1.0 → 0.1.1

package/src/transforms/line-item/detect-toc.ts

@@ -0,0 +1,459 @@
+/**
+ * @description Line-item transformation that identifies Table of Contents pages
+ * (≥75% of lines end with a page number) and extracts TOC links with hierarchy
+ * levels determined by x-position or font differences. Then cross-references
+ * each TOC link against document pages via `HeadlineFinder` to tag the actual
+ * heading text with H2–H6 types, falling back to font-height range matching for
+ * headings that could not be located by exact text.
+ */
+
+import ToLineItemTransformation from "../base/to-line-item-transform";
+import ParseResult from "../../models/parse-result";
+import LineItem from "../../models/line-item";
+import Word from "../../models/word";
+import HeadlineFinder from "../../models/headline-finder";
+import { REMOVED_ANNOTATION, ADDED_ANNOTATION } from "../../models/annotation";
+import BlockType from "../../models/block-type";
+import {
+  isDigit,
+  isNumber,
+  wordMatch,
+  hasOnly,
+} from "../../utils/string-functions";
+
+// Detect table of contents pages plus linked headlines
+export default class DetectTOC extends ToLineItemTransformation {
+  constructor() {
+    super("Detect TOC");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    const tocPages = [];
+    const maxPagesToEvaluate = Math.min(20, parseResult.pages.length);
+    const linkLeveler = new LinkLeveler();
+
+    var tocLinks = [];
+    var lastTocPage;
+    var headlineItem;
+    parseResult.pages.slice(0, maxPagesToEvaluate).forEach((page) => {
+      var lineItemsWithDigits = 0;
+      const unknownLines = new Set();
+      const pageTocLinks = [];
+      var lastWordsWithoutNumber;
+      var lastLine;
+      // find lines with words containing only "." ...
+      const tocLines = page.items.filter((line) =>
+        line.words.includes((word) => hasOnly(word.string, ".")),
+      );
+      // ... and ending with a number per page
+      tocLines.forEach((line) => {
+        var words = line.words.filter((word) => !hasOnly(word.string, "."));
+        const digits = [];
+        while (words.length > 0 && isNumber(words[words.length - 1].string)) {
+          const lastWord = words.pop();
+          digits.unshift(lastWord.string);
+        }
+
+        if (digits.length === 0 && words.length > 0) {
+          const lastWord = words[words.length - 1];
+          while (
+            isDigit(lastWord.string.charCodeAt(lastWord.string.length - 1))
+          ) {
+            digits.unshift(lastWord.string.charAt(lastWord.string.length - 1));
+            lastWord.string = lastWord.string.substring(
+              0,
+              lastWord.string.length - 1,
+            );
+          }
+        }
+        var endsWithDigit = digits.length > 0;
+        if (endsWithDigit) {
+          endsWithDigit = true;
+          if (lastWordsWithoutNumber) {
+            // 2-line item ?
+            words.push(...lastWordsWithoutNumber);
+            lastWordsWithoutNumber = null;
+          }
+          pageTocLinks.push(
+            new TocLink({
+              pageNumber: parseInt(digits.join("")),
+              lineItem: new LineItem({ ...line, words }),
+            }),
+          );
+          lineItemsWithDigits++;
+        } else {
+          if (!headlineItem) {
+            headlineItem = line;
+          } else {
+            if (lastWordsWithoutNumber) {
+              unknownLines.add(lastLine);
+            }
+            lastWordsWithoutNumber = words;
+            lastLine = line;
+          }
+        }
+      });
+
+      // page has been processed
+      if ((lineItemsWithDigits * 100) / page.items.length > 75) {
+        tocPages.push(page.index + 1);
+        lastTocPage = page;
+        linkLeveler.levelPageItems(pageTocLinks);
+        tocLinks.push(...pageTocLinks);
+
+        const newBlocks = [];
+        page.items.forEach((line) => {
+          if (!unknownLines.has(line)) {
+            line.annotation = REMOVED_ANNOTATION;
+          }
+          newBlocks.push(line);
+          if (line === headlineItem) {
+            newBlocks.push(
+              new LineItem({
+                ...line,
+                type: BlockType.H2,
+                annotation: ADDED_ANNOTATION,
+              }),
+            );
+          }
+        });
+        page.items = newBlocks;
+      } else {
+        headlineItem = null;
+      }
+    });
+
+    // all pages have been processed
+    var foundHeadlines = tocLinks.length;
+    const notFoundHeadlines = [];
+    const foundBySize = [];
+    const headlineTypeToHeightRange = {}; // H1={min:23, max:25}
+
+    if (tocPages.length > 0) {
+      // Add TOC items
+      tocLinks.forEach((tocLink) => {
+        lastTocPage.items.push(
+          new LineItem({
+            words: [
+              new Word({
+                string: " ".repeat(tocLink.level * 3) + "-",
+              }),
+            ].concat(tocLink.lineItem.words),
+            type: BlockType.TOC,
+            annotation: ADDED_ANNOTATION,
+          }),
+        );
+      });
+
+      // Add linked headers
+      const pageMapping = detectPageMappingNumber(
+        parseResult.pages.filter((page) => page.index > lastTocPage.index),
+        tocLinks,
+      );
+      tocLinks.forEach((tocLink) => {
+        var linkedPage = parseResult.pages[tocLink.pageNumber + pageMapping];
+        var foundHealineItems;
+        if (linkedPage) {
+          foundHealineItems = findHeadlineItems(
+            linkedPage,
+            tocLink.lineItem.text(),
+          );
+          if (!foundHealineItems) {
+            // pages are off by 1 ?
+            linkedPage =
+              parseResult.pages[tocLink.pageNumber + pageMapping + 1];
+            if (linkedPage) {
+              foundHealineItems = findHeadlineItems(
+                linkedPage,
+                tocLink.lineItem.text(),
+              );
+            }
+          }
+        }
+        if (foundHealineItems) {
+          addHeadlineItems(
+            linkedPage,
+            tocLink,
+            foundHealineItems,
+            headlineTypeToHeightRange,
+          );
+        } else {
+          notFoundHeadlines.push(tocLink);
+        }
+      });
+
+      // Try to find linked headers by height
+      var fromPage = lastTocPage.index + 2;
+      var lastNotFound = [];
+      const rollupLastNotFound = (currentPageNumber) => {
+        if (lastNotFound.length > 0) {
+          lastNotFound.forEach((notFoundTocLink) => {
+            const headlineType = BlockType.headlineByLevel(
+              notFoundTocLink.level + 2,
+            );
+            const heightRange = headlineTypeToHeightRange[headlineType.name];
+            if (heightRange) {
+              const [pageIndex, lineIndex] = findPageAndLineFromHeadline(
+                parseResult.pages,
+                notFoundTocLink,
+                heightRange,
+                fromPage,
+                currentPageNumber,
+              );
+              if (lineIndex > -1) {
+                const page = parseResult.pages[pageIndex];
+                page.items[lineIndex].annotation = REMOVED_ANNOTATION;
+                page.items.splice(
+                  lineIndex + 1,
+                  0,
+                  new LineItem({
+                    ...notFoundTocLink.lineItem,
+                    type: headlineType,
+                    annotation: ADDED_ANNOTATION,
+                  }),
+                );
+                foundBySize.push(notFoundTocLink);
+              }
+            }
+          });
+          lastNotFound = [];
+        }
+      };
+      if (notFoundHeadlines.length > 0) {
+        tocLinks.forEach((tocLink) => {
+          if (notFoundHeadlines.includes(tocLink)) {
+            lastNotFound.push(tocLink);
+          } else {
+            rollupLastNotFound(tocLink.pageNumber);
+            fromPage = tocLink.pageNumber;
+          }
+        });
+        if (lastNotFound.length > 0) {
+          rollupLastNotFound(parseResult.pages.length);
+        }
+      }
+    }
+
+    const messages = [];
+    messages.push("Detected " + tocPages.length + " table of content pages");
+    if (tocPages.length > 0) {
+      messages.push(
+        "TOC headline heights: " + JSON.stringify(headlineTypeToHeightRange),
+      );
+      messages.push(
+        "Found TOC headlines: " +
+          (foundHeadlines - notFoundHeadlines.length + foundBySize.length) +
+          "/" +
+          foundHeadlines,
+      );
+    }
+    if (notFoundHeadlines.length > 0) {
+      messages.push(
+        "Found TOC headlines (by size): " +
+          foundBySize.map((tocLink) => tocLink.lineItem.text()),
+      );
+      messages.push(
+        "Missing TOC headlines: " +
+          notFoundHeadlines
+            .filter((fTocLink) => !foundBySize.includes(fTocLink))
+            .map(
+              (tocLink) => tocLink.lineItem.text() + "=>" + tocLink.pageNumber,
+            ),
+      );
+    }
+    return new ParseResult({
+      ...parseResult,
+      globals: {
+        ...parseResult.globals,
+        tocPages,
+        headlineTypeToHeightRange,
+      },
+      messages,
+    });
+  }
+}
+
+// Find out how the TOC page link actualy translates to the page.index
+function detectPageMappingNumber(pages, tocLinks) {
+  for (var tocLink of tocLinks) {
+    const page = findPageWithHeadline(pages, tocLink.lineItem.text());
+    if (page) {
+      return page.index - tocLink.pageNumber;
+    }
+  }
+  return null;
+}
+
+function findPageWithHeadline(pages, headline) {
+  for (var page of pages) {
+    if (findHeadlineItems(page, headline)) {
+      return page;
+    }
+  }
+  return null;
+}
+
+function findHeadlineItems(page, headline) {
+  const headlineFinder = new HeadlineFinder({ headline });
+  var lineIndex = 0;
+  for (var line of page.items) {
+    const headlineItems = headlineFinder.consume(line);
+    if (headlineItems) {
+      return { lineIndex, headlineItems };
+    }
+    lineIndex++;
+  }
+  return null;
+}
+
+function addHeadlineItems(
+  page,
+  tocLink,
+  foundItems,
+  headlineTypeToHeightRange,
+) {
+  foundItems.headlineItems.forEach(
+    (item) => (item.annotation = REMOVED_ANNOTATION),
+  );
+  const headlineType = BlockType.headlineByLevel(tocLink.level + 2);
+  const headlineHeight = foundItems.headlineItems.reduce(
+    (max, item) => Math.max(max, item.height),
+    0,
+  );
+  page.items.splice(
+    foundItems.lineIndex + 1,
+    0,
+    new LineItem({
+      ...foundItems.headlineItems[0],
+      words: tocLink.lineItem.words,
+      height: headlineHeight,
+      type: headlineType,
+      annotation: ADDED_ANNOTATION,
+    }),
+  );
+  var range = headlineTypeToHeightRange[headlineType.name];
+  if (range) {
+    range.min = Math.min(range.min, headlineHeight);
+    range.max = Math.max(range.max, headlineHeight);
+  } else {
+    range = {
+      min: headlineHeight,
+      max: headlineHeight,
+    };
+    headlineTypeToHeightRange[headlineType.name] = range;
+  }
+}
+
+function findPageAndLineFromHeadline(
+  pages,
+  tocLink,
+  heightRange,
+  fromPage,
+  toPage,
+) {
+  const linkText = tocLink.lineItem.text().toUpperCase();
+  for (var i = fromPage; i <= toPage; i++) {
+    const page = pages[i - 1];
+    if (page) {
+      const lineIndex = page.items.findIndex((line) => {
+        if (
+          !line.type &&
+          !line.annotation &&
+          line.height >= heightRange.min &&
+          line.height <= heightRange.max
+        ) {
+          const match = wordMatch(linkText, line.text());
+          return match >= 0.5;
+        }
+        return false;
+      });
+      if (lineIndex > -1) return [i - 1, lineIndex];
+    }
+  }
+  return [-1, -1];
+}
+
+class LinkLeveler {
+  levelByMethod: any;
+  uniqueFonts: any[];
+
+  constructor() {
+    this.levelByMethod = null;
+    this.uniqueFonts = [];
+  }
+
+  levelPageItems(tocLinks /*: TocLink[] */) {
+    if (!this.levelByMethod) {
+      const uniqueX = this.calculateUniqueX(tocLinks);
+      if (uniqueX.length > 1) {
+        this.levelByMethod = this.levelByXDiff;
+      } else {
+        const uniqueFonts = this.calculateUniqueFonts(tocLinks);
+        if (uniqueFonts.length > 1) {
+          this.uniqueFonts = uniqueFonts;
+          this.levelByMethod = this.levelByFont;
+        } else {
+          this.levelByMethod = this.levelToZero;
+        }
+      }
+    }
+    this.levelByMethod(tocLinks);
+  }
+
+  levelByXDiff(tocLinks) {
+    const uniqueX = this.calculateUniqueX(tocLinks);
+    tocLinks.forEach((link) => {
+      link.level = uniqueX.indexOf(link.lineItem.x);
+    });
+  }
+
+  levelByFont(tocLinks) {
+    tocLinks.forEach((link) => {
+      link.level = this.uniqueFonts.indexOf(link.lineItem.font);
+    });
+  }
+
+  levelToZero(tocLinks) {
+    tocLinks.forEach((link) => {
+      link.level = 0;
+    });
+  }
+
+  calculateUniqueX(tocLinks) {
+    var uniqueX = tocLinks.reduce(function (uniquesArray, link) {
+      if (uniquesArray.indexOf(link.lineItem.x) < 0)
+        uniquesArray.push(link.lineItem.x);
+      return uniquesArray;
+    }, []);
+
+    uniqueX.sort((a, b) => {
+      return a - b;
+    });
+
+    return uniqueX;
+  }
+
+  calculateUniqueFonts(tocLinks) {
+    var uniqueFont = tocLinks.reduce(function (uniquesArray, link) {
+      if (uniquesArray.indexOf(link.lineItem.font) < 0)
+        uniquesArray.push(link.lineItem.font);
+      return uniquesArray;
+    }, []);
+
+    return uniqueFont;
+  }
+}
+
+class TocLink {
+  lineItem: any;
+  pageNumber: number;
+  level: number;
+
+  constructor(options: any) {
+    this.lineItem = options.lineItem;
+    this.pageNumber = options.pageNumber;
+    this.level = 0;
+  }
+}
+
+

package/src/transforms/line-item/remove-repetitive-elements.ts

@@ -0,0 +1,101 @@
+/**
+ * @description Line-item transformation that removes running page headers and
+ * footers. Hashes the topmost and bottommost line of each page (ignoring digits
+ * and whitespace so page numbers don't break the match), then marks lines whose
+ * hash appears on more than two-thirds of all pages with `REMOVED_ANNOTATION`.
+ */
+import ToLineItemTransformation from '../base/to-line-item-transform'
+import ParseResult from '../../models/parse-result'
+import { REMOVED_ANNOTATION } from '../../models/annotation'
+
+import { isDigit } from '../../utils/string-functions'
+
+function hashCodeIgnoringSpacesAndNumbers(string: string): number {
+  var hash = 0
+  if (string.trim().length === 0) return hash
+  for (var i = 0; i < string.length; i++) {
+    const charCode = string.charCodeAt(i)
+    if (!isDigit(charCode) && charCode !== 32 && charCode !== 160) {
+      hash = ((hash << 5) - hash) + charCode
+      hash |= 0 // Convert to 32bit integer
+    }
+  }
+  return hash
+}
+
+// Remove elements with similar content on same page positions, like page numbers, licenes information, etc...
+export default class RemoveRepetitiveElements extends ToLineItemTransformation {
+  constructor () {
+    super('Remove Repetitive Elements')
+  }
+
+  // The idea is the following:
+  // - For each page, collect all items of the first, and all items of the last line
+  // - Calculate how often these items occur accros all pages (hash ignoring numbers, whitespace, upper/lowercase)
+  // - Delete items occuring on more then 2/3 of all pages
+  transform(parseResult: ParseResult): ParseResult {
+    // find first and last lines per page
+    const pageStore = []
+    const minLineHashRepetitions = {}
+    const maxLineHashRepetitions = {}
+    parseResult.pages.forEach(page => {
+      const minMaxItems = page.items.reduce((itemStore, item) => {
+        if (item.y < itemStore.minY) {
+          itemStore.minElements = [item]
+          itemStore.minY = item.y
+        } else if (item.y === itemStore.minY) {
+          itemStore.minElements.push(item)
+        }
+        if (item.y > itemStore.maxY) {
+          itemStore.maxElements = [item]
+          itemStore.maxY = item.y
+        } else if (item.y === itemStore.maxY) {
+          itemStore.maxElements.push(item)
+        }
+        return itemStore
+      }, {
+        minY: 999,
+        maxY: 0,
+        minElements: [],
+        maxElements: [],
+      })
+
+      const minLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.minElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
+      const maxLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.maxElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
+      pageStore.push({
+        minElements: minMaxItems.minElements,
+        maxElements: minMaxItems.maxElements,
+        minLineHash: minLineHash,
+        maxLineHash: maxLineHash,
+      })
+      minLineHashRepetitions[minLineHash] = minLineHashRepetitions[minLineHash] ? minLineHashRepetitions[minLineHash] + 1 : 1
+      maxLineHashRepetitions[maxLineHash] = maxLineHashRepetitions[maxLineHash] ? maxLineHashRepetitions[maxLineHash] + 1 : 1
+    })
+
+    // now annoate all removed items
+    var removedHeader = 0
+    var removedFooter = 0
+    parseResult.pages.forEach((page, i) => {
+      if (minLineHashRepetitions[pageStore[i].minLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
+        pageStore[i].minElements.forEach(item => {
+          item.annotation = REMOVED_ANNOTATION
+        })
+        removedFooter++
+      }
+      if (maxLineHashRepetitions[pageStore[i].maxLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
+        pageStore[i].maxElements.forEach(item => {
+          item.annotation = REMOVED_ANNOTATION
+        })
+        removedHeader++
+      }
+    })
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        'Removed Header: ' + removedHeader,
+        'Removed Footers: ' + removedFooter,
+      ],
+    })
+  }
+}

package/src/transforms/line-item/vertical-to-horizontal.ts

@@ -0,0 +1,90 @@
+/**
+ * @description Line-item transformation that recovers rotated sidebar text.
+ * PDF extracts vertically-oriented labels as a sequence of single-character
+ * lines. `VerticalsStream` (a `StashingStream` subclass) detects runs of 6+
+ * such lines and merges them into one horizontal `LineItem`, combining their
+ * words and computing the correct bounding box.
+ */
+import ToLineItemTransformation from "../base/to-line-item-transform";
+import ParseResult from "../../models/parse-result";
+import LineItem from "../../models/line-item";
+import StashingStream from "../../models/stashing-stream";
+import { REMOVED_ANNOTATION, ADDED_ANNOTATION } from "../../models/annotation";
+
+// Converts vertical text to horizontal
+export default class VerticalToHorizontal extends ToLineItemTransformation {
+  constructor() {
+    super("Vertical to Horizontal Text");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    var foundVerticals = 0;
+    parseResult.pages.forEach((page) => {
+      const stream = new VerticalsStream();
+      stream.consumeAll(page.items);
+      page.items = stream.complete();
+      foundVerticals += stream.foundVerticals;
+    });
+
+    return new ParseResult({
+      ...parseResult,
+      messages: ["Converted " + foundVerticals + " verticals"],
+    });
+  }
+}
+
+class VerticalsStream extends StashingStream {
+  foundVerticals: number;
+
+  constructor() {
+    super();
+    this.foundVerticals = 0;
+  }
+
+  shouldStash(item: any): boolean {
+    return item.words.length === 1 && item.words[0].string.length === 1;
+  }
+
+  doMatchesStash(lastItem: any, item: any): boolean {
+    return (
+      lastItem.y - item.y > 5 && lastItem.words[0].type === item.words[0].type
+    );
+  }
+
+  doFlushStash(stash: any[], results: any[]): void {
+    if (stash.length > 5) {
+      // unite
+      var combinedWords = [];
+      var minX = 999;
+      var maxY = 0;
+      var sumWidth = 0;
+      var maxHeight = 0;
+      stash.forEach((oneCharacterLine) => {
+        oneCharacterLine.annotation = REMOVED_ANNOTATION;
+        results.push(oneCharacterLine);
+        combinedWords.push(oneCharacterLine.words[0]);
+        minX = Math.min(minX, oneCharacterLine.x);
+        maxY = Math.max(maxY, oneCharacterLine.y);
+        sumWidth += oneCharacterLine.width;
+        maxHeight = Math.max(maxHeight, oneCharacterLine.height);
+      });
+      results.push(
+        new LineItem({
+          ...stash[0],
+          x: minX,
+          y: maxY,
+          width: sumWidth,
+          height: maxHeight,
+          words: combinedWords,
+          annotation: ADDED_ANNOTATION,
+        }),
+      );
+      this.foundVerticals++;
+    } else {
+      // add as singles
+      results.push(...stash);
+    }
+  }
+}
+
+

package/src/transforms/to-html.ts

@@ -0,0 +1,46 @@
+/**
+ * @description Final serialization stage that converts each page's text block
+ * items into `<p>`-wrapped HTML. TOC blocks are emitted verbatim; hyphenated
+ * line-break artifacts are stripped from non-list blocks; `<code>` fencing is
+ * removed from CODE-category blocks (treated as prose in this simplified output).
+ */
+
+import Transformation from './base/transformation'
+import ParseResult from '../models/parse-result'
+
+export default class ToHTML extends Transformation {
+  constructor () {
+    super('To HTML', 'String')
+  }
+  transform(parseResult: ParseResult): ParseResult {
+    parseResult.pages.forEach(page => {
+      var text = ''
+      page.items.forEach(block => {
+        // Concatenate all words in the same block, unless it's a Table of Contents block
+        let concatText
+        if (block.category === 'TOC') {
+          concatText = block.text
+        } else {
+          concatText = block.text.replace(/(\r\n|\n|\r)/gm, '\n')
+        }
+
+        // Concatenate words that were previously broken up by newline
+        if (block.category !== 'LIST') {
+          concatText = concatText.split('- ').join('')  
+        }
+
+        // Assume there are no code blocks in our documents
+        if (block.category === 'CODE') {
+          concatText = concatText.split('<code>').join('').split('</code>').join('')
+        }
+
+        text += `<p>${concatText}</p>\n\n`
+      })
+
+      page.items = [text]
+    })
+    return new ParseResult({
+      ...parseResult,
+    })
+  }
+}