extract-pdf 0.1.21 → 0.1.22

package/src/transforms/line-item/remove-repetitive-elements.ts

@@ -1,101 +1,101 @@
-/**
- * @description Line-item transformation that removes running page headers and
- * footers. Hashes the topmost and bottommost line of each page (ignoring digits
- * and whitespace so page numbers don't break the match), then marks lines whose
- * hash appears on more than two-thirds of all pages with `REMOVED_ANNOTATION`.
- */
-import ToLineItemTransformation from '../base/to-line-item-transform'
-import ParseResult from '../../models/parse-result'
-import { REMOVED_ANNOTATION } from '../../models/annotation'
-
-import { isDigit } from '../../utils/string-functions'
-
-function hashCodeIgnoringSpacesAndNumbers(string: string): number {
-  var hash = 0
-  if (string.trim().length === 0) return hash
-  for (var i = 0; i < string.length; i++) {
-    const charCode = string.charCodeAt(i)
-    if (!isDigit(charCode) && charCode !== 32 && charCode !== 160) {
-      hash = ((hash << 5) - hash) + charCode
-      hash |= 0 // Convert to 32bit integer
-    }
-  }
-  return hash
-}
-
-// Remove elements with similar content on same page positions, like page numbers, licenes information, etc...
-export default class RemoveRepetitiveElements extends ToLineItemTransformation {
-  constructor () {
-    super('Remove Repetitive Elements')
-  }
-
-  // The idea is the following:
-  // - For each page, collect all items of the first, and all items of the last line
-  // - Calculate how often these items occur accros all pages (hash ignoring numbers, whitespace, upper/lowercase)
-  // - Delete items occuring on more then 2/3 of all pages
-  transform(parseResult: ParseResult): ParseResult {
-    // find first and last lines per page
-    const pageStore = []
-    const minLineHashRepetitions = {}
-    const maxLineHashRepetitions = {}
-    parseResult.pages.forEach(page => {
-      const minMaxItems = page.items.reduce((itemStore, item) => {
-        if (item.y < itemStore.minY) {
-          itemStore.minElements = [item]
-          itemStore.minY = item.y
-        } else if (item.y === itemStore.minY) {
-          itemStore.minElements.push(item)
-        }
-        if (item.y > itemStore.maxY) {
-          itemStore.maxElements = [item]
-          itemStore.maxY = item.y
-        } else if (item.y === itemStore.maxY) {
-          itemStore.maxElements.push(item)
-        }
-        return itemStore
-      }, {
-        minY: 999,
-        maxY: 0,
-        minElements: [],
-        maxElements: [],
-      })
-
-      const minLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.minElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
-      const maxLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.maxElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
-      pageStore.push({
-        minElements: minMaxItems.minElements,
-        maxElements: minMaxItems.maxElements,
-        minLineHash: minLineHash,
-        maxLineHash: maxLineHash,
-      })
-      minLineHashRepetitions[minLineHash] = minLineHashRepetitions[minLineHash] ? minLineHashRepetitions[minLineHash] + 1 : 1
-      maxLineHashRepetitions[maxLineHash] = maxLineHashRepetitions[maxLineHash] ? maxLineHashRepetitions[maxLineHash] + 1 : 1
-    })
-
-    // now annoate all removed items
-    var removedHeader = 0
-    var removedFooter = 0
-    parseResult.pages.forEach((page, i) => {
-      if (minLineHashRepetitions[pageStore[i].minLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
-        pageStore[i].minElements.forEach(item => {
-          item.annotation = REMOVED_ANNOTATION
-        })
-        removedFooter++
-      }
-      if (maxLineHashRepetitions[pageStore[i].maxLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
-        pageStore[i].maxElements.forEach(item => {
-          item.annotation = REMOVED_ANNOTATION
-        })
-        removedHeader++
-      }
-    })
-
-    return new ParseResult({
-      ...parseResult,
-      messages: [
-        'Removed Header: ' + removedHeader,
-        'Removed Footers: ' + removedFooter,
-      ],
-    })
-  }
-}
+/**
+ * @description Line-item transformation that removes running page headers and
+ * footers. Hashes the topmost and bottommost line of each page (ignoring digits
+ * and whitespace so page numbers don't break the match), then marks lines whose
+ * hash appears on more than two-thirds of all pages with `REMOVED_ANNOTATION`.
+ */
+import ToLineItemTransformation from '../base/to-line-item-transform'
+import ParseResult from '../../models/parse-result'
+import { REMOVED_ANNOTATION } from '../../models/annotation'
+
+import { isDigit } from '../../utils/string-functions'
+
+function hashCodeIgnoringSpacesAndNumbers(string: string): number {
+  var hash = 0
+  if (string.trim().length === 0) return hash
+  for (var i = 0; i < string.length; i++) {
+    const charCode = string.charCodeAt(i)
+    if (!isDigit(charCode) && charCode !== 32 && charCode !== 160) {
+      hash = ((hash << 5) - hash) + charCode
+      hash |= 0 // Convert to 32bit integer
+    }
+  }
+  return hash
+}
+
+// Remove elements with similar content on same page positions, like page numbers, licenes information, etc...
+export default class RemoveRepetitiveElements extends ToLineItemTransformation {
+  constructor () {
+    super('Remove Repetitive Elements')
+  }
+
+  // The idea is the following:
+  // - For each page, collect all items of the first, and all items of the last line
+  // - Calculate how often these items occur accros all pages (hash ignoring numbers, whitespace, upper/lowercase)
+  // - Delete items occuring on more then 2/3 of all pages
+  transform(parseResult: ParseResult): ParseResult {
+    // find first and last lines per page
+    const pageStore = []
+    const minLineHashRepetitions = {}
+    const maxLineHashRepetitions = {}
+    parseResult.pages.forEach(page => {
+      const minMaxItems = page.items.reduce((itemStore, item) => {
+        if (item.y < itemStore.minY) {
+          itemStore.minElements = [item]
+          itemStore.minY = item.y
+        } else if (item.y === itemStore.minY) {
+          itemStore.minElements.push(item)
+        }
+        if (item.y > itemStore.maxY) {
+          itemStore.maxElements = [item]
+          itemStore.maxY = item.y
+        } else if (item.y === itemStore.maxY) {
+          itemStore.maxElements.push(item)
+        }
+        return itemStore
+      }, {
+        minY: 999,
+        maxY: 0,
+        minElements: [],
+        maxElements: [],
+      })
+
+      const minLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.minElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
+      const maxLineHash = hashCodeIgnoringSpacesAndNumbers(minMaxItems.maxElements.reduce((combinedString, item) => combinedString + item.text().toUpperCase(), ''))
+      pageStore.push({
+        minElements: minMaxItems.minElements,
+        maxElements: minMaxItems.maxElements,
+        minLineHash: minLineHash,
+        maxLineHash: maxLineHash,
+      })
+      minLineHashRepetitions[minLineHash] = minLineHashRepetitions[minLineHash] ? minLineHashRepetitions[minLineHash] + 1 : 1
+      maxLineHashRepetitions[maxLineHash] = maxLineHashRepetitions[maxLineHash] ? maxLineHashRepetitions[maxLineHash] + 1 : 1
+    })
+
+    // now annoate all removed items
+    var removedHeader = 0
+    var removedFooter = 0
+    parseResult.pages.forEach((page, i) => {
+      if (minLineHashRepetitions[pageStore[i].minLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
+        pageStore[i].minElements.forEach(item => {
+          item.annotation = REMOVED_ANNOTATION
+        })
+        removedFooter++
+      }
+      if (maxLineHashRepetitions[pageStore[i].maxLineHash] >= Math.max(3, parseResult.pages.length * 2 / 3)) {
+        pageStore[i].maxElements.forEach(item => {
+          item.annotation = REMOVED_ANNOTATION
+        })
+        removedHeader++
+      }
+    })
+
+    return new ParseResult({
+      ...parseResult,
+      messages: [
+        'Removed Header: ' + removedHeader,
+        'Removed Footers: ' + removedFooter,
+      ],
+    })
+  }
+}

package/src/transforms/line-item/vertical-to-horizontal.ts

@@ -1,90 +1,90 @@
-/**
- * @description Line-item transformation that recovers rotated sidebar text.
- * PDF extracts vertically-oriented labels as a sequence of single-character
- * lines. `VerticalsStream` (a `StashingStream` subclass) detects runs of 6+
- * such lines and merges them into one horizontal `LineItem`, combining their
- * words and computing the correct bounding box.
- */
-import ToLineItemTransformation from "../base/to-line-item-transform";
-import ParseResult from "../../models/parse-result";
-import LineItem from "../../models/line-item";
-import StashingStream from "../../models/stashing-stream";
-import { REMOVED_ANNOTATION, ADDED_ANNOTATION } from "../../models/annotation";
-
-// Converts vertical text to horizontal
-export default class VerticalToHorizontal extends ToLineItemTransformation {
-  constructor() {
-    super("Vertical to Horizontal Text");
-  }
-
-  transform(parseResult: ParseResult): ParseResult {
-    var foundVerticals = 0;
-    parseResult.pages.forEach((page) => {
-      const stream = new VerticalsStream();
-      stream.consumeAll(page.items);
-      page.items = stream.complete();
-      foundVerticals += stream.foundVerticals;
-    });
-
-    return new ParseResult({
-      ...parseResult,
-      messages: ["Converted " + foundVerticals + " verticals"],
-    });
-  }
-}
-
-class VerticalsStream extends StashingStream {
-  foundVerticals: number;
-
-  constructor() {
-    super();
-    this.foundVerticals = 0;
-  }
-
-  shouldStash(item: any): boolean {
-    return item.words.length === 1 && item.words[0].string.length === 1;
-  }
-
-  doMatchesStash(lastItem: any, item: any): boolean {
-    return (
-      lastItem.y - item.y > 5 && lastItem.words[0].type === item.words[0].type
-    );
-  }
-
-  doFlushStash(stash: any[], results: any[]): void {
-    if (stash.length > 5) {
-      // unite
-      var combinedWords = [];
-      var minX = 999;
-      var maxY = 0;
-      var sumWidth = 0;
-      var maxHeight = 0;
-      stash.forEach((oneCharacterLine) => {
-        oneCharacterLine.annotation = REMOVED_ANNOTATION;
-        results.push(oneCharacterLine);
-        combinedWords.push(oneCharacterLine.words[0]);
-        minX = Math.min(minX, oneCharacterLine.x);
-        maxY = Math.max(maxY, oneCharacterLine.y);
-        sumWidth += oneCharacterLine.width;
-        maxHeight = Math.max(maxHeight, oneCharacterLine.height);
-      });
-      results.push(
-        new LineItem({
-          ...stash[0],
-          x: minX,
-          y: maxY,
-          width: sumWidth,
-          height: maxHeight,
-          words: combinedWords,
-          annotation: ADDED_ANNOTATION,
-        }),
-      );
-      this.foundVerticals++;
-    } else {
-      // add as singles
-      results.push(...stash);
-    }
-  }
-}
-
-
+/**
+ * @description Line-item transformation that recovers rotated sidebar text.
+ * PDF extracts vertically-oriented labels as a sequence of single-character
+ * lines. `VerticalsStream` (a `StashingStream` subclass) detects runs of 6+
+ * such lines and merges them into one horizontal `LineItem`, combining their
+ * words and computing the correct bounding box.
+ */
+import ToLineItemTransformation from "../base/to-line-item-transform";
+import ParseResult from "../../models/parse-result";
+import LineItem from "../../models/line-item";
+import StashingStream from "../../models/stashing-stream";
+import { REMOVED_ANNOTATION, ADDED_ANNOTATION } from "../../models/annotation";
+
+// Converts vertical text to horizontal
+export default class VerticalToHorizontal extends ToLineItemTransformation {
+  constructor() {
+    super("Vertical to Horizontal Text");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    var foundVerticals = 0;
+    parseResult.pages.forEach((page) => {
+      const stream = new VerticalsStream();
+      stream.consumeAll(page.items);
+      page.items = stream.complete();
+      foundVerticals += stream.foundVerticals;
+    });
+
+    return new ParseResult({
+      ...parseResult,
+      messages: ["Converted " + foundVerticals + " verticals"],
+    });
+  }
+}
+
+class VerticalsStream extends StashingStream {
+  foundVerticals: number;
+
+  constructor() {
+    super();
+    this.foundVerticals = 0;
+  }
+
+  shouldStash(item: any): boolean {
+    return item.words.length === 1 && item.words[0].string.length === 1;
+  }
+
+  doMatchesStash(lastItem: any, item: any): boolean {
+    return (
+      lastItem.y - item.y > 5 && lastItem.words[0].type === item.words[0].type
+    );
+  }
+
+  doFlushStash(stash: any[], results: any[]): void {
+    if (stash.length > 5) {
+      // unite
+      var combinedWords = [];
+      var minX = 999;
+      var maxY = 0;
+      var sumWidth = 0;
+      var maxHeight = 0;
+      stash.forEach((oneCharacterLine) => {
+        oneCharacterLine.annotation = REMOVED_ANNOTATION;
+        results.push(oneCharacterLine);
+        combinedWords.push(oneCharacterLine.words[0]);
+        minX = Math.min(minX, oneCharacterLine.x);
+        maxY = Math.max(maxY, oneCharacterLine.y);
+        sumWidth += oneCharacterLine.width;
+        maxHeight = Math.max(maxHeight, oneCharacterLine.height);
+      });
+      results.push(
+        new LineItem({
+          ...stash[0],
+          x: minX,
+          y: maxY,
+          width: sumWidth,
+          height: maxHeight,
+          words: combinedWords,
+          annotation: ADDED_ANNOTATION,
+        }),
+      );
+      this.foundVerticals++;
+    } else {
+      // add as singles
+      results.push(...stash);
+    }
+  }
+}
+
+

package/src/transforms/to-html.ts

@@ -1,46 +1,46 @@
-/**
- * @description Final serialization stage that converts each page's text block
- * items into `<p>`-wrapped HTML. TOC blocks are emitted verbatim; hyphenated
- * line-break artifacts are stripped from non-list blocks; `<code>` fencing is
- * removed from CODE-category blocks (treated as prose in this simplified output).
- */
-
-import Transformation from './base/transformation'
-import ParseResult from '../models/parse-result'
-
-export default class ToHTML extends Transformation {
-  constructor () {
-    super('To HTML', 'String')
-  }
-  transform(parseResult: ParseResult): ParseResult {
-    parseResult.pages.forEach(page => {
-      var text = ''
-      page.items.forEach(block => {
-        // Concatenate all words in the same block, unless it's a Table of Contents block
-        let concatText
-        if (block.category === 'TOC') {
-          concatText = block.text
-        } else {
-          concatText = block.text.replace(/(\r\n|\n|\r)/gm, '\n')
-        }
-
-        // Concatenate words that were previously broken up by newline
-        if (block.category !== 'LIST') {
-          concatText = concatText.split('- ').join('')  
-        }
-
-        // Assume there are no code blocks in our documents
-        if (block.category === 'CODE') {
-          concatText = concatText.split('<code>').join('').split('</code>').join('')
-        }
-
-        text += `<p>${concatText}</p>\n\n`
-      })
-
-      page.items = [text]
-    })
-    return new ParseResult({
-      ...parseResult,
-    })
-  }
-}
+/**
+ * @description Final serialization stage that converts each page's text block
+ * items into `<p>`-wrapped HTML. TOC blocks are emitted verbatim; hyphenated
+ * line-break artifacts are stripped from non-list blocks; `<code>` fencing is
+ * removed from CODE-category blocks (treated as prose in this simplified output).
+ */
+
+import Transformation from './base/transformation'
+import ParseResult from '../models/parse-result'
+
+export default class ToHTML extends Transformation {
+  constructor () {
+    super('To HTML', 'String')
+  }
+  transform(parseResult: ParseResult): ParseResult {
+    parseResult.pages.forEach(page => {
+      var text = ''
+      page.items.forEach(block => {
+        // Concatenate all words in the same block, unless it's a Table of Contents block
+        let concatText
+        if (block.category === 'TOC') {
+          concatText = block.text
+        } else {
+          concatText = block.text.replace(/(\r\n|\n|\r)/gm, '\n')
+        }
+
+        // Concatenate words that were previously broken up by newline
+        if (block.category !== 'LIST') {
+          concatText = concatText.split('- ').join('')  
+        }
+
+        // Assume there are no code blocks in our documents
+        if (block.category === 'CODE') {
+          concatText = concatText.split('<code>').join('').split('</code>').join('')
+        }
+
+        text += `<p>${concatText}</p>\n\n`
+      })
+
+      page.items = [text]
+    })
+    return new ParseResult({
+      ...parseResult,
+    })
+  }
+}

package/src/utils/is-url-pdf.ts

@@ -1,33 +1,33 @@
-/**
- * Detects if a given URL points to a PDF file by checking
- * the stream's first bytes for %PDF-  then ends  the request.
- * Useful for hidden pdf url that does not end with pdf
- * @category Extract
- */
-import grab from "grab-url";
-
-export async function isUrlPDF(url: string) {
-  try {
-    // Fetch the URL as an arraybuffer
-    const buffer = await grab(url, {
-      responseType: "arraybuffer",
-      timeout: 10,
-    });
-
-    if (!buffer || buffer.byteLength < 5) return false;
-
-    const chunk = new Uint8Array(buffer);
-
-    // Check if the bytes match the PDF signature
-    return (
-      chunk[0] === 0x25 && // %
-      chunk[1] === 0x50 && // P
-      chunk[2] === 0x44 && // D
-      chunk[3] === 0x46 && // F
-      chunk[4] === 0x2d
-    ); // -
-  } catch (error) {
-    console.error("Error checking URL:", error);
-    return false;
-  }
-}
+/**
+ * Detects if a given URL points to a PDF file by checking
+ * the stream's first bytes for %PDF-  then ends  the request.
+ * Useful for hidden pdf url that does not end with pdf
+ * @category Extract
+ */
+import grab from "grab-url";
+
+export async function isUrlPDF(url: string) {
+  try {
+    // Fetch the URL as an arraybuffer
+    const buffer = await grab(url, {
+      responseType: "arraybuffer",
+      timeout: 10,
+    });
+
+    if (!buffer || buffer.byteLength < 5) return false;
+
+    const chunk = new Uint8Array(buffer);
+
+    // Check if the bytes match the PDF signature
+    return (
+      chunk[0] === 0x25 && // %
+      chunk[1] === 0x50 && // P
+      chunk[2] === 0x44 && // D
+      chunk[3] === 0x46 && // F
+      chunk[4] === 0x2d
+    ); // -
+  } catch (error) {
+    console.error("Error checking URL:", error);
+    return false;
+  }
+}

package/src/utils/page-item-functions.ts

@@ -1,35 +1,35 @@
-/**
- * @description Geometry utility functions shared across the transformation pipeline.
- * `minXFromBlocks` finds the minimum x across all items inside an array of blocks;
- * `minXFromPageItems` finds the minimum x across a flat page item array; `sortByX`
- * sorts items in place by x coordinate for left-to-right reading order.
- */
-import LineItemBlock from '../models/line-item-block'
-
-export function minXFromBlocks(blocks: LineItemBlock[]): number | null {
-  var minX = 999
-  blocks.forEach(block => {
-    block.items.forEach(item => {
-      minX = Math.min(minX, item.x)
-    })
-  })
-  if (minX === 999) {
-    return null
-  }
-  return minX
-}
-
-export function minXFromPageItems(items: Array<{ x: number }>): number | null {
-  var minX = 999
-  items.forEach(item => {
-    minX = Math.min(minX, item.x)
-  })
-  if (minX === 999) {
-    return null
-  }
-  return minX
-}
-
-export function sortByX(items: Array<{ x: number }>): void {
-  items.sort((a, b) => a.x - b.x)
-}
+/**
+ * @description Geometry utility functions shared across the transformation pipeline.
+ * `minXFromBlocks` finds the minimum x across all items inside an array of blocks;
+ * `minXFromPageItems` finds the minimum x across a flat page item array; `sortByX`
+ * sorts items in place by x coordinate for left-to-right reading order.
+ */
+import LineItemBlock from '../models/line-item-block'
+
+export function minXFromBlocks(blocks: LineItemBlock[]): number | null {
+  var minX = 999
+  blocks.forEach(block => {
+    block.items.forEach(item => {
+      minX = Math.min(minX, item.x)
+    })
+  })
+  if (minX === 999) {
+    return null
+  }
+  return minX
+}
+
+export function minXFromPageItems(items: Array<{ x: number }>): number | null {
+  var minX = 999
+  items.forEach(item => {
+    minX = Math.min(minX, item.x)
+  })
+  if (minX === 999) {
+    return null
+  }
+  return minX
+}
+
+export function sortByX(items: Array<{ x: number }>): void {
+  items.sort((a, b) => a.x - b.x)
+}