extract-pdf 0.1.0 → 0.1.1

package/src/transforms/to-text-blocks.ts

@@ -0,0 +1,38 @@
+/**
+ * @description Flattens each page's parsed block items into plain text objects,
+ * replacing structured PDF block nodes with `{ category, text }` pairs. The
+ * `category` is derived from the block's `BlockType` name (e.g. "Paragraph",
+ * "Heading", "List") or falls back to "Unknown" when no type is assigned.
+ * `text` is extracted via `BlockType.blockToText`, which concatenates all
+ * inline text spans within the block. The result is a new `ParseResult` whose
+ * pages contain only these lightweight text items, discarding geometry and
+ * style metadata — making downstream NLP processing and serialization simpler.
+ */
+import Transformation from "./base/transformation";
+import ParseResult from "../models/parse-result";
+import BlockType from "../models/block-type";
+
+export default class ToTextBlocks extends Transformation {
+  constructor() {
+    super("To Text Blocks", "TextBlock");
+  }
+
+  transform(parseResult: ParseResult): ParseResult {
+    parseResult.pages.forEach((page) => {
+      const textItems: Array<{ category: string; text: string }> = [];
+      page.items.forEach((block) => {
+        // TODO category to type (before have no unknowns, have paragraph)
+        const category = block.type ? block.type.name : "Unknown";
+        textItems.push({
+          category: category,
+          text: BlockType.blockToText(block),
+        });
+      });
+      page.items = textItems;
+    });
+    return new ParseResult({
+      ...parseResult,
+    });
+  }
+}
+

package/src/utils/is-url-pdf.ts

@@ -0,0 +1,33 @@
+/**
+ * Detects if a given URL points to a PDF file by checking
+ * the stream's first bytes for %PDF-  then ends  the request.
+ * Useful for hidden pdf url that does not end with pdf
+ * @category Extract
+ */
+import grab from "grab-url";
+
+export async function isUrlPDF(url: string) {
+  try {
+    // Fetch the URL as an arraybuffer
+    const buffer = await grab(url, {
+      responseType: "arraybuffer",
+      timeout: 10,
+    });
+
+    if (!buffer || buffer.byteLength < 5) return false;
+
+    const chunk = new Uint8Array(buffer);
+
+    // Check if the bytes match the PDF signature
+    return (
+      chunk[0] === 0x25 && // %
+      chunk[1] === 0x50 && // P
+      chunk[2] === 0x44 && // D
+      chunk[3] === 0x46 && // F
+      chunk[4] === 0x2d
+    ); // -
+  } catch (error) {
+    console.error("Error checking URL:", error);
+    return false;
+  }
+}

package/src/utils/page-item-functions.ts

@@ -0,0 +1,35 @@
+/**
+ * @description Geometry utility functions shared across the transformation pipeline.
+ * `minXFromBlocks` finds the minimum x across all items inside an array of blocks;
+ * `minXFromPageItems` finds the minimum x across a flat page item array; `sortByX`
+ * sorts items in place by x coordinate for left-to-right reading order.
+ */
+import LineItemBlock from '../models/line-item-block'
+
+export function minXFromBlocks(blocks: LineItemBlock[]): number | null {
+  var minX = 999
+  blocks.forEach(block => {
+    block.items.forEach(item => {
+      minX = Math.min(minX, item.x)
+    })
+  })
+  if (minX === 999) {
+    return null
+  }
+  return minX
+}
+
+export function minXFromPageItems(items: Array<{ x: number }>): number | null {
+  var minX = 999
+  items.forEach(item => {
+    minX = Math.min(minX, item.x)
+  })
+  if (minX === 999) {
+    return null
+  }
+  return minX
+}
+
+export function sortByX(items: Array<{ x: number }>): void {
+  items.sort((a, b) => a.x - b.x)
+}

package/src/utils/page-number-functions.ts

@@ -0,0 +1,109 @@
+/**
+ * @description Detects and strips printed page numbers from PDF text content.
+ * Scans the top and bottom sixths of each page's item list for standalone numeric
+ * strings, builds a `pageIndex → pageNum` map, finds the first page where numbers
+ * begin incrementing consecutively, and provides `removePageNumber` to filter
+ * those items out before further processing.
+ */
+import {
+  removeLeadingWhitespaces,
+  removeTrailingWhitespaces,
+  isNumber,
+} from "./string-functions";
+
+interface TextContentItem {
+  str: string;
+}
+
+interface TextContent {
+  items: TextContentItem[];
+}
+
+type PageIndexNumMap = Record<string, number[]>;
+
+const searchRange = (numerator: number, denominator: number, length: number): number => {
+  return Math.floor((numerator / denominator) * length);
+};
+
+const searchArea = (range: TextContentItem[], pageIndexNumMap: PageIndexNumMap, pageIndex: number): PageIndexNumMap => {
+  for (const { str } of range) {
+    const trimLeadingWhitespaces = removeLeadingWhitespaces(str);
+    const trimWhitespaces = removeTrailingWhitespaces(trimLeadingWhitespaces);
+    if (isNumber(trimWhitespaces)) {
+      if (!pageIndexNumMap[pageIndex]) {
+        pageIndexNumMap[pageIndex] = [];
+      }
+      pageIndexNumMap[pageIndex].push(Number(trimWhitespaces));
+    }
+  }
+  return pageIndexNumMap;
+};
+
+const findPageNumbers = (pageIndexNumMap: PageIndexNumMap, pageIndex: number, items: TextContentItem[]): PageIndexNumMap => {
+  const topArea = searchRange(1, 6, items.length);
+  const bottomArea = searchRange(5, 6, items.length);
+
+  const topAreaResult = searchArea(
+    items.slice(0, topArea),
+    pageIndexNumMap,
+    pageIndex,
+  );
+  return searchArea(items.slice(bottomArea), topAreaResult, pageIndex);
+};
+
+const findFirstPage = (pageIndexNumMap: PageIndexNumMap): { pageIndex: number; pageNum: number } | undefined => {
+  let counter = 0;
+  const keys = Object.keys(pageIndexNumMap);
+  if (keys.length === 0 || keys.length === 1) {
+    return;
+  }
+
+  for (let x = 0; x < keys.length - 1; x++) {
+    const firstPage = pageIndexNumMap[keys[x]];
+    const secondPage = pageIndexNumMap[keys[x + 1]];
+    const prevCounter = counter;
+
+    for (let y = 0; y < firstPage.length && counter < 2; y++) {
+      for (let z = 0; z < secondPage.length && counter < 2; z++) {
+        const pageDifference = Number(keys[x + 1]) - Number(keys[x]);
+        if (firstPage[y] + 1 === secondPage[z]) {
+          counter++;
+        } else if (
+          pageDifference > 1 &&
+          firstPage[y] + pageDifference === secondPage[z]
+        ) {
+          counter++;
+        }
+      }
+    }
+
+    let pageDetails =
+      x > 0
+        ? Object.entries(pageIndexNumMap)[x - 1]
+        : Object.entries(pageIndexNumMap)[x];
+    if (prevCounter === counter) {
+      counter = 0;
+      pageDetails = Object.entries(pageIndexNumMap)[x];
+    } else if (counter >= 2) {
+      return { pageIndex: Number(pageDetails[0]), pageNum: pageDetails[1][0] };
+    }
+  }
+};
+
+const removePageNumber = (textContent: TextContent, pageNum: number): TextContent => {
+  const filteredContent = { items: [...textContent.items] };
+  const topArea = searchRange(1, 6, filteredContent.items.length);
+  const bottomArea = searchRange(5, 6, filteredContent.items.length);
+
+  filteredContent.items = filteredContent.items.filter((item, index) => {
+    const isAtTop = index > 0 && index < topArea;
+    const isAtBottom =
+      index > bottomArea && index < filteredContent.items.length;
+
+    return isAtTop || isAtBottom ? Number(item.str) !== Number(pageNum) : item;
+  });
+  return filteredContent;
+};
+
+export { findPageNumbers, findFirstPage, removePageNumber };
+

package/src/utils/string-functions.ts

@@ -0,0 +1,124 @@
+/**
+ * @description Character-level string utilities for the PDF text pipeline:
+ * digit and number detection, leading/trailing whitespace trimming, list-item
+ * character and pattern matching (bullet chars, numbered items), word-overlap
+ * scoring (`wordMatch`), char-code normalisation for headline matching
+ * (`normalizedCharCodeArray`), and camel-case detection
+ * (`hasUpperCaseCharacterInMiddleOfWord`).
+ */
+const MIN_DIGIT_CHAR_CODE = 48
+const MAX_DIGIT_CHAR_CODE = 57
+const WHITESPACE_CHAR_CODE = 32
+const TAB_CHAR_CODE = 9
+const DOT_CHAR_CODE = 46
+
+export function removeLeadingWhitespaces(string: string): string {
+  while (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) {
+    string = string.substring(1, string.length)
+  }
+  return string
+}
+
+export function removeTrailingWhitespaces(string: string): string {
+  while (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) {
+    string = string.substring(0, string.length - 1)
+  }
+  return string
+}
+
+export function isDigit(charCode: number): boolean {
+  return charCode >= MIN_DIGIT_CHAR_CODE && charCode <= MAX_DIGIT_CHAR_CODE
+}
+
+export function isNumber(string: string): boolean {
+  for (var i = 0; i < string.length; i++) {
+    const charCode = string.charCodeAt(i)
+    if (!isDigit(charCode)) {
+      return false
+    }
+  }
+  return true
+}
+
+export function hasOnly(string: string, char: string): boolean {
+  const charCode = char.charCodeAt(0)
+  for (var i = 0; i < string.length; i++) {
+    const aCharCode = string.charCodeAt(i)
+    if (aCharCode !== charCode) {
+      return false
+    }
+  }
+  return true
+}
+
+export function hasUpperCaseCharacterInMiddleOfWord(text: string): boolean {
+  var beginningOfWord = true
+  for (var i = 0; i < text.length; i++) {
+    const character = text.charAt(i)
+    if (character === ' ') {
+      beginningOfWord = true
+    } else {
+      if (!beginningOfWord && isNaN(character as any * 1) && character === character.toUpperCase() && character.toUpperCase() !== character.toLowerCase()) {
+        return true
+      }
+      beginningOfWord = false
+    }
+  }
+  return false
+}
+
+// Remove whitespace/dots + to uppercase
+export function normalizedCharCodeArray(string: string): number[] {
+  string = string.toUpperCase()
+  return charCodeArray(string).filter(charCode => charCode !== WHITESPACE_CHAR_CODE && charCode !== TAB_CHAR_CODE && charCode !== DOT_CHAR_CODE)
+}
+
+export function charCodeArray(string: string): number[] {
+  const charCodes: number[] = []
+  for (var i = 0; i < string.length; i++) {
+    charCodes.push(string.charCodeAt(i))
+  }
+  return charCodes
+}
+
+export function prefixAfterWhitespace(prefix: string, string: string): string {
+  if (string.charCodeAt(0) === WHITESPACE_CHAR_CODE) {
+    string = removeLeadingWhitespaces(string)
+    return ' ' + prefix + string
+  } else {
+    return prefix + string
+  }
+}
+
+export function suffixBeforeWhitespace(string: string, suffix: string): string {
+  if (string.charCodeAt(string.length - 1) === WHITESPACE_CHAR_CODE) {
+    string = removeTrailingWhitespaces(string)
+    return string + suffix + ' '
+  } else {
+    return string + suffix
+  }
+}
+
+export function isListItemCharacter(string: string): boolean {
+  if (string.length > 1) {
+    return false
+  }
+  const char = string.charAt(0)
+  return char === '-' || char === '•' || char === '–'
+}
+
+export function isListItem(string: string): boolean {
+  return /^[\s]*[-•–][\s].*$/g.test(string)
+}
+
+export function isNumberedListItem(string: string): boolean {
+  return /^[\s]*[\d]*[.][\s].*$/g.test(string)
+}
+
+export function wordMatch(string1: string, string2: string): number {
+  const words1 = new Set(string1.toUpperCase().split(' '))
+  const words2 = new Set(string2.toUpperCase().split(' '))
+  const intersection = new Set(
+    [...words1].filter(x => words2.has(x)))
+  return intersection.size / Math.max(words1.size, words2.size)
+}