@dragon708/docmind-markdown 1.2.1 → 1.2.2

package/node_modules/@opendocsg/pdf2md/README.md

@@ -0,0 +1,52 @@
+# pdf2md
+
+JavaScript npm library to parse PDF files and convert them into Markdown
+
+## Major Changes
+
+See [Releases](https://github.com/opendocsg/pdf2md/releases)
+
+## Usage
+
+### Library
+
+```js
+const path = require('path');
+const fs = require('fs')
+const pdf2md = require('@opendocsg/pdf2md')
+
+const pdfBuffer = fs.readFileSync(filePath)
+pdf2md(pdfBuffer, callbacks)
+  .then(text => {
+    let outputFile = allOutputPaths[i] + '.md'
+    console.log(`Writing to ${outputFile}...`)
+    fs.writeFileSync(path.resolve(outputFile), text)
+    console.log('Done.')
+  })
+  .catch(err => {
+    console.error(err)
+  })
+```
+
+### CLI tool
+
+```
+$ cd [project_folder]
+$ npx @opendocsg/pdf2md --inputFolderPath=[your input folder path] --outputFolderPath=[your output folder path] --recursive
+```
+
+If you are converting recursively on a large number of files you might encounter the error "Allocation failed - JavaScript heap out of memory”. Instead, run the command
+
+```
+$ node lib/pdf2md-cli.js --max-old-space-size=4096 --inputFolderPath=[your input folder path] --outputFolderPath=[your output folder path] --recursive
+```
+
+Options:
+1. Input folder path (should exist)
+2. Output folder path (should exist)
+3. Recursive - convert all PDFs for folders within folders. Specify the tag if you require recursive, and omit if you don't
+
+## Credits
+
+[pdf-to-markdown](https://github.com/jzillmann/pdf-to-markdown) - original project by Johannes Zillmann  
+[pdf.js](https://mozilla.github.io/pdf.js/) - Mozilla's PDF parsing & rendering platform which is used as a raw parser

package/node_modules/@opendocsg/pdf2md/lib/models/Annotation.js

@@ -0,0 +1,34 @@
+// Annotation for a text item
+class Annotation {
+  constructor (options) {
+    this.category = options.category
+    this.color = options.color
+  }
+}
+
+exports.default = Annotation
+
+exports.ADDED_ANNOTATION = new Annotation({
+  category: 'Added',
+  color: 'green',
+})
+
+exports.REMOVED_ANNOTATION = new Annotation({
+  category: 'Removed',
+  color: 'red',
+})
+
+exports.UNCHANGED_ANNOTATION = new Annotation({
+  category: 'Unchanged',
+  color: 'brown',
+})
+
+exports.DETECTED_ANNOTATION = new Annotation({
+  category: 'Detected',
+  color: 'green',
+})
+
+exports.MODIFIED_ANNOTATION = new Annotation({
+  category: 'Modified',
+  color: 'green',
+})

package/node_modules/@opendocsg/pdf2md/lib/models/HeadlineFinder.js

@@ -0,0 +1,40 @@
+const { normalizedCharCodeArray } = require('../util/string-functions')
+
+module.exports = class HeadlineFinder {
+  constructor (options) {
+    this.headlineCharCodes = normalizedCharCodeArray(options.headline)
+    this.stackedLineItems = []
+    this.stackedChars = 0
+  }
+
+  consume (lineItem) {
+    // TODO avoid join
+    const normalizedCharCodes = normalizedCharCodeArray(lineItem.text())
+    const matchAll = this.matchAll(normalizedCharCodes)
+    if (matchAll) {
+      this.stackedLineItems.push(lineItem)
+      this.stackedChars += normalizedCharCodes.length
+      if (this.stackedChars === this.headlineCharCodes.length) {
+        return this.stackedLineItems
+      }
+    } else {
+      if (this.stackedChars > 0) {
+        this.stackedChars = 0
+        this.stackedLineItems = []
+        this.consume(lineItem) // test again without stack
+      }
+    }
+    return null
+  }
+
+  matchAll (normalizedCharCodes) {
+    for (var i = 0; i < normalizedCharCodes.length; i++) {
+      const headlineChar = this.headlineCharCodes[this.stackedChars + i]
+      const textItemChar = normalizedCharCodes[i]
+      if (textItemChar !== headlineChar) {
+        return false
+      }
+    }
+    return true
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/LineConverter.js

@@ -0,0 +1,156 @@
+const TextItem = require('./TextItem')
+const Word = require('./Word')
+const WordType = require('./markdown/WordType')
+const WordFormat = require('./markdown/WordFormat')
+const LineItem = require('./LineItem')
+const StashingStream = require('./StashingStream')
+const ParsedElements = require('./ParsedElements')
+const { isNumber, isListItemCharacter } = require('../util/string-functions')
+const { sortByX } = require('../util/page-item-functions')
+
+// Converts text items which have been grouped to a line (through TextItemLineGrouper) to a single LineItem doing inline transformations like
+// 'whitespace removal', bold/emphasis annotation, link-detection, etc..
+module.exports = class LineConverter {
+  constructor (fontToFormats) {
+    this.fontToFormats = fontToFormats
+  }
+
+  // returns a CombineResult
+  compact (textItems /*: TextItem[] */) /*: LineItem */ {
+    // we can't trust order of occurence, esp. footnoteLinks like to come last
+    sortByX(textItems)
+
+    const wordStream = new WordDetectionStream(this.fontToFormats)
+    wordStream.consumeAll(textItems.map(item => new TextItem({ ...item })))
+    const words = wordStream.complete()
+
+    var maxHeight = 0
+    var widthSum = 0
+    textItems.forEach(item => {
+      maxHeight = Math.max(maxHeight, item.height)
+      widthSum += item.width
+    })
+    return new LineItem({
+      x: textItems[0].x,
+      y: textItems[0].y,
+      height: maxHeight,
+      width: widthSum,
+      words: words,
+      parsedElements: new ParsedElements({
+        footnoteLinks: wordStream.footnoteLinks,
+        footnotes: wordStream.footnotes,
+        containLinks: wordStream.containLinks,
+        formattedWords: wordStream.formattedWords,
+      }),
+    })
+  }
+}
+
+class WordDetectionStream extends StashingStream {
+  constructor (fontToFormats) {
+    super()
+    this.fontToFormats = fontToFormats
+    this.footnoteLinks = []
+    this.footnotes = []
+    this.formattedWords = 0
+    this.containLinks = false
+    this.stashedNumber = false
+  }
+
+  shouldStash (item) { // eslint-disable-line no-unused-vars
+    if (!this.firstY) {
+      this.firstY = item.y
+    }
+    this.currentItem = item
+    return true
+  }
+
+  onPushOnStash (item) { // eslint-disable-line no-unused-vars
+    this.stashedNumber = isNumber(item.text.trim())
+  }
+
+  doMatchesStash (lastItem, item) {
+    const lastItemFormat = this.fontToFormats.get(lastItem.font)
+    const itemFormat = this.fontToFormats.get(item.font)
+    if (lastItemFormat !== itemFormat) {
+      return false
+    }
+    const itemIsANumber = isNumber(item.text.trim())
+    return this.stashedNumber === itemIsANumber
+  }
+
+  doFlushStash (stash, results) {
+    if (this.stashedNumber) {
+      const joinedNumber = stash.map(item => item.text)
+        .join('')
+        .trim()
+      if (stash[0].y > this.firstY) { // footnote link
+        results.push(new Word({
+          string: `${joinedNumber}`,
+          type: WordType.FOOTNOTE_LINK,
+        }))
+        this.footnoteLinks.push(parseInt(joinedNumber))
+      } else if (this.currentItem && this.currentItem.y < stash[0].y) { // footnote
+        results.push(new Word({
+          string: `${joinedNumber}`,
+          type: WordType.FOOTNOTE,
+        }))
+        this.footnotes.push(joinedNumber)
+      } else {
+        this.copyStashItemsAsText(stash, results)
+      }
+    } else {
+      this.copyStashItemsAsText(stash, results)
+    }
+  }
+
+  copyStashItemsAsText (stash, results) {
+    const format = this.fontToFormats.get(stash[0].font)
+    results.push(...this.itemsToWords(stash, format))
+  }
+
+  itemsToWords (items, formatName) {
+    const combinedText = combineText(items)
+    const words = combinedText.split(' ')
+    const format = formatName ? WordFormat.enumValueOf(formatName) : null
+    return words.filter(w => w.trim().length > 0).map(word => {
+      var type = null
+      if (word.startsWith('http:')) {
+        this.containLinks = true
+        type = WordType.LINK
+      } else if (word.startsWith('www.')) {
+        this.containLinks = true
+        word = `http://${word}`
+        type = WordType.LINK
+      }
+
+      if (format) {
+        this.formattedWords++
+      }
+      return new Word({ string: word, type, format })
+    })
+  }
+}
+
+function combineText (textItems) {
+  var text = ''
+  var lastItem
+  textItems.forEach(textItem => {
+    var textToAdd = textItem.text
+    if (!text.endsWith(' ') && !textToAdd.startsWith(' ')) {
+      if (lastItem) {
+        const xDistance = textItem.x - lastItem.x - lastItem.width
+        if (xDistance > 5) {
+          text += ' '
+        }
+      } else {
+        if (isListItemCharacter(textItem.text)) {
+          textToAdd += ' '
+        }
+      }
+    }
+    text += textToAdd
+    lastItem = textItem
+  })
+  return text
+}

package/node_modules/@opendocsg/pdf2md/lib/models/LineItem.js

@@ -0,0 +1,29 @@
+const PageItem = require('./PageItem')
+const Word = require('./Word')
+
+// A line within a page
+module.exports = class LineItem extends PageItem {
+  constructor (options) {
+    super(options)
+    this.x = options.x
+    this.y = options.y
+    this.width = options.width
+    this.height = options.height
+    this.words = options.words || []
+    if (options.text && !options.words) {
+      this.words = options.text.split(' ')
+        .filter(string => string.trim().length > 0)
+        .map(wordAsString => new Word({
+          string: wordAsString,
+        }))
+    }
+  }
+
+  text () {
+    return this.wordStrings().join(' ')
+  }
+
+  wordStrings () {
+    return this.words.map(word => word.string)
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/LineItemBlock.js

@@ -0,0 +1,32 @@
+const PageItem = require('./PageItem')
+const LineItem = require('./LineItem')
+
+// A block of LineItem[] within a Page
+module.exports = class LineItemBlock extends PageItem {
+  constructor (options) {
+    super(options)
+    this.items = []
+    if (options.items) {
+      options.items.forEach(item => this.addItem(item))
+    }
+  }
+
+  addItem (item /*: LineItem */) {
+    if (this.type && item.type && this.type !== item.type) {
+      throw new Error(`Adding item of type ${item.type} to block of type ${this.type}`)
+    }
+    if (!this.type) {
+      this.type = item.type
+    }
+    if (item.parsedElements) {
+      if (this.parsedElements) {
+        this.parsedElements.add(item.parsedElements)
+      } else {
+        this.parsedElements = item.parsedElements
+      }
+    }
+    const copiedItem = new LineItem({ ...item })
+    copiedItem.type = null
+    this.items.push(copiedItem)
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/Metadata.js

@@ -0,0 +1,15 @@
+// Metadata of the PDF document
+module.exports = class Metadata {
+  constructor (originalMetadata) {
+    if (originalMetadata.metadata) {
+      this.title = originalMetadata.metadata.get('dc:title')
+      this.creator = originalMetadata.metadata.get('xap:creatortool')
+      this.producer = originalMetadata.metadata.get('pdf:producer')
+    } else {
+      this.title = originalMetadata.info.Title
+      this.author = originalMetadata.info.Author
+      this.creator = originalMetadata.info.Creator
+      this.producer = originalMetadata.info.Producer
+    }
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/Page.js

@@ -0,0 +1,7 @@
+// A page which holds PageItems displayable via PdfPageView
+module.exports = class Page {
+  constructor (options) {
+    this.index = options.index
+    this.items = options.items || [] // PageItem
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/PageItem.js

@@ -0,0 +1,11 @@
+// A abstract PageItem class, can be TextItem, LineItem or LineItemBlock
+module.exports = class PageItem {
+  constructor (options) {
+    if (this.constructor === PageItem) {
+      throw new TypeError('Can not construct abstract class.')
+    }
+    this.type = options.type
+    this.annotation = options.annotation
+    this.parsedElements = options.parsedElements
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/ParseResult.js

@@ -0,0 +1,8 @@
+// The result of a PDF parse respectively a Transformation
+module.exports = class ParseResult {
+  constructor (options) {
+    this.pages = options.pages // like Page[]
+    this.globals = options.globals // properties accasable for all the following transformations in debug mode
+    this.messages = options.messages // something to show only for the transformation in debug mode
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/ParsedElements.js

@@ -0,0 +1,15 @@
+module.exports = class ParsedElements {
+  constructor (options) {
+    this.footnoteLinks = options.footnoteLinks || []
+    this.footnotes = options.footnotes || []
+    this.containLinks = options.containLinks
+    this.formattedWords = options.formattedWords
+  }
+
+  add (parsedElements) {
+    this.footnoteLinks = this.footnoteLinks.concat(parsedElements.footnoteLinks)
+    this.footnotes = this.footnotes.concat(parsedElements.footnotes)
+    this.containLinks = this.containLinks || parsedElements.containLinks
+    this.formattedWords += parsedElements.formattedWords
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/StashingStream.js

@@ -0,0 +1,72 @@
+// Abstract stream which allows stash items temporarily
+module.exports = class StashingStream {
+  constructor () {
+    if (this.constructor === StashingStream) {
+      throw new TypeError('Can not construct abstract class.')
+    }
+    this.results = []
+    this.stash = []
+  }
+
+  consumeAll (items) {
+    items.forEach(item => this.consume(item))
+  }
+
+  consume (item) {
+    if (this.shouldStash(item)) {
+      if (!this.matchesStash(item)) {
+        this.flushStash()
+      }
+      this.pushOnStash(item)
+    } else {
+      if (this.stash.length > 0) {
+        this.flushStash()
+      }
+      this.results.push(item)
+    }
+  }
+
+  pushOnStash (item) {
+    this.onPushOnStash(item)
+    this.stash.push(item)
+  }
+
+  complete () {
+    if (this.stash.length > 0) {
+      this.flushStash()
+    }
+    return this.results
+  }
+
+  // return true if the item matches the items of the stack
+  matchesStash (item) {
+    if (this.stash.length === 0) {
+      return true
+    }
+    const lastItem = this.stash[this.stash.length - 1]
+    return this.doMatchesStash(lastItem, item)
+  }
+
+  flushStash () {
+    if (this.stash.length > 0) {
+      this.doFlushStash(this.stash, this.results)
+      this.stash = []
+    }
+  }
+
+  onPushOnStash (item) { // eslint-disable-line no-unused-vars
+    // sub-classes may override
+  }
+
+  shouldStash (item) {
+    throw new TypeError(' Do not call abstract method foo from child.' + item)
+  }
+
+  doMatchesStash (lastItem, item) {
+    throw new TypeError(' Do not call abstract method foo from child.' + lastItem + item)
+  }
+
+  doFlushStash (stash, results) {
+    throw new TypeError(' Do not call abstract method foo from child.' + stash + results)
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/TextItem.js

@@ -0,0 +1,18 @@
+const PageItem = require('./PageItem')
+
+// A text item, i.e. a line or a word within a page
+module.exports = class TextItem extends PageItem {
+  constructor (options) {
+    super(options)
+    this.x = options.x
+    this.y = options.y
+    this.width = options.width
+    this.height = options.height
+    this.text = options.text
+    this.font = options.font
+
+    this.lineFormat = options.lineFormat
+    this.unopenedFormat = options.unopenedFormat
+    this.unclosedFormat = options.unclosedFormat
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/TextItemLineGrouper.js

@@ -0,0 +1,34 @@
+// @flow
+
+/*::
+import TextItem from './TextItem'
+*/
+
+const { sortByX } = require('../util/page-item-functions')
+
+// Groups all text items which are on the same y line
+module.exports = class TextItemLineGrouper {
+  constructor (options) {
+    this.mostUsedDistance = options.mostUsedDistance || 12
+  }
+
+  // returns a CombineResult
+  group (textItems /*: TextItem[] */) /*: TextItem[][] */ {
+    const lines = []
+    var currentLine = []
+    textItems.forEach(item => {
+      if (currentLine.length > 0 && Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2) {
+        lines.push(currentLine)
+        currentLine = []
+      }
+      currentLine.push(item)
+    })
+    lines.push(currentLine)
+
+    lines.forEach(textItems => {
+      // we can't trust order of occurence, esp. footnoteLinks like to come last
+      sortByX(textItems)
+    })
+    return lines
+  }
+}

package/node_modules/@opendocsg/pdf2md/lib/models/Word.js

@@ -0,0 +1,7 @@
+module.exports = class Word {
+  constructor (options) {
+    this.string = options.string
+    this.type = options.type // WordType
+    this.format = options.format // WordFormat
+  }
+}