uploadconvert 0.0.1

data/lib/uploadconvert.rb

@@ -0,0 +1,100 @@
+require 'json'
+require 'docsplit'
+require 'open3'
+
+class UploadConvert
+
+  def initialize(input)
+    @input = input
+    @output = ""
+    @text = ""
+  end
+
+  # Convert PDFs to JSON
+  def pdfTojson
+    # Extract and clean text
+    @text = detectPDFType
+
+    # Extract metadata and generate output                                                                                          
+    extractMetadataPDF
+    outhash = Hash.new
+    @metadata.each{|k, v| outhash[k] = v}
+    outhash[:text] = @text
+    outhash[:input] = @input
+    @output = JSON.pretty_generate(outhash)
+  end
+
+  # Use embedded fonts to detect the type of PDF
+  def detectPDFType
+    out = `pdffonts #{@input}`.split("\n")
+    if out.length > 4
+      return embedPDF
+    else
+      return ocrPDF
+    end
+  end
+
+  # Extract text from embedded text PDFs
+  def embedPDF
+    Docsplit.extract_text(@input, :ocr => false)
+    outfile = @input.split(".")
+    text = File.read(outfile[0]+".txt")
+    
+    # Clean up text and delete file
+    File.delete(outfile[0]+".txt")
+    cleanPDF(text)
+  end
+
+  # OCR PDFs and turn that text into a JSON
+  def ocrPDF
+    # Extract individual pages
+    Docsplit.extract_images(@input)
+    
+    # OCR
+    docs = Dir["*.png"]
+    Docsplit.extract_text(@input, :ocr => true, :output => 'text')
+    outfile = @input.split(".")
+    text = File.read("text/" + outfile[0] + ".txt")
+
+    # Clean up text and files
+    File.delete("text/" + outfile[0]+".txt")
+    Dir.delete("text")
+    docs.each do |d|
+      File.delete(d)
+    end
+    cleanPDF(text)
+  end
+
+  # Removes numbers from edges of legal documents
+  def cleanPDF(text)
+    text.gsub!(/\r?\n/, "\n")
+    text.each_line do |l|
+      lflag = 0
+      (1..28).each do |i|
+        if l == i.to_s+"\n"
+          lflag = 1
+        end
+      end
+
+      if lflag != 1 && l
+        @text += l
+      end
+    end
+    
+    return @text
+  end
+
+  # Extract PDF metadata
+  def extractMetadataPDF
+    @metadata = Hash.new
+    @metadata[:author] = Docsplit.extract_author(@input)
+    @metadata[:creator] =  Docsplit.extract_creator(@input)
+    @metadata[:producer] = Docsplit.extract_producer(@input)
+    @metadata[:title] = Docsplit.extract_title(@input)
+    @metadata[:subject] = Docsplit.extract_subject(@input)
+    @metadata[:date] = Docsplit.extract_date(@input)
+    @metadata[:keywords] = Docsplit.extract_keywords(@input)
+    @metadata[:length] = Docsplit.extract_length(@input)
+  end
+end
+

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: uploadconvert
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-12 00:00:00.000000000 Z
+dependencies: []
+description: Converts documents to the appropriate format for Transparency Toolkit.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/uploadconvert.rb
+homepage: https://github.com/Shidash/UploadConvert
+licenses:
+- GPL
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Collected conversion tools for Transparency Toolkit
+test_files: []
+has_rdoc: