parsefile 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06411fba942946a2f8946a545d75018c3638fe3c
-  data.tar.gz: 47108b8867072ecffe7d383a991712b95b47d651
+  metadata.gz: 8e7561f81277ae170560990c0efd6f1fac6df28d
+  data.tar.gz: 0efdf3f4c2c98114be285ac6c7c60d9d728aa0e3
 SHA512:
-  metadata.gz: ed1ab41eef012a454b2b6f694b2829c1360627521e1f9964cf3a3d5588e9cf10a2f406627cc0592017b8bf09ff3967e1b92c7cae2b08ea582f7b4321834efce2
-  data.tar.gz: 469780b671827136b57b43e36870c3584e09f18dcfd332e638f42eec5b7003e8361cdd1ab5cedfec09c122bf91110a294f019713c46987c121a22b397772ac4e
+  metadata.gz: 094423025dd474a3aaf7ce4a7c7ee3f1ba7b39946eaee75f25b4e65fe9ea3117fc8fab7a1fd9afd3680a4a722b25eee9bfe6ed92be61be815a1e3f78f39eb6d2
+  data.tar.gz: 5f41812876112bfaa81df0fcf2d90842591abe821429af585d8d90702fe0b8822c4885ae6852c979226884ecc5a7fc1832ab072a36f58c8deaf03226ae6205d7

data/lib/extractmetadata.rb

@@ -22,7 +22,10 @@ class ExtractMetadata
     outhash[:filetype] = get_file_type
 
     # Extract file metadata, merge. and return
-    outhash.merge!(extract_file_metadata)
+    begin
+      outhash.merge!(extract_file_metadata)
+    rescue
+    end
     return outhash
   end
 

data/lib/ocrfile.rb

@@ -14,8 +14,8 @@ class OCRFile
   # OCR file
   def ocr
     begin
-      if File.exist?(@output_dir+@rel_path)
-        load_extracted_text(@output_dir+@rel_path)
+      if File.exist?(@output_dir+@rel_path+".json")
+        load_extracted_text(@output_dir+@rel_path+".json")
       elsif @path.include?(".pdf")
         ocr_pdf
       else
@@ -24,7 +24,7 @@ class OCRFile
     rescue # Detect errors
       binding.pry
     end
-
+    
     return @text
   end
 
@@ -36,7 +36,7 @@ class OCRFile
 
   # Load text that is already extracted
   def load_extracted_text(file)
-    @text = File.read(file)
+    @text = JSON.parse(File.read(file))["text"]
   end
 
   # Send file to give me text
@@ -55,23 +55,34 @@ class OCRFile
 
   # OCR with tesseract
   def ocr_pdf
+    # Dir_paths
+    base = Dir.pwd+"/"
+    
     # Split pages to handle large PDFs
-    Docsplit.extract_pages(@path, :output => 'pages')
+    Docsplit.extract_pages(@path, :output => base+'pages')
     filename = @path.split("/").last.gsub(".pdf", "")
-    docs = Dir['pages/'+filename+'*']
+    docs = Dir[base+'pages/'+filename+'*']
 
+    # Rename pages so that they can be processed with spaces
+    docs.each do |d|
+      new_name = d.split("/").last.gsub(" ", "_").gsub("(", "").gsub(")", "")
+      File.rename(d, base+'pages/'+new_name)
+    end
+    filename = filename.gsub(" ", "_").gsub("(", "").gsub(")", "")
+    docs_no_spaces = Dir[base+'pages/'+filename+'*']
+    
     # Extract text and save
-    Docsplit.extract_text(docs, :output => 'text')
-    text_files = Dir['text/'+filename+'*']
+    Docsplit.extract_text(docs_no_spaces, :output => base+'text')
+    text_files = Dir[base+'text/'+filename+'*']
     sorted_text = text_files.sort_by {|f| f.split(filename).last.gsub("_", "").gsub(".txt", "").to_i }
     sorted_text.each do |f|
       @text += File.read(f)
     end
 
     # Clean up
-    FileUtils.rm_f Dir.glob("pages/*")
-    Dir.delete("pages")
-    FileUtils.rm_f Dir.glob("text/*")
-    Dir.delete("text")
+    FileUtils.rm_f Dir.glob(base+"pages/*")
+    Dir.delete(base+"pages")
+    FileUtils.rm_f Dir.glob(base+"text/*")
+    Dir.delete(base+"text")
   end
 end

data/lib/parsefile.rb

@@ -2,9 +2,9 @@ require 'json'
 require 'docsplit'
 require 'fileutils'
 require 'pry'
-require 'dircrawl'
-load 'ocrfile.rb'
-load 'extractmetadata.rb'
+#require 'dircrawl'
+load '/home/shidash/Code/ParseFile/lib/ocrfile.rb' #FIX
+load '/home/shidash/Code/ParseFile/lib/extractmetadata.rb' # FIX
 
 class ParseFile
   def initialize(file, input_dir, output_dir)
@@ -15,6 +15,7 @@ class ParseFile
 
   # Parse the file
   def parse_file
+    begin
     # Get metadata
     m = ExtractMetadata.new(@path, @input_dir, @output_dir)
     @metadata = m.extract
@@ -25,6 +26,9 @@ class ParseFile
 
     # Generate output and return
     gen_output
+    rescue #TODO: Fix!
+      binding.pry
+    end
   end
 
   # Generate output
@@ -32,7 +36,16 @@ class ParseFile
     outhash = Hash.new
     outhash[:full_path] = @path
     outhash.merge!(@metadata)
-    outhash[:text] = @text
-    return JSON.pretty_generate(outhash)
+    begin
+      outhash[:text] = @text.to_s.encode('UTF-8', {
+                                           :invalid => :replace,
+                                           :undef   => :replace,
+                                           :replace => '?'
+                                         })
+      return JSON.pretty_generate(outhash)
+    rescue
+      binding.pry
+    end
   end
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parsefile
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - M. C. McGrath
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-05-16 00:00:00.000000000 Z
 dependencies: []
 description: OCR file and extract metadata
 email: shidash@shidash.com