docsplit 0.5.2 → 0.6.0

data/docsplit.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'docsplit'
-  s.version   = '0.5.2'         # Keep version in sync with docsplit.rb
-  s.date      = '2011-05-13'
+  s.version   = '0.6.0'         # Keep version in sync with docsplit.rb
+  s.date      = '2011-09-13'
 
   s.homepage    = "http://documentcloud.github.com/docsplit/"
   s.summary     = "Break Apart Documents into Images, Text, Pages and PDFs"

data/lib/docsplit.rb

@@ -1,7 +1,7 @@
 # The Docsplit module delegates to the Java PDF extractors.
 module Docsplit
 
-  VERSION       = '0.5.2' # Keep in sync with gemspec.
+  VERSION       = '0.6.0' # Keep in sync with gemspec.
 
   ROOT          = File.expand_path(File.dirname(__FILE__) + '/..')
 
@@ -19,7 +19,9 @@ module Docsplit
 
   DEPENDENCIES  = {:java => false, :gm => false, :pdftotext => false, :pdftk => false, :tesseract => false}
 
-  # Check for all dependencies, and warn of their absence.
+  ESCAPE        = lambda {|x| Shellwords.shellescape(x) }
+
+  # Check for all dependencies, and note their absence.
   dirs = ENV['PATH'].split(File::PATH_SEPARATOR)
   DEPENDENCIES.each_key do |dep|
     dirs.each do |dir|
@@ -28,7 +30,6 @@ module Docsplit
         break
       end
     end
-    warn "Warning: Docsplit dependency #{dep} not found." if !DEPENDENCIES[dep]
   end
 
   # Raise an ExtractionFailed exception when the PDF is encrypted, or otherwise
@@ -62,11 +63,13 @@ module Docsplit
     [docs].flatten.each do |doc|
       ext = File.extname(doc)
       basename = File.basename(doc, ext)
-      if GM_FORMATS.include?(ext.sub(/^\./, '').downcase.to_sym)
-        `gm convert "#{doc}" "#{out}/#{basename}.pdf"`
+      escaped_doc, escaped_out, escaped_basename = [doc, out, basename].map(&ESCAPE)
+
+      if ext.length > 0 && GM_FORMATS.include?(ext.sub(/^\./, '').downcase.to_sym)
+        `gm convert #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf`
       else
         options = "-jar #{ROOT}/vendor/jodconverter/jodconverter-core-3.0-beta-3.jar -r #{ROOT}/vendor/conf/document-formats.js"
-        run "#{options} \"#{doc}\" \"#{out}/#{basename}.pdf\"", [], {}
+        run "#{options} #{escaped_doc} #{escaped_out}/#{escaped_basename}.pdf", [], {}
       end
     end
   end
@@ -113,6 +116,7 @@ end
 
 require 'tmpdir'
 require 'fileutils'
+require 'shellwords'
 require "#{Docsplit::ROOT}/lib/docsplit/image_extractor"
 require "#{Docsplit::ROOT}/lib/docsplit/transparent_pdfs"
 require "#{Docsplit::ROOT}/lib/docsplit/text_extractor"

data/lib/docsplit/command_line.rb

@@ -85,6 +85,9 @@ Options:
         opts.on('-f', '--format [FORMAT]', 'set image format (pdf, jpg, gif...)') do |t|
           @options[:format] = t.split(',')
         end
+        opts.on('-d', '--density [NUM]', 'set image density (DPI) when rasterizing') do |d|
+          @options[:density] = d
+        end
         opts.on('--[no-]ocr', 'force OCR to be used, or disable OCR') do |o|
           @options[:ocr] = o
         end

data/lib/docsplit/image_extractor.rb

@@ -4,9 +4,9 @@ module Docsplit
   # nicely sized images.
   class ImageExtractor
 
-    DENSITY_ARG     = "-density 150"
     MEMORY_ARGS     = "-limit memory 256MiB -limit map 512MiB"
     DEFAULT_FORMAT  = :png
+    DEFAULT_DENSITY = '150'
 
     # Extract a list of PDFs as rasterized page images, according to the
     # configuration in options.
@@ -32,16 +32,17 @@ module Docsplit
       basename  = File.basename(pdf, File.extname(pdf))
       directory = directory_for(size)
       pages     = @pages || '1-' + Docsplit.extract_length(pdf).to_s
+      escaped_pdf = ESCAPE[pdf]
       FileUtils.mkdir_p(directory) unless File.exists?(directory)
-      common    = "#{MEMORY_ARGS} #{DENSITY_ARG} #{resize_arg(size)} #{quality_arg(format)}"
+      common    = "#{MEMORY_ARGS} -density #{@density} #{resize_arg(size)} #{quality_arg(format)}"
       if previous
         FileUtils.cp(Dir[directory_for(previous) + '/*'], directory)
         result = `MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm mogrify #{common} -unsharp 0x0.5+0.75 \"#{directory}/*.#{format}\" 2>&1`.chomp
         raise ExtractionFailed, result if $? != 0
       else
         page_list(pages).each do |page|
-          out_file  = File.join(directory, "#{basename}_#{page}.#{format}")
-          cmd = "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert +adjoin #{common} \"#{pdf}[#{page - 1}]\" \"#{out_file}\" 2>&1".chomp
+          out_file  = ESCAPE[File.join(directory, "#{basename}_#{page}.#{format}")]
+          cmd = "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert +adjoin #{common} #{escaped_pdf}[#{page - 1}] #{out_file} 2>&1".chomp
           result = `#{cmd}`.chomp
           raise ExtractionFailed, result if $? != 0
         end
@@ -57,6 +58,7 @@ module Docsplit
     def extract_options(options)
       @output  = options[:output]  || '.'
       @pages   = options[:pages]
+      @density = options[:density] || DEFAULT_DENSITY
       @formats = [options[:format] || DEFAULT_FORMAT].flatten
       @sizes   = [options[:size]].flatten.compact
       @sizes   = [nil] if @sizes.empty?

data/lib/docsplit/info_extractor.rb

@@ -18,7 +18,7 @@ module Docsplit
     # Pull out a single datum from a pdf.
     def extract(key, pdfs, opts)
       pdf = [pdfs].flatten.first
-      cmd = "pdfinfo #{pdf} 2>&1"
+      cmd = "pdfinfo #{ESCAPE[pdf]} 2>&1"
       result = `#{cmd}`.chomp
       raise ExtractionFailed, result if $? != 0
       match = result.match(MATCHERS[key])

data/lib/docsplit/page_extractor.rb

@@ -11,7 +11,7 @@ module Docsplit
         pdf_name = File.basename(pdf, File.extname(pdf))
         page_path = File.join(@output, "#{pdf_name}_%d.pdf")
         FileUtils.mkdir_p @output unless File.exists?(@output)
-        cmd = "pdftk #{pdf} burst output #{page_path} 2>&1"
+        cmd = "pdftk #{ESCAPE[pdf]} burst output #{ESCAPE[page_path]} 2>&1"
         result = `#{cmd}`.chomp
         FileUtils.rm('doc_data.txt') if File.exists?('doc_data.txt')
         raise ExtractionFailed, result if $? != 0

data/lib/docsplit/text_extractor.rb

@@ -16,7 +16,7 @@ module Docsplit
 
     NO_TEXT_DETECTED = /---------\n\Z/
 
-    OCR_FLAGS   = '-density 200x200 -colorspace GRAY'
+    OCR_FLAGS   = '-density 400x400 -colorspace GRAY'
     MEMORY_ARGS = '-limit memory 256MiB -limit map 512MiB'
 
     MIN_TEXT_PER_PAGE = 100 # in bytes
@@ -45,7 +45,7 @@ module Docsplit
 
     # Does a PDF have any text embedded?
     def contains_text?(pdf)
-      fonts = `pdffonts #{pdf} 2>&1`
+      fonts = `pdffonts #{ESCAPE[pdf]} 2>&1`
       !fonts.match(NO_TEXT_DETECTED)
     end
 
@@ -59,19 +59,22 @@ module Docsplit
     def extract_from_ocr(pdf, pages)
       tempdir = Dir.mktmpdir
       base_path = File.join(@output, @pdf_name)
+      escaped_pdf = ESCAPE[pdf]
       if pages
         pages.each do |page|
           tiff = "#{tempdir}/#{@pdf_name}_#{page}.tif"
+          escaped_tiff = ESCAPE[tiff]
           file = "#{base_path}_#{page}"
-          run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert +adjoin #{MEMORY_ARGS} #{OCR_FLAGS} #{pdf}[#{page - 1}] #{tiff} 2>&1"
-          run "tesseract #{tiff} #{file} -l eng 2>&1"
+          run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert -despeckle +adjoin #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf}[#{page - 1}] #{escaped_tiff} 2>&1"
+          run "tesseract #{escaped_tiff} #{ESCAPE[file]} -l eng 2>&1"
           clean_text(file + '.txt') if @clean_ocr
           FileUtils.remove_entry_secure tiff
         end
       else
         tiff = "#{tempdir}/#{@pdf_name}.tif"
-        run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert #{MEMORY_ARGS} #{OCR_FLAGS} #{pdf} #{tiff} 2>&1"
-        run "tesseract #{tiff} #{base_path} -l eng 2>&1"
+        escaped_tiff = ESCAPE[tiff]
+        run "MAGICK_TMPDIR=#{tempdir} OMP_NUM_THREADS=2 gm convert -despeckle #{MEMORY_ARGS} #{OCR_FLAGS} #{escaped_pdf} #{escaped_tiff} 2>&1"
+        run "tesseract #{escaped_tiff} #{base_path} -l eng 2>&1"
         clean_text(base_path + '.txt') if @clean_ocr
       end
     ensure
@@ -100,14 +103,14 @@ module Docsplit
     # Extract the full contents of a pdf as a single file, directly.
     def extract_full(pdf)
       text_path = File.join(@output, "#{@pdf_name}.txt")
-      run "pdftotext -enc UTF-8 #{pdf} #{text_path} 2>&1"
+      run "pdftotext -enc UTF-8 #{ESCAPE[pdf]} #{ESCAPE[text_path]} 2>&1"
     end
 
     # Extract the contents of a single page of text, directly, adding it to
     # the `@pages_to_ocr` list if the text length is inadequate.
     def extract_page(pdf, page)
       text_path = File.join(@output, "#{@pdf_name}_#{page}.txt")
-      run "pdftotext -enc UTF-8 -f #{page} -l #{page} #{pdf} #{text_path} 2>&1"
+      run "pdftotext -enc UTF-8 -f #{page} -l #{page} #{ESCAPE[pdf]} #{ESCAPE[text_path]} 2>&1"
       unless @forbid_ocr
         @pages_to_ocr.push(page) if File.read(text_path).length < MIN_TEXT_PER_PAGE
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: docsplit
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 7
   prerelease: 
   segments: 
   - 0
-  - 5
-  - 2
-  version: 0.5.2
+  - 6
+  - 0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-13 00:00:00 Z
+date: 2011-09-13 00:00:00 Z
 dependencies: []
 
 description: "    Docsplit is a command-line utility and Ruby library for splitting apart\n    documents into their component parts: searchable UTF-8 plain text, page\n    images or thumbnails in any format, PDFs, single pages, and document\n    metadata (title, author, number of pages...)\n"