tabula-extractor 0.6.3-java → 0.6.4-java

data/Rakefile

@@ -6,7 +6,7 @@ require 'rake'
 Bundler::GemHelper.install_tasks
 
 task :test do
-  ruby %{-J-Xmx512m test/tests.rb}
+  ruby %{--debug -X-C -J-Xmx512m test/tests.rb}
 end
 
 task :default => [:test]

data/bin/tabula

@@ -34,11 +34,13 @@ EOS
 
     opt :pages, 'Comma separated list of ranges. Examples: --pages 1-3,5-7 or --pages 3. Default is --pages 1', :default => '1', :type => String
     opt :area, 'Portion of the page to analyze (top,left,bottom,right). Example: --area 269.875,12.75,790.5,561. Default is entire page', :type => String, :default => nil
+    opt :password, 'Password to decrypt document. Default is empty', :default => ''
     opt :guess, 'Guess the portion of the page to analyze per page. Slow.'
+    opt :debug, 'Print detected table areas instead of processing.'
     opt :format, "Output format (#{FORMATS.join(",")})", :default => 'CSV'
     opt :outfile, 'Write output to <file> instead of STDOUT', :default => '-'
   end
-  
+
   if !opts[:area].nil?
     unless opts[:area].split(',').size == 4 \
       && opts[:area].split(',').all? { |x| x.strip =~ /(\d+\.?\d*)/ }
@@ -59,25 +61,29 @@ def main
 
   area = opts[:area].nil? ? nil : opts[:area].split(',').map(&:to_f)
   out = opts[:outfile] == '-' ? $stdout : File.new(opts[:outfile], 'w')
-  extractor = Tabula::Extraction::CharacterExtractor.new(filename, parse_pages_arg(opts[:pages]))
+  extractor = Tabula::Extraction::CharacterExtractor.new(filename, parse_pages_arg(opts[:pages]), opts[:password])
   extractor.extract.each_with_index do |page, page_index|
-    if opts[:guess] 
+    if opts[:guess]
       lines = Tabula::Ruling::clean_rulings(Tabula::LSD::detect_lines_in_pdf_page(filename, page_index))
-      page_areas = Tabula::TableGuesser::find_rects_from_lines(lines)      
+      page_areas = Tabula::TableGuesser::find_rects_from_lines(lines)
       page_areas.map!{|rect| rect.dims(:top, :left, :bottom, :right)}
     else
       page_areas = [area]
     end
 
     page_areas.each do |page_area|
-      text = page.get_text( page_area )
-      Tabula::Writers.send(opts[:format].to_sym,
-                           Tabula.make_table(text),
-                           out)
+      if opts[:guess] && opts[:debug]
+        puts (page_index + 1).to_s + ', ' + page_area.to_s
+      else
+        text = page.get_text( page_area )
+        Tabula::Writers.send(opts[:format].to_sym,
+                            Tabula.make_table(text),
+                            out)
+      end
+
     end
   end
   out.close
 end
 
 main
-

data/ext/Makefile.OSX

@@ -1,9 +1,12 @@
 include Makefile.defaults
 
+
+CFLAGS := -arch i386 -arch x86_64 -fPIC -O3 -g -Wall -Werror
+
 lib: lib$(NAME).$(VERSION).dylib
 
 lib$(NAME).$(VERSION).dylib: $(NAME).o
-	$(CC) -dynamiclib -lm -o lib$(NAME).dylib $^
+	$(CC) -arch i386 -arch x86_64 -dynamiclib -lm -o lib$(NAME).dylib $^
 
 clean:
 	$(RM) *.o

data/lib/tabula.rb

@@ -10,3 +10,4 @@ require_relative './tabula/writers'
 require_relative './tabula/table_guesser'
 require_relative './tabula/line_segment_detector'
 require_relative './tabula/pdf_render'
+#require_relative './tabula/whitespace'

data/lib/tabula/core_ext.rb

@@ -10,7 +10,7 @@ module Enumerable
 
   def sample_variance
     m = self.mean
-    sum = self.inject(0){|accum, i| accum +(i-m)**2 }
+    sum = self.inject(0) {|accum, i| accum + (i-m)**2 }
     sum/(self.length - 1).to_f
   end
 
@@ -18,4 +18,8 @@ module Enumerable
     return Math.sqrt(self.sample_variance)
   end
 
-end 
+  def sorted?
+    each_cons(2).all? { |a, b| (a <=> b) <= 0 }
+  end
+
+end

data/lib/tabula/entities.rb

@@ -70,6 +70,22 @@ module Tabula
       intersection_area / union_area
     end
 
+    # as defined by PDF-TREX paper
+    def horizontal_overlap_ratio(other)
+      delta = [self.bottom - self.top, other.bottom - other.top].min
+      if [other.top, self.top, other.bottom, self.bottom].sorted?
+        (other.bottom - self.top) / delta
+      elsif [self.top, other.top, self.bottom, other.bottom].sorted?
+        (self.bottom - other.top) / delta
+      elsif [self.top, other.top, other.bottom, self.bottom].sorted?
+        (other.bottom - other.top) / delta
+      elsif [other.top, self.top, self.bottom, other.bottom].sorted?
+        (self.bottom - self.top) / delta
+      else
+        0
+      end
+    end
+
     def to_h
       hash = {}
       [:top, :left, :width, :height].each do |m|
@@ -99,8 +115,8 @@ module Tabula
 
       # spaces are not detected, b/c they have height == 0
       # ze = ZoneEntity.new(area[0], area[1], area[3] - area[1], area[2] - area[0])
-      # self.texts.select { |t| t.overlaps? ze } 
-      self.texts.select do |t| 
+      # self.texts.select { |t| t.overlaps? ze }
+      self.texts.select do |t|
         t.top > area[0] && t.top + t.height < area[2] && t.left > area[1] && t.left + t.width < area[3]
       end
     end
@@ -179,12 +195,32 @@ module Tabula
     end
   end
 
+  class Table
+    attr_reader :lines
+    def initialize(line_count, separators)
+      @separators = separators
+      @lines = (0...line_count).inject([]) { |m| m << Line.new }
+    end
+
+    def add_text_element(text_element, i, j)
+      if @lines.size <= i
+        @lines[i] = Line.new
+      end
+      if @lines[i].text_elements[j]
+        @lines[i].text_elements[j].merge!(text_element)
+      else
+        @lines[i].text_elements[j] = text_element
+      end
+    end
+  end
 
   class Line < ZoneEntity
     attr_accessor :text_elements
+    attr_reader :index
 
-    def initialize
+    def initialize(index=nil)
       self.text_elements = []
+      @index = index
     end
 
     def <<(t)

data/lib/tabula/line_segment_detector.rb

@@ -5,6 +5,7 @@ require 'ffi'
 
 require_relative './entities'
 require_relative './pdf_render'
+require_relative './pdf_dump'
 require File.join(File.dirname(__FILE__), '../../target/', Tabula::PDFBOX)
 
 java_import javax.imageio.ImageIO
@@ -45,7 +46,7 @@ module Tabula
     def LSD.detect_lines_in_pdf_page(pdf_path, page_number, options={})
       options = DETECT_LINES_DEFAULTS.merge(options)
 
-      pdf_file = PDDocument.loadNonSeq(java.io.File.new(pdf_path), nil)
+      pdf_file = Extraction.openPDF(pdf_path)
       page = pdf_file.getDocumentCatalog.getAllPages[page_number]
       bi = Tabula::Render.pageToBufferedImage(page,
                                               options[:image_size])
@@ -62,9 +63,14 @@ module Tabula
                  image
                elsif image.class == String
                  ImageIO.read(java.io.File.new(image))
-                 else
+               else
                  raise ArgumentError, 'image must be a string or a BufferedImage'
                end
+
+      ImageIO.write(bimage,
+                    'png',
+                    java.io.File.new("/tmp/white.png"))
+
       image = LSD.image_to_image_double(bimage)
 
       lines_found_ptr = FFI::MemoryPointer.new(:int, 1)

data/lib/tabula/pdf_dump.rb

@@ -7,9 +7,22 @@ require File.join(File.dirname(__FILE__), '../../target/', Tabula::PDFBOX)
 java_import org.apache.pdfbox.pdfparser.PDFParser
 java_import org.apache.pdfbox.pdmodel.PDDocument
 java_import org.apache.pdfbox.util.PDFTextStripper
+java_import org.apache.pdfbox.pdmodel.encryption.StandardDecryptionMaterial
 
 module Tabula
+
   module Extraction
+
+    def Extraction.openPDF(pdf_filename, password='')
+      raise Errno::ENOENT unless File.exists?(pdf_filename)
+      document = PDDocument.load(pdf_filename)
+      if document.isEncrypted
+        sdm = StandardDecryptionMaterial.new(password)
+        document.openProtection(sdm)
+      end
+      document
+    end
+
     class TextExtractor < org.apache.pdfbox.util.PDFTextStripper
 
       attr_accessor :characters, :fonts
@@ -28,8 +41,9 @@ module Tabula
       end
 
 
+
       def processTextPosition(text)
-        #    return if text.getCharacter == ' '
+        # return if text.getCharacter == ' '
 
         # text_font = text.getFont
         # text_size = text.getFontSize
@@ -49,9 +63,8 @@ module Tabula
     end
 
     class PagesInfoExtractor
-      def initialize(pdf_filename)
-        raise Errno::ENOENT unless File.exists?(pdf_filename)
-        @pdf_file = PDDocument.load(java.io.File.new(pdf_filename))
+      def initialize(pdf_filename, password='')
+        @pdf_file = Extraction.openPDF(pdf_filename, password)
         @all_pages = @pdf_file.getDocumentCatalog.getAllPages
       end
 
@@ -60,7 +73,7 @@ module Tabula
           begin
             @all_pages.each_with_index do |page, i|
               contents = page.getContents
-              next if contents.nil?
+#              next if contents.nil?
               y.yield Tabula::Page.new(page.findCropBox.width,
                                        page.findCropBox.height,
                                        page.getRotation.to_i,
@@ -78,9 +91,9 @@ module Tabula
       include Observable
 
       #N.B. pages can be :all, a list of pages or a range.
-      def initialize(pdf_filename, pages=[1])
+      def initialize(pdf_filename, pages=[1], password='')
         raise Errno::ENOENT unless File.exists?(pdf_filename)
-        @pdf_file = PDDocument.loadNonSeq(java.io.File.new(pdf_filename), nil)
+        @pdf_file = Extraction.openPDF(pdf_filename, password)
         @all_pages = @pdf_file.getDocumentCatalog.getAllPages
         @pages = pages == :all ?  (1..@all_pages.size) : pages
         @extractor = TextExtractor.new
@@ -105,7 +118,7 @@ module Tabula
                                                                  char.getXDirAdj.round(2),
                                                                  char.getWidthDirAdj.round(2),
                                                                  char.getHeightDir.round(2),
-                                                                 nil,
+                                                                 char.getFont,
                                                                  char.getFontSize.round(2),
                                                                  char.getCharacter,
                                                                  char.getWidthOfSpace)

data/lib/tabula/table_extractor.rb

@@ -115,12 +115,9 @@ module Tabula
 
         char2 = self.text_elements[i+1]
 
-
-
         next if char2.nil? or char1.nil?
 
         if self.text_elements[current_word_index].should_merge?(char2)
-          #puts "merging: #{self.text_elements[current_word_index].text}/#{self.text_elements[current_word_index].width}"
           self.text_elements[current_word_index].merge!(char2)
           char1 = char2
           self.text_elements[i+1] = nil
@@ -166,92 +163,60 @@ module Tabula
 
   ONLY_SPACES_RE = Regexp.new('^\s+$')
 
-  # Returns an array of Tabula::Line
-  def Tabula.make_table(text_elements, options={})
-    extractor = TableExtractor.new(text_elements, options)
-
-    # group by lines
+  def Tabula.group_by_lines(text_elements)
     lines = []
-    line_boundaries = extractor.get_line_boundaries
-
-    # find all the text elements
-    # contained within each detected line (table row) boundary
-    line_boundaries.each do |lb|
-      line = Line.new
-
-      line_members = text_elements.find_all do |te|
-        te.vertically_overlaps?(lb)
-      end
-
-      text_elements -= line_members
-
-      line_members.sort_by(&:left).each do |te|
-        # skip text_elements that only contain spaces
-        next if te.text =~ ONLY_SPACES_RE
-        line << te
+    text_elements.each do |te|
+      next if te.text =~ ONLY_SPACES_RE
+      l = lines.find { |line| line.horizontal_overlap_ratio(te) >= 0.01 }
+      if l.nil?
+        l = Line.new
+        lines << l
       end
-
-      lines << line if line.text_elements.size > 0
+      l << te
     end
+    lines
+  end
 
-    lines.sort_by!(&:top)
-
-    columns = TableExtractor.new(lines.map(&:text_elements).flatten.compact.uniq, {:merge_words => options[:merge_words]}).group_by_columns.sort_by(&:left)
-
-    # # insert empty cells if needed
-    lines.each_with_index do |l, line_index|
-      next if l.text_elements.nil?
-      l.text_elements.compact! # TODO WHY do I have to do this?
-      l.text_elements.uniq!  # TODO WHY do I have to do this?
-      l.text_elements.sort_by!(&:left)
-
-      #next unless l.text_elements.size < columns.size
-
-      columns.each_with_index do |c, i|
-        if (i > l.text_elements.size - 1) or (!l.text_elements[i].nil? and !c.text_elements.include?(l.text_elements[i]))
-          l.text_elements.insert(i, TextElement.new(l.top, c.left, c.width, l.height, nil, 0, '', 0))
+  # Returns an array of Tabula::Line
+  def Tabula.make_table(text_elements, options={})
+    default_options = {:separators => []}
+    options = default_options.merge(options)
+
+    extractor = TableExtractor.new(text_elements, options).text_elements
+    lines = group_by_lines(text_elements)
+    top = lines[0].text_elements.map(&:top).min
+    right = 0
+    columns = []
+
+    text_elements.sort_by(&:left).each do |te|
+      next if te.text =~ ONLY_SPACES_RE
+      if te.top >= top
+        left = te.left
+        if (left > right)
+          columns << right
+          right = te.right
+        elsif te.right > right
+          right = te.right
         end
       end
     end
 
-    # # merge elements that are in the same column
-    lines.each_with_index do |l, line_index|
-      next if l.text_elements.nil?
-
-      (0..l.text_elements.size-1).to_a.combination(2).each do |t1, t2|
-        next if l.text_elements[t1].nil? or l.text_elements[t2].nil? or l.text_elements[t1].text.empty? or l.text_elements[t2].text.empty?
+    separators = columns[1..-1].sort.reverse
 
-        # if same column...
-        if columns.detect { |c| c.text_elements.include? l.text_elements[t1] } \
-          == columns.detect { |c| c.text_elements.include? l.text_elements[t2] }
-          if l.text_elements[t1].bottom <= l.text_elements[t2].bottom
-            l.text_elements[t1].merge!(l.text_elements[t2])
-            l.text_elements[t2] = nil
-          else
-            l.text_elements[t2].merge!(l.text_elements[t1])
-            l.text_elements[t1] = nil
-          end
-        end
+    table = Table.new(lines.count, separators)
+    lines.each_with_index do |line, i|
+      line.text_elements.each do |te|
+        j = separators.find_index { |s| te.left > s } || separators.count
+        table.add_text_element(te, i, separators.count - j)
       end
-
-      l.text_elements.compact!
     end
 
-    # remove duplicate lines
-    # TODO this shouldn't have happened here, check why we have to do
-    # this (maybe duplication is happening in the column merging phase?)
-    (0..lines.size - 2).each do |i|
-      next if lines[i].nil?
-      # if any of the elements on the next line is duplicated, kill
-      # the next line
-      if (0..lines[i].text_elements.size-1).any? { |j| lines[i].text_elements[j] == lines[i+1].text_elements[j] }
-        lines[i+1] = nil
-      end
+    table.lines.map do |l|
+      l.text_elements.map! { |te|
+        te.nil? ? TextElement.new(nil, nil, nil, nil, nil, nil, '', nil) : te
+      }
     end
 
-    lines.compact.map do |line|
-      line.text_elements.sort_by(&:left)
-    end
   end
 
 
@@ -340,9 +305,4 @@ module Tabula
       line.text_elements.sort_by(&:left)
     end
   end
-
-
-
-
-
 end

data/lib/tabula/version.rb

@@ -1,3 +1,3 @@
 module Tabula
-  VERSION = '0.6.3'
+  VERSION = '0.6.4'
 end

data/lib/tabula/writers.rb

@@ -15,7 +15,7 @@ module Tabula
     end
 
     def Writers.TSV(lines, output=$stdout)
-      tsv_string = lines.each { |l|
+      lines.each { |l|
         output.write(l.map(&:text).join("\t") + "\n")
       }
     end

data/tabula-extractor.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'bundler', '>= 1.3.4'
+  s.add_development_dependency 'ruby-debug'
 
   s.add_runtime_dependency "trollop", ["~> 2.0"]
+#  s.add_runtime_dependency "algorithms", ["~> 0.6.1"]
 end

data/test/tests.rb

@@ -114,7 +114,6 @@ class TestExtractor < Minitest::Test
     lines = Tabula::TableGuesser.find_lines_on_page(pdf_file_path, 0)
     vertical_rulings = lines.select(&:vertical?).uniq{|line| (line.left / 10).round }
 
-
     characters = character_extractor.extract.next.get_text([110, 28, 218, 833])
                                                            #top left bottom right
     expected = [['AANONSEN, DEBORAH, A', '', 'STATEN ISLAND, NY', 'MEALS', '$85.00'],

metadata

@@ -2,7 +2,7 @@
 name: tabula-extractor
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.6.3
+  version: 0.6.4
 platform: java
 authors:
 - Manuel Aristarán
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-06-29 00:00:00.000000000 Z
+date: 2013-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -45,6 +45,22 @@ dependencies:
     none: false
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: ruby-debug
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
   name: trollop
   version_requirements: !ruby/object:Gem::Requirement
@@ -126,12 +142,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 2
       version: '0'
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 2
       version: '0'
   none: false
 requirements: []