tabula-extractor 0.7.2-java → 0.7.4-java

data/lib/tabula/extraction.rb

@@ -31,6 +31,8 @@ module Tabula
         :extract_ruling_lines => true
       }
 
+      # TODO: the +pages+ constructor argument does not make sense
+      # now that we have +extract_page+ and +extract_pages+
       def initialize(pdf_filename, pages=[1], password='', options={})
         raise Errno::ENOENT unless File.exists?(pdf_filename)
         @pdf_filename = pdf_filename
@@ -47,39 +49,67 @@ module Tabula
         @transformed_clipping_path = nil
         self.clipping_paths = []
         @rulings = []
-        @min_char_width = @min_char_height = 1000000
+        @min_char_width = @min_char_height = Float::MAX
       end
 
-      def extract
+      def close!
+        self.ensure_open!
+        @pdf_file.close
+        @pdf_file_closed = true
+      end
+
+      def ensure_open!
+        raise "Document is closed" if @pdf_file_closed
+      end
+
+      ##
+      # extract objects from a page. Returns an instance of +Tabula::Page+
+      # (+page_number+ is 1-based. i.e., first page is number 1)
+      def extract_page(page_number)
+        self.ensure_open!
+
+        if page_number-1 >= @all_pages.size || (page_number) < 0
+          raise IndexError, "Page #{page_number} doesn't exist. Skipping. Valid pages are 1..#{@all_pages.size}"
+        end
+
+        page = @all_pages.get(page_number-1)
+        contents = page.getContents
+        return nil if contents.nil?
+
+        self.clear!
+        self.drawPage(page)
+        Tabula::Page.new(@pdf_filename,
+                         page.findCropBox.width,
+                         page.findCropBox.height,
+                         page.getRotation.to_i,
+                         page_number, #one-indexed, just like +page_number+ is.
+                         self.characters,
+                         self.rulings,
+                         @min_char_width,
+                         @min_char_height)
+      end
+
+      def extract(pages=nil)
+        self.ensure_open!
+        pages = if pages == :all
+                  (1..@all_pages.size)
+                elsif pages.nil?
+                  @pages
+                else
+                  pages
+                end
+
         Enumerator.new do |y|
-          begin
-            @pages.each do |i|
-              page = @all_pages.get(i-1)
-              contents = page.getContents
-              next if contents.nil?
-
-              self.clear!
-              self.drawPage(page)
-              p = Tabula::Page.new(@pdf_filename,
-                                   page.findCropBox.width,
-                                   page.findCropBox.height,
-                                   page.getRotation.to_i,
-                                   i, #one-indexed, just like `i` is.
-                                   self.characters,
-                                   self.rulings,
-                                   @min_char_width,
-                                   @min_char_height)
-              y.yield p
-            end
-          ensure
-            @pdf_file.close
-          end # begin
+          pages.each do |i|
+            y.yield self.extract_page(i)
+          end
         end
       end
 
       def clear!
         self.characters.clear
         self.clipping_paths.clear
+        @min_char_width = @min_char_height = Float::MAX
         @page_transform = nil
         @rulings.clear
       end
@@ -118,8 +148,14 @@ module Tabula
 
         path = self.pathToList(self.getLinePath)
 
+        # skip paths whose first operation is not a MOVETO
+        # or contains operations other than LINETO, MOVETO or CLOSE
         if path[0][0] != java.awt.geom.PathIterator::SEG_MOVETO \
-          || path[1..-1].any? { |p| p.first != java.awt.geom.PathIterator::SEG_LINETO && p.first != java.awt.geom.PathIterator::SEG_MOVETO && p.first != java.awt.geom.PathIterator::SEG_CLOSE }
+           || path[1..-1].any? { |p|
+            p.first != java.awt.geom.PathIterator::SEG_LINETO \
+            && p.first != java.awt.geom.PathIterator::SEG_MOVETO \
+            && p.first != java.awt.geom.PathIterator::SEG_CLOSE
+          }
           self.getLinePath.reset
           return
         end
@@ -129,26 +165,57 @@ module Tabula
         strokeColorComps = filter_by_color || self.getGraphicsState.getStrokingColor.getJavaColor.getRGBColorComponents(nil)
         color_filter = self.options[:line_color_filter]
 
+        if !color_filter.nil? && !color_filter.call(strokeColorComps)
+          self.getLinePath.reset
+          return
+        end
+
+        # skip the first path operation save it as the starting position
         first = path.shift
-        start_pos = java.awt.geom.Point2D::Float.new(first[1][0], first[1][1])
+        # last_move
+        start_pos = last_move = java.awt.geom.Point2D::Float.new(first[1][0], first[1][1])
+
+        end_pos = nil
 
         path.each do |p|
-          end_pos = java.awt.geom.Point2D::Float.new(p[1][0], p[1][1])
-          line = (start_pos <=> end_pos) == -1 \
-            ? java.awt.geom.Line2D::Float.new(start_pos, end_pos) \
-            : java.awt.geom.Line2D::Float.new(end_pos, start_pos)
-
-          if p[0] == java.awt.geom.PathIterator::SEG_LINETO \
-            && (color_filter.nil? ? true : color_filter.call(strokeColorComps)) \
-            && line.intersects(ccp_bounds)
-            # convert line to rectangle for clipping it to the current clippath
-            # sucks, but awt doesn't have methods for this
-            tmp = line.getBounds2D.createIntersection(ccp_bounds).getBounds2D
-            @rulings << ::Tabula::Ruling.new(tmp.getY,
-                                             tmp.getX,
-                                             tmp.getWidth,
-                                             tmp.getHeight,
-                                             filter_by_color.to_a)
+          case p[0]
+          when java.awt.geom.PathIterator::SEG_LINETO
+            end_pos = java.awt.geom.Point2D::Float.new(p[1][0], p[1][1])
+            line = (start_pos <=> end_pos) == -1 \
+                   ? java.awt.geom.Line2D::Float.new(start_pos, end_pos) \
+                   : java.awt.geom.Line2D::Float.new(end_pos, start_pos)
+
+            if line.intersects(ccp_bounds)
+              # convert line to rectangle for clipping it to the current clippath
+              # sucks, but awt doesn't have methods for this
+              tmp = line.getBounds2D.createIntersection(ccp_bounds).getBounds2D
+              @rulings << ::Tabula::Ruling.new(tmp.getY,
+                                               tmp.getX,
+                                               tmp.getWidth,
+                                               tmp.getHeight,
+                                               filter_by_color.to_a)
+            end
+          when java.awt.geom.PathIterator::SEG_MOVETO
+            last_move = java.awt.geom.Point2D::Float.new(p[1][0], p[1][1])
+          when java.awt.geom.PathIterator::SEG_CLOSE
+            # according to PathIterator docs:
+            # "the preceding subpath should be closed by appending a line segment
+            # back to the point corresponding to the most recent SEG_MOVETO."
+
+            line = (end_pos <=> last_move) == -1 \
+                   ? java.awt.geom.Line2D::Float.new(end_pos, last_move) \
+                   : java.awt.geom.Line2D::Float.new(last_move, end_pos)
+
+            if line.intersects(ccp_bounds)
+              # convert line to rectangle for clipping it to the current clippath
+              # sucks, but awt doesn't have methods for this
+              tmp = line.getBounds2D.createIntersection(ccp_bounds).getBounds2D
+              @rulings << ::Tabula::Ruling.new(tmp.getY,
+                                               tmp.getX,
+                                               tmp.getWidth,
+                                               tmp.getHeight,
+                                               filter_by_color.to_a)
+            end
           end
           start_pos = end_pos
         end
@@ -201,22 +268,21 @@ module Tabula
         c = text.getCharacter
         h = text.getHeightDir.round(2)
 
-        if c == ' ' || c == ' ' # replace non-breaking space for space
+        if c == ' ' # replace non-breaking space for space
           c = ' '
-          h = text.getWidth.round(2)
         end
 
         te = Tabula::TextElement.new(text.getY.round(2) - h,
                                      text.getX.round(2),
-                                     text.getWidth.round(2),
+                                     text.getWidthDirAdj,
                                      # ugly hack follows: we need spaces to have a height, so we can
                                      # test for vertical overlap. height == width seems a safe bet.
-                                     h,
+                                     text.getHeightDir,
                                      text.getFont,
-                                     text.getFontSize.round(2),
+                                     text.getFontSize,
                                      c,
                                      # workaround a possible bug in PDFBox: https://issues.apache.org/jira/browse/PDFBOX-1755
-                                     text.getWidthOfSpace == 0 ? self.currentSpaceWidth : text.getWidthOfSpace,
+                                     (text.getWidthOfSpace.nan? || text.getWidthOfSpace == 0) ? self.currentSpaceWidth : text.getWidthOfSpace,
                                      text.getDir)
 
         ccp_bounds = self.currentClippingPath
@@ -246,7 +312,6 @@ module Tabula
       end
 
       def rulings
-        return [] if @rulings.empty?
         @rulings.reject { |l| (l.left == l.right && l.top == l.bottom) || [l.top, l.left, l.bottom, l.right].any? { |p| p < 0 } }
       end
 

data/lib/tabula/line_segment_detector.rb

@@ -1,11 +1,6 @@
-require 'java'
 require 'rbconfig'
-
 require 'ffi'
 
-require_relative './entities'
-require_relative './pdf_render'
-require_relative './extraction'
 
 java_import javax.imageio.ImageIO
 java_import java.awt.image.BufferedImage

data/lib/tabula/table_extractor.rb

@@ -42,10 +42,12 @@ module Tabula
       page = [page]
     end
 
-    pdf_page = Extraction::ObjectExtractor.new(pdf_path,
-                                               page,
-                                               options[:password]) \
-      .extract.next
+    extractor = Extraction::ObjectExtractor.new(pdf_path,
+                                                page,
+                                                options[:password])
+
+    pdf_page = extractor.extract.next
+    extractor.close!
 
     if ["spreadsheet", "original"].include? options[:extraction_method]
       use_spreadsheet_extraction_method = options[:extraction_method] == "spreadsheet"
@@ -54,39 +56,32 @@ module Tabula
     end
 
     if use_spreadsheet_extraction_method
-      table = pdf_page.get_area(area).spreadsheets.inject(&:+)
-    else
-      use_detected_lines = false
-      if options[:detect_ruling_lines] && options[:vertical_rulings].empty?
-        detected_vertical_rulings = Ruling.crop_rulings_to_area(pdf_page.vertical_ruling_lines,
-                                                                area)
-
-        # only use lines if at least 80% of them cover at least 90%
-        # of the height of area of interest
-
-        # TODO this heuristic SUCKS
-        # what if only a couple columns is delimited with vertical rulings?
-        # ie: https://www.dropbox.com/s/lpydler5c3pn408/S2MNCEbirdisland.pdf (see 7th column)
-        # idea: detect columns without considering rulings, detect vertical rulings
-        # calculate ratio and try to come up with a threshold
-        use_detected_lines = detected_vertical_rulings.size > 2 \
-        && (detected_vertical_rulings.count { |vl|
-              vl.height / area.height > 0.9
-            } / detected_vertical_rulings.size.to_f) >= 0.8
-
-      end
-
-      table = pdf_page.get_area(area).get_table(:vertical_rulings => use_detected_lines ? detected_vertical_rulings : options[:vertical_rulings])
-
-      # fixes up the table a little bit, replacing nils with empty TextElements
-      # and sorting the lines.
-      table.lines.each do |l|
-        l.text_elements = l.text_elements.map do |te|
-          te || TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
-        end
-      end
-      table.lines.sort_by! { |l| l.text_elements.map { |te| te.top or 0 }.max }
-      table
+      return (spreadsheets = pdf_page.get_area(area).spreadsheets).empty? ? Spreadsheet.empty(pdf_page) : spreadsheets.inject(&:+)
     end
+
+    use_detected_lines = false
+    if options[:detect_ruling_lines] && options[:vertical_rulings].empty?
+      detected_vertical_rulings = Ruling.crop_rulings_to_area(pdf_page.vertical_ruling_lines,
+                                                              area)
+
+      # only use lines if at least 80% of them cover at least 90%
+      # of the height of area of interest
+
+      # TODO this heuristic SUCKS
+      # what if only a couple columns is delimited with vertical rulings?
+      # ie: https://www.dropbox.com/s/lpydler5c3pn408/S2MNCEbirdisland.pdf (see 7th column)
+      # idea: detect columns without considering rulings, detect vertical rulings
+      # calculate ratio and try to come up with a threshold
+      use_detected_lines = detected_vertical_rulings.size > 2 \
+      && (detected_vertical_rulings.count { |vl|
+            vl.height / area.height > 0.9
+          } / detected_vertical_rulings.size.to_f) >= 0.8
+
+    end
+
+    pdf_page
+      .get_area(area)
+      .get_table(:vertical_rulings => use_detected_lines ? detected_vertical_rulings : options[:vertical_rulings])
+
   end
 end

data/lib/tabula/version.rb

@@ -1,3 +1,3 @@
 module Tabula
-  VERSION = '0.7.2'
+  VERSION = '0.7.4'
 end

data/tabula-extractor.gemspec

@@ -15,17 +15,14 @@ Gem::Specification.new do |s|
 
   s.platform = 'java'
 
-  shared_libs = ['liblsd.dylib', 'liblsd-linux64.so', 'liblsd-linux32.so', 'liblsd.dll', 'liblsd64.dll'].map { |f| 'ext/' + f }
-  s.files         = `git ls-files`.split("\n") + shared_libs.map.reject { |f| !File.exists?(f) }
-  s.test_files    = `git ls-files -- {test,features}/*`.split("\n")
+  s.files         = `git ls-files`.split("\n").reject { |f| f =~ /^test\// }
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_development_dependency 'minitest'
   s.add_development_dependency 'bundler', '>= 1.3.4'
   s.add_development_dependency 'ruby-debug'
   s.add_development_dependency 'pry'
+  s.add_development_dependency 'minitest'
 
   s.add_runtime_dependency "trollop", ["~> 2.0"]
-#  s.add_runtime_dependency "algorithms", ["~> 0.6.1"]
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tabula-extractor
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.4
 platform: java
 authors:
 - Manuel Aristarán
@@ -10,38 +10,38 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2014-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: bundler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.4
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.4
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: ruby-debug
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: '0'
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
-  name: ruby-debug
+  name: pry
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -55,7 +55,7 @@ dependencies:
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: minitest
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -99,21 +99,6 @@ files:
 - README.md
 - Rakefile
 - bin/tabula
-- ext/COPYING
-- ext/Makefile.OSX
-- ext/Makefile.defaults
-- ext/Makefile.linux32
-- ext/Makefile.linux64
-- ext/Makefile.mingw
-- ext/Makefile.mingw64
-- ext/liblsd-linux32.so
-- ext/liblsd-linux64.so
-- ext/liblsd.def
-- ext/liblsd.dll
-- ext/liblsd.dylib
-- ext/liblsd64.dll
-- ext/lsd.c
-- ext/lsd.h
 - lib/tabula.rb
 - lib/tabula/core_ext.rb
 - lib/tabula/entities.rb
@@ -125,6 +110,7 @@ files:
 - lib/tabula/entities/ruling.rb
 - lib/tabula/entities/spreadsheet.rb
 - lib/tabula/entities/table.rb
+- lib/tabula/entities/tabular.rb
 - lib/tabula/entities/text_chunk.rb
 - lib/tabula/entities/text_element.rb
 - lib/tabula/entities/text_element_index.rb
@@ -143,40 +129,6 @@ files:
 - target/pdfbox-app-2.0.0-SNAPSHOT.jar
 - target/slf4j-api-1.6.3.jar
 - target/trove4j-3.0.3.jar
-- test/data/47008204D_USA.page4.pdf
-- test/data/560015757GV_China.page1.pdf
-- test/data/ClinicalResearchDisclosureReport2012Q2.pdf
-- test/data/GSK_2012_Q4.page437.pdf
-- test/data/S2MNCEbirdisland.pdf
-- test/data/argentina_diputados_voting_record.pdf
-- test/data/bo_page24.pdf
-- test/data/campaign_donors.pdf
-- test/data/frx_2012_disclosure.pdf
-- test/data/frx_2012_disclosure.tsv
-- test/data/gre.pdf
-- test/data/no_tables.pdf
-- test/data/nyc_2013fiscalreporttables.pdf
-- test/data/puertos1.pdf
-- test/data/spanning_cells.csv
-- test/data/spanning_cells.pdf
-- test/data/strongschools.pdf
-- test/data/sydney_disclosure_contract.pdf
-- test/data/tabla_subsidios.pdf
-- test/data/vertical_rulings_bug.pdf
-- test/data/vietnam3.pdf
-- test/data/wc2012.pdf
-- test/heuristic-test-set/original/560015757GV_China.page1.pdf
-- test/heuristic-test-set/original/S2MNCEbirdisland.pdf
-- test/heuristic-test-set/original/bo_page24.pdf
-- test/heuristic-test-set/original/campaign_donors.pdf
-- test/heuristic-test-set/original/cs076pct.pdf
-- test/heuristic-test-set/spreadsheet/47008204D_USA.page4.pdf
-- test/heuristic-test-set/spreadsheet/GSK_2012_Q4.page437.pdf
-- test/heuristic-test-set/spreadsheet/strongschools.pdf
-- test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf
-- test/heuristic.rb
-- test/test_bin_tabula.sh
-- test/tests.rb
 homepage: https://github.com/jazzido/tabula-extractor
 licenses:
 - MIT
@@ -197,42 +149,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.2.2
 signing_key:
 specification_version: 4
 summary: extract tables from PDF files
-test_files:
-- test/data/47008204D_USA.page4.pdf
-- test/data/560015757GV_China.page1.pdf
-- test/data/ClinicalResearchDisclosureReport2012Q2.pdf
-- test/data/GSK_2012_Q4.page437.pdf
-- test/data/S2MNCEbirdisland.pdf
-- test/data/argentina_diputados_voting_record.pdf
-- test/data/bo_page24.pdf
-- test/data/campaign_donors.pdf
-- test/data/frx_2012_disclosure.pdf
-- test/data/frx_2012_disclosure.tsv
-- test/data/gre.pdf
-- test/data/no_tables.pdf
-- test/data/nyc_2013fiscalreporttables.pdf
-- test/data/puertos1.pdf
-- test/data/spanning_cells.csv
-- test/data/spanning_cells.pdf
-- test/data/strongschools.pdf
-- test/data/sydney_disclosure_contract.pdf
-- test/data/tabla_subsidios.pdf
-- test/data/vertical_rulings_bug.pdf
-- test/data/vietnam3.pdf
-- test/data/wc2012.pdf
-- test/heuristic-test-set/original/560015757GV_China.page1.pdf
-- test/heuristic-test-set/original/S2MNCEbirdisland.pdf
-- test/heuristic-test-set/original/bo_page24.pdf
-- test/heuristic-test-set/original/campaign_donors.pdf
-- test/heuristic-test-set/original/cs076pct.pdf
-- test/heuristic-test-set/spreadsheet/47008204D_USA.page4.pdf
-- test/heuristic-test-set/spreadsheet/GSK_2012_Q4.page437.pdf
-- test/heuristic-test-set/spreadsheet/strongschools.pdf
-- test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf
-- test/heuristic.rb
-- test/test_bin_tabula.sh
-- test/tests.rb
+test_files: []