tabula-extractor 0.6.6-java → 0.7.0-java

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tabula-extractor
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 0.6.6
+  version: 0.7.0
 platform: java
 authors:
 - Manuel Aristarán
@@ -11,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-08-23 00:00:00.000000000 Z
+date: 2014-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -20,13 +19,11 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -36,13 +33,11 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 1.3.4
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: 1.3.4
-    none: false
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -52,13 +47,25 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: pry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -68,13 +75,11 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
-    none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
-    none: false
   prerelease: false
   type: :runtime
 description: extract tables from PDF files
@@ -109,34 +114,65 @@ files:
 - ext/liblsd64.dll
 - ext/lsd.c
 - ext/lsd.h
-- lib/geom/point.rb
-- lib/geom/rectangle.rb
-- lib/geom/segment.rb
 - lib/tabula.rb
 - lib/tabula/core_ext.rb
 - lib/tabula/entities.rb
+- lib/tabula/entities/cell.rb
+- lib/tabula/entities/has_cells.rb
+- lib/tabula/entities/line.rb
+- lib/tabula/entities/page.rb
+- lib/tabula/entities/page_area.rb
+- lib/tabula/entities/ruling.rb
+- lib/tabula/entities/spreadsheet.rb
+- lib/tabula/entities/table.rb
+- lib/tabula/entities/text_chunk.rb
+- lib/tabula/entities/text_element.rb
+- lib/tabula/entities/zone_entity.rb
+- lib/tabula/extraction.rb
 - lib/tabula/line_segment_detector.rb
-- lib/tabula/pdf_dump.rb
+- lib/tabula/pdf_line_extractor.rb
 - lib/tabula/pdf_render.rb
+- lib/tabula/spreadsheet_extractor.rb
 - lib/tabula/table_extractor.rb
 - lib/tabula/table_guesser.rb
 - lib/tabula/version.rb
-- lib/tabula/whitespace.rb
 - lib/tabula/writers.rb
 - tabula-extractor.gemspec
 - target/pdfbox-app-2.0.0-SNAPSHOT.jar
+- test/data/47008204D_USA.page4.pdf
+- test/data/560015757GV_China.page1.pdf
 - test/data/ClinicalResearchDisclosureReport2012Q2.pdf
+- test/data/GSK_2012_Q4.page437.pdf
+- test/data/S2MNCEbirdisland.pdf
 - test/data/argentina_diputados_voting_record.pdf
 - test/data/bo_page24.pdf
+- test/data/campaign_donors.pdf
 - test/data/frx_2012_disclosure.pdf
+- test/data/frx_2012_disclosure.tsv
 - test/data/gre.pdf
+- test/data/no_tables.pdf
+- test/data/puertos1.pdf
+- test/data/spanning_cells.csv
+- test/data/spanning_cells.pdf
+- test/data/strongschools.pdf
 - test/data/tabla_subsidios.pdf
+- test/data/vertical_rulings_bug.pdf
+- test/data/vietnam3.pdf
+- test/heuristic-test-set/original/560015757GV_China.page1.pdf
+- test/heuristic-test-set/original/S2MNCEbirdisland.pdf
+- test/heuristic-test-set/original/bo_page24.pdf
+- test/heuristic-test-set/original/campaign_donors.pdf
+- test/heuristic-test-set/spreadsheet/47008204D_USA.page4.pdf
+- test/heuristic-test-set/spreadsheet/GSK_2012_Q4.page437.pdf
+- test/heuristic-test-set/spreadsheet/strongschools.pdf
+- test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf
+- test/heuristic.rb
+- test/test_bin_tabula.sh
 - test/tests.rb
-- vertical_rulings_bug.pdf
-- vertical_rulings_bug.rb
 homepage: https://github.com/jazzido/tabula-extractor
 licenses:
 - MIT
+metadata: {}
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -145,31 +181,46 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      segments:
-      - 0
-      hash: 2
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
-      segments:
-      - 0
-      hash: 2
       version: '0'
-  none: false
 requirements: []
 rubyforge_project:
-rubygems_version: 1.8.24
+rubygems_version: 2.1.9
 signing_key:
-specification_version: 3
+specification_version: 4
 summary: extract tables from PDF files
 test_files:
+- test/data/47008204D_USA.page4.pdf
+- test/data/560015757GV_China.page1.pdf
 - test/data/ClinicalResearchDisclosureReport2012Q2.pdf
+- test/data/GSK_2012_Q4.page437.pdf
+- test/data/S2MNCEbirdisland.pdf
 - test/data/argentina_diputados_voting_record.pdf
 - test/data/bo_page24.pdf
+- test/data/campaign_donors.pdf
 - test/data/frx_2012_disclosure.pdf
+- test/data/frx_2012_disclosure.tsv
 - test/data/gre.pdf
+- test/data/no_tables.pdf
+- test/data/puertos1.pdf
+- test/data/spanning_cells.csv
+- test/data/spanning_cells.pdf
+- test/data/strongschools.pdf
 - test/data/tabla_subsidios.pdf
+- test/data/vertical_rulings_bug.pdf
+- test/data/vietnam3.pdf
+- test/heuristic-test-set/original/560015757GV_China.page1.pdf
+- test/heuristic-test-set/original/S2MNCEbirdisland.pdf
+- test/heuristic-test-set/original/bo_page24.pdf
+- test/heuristic-test-set/original/campaign_donors.pdf
+- test/heuristic-test-set/spreadsheet/47008204D_USA.page4.pdf
+- test/heuristic-test-set/spreadsheet/GSK_2012_Q4.page437.pdf
+- test/heuristic-test-set/spreadsheet/strongschools.pdf
+- test/heuristic-test-set/spreadsheet/tabla_subsidios.pdf
+- test/heuristic.rb
+- test/test_bin_tabula.sh
 - test/tests.rb

data/lib/geom/point.rb

@@ -1,21 +0,0 @@
-#
-# Cribbed shamelessly from Daniel Vartanov's [ruby-geometry](https://github.com/DanielVartanov/ruby-geometry/)
-# MIT License (c) 2008 Daniel Vartanov, modifications (c) 2013 Jeremy B. Merrill
-#
-
-
-module Geometry
-  class Point < Struct.new(:x, :y)
-    def self.new_by_array(array)      
-      self.new(array[0], array[1])
-    end
-
-    def ==(another_point)
-      x === another_point.x && y === another_point.y
-    end
-  end
-end
-
-def Point(x, y)
-  Geometry::Point.new(x, y)
-end

data/lib/geom/rectangle.rb

@@ -1,101 +0,0 @@
-#
-# Cribbed shamelessly from Daniel Vartanov's [ruby-geometry](https://github.com/DanielVartanov/ruby-geometry/)
-# MIT License (c) 2008 Daniel Vartanov, modifications (c) 2013 Jeremy B. Merrill
-#
-
-
-module Geometry
-  class Rectangle < Struct.new(:point1, :point2)
-    SIMILARITY_DIVISOR = 20
-
-    def Rectangle.unionize(non_overlapping_rectangles, next_rect)
-      #if next_rect doesn't overlap any of non_overlapping_rectangles
-      if (overlapping = non_overlapping_rectangles.select{|r| next_rect.overlaps? r}) && !non_overlapping_rectangles.empty?
-        #remove all of those that it overlaps from non_overlapping_rectangles and 
-        non_overlapping_rectangles -= overlapping
-        #add to non_overlapping_rectangles the bounding box of the overlapping rectangles.
-        non_overlapping_rectangles << overlapping.inject(next_rect){|memo, overlap| memo.bounding_box(overlap) }
-      
-      else
-        non_overlapping_rectangles << next_rect
-      end
-    end
-
-    def self.new_by_x_y_dims(x, y, width, height)
-      self.new( Point.new_by_array([x, y]), Point.new_by_array([x + width, y + height]) )
-    end
-
-    def x
-      [point1.x, point2.x].min
-    end
-
-    alias_method :left, :x
-
-    def y
-      #puts "y: [#{point1.y} #{point2.y}].min" 
-      [point1.y, point2.y].min
-    end
-
-    alias_method :top, :y
-
-    def x2
-      [point1.x, point2.x].max
-    end
-
-    alias_method :right, :x2
-
-    def y2
-      #puts "y2: [#{point1.y} #{point2.y}].max" 
-      [point1.y, point2.y].max
-    end
-
-    alias_method :bottom, :y2
-
-
-    def width
-      (point1.x - point2.x).abs
-    end
-
-    def height
-      (point1.y - point2.y).abs
-    end
-
-    def area
-      self.width * self.height
-    end
-
-    def similarity_hash
-      [self.x.to_i / SIMILARITY_DIVISOR, self.y.to_i / SIMILARITY_DIVISOR, self.width.to_i / SIMILARITY_DIVISOR, self.height.to_i / SIMILARITY_DIVISOR].to_s
-    end
-
-    def dims(*format)
-      if format
-        format.map{|method| self.send(method)}
-      else
-        [self.x, self.y, self.width, self.height]
-      end
-    end
-
-    def contains?(other_x, other_y)
-      (other_x <= x2 && other_x >= x ) && (other_y <= y2 && other_y > y)
-    end
-
-    def overlaps?(other_rect)
-      return contains?(other_rect.x, other_rect.y) || contains?(other_rect.x2, other_rect.y2) || 
-                contains?(other_rect.x, other_rect.y2) || contains?(other_rect.x2, other_rect.y) ||
-                other_rect.contains?(x, y) || other_rect.contains?(x2, y2) || 
-                other_rect.contains?(x, y2) || other_rect.contains?(x2, y)
-    end 
-    
-    def bounding_box(other_rect)
-      #new rect with bounding box of these two
-      new_x1 = [x, other_rect.x].min
-      new_y1 = [x, other_rect.y].min
-      new_x2 = [x2, other_rect.x2].max
-      new_y2 = [y2, other_rect.y2].max
-      new_width = (new_x2 - new_x1).abs
-      new_height = (new_y2 - new_y1).abs
-      Rectangle.new_by_x_y_dims(new_x1, new_y1, new_width, new_height)
-    end
-  end
-end

data/lib/geom/segment.rb

@@ -1,82 +0,0 @@
-#
-# Cribbed shamelessly from Daniel Vartanov's [ruby-geometry](https://github.com/DanielVartanov/ruby-geometry/)
-# MIT License (c) 2008 Daniel Vartanov, modifications (c) 2013 Jeremy B. Merrill
-#
-
-
-module Geometry
-  include Math
-  extend Math
-
-  def Geometry.distance(point1, point2)
-    hypot point1.x - point2.x, point1.y - point2.y
-  end
-
-
-  class Segment < Struct.new(:point1, :point2)
-    def self.new_by_arrays(point1_coordinates, point2_coordinates)
-      self.new(Point.new_by_array(point1_coordinates), 
-               Point.new_by_array(point2_coordinates))
-    end
-
-    def scale!(scale_factor)
-      self.point1.x = self.point1.x * scale_factor
-      self.point1.y = self.point1.y * scale_factor
-      self.point2.x = self.point2.x * scale_factor
-      self.point2.y = self.point2.y * scale_factor
-    end
-
-    def vertical?
-      point1.x == point2.x
-    end
-
-    def horizontal?
-      point1.y == point2.y
-    end
-
-    def leftmost_endpoint
-      ((point1.x <=> point2.x) == -1) ? point1 : point2
-    end
-
-    def rightmost_endpoint
-      ((point1.x <=> point2.x) == 1) ? point1 : point2
-    end
-
-    def topmost_endpoint
-      ((point1.y <=> point2.y) == 1) ? point1 : point2
-    end
-
-    def bottommost_endpoint
-      ((point1.y <=> point2.y) == -1) ? point1 : point2
-    end
-
-    def top
-      topmost_endpoint.y
-    end
-
-    def bottom
-      bottommost_endpoint.y
-    end
-    def width
-      (left - right).abs
-    end
-    def height
-      (bottom - top).abs
-    end
-
-    def left
-      leftmost_endpoint.x
-    end
-
-    def right
-      rightmost_endpoint.x
-    end
-    def length      
-      Geometry.distance(point1, point2)
-    end
-  end
-end
-
-def Segment(point1, point2)
-  Geometry::Segment.new point1, point2
-end

data/lib/tabula/pdf_dump.rb

@@ -1,132 +0,0 @@
-require 'observer'
-
-require_relative './entities.rb'
-
-require 'java'
-require File.join(File.dirname(__FILE__), '../../target/', Tabula::PDFBOX)
-java_import org.apache.pdfbox.pdfparser.PDFParser
-java_import org.apache.pdfbox.pdmodel.PDDocument
-java_import org.apache.pdfbox.util.PDFTextStripper
-java_import org.apache.pdfbox.pdmodel.encryption.StandardDecryptionMaterial
-
-module Tabula
-
-  module Extraction
-
-    def Extraction.openPDF(pdf_filename, password='')
-      raise Errno::ENOENT unless File.exists?(pdf_filename)
-      document = PDDocument.load(pdf_filename)
-      if document.isEncrypted
-        sdm = StandardDecryptionMaterial.new(password)
-        document.openProtection(sdm)
-      end
-      document
-    end
-
-    class TextExtractor < org.apache.pdfbox.util.PDFTextStripper
-
-      attr_accessor :characters, :fonts
-
-      PRINTABLE_RE = /[[:print:]]/
-
-      def initialize
-        super
-        self.fonts = {}
-        self.characters = []
-        self.setSortByPosition(true)
-      end
-
-      def clear!
-        self.characters = []; self.fonts = {}
-      end
-
-      def processTextPosition(text)
-        # return if text.getCharacter == ' '
-
-        # text_font = text.getFont
-        # text_size = text.getFontSize
-        # font_plus_size = self.fonts.select { |k, v| v == text_font }.first.first + "-" + text_size.to_i.to_s
-
-        # $fonts[$current_page].merge!({
-        #   font_plus_size => { :family => text_font.getBaseFont, :size => text_size }
-        # })
-
-        #    $page_contents[$current_page] += "  <text top=\"%.2f\" left=\"%.2f\" width=\"%.2f\" height=\"%.2f\" font=\"#{font_plus_size}\" dir=\"#{text.getDir}\">#{text.getCharacter}</text>\n" % [text.getYDirAdj - text.getHeightDir, text.getXDirAdj, text.getWidthDirAdj, text.getHeightDir]
-
-        c = text.getCharacter
-        # probably not the fastest way of detecting printable chars
-        self.characters << text  if c =~ PRINTABLE_RE
-
-      end
-    end
-
-    class PagesInfoExtractor
-      def initialize(pdf_filename, password='')
-        @pdf_file = Extraction.openPDF(pdf_filename, password)
-        @all_pages = @pdf_file.getDocumentCatalog.getAllPages
-      end
-
-      def pages
-        Enumerator.new do |y|
-          begin
-            @all_pages.each_with_index do |page, i|
-              contents = page.getContents
-#              next if contents.nil?
-              y.yield Tabula::Page.new(page.findCropBox.width,
-                                       page.findCropBox.height,
-                                       page.getRotation.to_i,
-                                       i+1)
-            end
-          ensure
-            @pdf_file.close
-          end
-        end
-      end
-    end
-
-
-    class CharacterExtractor
-      include Observable
-
-      #N.B. pages can be :all, a list of pages or a range.
-      def initialize(pdf_filename, pages=[1], password='')
-        raise Errno::ENOENT unless File.exists?(pdf_filename)
-        @pdf_file = Extraction.openPDF(pdf_filename, password)
-        @all_pages = @pdf_file.getDocumentCatalog.getAllPages
-        @pages = pages == :all ?  (1..@all_pages.size) : pages
-        @extractor = TextExtractor.new
-      end
-
-      def extract
-        Enumerator.new do |y|
-          begin
-            @pages.each do |i|
-              page = @all_pages.get(i-1)
-              contents = page.getContents
-              next if contents.nil?
-              @extractor.clear!
-              @extractor.processStream(page, page.findResources, contents.getStream)
-
-              y.yield Tabula::Page.new(page.findCropBox.width,
-                                       page.findCropBox.height,
-                                       page.getRotation.to_i,
-                                       i+1,
-                                       @extractor.characters.map { |char|
-                                         Tabula::TextElement.new(char.getYDirAdj.round(2),
-                                                                 char.getXDirAdj.round(2),
-                                                                 char.getWidthDirAdj.round(2),
-                                                                 char.getHeightDir.round(2),
-                                                                 char.getFont,
-                                                                 char.getFontSize.round(2),
-                                                                 char.getCharacter,
-                                                                 char.getWidthOfSpace)
-                                       })
-            end
-          ensure
-            @pdf_file.close
-          end # begin
-        end
-      end
-    end
-  end
-end