tabula-extractor 0.7.2-java → 0.7.4-java

data/lib/tabula/entities/ruling.rb

@@ -37,9 +37,11 @@ module Tabula
       bottom - top
     end
 
-
     # attributes that make sense only for non-oblique lines
     # these are used to have a single collapse method (in page, currently)
+
+    ##
+    # `x` (left) coordinate if line vertical, `y` (top) if horizontal
     def position
       raise NoMethodError, "Oblique line #{self.inspect} has no #position method." if oblique?
       vertical? ? left : top
@@ -153,6 +155,10 @@ module Tabula
         point.y >= top && point.y <= bottom
     end
 
+    def ==(other)
+      return self.getX1 == other.getX1 && self.getY1 == other.getY1 && self.getX2 == other.getX2 && self.getY2 == other.getY2
+    end
+
     ##
     # calculate the intersection point between +self+ and other Ruling
     def intersection_point(other)
@@ -164,40 +170,71 @@ module Tabula
 
       return nil if !self_l.intersectsLine(other_l)
 
-      x1 = self_l.getX1; y1 = self_l.getY1
-      x2 = self_l.getX2; y2 = self_l.getY2
-      x3 = other_l.getX1; y3 = other_l.getY1
-      x4 = other_l.getX2; y4 = other_l.getY2
-
-      det = lambda { |a,b,c,d| a * d - b * c }
+      horizontal, vertical = if self_l.horizontal? && other_l.vertical?
+                               [self_l, other]
+                             elsif self_l.vertical? && other_l.horizontal?
+                               [other_l, self_l]
+                             else
+                               raise ArgumentError, "must be orthogonal, horizontal and vertical"
+                             end
 
-      int_x = det.call(det.call(x1, y1, x2, y2), x1 - x2, det.call(x3, y3, x4, y4), x3 - x4) /
-        det.call(x1 - x2, y1 - y2, x3 - x4, y3 - y4)
 
-      int_y = det.call(det.call(x1, y1, x2, y2), y1 - y2,
-                       det.call(x3, y3, x4, y4), y3 - y4) /
-        det.call(x1 - x2, y1 - y2, x3 - x4, y3 - y4)
-
-      return nil if int_x.nan? || int_y.nan? # TODO is this right?
+      java.awt.geom.Point2D::Float.new(vertical.getX1, horizontal.getY1)
 
+    end
 
-      java.awt.geom.Point2D::Float.new(int_x, int_y)
+    class HSegmentComparator
+      include java.util.Comparator
+      def compare(o1, o2)
+        o1.top <=> o2.top
+      end
     end
 
     ##
-    # Find all intersection points between two list of +Ruling+
-    # (+horizontals+ and +verticals+)
-    # TODO: this is O(n^2) - optimize.
+    # log(n) implementation of find_intersections
+    # based on http://people.csail.mit.edu/indyk/6.838-old/handouts/lec2.pdf
     def self.find_intersections(horizontals, verticals)
-      horizontals.product(verticals).inject({}) do |memo, (h, v)|
-        ip = h.intersection_point(v)
-        unless ip.nil?
-          memo[ip] ||= []
-          # TODO: stupid hack for FLA pdfs where lines appear to intersect, but don't.
-          memo[ip] << [h.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT), v.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)]
-        end
-        memo
-      end
+      tree = java.util.TreeMap.new(HSegmentComparator.new)
+      sort_obj = Struct.new(:type, :pos, :obj)
+
+      (horizontals + verticals)
+        .flat_map { |r|
+          r.vertical? ? sort_obj.new(:v, r.left, r) : [sort_obj.new(:hl, r.left, r),
+                                                       sort_obj.new(:hr, r.right, r)]
+        }
+        .sort { |a,b|
+          if a.pos == b.pos
+            if a.type == :v && b.type == :hl
+              1
+            elsif a.type == :v && b.type == :hr
+              -1
+            elsif a.type == :hl && b.type == :v
+              -1
+            elsif a.type == :hr && b.type == :v
+              1
+            else
+              a.pos <=> b.pos
+            end
+          else
+            a.pos <=> b.pos
+          end
+        }
+        .inject({}) { |memo, e|
+          case e.type
+            when :v
+            tree.each { |h,_|
+              i = h.intersection_point(e.obj)
+              next memo if i.nil?
+              memo[i] = [h.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT),
+                         e.obj.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)]
+            }
+            when :hr
+              tree.remove(e.obj)
+            when :hl
+              tree[e.obj] = 1
+          end
+          memo
+        }
     end
 
     ##
@@ -211,90 +248,31 @@ module Tabula
       end
     end
 
-    # TODO do we really need this one anymore?
-    def self.clean_rulings(rulings, max_distance=4)
-
-      # merge horizontal and vertical lines
-      # TODO this should be iterative
-
-      skip = false
-
-      horiz = rulings.select { |r| r.horizontal? }
-        .group_by(&:top)
-        .values.reduce([]) do |memo, rs|
-
-        rs = rs.sort_by(&:left)
-        if rs.size > 1
-          memo +=
-            rs.each_cons(2)
-            .chunk { |p| p[1].left - p[0].right < 7 }
-            .select { |c| c[0] }
-            .map { |group|
-            group = group.last.flatten.uniq
-            Tabula::Ruling.new(group[0].top,
-                               group[0].left,
-                               group[-1].right - group[0].left,
-                               0)
-          }
-          Tabula::Ruling.new(rs[0].top, rs[0].left, rs[-1].right - rs[0].left, 0)
-        else
-          memo << rs.first
-        end
-        memo
+    def self.collapse_oriented_rulings(lines)
+      # lines must all be of one orientation (i.e. horizontal, vertical)
+
+      if lines.empty?
+        return []
       end
-        .sort_by(&:top)
 
-      h = []
-      horiz.size.times do |i|
+      lines.sort! {|a, b| a.position != b.position ? a.position <=> b.position : a.start <=> b.start }
 
-        if i == horiz.size - 1
-          h << horiz[-1]
-          break
-        end
+      lines = lines.inject([lines.shift]) do |memo, next_line|
+        last = memo.last
 
-        if skip
-          skip = false;
-          next
-        end
-        d = (horiz[i+1].top - horiz[i].top).abs
-
-        h << if d < max_distance # THRESHOLD DISTANCE between horizontal lines
-               skip = true
-               Tabula::Ruling.new(horiz[i].top + d / 2, [horiz[i].left, horiz[i+1].left].min, [horiz[i+1].width.abs, horiz[i].width.abs].max, 0)
-             else
-               horiz[i]
-             end
-      end
-      horiz = h
-
-      vert = rulings.select { |r| r.vertical? }
-        .group_by(&:left)
-        .values
-        .reduce([]) do |memo, rs|
-
-        rs = rs.sort_by(&:top)
-
-        if rs.size > 1
-          # Here be dragons:
-          # merge consecutive segments of lines that are close enough
-          memo +=
-            rs.each_cons(2)
-            .chunk { |p| p[1].top - p[0].bottom < 7 }
-            .select { |c| c[0] }
-            .map { |group|
-            group = group.last.flatten.uniq
-            Tabula::Ruling.new(group[0].top,
-                               group[0].left,
-                               0,
-                               group[-1].bottom - group[0].top)
-          }
+        # if current line colinear with next, and are "close enough": expand current line
+        if next_line.position == last.position && last.nearlyIntersects?(next_line)
+          memo.last.start = next_line.start < last.start ? next_line.start : last.start
+          memo.last.end = next_line.end < last.end ? last.end : next_line.end
+          memo
+        # if next line has no length, ignore it
+        elsif next_line.length == 0
+          memo
+        # otherwise, add it to the returned collection
         else
-          memo << rs.first
+          memo << next_line
         end
-        memo
-      end.sort_by(&:left)
-
-      return horiz += vert
+      end
     end
   end
 end

data/lib/tabula/entities/spreadsheet.rb

@@ -5,7 +5,7 @@ module Tabula
   # the both should implement `cells`, `rows`, `cols`, `extraction_method`
 
   class Spreadsheet < ZoneEntity
-    include Tabula::HasCells
+    include Tabula::Tabular
     attr_accessor :cells, :vertical_ruling_lines, :horizontal_ruling_lines, :cells_resolved
     attr_reader :extraction_method, :page
 
@@ -18,6 +18,10 @@ module Tabula
       @extraction_method = "spreadsheet"
     end
 
+    def self.empty(page)
+      Spreadsheet.new(0, 0, 0, 0, page, [], nil, nil) 
+    end
+
     def ruling_lines
       @vertical_ruling_lines + @horizontal_ruling_lines
     end
@@ -27,15 +31,6 @@ module Tabula
       @horizontal_ruling_lines = lines.select{|hl| hl.horizontal? && spr.intersectsLine(hl) }
     end
 
-    def fill_in_cells!
-      unless @cells_resolved
-        @cells_resolved = true
-        cells.each do |cell|
-          cell.text_elements = @page.get_cell_text(cell)
-        end
-      end
-    end
-
     # call `rows` with `evaluate_cells` as `false` to defer filling in the text in
     # each cell, which can be computationally intensive.
     def rows(evaluate_cells=true)
@@ -77,6 +72,64 @@ module Tabula
       end
     end
 
+    #######################################################
+    # Chapter 2 of Spreadsheet extraction, Spanning Cells #
+    #######################################################
+    #if c is a "spanning cell", that is
+    #              if there are N>0 vertical lines strictly between this cell's left and right
+    #insert N placeholder cells after it with zero size (but same top)
+    def add_spanning_cells!
+      #rounding: because Cell.new_from_points, using in #find_cells above, has
+      # a float precision error where, for instance, a cell whose x2 coord is
+      # supposed to be 160.137451171875 comes out as 160.13745498657227 because
+      # of minus. :(
+      vertical_uniq_locs = vertical_ruling_lines.map{|l| l.left.round(5)}.uniq    #already sorted
+      horizontal_uniq_locs = horizontal_ruling_lines.map{|l| l.top.round(5)}.uniq #already sorted
+
+      cells.each do |c|
+        vertical_rulings_spanned_over = vertical_uniq_locs.select{|l| l > c.left.round(5) && l < c.right.round(5) }
+        horizontal_rulings_spanned_over = horizontal_uniq_locs.select{|t| t > c.top.round(5) && t < c.bottom.round(5) }
+
+        unless vertical_rulings_spanned_over.empty?
+          c.spanning = true
+          vertical_rulings_spanned_over.each do |spanned_over_line_loc|
+            placeholder = Cell.new(c.top, spanned_over_line_loc, 0, c.height)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+        unless horizontal_rulings_spanned_over.empty?
+          c.spanning = true
+          horizontal_rulings_spanned_over.each do |spanned_over_line_loc|
+            placeholder = Cell.new(spanned_over_line_loc, c.left, c.width, 0)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+
+        #if there's a spanning cell that's spans over both rows and columns, then it has "double placeholder" cells
+        # e.g. -------------------
+        #      | C |  C |  C | C |         (this is some pretty sweet ASCII art, eh?)
+        #      |-----------------|
+        #      | C |  C |  C | C |
+        #      |-----------------|
+        #      | C | SC    P | C |   where MC is the "spanning cell" that holds all the text within its bounds
+        #      |----    +    ----|         P is a "placeholder" cell with either zero width or zero height
+        #      | C | P    DP | C |         DP is a "double placeholder" cell with zero width and zero height
+        #      |----    +    ----|         C is an ordinary cell.
+        #      | C | P    DP | C |
+        #      |-----------------|
+
+        unless (double_placeholders = vertical_rulings_spanned_over.product(horizontal_rulings_spanned_over)).empty?
+          double_placeholders.each do |vert_spanned_over, horiz_spanned_over|
+            placeholder = Cell.new(horiz_spanned_over, vert_spanned_over, 0, 0)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+      end
+    end
+
     def to_a
       fill_in_cells!
       rows.map{ |row_cells| row_cells.map(&:text) }
@@ -103,8 +156,18 @@ module Tabula
     end
 
     def +(other)
-      raise ArgumentError unless other.page == @page
+      raise ArgumentError, "Data can only be added if it's from the same PDF page" unless other.page == @page
       Spreadsheet.new(nil, nil, nil, nil, @page, @cells + other.cells, nil, nil )
     end
+
+    protected
+    def fill_in_cells!
+      unless @cells_resolved
+        @cells_resolved = true
+        cells.each do |cell|
+          cell.text_elements = @page.get_cell_text(cell)
+        end
+      end
+    end
   end
 end

data/lib/tabula/entities/table.rb

@@ -1,7 +1,8 @@
 module Tabula
   class Table
+    include Tabula::Tabular
     attr_reader :extraction_method
-    attr_accessor :lines
+
     def initialize(line_count, separators)
       @separators = separators
       @lines = (0...line_count).inject([]) { |m| m << Line.new }
@@ -19,27 +20,24 @@ module Tabula
       end
     end
 
-    def rpad!
-      max = lines.map{|l| l.text_elements.size}.max
-      lines.each do |line|
-        needed = max - line.text_elements.size
-        needed.times do 
-          line.text_elements << TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
-        end
-      end
-    end
 
     def cols
       rows.transpose
     end
 
+    # TODO: this is awful, refactor
     def rows
-      self.rpad!
-      lines.map do |l|
+      rpad!
+      lstrip_lines!
+      li = lines.map do |l|
         l.text_elements.map! do |te|
           te || TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
         end
-      end.sort_by { |l| l.map { |te| te.top || 0 }.max }    
+      end.select do
+        |l| !l.all? { |te| te.text.empty? }
+      end.sort_by do |l|
+        l.map { |te| te.top || 0 }.max
+      end
     end
 
     # create a new Table object from an array of arrays, representing a list of rows in a spreadsheet
@@ -47,41 +45,26 @@ module Tabula
     def self.new_from_array(array_of_rows)
       t = Table.new(array_of_rows.size, [])
       @extraction_method = "testing"
+      tlines = []
       array_of_rows.each_with_index do |row, index|
-        t.lines[index].text_elements = row.each_with_index.map{|cell, inner_index| TextElement.new(index, inner_index, 1, 1, nil, nil, cell, nil)}
+        l = Line.new
+        l.text_elements = row.each_with_index.map{|cell, inner_index| TextElement.new(index, inner_index, 1, 1, nil, nil, cell, nil)}
+        tlines << l
       end
-      t.rpad!
+      t.instance_variable_set(:@lines, tlines)
+      t.send(:rpad!)
       t
     end
 
-    #for equality testing, return @lines stripped of leading columns of empty strings
-    #TODO: write a method to strip all totally-empty columns (or not?)
-    def lstrip_lines
-      return @lines if @lines.include?(nil)
-      min_leading_empty_strings = Float::INFINITY
-      @lines.each do |line|
-        empties = line.text_elements.map{|t| t.nil? || t.text.empty? }
-        min_leading_empty_strings = [min_leading_empty_strings, empties.index(false)].min
-      end
-      if min_leading_empty_strings == 0
-        @lines
-      else
-        @lines.each{|line| line.text_elements = line.text_elements[min_leading_empty_strings..-1]}
-        @lines
-      end
-    end
-    def lstrip_lines!
-      @lines = self.lstrip_lines
-    end
 
     #used for testing, ignores separator locations (they'll sometimes be nil/empty)
     def ==(other)
       self.instance_variable_set(:@lines, self.lstrip_lines)
       other.instance_variable_set(:@lines, other.lstrip_lines)
-      self.instance_variable_set(:@lines, self.lines.rpad(nil, other.lines.size))
-      other.instance_variable_set(:@lines, other.lines.rpad(nil, self.lines.size))
+      self.instance_variable_set(:@lines, self.lines.rpad(Line.new, other.lines.size))
+      other.instance_variable_set(:@lines, other.lines.rpad(Line.new, self.lines.size))
 
-      self.lines.zip(other.lines).all? { |my, yours| my == yours }
+      self.rows.zip(other.rows).all? { |my, yours| my == yours }
 
     end
 
@@ -89,6 +72,7 @@ module Tabula
       {
         'json_class'   => self.class.name,
         'extraction_method' => @extraction_method,
+        'vertical_separators' => @separators,
         'data' => rows,
       }.to_json(*a)
     end
@@ -104,5 +88,39 @@ module Tabula
       Tabula::Writers.TSV(rows, out)
       out.string
     end
+
+    protected
+    def rpad!
+      max = lines.map{|l| l.text_elements.size}.max
+      lines.each do |line|
+        needed = max - line.text_elements.size
+        needed.times do
+          line.text_elements << TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
+        end
+      end
+    end
+
+    #for equality testing, return @lines stripped of leading columns of empty strings
+    #TODO: write a method to strip all totally-empty columns (or not?)
+    def lstrip_lines
+      min_leading_empty_strings = Float::INFINITY
+      @lines.each do |line|
+        empties = line.text_elements.map{|t| t.nil? || t.text.empty? }
+        min_leading_empty_strings = [min_leading_empty_strings,
+                                     empties.index(false) || 0].min
+      end
+      if min_leading_empty_strings == 0
+        @lines
+      else
+        @lines.each{|line| line.text_elements = line.text_elements[min_leading_empty_strings..-1]}
+        @lines
+      end
+    end
+    def lstrip_lines!
+      @lines = self.lstrip_lines
+    end
+
+    attr_accessor :lines
+
   end
 end