tabula-extractor 0.6.6-java → 0.7.0-java

data/lib/tabula/entities/cell.rb

@@ -0,0 +1,42 @@
+module Tabula
+
+  #cells are components of spreadsheets
+
+  class Cell < ZoneEntity
+
+    NORMAL = 0
+    DEBUG = 1
+    SUPERDEBUG = 2
+
+    attr_accessor :text_elements, :placeholder, :spanning, :options
+
+    def initialize(top, left, width, height, options={})
+      super(top, left, width, height)
+      @placeholder = false
+      @spanning = false
+      @text_elements = []
+      @options = ({:use_line_returns => false, :cell_debug => NORMAL}).merge options
+    end
+
+    def self.new_from_points(topleft, bottomright, options={})
+      width = bottomright.x - topleft.x
+      height = bottomright.y - topleft.y
+      Cell.new(topleft.y, topleft.x, width, height, options)
+    end
+
+    def text
+      return "placeholder" if @placeholder && @options[:cell_debug] >= DEBUG
+      output = ""
+      text_elements.sort #use the default sort for ZoneEntity
+      text_elements.group_by(&:top).values.each do |row|
+        output << row.map{|el| el.text}.join('') + (@options[:use_line_returns] ? "\n" : '')
+      end 
+      if (output.empty? && @options[:cell_debug] >= DEBUG) || @options[:cell_debug] >= SUPERDEBUG
+        text_output = output.dup
+        output = "top: #{top} left: #{left} \n w: #{width} h: #{height}" 
+        output += " \n #{text_output}"
+      end
+      output.strip
+    end
+  end
+end

data/lib/tabula/entities/has_cells.rb

@@ -0,0 +1,244 @@
+require 'set'
+java_import java.awt.Polygon
+java_import java.awt.geom.Area
+
+module Tabula
+  # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors; ruling_lines reader
+  module HasCells
+
+    IS_TABULAR_HEURISTIC_RATIO = 0.8
+    ANOTHER_MAGIC_NUMBER = 0.75
+
+    def is_tabular?
+      #spreadsheet extraction
+      spreadsheet = spreadsheets.first
+      return false if spreadsheet.nil?
+      rows_defined_by_lines = spreadsheet.rows.size #rows filled in automatically
+      columns_defined_by_lines = spreadsheet.cols.size
+
+      table = self.get_table
+      columns_defined_without_lines = table.cols.size
+      rows_defined_without_lines = table.rows.size
+      ratio = ((columns_defined_by_lines.to_f / columns_defined_without_lines) + (rows_defined_by_lines.to_f / rows_defined_without_lines)) / 2
+
+      return ratio > ANOTHER_MAGIC_NUMBER && ratio < (1 / ANOTHER_MAGIC_NUMBER)
+    end
+
+    # finds cells from the ruling lines on the page.
+    # implements Nurminen thesis algorithm cf. https://github.com/jazzido/tabula-extractor/issues/16
+    # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors
+    def find_cells!(options={})
+      # All lines need to been sorted from up to down,
+      # and left to right in ascending order
+
+      cellsFound = []
+
+      intersection_points = Ruling.find_intersections(horizontal_ruling_lines, vertical_ruling_lines)
+
+      # All crossing-points have been sorted from up to down,
+      # and left to right in ascending order
+      # depending on the Point2D default sort here.
+      intersection_points_array = intersection_points.keys.sort
+
+      intersection_points.each_with_index do |(topLeft, ((horizontal, vertical))), i|
+        # Fetch all points on the same vertical and horizontal
+        # line with current crossing point
+
+        # this lets us go to the next intersection_point in intersection_points_array
+        # it is bad and I feel bad.
+        catch :cellCreated do
+
+          # CrossingPointsDirectlyBelow( topLeft );
+          x_points = intersection_points_array[i..-1].select{|pt| pt.x == topLeft.x && pt.y > topLeft.y }
+          # CrossingPointsDirectlyToTheRight( topLeft );
+          y_points = intersection_points_array[i..-1].select{|pt| pt.y == topLeft.y && pt.x > topLeft.x }
+
+
+          x_points.each do |x_point|
+            #                                Skip to next crossing-point
+            # if( NOT EdgeExistsBetween( topLeft, x_point)) next crossing-
+            #                                                    point;
+            next unless vertical.colinear?(x_point)
+            y_points.each do |y_point|
+
+              # if( NOT EdgeExistsBetween( topLeft, y_point)) next crossing-
+              #                                                    point;
+              next unless horizontal.colinear?(y_point)
+              #Hypothetical bottom right point of rectangle
+              btmRight = Point2D::Float.new( y_point.x, x_point.y )
+              if intersection_points.include?(btmRight)
+                intersection_points[btmRight].each do |btmRightHorizontal, btmRightVertical|
+                  if btmRightHorizontal.colinear?( x_point ) &&
+                    btmRightVertical.colinear?( y_point )
+                    # Rectangle is confirmed to have 4 sides
+                    cellsFound << Cell.new_from_points( topLeft, btmRight, options)
+                    # Each crossing point can be the top left corner
+                    # of only a single rectangle
+                    #next crossing-point; we need to "next" out of the outer loop here
+                    # to avoid creating non-minimal cells, I htink.
+                    throw :cellCreated
+                  end
+                end
+              end
+            end
+          end
+        end #cellCreated
+      end
+      self.cells = cellsFound
+      cellsFound
+    end
+
+    #############################
+    # Chapter 2, Spanning Cells #
+    #############################
+    #if c is a "spanning cell", that is
+    #              if there are N>0 vertical lines strictly between this cell's left and right
+    #insert N placeholder cells after it with zero size (but same top)
+
+    # subclasses must define cells, vertical_ruling_lines, horizontal_ruling_lines accessors
+    def add_spanning_cells!
+      #rounding: because Cell.new_from_points, using in #find_cells above, has
+      # a float precision error where, for instance, a cell whose x2 coord is
+      # supposed to be 160.137451171875 comes out as 160.13745498657227 because
+      # of minus. :(
+      vertical_uniq_locs = vertical_ruling_lines.map{|l| l.left.round(5)}.uniq    #already sorted
+      horizontal_uniq_locs = horizontal_ruling_lines.map{|l| l.top.round(5)}.uniq #already sorted
+
+      cells.each do |c|
+        vertical_rulings_spanned_over = vertical_uniq_locs.select{|l| l > c.left.round(5) && l < c.right.round(5) }
+        horizontal_rulings_spanned_over = horizontal_uniq_locs.select{|t| t > c.top.round(5) && t < c.bottom.round(5) }
+
+        unless vertical_rulings_spanned_over.empty?
+          c.spanning = true
+          vertical_rulings_spanned_over.each do |spanned_over_line_loc|
+            placeholder = Cell.new(c.top, spanned_over_line_loc, 0, c.height)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+        unless horizontal_rulings_spanned_over.empty?
+          c.spanning = true
+          horizontal_rulings_spanned_over.each do |spanned_over_line_loc|
+            placeholder = Cell.new(spanned_over_line_loc, c.left, c.width, 0)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+
+        #if there's a spanning cell that's spans over both rows and columns, then it has "double placeholder" cells
+        # e.g. -------------------
+        #      | C |  C |  C | C |         (this is some pretty sweet ASCII art, eh?)
+        #      |-----------------|
+        #      | C |  C |  C | C |
+        #      |-----------------|
+        #      | C | SC    P | C |   where MC is the "spanning cell" that holds all the text within its bounds
+        #      |----    +    ----|         P is a "placeholder" cell with either zero width or zero height
+        #      | C | P    DP | C |         DP is a "double placeholder" cell with zero width and zero height
+        #      |----    +    ----|         C is an ordinary cell.
+        #      | C | P    DP | C |
+        #      |-----------------|
+
+        unless (double_placeholders = vertical_rulings_spanned_over.product(horizontal_rulings_spanned_over)).empty?
+          double_placeholders.each do |vert_spanned_over, horiz_spanned_over|
+            placeholder = Cell.new(horiz_spanned_over, vert_spanned_over, 0, 0)
+            placeholder.placeholder = true
+            cells << placeholder
+          end
+        end
+      end
+    end
+
+    #TODO:
+    #returns array of Spreadsheet objects constructed (or spreadsheet_areas => cells)
+    #maybe placeholders should be added after cells is split into spreadsheets
+    def find_spreadsheets_from_cells
+      cells.sort!
+
+      # via http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
+
+      points = Set.new
+      cells.each do |cell|
+        #TODO: keep track of cells for each point here for more efficiently keeping track of cells inside a polygon
+        cell.points.each do |pt|
+          if points.include?(pt) # Shared vertex, remove it.
+            points.delete(pt)
+          else
+            points << pt
+          end
+        end
+      end
+      points = points.to_a
+
+      #x first sort
+      points_sort_x = points.sort{ |s, other| s.x_first_cmp(other) }
+      points_sort_y = points.sort
+
+      edges_h = {}
+      edges_v = {}
+
+      i = 0
+      while i < points.size do
+        curr_y = points_sort_y[i].y
+        while i < points.size && points_sort_y[i].y == curr_y do
+          edges_h[points_sort_y[i]] = points_sort_y[i + 1]
+          edges_h[points_sort_y[i + 1]] = points_sort_y[i]
+          i += 2
+        end
+      end
+
+      i = 0
+      while i < points.size do
+        curr_x = points_sort_x[i].x
+        while i < points.size && points_sort_x[i].x == curr_x do
+          edges_v[points_sort_x[i]] = points_sort_x[i + 1]
+          edges_v[points_sort_x[i + 1]] = points_sort_x[i]
+          i += 2
+        end
+      end
+
+      # Get all the polygons.
+      polygons = []
+      while !edges_h.empty?
+        # We can start with any point.
+        #TODO: should the polygon be represented just by an ordered array of points?
+        polygon = [[edges_h.shift[0], :horiz]] #popitem removes and returns a random key-value pair
+        loop do
+          curr, e = polygon.last
+          if e == :horiz
+            next_vertex = edges_v.delete(curr)
+            polygon << [next_vertex, :vert]
+          else
+            next_vertex = edges_h.delete(curr) #pop removes and returns the value at key `curr`
+            polygon << [next_vertex, :horiz]
+          end
+          if polygon[-1] == polygon[0]
+            # Closed polygon
+            polygon.pop()
+            break
+          end
+        end
+
+        # Remove implementation-markers (:horiz and :vert) from the polygon.
+        polygon.map!{|point, _| point}
+        polygon.each do |vertex|
+          edges_h.delete(vertex) if edges_h.include?(vertex)
+          edges_v.delete(vertex) if edges_v.include?(vertex)
+        end
+        polygons << polygon
+      end
+
+      # for efficiency's sake, we maybe ought to use java Polygon objects internally
+      # for flexibility, we don't.
+
+      polygons.map do |polygon|
+        xpoints = []
+        ypoints = []
+        polygon.each do |pt|
+          xpoints << pt.x
+          ypoints << pt.y
+        end
+        Area.new(Polygon.new(xpoints.to_java(Java::int), ypoints.to_java(Java::int), xpoints.size)) #lol jruby
+      end
+    end
+  end
+end

data/lib/tabula/entities/line.rb

@@ -0,0 +1,39 @@
+module Tabula
+  class Line < ZoneEntity
+    attr_accessor :text_elements
+    attr_reader :index
+
+    def initialize(index=nil)
+      @text_elements = []
+      @index = index
+    end
+
+    def <<(t)
+      if @text_elements.size == 0
+        @text_elements << t
+        self.top = t.top
+        self.left = t.left
+        self.width = t.width
+        self.height = t.height
+      else
+        if in_same_column = @text_elements.find { |te| te.horizontally_overlaps?(t) }
+          in_same_column.merge!(t)
+        else
+          self.text_elements << t
+          self.merge!(t)
+        end
+      end
+    end
+
+    #used for testing, ignores text element stuff besides stripped text.
+    def ==(other)
+      return false if other.nil?
+      self.text_elements = self.text_elements.rpad(TextElement::EMPTY, other.text_elements.size)
+      other.text_elements = other.text_elements.rpad(TextElement::EMPTY, self.text_elements.size)
+      self.text_elements.zip(other.text_elements).inject(true) do |memo, my_yours|
+        my, yours = my_yours
+        memo && my == yours
+      end
+    end
+  end
+end

data/lib/tabula/entities/page.rb

@@ -0,0 +1,269 @@
+module Tabula
+  class Page < ZoneEntity
+    include Tabula::HasCells
+
+    attr_reader :rotation, :number_one_indexed, :file_path
+    attr_writer :min_char_width, :min_char_height
+    attr_accessor :cells
+
+    def initialize(file_path, width, height, rotation, number, texts=[], ruling_lines=[], min_char_width=nil, min_char_height=nil)
+      super(0, 0, width, height)
+      @rotation = rotation
+      if number < 1
+        raise ArgumentError, "Tabula::Page numbers are one-indexed; numbers < 1 are invalid."
+      end
+      @ruling_lines = ruling_lines
+      @file_path = file_path
+      @number_one_indexed = number
+      self.texts = texts
+      @cells = []
+      @spreadsheets = nil
+      @min_char_width = min_char_width
+      @min_char_height = min_char_height
+    end
+
+    def min_char_width
+      @min_char_width ||= texts.map(&:width).min
+    end
+
+    def min_char_height
+      @min_char_height ||= texts.map(&:height).min
+    end
+
+    def get_area(area)
+      if area.is_a?(Array)
+        top, left, bottom, right = area
+        area = Tabula::ZoneEntity.new(top, left,
+                                      right - left, bottom - top)
+      end
+
+      texts = self.get_text(area)
+      page_area = PageArea.new(file_path,
+                               area.width,
+                               area.height,
+                               rotation,
+                               number,
+                               texts,
+                               Ruling.crop_rulings_to_area(@ruling_lines, area),
+                               texts.map(&:width).min,
+                               texts.map(&:height).min)
+      return page_area
+    end
+
+    #returns a Table object
+    def get_table(options={})
+      options = {:vertical_rulings => []}.merge(options)
+      if texts.empty?
+        return []
+      end
+
+      text_chunks = TextElement.merge_words(self.texts, options).sort
+
+      lines = TextChunk.group_by_lines(text_chunks)
+
+      unless options[:vertical_rulings].empty?
+        columns = options[:vertical_rulings].map(&:left) #pixel locations, not entities
+        separators = columns.sort.reverse
+      else
+        columns = TextChunk.column_positions(text_chunks)
+        separators = columns[1..-1].sort.reverse
+      end
+
+      table = Table.new(lines.count, separators)
+      lines.each_with_index do |line, i|
+        line.text_elements.each do |te|
+          j = separators.find_index { |s| te.left > s } || separators.count
+          table.add_text_element(te, i, separators.count - j)
+        end
+      end
+
+      table.lstrip_lines!
+      table
+    end
+
+    #for API backwards-compatibility reasons, this returns an array of arrays.
+    def make_table(options={})
+      get_table(options).lines.map do |l|
+        l.text_elements.map! do |te|
+          te || TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
+        end
+      end.sort_by { |l| l.map { |te| te.top or 0 }.max }
+    end
+
+    # returns the Spreadsheets; creating them if they're not memoized
+    def spreadsheets(options={})
+      unless @spreadsheets.nil?
+        return @spreadsheets
+      end
+      get_ruling_lines!(options)
+      self.find_cells!(options)
+
+      spreadsheet_areas = find_spreadsheets_from_cells #literally, java.awt.geom.Area objects. lol sorry. polygons.
+
+      #transform each spreadsheet area into a rectangle
+      # and get the cells contained within it.
+      spreadsheet_rectangle_areas = spreadsheet_areas.map{|a| a.getBounds } #getBounds2D is theoretically better, but returns a Rectangle2D.Double, which doesn't have our Ruby sugar on it.
+
+      @spreadsheets = spreadsheet_rectangle_areas.map do |rect|
+        spr = Spreadsheet.new(rect.y, rect.x,
+                        rect.width, rect.height,
+                        self,
+                        #TODO: keep track of the cells, instead of getting them again inefficiently.
+                        [],
+                        vertical_ruling_lines.select{|vl| rect.intersectsLine(vl) },
+                        horizontal_ruling_lines.select{|hl| rect.intersectsLine(hl) }
+                        )
+        spr.cells = @cells.select{|c| spr.overlaps?(c) }
+        spr.add_spanning_cells!
+        spr
+      end
+      if options[:fill_in_cells]
+        fill_in_cells!
+      end
+      spreadsheets
+    end
+
+    def fill_in_cells!(options={})
+      spreadsheets(options).each do |spreadsheet|
+        spreadsheet.cells.each do |cell|
+          cell.text_elements = page.get_cell_text(cell)
+          spreadsheet.cells_resolved = true
+        end
+      end
+    end
+
+    def number(indexing_base=:one_indexed)
+      if indexing_base == :zero_indexed
+        return @number_one_indexed - 1
+      else
+        return @number_one_indexed
+      end
+    end
+
+    # TODO no need for this, let's choose one name
+    def ruling_lines
+      get_ruling_lines!
+    end
+
+    def horizontal_ruling_lines
+      get_ruling_lines!
+      @horizontal_ruling_lines.nil? ? [] : @horizontal_ruling_lines
+    end
+
+    def vertical_ruling_lines
+      get_ruling_lines!
+      @vertical_ruling_lines.nil? ? [] : @vertical_ruling_lines
+    end
+
+    #returns ruling lines, memoizes them in
+    def get_ruling_lines!(options={})
+      if !@ruling_lines.nil? && !@ruling_lines.empty?
+        self.snap_points!
+        @vertical_ruling_lines ||= self.collapse_oriented_rulings(@ruling_lines.select(&:vertical?))
+        @horizontal_ruling_lines ||= self.collapse_oriented_rulings(@ruling_lines.select(&:horizontal?))
+        @vertical_ruling_lines + @horizontal_ruling_lines
+      else
+        []
+      end
+    end
+
+    ##
+    # get text insidea area
+    # area can be an Array ([top, left, width, height])
+    # or a Rectangle2D
+    def get_text(area=nil)
+      if area.instance_of?(Array)
+        top, left, bottom, right = area
+        area = Tabula::ZoneEntity.new(top, left,
+                                      right - left, bottom - top)
+      end
+      if area.nil?
+        texts
+      else
+        texts.select do |t|
+          area.contains(t)
+        end
+      end
+    end
+
+    def get_cell_text(area=nil)
+      TextElement.merge_words(self.get_text(area))
+    end
+
+    def to_json(options={})
+      { :width => self.width,
+        :height => self.height,
+        :number => self.number,
+        :rotation => self.rotation,
+        :texts => self.texts
+      }.to_json(options)
+    end
+
+    def snap_points!
+      lines_to_points = {}
+      points = []
+      @ruling_lines.each do |line|
+        point1 = line.p1 #comptooters are the wurst
+        point2 = line.p2
+        # for a given line, each call to #p1 and #p2 creates a new
+        # Point2D::Float object, rather than returning the same one over and
+        # over again.
+        # so we have to get it, store it in memory as `point1` and `point2`
+        # and then store those in various places (and now, modifying one will
+        # modify the reference and thereby modify the other)
+        lines_to_points[line] = [point1, point2]
+        points += [point1, point2]
+      end
+
+      # lines are stored separately from their constituent points
+      # so you can't modify the points and then modify the lines.
+      # ah, but perhaps I can stick the points in a hash AND in an array
+      # and then modify the lines by means of the points in the hash.
+
+      [[:x, :x=, self.min_char_width], [:y, :y=, self.min_char_height]].each do |getter, setter, cell_size|
+        sorted_points = points.sort_by(&getter)
+        first_point = sorted_points.shift
+        grouped_points = sorted_points.inject([[first_point]] ) do |memo, next_point|
+          last = memo.last
+
+          if (next_point.send(getter) - last.first.send(getter)).abs < cell_size
+            memo[-1] << next_point
+          else
+            memo << [next_point]
+          end
+          memo
+        end
+        grouped_points.each do |group|
+          uniq_locs = group.map(&getter).uniq
+          avg_loc = uniq_locs.sum / uniq_locs.size
+          group.each{|p| p.send(setter, avg_loc) }
+        end
+      end
+
+      lines_to_points.each do |l, p1_p2|
+        l.java_send :setLine, [java.awt.geom.Point2D, java.awt.geom.Point2D], p1_p2[0],
+ p1_p2[1]
+      end
+    end
+
+    def collapse_oriented_rulings(lines)
+      # lines must all be of one orientation (i.e. horizontal, vertical)
+      lines.sort! {|a, b| a.position != b.position ? a.position <=> b.position : a.start <=> b.start }
+
+      lines = lines.inject([lines.shift]) do |memo, next_line|
+        last = memo.last
+        if next_line.position == last.position && last.nearlyIntersects?(next_line)
+          memo.last.start = next_line.start < last.start ? next_line.start : last.start
+          memo.last.end = next_line.end < last.end ? last.end : next_line.end
+          memo
+        elsif next_line.length == 0
+          memo
+        else
+          memo << next_line
+        end
+      end
+      lines
+    end
+  end
+
+end