tabula-extractor 0.6.6-java → 0.7.0-java

data/lib/tabula/entities/page_area.rb

@@ -0,0 +1,7 @@
+module Tabula
+  class PageArea < Page
+
+
+  end
+
+end

data/lib/tabula/entities/ruling.rb

@@ -0,0 +1,300 @@
+module Tabula
+  class Ruling < java.awt.geom.Line2D::Float
+
+    attr_accessor :stroking_color
+
+    def initialize(top, left, width, height, stroking_color=nil)
+      super(left, top, left+width, top+height)
+      self.stroking_color = stroking_color
+    end
+
+    alias :top :getY1
+    alias :left :getX1
+    alias :bottom :getY2
+    alias :right :getX2
+
+    def top=(v)
+      self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, v, right, bottom
+    end
+
+    def left=(v)
+      self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], v, top, right, bottom
+    end
+
+    def bottom=(v)
+      self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, top, right, v
+    end
+
+    def right=(v)
+      self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], left, top, v, bottom
+    end
+
+    def width
+      right - left
+    end
+
+    def height
+      bottom - top
+    end
+
+
+    # attributes that make sense only for non-oblique lines
+    # these are used to have a single collapse method (in page, currently)
+    def position
+      raise NoMethodError, "Oblique line #{self.inspect} has no #position method." if oblique?
+      vertical? ? left : top
+    end
+    def start
+      raise NoMethodError, "Oblique line #{self.inspect} has no #start method." if oblique?
+      vertical? ? top : left
+    end
+    def end
+      raise NoMethodError, "Oblique line #{self.inspect} has no #end method." if oblique?
+      vertical? ? bottom : right
+    end
+    def position=(coord)
+      raise NoMethodError, "Oblique line #{self.inspect} has no #position= method." if oblique?
+      if vertical?
+        self.left = coord
+        self.right = coord
+      else
+        self.top = coord
+        self.bottom = coord
+      end
+    end
+    def start=(coord)
+      raise NoMethodError, "Oblique line #{self.inspect} has no #start= method." if oblique?
+      if vertical?
+        self.top = coord
+      else
+        self.left = coord
+      end
+    end
+    def end=(coord)
+      raise NoMethodError, "Oblique line #{self.inspect} has no #end= method." if oblique?
+      if vertical?
+        self.bottom = coord
+      else
+        self.right = coord
+      end
+    end
+
+    #ok wtf are you doing, Jeremy?
+    # some PDFs (garment factory audits, precise link TK) make tables by drawing lines that
+    # very nearly intersect each other, but not quite. E.g. a horizontal line spans the table at a Y val of 100
+    # and each vertical line (i.e. column separating ruling line) starts at 101 or 102.
+    # this is very annoying. so we check if those lines nearly overlap by expanding each pair
+    # by 2 pixels in each direction (so the vertical lines' top becomes 99 or 100, and then the expanded versions overlap)
+
+    PERPENDICULAR_PIXEL_EXPAND_AMOUNT = 2
+    COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT = 1
+
+    # if the lines we're comparing are colinear or parallel, we expand them by a only 1 pixel,
+    # because the expansions are additive
+    # (e.g. two vertical lines, at x = 100, with one having y2 of 98 and the other having y1 of 102 would
+    # erroneously be said to nearlyIntersect if they were each expanded by 2 (since they'd both terminate at 100).
+    # The COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT is only 1 so the total expansion is 2.
+    # A total expansion amount of 2 is empirically verified to work sometimes. It's not a magic number from any
+    # source other than a little bit of experience.)
+
+    def nearlyIntersects?(another)
+      if self.intersectsLine(another)
+        true
+      elsif self.perpendicular_to?(another)
+        self.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT).intersectsLine(another.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT))
+      else
+        self.expand(COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT).intersectsLine(another.expand(COLINEAR_OR_PARALLEL_PIXEL_EXPAND_AMOUNT))
+      end
+    end
+
+    ##
+    # intersect this Ruling with a java.awt.geom.Rectangle2D
+    def intersect(area)
+      i = self.getBounds2D.createIntersection(area)
+      self.java_send :setLine, [Java::float, Java::float, Java::float, Java::float,], i.getX, i.getY, i.getX + i.getWidth, i.getY + i.getHeight
+      self
+    end
+
+    def expand(amt)
+      raise NoMethodError, "Oblique line #{self.inspect} has no #expand method." if oblique?
+      r = Ruling.new(self.top, self.left, self.width, self.height)
+      r.start = r.start - amt
+      r.end = r.end + amt
+      r
+    end
+
+
+    def length
+      Math.sqrt( (self.right - self.left).abs ** 2 + (self.bottom - self.top).abs ** 2 )
+    end
+
+    def vertical?
+      left == right
+    end
+
+    def horizontal?
+      top == bottom
+    end
+
+    def oblique?
+      !(vertical? || horizontal?)
+    end
+
+    def perpendicular_to?(other)
+      return self.vertical? == other.horizontal?
+    end
+
+    def to_json(arg)
+      [left, top, right, bottom].to_json
+    end
+
+    def colinear?(point)
+      point.x >= left && point.x <= right &&
+        point.y >= top && point.y <= bottom
+    end
+
+    ##
+    # calculate the intersection point between +self+ and other Ruling
+    def intersection_point(other)
+      # algo taken from http://mathworld.wolfram.com/Line-LineIntersection.html
+
+      #self and other should always be perpendicular, since one should be horizontal and one should be vertical
+      self_l  = self.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)
+      other_l = other.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)
+
+      return nil if !self_l.intersectsLine(other_l)
+
+      x1 = self_l.getX1; y1 = self_l.getY1
+      x2 = self_l.getX2; y2 = self_l.getY2
+      x3 = other_l.getX1; y3 = other_l.getY1
+      x4 = other_l.getX2; y4 = other_l.getY2
+
+      det = lambda { |a,b,c,d| a * d - b * c }
+
+      int_x = det.call(det.call(x1, y1, x2, y2), x1 - x2, det.call(x3, y3, x4, y4), x3 - x4) /
+        det.call(x1 - x2, y1 - y2, x3 - x4, y3 - y4)
+
+      int_y = det.call(det.call(x1, y1, x2, y2), y1 - y2,
+                       det.call(x3, y3, x4, y4), y3 - y4) /
+        det.call(x1 - x2, y1 - y2, x3 - x4, y3 - y4)
+
+      return nil if int_x.nan? || int_y.nan? # TODO is this right?
+
+
+      java.awt.geom.Point2D::Float.new(int_x, int_y)
+    end
+
+    ##
+    # Find all intersection points between two list of +Ruling+
+    # (+horizontals+ and +verticals+)
+    # TODO: this is O(n^2) - optimize.
+    def self.find_intersections(horizontals, verticals)
+      horizontals.product(verticals).inject({}) do |memo, (h, v)|
+        ip = h.intersection_point(v)
+        unless ip.nil?
+          memo[ip] ||= []
+          # TODO: stupid hack for FLA pdfs where lines appear to intersect, but don't.
+          memo[ip] << [h.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT), v.expand(PERPENDICULAR_PIXEL_EXPAND_AMOUNT)]
+        end
+        memo
+      end
+    end
+
+    ##
+    # crop an enumerable of +Ruling+ to an +area+
+    def self.crop_rulings_to_area(rulings, area)
+      rulings.reduce([]) do |memo, r|
+        if r.intersects(area)
+          memo << r.clone.intersect(area)
+        end
+        memo
+      end
+    end
+
+    # TODO do we really need this one anymore?
+    def self.clean_rulings(rulings, max_distance=4)
+
+      # merge horizontal and vertical lines
+      # TODO this should be iterative
+
+      skip = false
+
+      horiz = rulings.select { |r| r.horizontal? }
+        .group_by(&:top)
+        .values.reduce([]) do |memo, rs|
+
+        rs = rs.sort_by(&:left)
+        if rs.size > 1
+          memo +=
+            rs.each_cons(2)
+            .chunk { |p| p[1].left - p[0].right < 7 }
+            .select { |c| c[0] }
+            .map { |group|
+            group = group.last.flatten.uniq
+            Tabula::Ruling.new(group[0].top,
+                               group[0].left,
+                               group[-1].right - group[0].left,
+                               0)
+          }
+          Tabula::Ruling.new(rs[0].top, rs[0].left, rs[-1].right - rs[0].left, 0)
+        else
+          memo << rs.first
+        end
+        memo
+      end
+        .sort_by(&:top)
+
+      h = []
+      horiz.size.times do |i|
+
+        if i == horiz.size - 1
+          h << horiz[-1]
+          break
+        end
+
+        if skip
+          skip = false;
+          next
+        end
+        d = (horiz[i+1].top - horiz[i].top).abs
+
+        h << if d < max_distance # THRESHOLD DISTANCE between horizontal lines
+               skip = true
+               Tabula::Ruling.new(horiz[i].top + d / 2, [horiz[i].left, horiz[i+1].left].min, [horiz[i+1].width.abs, horiz[i].width.abs].max, 0)
+             else
+               horiz[i]
+             end
+      end
+      horiz = h
+
+      vert = rulings.select { |r| r.vertical? }
+        .group_by(&:left)
+        .values
+        .reduce([]) do |memo, rs|
+
+        rs = rs.sort_by(&:top)
+
+        if rs.size > 1
+          # Here be dragons:
+          # merge consecutive segments of lines that are close enough
+          memo +=
+            rs.each_cons(2)
+            .chunk { |p| p[1].top - p[0].bottom < 7 }
+            .select { |c| c[0] }
+            .map { |group|
+            group = group.last.flatten.uniq
+            Tabula::Ruling.new(group[0].top,
+                               group[0].left,
+                               0,
+                               group[-1].bottom - group[0].top)
+          }
+        else
+          memo << rs.first
+        end
+        memo
+      end.sort_by(&:left)
+
+      return horiz += vert
+    end
+  end
+end

data/lib/tabula/entities/spreadsheet.rb

@@ -0,0 +1,92 @@
+module Tabula
+  # a counterpart of Table, to be sure.
+  # not sure yet what their relationship ought to be.
+  class Spreadsheet < ZoneEntity
+    include Tabula::HasCells
+    attr_accessor :cells, :vertical_ruling_lines, :horizontal_ruling_lines, :cells_resolved
+
+    def initialize(top, left, width, height, page, cells, vertical_ruling_lines, horizontal_ruling_lines) #, lines)
+      super(top, left, width, height)
+      @cells = cells
+      @page = page
+      @vertical_ruling_lines = vertical_ruling_lines
+      @horizontal_ruling_lines = horizontal_ruling_lines
+    end
+
+    def ruling_lines
+      @vertical_ruling_lines + @horizontal_ruling_lines
+    end
+
+    def ruling_lines=(lines)
+      @vertical_ruling_lines = lines.select{|vl| vl.vertical? && spr.intersectsLine(vl) }
+      @horizontal_ruling_lines = lines.select{|hl| hl.horizontal? && spr.intersectsLine(hl) }
+    end
+
+    def fill_in_cells!
+      unless @cells_resolved
+        @cells_resolved = true
+        cells.each do |cell|
+          cell.text_elements = @page.get_cell_text(cell)
+        end
+      end
+    end
+
+    # call `rows` with `evaluate_cells` as `false` to defer filling in the text in
+    # each cell, which can be computationally intensive.
+    def rows(evaluate_cells=true)
+      if evaluate_cells
+        fill_in_cells!
+      end
+      tops = cells.map(&:top).uniq.sort
+      array_of_rows = tops.map do |top|
+        cells.select{|c| c.top == top }.sort_by(&:left)
+      end
+      #here, insert another kind of placeholder for empty corners
+      # like in 01001523B_China.pdf
+      #TODO: support placeholders for "empty" cells in rows other than row 1, and in #cols
+      # puts array_of_rows[0].inspect
+      if array_of_rows.size > 2
+        if array_of_rows[0].map(&:left).uniq.size < array_of_rows[1].map(&:left).uniq.size
+          missing_spots = array_of_rows[1].map(&:left) - array_of_rows[0].map(&:left)
+          # puts missing_spots.inspect
+          missing_spots.each do |missing_spot|
+            missing_spot_placeholder = Cell.new(array_of_rows[0][0].top, missing_spot, 0, 0)
+            missing_spot_placeholder.placeholder = true
+            array_of_rows[0] << missing_spot_placeholder
+          end
+        end
+        array_of_rows[0].sort_by!(&:left)
+      end
+      array_of_rows
+    end
+
+    # call `cols` with `evaluate_cells` as `false` to defer filling in the text in
+    # each cell, which can be computationally intensive.
+    def cols(evaluate_cells=true)
+      if evaluate_cells
+        fill_in_cells!
+      end
+      lefts = cells.map(&:left).uniq.sort
+      lefts.map do |left|
+        cells.select{|c| c.left == left }.sort_by(&:top)
+      end
+    end
+
+    def to_a
+      fill_in_cells!
+      rows.map{ |row_cells| row_cells.map(&:text) }
+    end
+
+    def to_csv
+      out = StringIO.new
+      Tabula::Writers.CSV(rows, out)
+      out.string
+    end
+
+    def to_tsv
+      out = StringIO.new
+      Tabula::Writers.TSV(rows, out)
+      out.string
+    end
+  end
+end

data/lib/tabula/entities/table.rb

@@ -0,0 +1,81 @@
+module Tabula
+  class Table
+    attr_reader :lines
+    def initialize(line_count, separators)
+      @separators = separators
+      @lines = (0...line_count).inject([]) { |m| m << Line.new }
+    end
+
+    def add_text_element(text_element, i, j)
+      if @lines.size <= i
+        @lines[i] = Line.new
+      end
+      if @lines[i].text_elements[j]
+        @lines[i].text_elements[j].merge!(text_element)
+      else
+        @lines[i].text_elements[j] = text_element
+      end
+    end
+
+    def rpad!
+      max = lines.map{|l| l.text_elements.size}.max
+      lines.each do |line|
+        needed = max - line.text_elements.size
+        needed.times do 
+          line.text_elements << TextElement.new(nil, nil, nil, nil, nil, nil, '', nil)
+        end
+      end
+    end
+
+    def cols
+      self.rpad!
+      lines.map(&:text_elements).transpose
+    end
+
+    def rows
+      self.rpad!
+      lines.map(&:text_elements)
+    end
+
+    # create a new Table object from an array of arrays, representing a list of rows in a spreadsheet
+    # probably only used for testing
+    def self.new_from_array(array_of_rows)
+      t = Table.new(array_of_rows.size, [])
+      array_of_rows.each_with_index do |row, index|
+        t.lines[index].text_elements = row.map{|cell| TextElement.new(nil, nil, nil, nil, nil, nil, cell, nil)}
+      end
+      t
+    end
+
+    #for equality testing, return @lines stripped of leading columns of empty strings
+    #TODO: write a method to strip all totally-empty columns (or not?)
+    def lstrip_lines
+      return @lines if @lines.include?(nil)
+      min_leading_empty_strings = Float::INFINITY
+      @lines.each do |line|
+        empties = line.text_elements.map{|t| t.nil? || t.text.empty? }
+        min_leading_empty_strings = [min_leading_empty_strings, empties.index(false)].min
+      end
+      if min_leading_empty_strings == 0
+        @lines
+      else
+        @lines.each{|line| line.text_elements = line.text_elements[min_leading_empty_strings..-1]}
+        @lines
+      end
+    end
+    def lstrip_lines!
+      @lines = self.lstrip_lines
+    end
+
+    #used for testing, ignores separator locations (they'll sometimes be nil/empty)
+    def ==(other)
+      self.instance_variable_set(:@lines, self.lstrip_lines)
+      other.instance_variable_set(:@lines, other.lstrip_lines)
+      self.instance_variable_set(:@lines, self.lines.rpad(nil, other.lines.size))
+      other.instance_variable_set(:@lines, other.lines.rpad(nil, self.lines.size))
+
+      self.lines.zip(other.lines).all? { |my, yours| my == yours }
+
+    end
+  end
+end