RubyGems - pagerecognizer - Versions diffs - 0.0.1 → 0.1.0 - Mend

pagerecognizer 0.0.1 → 0.1.0

Files changed (5) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61a30cc5e39e171b8eabdf26490475ce3fb041b9
-  data.tar.gz: e64d2e3700730de8e9d3fe4d4de57f4743ca98b9
+  metadata.gz: 17ef706811d7513a3f7f6a109feacb59bbae91dc
+  data.tar.gz: e6890fcd6c6bfdd6d042513f02dea280e5c436ae
 SHA512:
-  metadata.gz: cd4370f97135ac3df6376c2df1dcbb39cd25a35fac0ad172d29fa86764d8fb82d7ec28da7d4aaf0a547f1684cdda34f2d4de1b14de5c66ed784bb798643e96cd
-  data.tar.gz: e517075c5eb9d4efdb5bc851865136776df7dda1eacaecb96e3d89f211c63c6a5429b0ee4286280d150f7ea338662ceebdc0f9b0039d7df8a6a2778b349629f6
+  metadata.gz: 74fc2c48871a01192e4ebbd80f603a170b1f953f61252f8bf21627bc3abb8555beb12572ab786754f6f015870bdaa6cec54daab23d15068c7aa0152b70c612aa
+  data.tar.gz: 3e6a080c6740075bab1c111127249ade7429ac844fcf1c0b99979c7a535cd5013cf9bd86629957ee7bcd4467828d023831a51df078037dff5cfc777c6a81be9e

data/lib/pagerecognizer.rb CHANGED Viewed

@@ -4,14 +4,16 @@ module PageRecognizer
   end
   require "logger"
   self.logger = Logger.new STDOUT
+  self.logger.formatter = ->(severity, datetime, progname, msg){ "#{datetime.strftime "%H%M%S"} #{severity.to_s[0]} #{msg}\n" }
+  self.logger.level = ENV.fetch("LOGLEVEL_PageRecognizer", "FATAL").to_sym
   module Dumpable
     def dump
-      "<html><body>#{
+      "<html><body style='white-space: nowrap'>#{
         map.with_index do |n, i|
-          "<div style='position: absolute; background-color: hsla(#{
+          "<div id='#{i}' style='position: absolute; background-color: hsla(#{
             360 * i / size
-          },100%,50%,0.5); top: #{n.top}; left: #{n.left}; width: #{n.width}; height: #{n.height}'>#{
+          },100%,50%,0.5); top: #{n.top}; left: #{n.left}; width: #{n.width}; height: #{n.height}'>#{i} #{
             n.node.tag_name.upcase
           }</div>"
         end.join
@@ -29,92 +31,110 @@ module PageRecognizer
     end.extend Dumpable
   end
-  def recognize
+  def self.rgb2hsv r, g, b   # [<256, <256, <256]
+    # http://stackoverflow.com/q/41926874/322020
+    r, g, b  = [r, g, b].map{ |_| _.fdiv 255 }
+    min, max = [r, g, b].minmax
+    chroma   = max - min
+    [
+      60.0 * ( chroma.zero? ? 0 : case max
+        when r ; (g - b) / chroma
+        when g ; (b - r) / chroma + 2
+        when b ; (r - g) / chroma + 4
+        else 0
+      end % 6 ),
+      chroma.zero? ? 0.0 : chroma / max,
+      max,
+    ]   # [<=360, <=1, <=1]
+  end
+  def self.dist h1, s1, v1, h2, s2, v2   # [<256, <256, <256]
+    # https://en.wikipedia.org/wiki/HSL_and_HSV#/media/File:Hsl-hsv_saturation-lightness_slices.svg
+    c1, c2 = s1 * v1 / 256.0, s2 * v2 / 256.0   # chroma
+    z1, z2 = v1 * (2 - c1 / 256), v2 * (2 - c2 / 256)
+    a = (((h2 - h1) * 360 / 256.0) % 360) / (180 / Math::PI)
+        x2 =     Math::sin(a) * c2
+    y1, y2 = c1, Math::cos(a) * c2
+    x2*x2 + (y1-y2)*(y1-y2) + (z1-z2)*(z1-z2)
+  end
+  private def recognize
     logger = Module.nesting.first.logger
+    logger.info "method #{__method__}..."
     nodes = []
     try = lambda do
-      prev = nodes
-      code = "( function(node) {
-        var x = scrollX, y = scrollY;
-        var _tap = function(x, f){ f(); return x };
-        var f = function(node) {
-          node.scrollIntoView();
-          var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
-          var child_nodes = Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 });
-          var clickable;
-          if (node.nodeName == 'svg') {
-            var states = child_nodes.map( function(n){
-              return _tap(n.style ? n.style.display : '', function(){ n.style.display = 'none' } );
-            } );
-            clickable = (node === document.elementFromPoint(rect.x + rect.width/2, rect.y + rect.height/2));
-            var _zip = function(a, b){ return a.map( function(e, i) { return [e, b[i]] } ) };
-            _zip(child_nodes, states).forEach( function(_){ _[0].style.display = _[1] } );
-          } else {
-            clickable = (node === document.elementFromPoint(rect.x + rect.width/2, rect.y + rect.height/2));
-          };
-          rect.top += scrollY;
-          rect.left += scrollX;
-          return [ [
-            rect.top, rect.left, rect.width, rect.height, clickable, node
-          ] ].concat(node.nodeName == 'svg' ? [] : child_nodes.flatMap(f));
-        };
-        return _tap(f(node), function(){ scrollTo(x, y) });
-      } )(arguments[0])"
-      str = Struct.new :top, :left, :width, :height, :clickable, :node
-      nodes = page.evaluate(code, self).map{ |s| str.new *s }
-      nodes.size == prev.size
-    end
+      str = Struct.new :node, :visible, :top, :left, :width, :height, :area do
+        def texts
+          node.page.evaluate(<<~HEREDOC, node).map(&JSON.method(:load)).map do |text, rect1, rect2, style|
+            (function(node){
+              let result = [], range = document.createRange();
+              for (
+                let iterator = document.evaluate('.//text()', node, null, XPathResult.ANY_TYPE, null);
+                text = iterator.iterateNext();
+              ) {
+                range.selectNode(text);
+                result.push(JSON.stringify( [
+                  text.wholeText,
+                  range.getBoundingClientRect(),
+                  text.parentNode.getBoundingClientRect(),
+                  getComputedStyle(text.parentNode),
+                ] ));
+              }
+              return result;
+            })(arguments[0])
+          HEREDOC
-    if defined? Selenium::WebDriver::Wait
-      Selenium::WebDriver::Wait.new(
-        message: "number of DOM elements didn't stop to change"
-      ).until &try
-    else
-      t = Time.now
-      until try.call
-        fail "number of DOM elements didn't stop to change" if Time.now > t + 5
-      end
-    end
-    logger.info "#{nodes.size} DOM nodes found"
+            # google SERP has 1x1 nodes with text _<>
+            next if rect1["width"] < 2 || rect1["height"] < 2
+            next if rect2["width"] < 2 || rect2["height"] < 2
-    nodes.select! &:clickable
-    nodes.reject do |n|
-      nodes.any? do |nn|
-        cs = [
-          nn.top <=> n.top,
-          nn.left <=> n.left,
-          n.left + n.width <=> nn.left + nn.width,
-          n.top + n.height <=> nn.top + nn.height,
-        ]
-        cs.include?(1) && !cs.include?(-1)
+            color = style["color"]
+            fail color unless /\Argba?\((?<red>\d+), (?<green>\d+), (?<blue>\d+)(, 0(\.\d+)?)?\)\z/ =~ color
+            closest_color = {   # https://en.wikipedia.org/wiki/Web_colors#Basic_colors
+              white: [0, 0, 100],
+              silver: [0, 0, 75],
+              gray: [0, 0, 50],
+              black: [0, 0, 0],
+              red: [0, 100, 100],
+              maroon: [0, 100, 50],
+              yellow: [60, 100, 100],
+              olive: [60, 100, 50],
+              lime: [120, 100, 100],
+              green: [120, 100, 50],
+              aqua: [180, 100, 100],
+              teal: [180, 100, 50],
+              blue: [240, 100, 100],
+              navy: [240, 100, 50],
+              fuchsia: [300, 100, 100],
+              purple: [300, 100, 50],
+            }.to_a.min_by do |_, (h1, s1, v1)|
+              h2, s2, v2 = PageRecognizer.rgb2hsv(red.to_i, green.to_i, blue.to_i)
+              PageRecognizer.dist h1*255/360, s1*256/100, v1*256/100, h2*255/360, s2*255, v2*255
+            end.first
+            [text, style, closest_color, rect1]
+          end.compact
+        end
       end
-    end.extend Dumpable
-  end
-  private def recognize_more
-    logger = Module.nesting.first.logger
-    nodes = []
-    try = lambda do
-      prev = nodes
-      code = "( function(node) {
-        var x = scrollX, y = scrollY;
-        var _tap = function(x, f){ f(); return x };
-        var f = function(node) {
-          node.scrollIntoView();
-          var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
-          rect.top += scrollY;
-          rect.left += scrollX;
-          return [ [
-            node, JSON.stringify([rect.top, rect.left, rect.width, rect.height])
-          ] ].concat(Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 }).flatMap(f));
-        };
-        return _tap(f(node), function(){ scrollTo(x, y) });
-      } )(arguments[0])"
-      str = Struct.new :node, :top, :left, :width, :height
-      nodes = page.evaluate(code, self).map{ |node, a| str.new node, *JSON.load(a) }
-      nodes.size == prev.size
+      prev = nodes.size
+      t = page.evaluate(<<~HEREDOC, self)
+        ( function(node) {
+          var x = scrollX, y = scrollY;
+          var _tap = function(x, f){ f(); return x };
+          var f = function(node) {
+            node.scrollIntoView();
+            var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
+            rect.top += scrollY;
+            rect.left += scrollX;
+            return [
+              node, JSON.stringify([rect.top, rect.left, rect.width, rect.height]), ("visible" == getComputedStyle(node).visibility)
+            ].concat(Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 }).flatMap(f));
+          };
+          return _tap(f(node), function(){ scrollTo(x, y) });
+        } )(arguments[0])
+      HEREDOC
+      logger.debug [t.size / 3, prev]
+      nodes = t.each_slice(3).map{ |node, rect, visible| str.new(node, visible, *JSON.load(rect)).tap{ |_| _.area = _.width * _.height } }
+      nodes.size == prev
     end
     if defined? Selenium::WebDriver::Wait
@@ -128,9 +148,9 @@ module PageRecognizer
       end
     end
     logger.info "#{nodes.size} DOM nodes found"
-    nodes.reject!{ |i| i.height.zero? || i.width.zero? }
-    nodes
+    nodes.reject!{ |_| _.height.zero? || _.width.zero? || !_.visible }
+    logger.info "visible nodes: #{nodes.size}"
+    nodes.extend Dumpable
   end
   logging_error = Class.new RuntimeError do
@@ -143,8 +163,9 @@ module PageRecognizer
   end
   class ErrorNotEnoughNodes < logging_error ; end
-  private def split heuristics, hh, ww, tt, ll
+  private def split hh, ww, tt, ll, heuristics, try_min, dump, &filter
     logger = Module.nesting.first.logger
+    logger.info heuristics
     unstale = unless defined? Selenium::WebDriver::Error::StaleElementReferenceError
       ->(&b){ b.call }
@@ -159,82 +180,253 @@ module PageRecognizer
         end
       end
     end
-    all = unstale.call do recognize_more end.sort_by(&tt)
-    logger.info "all nodes: #{all.size}"
-    rect = page.evaluate("( function(node) { return JSON.parse(JSON.stringify(node.getBoundingClientRect())) } )(arguments[0])", self)
-    inside = all.reject{ |i| i.left < rect["left"] || i.left + i.width > rect["right"] || i.top < rect["top"] || i.top + i.height > rect["bottom"] }
-    raise ErrorNotEnoughNodes.new "no inside nodes", all: all, inside: inside if inside.empty?
-    logger.info "inside nodes: #{inside.size}"
-    nodes = unstale.call do inside.reject{ |i| %w{ button script svg path a img span }.include? i.node.tag_name } end.uniq{ |i| [i[hh], i[ww], i[tt], i[ll]] }
-    logger.info "good nodes: #{nodes.size}"   # only those that might be containers
-    large = nodes#.select{ |i| i[ww] > nodes.map(&ww).max / 4 }
-    logger.info "large enough and unique: #{large.size}"
+    nodes = unstale.call do recognize end.sort_by{ |_| [_[tt], _[ll]] }
+    File.write "#{dump}.all.htm", nodes.extend(Dumpable).dump if dump
+    nodes = unstale.call do nodes.reject{ |i| %w{ button script svg path a img }.include? i.node.tag_name } end.uniq{ |_| [_[hh], _[ww], _[tt], _[ll]] }
+    logger.info "good and unique: #{nodes.size}"   # only those that might be containers
+    File.write "#{dump}.nodes.htm", nodes.extend(Dumpable).dump if dump
     interfere = lambda do |a, b|
       a[tt] < b[tt] + b[hh] &&
       b[tt] < a[tt] + a[hh]
     end
-    rest = large.select.with_index do |a, i|
-      large.each_with_index.none? do |b, j|
+    rest = nodes.select.with_index do |a, i|
+      nodes.each_with_index.none? do |b, j|
         next if i == j
         a[tt] >= b[tt] && a[tt] + a[hh] <= b[tt] + b[hh] &&
-        large.all?{ |c| interfere[a, c] == interfere[b, c] }
+        a[ll] >= b[ll] && a[ll] + a[ww] <= b[ll] + b[ww] &&
+        nodes.all?{ |c| interfere[a, c] == interfere[b, c] }
       end
     end
     logger.info "not nested: #{rest.size}"
-    # rest = rest.sample 50
+    File.write "#{dump}.rest1.htm", rest.extend(Dumpable).dump if dump
+    # 8 = max_results - 1, 3 = (from row size diff euristic)
+    if try_min
+      rest = rest.reject{ |_| _[hh] + _[hh]/3*(try_min - 1) > (rest.map{ |_| _[tt] + _[hh] }.max - rest.map(&tt).min) }
+      logger.info "small enough: #{rest.size}"
+    end
+    File.write "#{dump}.rest2.htm", rest.extend(Dumpable).dump if dump
-    # adding the :area field for faster upcoming computations
-    struct = Struct.new *large.first.members, :area
-    rest.map!{ |i| struct.new *i.values, i.width * i.height }
+    rest.select! &filter
+    logger.info "filtered: #{rest.size}"
+    File.write "#{dump}.filtered.htm", rest.extend(Dumpable).dump if dump
+    rest.sort_by!(&:area).reverse!
+    File.write "#{dump}.sorted.htm", rest.extend(Dumpable).dump if dump
     require "pcbr"
     pcbr = PCBR.new
     is = []
-    max, past = 0, []
+    max, past = 0, Set.new
     prev = nil
     time = Time.now
     loop do
-      rest.each_with_index do |node, i|
-        next if is.any?{ |j| i == j || interfere[rest[i], rest[j]] }
+      si = (0...rest.size).reject do |i|
+        # I don't shrink pcbr so this should be a safe optimization
+        next true if is.last > i unless is.empty?
+        # also we've sorted from large to small so it does not get stuck with the half of the page below the largest node
+        next (logger.debug [i, 2]; true) if is.any?{ |j| i == j || interfere[rest[i], rest[j]] }
+        next (logger.debug [i, 3]; true) if is.any?{ |j| rest[i][ww] > rest[j][ww] * 2 } if heuristics.include? :WIDTH
+        next (logger.debug [i, 4]; true) if is.any?{ |j| rest[j][ww] > rest[i][ww] * 2 } if heuristics.include? :WIDTH
+        next (logger.debug [i, 5]; true) if is.any?{ |j| rest[i][hh] > rest[j][hh] * 3 }
+        next (logger.debug [i, 6]; true) if is.any?{ |j| rest[j][hh] > rest[i][hh] * 3 }
+      end
+      logger.debug [is, si]
+      si.each do |i|
         sol = rest.values_at *is, i
+        unless pcbr.set.include? [*is, i].sort
+        logger.debug [is, i, sol.map(&:area).reduce(:+)]
         pcbr.store [*is, i].sort, [
           *( is.size                                                                                                                if heuristics.include? :SIZE   ),
-          *( sol.map(&:area).inject(:+)                                                                                             if heuristics.include? :AREA   ),
-          *( -sol.product(sol).map{ |s1, s2| (s1.width              - s2.width             ).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :WIDTH  ),
-          *( -sol.product(sol).map{ |s1, s2| (s1.height             - s2.height            ).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :HEIGHT ),
-          *( -sol.product(sol).map{ |s1, s2| (s1[ll] + s1[ww] / 2.0 - s2[ll] - s2[ww] / 2.0).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :MIDDLE ),
-        ] unless pcbr.table.assoc [*is, i].sort
+          *( sol.map(&:area).reduce(:+)                                                                                             if heuristics.include? :AREA   ),
+          # https://en.wikipedia.org/wiki/Mean_absolute_difference
+          *( -sol.product(sol).map{ |s1, s2| (s1.height             - s2.height            ).abs }.reduce(:+) / sol.size / sol.size if heuristics.include? :HEIGHT ),
+          *( -sol.product(sol).map{ |s1, s2| (s1[ll] + s1[ww] / 2.0 - s2[ll] - s2[ww] / 2.0).abs }.reduce(:+) / sol.size / sol.size if heuristics.include? :MIDDLE ),
+        ]
+          logger.debug "pcbr.table.size: #{pcbr.table.size}"
+          if si.none? do |j|
+            next if j <= i
+            next true if interfere[rest[i], rest[j]]
+            next true if rest[i][ww] > rest[j][ww] * 2 if heuristics.include? :WIDTH
+            next true if rest[j][ww] > rest[i][ww] * 2 if heuristics.include? :WIDTH
+            next true if rest[i][hh] > rest[j][hh] * 3
+            next true if rest[j][hh] > rest[i][hh] * 3
+          end
+            logger.debug "forced"
+            break
+          end
+        end
       end
-      if prev && Time.now - time > 1 && (Time.now - prev > (prev - time))
-        m = pcbr.table.reject{ |i| i.first.size == 1 }.map(&:last).max
-        break if 1 == pcbr.table.count{ |i| i.last == m } || Time.now - time > 5
+      if prev && Time.now - time > 5
+        logger.debug "check"
+        break logger.info "break 0" if Time.now - time > 30
+        break logger.info "break 1" if Time.now - prev > 10
+        m = pcbr.table.reject{ |i| i.first.size < 2 }.map(&:last).max
+        break logger.info "break 2" if Time.now - prev > (prev - time) && 1 == pcbr.table.count{ |i| i.last == m }
       end
-      break unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.inject(:+) }.max_by(&:last)
+      break logger.info "done" unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.reduce(:+) }.max_by(&:last)
+      logger.debug "next: #{t}"
+      past.add (is = t.first).map{ |i| 2**i }.reduce(:+)
       if t.last > max
         prev, max = Time.now, t.last
+        logger.debug "new max: #{max}"
         logger.debug [Time.now - time, max, t.first]
       end
-      past.push (is = t.first).map{ |i| 2**i }.inject(:+)
     end
     # TODO: if multiple with max score, take the max by area
-    unless best = pcbr.table.reject{ |is,| is.size == 1 }.max_by(&:last)
-      raise ErrorNotEnoughNodes.new "failed to split <#{tag_name}>", all: all, inside: inside, nodes: nodes, large: large, rest: rest
+    unless best = pcbr.table.reject{ |is,| is.size < 2 }.max_by(&:last)
+      raise ErrorNotEnoughNodes.new "failed to split <#{tag_name}>", all: all, nodes: nodes, rest: rest
     end
-    rest.values_at(*best.first).extend(Dumpable)
+    pcbr.table.max_by(20, &:last).each_with_index{ |_, i| logger.debug "##{i} #{_}" }
+    logger.info best
+    logger.info "splitted in #{best.first.size}"
+    rest.values_at(*best.first).sort_by(&tt).extend Dumpable
   end
-  def rows *heuristics
-    heuristics = %i{ AREA HEIGHT WIDTH } if heuristics.empty?
-    split heuristics, :height, :width, :top, :left
+  def rows heuristics, try_min: nil, dump: nil, &b
+    split :height, :width, :top, :left, heuristics, try_min, dump, &b
   end
-  def cols *heuristics
-    heuristics = %i{ AREA HEIGHT WIDTH } if heuristics.empty?
-    split heuristics, :width, :height, :left, :top
+  def cols heuristics, try_min: nil, dump: nil, &b
+    split :width, :height, :left, :top, heuristics, try_min, dump, &b
   end
+  def self.piles z
+    max = nil
+    result = [current = []]
+    z.map.with_index.sort.each do |x|
+      if !max || max > x[0][0]
+        current.push x
+        max = x[0][0] + x[0][1] if !max || max < x[0][0] + x[0][1]
+      else
+        result.push current = [x]
+        max = x[0][0] + x[0][1]
+      end
+    end
+    result.map{ |_| _.map &:last }
+  end
+  module Gridable
+    def rows
+      Module.nesting[1].piles(map{ |n| [n.top, n.height] }).map{ |s| values_at(*s).extend Module.nesting[1]::Dumpable }
+    end
+    def cols
+      Module.nesting[1].piles(map{ |n| [n.left, n.width] }).map{ |s| values_at(*s).extend Module.nesting[1]::Dumpable }
+    end
+  end
+  def grid dump = nil
+    logger = Module.nesting.first.logger
+    all = recognize
+    logger.info "all nodes: #{all.size}"
+    File.write "#{dump}.all.htm", all.extend(Dumpable).dump if dump
+    # adding the fields for faster upcoming computations
+    struct = Struct.new *all.first.members, :midx, :midy
+    all.map!{ |i| struct.new *i.values, i.left + i.width / 2.0, i.top * i.height / 2.0 }
+    all = all.sort_by{ |_| [_.area, _.top, _.left] }.reverse
+    rect = page.evaluate("( function(node) { return JSON.parse(JSON.stringify(node.getBoundingClientRect())) } )(arguments[0])", self)
+    inside = all.reject{ |i| i.left < rect["left"] || i.left + i.width > rect["right"] || i.top < rect["top"] || i.top + i.height > rect["bottom"] }
+    raise ErrorNotEnoughNodes.new "no inside nodes", all: all, inside: inside if inside.empty?
+    logger.info "inside nodes: #{inside.size}"
+    File.write "#{dump}.inside.htm", inside.extend(Dumpable).dump if dump
+    good = inside.reject{ |i| %w{ button script svg path a img }.include? i.node.tag_name }.uniq{ |i| [i.height, i.width, i.top, i.left] }
+    logger.info "good and unique: #{good.size}"   # only those that might be containers
+    File.write "#{dump}.good.htm", good.extend(Dumpable).dump if dump
+    # large = good#.select{ |i| i[ww] > good.map(&ww).max / 4 }
+    # logger.info "large enough: #{large.size}"
+    interfere = lambda do |a, b|
+      a.top < b.top + b.height &&
+      b.top < a.top + a.height &&
+      a.left < b.left + b.width &&
+      b.left < a.left + a.width
+    end
+    rest = good.select.with_index do |a, i|
+      good.each_with_index.none? do |b, j|
+        next if i == j
+        a.top >= b.top && a.top + a.height <= b.top + b.height &&
+        a.left >= b.left && a.left + a.width <= b.left + b.width &&
+        good.all?{ |c| interfere[a, c] == interfere[b, c] }
+      end
+    end
+    logger.info "not nested: #{rest.size}"
+    File.write "#{dump}.rest.htm", rest.extend(Dumpable).dump if dump
+    begin
+      prev = rest.size
+      rest.select!.with_index do |a, i|
+        rest.each_with_index.any? do |b, j|
+          cw = [[a.left + a.width, b.left + b.width].min - [a.left, b.left].max, 0].max
+          i != j && !interfere[a, b] && [cw, a.width].min.fdiv(a.width) * [cw, b.width].min.fdiv(b.width) > 0.9
+        end and
+        rest.each_with_index.any? do |b, j|
+          ch = [[a.top + a.height, b.top + b.height].min - [a.top, b.top].max, 0].max
+          i != j && !interfere[a, b] && [ch, a.height].min.fdiv(a.height) * [ch, b.height].min.fdiv(b.height) > 0.9
+        end
+      end
+    end until prev == rest.size
+    logger.info "gridable: #{rest.size}"
+    File.write "#{dump}.griddable.htm", rest.extend(Dumpable).dump if dump
+    require "pcbr"
+    pcbr = PCBR.new
+    max, past = 0, []
+    prev = nil
+    prev_max = nil
+    time = Time.now
+    heuristics = %i{ SIZE AREA }
+    inter = lambda do |a1, a2, b1, b2|
+      c = [[a1 + a2, b1 + b2].min - [a1, b1].max, 0].max
+      [c, a2].min.fdiv(a2) * [c, b2].min.fdiv(b2)
+    end
+    lp = lambda do |is|
+      past.push is.map{ |i| 2**i }.reduce(:+)
+      rest.size.times do |ij|
+        next if ij <= is.last unless is.empty?
+        sorted = is + [ij]
+        next if pcbr.set.include? sorted
+        next if is.any?{ |j| interfere[rest[ij], rest[j]] }
+        sol = rest.values_at *sorted
+        xn = Module.nesting.first.piles sol.map{ |s| [s.left, s.width] }
+        yn = Module.nesting.first.piles sol.map{ |s| [s.top, s.height] }
+        next if xn.product(yn).any?{ |i,j| (i & j).size > 1 } if sorted.size >= 4
+        pcbr.store sorted, [
+          *( sol.map(&:area).reduce(:+) if heuristics.include? :AREA ),
+          xn.map{ |g| sosol = sol.values_at *g; next 0 if sosol.size == 1; sosol.combination(2).map{ |s1, s2| inter[s1.left, s1.width, s2.left, s2.width] }.reduce(:+) / sosol.size / (sosol.size - 1) * 2 }.reduce(:+) / xn.size,
+          yn.map{ |g| sosol = sol.values_at *g; next 0 if sosol.size == 1; sosol.combination(2).map{ |s1, s2| inter[s1.top, s1.height, s2.top, s2.height] }.reduce(:+) / sosol.size / (sosol.size - 1) * 2 }.reduce(:+) / yn.size,
+        ]
+        if prev && Time.now - time > 3
+          logger.debug "check"
+          break logger.info "break 0" if Time.now - time > 30
+          break logger.info "break 1" if Time.now - prev > 10
+          m = pcbr.table.reject{ |i| i.first.size < 3 }.map(&:last).max
+          break logger.debug "break 2" if Time.now - prev > (prev - time) * 2 && 1 == pcbr.table.count{ |i| i.last == m }
+        end
+        break logger.info "break 3" unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.reduce(:+) }.max_by(&:last)
+        logger.debug [t.last, max, t.first == prev_max, t.first.map{ |i| 2**i }.reduce(:+)]
+        if t.last > max && t.first != prev_max
+          prev, max, prev_max = Time.now, t.last, t.first
+          logger.debug [pcbr.table.size, max, t.first]
+        end
+        lp.call t.first
+      end
+    end
+    lp.call []
+    # TODO: if multiple with max score, take the max by area
+    pcbr.table.max_by(20, &:last).each_with_index{ |_, i| logger.debug "##{i} #{_}" }
+    rest.values_at(*pcbr.table.max_by(&:last).first).extend Dumpable, Gridable
+  end
 end

data/pagerecognizer.gemspec CHANGED Viewed

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "pagerecognizer"
-  spec.version      = "0.0.1"
+  spec.version      = "0.1.0"
   spec.summary      = "visual HTML page structure recognizer"
   spec.author       = "Victor Maslov aka Nakilon"
@@ -8,16 +8,15 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
   spec.metadata     = {"source_code_uri" => "https://github.com/nakilon/pagerecognizer"}
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "pcbr"
   spec.add_dependency "ferrum"
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "pcbr", "~>0.4.2"
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "mll"
-  spec.require_path = "lib"
   spec.test_file    = "test.rb"
   spec.files        = %w{ LICENSE pagerecognizer.gemspec lib/pagerecognizer.rb }
 end

data/test.rb CHANGED Viewed

@@ -1,28 +1,72 @@
 require "minitest/autorun"
 require "ferrum"
 require_relative "lib/pagerecognizer"
-Ferrum::Node.include PageRecognizer
+PageRecognizer.logger.level = :INFO
 describe PageRecognizer do
-  it "google" do
-    browser = Ferrum::Browser.new **(ENV.has_key?("FERRUM_NO_SANDBOX") ? {browser_options: {"no-sandbox": nil}} : {})
-    browser.goto "about:blank"
-    browser.execute "document.write(#{File.read("google.htm").inspect})"
-    results = browser.at_css("body").rows
-    width = results.group_by(&:width).max_by{ |w, g| g.size }.first
-    assert_equal [
-      ["https://www.ruby-lang.org/ru/", "Ruby это... динамический язык программирования с о"],
-      ["https://ru.wikibooks.org/wiki/Ruby", "Этот учебник намерен осветить все тонкости програм"],
-      ["https://habr.com/ru/post/433672/", "19 дек. 2018 г. - Взрывной рост интереса к Ruby ос"],
-      ["https://habr.com/ru/hub/ruby/", "Ruby (англ. Ruby — «Рубин») — динамический, рефлек"],
-      ["https://web-creator.ru/articles/ruby", "Ruby разрабатывался на Linux, но работает на многи"],
-      ["http://rusrails.ru/", "Ruby on Rails руководства, учебники, статьи на рус"],
-      ["https://vc.ru/dev/72391-pochemu-my-vybiraem-ruby-d", "20 июн. 2019 г. - Ruby on Rails одним из первых на"],
-      ["https://tproger.ru/tag/ruby/", "Django или Ruby on Rails: какой фреймворк выбрать?"],
-      ["https://rubyrussia.club/", "Главная российская конференция о Ruby. Расширяем г"]
-    ], results.select{ |r| r.width == width }.map(&:node).map(&:rows).map{ |link, desc| [
-      link.node.at_css("a").property("href")[0,50],
-      desc.node.text[0,50],
+  before do
+    options = {}
+    options[:browser_options] = {"no-sandbox": nil} if ENV.has_key? "FERRUM_NO_SANDBOX"
+    options[:headless] = false if ENV.has_key? "HEADFULL"
+    @browser = Ferrum::Browser.new **options
+  end
+  after do
+    @browser&.quit
+  end
+  [
+      ["google1.htm", [
+        ["https://ru.wikipedia.org/wiki/Ruby#:~:te", "Ruby — Википедия"],
+        ["https://www.ruby-lang.org/ru/", "Язык программирования Ruby"],
+        ["https://ru.wikibooks.org/wiki/Ruby", "Ruby — Викиучебник"],
+        ["https://habr.com/ru/post/433672/", "Пацаны, так Ruby умер или нет? / Хабр - Habr"],
+        ["https://habr.com/ru/hub/ruby/", "Ruby – Динамический высокоуровневый язык..."],
+        ["https://web-creator.ru/articles/ruby", "Язык программирования Ruby - Веб Креатор"],
+        ["http://rusrails.ru/", "Rusrails: Ruby on Rails по-русски"],
+        ["https://vc.ru/dev/72391-pochemu-my-vybir", "Почему мы выбираем Ruby для наших проектов..."],
+        ["https://tproger.ru/tag/ruby/", "Ruby — всё по этой теме для программистов..."],
+        ["https://rubyrussia.club/", "RubyRussia"],
+      ] ],
+      ["google2.mht", [
+        ["https://www.ruby-lang.org/ru/", "Язык программирования Ruby"],
+        ["https://ru.wikipedia.org/wiki/Ruby", "Ruby - Википедия"],
+        ["https://evrone.ru/why-ruby", "5 причин, почему мы выбираем Ruby - evrone.ru"],
+        ["https://habr.com/ru/hub/ruby/", "Ruby — Динамический высокоуровневый язык..."],
+        ["https://ru.wikibooks.org/wiki/Ruby", "Ruby - Викиучебник"],
+        ["https://context.reverso.net/%D0%BF%D0%B5", "ruby - Перевод на русский - примеры английский..."],
+        ["https://web-creator.ru/articles/ruby", "Язык программирования Ruby - Веб Креатор"],
+        ["https://ru.hexlet.io/courses/ruby", "Введение в Ruby - Хекслет"],
+        ["https://rubyrush.ru/articles/what-is-rub", "Что такое Ruby on Rails?"],
+      ] ],
+  ].each do |filename, expectation|
+    it "google rows #{filename}" do
+    @browser.goto "file://#{File.expand_path filename}"
+    results = @browser.at_css("body").rows([:AREA, :SIZE], try_min: 9) do |node|
+      texts = node.texts
+      next if texts.none?{ |_, _, color, | :black == color }
+      _, group = texts.group_by{ |_, style, | style["fontSize"].to_i }.to_a.max_by(&:first)
+      next unless group
+      next unless group.size == 1 && %i{ blue navy }.include?(group[0][2])
+      true
+    end
+    assert_equal expectation, results.reject{ |_| _.node.at_css "img" }.map{ |result| [
+      result.node.at_css("a").property("href")[0,40],
+      result.texts.max_by{ |_, style, | style["fontStyle"].to_i }[0].sub(/(.{40}) .+/, "\\1..."),
     ] }
+    end
+  end
+  [
+      ["youtube.htm", %w{ Главная В\ тренде Подписки Библиотека История }, 8],
+      ["youtube2.mht", %w{ Главная Навигатор Shorts Подписки Библиотека История }, 10],
+  ].each do |filename, expected_navigation, rows|
+    it "youtube rows grid #{filename}" do
+      @browser.goto "file://#{File.expand_path filename}"
+      assert_equal expected_navigation, @browser.at_css("ytd-mini-guide-renderer").rows([:AREA, :SIZE]){ |_| !_.node.text.strip.empty? }.map{ |nav| nav.texts.first[0] }
+      grid = @browser.at_css("#content").grid
+      assert_equal 3*rows, grid.size
+      assert_equal [3]*rows, grid.rows.map(&:size)
+      assert_equal [rows]*3, grid.cols.map(&:size)
+      grid.each{ |n| n.to_h.values_at(:width, :height).each{ |_| assert_in_delta 250, _, 50 } }
+    end
   end
 end

metadata CHANGED Viewed

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: pagerecognizer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-24 00:00:00.000000000 Z
+date: 2022-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pcbr
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: pcbr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement