RubyGems - pagerecognizer - Versions diffs - 0.0.1 → 0.1.0 - Mend

pagerecognizer 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61a30cc5e39e171b8eabdf26490475ce3fb041b9
-  data.tar.gz: e64d2e3700730de8e9d3fe4d4de57f4743ca98b9
+  metadata.gz: 17ef706811d7513a3f7f6a109feacb59bbae91dc
+  data.tar.gz: e6890fcd6c6bfdd6d042513f02dea280e5c436ae
 SHA512:
-  metadata.gz: cd4370f97135ac3df6376c2df1dcbb39cd25a35fac0ad172d29fa86764d8fb82d7ec28da7d4aaf0a547f1684cdda34f2d4de1b14de5c66ed784bb798643e96cd
-  data.tar.gz: e517075c5eb9d4efdb5bc851865136776df7dda1eacaecb96e3d89f211c63c6a5429b0ee4286280d150f7ea338662ceebdc0f9b0039d7df8a6a2778b349629f6
+  metadata.gz: 74fc2c48871a01192e4ebbd80f603a170b1f953f61252f8bf21627bc3abb8555beb12572ab786754f6f015870bdaa6cec54daab23d15068c7aa0152b70c612aa
+  data.tar.gz: 3e6a080c6740075bab1c111127249ade7429ac844fcf1c0b99979c7a535cd5013cf9bd86629957ee7bcd4467828d023831a51df078037dff5cfc777c6a81be9e

data/lib/pagerecognizer.rb CHANGED Viewed

@@ -4,14 +4,16 @@ module PageRecognizer
   end
   require "logger"
   self.logger = Logger.new STDOUT
+  self.logger.formatter = ->(severity, datetime, progname, msg){ "#{datetime.strftime "%H%M%S"} #{severity.to_s[0]} #{msg}\n" }
+  self.logger.level = ENV.fetch("LOGLEVEL_PageRecognizer", "FATAL").to_sym
   module Dumpable
     def dump
-      "<html><body>#{
+      "<html><body style='white-space: nowrap'>#{
         map.with_index do |n, i|
-          "<div style='position: absolute; background-color: hsla(#{
+          "<div id='#{i}' style='position: absolute; background-color: hsla(#{
             360 * i / size
-          },100%,50%,0.5); top: #{n.top}; left: #{n.left}; width: #{n.width}; height: #{n.height}'>#{
+          },100%,50%,0.5); top: #{n.top}; left: #{n.left}; width: #{n.width}; height: #{n.height}'>#{i} #{
             n.node.tag_name.upcase
           }</div>"
         end.join
@@ -29,92 +31,110 @@ module PageRecognizer
     end.extend Dumpable
   end
-  def recognize
+  def self.rgb2hsv r, g, b   # [<256, <256, <256]
+    # http://stackoverflow.com/q/41926874/322020
+    r, g, b  = [r, g, b].map{ |_| _.fdiv 255 }
+    min, max = [r, g, b].minmax
+    chroma   = max - min
+    [
+      60.0 * ( chroma.zero? ? 0 : case max
+        when r ; (g - b) / chroma
+        when g ; (b - r) / chroma + 2
+        when b ; (r - g) / chroma + 4
+        else 0
+      end % 6 ),
+      chroma.zero? ? 0.0 : chroma / max,
+      max,
+    ]   # [<=360, <=1, <=1]
+  end
+  def self.dist h1, s1, v1, h2, s2, v2   # [<256, <256, <256]
+    # https://en.wikipedia.org/wiki/HSL_and_HSV#/media/File:Hsl-hsv_saturation-lightness_slices.svg
+    c1, c2 = s1 * v1 / 256.0, s2 * v2 / 256.0   # chroma
+    z1, z2 = v1 * (2 - c1 / 256), v2 * (2 - c2 / 256)
+    a = (((h2 - h1) * 360 / 256.0) % 360) / (180 / Math::PI)
+        x2 =     Math::sin(a) * c2
+    y1, y2 = c1, Math::cos(a) * c2
+    x2*x2 + (y1-y2)*(y1-y2) + (z1-z2)*(z1-z2)
+  end
+  private def recognize
     logger = Module.nesting.first.logger
+    logger.info "method #{__method__}..."
     nodes = []
     try = lambda do
-      prev = nodes
-      code = "( function(node) {
-        var x = scrollX, y = scrollY;
-        var _tap = function(x, f){ f(); return x };
-        var f = function(node) {
-          node.scrollIntoView();
-          var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
-          var child_nodes = Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 });
-          var clickable;
-          if (node.nodeName == 'svg') {
-            var states = child_nodes.map( function(n){
-              return _tap(n.style ? n.style.display : '', function(){ n.style.display = 'none' } );
-            } );
-            clickable = (node === document.elementFromPoint(rect.x + rect.width/2, rect.y + rect.height/2));
-            var _zip = function(a, b){ return a.map( function(e, i) { return [e, b[i]] } ) };
-            _zip(child_nodes, states).forEach( function(_){ _[0].style.display = _[1] } );
-          } else {
-            clickable = (node === document.elementFromPoint(rect.x + rect.width/2, rect.y + rect.height/2));
-          };
-          rect.top += scrollY;
-          rect.left += scrollX;
-          return [ [
-            rect.top, rect.left, rect.width, rect.height, clickable, node
-          ] ].concat(node.nodeName == 'svg' ? [] : child_nodes.flatMap(f));
-        };
-        return _tap(f(node), function(){ scrollTo(x, y) });
-      } )(arguments[0])"
-      str = Struct.new :top, :left, :width, :height, :clickable, :node
-      nodes = page.evaluate(code, self).map{ |s| str.new *s }
-      nodes.size == prev.size
-    end
+      str = Struct.new :node, :visible, :top, :left, :width, :height, :area do
+        def texts
+          node.page.evaluate(<<~HEREDOC, node).map(&JSON.method(:load)).map do |text, rect1, rect2, style|
+            (function(node){
+              let result = [], range = document.createRange();
+              for (
+                let iterator = document.evaluate('.//text()', node, null, XPathResult.ANY_TYPE, null);
+                text = iterator.iterateNext();
+              ) {
+                range.selectNode(text);
+                result.push(JSON.stringify( [
+                  text.wholeText,
+                  range.getBoundingClientRect(),
+                  text.parentNode.getBoundingClientRect(),
+                  getComputedStyle(text.parentNode),
+                ] ));
+              }
+              return result;
+            })(arguments[0])
+          HEREDOC
-    if defined? Selenium::WebDriver::Wait
-      Selenium::WebDriver::Wait.new(
-        message: "number of DOM elements didn't stop to change"
-      ).until &try
-    else
-      t = Time.now
-      until try.call
-        fail "number of DOM elements didn't stop to change" if Time.now > t + 5
-      end
-    end
-    logger.info "#{nodes.size} DOM nodes found"
+            # google SERP has 1x1 nodes with text _<>
+            next if rect1["width"] < 2 || rect1["height"] < 2
+            next if rect2["width"] < 2 || rect2["height"] < 2
-    nodes.select! &:clickable
-    nodes.reject do |n|
-      nodes.any? do |nn|
-        cs = [
-          nn.top <=> n.top,
-          nn.left <=> n.left,
-          n.left + n.width <=> nn.left + nn.width,
-          n.top + n.height <=> nn.top + nn.height,
-        ]
-        cs.include?(1) && !cs.include?(-1)
+            color = style["color"]
+            fail color unless /\Argba?\((?<red>\d+), (?<green>\d+), (?<blue>\d+)(, 0(\.\d+)?)?\)\z/ =~ color
+            closest_color = {   # https://en.wikipedia.org/wiki/Web_colors#Basic_colors
+              white: [0, 0, 100],
+              silver: [0, 0, 75],
+              gray: [0, 0, 50],
+              black: [0, 0, 0],
+              red: [0, 100, 100],
+              maroon: [0, 100, 50],
+              yellow: [60, 100, 100],
+              olive: [60, 100, 50],
+              lime: [120, 100, 100],
+              green: [120, 100, 50],
+              aqua: [180, 100, 100],
+              teal: [180, 100, 50],
+              blue: [240, 100, 100],
+              navy: [240, 100, 50],
+              fuchsia: [300, 100, 100],
+              purple: [300, 100, 50],
+            }.to_a.min_by do |_, (h1, s1, v1)|
+              h2, s2, v2 = PageRecognizer.rgb2hsv(red.to_i, green.to_i, blue.to_i)
+              PageRecognizer.dist h1*255/360, s1*256/100, v1*256/100, h2*255/360, s2*255, v2*255
+            end.first
+            [text, style, closest_color, rect1]
+          end.compact
+        end
       end
-    end.extend Dumpable
-  end
-  private def recognize_more
-    logger = Module.nesting.first.logger
-    nodes = []
-    try = lambda do
-      prev = nodes
-      code = "( function(node) {
-        var x = scrollX, y = scrollY;
-        var _tap = function(x, f){ f(); return x };
-        var f = function(node) {
-          node.scrollIntoView();
-          var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
-          rect.top += scrollY;
-          rect.left += scrollX;
-          return [ [
-            node, JSON.stringify([rect.top, rect.left, rect.width, rect.height])
-          ] ].concat(Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 }).flatMap(f));
-        };
-        return _tap(f(node), function(){ scrollTo(x, y) });
-      } )(arguments[0])"
-      str = Struct.new :node, :top, :left, :width, :height
-      nodes = page.evaluate(code, self).map{ |node, a| str.new node, *JSON.load(a) }
-      nodes.size == prev.size
+      prev = nodes.size
+      t = page.evaluate(<<~HEREDOC, self)
+        ( function(node) {
+          var x = scrollX, y = scrollY;
+          var _tap = function(x, f){ f(); return x };
+          var f = function(node) {
+            node.scrollIntoView();
+            var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
+            rect.top += scrollY;
+            rect.left += scrollX;
+            return [
+              node, JSON.stringify([rect.top, rect.left, rect.width, rect.height]), ("visible" == getComputedStyle(node).visibility)
+            ].concat(Array.from(node.childNodes).filter(function(node) { return node.nodeType == 1 }).flatMap(f));
+          };
+          return _tap(f(node), function(){ scrollTo(x, y) });
+        } )(arguments[0])
+      HEREDOC
+      logger.debug [t.size / 3, prev]
+      nodes = t.each_slice(3).map{ |node, rect, visible| str.new(node, visible, *JSON.load(rect)).tap{ |_| _.area = _.width * _.height } }
+      nodes.size == prev
     end
     if defined? Selenium::WebDriver::Wait
@@ -128,9 +148,9 @@ module PageRecognizer
       end
     end
     logger.info "#{nodes.size} DOM nodes found"
-    nodes.reject!{ |i| i.height.zero? || i.width.zero? }
-    nodes
+    nodes.reject!{ |_| _.height.zero? || _.width.zero? || !_.visible }
+    logger.info "visible nodes: #{nodes.size}"
+    nodes.extend Dumpable
   end
   logging_error = Class.new RuntimeError do
@@ -143,8 +163,9 @@ module PageRecognizer
   end
   class ErrorNotEnoughNodes < logging_error ; end
-  private def split heuristics, hh, ww, tt, ll
+  private def split hh, ww, tt, ll, heuristics, try_min, dump, &filter
     logger = Module.nesting.first.logger
+    logger.info heuristics
     unstale = unless defined? Selenium::WebDriver::Error::StaleElementReferenceError
       ->(&b){ b.call }
@@ -159,82 +180,253 @@ module PageRecognizer
         end
       end
     end
-    all = unstale.call do recognize_more end.sort_by(&tt)
-    logger.info "all nodes: #{all.size}"
-    rect = page.evaluate("( function(node) { return JSON.parse(JSON.stringify(node.getBoundingClientRect())) } )(arguments[0])", self)
-    inside = all.reject{ |i| i.left < rect["left"] || i.left + i.width > rect["right"] || i.top < rect["top"] || i.top + i.height > rect["bottom"] }
-    raise ErrorNotEnoughNodes.new "no inside nodes", all: all, inside: inside if inside.empty?
-    logger.info "inside nodes: #{inside.size}"
-    nodes = unstale.call do inside.reject{ |i| %w{ button script svg path a img span }.include? i.node.tag_name } end.uniq{ |i| [i[hh], i[ww], i[tt], i[ll]] }
-    logger.info "good nodes: #{nodes.size}"   # only those that might be containers
-    large = nodes#.select{ |i| i[ww] > nodes.map(&ww).max / 4 }
-    logger.info "large enough and unique: #{large.size}"
+    nodes = unstale.call do recognize end.sort_by{ |_| [_[tt], _[ll]] }
+    File.write "#{dump}.all.htm", nodes.extend(Dumpable).dump if dump
+    nodes = unstale.call do nodes.reject{ |i| %w{ button script svg path a img }.include? i.node.tag_name } end.uniq{ |_| [_[hh], _[ww], _[tt], _[ll]] }
+    logger.info "good and unique: #{nodes.size}"   # only those that might be containers
+    File.write "#{dump}.nodes.htm", nodes.extend(Dumpable).dump if dump
     interfere = lambda do |a, b|
       a[tt] < b[tt] + b[hh] &&
       b[tt] < a[tt] + a[hh]
     end
-    rest = large.select.with_index do |a, i|
-      large.each_with_index.none? do |b, j|
+    rest = nodes.select.with_index do |a, i|
+      nodes.each_with_index.none? do |b, j|
         next if i == j
         a[tt] >= b[tt] && a[tt] + a[hh] <= b[tt] + b[hh] &&
-        large.all?{ |c| interfere[a, c] == interfere[b, c] }
+        a[ll] >= b[ll] && a[ll] + a[ww] <= b[ll] + b[ww] &&
+        nodes.all?{ |c| interfere[a, c] == interfere[b, c] }
       end
     end
     logger.info "not nested: #{rest.size}"
-    # rest = rest.sample 50
+    File.write "#{dump}.rest1.htm", rest.extend(Dumpable).dump if dump
+    # 8 = max_results - 1, 3 = (from row size diff euristic)
+    if try_min
+      rest = rest.reject{ |_| _[hh] + _[hh]/3*(try_min - 1) > (rest.map{ |_| _[tt] + _[hh] }.max - rest.map(&tt).min) }
+      logger.info "small enough: #{rest.size}"
+    end
+    File.write "#{dump}.rest2.htm", rest.extend(Dumpable).dump if dump
-    # adding the :area field for faster upcoming computations
-    struct = Struct.new *large.first.members, :area
-    rest.map!{ |i| struct.new *i.values, i.width * i.height }
+    rest.select! &filter
+    logger.info "filtered: #{rest.size}"
+    File.write "#{dump}.filtered.htm", rest.extend(Dumpable).dump if dump
+    rest.sort_by!(&:area).reverse!
+    File.write "#{dump}.sorted.htm", rest.extend(Dumpable).dump if dump
     require "pcbr"
     pcbr = PCBR.new
     is = []
-    max, past = 0, []
+    max, past = 0, Set.new
     prev = nil
     time = Time.now
     loop do
-      rest.each_with_index do |node, i|
-        next if is.any?{ |j| i == j || interfere[rest[i], rest[j]] }
+      si = (0...rest.size).reject do |i|
+        # I don't shrink pcbr so this should be a safe optimization
+        next true if is.last > i unless is.empty?
+        # also we've sorted from large to small so it does not get stuck with the half of the page below the largest node
+        next (logger.debug [i, 2]; true) if is.any?{ |j| i == j || interfere[rest[i], rest[j]] }
+        next (logger.debug [i, 3]; true) if is.any?{ |j| rest[i][ww] > rest[j][ww] * 2 } if heuristics.include? :WIDTH
+        next (logger.debug [i, 4]; true) if is.any?{ |j| rest[j][ww] > rest[i][ww] * 2 } if heuristics.include? :WIDTH
+        next (logger.debug [i, 5]; true) if is.any?{ |j| rest[i][hh] > rest[j][hh] * 3 }
+        next (logger.debug [i, 6]; true) if is.any?{ |j| rest[j][hh] > rest[i][hh] * 3 }
+      end
+      logger.debug [is, si]
+      si.each do |i|
         sol = rest.values_at *is, i
+        unless pcbr.set.include? [*is, i].sort
+        logger.debug [is, i, sol.map(&:area).reduce(:+)]
         pcbr.store [*is, i].sort, [
           *( is.size                                                                                                                if heuristics.include? :SIZE   ),
-          *( sol.map(&:area).inject(:+)                                                                                             if heuristics.include? :AREA   ),
-          *( -sol.product(sol).map{ |s1, s2| (s1.width              - s2.width             ).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :WIDTH  ),
-          *( -sol.product(sol).map{ |s1, s2| (s1.height             - s2.height            ).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :HEIGHT ),
-          *( -sol.product(sol).map{ |s1, s2| (s1[ll] + s1[ww] / 2.0 - s2[ll] - s2[ww] / 2.0).abs }.inject(:+) / sol.size / sol.size if heuristics.include? :MIDDLE ),
-        ] unless pcbr.table.assoc [*is, i].sort
+          *( sol.map(&:area).reduce(:+)                                                                                             if heuristics.include? :AREA   ),
+          # https://en.wikipedia.org/wiki/Mean_absolute_difference
+          *( -sol.product(sol).map{ |s1, s2| (s1.height             - s2.height            ).abs }.reduce(:+) / sol.size / sol.size if heuristics.include? :HEIGHT ),
+          *( -sol.product(sol).map{ |s1, s2| (s1[ll] + s1[ww] / 2.0 - s2[ll] - s2[ww] / 2.0).abs }.reduce(:+) / sol.size / sol.size if heuristics.include? :MIDDLE ),
+        ]
+          logger.debug "pcbr.table.size: #{pcbr.table.size}"
+          if si.none? do |j|
+            next if j <= i
+            next true if interfere[rest[i], rest[j]]
+            next true if rest[i][ww] > rest[j][ww] * 2 if heuristics.include? :WIDTH
+            next true if rest[j][ww] > rest[i][ww] * 2 if heuristics.include? :WIDTH
+            next true if rest[i][hh] > rest[j][hh] * 3
+            next true if rest[j][hh] > rest[i][hh] * 3
+          end
+            logger.debug "forced"
+            break
+          end
+        end
       end
-      if prev && Time.now - time > 1 && (Time.now - prev > (prev - time))
-        m = pcbr.table.reject{ |i| i.first.size == 1 }.map(&:last).max
-        break if 1 == pcbr.table.count{ |i| i.last == m } || Time.now - time > 5
+      if prev && Time.now - time > 5
+        logger.debug "check"
+        break logger.info "break 0" if Time.now - time > 30
+        break logger.info "break 1" if Time.now - prev > 10
+        m = pcbr.table.reject{ |i| i.first.size < 2 }.map(&:last).max
+        break logger.info "break 2" if Time.now - prev > (prev - time) && 1 == pcbr.table.count{ |i| i.last == m }
       end
-      break unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.inject(:+) }.max_by(&:last)
+      break logger.info "done" unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.reduce(:+) }.max_by(&:last)
+      logger.debug "next: #{t}"
+      past.add (is = t.first).map{ |i| 2**i }.reduce(:+)
       if t.last > max
         prev, max = Time.now, t.last
+        logger.debug "new max: #{max}"
         logger.debug [Time.now - time, max, t.first]
       end
-      past.push (is = t.first).map{ |i| 2**i }.inject(:+)
     end
     # TODO: if multiple with max score, take the max by area
-    unless best = pcbr.table.reject{ |is,| is.size == 1 }.max_by(&:last)
-      raise ErrorNotEnoughNodes.new "failed to split <#{tag_name}>", all: all, inside: inside, nodes: nodes, large: large, rest: rest
+    unless best = pcbr.table.reject{ |is,| is.size < 2 }.max_by(&:last)
+      raise ErrorNotEnoughNodes.new "failed to split <#{tag_name}>", all: all, nodes: nodes, rest: rest
     end
-    rest.values_at(*best.first).extend(Dumpable)
+    pcbr.table.max_by(20, &:last).each_with_index{ |_, i| logger.debug "##{i} #{_}" }
+    logger.info best
+    logger.info "splitted in #{best.first.size}"
+    rest.values_at(*best.first).sort_by(&tt).extend Dumpable
   end
-  def rows *heuristics
-    heuristics = %i{ AREA HEIGHT WIDTH } if heuristics.empty?
-    split heuristics, :height, :width, :top, :left
+  def rows heuristics, try_min: nil, dump: nil, &b
+    split :height, :width, :top, :left, heuristics, try_min, dump, &b
   end
-  def cols *heuristics
-    heuristics = %i{ AREA HEIGHT WIDTH } if heuristics.empty?
-    split heuristics, :width, :height, :left, :top
+  def cols heuristics, try_min: nil, dump: nil, &b
+    split :width, :height, :left, :top, heuristics, try_min, dump, &b
   end
+  def self.piles z
+    max = nil
+    result = [current = []]
+    z.map.with_index.sort.each do |x|
+      if !max || max > x[0][0]
+        current.push x
+        max = x[0][0] + x[0][1] if !max || max < x[0][0] + x[0][1]
+      else
+        result.push current = [x]
+        max = x[0][0] + x[0][1]
+      end
+    end
+    result.map{ |_| _.map &:last }
+  end
+  module Gridable
+    def rows
+      Module.nesting[1].piles(map{ |n| [n.top, n.height] }).map{ |s| values_at(*s).extend Module.nesting[1]::Dumpable }
+    end
+    def cols
+      Module.nesting[1].piles(map{ |n| [n.left, n.width] }).map{ |s| values_at(*s).extend Module.nesting[1]::Dumpable }
+    end
+  end
+  def grid dump = nil
+    logger = Module.nesting.first.logger
+    all = recognize
+    logger.info "all nodes: #{all.size}"
+    File.write "#{dump}.all.htm", all.extend(Dumpable).dump if dump
+    # adding the fields for faster upcoming computations
+    struct = Struct.new *all.first.members, :midx, :midy
+    all.map!{ |i| struct.new *i.values, i.left + i.width / 2.0, i.top * i.height / 2.0 }
+    all = all.sort_by{ |_| [_.area, _.top, _.left] }.reverse
+    rect = page.evaluate("( function(node) { return JSON.parse(JSON.stringify(node.getBoundingClientRect())) } )(arguments[0])", self)
+    inside = all.reject{ |i| i.left < rect["left"] || i.left + i.width > rect["right"] || i.top < rect["top"] || i.top + i.height > rect["bottom"] }
+    raise ErrorNotEnoughNodes.new "no inside nodes", all: all, inside: inside if inside.empty?
+    logger.info "inside nodes: #{inside.size}"
+    File.write "#{dump}.inside.htm", inside.extend(Dumpable).dump if dump
+    good = inside.reject{ |i| %w{ button script svg path a img }.include? i.node.tag_name }.uniq{ |i| [i.height, i.width, i.top, i.left] }
+    logger.info "good and unique: #{good.size}"   # only those that might be containers
+    File.write "#{dump}.good.htm", good.extend(Dumpable).dump if dump
+    # large = good#.select{ |i| i[ww] > good.map(&ww).max / 4 }
+    # logger.info "large enough: #{large.size}"
+    interfere = lambda do |a, b|
+      a.top < b.top + b.height &&
+      b.top < a.top + a.height &&
+      a.left < b.left + b.width &&
+      b.left < a.left + a.width
+    end
+    rest = good.select.with_index do |a, i|
+      good.each_with_index.none? do |b, j|
+        next if i == j
+        a.top >= b.top && a.top + a.height <= b.top + b.height &&
+        a.left >= b.left && a.left + a.width <= b.left + b.width &&
+        good.all?{ |c| interfere[a, c] == interfere[b, c] }
+      end
+    end
+    logger.info "not nested: #{rest.size}"
+    File.write "#{dump}.rest.htm", rest.extend(Dumpable).dump if dump
+    begin
+      prev = rest.size
+      rest.select!.with_index do |a, i|
+        rest.each_with_index.any? do |b, j|
+          cw = [[a.left + a.width, b.left + b.width].min - [a.left, b.left].max, 0].max
+          i != j && !interfere[a, b] && [cw, a.width].min.fdiv(a.width) * [cw, b.width].min.fdiv(b.width) > 0.9
+        end and
+        rest.each_with_index.any? do |b, j|
+          ch = [[a.top + a.height, b.top + b.height].min - [a.top, b.top].max, 0].max
+          i != j && !interfere[a, b] && [ch, a.height].min.fdiv(a.height) * [ch, b.height].min.fdiv(b.height) > 0.9
+        end
+      end
+    end until prev == rest.size
+    logger.info "gridable: #{rest.size}"
+    File.write "#{dump}.griddable.htm", rest.extend(Dumpable).dump if dump
+    require "pcbr"
+    pcbr = PCBR.new
+    max, past = 0, []
+    prev = nil
+    prev_max = nil
+    time = Time.now
+    heuristics = %i{ SIZE AREA }
+    inter = lambda do |a1, a2, b1, b2|
+      c = [[a1 + a2, b1 + b2].min - [a1, b1].max, 0].max
+      [c, a2].min.fdiv(a2) * [c, b2].min.fdiv(b2)
+    end
+    lp = lambda do |is|
+      past.push is.map{ |i| 2**i }.reduce(:+)
+      rest.size.times do |ij|
+        next if ij <= is.last unless is.empty?
+        sorted = is + [ij]
+        next if pcbr.set.include? sorted
+        next if is.any?{ |j| interfere[rest[ij], rest[j]] }
+        sol = rest.values_at *sorted
+        xn = Module.nesting.first.piles sol.map{ |s| [s.left, s.width] }
+        yn = Module.nesting.first.piles sol.map{ |s| [s.top, s.height] }
+        next if xn.product(yn).any?{ |i,j| (i & j).size > 1 } if sorted.size >= 4
+        pcbr.store sorted, [
+          *( sol.map(&:area).reduce(:+) if heuristics.include? :AREA ),
+          xn.map{ |g| sosol = sol.values_at *g; next 0 if sosol.size == 1; sosol.combination(2).map{ |s1, s2| inter[s1.left, s1.width, s2.left, s2.width] }.reduce(:+) / sosol.size / (sosol.size - 1) * 2 }.reduce(:+) / xn.size,
+          yn.map{ |g| sosol = sol.values_at *g; next 0 if sosol.size == 1; sosol.combination(2).map{ |s1, s2| inter[s1.top, s1.height, s2.top, s2.height] }.reduce(:+) / sosol.size / (sosol.size - 1) * 2 }.reduce(:+) / yn.size,
+        ]
+        if prev && Time.now - time > 3
+          logger.debug "check"
+          break logger.info "break 0" if Time.now - time > 30
+          break logger.info "break 1" if Time.now - prev > 10
+          m = pcbr.table.reject{ |i| i.first.size < 3 }.map(&:last).max
+          break logger.debug "break 2" if Time.now - prev > (prev - time) * 2 && 1 == pcbr.table.count{ |i| i.last == m }
+        end
+        break logger.info "break 3" unless t = pcbr.table.reject{ |is,| past.include? is.map{ |i| 2**i }.reduce(:+) }.max_by(&:last)
+        logger.debug [t.last, max, t.first == prev_max, t.first.map{ |i| 2**i }.reduce(:+)]
+        if t.last > max && t.first != prev_max
+          prev, max, prev_max = Time.now, t.last, t.first
+          logger.debug [pcbr.table.size, max, t.first]
+        end
+        lp.call t.first
+      end
+    end
+    lp.call []
+    # TODO: if multiple with max score, take the max by area
+    pcbr.table.max_by(20, &:last).each_with_index{ |_, i| logger.debug "##{i} #{_}" }
+    rest.values_at(*pcbr.table.max_by(&:last).first).extend Dumpable, Gridable
+  end
 end

data/pagerecognizer.gemspec CHANGED Viewed

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "pagerecognizer"
-  spec.version      = "0.0.1"
+  spec.version      = "0.1.0"
   spec.summary      = "visual HTML page structure recognizer"
   spec.author       = "Victor Maslov aka Nakilon"
@@ -8,16 +8,15 @@ Gem::Specification.new do |spec|
   spec.license      = "MIT"
   spec.metadata     = {"source_code_uri" => "https://github.com/nakilon/pagerecognizer"}
-  spec.add_dependency "nokogiri"
-  spec.add_dependency "pcbr"
   spec.add_dependency "ferrum"
+  spec.add_dependency "nokogiri"
+  spec.add_dependency "pcbr", "~>0.4.2"
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "ruby-prof"
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "mll"
-  spec.require_path = "lib"
   spec.test_file    = "test.rb"
   spec.files        = %w{ LICENSE pagerecognizer.gemspec lib/pagerecognizer.rb }
 end

data/test.rb CHANGED Viewed

@@ -1,28 +1,72 @@
 require "minitest/autorun"
 require "ferrum"
 require_relative "lib/pagerecognizer"
-Ferrum::Node.include PageRecognizer
+PageRecognizer.logger.level = :INFO
 describe PageRecognizer do
-  it "google" do
-    browser = Ferrum::Browser.new **(ENV.has_key?("FERRUM_NO_SANDBOX") ? {browser_options: {"no-sandbox": nil}} : {})
-    browser.goto "about:blank"
-    browser.execute "document.write(#{File.read("google.htm").inspect})"
-    results = browser.at_css("body").rows
-    width = results.group_by(&:width).max_by{ |w, g| g.size }.first
-    assert_equal [
-      ["https://www.ruby-lang.org/ru/", "Ruby это... динамический язык программирования с о"],
-      ["https://ru.wikibooks.org/wiki/Ruby", "Этот учебник намерен осветить все тонкости програм"],
-      ["https://habr.com/ru/post/433672/", "19 дек. 2018 г. - Взрывной рост интереса к Ruby ос"],
-      ["https://habr.com/ru/hub/ruby/", "Ruby (англ. Ruby — «Рубин») — динамический, рефлек"],
-      ["https://web-creator.ru/articles/ruby", "Ruby разрабатывался на Linux, но работает на многи"],
-      ["http://rusrails.ru/", "Ruby on Rails руководства, учебники, статьи на рус"],
-      ["https://vc.ru/dev/72391-pochemu-my-vybiraem-ruby-d", "20 июн. 2019 г. - Ruby on Rails одним из первых на"],
-      ["https://tproger.ru/tag/ruby/", "Django или Ruby on Rails: какой фреймворк выбрать?"],
-      ["https://rubyrussia.club/", "Главная российская конференция о Ruby. Расширяем г"]
-    ], results.select{ |r| r.width == width }.map(&:node).map(&:rows).map{ |link, desc| [
-      link.node.at_css("a").property("href")[0,50],
-      desc.node.text[0,50],
+  before do
+    options = {}
+    options[:browser_options] = {"no-sandbox": nil} if ENV.has_key? "FERRUM_NO_SANDBOX"
+    options[:headless] = false if ENV.has_key? "HEADFULL"
+    @browser = Ferrum::Browser.new **options
+  end
+  after do
+    @browser&.quit
+  end
+  [
+      ["google1.htm", [
+        ["https://ru.wikipedia.org/wiki/Ruby#:~:te", "Ruby — Википедия"],
+        ["https://www.ruby-lang.org/ru/", "Язык программирования Ruby"],
+        ["https://ru.wikibooks.org/wiki/Ruby", "Ruby — Викиучебник"],
+        ["https://habr.com/ru/post/433672/", "Пацаны, так Ruby умер или нет? / Хабр - Habr"],
+        ["https://habr.com/ru/hub/ruby/", "Ruby – Динамический высокоуровневый язык..."],
+        ["https://web-creator.ru/articles/ruby", "Язык программирования Ruby - Веб Креатор"],
+        ["http://rusrails.ru/", "Rusrails: Ruby on Rails по-русски"],
+        ["https://vc.ru/dev/72391-pochemu-my-vybir", "Почему мы выбираем Ruby для наших проектов..."],
+        ["https://tproger.ru/tag/ruby/", "Ruby — всё по этой теме для программистов..."],
+        ["https://rubyrussia.club/", "RubyRussia"],
+      ] ],
+      ["google2.mht", [
+        ["https://www.ruby-lang.org/ru/", "Язык программирования Ruby"],
+        ["https://ru.wikipedia.org/wiki/Ruby", "Ruby - Википедия"],
+        ["https://evrone.ru/why-ruby", "5 причин, почему мы выбираем Ruby - evrone.ru"],
+        ["https://habr.com/ru/hub/ruby/", "Ruby — Динамический высокоуровневый язык..."],
+        ["https://ru.wikibooks.org/wiki/Ruby", "Ruby - Викиучебник"],
+        ["https://context.reverso.net/%D0%BF%D0%B5", "ruby - Перевод на русский - примеры английский..."],
+        ["https://web-creator.ru/articles/ruby", "Язык программирования Ruby - Веб Креатор"],
+        ["https://ru.hexlet.io/courses/ruby", "Введение в Ruby - Хекслет"],
+        ["https://rubyrush.ru/articles/what-is-rub", "Что такое Ruby on Rails?"],
+      ] ],
+  ].each do |filename, expectation|
+    it "google rows #{filename}" do
+    @browser.goto "file://#{File.expand_path filename}"
+    results = @browser.at_css("body").rows([:AREA, :SIZE], try_min: 9) do |node|
+      texts = node.texts
+      next if texts.none?{ |_, _, color, | :black == color }
+      _, group = texts.group_by{ |_, style, | style["fontSize"].to_i }.to_a.max_by(&:first)
+      next unless group
+      next unless group.size == 1 && %i{ blue navy }.include?(group[0][2])
+      true
+    end
+    assert_equal expectation, results.reject{ |_| _.node.at_css "img" }.map{ |result| [
+      result.node.at_css("a").property("href")[0,40],
+      result.texts.max_by{ |_, style, | style["fontStyle"].to_i }[0].sub(/(.{40}) .+/, "\\1..."),
     ] }
+    end
+  end
+  [
+      ["youtube.htm", %w{ Главная В\ тренде Подписки Библиотека История }, 8],
+      ["youtube2.mht", %w{ Главная Навигатор Shorts Подписки Библиотека История }, 10],
+  ].each do |filename, expected_navigation, rows|
+    it "youtube rows grid #{filename}" do
+      @browser.goto "file://#{File.expand_path filename}"
+      assert_equal expected_navigation, @browser.at_css("ytd-mini-guide-renderer").rows([:AREA, :SIZE]){ |_| !_.node.text.strip.empty? }.map{ |nav| nav.texts.first[0] }
+      grid = @browser.at_css("#content").grid
+      assert_equal 3*rows, grid.size
+      assert_equal [3]*rows, grid.rows.map(&:size)
+      assert_equal [rows]*3, grid.cols.map(&:size)
+      grid.each{ |n| n.to_h.values_at(:width, :height).each{ |_| assert_in_delta 250, _, 50 } }
+    end
   end
 end

metadata CHANGED Viewed

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: pagerecognizer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-24 00:00:00.000000000 Z
+date: 2022-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: ferrum
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pcbr
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,19 +39,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ferrum
+  name: pcbr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.4.2
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement