ferrum_common 0.0.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e8ef82a6987019ce9a01dffedf29704905fc53d7f3534cc1654b6a9a7c9220d
-  data.tar.gz: b9d2a38f62cc43372635cfcf679f95868c735b5ad8fbf2d482ac9b640df2e06a
+  metadata.gz: e128a92c5b106f583d12cf8cc772b95637272dae11711e1095916e8485ccd9c5
+  data.tar.gz: c52cb59ea98be841986e6930e6a21456636491684cbe2d812f474dff996bb105
 SHA512:
-  metadata.gz: 4a7044da6e7ec7c03d56d5b1b2e99f72988476cd93323af8fc9601db4fe9161b534dd8188f9ea57548676776ab05394f5d5c60d7a43fed6bc26ecbd21a93eb2e
-  data.tar.gz: 474977a1cb5ec61a2f1ac62a47d275e34e803754c3554a8f5393c7634e0fe23abbf02fae4c7b1b9d9f48763266868fd00ec8e3df139a0ef3297dbc1a0bbeaaba
+  metadata.gz: aee22d614c6b08f217fcf0786083ef0b7a4aa0c154f79948916fe3433796c1255168df736ae6fdac82e3e0fd4d91ce309aaddadd0fc80e93087856e4cc355789
+  data.tar.gz: 6a2939158a8d3dc9da7938d63cad598fb005c88ef0307418ea9dd8a6f5eb3a8af34a1ad5c59257e23d7a99150a7c627520f9d26e1abe9c69ca4c3a0292f52546

data/ferrum_common.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "ferrum_common"
-  spec.version      = "0.0.0"
+  spec.version      = "0.2.0"
   spec.summary      = "[WIP] common useful extensions for ferrum or cuprite"
 
   spec.author       = "Victor Maslov aka Nakilon"
@@ -9,7 +9,9 @@ Gem::Specification.new do |spec|
   spec.metadata     = {"source_code_uri" => "https://github.com/nakilon/ferrum_common"}
 
   spec.add_dependency "ferrum"
-  spec.required_ruby_version = ">=2.5"
+  spec.add_dependency "browser_reposition"
+  spec.add_dependency "nokogiri"
+  spec.required_ruby_version = ">=2.5"  # why?
 
   spec.files        = %w{ LICENSE ferrum_common.gemspec lib/ferrum_common.rb }
 end

data/lib/ferrum_common.rb

@@ -1,28 +1,168 @@
 require "ferrum"
 module FerrumCommon
-  def self.new **_
-    Ferrum::Browser.new(**_).tap do |browser|
-      require "browser_reposition"
-      browser.extend(BrowserReposition).reposition
-      browser.define_singleton_method :redo_until_true do |timeout, msg = nil, &block|
-        Timeout.timeout timeout do
-          begin
-            block.call
-          rescue Ferrum::NodeNotFoundError
-            redo
-          end or (sleep timeout*0.1; redo)
+
+  module Common
+
+    private def mhtml browser, timeout, mtd, msg = nil
+      Timeout.timeout(timeout){ yield }
+    rescue Timeout::Error
+      browser.mhtml path: "temp.mhtml"
+      STDERR.puts "dumped to ./temp.mhtml"
+      $!.backtrace.reject!{ |_| _[/\/gems\/concurrent-ruby-/] }
+      $!.backtrace.reject!{ |_| _[/\/gems\/ferrum-/] }
+      raise Timeout::Error, "#{$!.to_s} after #{timeout} sec in #{mtd}#{" (#{msg.respond_to?(:call) ? msg.call : msg})" if msg}"
+    end
+
+    def until_true timeout, msg = nil
+      mhtml self, timeout, __method__, msg do
+        begin
+          yield
+        rescue Ferrum::NodeNotFoundError
+          redo
+        end or (sleep timeout*0.1; redo)
+      end
+    end
+
+    def until_one type, selector, timeout
+      t = nil
+      mhtml self, timeout, __method__, ->{ "expected exactly one node for #{type} #{selector.inspect}, got #{t ? t.size : "none"}" } do
+        t = begin
+          public_method(type).call selector
+        end
+        unless 1 == t.size
+          sleep timeout * 0.1
+          redo
         end
-      rescue Timeout::Error
-        browser.mhtml path: "temp.mhtml"
-        $!.backtrace.reject!{ |_| _[/\/gems\/concurrent-ruby-/] }
-        $!.backtrace.reject!{ |_| _[/\/gems\/ferrum-/] }
-        raise Timeout::Error, "#{$!.to_s} in redo_until_true #{" (#{msg})" if msg}"
       end
-      browser.define_singleton_method :abort do |msg|
-        Browser.mhtml path: "temp.mhtml"
-        puts Thread.current.backtrace
-        abort msg
+      t.first
+    end
+
+    def abort msg_or_cause
+      # puts (msg_or_cause.respond_to?(:backtrace) ? msg_or_cause : Thread.current).backtrace
+      puts (msg_or_cause.respond_to?(:full_message) ? msg_or_cause.full_message : Thread.current.backtrace)
+      mhtml path: "temp.mhtml"
+      STDERR.puts "dumped to ./temp.mhtml"
+      Kernel.abort msg_or_cause.to_s
+    end
+
+  end
+  Ferrum::Page.include Common
+  Ferrum::Frame.include Common
+
+  if "darwin" == Gem::Platform.local.os
+    require "browser_reposition"
+    Ferrum::Browser.include Common, BrowserReposition
+    def self.new **_
+      Ferrum::Browser.new(**_).tap(&:reposition)
+    end
+  else
+    Ferrum::Browser.include Common
+    def self.new **_
+      Ferrum::Browser.new **_
+    end
+  end
+
+  # https://datatracker.ietf.org/doc/html/rfc2557
+  # https://en.wikipedia.org/wiki/Quoted-printable
+  # require "strscan"
+  require "nokogiri"  # Oga crashes on vk charset
+  def self.process_mhtml mht
+    scanner = ::StringScanner.new mht
+    fail scanner.peek(100).inspect unless scanner.scan(/\AFrom: <Saved by Blink>\r
+Snapshot-Content-Location: \S+\r
+Subject:(?: \S+\r\n)+Date: [A-Z][a-z][a-z], \d\d? [A-Z][a-z][a-z] 20\d\d \d\d:\d\d:\d\d -0000\r
+MIME-Version: 1\.0\r
+Content-Type: multipart\/related;\r
+\ttype="text\/html";\r
+\tboundary="(----MultipartBoundary--[a-zA-Z0-9]{42}----)"\r\n\r\n\r\n--\1/)
+    delimeter = scanner[1]
+    fail unless scanner.charpos == prev = scanner.pos
+    reps = []
+    while s = scanner.search_full(::Regexp.new(delimeter), true, true)
+      doc = s[0...-delimeter.size-4]
+      case doc
+      when /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: chrome-error:\/\/chromewebdata\/\r\n\r\n/,
+           /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r\n\r\n/
+        puts "trash #{$'.size}"
+        reps.push [prev-delimeter.size-2, scanner.pos-delimeter.size-4, "", ""]
+      when /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        puts "html #{$'.size}"
+        header = $&
+        t = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
+        puts "unpacked #{t.size}"
+        html = ::Nokogiri::HTML t#.force_encoding "utf-8"
+
+        puts ".to_s.size #{html.to_s.size}"
+
+        html.xpath("//*[not(*)]").group_by(&:name).
+          map{ |_, g| [_, g.map(&:to_s).map(&:size).reduce(:+)] }.
+          sort_by(&:last).reverse.take(5).each &method(:p)
+
+        if block_given?
+          yield html
+          puts "yielded"
+          puts "yield #{html.to_s.size}"
+        end
+
+        reps.push [prev, scanner.pos-delimeter.size-4, header, html.to_s, true, :html]
+      when /\A\r\nContent-Type: text\/css\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        puts "css > #{$'.size}"
+        header = $&
+        css = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
+        css.gsub!(/[\r\n]+/, "\n")
+
+        puts "css < #{css.size}"
+        reps.push [prev, scanner.pos-delimeter.size-4, header, css, true, :css]
+
+      when /\A\r\nContent-Type: image\/(webp|png|gif|jpeg)\r
+Content-Transfer-Encoding: base64\r
+Content-Location: \S+\r\n\r\n/
+        puts "#{$1} #{$'.size}"
+      when /\A\r\nContent-Type: image\/svg\+xml\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        puts "svg #{$'.size}"
+      else
+        puts doc[0..300]
+        fail
       end
+      fail unless scanner.charpos == prev = scanner.pos
+    end
+
+    p is = reps.map.with_index{ |(_, _, _, _, _, type), i| i if :html == type }.compact
+    p cs = reps.map.with_index{ |(_, _, _, _, _, type), i| i if :css == type }.compact
+    cs.each_cons(2){ |i,j| fail unless i+1==j }
+    fail unless is == [cs[0]-1]
+    File.write "temp.htm", reps[is[0]][3]
+    puts "css > #{File.size "temp.css"}"
+    File.open("temp.css", "w"){ |f| cs.each{ |i| f.puts reps[i][3] } }
+    system "uncss temp.htm -s temp.css -o out.css"
+    puts "css < #{File.size "out.css"}"
+    reps[cs[0]][1] = reps[cs[-1]][1]
+    reps[cs[0]+1..cs[-1]] = []
+    reps[cs[0]][3] = File.read "out.css"
+
+    reps.reverse_each do |from, to, header, str, qp|
+      str = qp ?
+        header + str.gsub("=", "=3D").
+          b.gsub(/[\x80-\xFF]/n){ |_| "=%02X" % _.ord }.
+          gsub(/.{73}[^=][^=](?=.)/, "\\0=\r\n") :
+        header + str.gsub("\n", "\r\n")
+      p [str.size, "to - from = #{to - from}"]
+      mht[from...to] = str
     end
+    p ::File.write "temp.mht", mht
+    puts "OK"
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum_common
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-07 00:00:00.000000000 Z
+date: 2023-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ferrum
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: browser_reposition
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: nakilon@gmail.com
 executables: []