ferrum_common 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 337f36c693834482dea4e00fcb2d5407b2787df08279bdba679b5e22b3482745
-  data.tar.gz: 01bceb7e7fa64fa3082040ecc11b7c83674ace1a2282153132172cc3e6c12a9f
+  metadata.gz: 3f66888a248e7235fdccb3510c7370d708c2052868d828c4553c41b6b0d3411e
+  data.tar.gz: e57b7c549817f6294703ee108d2b54d400d34735fc8cde5a8f63f27546aa9562
 SHA512:
-  metadata.gz: 56f406f33b726a82f4aa2f7e8b1c4e39b53ddc14dedea932f550bb1e4d7586d09d835915fd2a219327c7fdfeae34e245ae4721b9f06eb66a2cbec1b743b1f977
-  data.tar.gz: afbcaa65ed2a25940d44290e3e82f4269eb5b74c6cdc80f6d402da7e7d87ff14fd3f704243d7e080f51c9014e74abba580ecc11df39a4c5a6838e9cf98bf5dd2
+  metadata.gz: b87cac7f5c32144f8955c436f37d1b98b4562a771fd12607bbfee6524c983288b0eba127474fa12ded126bb5abf0673740eda70dc2e0144a7fb8c2c0f24b04f4
+  data.tar.gz: 7731364485b2bf8be92c6d4e4aaa4cbd9465d1cc9386cd0cbfa9d5e24f26f33065f3c68ebbcb55cea6c30fce320b50a719a62b3cbddfce69282952f39ebad6e7

data/ferrum_common.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name         = "ferrum_common"
-  spec.version      = "0.1.0"
+  spec.version      = "0.3.0"
   spec.summary      = "[WIP] common useful extensions for ferrum or cuprite"
 
   spec.author       = "Victor Maslov aka Nakilon"
@@ -10,6 +10,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "ferrum"
   spec.add_dependency "browser_reposition"
+  spec.add_dependency "nokogiri"
   spec.required_ruby_version = ">=2.5"  # why?
 
   spec.files        = %w{ LICENSE ferrum_common.gemspec lib/ferrum_common.rb }

data/lib/ferrum_common.rb

@@ -3,17 +3,18 @@ module FerrumCommon
 
   module Common
 
-    def self.mhtml browser, timeout, mtd, msg = nil
+    private def yield_with_timeout browser, timeout, mtd, msg = nil
       Timeout.timeout(timeout){ yield }
     rescue Timeout::Error
       browser.mhtml path: "temp.mhtml"
+      STDERR.puts "dumped to ./temp.mhtml"
       $!.backtrace.reject!{ |_| _[/\/gems\/concurrent-ruby-/] }
       $!.backtrace.reject!{ |_| _[/\/gems\/ferrum-/] }
       raise Timeout::Error, "#{$!.to_s} after #{timeout} sec in #{mtd}#{" (#{msg.respond_to?(:call) ? msg.call : msg})" if msg}"
     end
 
     def until_true timeout, msg = nil
-      Module.nesting.first.mhtml self, timeout, __method__, msg do
+      yield_with_timeout self, timeout, __method__, msg do
         begin
           yield
         rescue Ferrum::NodeNotFoundError
@@ -24,12 +25,9 @@ module FerrumCommon
 
     def until_one type, selector, timeout
       t = nil
-      Module.nesting.first.mhtml self, timeout, __method__, ->{ "expected exactly one node for #{type} #{selector.inspect}, got #{t ? t.size : "none"}" } do
+      yield_with_timeout self, timeout, __method__, ->{ "expected exactly one node for #{type} #{selector.inspect}, got #{t ? t.size : "none"}" } do
         t = begin
           public_method(type).call selector
-        rescue Ferrum::NodeNotFoundError
-          sleep timeout * 0.1
-          redo
         end
         unless 1 == t.size
           sleep timeout * 0.1
@@ -43,7 +41,7 @@ module FerrumCommon
       # puts (msg_or_cause.respond_to?(:backtrace) ? msg_or_cause : Thread.current).backtrace
       puts (msg_or_cause.respond_to?(:full_message) ? msg_or_cause.full_message : Thread.current.backtrace)
       mhtml path: "temp.mhtml"
-      puts "dumped to ./temp.mhtml"
+      STDERR.puts "dumped to ./temp.mhtml"
       Kernel.abort msg_or_cause.to_s
     end
 
@@ -51,10 +49,122 @@ module FerrumCommon
   Ferrum::Page.include Common
   Ferrum::Frame.include Common
 
-  require "browser_reposition"
-  Ferrum::Browser.include Common, BrowserReposition
-  def self.new **_
-    Ferrum::Browser.new(**_).tap(&:reposition)
+  if "darwin" == Gem::Platform.local.os
+    require "browser_reposition"
+    Ferrum::Browser.include Common, BrowserReposition
+    def self.new **_
+      Ferrum::Browser.new(**_).tap(&:reposition)
+    end
+  else
+    Ferrum::Browser.include Common
+    def self.new **_
+      Ferrum::Browser.new **_
+    end
+  end
+
+  # https://datatracker.ietf.org/doc/html/rfc2557
+  # https://en.wikipedia.org/wiki/Quoted-printable
+  # require "strscan"
+  require "nokogiri"  # Oga crashes on vk charset
+  def self.process_mhtml
+    scanner = ::StringScanner.new(mht = ARGF.read)
+    fail scanner.peek(400).inspect unless scanner.scan(/\AFrom: <Saved by Blink>\r
+Snapshot-Content-Location: \S+\r
+Subject:(?: \S.*\r\n)+Date: [A-Z][a-z][a-z], \d\d? [A-Z][a-z][a-z] 20\d\d \d\d:\d\d:\d\d -0000\r
+MIME-Version: 1\.0\r
+Content-Type: multipart\/related;\r
+\ttype="text\/html";\r
+\tboundary="(----MultipartBoundary--[a-zA-Z0-9]{42}----)"\r\n\r\n\r\n--\1/)
+    delimeter = scanner[1]
+    fail unless scanner.charpos == prev = scanner.pos
+    reps = []
+    while s = scanner.search_full(::Regexp.new(delimeter), true, true)
+      doc = s[0...-delimeter.size-4]
+      case doc
+      when /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: chrome-error:\/\/chromewebdata\/\r\n\r\n/,
+           /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r\n\r\n/
+        STDERR.puts "trash #{$'.size}"
+        reps.push [prev-delimeter.size-2, scanner.pos-delimeter.size-4, "", ""]
+      when /\A\r\nContent-Type: text\/html\r
+Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        STDERR.puts "html #{$'.size}"
+        header = $&
+        t = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
+        STDERR.puts "unpacked #{t.size}"
+        html = ::Nokogiri::HTML t#.force_encoding "utf-8"
+
+        STDERR.puts ".to_s.size #{html.to_s.size}"
+
+        html.xpath("//*[not(*)]").group_by(&:name).
+          map{ |_, g| [_, g.map(&:to_s).map(&:size).reduce(:+)] }.
+          sort_by(&:last).reverse.take(5).each &method(:p)
+
+        if block_given?
+          yield html
+          STDERR.puts "yielded"
+          STDERR.puts "yield #{html.to_s.size}"
+        end
+
+        reps.push [prev, scanner.pos-delimeter.size-4, header, html.to_s, true, :html]
+      when /\A\r\nContent-Type: text\/css\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        STDERR.puts "css > #{$'.size}"
+        header = $&
+        css = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
+        css.gsub!(/[\r\n]+/, "\n")
+
+        STDERR.puts "css < #{css.size}"
+        reps.push [prev, scanner.pos-delimeter.size-4, header, css, true, :css]
+
+      when /\A\r\nContent-Type: image\/(webp|png|gif|jpeg)\r
+Content-Transfer-Encoding: base64\r
+Content-Location: \S+\r\n\r\n/
+        STDERR.puts "#{$1} #{$'.size}"
+      when /\A\r\nContent-Type: image\/svg\+xml\r
+Content-Transfer-Encoding: quoted-printable\r
+Content-Location: \S+\r\n\r\n/
+        STDERR.puts "svg #{$'.size}"
+      else
+        STDERR.puts doc[0..300]
+        fail
+      end
+      fail unless scanner.charpos == prev = scanner.pos
+    end
+
+    is = reps.map.with_index{ |(_, _, _, _, _, type), i| i if :html == type }.compact
+    STDERR.puts is.inspect
+    cs = reps.map.with_index{ |(_, _, _, _, _, type), i| i if :css == type }.compact
+    STDERR.puts cs.inspect
+    cs.each_cons(2){ |i,j| fail unless i+1==j }
+    fail unless is == [cs[0]-1]
+    File.write "temp.htm", reps[is[0]][3]
+    puts "css > #{File.size "temp.css"}"
+    File.open("temp.css", "w"){ |f| cs.each{ |i| f.puts reps[i][3] } }
+    system "uncss temp.htm -s temp.css -o out.css"
+    STDERR.puts "css < #{File.size "out.css"}"
+    reps[cs[0]][1] = reps[cs[-1]][1]
+    reps[cs[0]+1..cs[-1]] = []
+    reps[cs[0]][3] = File.read "out.css"
+
+    reps.reverse_each do |from, to, header, str, qp|
+      str = qp ?
+        header + str.gsub("=", "=3D").
+          b.gsub(/[\x80-\xFF]/n){ |_| "=%02X" % _.ord }.
+          gsub(/.{73}[^=][^=](?=.)/, "\\0=\r\n") :
+        header + str.gsub("\n", "\r\n")
+      STDERR.puts [str.size, "to - from = #{to - from}"].inspect
+      mht[from...to] = str
+    end
+    puts mht
+    STDERR.puts "OK"
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum_common
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Victor Maslov aka Nakilon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-09 00:00:00.000000000 Z
+date: 2023-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ferrum
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: nakilon@gmail.com
 executables: []