ferrum_common 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ferrum_common.gemspec +2 -2
- data/lib/ferrum_common.rb +142 -27
- metadata +10 -10
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '05923a936f2ca3b407d7313fbf4ad661ecf51eb1d1bab8870204d48c26e85564'
|
|
4
|
+
data.tar.gz: 1523dd2d0b471909b4424e61e314ee8b4def0ab9ec7e6894a8be2894b94d2ba2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0c3b3a806de76805483ae3bd4d129b59afd12e4267096ec95221a012e6935140fafb1cf0df2965a1756cd649c83ef725e1e8785091afd130f74da5a8ecd3f6af
|
|
7
|
+
data.tar.gz: 91037fe48c613a829a95deae97d7a6658f3b4a46f31a7301d4a43d6b79346c424b0e47c05ef63363af01b70fbdbccaedef1a35d365676773050d8655d9d21c7d
|
data/ferrum_common.gemspec
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |spec|
|
|
2
2
|
spec.name = "ferrum_common"
|
|
3
|
-
spec.version = "0.
|
|
3
|
+
spec.version = "0.4.0"
|
|
4
4
|
spec.summary = "[WIP] common useful extensions for ferrum or cuprite"
|
|
5
5
|
|
|
6
6
|
spec.author = "Victor Maslov aka Nakilon"
|
|
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
|
|
|
9
9
|
spec.metadata = {"source_code_uri" => "https://github.com/nakilon/ferrum_common"}
|
|
10
10
|
|
|
11
11
|
spec.add_dependency "ferrum"
|
|
12
|
-
spec.add_dependency "browser_reposition"
|
|
13
12
|
spec.add_dependency "nokogiri"
|
|
13
|
+
spec.add_dependency "ffi"
|
|
14
14
|
spec.required_ruby_version = ">=2.5" # why?
|
|
15
15
|
|
|
16
16
|
spec.files = %w{ LICENSE ferrum_common.gemspec lib/ferrum_common.rb }
|
data/lib/ferrum_common.rb
CHANGED
|
@@ -1,16 +1,36 @@
|
|
|
1
1
|
require "ferrum"
|
|
2
|
+
::Ferrum::Node.module_eval do
|
|
3
|
+
def rect
|
|
4
|
+
::Struct.new(:x, :y, :width, :height).new(*::JSON.load(page.evaluate(<<~HEREDOC, self)))
|
|
5
|
+
( function(node) {
|
|
6
|
+
var x = scrollX, y = scrollY;
|
|
7
|
+
var rect = JSON.parse(JSON.stringify(node.getBoundingClientRect()));
|
|
8
|
+
rect.top += scrollY;
|
|
9
|
+
rect.left += scrollX;
|
|
10
|
+
var t = JSON.stringify( [rect.left, rect.top, rect.width, rect.height, ] );
|
|
11
|
+
return t;
|
|
12
|
+
} )(arguments[0])
|
|
13
|
+
HEREDOC
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
2
17
|
module FerrumCommon
|
|
3
18
|
|
|
4
19
|
module Common
|
|
5
20
|
|
|
6
21
|
private def yield_with_timeout browser, timeout, mtd, msg = nil
|
|
7
|
-
Timeout.timeout(timeout){ yield }
|
|
8
|
-
rescue Timeout::Error
|
|
22
|
+
::Timeout.timeout(timeout){ yield }
|
|
23
|
+
rescue ::Timeout::Error
|
|
24
|
+
begin
|
|
9
25
|
browser.mhtml path: "temp.mhtml"
|
|
10
|
-
|
|
26
|
+
rescue ::Ferrum::BrowserError
|
|
27
|
+
sleep 1
|
|
28
|
+
browser.mhtml path: "temp.mhtml"
|
|
29
|
+
end
|
|
30
|
+
::STDERR.puts "dumped to ./temp.mhtml"
|
|
11
31
|
$!.backtrace.reject!{ |_| _[/\/gems\/concurrent-ruby-/] }
|
|
12
32
|
$!.backtrace.reject!{ |_| _[/\/gems\/ferrum-/] }
|
|
13
|
-
raise Timeout::Error, "#{$!.to_s} after #{timeout} sec in #{mtd}#{" (#{msg.respond_to?(:call) ? msg.call : msg})" if msg}"
|
|
33
|
+
raise ::Timeout::Error, "#{$!.to_s} after #{timeout} sec in #{mtd}#{" (#{msg.respond_to?(:call) ? msg.call : msg})" if msg}"
|
|
14
34
|
end
|
|
15
35
|
|
|
16
36
|
def until_true timeout, msg = nil
|
|
@@ -19,26 +39,40 @@ module FerrumCommon
|
|
|
19
39
|
yield
|
|
20
40
|
rescue Ferrum::NodeNotFoundError
|
|
21
41
|
redo
|
|
22
|
-
end or (sleep timeout*0.1; redo)
|
|
42
|
+
end or (sleep timeout * 0.1; redo)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def find_any type, selector, timeout
|
|
47
|
+
Timeout.timeout timeout do
|
|
48
|
+
t = public_method(type).call selector
|
|
49
|
+
return t unless t.empty?
|
|
50
|
+
sleep timeout * 0.1
|
|
51
|
+
redo
|
|
23
52
|
end
|
|
53
|
+
rescue Timeout::Error
|
|
24
54
|
end
|
|
25
55
|
|
|
26
|
-
def
|
|
56
|
+
def until_n type, selector, timeout, n, node = nil
|
|
27
57
|
t = nil
|
|
28
|
-
yield_with_timeout self, timeout, __method__, ->{ "expected exactly
|
|
58
|
+
yield_with_timeout self, timeout, __method__, ->{ "expected exactly #{n} nodes for #{type} #{selector.inspect}, got #{t ? t.size : "none"}" } do
|
|
29
59
|
t = begin
|
|
30
|
-
public_method(type).call selector
|
|
60
|
+
public_method(type).call selector, within: node
|
|
31
61
|
end
|
|
32
|
-
unless
|
|
62
|
+
unless n == t.size
|
|
33
63
|
sleep timeout * 0.1
|
|
34
64
|
redo
|
|
35
65
|
end
|
|
36
66
|
end
|
|
37
|
-
t
|
|
67
|
+
t
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def until_one type, selector, timeout, node = nil
|
|
71
|
+
until_n(type, selector, timeout, 1, node).first
|
|
38
72
|
end
|
|
39
73
|
|
|
40
74
|
def abort msg_or_cause
|
|
41
|
-
|
|
75
|
+
puts msg_or_cause
|
|
42
76
|
puts (msg_or_cause.respond_to?(:full_message) ? msg_or_cause.full_message : Thread.current.backtrace)
|
|
43
77
|
mhtml path: "temp.mhtml"
|
|
44
78
|
STDERR.puts "dumped to ./temp.mhtml"
|
|
@@ -49,28 +83,100 @@ module FerrumCommon
|
|
|
49
83
|
Ferrum::Page.include Common
|
|
50
84
|
Ferrum::Frame.include Common
|
|
51
85
|
|
|
52
|
-
|
|
53
|
-
require "
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
86
|
+
module CoreGraphics
|
|
87
|
+
require "ffi"
|
|
88
|
+
extend ::FFI::Library
|
|
89
|
+
ffi_lib "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics"
|
|
90
|
+
attach_function :CGGetActiveDisplayList, [:uint32, :pointer, :pointer], :uint32
|
|
91
|
+
class CGRect < ::FFI::Struct
|
|
92
|
+
layout :x, :double,
|
|
93
|
+
:y, :double,
|
|
94
|
+
:width, :double,
|
|
95
|
+
:height, :double
|
|
57
96
|
end
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
97
|
+
attach_function :CGDisplayBounds, [:uint32], CGRect.by_value
|
|
98
|
+
end if "Darwin" == ::Etc.uname[:sysname]
|
|
99
|
+
def self.browser_reposition ferrum_browser, mtd, dx = 0, dy = 0
|
|
100
|
+
case ::Etc.uname[:sysname]
|
|
101
|
+
when "Darwin"
|
|
102
|
+
count_ptr = ::FFI::MemoryPointer.new :uint32
|
|
103
|
+
CoreGraphics.CGGetActiveDisplayList 0, nil, count_ptr
|
|
104
|
+
display_ids = ::FFI::MemoryPointer.new :uint32, count_ptr.read_uint32
|
|
105
|
+
CoreGraphics.CGGetActiveDisplayList count_ptr.read_uint32, display_ids, nil
|
|
106
|
+
displays = display_ids.read_array_of_uint32(count_ptr.read_uint32).map do |display_id|
|
|
107
|
+
CoreGraphics.CGDisplayBounds display_id
|
|
108
|
+
end
|
|
109
|
+
best = displays.map{ |_| _[:x] }.method(mtd).call
|
|
110
|
+
display = displays.find{ |_| best == _[:x] }
|
|
111
|
+
ferrum_browser.position = {left: display[:x] + dx, top: display[:y] + dy}
|
|
112
|
+
else
|
|
113
|
+
"#{::Etc.uname[:sysname].inspect} isn't supported yet"
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def self.parse_mhtml mhtml_string
|
|
118
|
+
scanner = ::StringScanner.new mhtml_string
|
|
119
|
+
fail scanner.peek(400).inspect unless scanner.scan(/\AFrom: <Saved by Blink>\r
|
|
120
|
+
Snapshot-Content-Location: \S+\r
|
|
121
|
+
Subject:(?: \S.*\r\n)+Date: [A-Z][a-z][a-z], \d\d? [A-Z][a-z][a-z] 20\d\d \d\d:\d\d:\d\d [-+]0\d00\r
|
|
122
|
+
MIME-Version: 1\.0\r
|
|
123
|
+
Content-Type: multipart\/related;\r
|
|
124
|
+
\ttype="text\/html";\r
|
|
125
|
+
\tboundary="(----MultipartBoundary--[a-zA-Z0-9]{42}----)"\r\n\r\n\r\n--\1/)
|
|
126
|
+
delimeter = scanner[1]
|
|
127
|
+
fail unless scanner.charpos == prev = scanner.pos
|
|
128
|
+
while s = scanner.search_full(::Regexp.new(delimeter), true, true)
|
|
129
|
+
case doc = s[0...-delimeter.size-4]
|
|
130
|
+
when /\A\r\nContent-Type: text\/html\r
|
|
131
|
+
Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
|
|
132
|
+
Content-Transfer-Encoding: quoted-printable\r
|
|
133
|
+
Content-Location: chrome-error:\/\/chromewebdata\/\r\n\r\n/,
|
|
134
|
+
/\A\r\nContent-Type: text\/html\r
|
|
135
|
+
Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
|
|
136
|
+
Content-Transfer-Encoding: quoted-printable\r\n\r\n/
|
|
137
|
+
# trash
|
|
138
|
+
when /\A\r\nContent-Type: text\/html\r
|
|
139
|
+
Content-ID: <frame-[A-Z0-9]{32}@mhtml\.blink>\r
|
|
140
|
+
Content-Transfer-Encoding: quoted-printable\r
|
|
141
|
+
Content-Location: \S+\r\n\r\n/
|
|
142
|
+
# html
|
|
143
|
+
when /\A\r\nContent-Type: text\/css\r
|
|
144
|
+
Content-Transfer-Encoding: quoted-printable\r
|
|
145
|
+
Content-Location: \S+\r\n\r\n/
|
|
146
|
+
# css
|
|
147
|
+
when /\A\r\nContent-Type: image\/(webp|png|gif|jpeg)\r
|
|
148
|
+
Content-Transfer-Encoding: base64\r
|
|
149
|
+
Content-Location: (https?:\S+)\r\n\r\n(([+\/0-9A-Za-z=]{,76}\r\n)+)\z/
|
|
150
|
+
yield $1, $2, $3
|
|
151
|
+
when /\A\r\nContent-Type: binary\/octet-stream\r
|
|
152
|
+
Content-Transfer-Encoding: base64\r
|
|
153
|
+
Content-Location: https:\/\/\S+\r\n\r\n/
|
|
154
|
+
# binary
|
|
155
|
+
when /\A\r\nContent-Type: image\/svg\+xml\r
|
|
156
|
+
Content-Transfer-Encoding: quoted-printable\r
|
|
157
|
+
Content-Location: https?:\S+\r\n\r\n/
|
|
158
|
+
# svg
|
|
159
|
+
when /\A\r\nContent-Type: image\/gif\r
|
|
160
|
+
Content-ID: <frame-[0-9A-F]{32}@mhtml\.blink>\r
|
|
161
|
+
Content-Transfer-Encoding: base64\r
|
|
162
|
+
Content-Location: https?:\S+\r\n\r\n/
|
|
163
|
+
# gif
|
|
164
|
+
else
|
|
165
|
+
STDERR.puts s[0..300]
|
|
166
|
+
fail
|
|
167
|
+
end
|
|
168
|
+
fail unless scanner.charpos == prev = scanner.pos
|
|
62
169
|
end
|
|
63
170
|
end
|
|
64
171
|
|
|
65
172
|
# https://datatracker.ietf.org/doc/html/rfc2557
|
|
66
173
|
# https://en.wikipedia.org/wiki/Quoted-printable
|
|
67
|
-
# require "strscan"
|
|
68
174
|
require "nokogiri" # Oga crashes on vk charset
|
|
69
175
|
def self.process_mhtml
|
|
70
176
|
scanner = ::StringScanner.new(mht = ARGF.read)
|
|
71
177
|
fail scanner.peek(400).inspect unless scanner.scan(/\AFrom: <Saved by Blink>\r
|
|
72
178
|
Snapshot-Content-Location: \S+\r
|
|
73
|
-
Subject:(?: \S.*\r\n)+Date: [A-Z][a-z][a-z], \d\d? [A-Z][a-z][a-z] 20\d\d \d\d:\d\d:\d\d
|
|
179
|
+
Subject:(?: \S.*\r\n)+Date: [A-Z][a-z][a-z], \d\d? [A-Z][a-z][a-z] 20\d\d \d\d:\d\d:\d\d [-+]0\d00\r
|
|
74
180
|
MIME-Version: 1\.0\r
|
|
75
181
|
Content-Type: multipart\/related;\r
|
|
76
182
|
\ttype="text\/html";\r
|
|
@@ -96,7 +202,7 @@ Content-Transfer-Encoding: quoted-printable\r
|
|
|
96
202
|
Content-Location: \S+\r\n\r\n/
|
|
97
203
|
STDERR.puts "html #{$'.size}"
|
|
98
204
|
header = $&
|
|
99
|
-
t = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
|
|
205
|
+
t = $'.gsub(/=([0-9A-F][0-9A-F])/){ fail $1 unless "3D" == $1 || "20" == $1 || "0A" == $1 || "09" == $1 unless "80" <= $1; $1.hex.chr }.gsub("=\r\n", "")
|
|
100
206
|
STDERR.puts "unpacked #{t.size}"
|
|
101
207
|
html = ::Nokogiri::HTML t#.force_encoding "utf-8"
|
|
102
208
|
|
|
@@ -104,7 +210,7 @@ Content-Location: \S+\r\n\r\n/
|
|
|
104
210
|
|
|
105
211
|
html.xpath("//*[not(*)]").group_by(&:name).
|
|
106
212
|
map{ |_, g| [_, g.map(&:to_s).map(&:size).reduce(:+)] }.
|
|
107
|
-
sort_by(&:last).reverse.take(5).each
|
|
213
|
+
sort_by(&:last).reverse.take(5).each{ |_| STDERR.puts _.inspect }
|
|
108
214
|
|
|
109
215
|
if block_given?
|
|
110
216
|
yield html
|
|
@@ -126,12 +232,21 @@ Content-Location: \S+\r\n\r\n/
|
|
|
126
232
|
|
|
127
233
|
when /\A\r\nContent-Type: image\/(webp|png|gif|jpeg)\r
|
|
128
234
|
Content-Transfer-Encoding: base64\r
|
|
129
|
-
Content-Location:
|
|
235
|
+
Content-Location: https?:\S+\r\n\r\n/
|
|
130
236
|
STDERR.puts "#{$1} #{$'.size}"
|
|
237
|
+
when /\A\r\nContent-Type: binary\/octet-stream\r
|
|
238
|
+
Content-Transfer-Encoding: base64\r
|
|
239
|
+
Content-Location: https:\/\/\S+\r\n\r\n/
|
|
240
|
+
STDERR.puts "binary #{$'.size}"
|
|
131
241
|
when /\A\r\nContent-Type: image\/svg\+xml\r
|
|
132
242
|
Content-Transfer-Encoding: quoted-printable\r
|
|
133
|
-
Content-Location:
|
|
243
|
+
Content-Location: https?:\S+\r\n\r\n/
|
|
134
244
|
STDERR.puts "svg #{$'.size}"
|
|
245
|
+
when /\A\r\nContent-Type: image\/gif\r
|
|
246
|
+
Content-ID: <frame-[0-9A-F]{32}@mhtml\.blink>\r
|
|
247
|
+
Content-Transfer-Encoding: base64\r
|
|
248
|
+
Content-Location: https?:\S+\r\n\r\n/
|
|
249
|
+
STDERR.puts "gif #{$'.size}"
|
|
135
250
|
else
|
|
136
251
|
STDERR.puts doc[0..300]
|
|
137
252
|
fail
|
|
@@ -146,8 +261,8 @@ Content-Location: \S+\r\n\r\n/
|
|
|
146
261
|
cs.each_cons(2){ |i,j| fail unless i+1==j }
|
|
147
262
|
fail unless is == [cs[0]-1]
|
|
148
263
|
File.write "temp.htm", reps[is[0]][3]
|
|
149
|
-
puts "css > #{File.size "temp.css"}"
|
|
150
264
|
File.open("temp.css", "w"){ |f| cs.each{ |i| f.puts reps[i][3] } }
|
|
265
|
+
STDERR.puts "css > #{File.size "temp.css"}"
|
|
151
266
|
system "uncss temp.htm -s temp.css -o out.css"
|
|
152
267
|
STDERR.puts "css < #{File.size "out.css"}"
|
|
153
268
|
reps[cs[0]][1] = reps[cs[-1]][1]
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ferrum_common
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Victor Maslov aka Nakilon
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-07-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: ferrum
|
|
@@ -25,7 +25,7 @@ dependencies:
|
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
|
-
name:
|
|
28
|
+
name: nokogiri
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
31
|
- - ">="
|
|
@@ -39,7 +39,7 @@ dependencies:
|
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
|
-
name:
|
|
42
|
+
name: ffi
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - ">="
|
|
@@ -52,7 +52,7 @@ dependencies:
|
|
|
52
52
|
- - ">="
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
54
|
version: '0'
|
|
55
|
-
description:
|
|
55
|
+
description:
|
|
56
56
|
email: nakilon@gmail.com
|
|
57
57
|
executables: []
|
|
58
58
|
extensions: []
|
|
@@ -61,12 +61,12 @@ files:
|
|
|
61
61
|
- LICENSE
|
|
62
62
|
- ferrum_common.gemspec
|
|
63
63
|
- lib/ferrum_common.rb
|
|
64
|
-
homepage:
|
|
64
|
+
homepage:
|
|
65
65
|
licenses:
|
|
66
66
|
- MIT
|
|
67
67
|
metadata:
|
|
68
68
|
source_code_uri: https://github.com/nakilon/ferrum_common
|
|
69
|
-
post_install_message:
|
|
69
|
+
post_install_message:
|
|
70
70
|
rdoc_options: []
|
|
71
71
|
require_paths:
|
|
72
72
|
- lib
|
|
@@ -81,8 +81,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
82
|
version: '0'
|
|
83
83
|
requirements: []
|
|
84
|
-
rubygems_version: 3.
|
|
85
|
-
signing_key:
|
|
84
|
+
rubygems_version: 3.1.6
|
|
85
|
+
signing_key:
|
|
86
86
|
specification_version: 4
|
|
87
87
|
summary: "[WIP] common useful extensions for ferrum or cuprite"
|
|
88
88
|
test_files: []
|