ferrum 0.11 → 0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +174 -30
- data/lib/ferrum/browser/binary.rb +46 -0
- data/lib/ferrum/browser/client.rb +17 -16
- data/lib/ferrum/browser/command.rb +10 -12
- data/lib/ferrum/browser/options/base.rb +2 -11
- data/lib/ferrum/browser/options/chrome.rb +29 -18
- data/lib/ferrum/browser/options/firefox.rb +13 -9
- data/lib/ferrum/browser/options.rb +84 -0
- data/lib/ferrum/browser/process.rb +45 -40
- data/lib/ferrum/browser/subscriber.rb +1 -3
- data/lib/ferrum/browser/version_info.rb +71 -0
- data/lib/ferrum/browser/web_socket.rb +9 -12
- data/lib/ferrum/browser/xvfb.rb +4 -8
- data/lib/ferrum/browser.rb +193 -47
- data/lib/ferrum/context.rb +9 -4
- data/lib/ferrum/contexts.rb +12 -10
- data/lib/ferrum/cookies/cookie.rb +126 -0
- data/lib/ferrum/cookies.rb +93 -55
- data/lib/ferrum/dialog.rb +30 -0
- data/lib/ferrum/errors.rb +115 -0
- data/lib/ferrum/frame/dom.rb +177 -0
- data/lib/ferrum/frame/runtime.rb +58 -75
- data/lib/ferrum/frame.rb +118 -23
- data/lib/ferrum/headers.rb +30 -2
- data/lib/ferrum/keyboard.rb +56 -13
- data/lib/ferrum/mouse.rb +92 -7
- data/lib/ferrum/network/auth_request.rb +7 -2
- data/lib/ferrum/network/exchange.rb +97 -12
- data/lib/ferrum/network/intercepted_request.rb +10 -8
- data/lib/ferrum/network/request.rb +69 -0
- data/lib/ferrum/network/response.rb +85 -3
- data/lib/ferrum/network.rb +285 -36
- data/lib/ferrum/node.rb +69 -23
- data/lib/ferrum/page/animation.rb +16 -1
- data/lib/ferrum/page/frames.rb +111 -30
- data/lib/ferrum/page/screenshot.rb +142 -65
- data/lib/ferrum/page/stream.rb +38 -0
- data/lib/ferrum/page/tracing.rb +97 -0
- data/lib/ferrum/page.rb +224 -60
- data/lib/ferrum/proxy.rb +147 -0
- data/lib/ferrum/{rbga.rb → rgba.rb} +4 -2
- data/lib/ferrum/target.rb +7 -4
- data/lib/ferrum/utils/attempt.rb +20 -0
- data/lib/ferrum/utils/elapsed_time.rb +27 -0
- data/lib/ferrum/utils/platform.rb +28 -0
- data/lib/ferrum/version.rb +1 -1
- data/lib/ferrum.rb +4 -146
- metadata +63 -51
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bde7c8e40700ace2d713cba69eee5828dcb888e5468c07a6b1e5e0d668e4c641
|
4
|
+
data.tar.gz: d52f7278dd76e670aa50721e6d70adc26297bbfb031ddac259dde5421c817a04
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7206d7a92d8483bd106fe262130492e7bf51e2db8b99f73c39ada0a674fb29c84bc948bdbb6f554b672ade9f4e3812a9158447b30d6f976cb4892b5e4e8df30
|
7
|
+
data.tar.gz: aef76b65c27dca2a5385d9881f9be8caccb85e804b31a26e421e35a43295baa3a5c818de856b49e3a7928139af4aeb588a5fae1cb67275440e5a0b3ddf97f76e
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
<img align="right"
|
4
4
|
width="320" height="241"
|
5
5
|
alt="Ferrum logo"
|
6
|
-
src="https://raw.githubusercontent.com/rubycdp/ferrum/
|
6
|
+
src="https://raw.githubusercontent.com/rubycdp/ferrum/main/logo.svg?sanitize=true">
|
7
7
|
|
8
8
|
#### As simple as Puppeteer, though even simpler.
|
9
9
|
|
@@ -23,12 +23,7 @@ going to crawl sites you better use Ferrum or
|
|
23
23
|
[Vessel](https://github.com/rubycdp/vessel) because you crawl, not test.
|
24
24
|
|
25
25
|
* [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework
|
26
|
-
based on Ferrum
|
27
|
-
a real browser in order to grab data.
|
28
|
-
|
29
|
-
Web design by [Evrone](https://evrone.com/), what else
|
30
|
-
[we build with Ruby on Rails](https://evrone.com/ruby), what else
|
31
|
-
[we do at Evrone](https://evrone.com/cases#case-studies).
|
26
|
+
based on Ferrum and Mechanize.
|
32
27
|
|
33
28
|
|
34
29
|
## Index
|
@@ -40,7 +35,9 @@ Web design by [Evrone](https://evrone.com/), what else
|
|
40
35
|
* [Navigation](https://github.com/rubycdp/ferrum#navigation)
|
41
36
|
* [Finders](https://github.com/rubycdp/ferrum#finders)
|
42
37
|
* [Screenshots](https://github.com/rubycdp/ferrum#screenshots)
|
38
|
+
* [Cleaning Up](https://github.com/rubycdp/ferrum#cleaning-up)
|
43
39
|
* [Network](https://github.com/rubycdp/ferrum#network)
|
40
|
+
* [Proxy](https://github.com/rubycdp/ferrum#proxy)
|
44
41
|
* [Mouse](https://github.com/rubycdp/ferrum#mouse)
|
45
42
|
* [Keyboard](https://github.com/rubycdp/ferrum#keyboard)
|
46
43
|
* [Cookies](https://github.com/rubycdp/ferrum#cookies)
|
@@ -48,9 +45,10 @@ Web design by [Evrone](https://evrone.com/), what else
|
|
48
45
|
* [JavaScript](https://github.com/rubycdp/ferrum#javascript)
|
49
46
|
* [Frames](https://github.com/rubycdp/ferrum#frames)
|
50
47
|
* [Frame](https://github.com/rubycdp/ferrum#frame)
|
51
|
-
* [
|
48
|
+
* [Dialogs](https://github.com/rubycdp/ferrum#dialogs)
|
52
49
|
* [Animation](https://github.com/rubycdp/ferrum#animation)
|
53
50
|
* [Node](https://github.com/rubycdp/ferrum#node)
|
51
|
+
* [Tracing](https://github.com/rubycdp/ferrum#tracing)
|
54
52
|
* [Thread safety](https://github.com/rubycdp/ferrum#thread-safety)
|
55
53
|
* [Development](https://github.com/rubycdp/ferrum#development)
|
56
54
|
* [Contributing](https://github.com/rubycdp/ferrum#contributing)
|
@@ -61,7 +59,8 @@ Web design by [Evrone](https://evrone.com/), what else
|
|
61
59
|
|
62
60
|
There's no official Chrome or Chromium package for Linux don't install it this
|
63
61
|
way because it's either outdated or unofficial, both are bad. Download it from
|
64
|
-
official [
|
62
|
+
official source for [Chrome](https://www.google.com/chrome/) or
|
63
|
+
[Chromium](https://www.chromium.org/getting-involved/download-chromium).
|
65
64
|
Chrome binary should be in the `PATH` or `BROWSER_PATH` or you can pass it as an
|
66
65
|
option to browser instance see `:browser_path` in
|
67
66
|
[Customization](https://github.com/rubycdp/ferrum#customization).
|
@@ -155,7 +154,7 @@ Ferrum::Browser.new(options)
|
|
155
154
|
* `:logger` (Object responding to `puts`) - When present, debug output is
|
156
155
|
written to this object.
|
157
156
|
* `:slowmo` (Integer | Float) - Set a delay in seconds to wait before sending command.
|
158
|
-
|
157
|
+
Useful companion of headless option, so that you have time to see changes.
|
159
158
|
* `:timeout` (Numeric) - The number of seconds we'll wait for a response when
|
160
159
|
communicating with browser. Default is 5.
|
161
160
|
* `:js_errors` (Boolean) - When true, JavaScript errors get re-raised in Ruby.
|
@@ -173,15 +172,18 @@ Ferrum::Browser.new(options)
|
|
173
172
|
options it passes to the browser, if you set this to `true` then only
|
174
173
|
options you put in `:browser_options` will be passed to the browser,
|
175
174
|
except required ones of course.
|
176
|
-
* `:port` (Integer) - Remote debugging port for headless Chrome
|
177
|
-
* `:host` (String) - Remote debugging address for headless Chrome
|
175
|
+
* `:port` (Integer) - Remote debugging port for headless Chrome.
|
176
|
+
* `:host` (String) - Remote debugging address for headless Chrome.
|
178
177
|
* `:url` (String) - URL for a running instance of Chrome. If this is set, a
|
179
178
|
browser process will not be spawned.
|
180
179
|
* `:process_timeout` (Integer) - How long to wait for the Chrome process to
|
181
|
-
respond on startup
|
180
|
+
respond on startup.
|
182
181
|
* `:ws_max_receive_size` (Integer) - How big messages to accept from Chrome
|
183
182
|
over the web socket, in bytes. Defaults to 64MB. Incoming messages larger
|
184
183
|
than this will cause a `Ferrum::DeadBrowserError`.
|
184
|
+
* `:proxy` (Hash) - Specify proxy settings, [read more](https://github.com/rubycdp/ferrum#proxy)
|
185
|
+
* `:save_path` (String) - Path to save attachments with [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) header.
|
186
|
+
* `:env` (Hash) - Environment variables you'd like to pass through to the process
|
185
187
|
|
186
188
|
|
187
189
|
## Navigation
|
@@ -407,9 +409,28 @@ browser.mhtml(path: "google.mhtml") # => 87742
|
|
407
409
|
```
|
408
410
|
|
409
411
|
|
412
|
+
## Cleaning Up
|
413
|
+
|
414
|
+
#### reset
|
415
|
+
|
416
|
+
Closes browser tabs opened by the `Browser` instance.
|
417
|
+
|
418
|
+
```ruby
|
419
|
+
# connect to a long-running Chrome process
|
420
|
+
browser = Ferrum::Browser.new(url: 'http://localhost:9222')
|
421
|
+
|
422
|
+
browser.go_to("https://github.com/")
|
423
|
+
|
424
|
+
# clean up, lest the tab stays there hanging forever
|
425
|
+
browser.reset
|
426
|
+
|
427
|
+
browser.quit
|
428
|
+
```
|
429
|
+
|
430
|
+
|
410
431
|
## Network
|
411
432
|
|
412
|
-
browser.network
|
433
|
+
`browser.network`
|
413
434
|
|
414
435
|
#### traffic `Array<Network::Exchange>`
|
415
436
|
|
@@ -542,17 +563,78 @@ end
|
|
542
563
|
browser.network.authorize(user: "login", password: "pass", type: :proxy)
|
543
564
|
|
544
565
|
browser.go_to("https://google.com")
|
545
|
-
|
546
566
|
```
|
547
567
|
|
548
568
|
You used to call `authorize` method without block, but since it's implemented using request interception there could be
|
549
569
|
a collision with another part of your code that also uses request interception, so that authorize allows the request
|
550
570
|
while your code denies but it's too late. The block is mandatory now.
|
551
571
|
|
572
|
+
#### emulate_network_conditions(\*\*options)
|
573
|
+
|
574
|
+
Activates emulation of network conditions.
|
575
|
+
|
576
|
+
* options `Hash`
|
577
|
+
* :offline `Boolean` emulate internet disconnection, `false` by default
|
578
|
+
* :latency `Integer` minimum latency from request sent to response headers received (ms), `0` by
|
579
|
+
default
|
580
|
+
* :download_throughput `Integer` maximal aggregated download throughput (bytes/sec), `-1`
|
581
|
+
by default, disables download throttling
|
582
|
+
* :upload_throughput `Integer` maximal aggregated upload throughput (bytes/sec), `-1`
|
583
|
+
by default, disables download throttling
|
584
|
+
* :connection_type `String` connection type if known, one of: none, cellular2g, cellular3g, cellular4g,
|
585
|
+
bluetooth, ethernet, wifi, wimax, other. `nil` by default
|
586
|
+
|
587
|
+
```ruby
|
588
|
+
browser.network.emulate_network_conditions(connection_type: "cellular2g")
|
589
|
+
browser.go_to("https://github.com/")
|
590
|
+
```
|
591
|
+
|
592
|
+
#### offline_mode
|
593
|
+
|
594
|
+
Activates offline mode for a page.
|
595
|
+
|
596
|
+
```ruby
|
597
|
+
browser.network.offline_mode
|
598
|
+
browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed to reach server, check DNS and server status)
|
599
|
+
```
|
600
|
+
|
601
|
+
|
602
|
+
## Proxy
|
603
|
+
|
604
|
+
You can set a proxy with a `:proxy` option:
|
605
|
+
|
606
|
+
```ruby
|
607
|
+
browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", password: "pa$$" })
|
608
|
+
```
|
609
|
+
|
610
|
+
`:bypass` can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
|
611
|
+
|
612
|
+
```ruby
|
613
|
+
browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
|
614
|
+
```
|
615
|
+
|
616
|
+
In general passing a proxy option when instantiating a browser results in a browser running with proxy command line
|
617
|
+
flags, so that it affects all pages and contexts. You can create a page in a new context which can use its own proxy
|
618
|
+
settings:
|
619
|
+
|
620
|
+
```ruby
|
621
|
+
browser = Ferrum::Browser.new
|
622
|
+
|
623
|
+
browser.create_page(proxy: { host: "x.x.x.x", port: 31337, user: "user", password: "password" }) do |page|
|
624
|
+
page.go_to("https://api.ipify.org?format=json")
|
625
|
+
page.body # => "x.x.x.x"
|
626
|
+
end
|
627
|
+
|
628
|
+
browser.create_page(proxy: { host: "y.y.y.y", port: 31337, user: "user", password: "password" }) do |page|
|
629
|
+
page.go_to("https://api.ipify.org?format=json")
|
630
|
+
page.body # => "y.y.y.y"
|
631
|
+
end
|
632
|
+
```
|
633
|
+
|
552
634
|
|
553
635
|
### Mouse
|
554
636
|
|
555
|
-
browser.mouse
|
637
|
+
`browser.mouse`
|
556
638
|
|
557
639
|
#### scroll_to(x, y)
|
558
640
|
|
@@ -639,13 +721,12 @@ Returns bitfield for a given keys
|
|
639
721
|
|
640
722
|
## Cookies
|
641
723
|
|
642
|
-
browser.cookies
|
724
|
+
`browser.cookies`
|
643
725
|
|
644
726
|
#### all : `Hash<String, Cookie>`
|
645
727
|
|
646
728
|
Returns cookies hash
|
647
729
|
|
648
|
-
|
649
730
|
```ruby
|
650
731
|
browser.cookies.all # => {"NID"=>#<Ferrum::Cookies::Cookie:0x0000558624b37a40 @attributes={"name"=>"NID", "value"=>"...", "domain"=>".google.com", "path"=>"/", "expires"=>1583211046.575681, "size"=>178, "httpOnly"=>true, "secure"=>false, "session"=>false}>}
|
651
732
|
```
|
@@ -660,11 +741,11 @@ Returns cookie
|
|
660
741
|
browser.cookies["NID"] # => <Ferrum::Cookies::Cookie:0x0000558624b67a88 @attributes={"name"=>"NID", "value"=>"...", "domain"=>".google.com", "path"=>"/", "expires"=>1583211046.575681, "size"=>178, "httpOnly"=>true, "secure"=>false, "session"=>false}>
|
661
742
|
```
|
662
743
|
|
663
|
-
#### set(
|
744
|
+
#### set(value) : `Boolean`
|
664
745
|
|
665
|
-
Sets
|
746
|
+
Sets a cookie
|
666
747
|
|
667
|
-
*
|
748
|
+
* value `Hash`
|
668
749
|
* :name `String`
|
669
750
|
* :value `String`
|
670
751
|
* :domain `String`
|
@@ -676,6 +757,13 @@ Sets given values as cookie
|
|
676
757
|
browser.cookies.set(name: "stealth", value: "omg", domain: "google.com") # => true
|
677
758
|
```
|
678
759
|
|
760
|
+
* value `Cookie`
|
761
|
+
|
762
|
+
```ruby
|
763
|
+
nid_cookie = browser.cookies["NID"] # => <Ferrum::Cookies::Cookie:0x0000558624b67a88>
|
764
|
+
browser.cookies.set(nid_cookie) # => true
|
765
|
+
```
|
766
|
+
|
679
767
|
#### remove(\*\*options) : `Boolean`
|
680
768
|
|
681
769
|
Removes given cookie
|
@@ -699,7 +787,7 @@ browser.cookies.clear # => true
|
|
699
787
|
|
700
788
|
## Headers
|
701
789
|
|
702
|
-
browser.headers
|
790
|
+
`browser.headers`
|
703
791
|
|
704
792
|
#### get : `Hash`
|
705
793
|
|
@@ -798,9 +886,10 @@ browser.add_script_tag(url: "http://example.com/stylesheet.css") # => true
|
|
798
886
|
browser.add_style_tag(content: "h1 { font-size: 40px; }") # => true
|
799
887
|
|
800
888
|
```
|
801
|
-
#### bypass_csp(
|
889
|
+
#### bypass_csp(\*\*options) : `Boolean`
|
802
890
|
|
803
|
-
*
|
891
|
+
* options `Hash`
|
892
|
+
* :enabled `Boolean`, `true` by default
|
804
893
|
|
805
894
|
```ruby
|
806
895
|
browser.bypass_csp # => true
|
@@ -941,7 +1030,7 @@ browser.go_to("https://www.w3schools.com/tags/tag_frame.asp")
|
|
941
1030
|
browser.main_frame.doctype # => "<!DOCTYPE html>"
|
942
1031
|
```
|
943
1032
|
|
944
|
-
####
|
1033
|
+
#### content = html
|
945
1034
|
|
946
1035
|
Sets a content of a given frame.
|
947
1036
|
|
@@ -951,12 +1040,12 @@ Sets a content of a given frame.
|
|
951
1040
|
browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
|
952
1041
|
frame = browser.frames[1]
|
953
1042
|
frame.body # <html lang="en"><head><style>body {transition: opacity ease-in 0.2s; }...
|
954
|
-
frame.
|
1043
|
+
frame.content = "<html><head></head><body><p>lol</p></body></html>"
|
955
1044
|
frame.body # => <html><head></head><body><p>lol</p></body></html>
|
956
1045
|
```
|
957
1046
|
|
958
1047
|
|
959
|
-
##
|
1048
|
+
## Dialogs
|
960
1049
|
|
961
1050
|
#### accept(text)
|
962
1051
|
|
@@ -1008,7 +1097,16 @@ browser.playback_rate # => 2000
|
|
1008
1097
|
|
1009
1098
|
#### node? : `Boolean`
|
1010
1099
|
#### frame_id
|
1011
|
-
#### frame
|
1100
|
+
#### frame : `Frame`
|
1101
|
+
|
1102
|
+
Returns [Frame](https://github.com/rubycdp/ferrum#frame) object for current node, you can keep using
|
1103
|
+
[Finders](https://github.com/rubycdp/ferrum#Finders) for that object:
|
1104
|
+
|
1105
|
+
```ruby
|
1106
|
+
frame = browser.at_xpath("//iframe").frame # => Frame
|
1107
|
+
frame.at_css("//a[text() = 'Log in']") # => Node
|
1108
|
+
```
|
1109
|
+
|
1012
1110
|
#### focus
|
1013
1111
|
#### focusable?
|
1014
1112
|
#### moving? : `Boolean`
|
@@ -1028,6 +1126,49 @@ browser.playback_rate # => 2000
|
|
1028
1126
|
#### property
|
1029
1127
|
#### attribute
|
1030
1128
|
#### evaluate
|
1129
|
+
#### selected : `Array<Node>`
|
1130
|
+
#### select
|
1131
|
+
|
1132
|
+
(chainable) Selects options by passed attribute.
|
1133
|
+
|
1134
|
+
```ruby
|
1135
|
+
browser.at_xpath("//*[select]").select(["1"]) # => Node (select)
|
1136
|
+
browser.at_xpath("//*[select]").select(["text"], by: :text) # => Node (select)
|
1137
|
+
```
|
1138
|
+
|
1139
|
+
Accept string, array or strings:
|
1140
|
+
```ruby
|
1141
|
+
browser.at_xpath("//*[select]").select("1")
|
1142
|
+
browser.at_xpath("//*[select]").select("1", "2")
|
1143
|
+
browser.at_xpath("//*[select]").select(["1", "2"])
|
1144
|
+
```
|
1145
|
+
|
1146
|
+
|
1147
|
+
## Tracing
|
1148
|
+
|
1149
|
+
You can use `tracing.record` to create a trace file which can be opened in Chrome DevTools or
|
1150
|
+
[timeline viewer](https://chromedevtools.github.io/timeline-viewer/).
|
1151
|
+
|
1152
|
+
```ruby
|
1153
|
+
page.tracing.record(path: "trace.json") do
|
1154
|
+
page.go_to("https://www.google.com")
|
1155
|
+
end
|
1156
|
+
```
|
1157
|
+
|
1158
|
+
#### tracing.record(\*\*options) : `String`
|
1159
|
+
|
1160
|
+
Accepts block, records trace and by default returns trace data from `Tracing.tracingComplete` event as output. When
|
1161
|
+
`path` is specified returns `true` and stores trace data into file.
|
1162
|
+
|
1163
|
+
* options `Hash`
|
1164
|
+
* :path `String` save data on the disk, `nil` by default
|
1165
|
+
* :encoding `Symbol` `:base64` | `:binary` encode output as Base64 or plain text. `:binary` by default
|
1166
|
+
* :timeout `Float` wait until file streaming finishes in the specified time or raise error, defaults to `nil`
|
1167
|
+
* :screenshots `Boolean` capture screenshots in the trace, `false` by default
|
1168
|
+
* :trace_config `Hash<String, Object>` config for
|
1169
|
+
[trace](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#type-TraceConfig), for categories
|
1170
|
+
see [getCategories](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#method-getCategories),
|
1171
|
+
only one trace config can be active at a time per browser.
|
1031
1172
|
|
1032
1173
|
|
1033
1174
|
## Thread safety ##
|
@@ -1091,9 +1232,12 @@ browser.quit
|
|
1091
1232
|
|
1092
1233
|
After checking out the repo, run `bundle install` to install dependencies.
|
1093
1234
|
|
1094
|
-
Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will
|
1235
|
+
Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will
|
1236
|
+
allow you to experiment.
|
1095
1237
|
|
1096
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
|
1238
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
|
1239
|
+
version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
|
1240
|
+
push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
1097
1241
|
|
1098
1242
|
|
1099
1243
|
## Contributing
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
module Binary
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def find(commands)
|
9
|
+
enum(commands).first
|
10
|
+
end
|
11
|
+
|
12
|
+
def all(commands)
|
13
|
+
enum(commands).force
|
14
|
+
end
|
15
|
+
|
16
|
+
def enum(commands)
|
17
|
+
paths, exts = prepare_paths
|
18
|
+
cmds = Array(commands).product(paths, exts)
|
19
|
+
lazy_find(cmds)
|
20
|
+
end
|
21
|
+
|
22
|
+
def prepare_paths
|
23
|
+
exts = (ENV.key?("PATHEXT") ? ENV.fetch("PATHEXT").split(";") : []) << ""
|
24
|
+
paths = ENV["PATH"].split(File::PATH_SEPARATOR)
|
25
|
+
raise EmptyPathError if paths.empty?
|
26
|
+
|
27
|
+
[paths, exts]
|
28
|
+
end
|
29
|
+
|
30
|
+
# rubocop:disable Style/CollectionCompact
|
31
|
+
def lazy_find(cmds)
|
32
|
+
cmds.lazy.map do |cmd, path, ext|
|
33
|
+
absolute_path = File.absolute_path(cmd)
|
34
|
+
is_absolute_path = absolute_path == cmd
|
35
|
+
cmd = File.expand_path("#{cmd}#{ext}", path) unless is_absolute_path
|
36
|
+
|
37
|
+
next unless File.executable?(cmd)
|
38
|
+
next if File.directory?(cmd)
|
39
|
+
|
40
|
+
cmd
|
41
|
+
end.reject(&:nil?) # .compact isn't defined on Enumerator::Lazy
|
42
|
+
end
|
43
|
+
# rubocop:enable Style/CollectionCompact
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "concurrent-ruby"
|
4
3
|
require "ferrum/browser/subscriber"
|
5
4
|
require "ferrum/browser/web_socket"
|
6
5
|
|
@@ -9,22 +8,23 @@ module Ferrum
|
|
9
8
|
class Client
|
10
9
|
INTERRUPTIONS = %w[Fetch.requestPaused Fetch.authRequired].freeze
|
11
10
|
|
12
|
-
def initialize(
|
13
|
-
@
|
11
|
+
def initialize(ws_url, connectable, logger: nil, ws_max_receive_size: nil, id_starts_with: 0)
|
12
|
+
@connectable = connectable
|
14
13
|
@command_id = id_starts_with
|
15
14
|
@pendings = Concurrent::Hash.new
|
16
|
-
@ws = WebSocket.new(ws_url,
|
17
|
-
@subscriber, @
|
15
|
+
@ws = WebSocket.new(ws_url, ws_max_receive_size, logger)
|
16
|
+
@subscriber, @interrupter = Subscriber.build(2)
|
18
17
|
|
19
18
|
@thread = Thread.new do
|
20
19
|
Thread.current.abort_on_exception = true
|
21
|
-
if Thread.current.respond_to?(:report_on_exception=)
|
22
|
-
|
23
|
-
|
20
|
+
Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
|
21
|
+
|
22
|
+
loop do
|
23
|
+
message = @ws.messages.pop
|
24
|
+
break unless message
|
24
25
|
|
25
|
-
while message = @ws.messages.pop
|
26
26
|
if INTERRUPTIONS.include?(message["method"])
|
27
|
-
@
|
27
|
+
@interrupter.async.call(message)
|
28
28
|
elsif message.key?("method")
|
29
29
|
@subscriber.async.call(message)
|
30
30
|
else
|
@@ -39,11 +39,12 @@ module Ferrum
|
|
39
39
|
message = build_message(method, params)
|
40
40
|
@pendings[message[:id]] = pending
|
41
41
|
@ws.send_message(message)
|
42
|
-
data = pending.value!(@
|
42
|
+
data = pending.value!(@connectable.timeout)
|
43
43
|
@pendings.delete(message[:id])
|
44
44
|
|
45
45
|
raise DeadBrowserError if data.nil? && @ws.messages.closed?
|
46
46
|
raise TimeoutError unless data
|
47
|
+
|
47
48
|
error, response = data.values_at("error", "result")
|
48
49
|
raise_browser_error(error) if error
|
49
50
|
response
|
@@ -52,14 +53,14 @@ module Ferrum
|
|
52
53
|
def on(event, &block)
|
53
54
|
case event
|
54
55
|
when *INTERRUPTIONS
|
55
|
-
@
|
56
|
+
@interrupter.on(event, &block)
|
56
57
|
else
|
57
58
|
@subscriber.on(event, &block)
|
58
59
|
end
|
59
60
|
end
|
60
61
|
|
61
62
|
def subscribed?(event)
|
62
|
-
[@
|
63
|
+
[@interrupter, @subscriber].any? { |s| s.subscribed?(event) }
|
63
64
|
end
|
64
65
|
|
65
66
|
def close
|
@@ -84,16 +85,16 @@ module Ferrum
|
|
84
85
|
# Node has disappeared while we were trying to get it
|
85
86
|
when "No node with given id found",
|
86
87
|
"Could not find node with given id"
|
87
|
-
raise NodeNotFoundError
|
88
|
+
raise NodeNotFoundError, error
|
88
89
|
# Context is lost, page is reloading
|
89
90
|
when "Cannot find context with specified id"
|
90
|
-
raise NoExecutionContextError
|
91
|
+
raise NoExecutionContextError, error
|
91
92
|
when "No target with given id found"
|
92
93
|
raise NoSuchPageError
|
93
94
|
when /Could not compute content quads/
|
94
95
|
raise CoordinatesNotFoundError
|
95
96
|
else
|
96
|
-
raise BrowserError
|
97
|
+
raise BrowserError, error
|
97
98
|
end
|
98
99
|
end
|
99
100
|
end
|
@@ -5,12 +5,12 @@ module Ferrum
|
|
5
5
|
class Command
|
6
6
|
NOT_FOUND = "Could not find an executable for the browser. Try to make " \
|
7
7
|
"it available on the PATH or set environment variable for " \
|
8
|
-
"example BROWSER_PATH=\"/usr/bin/chrome\""
|
8
|
+
"example BROWSER_PATH=\"/usr/bin/chrome\""
|
9
9
|
|
10
10
|
# Currently only these browsers support CDP:
|
11
11
|
# https://github.com/cyrus-and/chrome-remote-interface#implementations
|
12
12
|
def self.build(options, user_data_dir)
|
13
|
-
defaults = case options
|
13
|
+
defaults = case options.browser_name
|
14
14
|
when :firefox
|
15
15
|
Options::Firefox.options
|
16
16
|
when :chrome, :opera, :edge, nil
|
@@ -27,14 +27,16 @@ module Ferrum
|
|
27
27
|
def initialize(defaults, options, user_data_dir)
|
28
28
|
@flags = {}
|
29
29
|
@defaults = defaults
|
30
|
-
@options
|
31
|
-
@
|
32
|
-
|
30
|
+
@options = options
|
31
|
+
@user_data_dir = user_data_dir
|
32
|
+
@path = options.browser_path || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
|
33
|
+
raise BinaryNotFoundError, NOT_FOUND unless @path
|
34
|
+
|
33
35
|
merge_options
|
34
36
|
end
|
35
37
|
|
36
38
|
def xvfb?
|
37
|
-
!!options
|
39
|
+
!!options.xvfb
|
38
40
|
end
|
39
41
|
|
40
42
|
def to_a
|
@@ -45,12 +47,8 @@ module Ferrum
|
|
45
47
|
|
46
48
|
def merge_options
|
47
49
|
@flags = defaults.merge_required(@flags, options, @user_data_dir)
|
48
|
-
|
49
|
-
|
50
|
-
@flags = defaults.merge_default(@flags, options)
|
51
|
-
end
|
52
|
-
|
53
|
-
@flags.merge!(options.fetch(:browser_options, {}))
|
50
|
+
@flags = defaults.merge_default(@flags, options) unless options.ignore_default_browser_options
|
51
|
+
@flags.merge!(options.browser_options)
|
54
52
|
end
|
55
53
|
end
|
56
54
|
end
|
@@ -4,11 +4,8 @@ require "singleton"
|
|
4
4
|
|
5
5
|
module Ferrum
|
6
6
|
class Browser
|
7
|
-
|
7
|
+
class Options
|
8
8
|
class Base
|
9
|
-
BROWSER_HOST = "127.0.0.1"
|
10
|
-
BROWSER_PORT = "0"
|
11
|
-
|
12
9
|
include Singleton
|
13
10
|
|
14
11
|
def self.options
|
@@ -24,13 +21,7 @@ module Ferrum
|
|
24
21
|
end
|
25
22
|
|
26
23
|
def detect_path
|
27
|
-
|
28
|
-
self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
|
29
|
-
else
|
30
|
-
self.class::LINUX_BIN_PATH.find do |name|
|
31
|
-
path = Cliver.detect(name) and break(path)
|
32
|
-
end
|
33
|
-
end
|
24
|
+
Binary.find(self.class::PLATFORM_PATH[Utils::Platform.name])
|
34
25
|
end
|
35
26
|
|
36
27
|
def merge_required(flags, options, user_data_dir)
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Ferrum
|
4
4
|
class Browser
|
5
|
-
|
5
|
+
class Options
|
6
6
|
class Chrome < Base
|
7
7
|
DEFAULT_OPTIONS = {
|
8
8
|
"headless" => nil,
|
@@ -36,34 +36,45 @@ module Ferrum
|
|
36
36
|
"metrics-recording-only" => nil,
|
37
37
|
"safebrowsing-disable-auto-update" => nil,
|
38
38
|
"password-store" => "basic",
|
39
|
-
"no-startup-window" => nil
|
40
|
-
#
|
39
|
+
"no-startup-window" => nil
|
40
|
+
# NOTE: --no-sandbox is not needed if you properly setup a user in the container.
|
41
41
|
# https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
|
42
42
|
# "no-sandbox" => nil,
|
43
43
|
}.freeze
|
44
44
|
|
45
45
|
MAC_BIN_PATH = [
|
46
|
-
"/Applications/
|
47
|
-
"/Applications/
|
46
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
47
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium"
|
48
48
|
].freeze
|
49
|
-
LINUX_BIN_PATH = %w[
|
50
|
-
google-chrome
|
51
|
-
|
49
|
+
LINUX_BIN_PATH = %w[chrome google-chrome google-chrome-stable google-chrome-beta
|
50
|
+
chromium chromium-browser google-chrome-unstable].freeze
|
51
|
+
WINDOWS_BIN_PATH = [
|
52
|
+
"C:/Program Files/Google/Chrome/Application/chrome.exe",
|
53
|
+
"C:/Program Files/Google/Chrome Dev/Application/chrome.exe"
|
54
|
+
].freeze
|
55
|
+
PLATFORM_PATH = {
|
56
|
+
mac: MAC_BIN_PATH,
|
57
|
+
windows: WINDOWS_BIN_PATH,
|
58
|
+
linux: LINUX_BIN_PATH
|
59
|
+
}.freeze
|
52
60
|
|
53
61
|
def merge_required(flags, options, user_data_dir)
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
62
|
+
flags = flags.merge("remote-debugging-port" => options.port,
|
63
|
+
"remote-debugging-address" => options.host,
|
64
|
+
# Doesn't work on MacOS, so we need to set it by CDP
|
65
|
+
"window-size" => options.window_size&.join(","),
|
66
|
+
"user-data-dir" => user_data_dir)
|
67
|
+
|
68
|
+
if options.proxy
|
69
|
+
flags.merge!("proxy-server" => "#{options.proxy[:host]}:#{options.proxy[:port]}")
|
70
|
+
flags.merge!("proxy-bypass-list" => options.proxy[:bypass]) if options.proxy[:bypass]
|
71
|
+
end
|
72
|
+
|
73
|
+
flags
|
61
74
|
end
|
62
75
|
|
63
76
|
def merge_default(flags, options)
|
64
|
-
|
65
|
-
defaults = except("headless", "disable-gpu")
|
66
|
-
end
|
77
|
+
defaults = except("headless", "disable-gpu") unless options.headless
|
67
78
|
|
68
79
|
defaults ||= DEFAULT_OPTIONS
|
69
80
|
defaults.merge(flags)
|