video_grabber 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +7 -5
- data/README.md +4 -3
- data/lib/video_grabber/config.rb +2 -1
- data/lib/video_grabber/scraper.rb +4 -3
- data/lib/video_grabber/version.rb +1 -1
- data/lib/video_grabber.rb +1 -0
- data/pkg/video_grabber-1.8.0.gem +0 -0
- data/video_grabber.gemspec +2 -1
- metadata +22 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: be3fad3a188a7f87df6e1da1aa2c2ab9047a4c3f
|
4
|
+
data.tar.gz: a166877391ef4641acd1c893dd47b12c8ff0f6ab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: edef2ec15170fd779860673c5ae004fe71c7bb786abc609db5633dbe1cebb64abca0266838582a75b55467e97d7f9b58fdbdaba8daeb89268552f78b64422ec9
|
7
|
+
data.tar.gz: 6e488a487db380a33869a39c2acf911842a1d4e909059672f86cf1c0dcbe00981b48282242c60c4d061d3d51bf5333adc8a4c750514b0275e69e4d9a555abf44
|
data/Gemfile.lock
CHANGED
@@ -1,16 +1,18 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
video_grabber (1.
|
4
|
+
video_grabber (1.8.0)
|
5
5
|
headless
|
6
6
|
nokogiri
|
7
|
-
|
7
|
+
phantomjs
|
8
|
+
pry
|
9
|
+
selenium-webdriver
|
8
10
|
watir
|
9
11
|
|
10
12
|
GEM
|
11
13
|
remote: https://rubygems.org/
|
12
14
|
specs:
|
13
|
-
childprocess (0.6.
|
15
|
+
childprocess (0.6.3)
|
14
16
|
ffi (~> 1.0, >= 1.0.11)
|
15
17
|
coderay (1.1.1)
|
16
18
|
commonwatir (4.0.0)
|
@@ -21,6 +23,7 @@ GEM
|
|
21
23
|
mini_portile2 (2.1.0)
|
22
24
|
nokogiri (1.7.1)
|
23
25
|
mini_portile2 (~> 2.1.0)
|
26
|
+
phantomjs (2.1.1.0)
|
24
27
|
pry (0.10.4)
|
25
28
|
coderay (~> 1.1.0)
|
26
29
|
method_source (~> 0.8.1)
|
@@ -57,10 +60,9 @@ PLATFORMS
|
|
57
60
|
|
58
61
|
DEPENDENCIES
|
59
62
|
bundler (~> 1.11)
|
60
|
-
pry
|
61
63
|
rake (~> 10.0)
|
62
64
|
rspec (~> 3.0)
|
63
65
|
video_grabber!
|
64
66
|
|
65
67
|
BUNDLED WITH
|
66
|
-
1.
|
68
|
+
1.14.6
|
data/README.md
CHANGED
@@ -30,7 +30,7 @@ irb(main):001:0> require 'video_grabber'
|
|
30
30
|
|
31
31
|
## Requirements
|
32
32
|
|
33
|
-
`video_grabber`
|
33
|
+
If you plan to use Firefox as your browser, `video_grabber` will then rely on [Headless](https://github.com/leonid-shevtsov/headless) (unless you specifically disable it). To get Headless working you will need linux and `xvfb`.
|
34
34
|
|
35
35
|
Install `xvfb` on Debian:
|
36
36
|
|
@@ -68,9 +68,10 @@ Or you can directly pass the param `keep_browser_open` during initialization.
|
|
68
68
|
- **url:** The url of the resource containing the video(s)
|
69
69
|
- **timeout:** *(default: 60)* The timeout for the scraper. Will trigger a `VideoGrabber::Timeout` if the delay is met.
|
70
70
|
- **keep_browser_open** *(default: false)* If activated, will keep the scraper's browser open as long as you do not stop it (using the `stop` public method.).
|
71
|
-
- **headless_enabled** *(default: true)* If disabled, will open your
|
71
|
+
- **headless_enabled** *(default: true)* If disabled, will open your browser to crawl your links.
|
72
|
+
- **browser** *(default: :firefox)* Set the browser which will be used to browse websites .
|
72
73
|
- **html_attributes** *(default: {controls: true})* This option enables you to pass html attributes that will be passed to your crawled links elements.
|
73
|
-
- **firefox_extension_path** If passed, your Scraper instance will run using the given extension (`.xpi` file). Useful if you want to benefit from an Adblocker for instance
|
74
|
+
- **firefox_extension_path** If passed, your Scraper instance will run using the given extension (`.xpi` file). Your browser has to be set at Firefox. Useful if you want to benefit from an Adblocker for instance
|
74
75
|
|
75
76
|
## Versioning
|
76
77
|
|
data/lib/video_grabber/config.rb
CHANGED
@@ -23,13 +23,14 @@ module VideoGrabber
|
|
23
23
|
class Configuration
|
24
24
|
|
25
25
|
attr_accessor :url, :keep_browser_open, :timeout, :headless_enabled,
|
26
|
-
:firefox_extension_path, :html_attributes
|
26
|
+
:firefox_extension_path, :html_attributes, :browser
|
27
27
|
|
28
28
|
def initialize
|
29
29
|
@keep_browser_open = false
|
30
30
|
@timeout = 60
|
31
31
|
@headless_enabled = true
|
32
32
|
@html_attributes = { controls: true }
|
33
|
+
@browser = :firefox
|
33
34
|
end
|
34
35
|
end
|
35
36
|
end
|
@@ -2,7 +2,7 @@ module VideoGrabber
|
|
2
2
|
class Scraper
|
3
3
|
|
4
4
|
attr_reader :url, :browser, :timeout, :keep_browser_open, :headless_enabled,
|
5
|
-
:firefox_extension_path, :profile, :html_attributes
|
5
|
+
:firefox_extension_path, :profile, :html_attributes, :browser_type
|
6
6
|
|
7
7
|
def initialize(config)
|
8
8
|
@keep_browser_open = config.keep_browser_open
|
@@ -11,6 +11,7 @@ module VideoGrabber
|
|
11
11
|
@headless_enabled = config.headless_enabled
|
12
12
|
@firefox_extension_path = config.firefox_extension_path
|
13
13
|
@html_attributes = config.html_attributes
|
14
|
+
@browser_type = config.browser
|
14
15
|
end
|
15
16
|
|
16
17
|
def start
|
@@ -53,10 +54,10 @@ module VideoGrabber
|
|
53
54
|
def open_browser
|
54
55
|
start_headless
|
55
56
|
|
56
|
-
@profile = ::Selenium::WebDriver::Firefox::Profile.new
|
57
|
+
@profile = ::Selenium::WebDriver::Firefox::Profile.new && load_extension if firefox_extension_path
|
57
58
|
client = Selenium::WebDriver::Remote::Http::Default.new
|
58
59
|
client.timeout = timeout
|
59
|
-
@browser = ::Watir::Browser.new(
|
60
|
+
@browser = ::Watir::Browser.new(browser_type, profile: profile, http_client: client)
|
60
61
|
end
|
61
62
|
|
62
63
|
def start_headless
|
data/lib/video_grabber.rb
CHANGED
Binary file
|
data/video_grabber.gemspec
CHANGED
@@ -26,10 +26,11 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.executables = ['video_grabber']
|
27
27
|
spec.require_paths = ['lib']
|
28
28
|
|
29
|
-
spec.add_dependency 'selenium-webdriver'
|
29
|
+
spec.add_dependency 'selenium-webdriver'
|
30
30
|
spec.add_dependency 'watir'
|
31
31
|
spec.add_dependency 'headless'
|
32
32
|
spec.add_dependency 'nokogiri'
|
33
|
+
spec.add_dependency 'phantomjs'
|
33
34
|
spec.add_dependency 'pry'
|
34
35
|
spec.add_development_dependency 'bundler', '~> 1.11'
|
35
36
|
spec.add_development_dependency 'rake', '~> 10.0'
|
metadata
CHANGED
@@ -1,29 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: video_grabber
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sidney
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: selenium-webdriver
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: watir
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: phantomjs
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: pry
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -147,6 +161,7 @@ files:
|
|
147
161
|
- lib/video_grabber/scraper.rb
|
148
162
|
- lib/video_grabber/version.rb
|
149
163
|
- pkg/video_grabber-1.1.0.gem
|
164
|
+
- pkg/video_grabber-1.8.0.gem
|
150
165
|
- pkg_checksum
|
151
166
|
- video_grabber-1.0.0.gem
|
152
167
|
- video_grabber.gemspec
|
@@ -171,9 +186,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
171
186
|
version: '0'
|
172
187
|
requirements: []
|
173
188
|
rubyforge_project:
|
174
|
-
rubygems_version: 2.5
|
189
|
+
rubygems_version: 2.4.5
|
175
190
|
signing_key:
|
176
191
|
specification_version: 4
|
177
192
|
summary: VideoGrabber is a simple tool to get video tags from a given URL
|
178
193
|
test_files: []
|
179
|
-
has_rdoc:
|