webstract 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +69 -2
- data/lib/webstract/errors.rb +10 -1
- data/lib/webstract/favicon.rb +8 -9
- data/lib/webstract/screen_capture.rb +9 -4
- data/lib/webstract/screenshot.rb +27 -3
- data/lib/webstract/screenshot_backend.rb +6 -2
- data/lib/webstract/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2acd9f72a410808ca0c13b251ea4a90c6a8f16fe
|
4
|
+
data.tar.gz: 9bf28dbbd5dd17df779ef11bb8172808a5ba0620
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f477129c1811a7ffce4d1bf3e8d2a77555102d8eaef1a82d9d6ef0a277236cef491e3c87c43e91b38bea994c8d2232d142349993bcca3b6e995002520b1893e0
|
7
|
+
data.tar.gz: 6ea30f55206ac01ccbb9a30d43928bdc7adfe13960510f84b27f49a27c61ae48e777e193c2c4f61cf1dba009f8d41f983034f116f594190b0299ddcb2a0d795c
|
data/README.md
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# Webstract
|
2
2
|
|
3
|
-
TODO: Write a gem description
|
4
3
|
|
5
4
|
## Installation
|
6
5
|
|
@@ -10,6 +9,34 @@ Add this line to your application's Gemfile:
|
|
10
9
|
gem 'webstract'
|
11
10
|
```
|
12
11
|
|
12
|
+
```bash
|
13
|
+
#
|
14
|
+
# MAC OSX
|
15
|
+
#
|
16
|
+
brew install phantomjs
|
17
|
+
|
18
|
+
#
|
19
|
+
# Linux (Ubuntu/Debian)
|
20
|
+
#
|
21
|
+
sudo apt-get install libfreetype6 libfreetype6-dev
|
22
|
+
sudo apt-get install libfontconfig1
|
23
|
+
|
24
|
+
# now download from phantomjs website
|
25
|
+
export PHANTOM_JS="phantomjs-1.9.7-linux-x86_64"
|
26
|
+
wget https://bitbucket.org/ariya/phantomjs/downloads/$PHANTOM_JS.tar.bz2
|
27
|
+
|
28
|
+
# create your symlinks
|
29
|
+
sudo mv $PHANTOM_JS.tar.bz2 /usr/local/share/
|
30
|
+
cd /usr/local/share/
|
31
|
+
sudo tar xvjf $PHANTOM_JS.tar.bz2
|
32
|
+
sudo ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/share/phantomjs
|
33
|
+
sudo ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/local/bin/phantomjs
|
34
|
+
sudo ln -sf /usr/local/share/$PHANTOM_JS/bin/phantomjs /usr/bin/phantomjs
|
35
|
+
|
36
|
+
# now try it
|
37
|
+
phantomjs --version
|
38
|
+
```
|
39
|
+
|
13
40
|
And then execute:
|
14
41
|
|
15
42
|
$ bundle
|
@@ -20,7 +47,47 @@ Or install it yourself as:
|
|
20
47
|
|
21
48
|
## Usage
|
22
49
|
|
23
|
-
|
50
|
+
```ruby
|
51
|
+
|
52
|
+
#
|
53
|
+
# SCREENSHOTS
|
54
|
+
#
|
55
|
+
|
56
|
+
# setup your session
|
57
|
+
ws = Webstract.screenshot(
|
58
|
+
url: "https://www.example.com/admin",
|
59
|
+
path: "/tmp/sidekiq.png", # where to save it
|
60
|
+
width: 1024,
|
61
|
+
height: 768,
|
62
|
+
user_agent: :ios, # also (:web, :android)
|
63
|
+
accept_language: 'es', # capture the page in a specific language
|
64
|
+
basic_auth: {
|
65
|
+
username: 'example-user',
|
66
|
+
password: 'example-pass'
|
67
|
+
}
|
68
|
+
)
|
69
|
+
|
70
|
+
# now capture the page and save it to the filesystem
|
71
|
+
ws.capture
|
72
|
+
|
73
|
+
#
|
74
|
+
# FAVICONS
|
75
|
+
#
|
76
|
+
|
77
|
+
fav = Webstract.favicon(url: 'https://www.google.com')
|
78
|
+
fav.fetch_and_save("/tmp/google.ico")
|
79
|
+
|
80
|
+
# or just return raw data
|
81
|
+
fav.fetch
|
82
|
+
```
|
83
|
+
|
84
|
+
## Debugging
|
85
|
+
|
86
|
+
Just set the WEBSTRACT_DEBUG environment variable
|
87
|
+
|
88
|
+
```bash
|
89
|
+
WEBSTRACT_DEBUG=1 ruby my_screenshot_script.rb
|
90
|
+
```
|
24
91
|
|
25
92
|
## Contributing
|
26
93
|
|
data/lib/webstract/errors.rb
CHANGED
@@ -1,4 +1,13 @@
|
|
1
1
|
module Webstract
|
2
|
-
|
2
|
+
module Errors
|
3
|
+
|
4
|
+
class PathNotWritableError < StandardError; ;end
|
5
|
+
|
6
|
+
# raised whenever a webpage capture fails due to browser/connection client errors
|
7
|
+
class CaptureError < StandardError; ;end
|
8
|
+
|
9
|
+
# raised when the target page could not be reached
|
10
|
+
# but due to server errors like 404, or 500.
|
11
|
+
class PageError < StandardError; ;end
|
3
12
|
end
|
4
13
|
end
|
data/lib/webstract/favicon.rb
CHANGED
@@ -1,14 +1,5 @@
|
|
1
1
|
require 'faviconduit'
|
2
2
|
|
3
|
-
=begin
|
4
|
-
|
5
|
-
usage:
|
6
|
-
|
7
|
-
favicon = Webstract::Favicon.new(url: 'https://example.com')
|
8
|
-
favicon.fetch
|
9
|
-
|
10
|
-
=end
|
11
|
-
|
12
3
|
module Webstract
|
13
4
|
class Favicon
|
14
5
|
attr_accessor :url, :favicon_data
|
@@ -22,5 +13,13 @@ module Webstract
|
|
22
13
|
@favicon_data = Faviconduit.get(url).data
|
23
14
|
end
|
24
15
|
|
16
|
+
def fetch_and_save(path)
|
17
|
+
data = fetch
|
18
|
+
File.open(path, 'w') do |f|
|
19
|
+
f.write(data)
|
20
|
+
f.close
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
25
24
|
end
|
26
25
|
end
|
@@ -4,14 +4,18 @@ module Webstract
|
|
4
4
|
class ScreenCapture
|
5
5
|
include Capybara::DSL
|
6
6
|
|
7
|
-
attr_reader :width, :height, :user_agent, :accept_language, :path
|
7
|
+
attr_reader :width, :height, :user_agent, :accept_language, :path, :basic_auth
|
8
8
|
|
9
9
|
def initialize(opts = {})
|
10
10
|
Webstract::ScreenshotBackend.capybara_setup!
|
11
11
|
@width = opts.fetch(:width, Webstract::ScreenshotBackend.width)
|
12
12
|
@height = opts.fetch(:height, Webstract::ScreenshotBackend.height)
|
13
|
-
@user_agent = opts.fetch(:user_agent, Webstract::ScreenshotBackend.user_agent)
|
14
13
|
@accept_language = opts.fetch(:accept_language, Webstract::ScreenshotBackend.accept_language)
|
14
|
+
@basic_auth = opts.fetch(:basic_auth, Webstract::ScreenshotBackend.basic_auth)
|
15
|
+
|
16
|
+
ua = opts.fetch(:user_agent, Webstract::ScreenshotBackend.user_agent)
|
17
|
+
@user_agent = Webstract::ScreenshotBackend::USER_AGENTS[ua] if ua.is_a?(Symbol)
|
18
|
+
|
15
19
|
|
16
20
|
# Browser settings
|
17
21
|
page.driver.resize(@width, @height)
|
@@ -19,6 +23,7 @@ module Webstract
|
|
19
23
|
"User-Agent" => @user_agent,
|
20
24
|
'Accept-Language' => @accept_language
|
21
25
|
}
|
26
|
+
page.driver.basic_authorize(basic_auth[:username], basic_auth[:password]) if basic_auth
|
22
27
|
end
|
23
28
|
|
24
29
|
def start_session(&block)
|
@@ -49,11 +54,11 @@ module Webstract
|
|
49
54
|
if page.driver.status_code.to_i == 200 || page.driver.status_code.to_i / 100 == 3
|
50
55
|
page.driver.save_screenshot(path, :full => true)
|
51
56
|
else
|
52
|
-
raise Webstract::
|
57
|
+
raise Webstract::Errors::PageError.new("Could not fetch page: #{url.inspect}, error code: #{page.driver.status_code}")
|
53
58
|
end
|
54
59
|
rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient, Capybara::Poltergeist::TimeoutError, Errno::EPIPE => e
|
55
60
|
# TODO: Handle Errno::EPIPE and Errno::ECONNRESET
|
56
|
-
raise Webstract::
|
61
|
+
raise Webstract::Errors::CaptureError.new("Capybara error: #{e.message.inspect}")
|
57
62
|
end
|
58
63
|
end
|
59
64
|
end
|
data/lib/webstract/screenshot.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module Webstract
|
2
2
|
class Screenshot
|
3
|
-
attr_accessor :url, :
|
4
|
-
attr_reader :handle
|
3
|
+
attr_accessor :url, :width, :height, :user_agent, :accept_language, :basic_auth
|
4
|
+
attr_reader :handle, :path
|
5
5
|
|
6
6
|
def initialize(options = {})
|
7
7
|
@handle = Webstract::ScreenCapture.new(options)
|
@@ -10,10 +10,34 @@ module Webstract
|
|
10
10
|
setter = "#{k}="
|
11
11
|
self.public_send(setter, value) if self.respond_to?(setter)
|
12
12
|
end
|
13
|
+
path = options[:path] || raise(ArgumentError.new('path required'))
|
14
|
+
@basic_auth ||= {}
|
15
|
+
|
16
|
+
raise(ArgumentError.new("basic_auth credentials must be a Hash: got #{@basic_auth.inspect}")) unless @basic_auth.is_a?(Hash)
|
17
|
+
end
|
18
|
+
|
19
|
+
def path=(path)
|
20
|
+
dir = File.dirname(path)
|
21
|
+
raise(Webstract::Errors::PathNotWritableError.new('path must be writable.')) unless File.writable?(dir)
|
22
|
+
@path = path
|
23
|
+
end
|
24
|
+
|
25
|
+
def username=(username)
|
26
|
+
@basic_auth[:username] = username
|
27
|
+
end
|
28
|
+
|
29
|
+
def password=(password)
|
30
|
+
@basic_auth[:password] = password
|
13
31
|
end
|
14
32
|
|
15
33
|
def capture
|
16
|
-
handle.capture(url, path,
|
34
|
+
handle.capture(url, path,
|
35
|
+
width: width,
|
36
|
+
height: height,
|
37
|
+
user_agent: user_agent,
|
38
|
+
accept_language: accept_language,
|
39
|
+
basic_auth: basic_auth
|
40
|
+
)
|
17
41
|
end
|
18
42
|
|
19
43
|
end
|
@@ -24,15 +24,18 @@ module Webstract
|
|
24
24
|
mattr_accessor :accept_language
|
25
25
|
@@accept_language = 'en-us,en;q=0.5'
|
26
26
|
|
27
|
+
mattr_accessor :basic_auth
|
28
|
+
@@basic_auth = nil
|
29
|
+
|
27
30
|
|
28
31
|
# User agent
|
29
32
|
class << self
|
30
33
|
|
31
34
|
def user_agent
|
32
|
-
@user_agent ||=
|
35
|
+
@user_agent ||= USER_AGENTS[:web]
|
33
36
|
end
|
34
37
|
def user_agent=(ua)
|
35
|
-
agent_string =
|
38
|
+
agent_string = USER_AGENTS[ua]
|
36
39
|
raise(ArgumentError.new('must be one of #{USER_AGENTS.inspect}')) unless agent_string
|
37
40
|
@user_agent = agent_string
|
38
41
|
end
|
@@ -54,6 +57,7 @@ module Webstract
|
|
54
57
|
Capybara::Poltergeist::Driver.new(app, {
|
55
58
|
# Raise JavaScript errors to Ruby
|
56
59
|
js_errors: false,
|
60
|
+
debug: ENV['WEBSTRACT_DEBUG'] || false,
|
57
61
|
# Additional command line options for PhantomJS
|
58
62
|
phantomjs_options: ['--ignore-ssl-errors=yes'],
|
59
63
|
})
|
data/lib/webstract/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webstract
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Faucett
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-10-
|
11
|
+
date: 2014-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|