webstract 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66bbaffe0a470b990d080a7b5a8abd8c52f1b504
-  data.tar.gz: 25f7ff96946ce8cf6ce57abf5a41a3352be3373b
+  metadata.gz: 6243396588bd377a63b7bde8eda4753f74580d2e
+  data.tar.gz: 1a3a31eb9db5c4e83b197798d00a8c049ca6acb6
 SHA512:
-  metadata.gz: dd4c4ad6619ce8542105e27ec8e29eeb589656a3f7225365ee48d6f6f7df05f5c9da4e6e7fd1ffc88374c11173967453780b73ccad179bbd28f8bba39683cda3
-  data.tar.gz: 6219e01b43583d324d523cb9a214d6d6e7b9d963333e3bf27cc39757b25d3c64769e371b2a6d8ff2e8e6e0c62d5b3b9c5169a11a20c6d4e99ab4ddd6bd56b80e
+  metadata.gz: 4a1b8514db48474677b8a2c51e59a4ad2da3a61d4f482372d2cc467dedfc98fbb8ad34abaee5fdca140b45afcf9f32cceefb473932da24a25761571e462c2a7e
+  data.tar.gz: 463d81732822cb8dea73e1c8055d71bc9f9824d8dc335d78ed1ef8557a5f48e2149da8132e74acc373e787e5c3658e4f05231a4816364304ee33d4e96e3db166

data/lib/webstract.rb

@@ -1,7 +1,10 @@
 module Webstract
 
+  autoload :Errors, 'webstract/errors'
+  autoload :ScreenCapture, 'webstract/screen_capture'
+  autoload :ScreenshotBackend, 'webstract/screenshot_backend'
   autoload :Screenshot, 'webstract/screenshot'
-  autoload :Favicon, 'webstract/favicon'
+  autoload :Favicon,    'webstract/favicon'
 
   def self.screenshot(options = {})
     Webstract::Screenshot.new(options)

data/lib/webstract/errors.rb

@@ -0,0 +1,4 @@
+module Webstract
+  class Error < StandardError
+  end
+end

data/lib/webstract/screen_capture.rb

@@ -0,0 +1,60 @@
+require 'capybara/dsl'
+
+module Webstract
+  class ScreenCapture
+    include Capybara::DSL
+
+    attr_reader :width, :height, :user_agent, :accept_language, :path
+
+    def initialize(opts = {})
+      Webstract::ScreenshotBackend.capybara_setup!
+      @width  = opts.fetch(:width, Webstract::ScreenshotBackend.width)
+      @height = opts.fetch(:height, Webstract::ScreenshotBackend.height)
+      @user_agent = opts.fetch(:user_agent, Webstract::ScreenshotBackend.user_agent)
+      @accept_language = opts.fetch(:accept_language, Webstract::ScreenshotBackend.accept_language)
+
+      # Browser settings
+      page.driver.resize(@width, @height)
+      page.driver.headers = {
+        "User-Agent" => @user_agent,
+        'Accept-Language' => @accept_language
+      }
+    end
+
+    def start_session(&block)
+      Capybara.reset_sessions!
+      Capybara.current_session.instance_eval(&block) if block_given?
+      @session_started = true
+      self
+    end
+
+    # Captures a screenshot of +url+ saving it to +path+.
+    def capture(url, path, opts = {})
+      begin
+        # Default settings
+        @width   = opts.fetch(:width, 120)        if opts[:width]
+        @height  = opts.fetch(:height, 90)        if opts[:width]
+
+        # Reset session before visiting url
+        Capybara.reset_sessions! unless @session_started
+        @session_started = false
+
+        # Open page
+        visit(url)
+
+        # Timeout
+        sleep opts[:timeout] if opts[:timeout]
+
+        # Check response code
+        if page.driver.status_code.to_i == 200 || page.driver.status_code.to_i / 100 == 3
+          page.driver.save_screenshot(path, :full => true)
+        else
+          raise Webstract::Error.new("Could not fetch page: #{url.inspect}, error code: #{page.driver.status_code}")
+        end
+      rescue Capybara::Poltergeist::BrowserError, Capybara::Poltergeist::DeadClient, Capybara::Poltergeist::TimeoutError, Errno::EPIPE => e
+        # TODO: Handle Errno::EPIPE and Errno::ECONNRESET
+        raise Webstract::Error.new("Capybara error: #{e.message.inspect}")
+      end
+    end
+  end
+end

data/lib/webstract/screenshot.rb

@@ -1,23 +1,19 @@
-require 'webshot'
-
 module Webstract
   class Screenshot
-    attr_accessor :url, :path, :width, :height, :quality
+    attr_accessor :url, :path, :width, :height, :user_agent, :accept_language
     attr_reader :handle
 
     def initialize(options = {})
-      @handle = Webshot::Screenshot.instance
+      @handle = Webstract::ScreenCapture.new(options)
 
       options.each do |k, value|
         setter = "#{k}="
         self.public_send(setter, value) if self.respond_to?(setter)
       end
-
-      @quality = 85 unless self.quality
     end
 
     def capture
-      handle.capture(url, path, width: width, height: height, quality: quality)
+      handle.capture(url, path, width: width, height: height, user_agent: user_agent, accept_language: accept_language)
     end
 
   end

data/lib/webstract/screenshot_backend.rb

@@ -0,0 +1,64 @@
+require "capybara/dsl"
+require "capybara/poltergeist"
+require "active_support"
+require "active_support/core_ext"
+
+module Webstract
+  module ScreenshotBackend
+
+    USER_AGENTS = {
+      web: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31',
+      android: 'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 4 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19',
+      ios: 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3'
+    }
+
+    ## Browser settings
+    # Width
+    mattr_accessor :width
+    @@width = 1024
+
+    # Height
+    mattr_accessor :height
+    @@height = 768
+
+    mattr_accessor :accept_language
+    @@accept_language = 'en-us,en;q=0.5'
+
+
+    # User agent
+    class << self
+
+      def user_agent
+        @user_agent ||= USER_AGENT[:web]
+      end
+      def user_agent=(ua)
+        agent_string = USER_AGENT[ua]
+        raise(ArgumentError.new('must be one of #{USER_AGENTS.inspect}')) unless agent_string
+        @user_agent = agent_string
+      end
+
+    end
+
+    # Customize settings
+    def self.setup
+      yield(self)
+    end
+
+    # Capibara setup
+    def self.capybara_setup!
+      # By default Capybara will try to boot a rack application
+      # automatically. You might want to switch off Capybara's
+      # rack server if you are running against a remote application
+      Capybara.run_server = false
+      Capybara.register_driver :poltergeist do |app|
+        Capybara::Poltergeist::Driver.new(app, {
+          # Raise JavaScript errors to Ruby
+          js_errors: false,
+          # Additional command line options for PhantomJS
+          phantomjs_options: ['--ignore-ssl-errors=yes'],
+        })
+      end
+      Capybara.current_driver = :poltergeist
+    end
+  end
+end

data/lib/webstract/version.rb

@@ -1,3 +1,3 @@
 module Webstract
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,15 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: webstract
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - John Faucett
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-08 00:00:00.000000000 Z
+date: 2014-10-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: faviconduit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -39,33 +87,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: webshot
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '3'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: faviconduit
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: 0.10.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.0
 description: Extract images, favicons, and meta info from websites
 email:
 - jwaterfaucett@gmail.com
@@ -73,16 +121,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
 - lib/webstract.rb
+- lib/webstract/errors.rb
 - lib/webstract/favicon.rb
+- lib/webstract/screen_capture.rb
 - lib/webstract/screenshot.rb
+- lib/webstract/screenshot_backend.rb
 - lib/webstract/version.rb
-- webstract.gemspec
 homepage: 
 licenses:
 - MIT

data/.gitignore

@@ -1,14 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/
-*.bundle
-*.so
-*.o
-*.a
-mkmf.log

data/Gemfile

@@ -1,4 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in webstract.gemspec
-gemspec

data/Rakefile

@@ -1,2 +0,0 @@
-require "bundler/gem_tasks"
-

data/webstract.gemspec

@@ -1,24 +0,0 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'webstract/version'
-
-Gem::Specification.new do |spec|
-  spec.name          = "webstract"
-  spec.version       = Webstract::VERSION
-  spec.authors       = ["John Faucett"]
-  spec.email         = ["jwaterfaucett@gmail.com"]
-  spec.summary       = 'Extract information from websites'
-  spec.description   = 'Extract images, favicons, and meta info from websites'
-  spec.license       = 'MIT'
-
-  spec.files         = `git ls-files -z`.split("\x0")
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
-
-  spec.add_development_dependency 'bundler', '~> 1.7'
-  spec.add_development_dependency 'rake', '~> 10.0'
-
-  spec.add_runtime_dependency 'webshot'
-  spec.add_runtime_dependency 'faviconduit'
-end