click_session 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8d180275fd38f126e826853293950674857701ee
+  data.tar.gz: 86ed88478e1de5c0b2fd245819617c82f56e5d4b
+SHA512:
+  metadata.gz: 481b241d4dab8796a7a9c8fcd78e535f184c396859f135369a6ac722dd768f03f3be35c2884dc7918c224ee5aec25695328756b04baf15f747707641da934856
+  data.tar.gz: 1abddccdb8cbbc18b1c87fb7444c836633c28835b6490d432368df4632197c61f135ee47bbd6c6f4bb64fa3e4688bb30e166a17b88d103ee8cc1a001a97d5989

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in click_session.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Tobias Talltorp
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,286 @@
+# click_session
+Turn any repeatable web navigation process into an api
+
+## Why? 
+Modern web apps rely more and more on html to be loaded asyncronously after the page has been loaded. The current solutions for automating a series of clicks, form posts and navigation changes relies on all html being rendered at once.
+
+The Capybara team has put a lot of thought into how these web apps can be tested and because of this, it also makes a good tool for scraping these web sites.
+
+## Installation
+
+## How to set up
+
+### Generate a migration
+`rails generate click_session:install`
+This will create a migration and generate an initializer with configuration parameters needed for click_session
+
+### Define the steps in a class
+Name the class ```ClickSessionRunner``` and add a method called ```run```.  
+
+This class must extend the ```WebRunner``` class
+
+The ```model``` is an ActiveRecord model which holds the data needed for the session. 
+
+```ruby
+class ClickSessionRunner < ClickSession::WebRunner
+  
+  # Steps to simulate
+  def run(model)
+    visit "https://www.stackoverflow.com"
+    fill_in "q", with: "Capybara"
+    press_enter_to_submit
+
+    model.name = first_search_result.text
+
+    model.save
+  end
+
+  private
+
+  def press_enter_to_submit
+    find_field('q').native.send_key(:enter)
+  end
+
+  def first_search_result
+    page.first(".summary")
+  end
+end
+```
+
+### Run session syncronously
+__Note:__ The response time for this type of request is totally dependant of the time it takes to visit all the pages. 
+
+```ruby
+user = User.new
+sync_click_session = ClickSession::Sync.new(user)
+result = sync_click_session.run
+# --> saves the User
+# --> run the steps in the ClickSessionRunner
+# --> result contains the serialized user data 
+```
+
+### Run session asyncronously
+```ruby
+user = User.new
+async_click_session = ClickSession::Async.new(user)
+result = async_click_session.run 
+# --> saves the User
+# --> saves the SessionState
+# --> result contains the ID of the saved SessionState
+
+# $ rake click_session:process_active
+# --> run the steps in the ClickSessionRunner
+
+# $ rake click_session:report_successful
+# --> the request sent contains the serialized user data 
+```
+
+### result hash
+Example:
+```
+{
+  id: 1234,   
+  status: {
+    success: true,          # Boolean
+  },
+  data: {                   # This is the output of the Serialized model
+    name: "Joe",
+    facebook_avatar: "http://fb.com/i/joe.png"
+  }
+}
+```
+The only optional part of the result is the ```data```.
+
+### Example of how to use it in a rails controller action
+```ruby
+def show
+  user = User.new
+  sync_click_session = ClickSession::Sync.new(user)
+  
+  result = sync_click_session.run
+
+  if result.status.success
+    render json: result.as_json, status: 201
+  else
+    render json: result.as_json, status: :unprocessable_entity
+  end
+end
+
+```
+
+## Mandatory configurations
+```ruby
+ClickSession.configure do | config |
+  config.model_class = YourModel
+end
+```
+
+## Optional configurations and extentions
+
+```ruby
+ClickSession.configure do | config |
+  config.processor_class = MyCustomProcessor
+  config.notifier_class = MyCustomNotifier
+  config.serializer_class = MyCustomSerializer
+  config.success_callback_url = "https://my.domain.com/webhook_success"
+  config.failure_callback_url = "https://my.domain.com/webhook_failure"
+  config.enable_screenshot = false # true
+  config.screenshot = {
+    s3_bucket: ENV['S3_BUCKET'],
+    s3_key_id: ENV['S3_KEY_ID'],
+    s3_access_key: ENV['S3_ACCESS_KEY']
+  }
+  config.driver_client = :poltergeist # :selenium
+end
+```
+
+Option  | Description 
+------- | ------------
+```notifier_class``` | The name of the class with your [custom notifications](#define-how-you-want-to-be-notified)
+```serializer_class``` | The name of the class with your [custom serializer](#define-how-you-want-to-serialize-the-result)
+```success_callback_url``` | The url you want us to ```POST``` to with the  successful result.  Only needed when using ```AsyncClickSession```
+```failure_callback_url``` | The url you want us to ```POST``` to with the error message.  Only needed when using ```AsyncClickSession```
+```enable_screenshot``` | Must be set to true if you want to save screenshots.
+```screenshot``` | A hash containing the configuration information needed to be able to save screenshots. ```s3_bucket```, ```s3_key_id``` and ```s3_access_key``` are all required.
+```driver_client``` | The driver you want to use to run the ClickSession. ```:poltergeist``` is the default, but ```:selenium``` is a good choice if you are developing in a local environment and want to see the browser appear.
+
+
+### Define how you want to serialize the result
+The serializer class takes the ```model``` that you accociated with the click_session and lets you transform it to whatever structure you like.  
+
+If you don't specify this class, we do a simple ```.as_json``` of the model and return that as the serialized result.
+
+This can be good when there might be things that you save on the model that are not needed in the result, such as generated tokens or simple placeholders of data.
+
+```ruby
+class MyUserSerializer
+  def serialize(model)
+    api_user = {
+      name: model.name,
+      facebook_avatar: model.user_image
+    }
+
+    api_user.as_json
+  end
+end
+```
+
+### Define how you want to be notified
+We will notify you when the following things happen
+* The ClickSession was successfully completed
+* The ClickSession failed because the max number of retries to run the SessionRunner has been exceeded
+* The status of the ClickSession was successfully reported back to the webhook
+* The max number of retries to report the asyncronous result (success or failure) back to your web hook has been exceeded.
+* Every time we rescue an error
+
+If this class is not defined, the information is logged to ```stdout```
+
+All of these notifications are executed after the model has been successfully persisted. 
+
+```ruby
+# Override any number of methods to 
+# customize the behaviour of the notifications
+
+class MyCustomNotifier < ClickSession::Notifier
+  def session_successful(session)
+    # Post to slack channel
+    # Send an email to the boss
+    super # log to stdout
+  end
+
+  def session_failed(session)
+    # Post to "alerts" channel on slack
+    # Send email to developers
+  end
+
+  def session_reported(session)
+    # Post to slack
+  end
+
+  def session_failed_to_report(session)
+    # Send email to developers
+    # Alert operations!
+  end
+
+  def rescued_error(e)
+    # Send the error to airbrake
+  end
+end
+```
+
+All other types of possible errors must be handled by your own code.
+
+### Save screen shots to S3
+If you have enabled screenshots in your configuration, we will take a screenshot after the run has been successful or failed.
+
+__Note:__ This requires you to add the S3 credentials and bucket name to the configuration
+
+
+## Rake tasks
+### click_session:process_active
+Processes all the active click sessions, meaning the ones that are ready to be run.
+
+__Note:__ Only needed for ```ClickSession::Async```
+
+### click_session:report_succesful
+Reports all click_sessions, which were successfully run, to the configured ```success_callback_url```
+
+__Note:__ Only needed for ```ClickSession::Async```
+
+### click_session:report_failed
+Reports all click_sessions, which failed to run, to the configured ```failure_callback_url```
+
+__Note:__ Only needed for ```ClickSession::Async```
+
+### click_session:report_failed
+Reports all click_sessions, which failed to run, to the configured ```failure_callback_url```
+
+__Note:__ Only needed for ```ClickSession::Async```
+
+### click_session:validate (not yet implemented)
+Runs the steps you defined that validates that the steps in the session has not changed.
+
+```ruby
+class ClickSessionRunner < WebRunner
+  def run(search_result_model)
+    # ...
+  end
+
+  def validate(model)
+    visit "http://www.google.com"
+
+    unless search_field_accesible?
+      raise ValidateClickSessionError("There are no results!!")
+    end
+  end
+
+  private
+  def search_field_accesible?
+    page.find("input[name='query']") != nil
+  end
+end
+```
+
+## Dependencies
+This gem is dependant on you having a browser installed which can be run by the capybara driver.
+
+We have tested it with
+* poltergeist (PhantomJS)
+* selenium (FireFox)
+
+## Deployment
+If you like to deploy your code to heroku, you need to use the ```build-pack-multi```.
+
+Create a file in the root of your application called ```.buildpacks```with this content
+```
+https://github.com/stomita/heroku-buildpack-phantomjs
+https://github.com/heroku/heroku-buildpack-ruby
+```
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/click_session/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/click_session.gemspec

@@ -0,0 +1,36 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'click_session/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "click_session"
+  spec.version       = ClickSession::VERSION
+  spec.authors       = ["Tobias Talltorp"]
+  spec.email         = ["tobias@talltorp.se"]
+  spec.summary       = "Navigates the web for you"
+  spec.description   = "Navigates the web for you"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'rails', '>= 4.1'
+  spec.add_dependency 'capybara', "~> 2.4"
+  spec.add_dependency 'poltergeist', "~> 1.6"
+  spec.add_dependency 'rest-client', "~> 1.7"
+  spec.add_dependency 'state_machine', "~> 1.2"
+  spec.add_dependency 'aws-sdk-v1', "~> 1.63"
+  spec.add_dependency 'selenium-webdriver', "~> 2.45"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec-rails", "~> 3.1"
+  spec.add_development_dependency "factory_girl_rails", "~> 4.2"
+  spec.add_development_dependency 'shoulda-matchers'
+  spec.add_development_dependency 'webmock', '~> 1.18'
+  spec.add_development_dependency "sqlite3"
+end

data/lib/click_session/async.rb

@@ -0,0 +1,45 @@
+module ClickSession
+  class Async
+    attr_reader :model
+    attr_accessor :click_session
+
+    def initialize(model)
+      @model = model
+    end
+
+    def run
+      validate_async_configuration
+
+      @click_session = SessionState.create(model: model)
+      serialize_success_response
+    end
+
+    private
+
+    def validate_async_configuration
+      if success_callback_missing? || failure_callback_missing?
+        raise ConfigurationError.new("You need to configure the callback URLs in order to use the AsyncClickSession")
+      end
+    end
+
+    def success_callback_missing?
+      ClickSession.configuration.success_callback_url == nil
+    end
+
+    def failure_callback_missing?
+      ClickSession.configuration.failure_callback_url == nil
+    end
+
+    def serialize_success_response
+      serializer.serialize_success(click_session)
+    end
+
+    def serialize_failure_response
+      serializer.serialize_failure(click_session)
+    end
+
+    def serializer
+      @serializer ||= ClickSession::ResponseSerializer.new
+    end
+  end
+end

data/lib/click_session/click_session_processor.rb

@@ -0,0 +1,64 @@
+require_relative './exceptions'
+
+module ClickSession
+  class ClickSessionProcessor
+    attr_accessor :click_session
+    attr_reader :web_runner_processor, :notifier, :screenshot_enabled, :screenshot_options
+
+    def initialize(click_session, web_runner_processor, notifier, options = {})
+      @click_session = click_session
+      @web_runner_processor = web_runner_processor
+      @notifier = notifier
+      @screenshot_enabled = options[:screenshot_enabled] || false
+      @screenshot_options = options[:screenshot_options] || nil
+    end
+
+    def process
+      validate_screenshot_configuration
+
+      begin
+        process_provided_steps_in_session
+      rescue TooManyRetriesError => e
+        take_care_of_failed_session
+        raise e
+      end
+
+      click_session
+    end
+
+    private
+
+    def validate_screenshot_configuration
+      if screenshot_enabled
+        if screenshot_options == nil
+          raise ConfigurationError.new(<<-ERROR.strip_heredoc)
+            In order to save screenshots, you need to enter s3 information
+            in the 'screenshot' option of the configuration
+            See https://github.com/talltorp/click_session for more information.
+          ERROR
+        end
+      end
+    end
+
+    def process_provided_steps_in_session
+      web_runner_processor.process(click_session.model)
+      click_session.success!
+      notifier.session_successful(click_session)
+
+      if screenshot_enabled
+        click_session.screenshot_url = web_runner_processor.save_screenshot(click_session.id)
+        click_session.save
+      end
+    end
+
+    def take_care_of_failed_session
+      click_session.failure!
+      notifier.session_failed(click_session)
+
+      if screenshot_enabled
+        click_session.screenshot_url = web_runner_processor.save_screenshot(click_session.id)
+        click_session.save
+      end
+    end
+  end
+end

data/lib/click_session/configuration.rb

@@ -0,0 +1,142 @@
+module ClickSession
+  @@configuration = nil
+
+  def self.configure
+    @@configuration = Configuration.new
+
+    if block_given?
+      yield configuration
+    end
+
+    configuration
+  end
+
+  def self.configuration
+    @@configuration || configure
+  end
+
+  class Configuration
+    attr_accessor :processor_class,
+      :success_callback_url,
+      :failure_callback_url,
+      :serializer_class
+
+    def model_class
+      if @model_class == nil
+        raise NameError.new(<<-ERROR.strip_heredoc, 'model_class')
+          To use ClickSession, you must define the name of the active model
+          you want ClickSession to operate on.
+          See https://github.com/talltorp/click_session for more information.
+        ERROR
+      end
+
+      @model_class.constantize
+    end
+
+    def model_class=(klass)
+      @model_class = klass.to_s
+    end
+
+    def processor_class
+      @processor_class ||=
+        begin
+          if Kernel.const_defined?(:ClickSessionRunner)
+            "ClickSessionRunner"
+          else
+            raise NameError.new(<<-ERROR.strip_heredoc, 'ClickSessionRunner')
+              To use ClickSession, you must either define `ClickSessionRunner` or configure a
+              different processor. See https://github.com/talltorp/click_session for
+              more information.
+            ERROR
+          end
+        end
+
+      @processor_class.constantize
+    end
+
+    def processor_class=(klass)
+      @processor_class = klass.to_s
+    end
+
+    def serializer_class
+      @serializer_class ||= "ClickSession::WebhookModelSerializer"
+      @serializer_class.constantize
+    end
+
+    def serializer_class=(klass)
+      @serializer_class = klass.to_s
+    end
+
+    def notifier_class
+      @notifier_class ||= "ClickSession::Notifier"
+      constantized_notifier = @notifier_class.constantize
+
+      if notifier_class_violates_interface(constantized_notifier)
+        raise ArgumentError.new(<<-ERROR.strip_heredoc)
+          Your custom notifier must inherit ClickSession::Notifier
+          See https://github.com/talltorp/click_session
+        ERROR
+      end
+
+      constantized_notifier
+    end
+
+    def notifier_class=(klass)
+      @notifier_class = klass.to_s
+    end
+
+    def driver_client
+      @driver_client ||= :poltergeist
+    end
+
+    def driver_client=(client)
+      @driver_client = client
+    end
+
+    def screenshot_enabled?
+      @screenshot_enabled ||= false
+    end
+
+    def enable_screenshot=(enable)
+      @screenshot_enabled = enable
+    end
+
+    def screenshot
+      if @screenshot == nil
+        raise ArgumentError.new("In order to save screenshots, you need to configure \
+          the information.
+          https://github.com/talltorp/click_session#optional-configurations-and-extentions
+          ")
+      end
+
+      @screenshot
+    end
+
+    def screenshot=(screenshot)
+      if screenshot[:s3_bucket] == nil
+        raise ArgumentError.new("The s3_bucket is required")
+      end
+
+      if screenshot[:s3_key_id] == nil
+        raise ArgumentError.new("The s3_key_id is required")
+      end
+
+      if screenshot[:s3_access_key] == nil
+        raise ArgumentError.new("The s3_access_key is required")
+      end
+
+      @screenshot = screenshot
+    end
+
+    private
+
+    def notifier_class_violates_interface(notifier_class)
+      required_methods = ClickSession::Notifier.instance_methods(false)
+      actual_methods = notifier_class.instance_methods
+
+      intersection = required_methods - actual_methods
+
+      intersection.length != 0
+    end
+  end
+end

data/lib/click_session/exceptions.rb

@@ -0,0 +1,12 @@
+module ClickSession
+  # A general ClickSession exception
+  class Error < StandardError; end
+
+  # Raised when the threshold has been reached for how many retries we
+  # make of running the steps provided before we consider the entire
+  # session a failure
+  class TooManyRetriesError < Error; end
+
+  # Raised when the configuration is not properly initialized
+  class ConfigurationError < Error; end
+end

data/lib/click_session/failure_status_reporter.rb

@@ -0,0 +1,15 @@
+module ClickSession
+  class FailureStatusReporter < StatusReporter
+    def initialize(
+      webhook = Webhook.new(ClickSession.configuration.failure_callback_url)
+    )
+      super(webhook)
+    end
+
+    def report(click_session)
+      raise ArgumentError unless click_session.failed_to_process?
+
+      super(click_session)
+    end
+  end
+end

data/lib/click_session/notifier.rb

@@ -0,0 +1,23 @@
+module ClickSession
+  class Notifier
+    def session_successful(click_session)
+      puts "SUCCESS: #{click_session.id} completed"
+    end
+
+    def session_failed(click_session)
+      $stderr.puts "FAILURE: #{click_session.id} failed"
+    end
+
+    def session_reported(click_session)
+      puts "REPORTED: #{click_session.id} successfully reported"
+    end
+
+    def session_failed_to_report(click_session)
+      $stderr.puts "REPORT_FAIL: #{click_session.id} failed to report"
+    end
+
+    def rescued_error(e)
+      puts "#{e.class.name}: #{e.message}"
+    end
+  end
+end