url-agent 1.0.0

data/lib/url-agent.rb

@@ -0,0 +1,20 @@
+require 'pathname'
+require 'logger'
+
+module URLAgent
+  class << self
+    attr_reader :root
+    attr_accessor :logger
+
+    def root=(path)
+      @root = Pathname.new("#{File.expand_path(path || Dir.pwd)}").realdirpath
+    end
+  end
+end
+
+require_relative 'url_agent/logger'
+require_relative 'url_agent/exceptions'
+require_relative 'url_agent/url'
+require_relative 'url_agent/url_set'
+require_relative 'url_agent/base'
+require_relative 'url_agent/pinger'

data/lib/url_agent/base.rb

@@ -0,0 +1,144 @@
+require 'singleton'
+require 'logger'
+require 'yaml'
+require 'uri'
+require 'active_support/core_ext/hash'
+
+require_relative 'dispatcher'
+
+class URLAgent::Base
+  include Singleton
+
+  attr_reader :logger, :pinger
+  attr_accessor :urls
+
+  def initialize(options = {})
+    @configured = false
+    @options = options
+  end
+
+  def configure(options = {})
+    return if configured?
+    configure!(options)
+  end
+
+  def configure!(options)
+    URLAgent.root = options[:path_root]
+
+    unless url_configuration_file.exist?
+      raise URLAgent::ConfigurationNotFoundException, "Cannot find #{url_configuration_file}"
+    end
+
+    @evented = begin
+      require_eventmachine_libraries
+    rescue LoadError
+      false
+    end
+
+    @urls ||= {}
+
+    url_configuration_load
+    initialize_logger
+
+    load_dispatcher
+    initialize_urls
+    configure_pinger
+
+    @configured = true
+  end
+
+  def configured?
+    @configured
+  end
+
+  def evented?
+    @evented
+  end
+
+  def [](identifier)
+    raise URLAgent::IdentifierNotFoundException unless @urls[identifier]
+
+    @urls[identifier].active
+  end
+
+  private
+
+  def initialize_logger
+    return unless @options[:logger]
+
+    logger_file = URLAgent.root.join(@options[:logger][:path])
+    log_level = if @options[:verbose]
+                  :DEBUG
+                else
+                  @options[:logger][:log_level]
+                end.upcase.to_sym
+
+    raise URLAgent::LogDirectoryNotPresent unless logger_file.dirname.exist?
+
+    @logger = URLAgent::Logger.new(logger_file)
+    @logger.info "URL Agent starting up..."
+    @logger.info "Log level is #{log_level}"
+    @logger.level = ActiveSupport::Logger::Severity.const_get(log_level)
+
+    # Set this up as a logger available at the root
+    URLAgent.logger = @logger
+  end
+
+  def initialize_urls
+    @urls = Hash[@urls.map { |key, data| [key, URLAgent::UrlSet.new(data, :url_set_identifier => key, :dispatcher => @dispatcher)] }]
+  end
+
+  def url_configuration_file
+    URLAgent.root.join('config', 'urls.yaml')
+  end
+
+  def require_eventmachine_libraries
+    require 'em-http-request'
+    require 'em-synchrony'
+    require 'em-synchrony/em-http'
+
+    # We don't test for reactor_running? here, that's defered to the call method
+    true
+  end
+
+  def url_configuration_load
+    @urls = {}
+    url_configuration = File.open(url_configuration_file, 'r') { |f| YAML.load(f.read) }
+    url_configuration.symbolize_keys!
+
+    url_configuration.each do |identifier, data|
+      case identifier
+      when :logger
+        @options[:logger] = data.with_indifferent_access
+      when :pinger
+        @options[:pinger] = data.with_indifferent_access
+      when :timeouts
+        @options[:timeouts] = data.with_indifferent_access
+      else
+        @urls[identifier] = data
+      end
+    end
+  end
+
+  def load_dispatcher
+    type = if evented?
+             :em_http
+           else
+             :net_http
+           end
+
+    logger.info "Using #{type} dispatcher"
+
+    @dispatcher = URLAgent::Dispatcher.new(@options.merge({ :type => type }))
+  end
+
+  def configure_pinger
+    return unless evented?
+    return unless @options[:pinger]
+
+    @pinger = URLAgent::Pinger.new(@urls, @options[:pinger])
+
+    logger.info "Pinger started"
+    @pinger.monitor
+  end
+end

data/lib/url_agent/dispatcher.rb

@@ -0,0 +1,20 @@
+class URLAgent::Dispatcher
+  attr_reader :proxy
+  
+  def initialize(options = {})
+    type = options.delete(:type) || :net_http
+    @proxy = case type
+      when :net_http
+        require_relative 'dispatcher/net_http'
+        URLAgent::NetHTTPDispatcher.new(options)
+      when :em_http
+        require_relative 'dispatcher/em_http'
+        URLAgent::EMHTTPDispatcher.new(options)
+      end
+  end
+  
+  def execute_request(http_verb, url_name, url, request_options = {})
+    @proxy.execute_request(http_verb, url_name, url, request_options)
+  end
+end
+

data/lib/url_agent/dispatcher/em_http.rb

@@ -0,0 +1,44 @@
+
+class URLAgent::EMHTTPDispatcher
+  def initialize(url_agent_options)
+    @connection_options ||= {} 
+    
+    if url_agent_options[:timeouts]
+      @connection_options[:connect_timeout] = url_agent_options[:timeouts][:connection]
+      @connection_options[:inactivity_timeout] = url_agent_options[:timeouts][:inactivity]
+    end
+  end
+  
+  def execute_request(http_verb, url_name, url, request_options = {})
+    check_for_eventmachine!
+    
+    logger.debug "#{http_verb.upcase}\t#{url_name}\t#{url}\t#{request_options.inspect}"
+    
+    request = EventMachine::HttpRequest.new(url, @connection_options).send(http_verb, request_options)
+    
+    unless request.error.to_s.empty?
+      logger.error "#{http_verb.upcase}\t#{url_name}\t#{url}\t#{request_options.inspect}\tConnection Error"
+      raise URLAgent::ConnectionError, request.error
+    end
+      
+    unless request.finished?
+      logger.error "#{http_verb.upcase}\t#{url_name}\t#{url}\t#{request_options.inspect}\tTimeout Exception"
+      raise URLAgent::TimeoutException
+    end
+    
+    logger.debug "#{http_verb.upcase}-RESPONSE\t#{url_name}\t#{url}\t#{request_options.inspect}\t#{request.response.length}\t#{request.response.inspect[0..200]}"
+    
+    request
+  end
+  
+  def logger
+    URLAgent.logger
+  end
+  
+  private
+  def check_for_eventmachine!(&block)
+    unless EM.reactor_running?
+      raise URLAgent::EMHTTPDispatcherReactorNotRunning, "The reactor is not running. You should wrap calls to the EMHTTPDispatcher around a EM.synchrony block"
+    end
+  end
+end

data/lib/url_agent/exceptions.rb

@@ -0,0 +1,23 @@
+class URLAgent::ConfigurationNotFoundException < Exception
+end
+
+class URLAgent::GemsNotFoundException < Exception
+end
+
+class URLAgent::IdentifierNotFoundException < Exception
+end
+
+class URLAgent::TimeoutException < Exception
+end
+
+class URLAgent::ConnectionError < Exception
+end
+
+class URLAgent::NoLiveURLFound < Exception
+end
+
+class URLAgent::EMHTTPDispatcherReactorNotRunning < Exception
+end
+
+class URLAgent::LogDirectoryNotPresent < Exception
+end

data/lib/url_agent/logger.rb

@@ -0,0 +1,11 @@
+require 'active_support/core_ext/date_time'
+require 'active_support/logger'
+
+class URLAgent::Logger < ActiveSupport::Logger
+
+  class ActiveSupport::Logger::SimpleFormatter
+    def call(severity, time, progname, msg)
+      msg = "#{Time.now.to_formatted_s(:db)}, #{severity} #{msg.strip}\n"
+    end
+  end
+end

data/lib/url_agent/pinger.rb

@@ -0,0 +1,49 @@
+require 'active_support/core_ext/hash'
+
+class URLAgent::Pinger
+  attr_accessor :url_sets, :pinger_configuration, :tick
+  
+  def initialize(url_sets, pinger_configuration)
+    @url_sets = url_sets
+    @pinger_configuration = pinger_configuration
+    @tick = @pinger_configuration[:tick] || 10
+  end
+  
+  def monitor
+    return unless EM.reactor_running?
+    
+    logger.debug "MONITOR"
+    EM.add_periodic_timer(@tick) { Fiber.new { check }.resume }
+  end
+  
+  def check
+    @url_sets.each do |url_set_identifier, set|     
+      url_configuration = pinger_configuration[url_set_identifier] 
+      url_parameters = url_configuration["params"].symbolize_keys
+      http_verb = (url_configuration['verb'] || "get").to_sym
+      
+      set.urls.each do |url_identifier, url|
+        begin
+          url_built = url.build(url_parameters)
+          
+          logger.debug "CHECK\t#{url_set_identifier}:#{url_identifier}\t#{url_built}"
+          connection = url_built.send(http_verb)
+          logger.debug "CHECK-RESPONSE\t#{url_set_identifier}:#{url_identifier}\t#{url_built}\t#{connection.response.length}\t#{connection.response.inspect[0..200]}"
+          
+          logger.info "ALIVE\t#{url_set_identifier}:#{url_identifier}\t#{url_built}"
+          url.alive!
+        rescue URLAgent::ConnectionError
+          logger.warn "DEAD\t#{url_set_identifier}:#{url_identifier}\t#{url_built}\tConnection Error"
+          url.dead!
+        rescue URLAgent::TimeoutException
+          logger.warn "DEAD\t#{url_set_identifier}:#{url_identifier}\t#{url_built}\tTimeout Exception"
+          url.dead!
+        end
+      end
+    end
+  end
+  
+  def logger
+    URLAgent.logger
+  end
+end

data/lib/url_agent/url.rb

@@ -0,0 +1,60 @@
+class URLAgent::Url
+  attr_accessor :live
+
+  def initialize(url, options = {})
+    @url = url
+
+    @dispatcher = options[:dispatcher]
+    @live = options[:live].nil? ? true : options[:live]
+    @url_set_identifier = options[:url_set_identifier]
+    @identifier = options[:identifier]
+
+    unless @dispatcher
+      raise ArgumentError, "Must provide a dispatcher to construct a callable URL."
+    end
+  end
+
+  def build(parameters = {})
+    url_with_parameters = @url.dup
+    parameters.each do |parameter_name, parameter_value|
+      url_with_parameters.gsub!("%%#{parameter_name}%%", URI.escape("#{parameter_value}"))
+    end
+
+    self.class.new(url_with_parameters, { :dispatcher => @dispatcher, :live => @live, :url_set_identifier => @url_set_identifier, :identifier => @identifier })
+  end
+
+  def to_s
+    @url.to_s
+  end
+
+  [:get, :head, :post, :put].each do |http_verb|
+    self.class_eval do
+      define_method http_verb do |request_options = {}|
+        logger.debug "DISPATCH\t#{url_name}\t#{self.to_s}\t#{__method__.upcase}"
+        @dispatcher.execute_request(http_verb, url_name, self.to_s, request_options)
+      end
+    end
+  end
+
+  alias :call :get
+
+  def dead!
+    @live = false
+  end
+
+  def alive!
+    @live = true
+  end
+
+  def alive?
+    @live
+  end
+
+  def logger
+    URLAgent.logger
+  end
+
+  def url_name
+    "#{@url_set_identifier}:#{@identifier}"
+  end
+end

data/lib/url_agent/url_set.rb

@@ -0,0 +1,26 @@
+class URLAgent::UrlSet
+  attr_reader :urls
+
+  def initialize(urls, options = {})
+    @urls = {}
+    urls.each do |identifier, url|
+      @urls[identifier] = URLAgent::Url.new(url, options.merge(:identifier => identifier))
+    end
+  end
+
+  def active
+    (@urls.select { |key, url| url.alive? }).values.first or raise URLAgent::NoLiveURLFound, "No active url found!"
+  end
+
+  def dead!(identifier)
+    @urls[identifier].dead!
+  end
+
+  def alive!(identifier)
+    @urls[identifier].alive!
+  end
+
+  def build(arguments = {})
+    active.build(arguments)
+  end
+end

metadata

@@ -0,0 +1,240 @@
+--- !ruby/object:Gem::Specification
+name: url-agent
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Vishnu Gopal
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-rcov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: flog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ci_reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0.beta
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: URL agent can handle common errors, concurrency and fallback for applications
+  that call a large number of URLs.
+email:
+- vishnu@mobme.in
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- examples/config/urls.yaml
+- examples/google_evented.rb
+- examples/log/urls.log
+- lib/url-agent.rb
+- lib/url_agent/base.rb
+- lib/url_agent/dispatcher.rb
+- lib/url_agent/dispatcher/em_http.rb
+- lib/url_agent/exceptions.rb
+- lib/url_agent/logger.rb
+- lib/url_agent/pinger.rb
+- lib/url_agent/url.rb
+- lib/url_agent/url_set.rb
+homepage: http://www.mobme.in/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: URL Agent acts as a proxy for managing URL access for applications
+test_files: []
+has_rdoc: