wordnik 4.07 → 4.08

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wordnik (4.06.15)
+    wordnik (4.08)
       activemodel (>= 3.0.3)
       addressable (>= 2.2.4)
       htmlentities (>= 4.2.4)
@@ -13,25 +13,26 @@ GEM
   remote: http://rubygems.org/
   specs:
     ZenTest (4.6.2)
-    activemodel (3.2.1)
-      activesupport (= 3.2.1)
+    activemodel (3.2.7)
+      activesupport (= 3.2.7)
       builder (~> 3.0.0)
-    activesupport (3.2.1)
+    activesupport (3.2.7)
       i18n (~> 0.6)
       multi_json (~> 1.0)
     addressable (2.2.6)
     autotest (4.4.6)
       ZenTest (>= 4.4.1)
     autotest-rails-pure (4.1.2)
-    builder (3.0.0)
+    builder (3.0.3)
     crack (0.1.8)
     diff-lcs (1.1.3)
     htmlentities (4.3.1)
-    i18n (0.6.0)
-    json (1.6.5)
-    mime-types (1.17.2)
-    multi_json (1.1.0)
-    nokogiri (1.5.0)
+    i18n (0.6.1)
+    json (1.7.5)
+    mime-types (1.19)
+    multi_json (1.3.6)
+    nokogiri (1.5.5)
+    rake (0.9.2.2)
     rspec (2.8.0)
       rspec-core (~> 2.8.0)
       rspec-expectations (~> 2.8.0)
@@ -40,6 +41,7 @@ GEM
     rspec-expectations (2.8.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.8.0)
+    ruby-prof (0.11.2)
     typhoeus (0.3.3)
       mime-types
     vcr (1.11.3)
@@ -53,7 +55,9 @@ PLATFORMS
 DEPENDENCIES
   autotest
   autotest-rails-pure
+  rake
   rspec (~> 2.8.0)
+  ruby-prof
   vcr (~> 1.11.3)
   webmock (>= 1.6.2)
   wordnik!

data/README.md

@@ -1,9 +1,9 @@
 wordnik rubygem
 ===============
 
-This is the official Wordnik rubygem. It fully wraps Wordnik's v4 API. Refer to 
-[developer.wordnik.com/docs](http://developer.wordnik.com/docs) to play around 
-in the live API sandbox. All the methods you see there are implemented in this 
+This is the official Wordnik rubygem. It fully wraps Wordnik's v4 API. Refer to
+[developer.wordnik.com/docs](http://developer.wordnik.com/docs) to play around
+in the live API sandbox. All the methods you see there are implemented in this
 ruby gem.
 
 Installation
@@ -69,7 +69,7 @@ Wordnik.words.search_words(:query => '*tin*', :include_part_of_speech => 'verb',
 ```
 
 For a full list of available methods, check out the [Wordnik API documentation](http://developer.wordnik.com/docs).
-When you make a request using our web-based API sandbox, the response output will show you how to make the 
+When you make a request using our web-based API sandbox, the response output will show you how to make the
 [equivalent ruby request](http://cl.ly/9FQY). w00t!
 
 Specs
@@ -77,14 +77,7 @@ Specs
 
 The wordnik gem uses rspec 2. To run the test suite, just type `rake` or `bundle exec rake spec` in the gem's base directory.
 
-Note
-----
-For testing locally, you will need to tunnel into the beta box
 
-  ssh -f -N -L 8001:localhost:8001 beta.wordnik.com
-
-And remember to update the spec_helper
-	
 Contributing
 ------------
 
@@ -95,16 +88,29 @@ Contributing
 * Commit and push until you are happy with your contribution
 * Make sure to add tests for the feature/bugfix. This is important so we don't break it in a future version unintentionally.
 
-Wishlist
---------
+Releasing
+---------
 
-* Go Kart
-* Helicopter
+```bash
+rake swagger
+open lib/version.rb           # bump the version number
+rake spec                     # test
+git commit -am "newness"      # commit
+git push origin master        # push
+rake release                  # release
+```
 
 Props
 -----
 
-* Thanks to [Jason Adams](http://twitter.com/#!/ealdent) for graciously turning 
+* Thanks to [Jason Adams](http://twitter.com/#!/ealdent) for graciously turning
 	over the [wordnik gem name](https://rubygems.org/gems/wordnik).
-* HTTP requests are made using [Typhoeus](https://github.com/dbalatero/typhoeus), 
-	a modern code version of the mythical beast with 100 serpent heads.
+* HTTP requests are made using [Typhoeus](https://github.com/dbalatero/typhoeus),
+	a modern code version of the mythical beast with 100 serpent heads.
+
+Notes
+-----
+
+* If you are using the Wordnik gem on [Heroku](http://www.heroku.com/), you'll need
+  to use a stack that is compatible with [Typhoeus](https://github.com/dbalatero/typhoeus).
+  As of 2012-08, this means the Cedar stack.

data/lib/wordnik.rb

@@ -6,6 +6,7 @@ require 'wordnik/resource'
 require 'wordnik/response'
 require 'wordnik/configuration'
 require 'wordnik/version'
+require 'wordnik/load_balancer'
 require 'logger'
 
 # http://blog.jayfields.com/2007/10/ruby-defining-class-methods.html
@@ -16,17 +17,17 @@ class Object
 end
 
 module Wordnik
-    
+
   class << self
-    
+
     # A Wordnik configuration object. Must act like a hash and return sensible
     # values for all Wordnik configuration options. See Wordnik::Configuration.
     attr_accessor :configuration
 
     attr_accessor :resources
-    
+
     attr_accessor :logger
-    
+
     # Call this method to modify defaults in your initializers.
     #
     # @example
@@ -43,7 +44,7 @@ module Wordnik
 
       # Configure logger.  Default to use Rails
       self.logger ||= configuration.logger || (defined?(Rails) ? Rails.logger : Logger.new(STDOUT))
-      
+
       # remove :// from scheme
       configuration.scheme.sub!(/:\/\//, '')
 
@@ -51,6 +52,15 @@ module Wordnik
       configuration.host.sub!(/https?:\/\//, '')
       configuration.host = configuration.host.split('/').first
 
+      # do the same if multiple hosts are specified
+      configuration.hosts = configuration.hosts.map{|host| host.sub(/https?:\/\//, '').split('/').first}
+
+      # create a load balancer if no load balancer specified && multiple hosts are specified ...
+      if !configuration.load_balancer && configuration.hosts.size > 0
+        self.logger.debug "Creating a load balancer from #{configuration.hosts.join(', ')}"
+        configuration.load_balancer = LoadBalancer.new(configuration.hosts)
+      end
+
       # Add leading and trailing slashes to base_path
       configuration.base_path = "/#{configuration.base_path}".gsub(/\/+/, '/')
       configuration.base_path = "" if configuration.base_path == "/"
@@ -59,17 +69,17 @@ module Wordnik
       # attach resources because they haven't been downloaded.
       if build
         self.instantiate_resources
-        self.create_resource_shortcuts 
+        self.create_resource_shortcuts
       end
     end
-    
+
     # Remove old JSON documentation and generated modules,
     # then download fresh JSON files.
     #
     def download_resource_descriptions
       system "rm api_docs/*.json"
       system "rm lib/wordnik/resource_modules/*.rb"
-      
+
       Wordnik::Request.new(:get, "resources.json").response.body['apis'].each do |api|
         resource_name = api['path'].split(".").first.gsub(/\//, '')
         description = api['description']
@@ -84,19 +94,19 @@ module Wordnik
     # 1. Instantiate a Resource object
     # 2. Stuff the Resource in Wordnik.resources
     #
-    def instantiate_resources   
+    def instantiate_resources
       self.resources = {}
       self.configuration.resource_names.each do |resource_name|
         name = resource_name.underscore.to_sym # 'fooBar' => :foo_bar
-        filename = File.join(File.dirname(__FILE__), "../api_docs/#{resource_name}.json")        
+        filename = File.join(File.dirname(__FILE__), "../api_docs/#{resource_name}.json")
         resource = Resource.new(
           :name => name,
           :raw_data => JSON.parse(File.read(filename))
         )
         self.resources[name] = resource
-      end      
+      end
     end
-    
+
     # Use some magic ruby dust to make nice method shortcuts.
     # Wordnik.word => Wordnik.resources[:word]
     # Wordnik.users => Wordnik.resources[:user]
@@ -109,39 +119,43 @@ module Wordnik
         end
       end
     end
-    
+
     def authenticated?
       Wordnik.configuration.user_id.present? && Wordnik.configuration.auth_token.present?
     end
-    
+
     def de_authenticate
       Wordnik.configuration.user_id = nil
       Wordnik.configuration.auth_token = nil
     end
-    
+
+    def clear_configuration
+      Wordnik.configuration = Configuration.new
+    end
+
     def authenticate
       return if Wordnik.authenticated?
-      
+
       if Wordnik.configuration.username.blank? || Wordnik.configuration.password.blank?
         raise ClientError, "Username and password are required to authenticate."
       end
-      
+
       request = Wordnik::Request.new(
-        :get, 
-        "account/authenticate/{username}", 
+        :get,
+        "account/authenticate/{username}",
         :params => {
-          :username => Wordnik.configuration.username, 
+          :username => Wordnik.configuration.username,
           :password => Wordnik.configuration.password
         }
       )
-      
+
       response_body = request.response.body
       Wordnik.configuration.user_id = response_body['userId']
       Wordnik.configuration.auth_token = response_body['token']
     end
 
   end
-  
+
 end
 
 class ServerError < StandardError

data/lib/wordnik/configuration.rb

@@ -7,35 +7,39 @@ module Wordnik
     attr_accessor :api_key
     attr_accessor :username
     attr_accessor :password
-    
+
     # TODO: Steal all the auth stuff from the old gem!
     attr_accessor :auth_token
     attr_accessor :user_id
-    
+
     # Response format can be 'json' (default) or 'xml'
     attr_accessor :response_format
-    
+
     # A comma-delimited list of the API's resources
     attr_accessor :resource_names
-    
+
     # The URL of the API server
     attr_accessor :scheme
     attr_accessor :host
+    attr_accessor :hosts # to do in process load balancing
+    attr_accessor :load_balancer
     attr_accessor :base_path
-    
+
     attr_accessor :user_agent
-    
+
     attr_accessor :proxy
     attr_accessor :proxy_username
     attr_accessor :proxy_password
 
     attr_accessor :logger
-    
+
     # Defaults go in here..
     def initialize
       @response_format = 'json'
       @scheme = 'http'
       @host = 'api.wordnik.com'
+      @hosts = []
+      @load_balancer = nil
       @base_path = '/v4'
       @user_agent = "ruby-#{Wordnik::VERSION}"
       # Build the default set of resource names from the filenames of the API documentation
@@ -47,7 +51,7 @@ module Wordnik
         raise "Problem loading the resource files in ./api_docs/"
       end
     end
-    
+
     def base_url
       Addressable::URI.new(
         :scheme => self.scheme,
@@ -56,6 +60,14 @@ module Wordnik
       )
     end
 
+    def clear
+      initialize
+    end
+
+    def host
+      @load_balancer ? @load_balancer.host : @host
+    end
+
   end
 
 end

data/lib/wordnik/load_balancer.rb

@@ -0,0 +1,71 @@
+module Wordnik
+
+  # the simple idea here of a load balancer is to keep a set of hosts
+  # around, and use the 'best' one. At Wordnik, we have a set of
+  # API hosts available to us (these servers are invisible to the general web)
+  # The class below implements least recently used.
+  #
+  # The Wordnik Configuration object will convert a :hosts specification
+  # into a LoadBalancer instance
+  #
+
+  #
+  # These should be thread safe.
+  class LoadBalancer
+
+    attr_reader :hosts
+    attr_accessor :all_hosts
+    attr_accessor :failed_hosts_table
+    attr_accessor :current_host
+
+    def initialize(hosts)
+      @all_hosts = hosts
+      @hosts = @all_hosts
+      @failed_hosts_table = {}
+      @current_host = nil
+    end
+
+    def host
+      @current_host = hosts.first
+      @hosts.rotate!
+      restore_failed_hosts_maybe
+      @current_host
+    end
+
+    def inform_failure
+        #Wordnik.logger.debug "Informing failure about #{@current_host}. table: #{@failed_hosts_table.inspect}"
+      if @failed_hosts_table.include?(@current_host)
+        failures, failed_time = @failed_hosts_table[@current_host]
+        @failed_hosts_table[@current_host] = [failures+1, failed_time]
+      else
+        @failed_hosts_table[@current_host] = [1, Time.now.to_f] # failure count, first failure time
+      end
+      #Wordnik.logger.debug "Informed failure about #{@current_host}. table now: #{@failed_hosts_table.inspect}"
+      @hosts.delete(@current_host)
+      @hosts = [@current_host] if @hosts.size == 0 # got to have something!
+    end
+
+    # success here means just that a successful connection was made
+    # and the website didn't time out.
+    def inform_success
+      @failed_hosts_table.delete(@current_host)
+      @hosts << @current_host unless @hosts.include? @current_host
+      @hosts
+    end
+
+    def restore_failed_hosts_maybe
+      return if @failed_hosts_table.size == 0
+      @failed_hosts_table.each do |host, pair|
+        failures, failed_time = pair
+        seconds_since_first_failure = (Time.now.to_f - failed_time)
+        #Wordnik.logger.debug "Seconds since #{host}'s first failure: #{seconds_since_first_failure} compared to #{2**(failures-1)}"
+        # exponential backoff, but try every hour...
+        if (seconds_since_first_failure > [3600, 2**(failures-1)].min)
+          @hosts << host # give it a chance to succeed ...
+          #Wordnik.logger.debug "Added #{host} to @hosts; now: #{@hosts}"
+        end
+      end
+    end
+  end
+
+end

data/lib/wordnik/request.rb

@@ -16,7 +16,7 @@ module Wordnik
 
     # All requests must have an HTTP method and a path
     # Optionals parameters are :params, :headers, :body, :format, :host
-    # 
+    #
     def initialize(http_method, path, attributes={})
       attributes[:format] ||= Wordnik.configuration.response_format
       attributes[:params] ||= {}
@@ -32,21 +32,21 @@ module Wordnik
       if attributes[:headers].present? && attributes[:headers].has_key?(:api_key)
         default_headers.delete(:api_key)
       end
-      
+
       # api_key from params hash trumps all others (headers and default_headers)
       if attributes[:params].present? && attributes[:params].has_key?(:api_key)
         default_headers.delete(:api_key)
         attributes[:headers].delete(:api_key) if attributes[:headers].present?
       end
-      
+
       # Merge argument headers into defaults
       attributes[:headers] = default_headers.merge(attributes[:headers] || {})
-      
+
       # Stick in the auth token if there is one
       if Wordnik.authenticated?
         attributes[:headers].merge!({:auth_token => Wordnik.configuration.auth_token})
       end
-            
+
       self.http_method = http_method.to_sym
       self.path = path
       attributes.each do |name, value|
@@ -56,20 +56,20 @@ module Wordnik
 
     # Construct a base URL
     #
-    def url(options = {})  
+    def url(options = {})
       u = Addressable::URI.new(
         :scheme => Wordnik.configuration.scheme,
         :host => Wordnik.configuration.host,
         :path => self.interpreted_path,
         :query => self.query_string.sub(/\?/, '')
       ).to_s
-      
+
       # Drop trailing question mark, if present
       u.sub! /\?$/, ''
-      
+
       # Obfuscate API key?
       u.sub! /api\_key=\w+/, 'api_key=YOUR_API_KEY' if options[:obfuscated]
-      
+
       u
     end
 
@@ -91,14 +91,14 @@ module Wordnik
       end
 
       p = p.sub("{format}", self.format.to_s)
-      
+
       URI.encode [Wordnik.configuration.base_path, p].join("/").gsub(/\/+/, '/')
     end
-  
+
     # Massage the request body into a state of readiness
     # If body is a hash, camelize all keys then convert to a json string
     #
-    def body=(value)      
+    def body=(value)
       if value.is_a?(Hash)
         value = value.inject({}) do |memo, (k,v)|
           memo[k.to_s.camelize(:lower).to_sym] = v
@@ -107,13 +107,13 @@ module Wordnik
       end
       @body = value
     end
-    
+
     # If body is an object, JSONify it before making the actual request.
-    # 
+    #
     def outgoing_body
       body.is_a?(String) ? body : body.to_json
     end
-    
+
     # Construct a query string from the query-string-type params
     def query_string
 
@@ -127,48 +127,69 @@ module Wordnik
         key = key.to_s.camelize(:lower).to_sym unless key.to_sym == :api_key    # api_key is not a camelCased param
         query_values[key] = value.to_s
       end
-    
+
       # We don't want to end up with '?' as our query string
       # if there aren't really any params
       return "" if query_values.blank?
-    
+
       # Addressable requires query_values to be set after initialization..
       qs = Addressable::URI.new
       qs.query_values = query_values
       qs.to_s
     end
-  
-    def make
-      request = Typhoeus::Request.new(self.url,
+
+    def make(attempt = 0)
+      # url is calculated, so we need to compute it once.
+      u = self.url
+      #Wordnik.logger.debug "Making attempt #{attempt}; now fetching #{u}" if attempt > 0
+      request = Typhoeus::Request.new(u,
         :headers => self.headers.stringify_keys,
         :method => self.http_method.to_sym)
-      
-      # Make request proxy-aware 
+
+      # Make request proxy-aware
       if Wordnik.configuration.proxy.present?
         request.proxy = Wordnik.configuration.proxy
         request.proxy_username = Wordnik.configuration.proxy_username if Wordnik.configuration.proxy_username.present?
         request.proxy_password = Wordnik.configuration.proxy_password if Wordnik.configuration.proxy_password.present?
       end
-      
-      Wordnik.logger.debug "\n  #{self.http_method.to_s.upcase} #{self.url}\n  body: #{self.outgoing_body}\n  headers: #{request.headers}\n\n"
-      
+
+      Wordnik.logger.debug "\n  #{self.http_method.to_s.upcase} #{u}\n  body: #{self.outgoing_body}\n  headers: #{request.headers}\n\n"
+
       request.body = self.outgoing_body unless self.http_method.to_sym == :get
 
-      # Execute the request
+      # Execute the request — blocking call here.
       Typhoeus::Hydra.hydra.queue request
-      Typhoeus::Hydra.hydra.run  
-      Response.new(request.response)
+      Typhoeus::Hydra.hydra.run
+
+      # if we are using local load balancing, check for timeouts and connection errors
+      resp = request.response
+
+      if Wordnik.configuration.load_balancer
+        if (resp.timed_out? || resp.code == 0)
+          # Wordnik.logger.debug "informing load balancer about failure"
+          Wordnik.configuration.load_balancer.inform_failure
+          if (attempt <= 3)
+            # Wordnik.logger.debug "Trying again after failing #{attempt} times..."
+            return make(attempt + 1) if attempt <= 3 # try three times to get a result...
+          end
+        else
+          # Wordnik.logger.debug "informing load balancer about success"
+          Wordnik.configuration.load_balancer.inform_success
+        end
+      end
+
+      Response.new(resp)
     end
-  
+
     def response
       self.make
     end
-  
+
     def response_code_pretty
       return unless @response.present?
-      @response.code.to_s    
+      @response.code.to_s
     end
-  
+
     def response_headers_pretty
       return unless @response.present?
       # JSON.pretty_generate(@response.headers).gsub(/\n/, '<br/>') # <- This was for RestClient