mloughran-api_cache 0.1.1

data/README.markdown

@@ -0,0 +1,59 @@
+api_cache
+=========
+You want to use the Twitter API but you don't want to die? I have the solution to API caching:
+
+    APICache.get("http://twitter.com/statuses/public_timeline.rss")
+
+You get the following functionality for free:
+
+* New data every 10 minutes
+* If the twitter API dies then keep using the last data received for a day. Then assume it's invalid and announce that Twitter has FAILED (optional).
+* Don't hit the rate limit (70 requests per 60 minutes)
+
+So what exactly does `APICache` do? Given cached data less than 10 minutes old, it returns that. Otherwise, assuming it didn't try to request the URL within the last minute (to avoid the rate limit), it makes a get request to the Twitter API. If the Twitter API timeouts or doesn't return a 2xx code (very likely) we're still fine: it just returns the last data fetched (as long as it's less than a day old). In the exceptional case that all is lost and no data can be returned, it raises an `APICache::NotAvailableError` exception. You're responsible for catching this exception and complaining bitterly to the internet.
+
+All very simple. What if you need to do something more complicated? Say you need authentication or the silly API you're using doesn't follow a nice convention of returning 2xx for success. Then you need a block:
+
+    APICache.get('twitter_replies', :cache => 3600) do
+      Net::HTTP.start('twitter.com') do |http|
+        req = Net::HTTP::Get.new('/statuses/replies.xml')
+        req.basic_auth 'username', 'password'
+        response = http.request(req)
+        case response
+        when Net::HTTPSuccess
+          # 2xx response code
+          response.body
+        else
+          raise APICache::Invalid
+        end
+      end
+    end
+
+All the caching is still handled for you. If you supply a block then the first argument to `APICache.get` is assumed to be a unique key rather than a URL. Throwing `APICache::Invalid` signals to `APICache` that the request was not successful.
+
+You can send any of the following options to `APICache.get(url, options = {}, &block)`. These are the default values (times are all in seconds):
+
+    {
+      :cache => 600,    # 10 minutes  After this time fetch new data
+      :valid => 86400,  # 1 day       Maximum time to use old data
+                        #             :forever is a valid option
+      :period => 60,    # 1 minute    Maximum frequency to call API
+      :timeout => 5     # 5 seconds   API response timeout
+    }
+
+Before using the APICache you need to initialize the caches. In merb, for example, put this in your `init.rb`:
+
+    APICache.start
+
+Currently there are two stores available: `MemcacheStore` and `MemoryStore`. `MemcacheStore` is the default but if you'd like to use `MemoryStore`, or another store - see `AbstractStore`, just supply it to the start method:
+
+    APICache.start(APICache::MemoryStore)
+
+I suppose you'll want to get your hands on this magic. For now just grab the source from [github](http://github.com/mloughran/api_cache/tree/master) and `rake install`. I'll get a gem sorted soon.
+
+Please send feedback if you think of any other functionality that would be handy.
+
+Copyright
+=========
+
+Copyright (c) 2008 Martyn Loughran. See LICENSE for details.

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 1
+:major: 0
+:minor: 1

data/lib/api_cache/abstract_store.rb

@@ -0,0 +1,25 @@
+class APICache::AbstractStore
+  def initialize
+    raise "Method not implemented. Called abstract class."
+  end
+  
+  # Set value. Returns true if success.
+  def set(key, value)
+    raise "Method not implemented. Called abstract class."
+  end
+  
+  # Get value.
+  def get(key)
+    raise "Method not implemented. Called abstract class."
+  end
+  
+  # Does a given key exist in the cache?
+  def exists?(key)
+    raise "Method not implemented. Called abstract class."
+  end
+  
+  # Has a given time passed since the key was set?
+  def expired?(key, timeout)
+    raise "Method not implemented. Called abstract class."
+  end
+end

data/lib/api_cache/api.rb

@@ -0,0 +1,65 @@
+require 'net/http'
+
+# This class wraps up querying the API and remembers when each API was 
+# last queried in case there is a limit to the number that can be made.
+class APICache::API
+  def initialize
+    @query_times = {}
+  end
+  
+  # Checks whether the API can be queried (i.e. whether retry_time has passed
+  # since the last query to the API).
+  # 
+  # If retry_time is 0 then there is no limit on how frequently queries can
+  # be made to the API.
+  def queryable?(key, retry_time)
+    if @query_times[key]
+      if Time.now - @query_times[key] > retry_time
+        APICache.logger.log "Queryable: true - retry_time has passed"
+        true
+      else
+        APICache.logger.log "Queryable: false - queried too recently"
+        false
+      end
+    else
+      APICache.logger.log "Queryable: true - never used API before"
+      true
+    end
+  end
+  
+  # Fetch data from the API.
+  # 
+  # If no block is given then the key is assumed to be a URL and which will
+  # be queried expecting a 200 response. Otherwise the return value of the
+  # block will be used.
+  # 
+  # If the block is unable to fetch the value from the API it should raise
+  # APICache::Invalid.
+  def get(key, timeout, &block)
+    APICache.logger.log "Fetching data from the API"
+    @query_times[key] = Time.now
+    Timeout::timeout(timeout) do
+      if block_given?
+        # This should raise APICache::Invalid if it is not correct
+        yield
+      else
+        get_via_http(key, timeout)
+      end
+    end
+  rescue Timeout::Error, APICache::Invalid
+    raise APICache::CannotFetch
+  end
+  
+private
+  
+  def get_via_http(key, timeout)
+    response = Net::HTTP.get_response(URI.parse(key))
+    case response
+    when Net::HTTPSuccess
+      # 2xx response code
+      response.body
+    else
+      raise APICache::Invalid
+    end
+  end
+end

data/lib/api_cache/cache.rb

@@ -0,0 +1,41 @@
+# Cache performs calculations relating to the status of items stored in the
+# cache and delegates storage to the various cache stores.
+require 'digest/md5'
+class APICache::Cache
+  attr_accessor :store
+  def initialize(store)
+    @store = store.send(:new)
+  end
+  
+  # Returns one of the following options depending on the state of the key:
+  # 
+  # * :current (key has been set recently)
+  # * :refetch (data should be refetched but is still available for use)
+  # * :invalid (data is too old to be useful)
+  # * :missing (do data for this key)
+  def state(key, refetch_time, invalid_time)
+    if @store.exists?(encode(key))
+      if !@store.expired?(encode(key), refetch_time)
+        :current
+      elsif (invalid_time == :forever) || !@store.expired?(encode(key), invalid_time)
+        :refetch
+      else
+        :invalid
+      end
+    else
+      :missing
+    end
+  end
+  
+  def get(key)
+    @store.get(encode(key))
+  end
+  
+  def set(key, value)
+    @store.set(encode(key), value)
+    true
+  end
+  def encode(key)
+    Digest::MD5.hexdigest key
+  end
+end

data/lib/api_cache/logger.rb

@@ -0,0 +1,9 @@
+class APICache::Logger
+  def initialize
+    
+  end
+  
+  def log(message)
+    # puts "APICache: #{message}"
+  end
+end

data/lib/api_cache/memcache_store.rb

@@ -0,0 +1,54 @@
+require 'memcache'
+
+class APICache::MemcacheStore < APICache::AbstractStore
+  class NotReady < Exception #:nodoc:
+    def initialize
+      super("Memcache server is not ready")
+    end
+  end
+
+  class NotDefined < Exception #:nodoc:
+    def initialize
+      super("Memcache is not defined")
+    end
+  end
+  
+  def initialize
+    APICache.logger.log "Using memcached store"
+    namespace = 'api_cache'
+    host = '127.0.0.1:11211'
+    @memcache = MemCache.new(host, {:namespace => namespace})
+    raise NotReady unless @memcache.active?
+    true
+  # rescue NameError
+  #   raise NotDefined
+  end
+  
+  def set(key, data)
+    @memcache.set(key, data)
+    @memcache.set("#{key}_created_at", Time.now)
+    APICache.logger.log("cache: set (#{key})")
+    true
+  end
+  
+  def get(key)
+    data = @memcache.get(key)
+    APICache.logger.log("cache: #{data.nil? ? "miss" : "hit"} (#{key})")
+    data
+  end
+  
+  def exists?(key)
+    # TODO: inefficient - is there a better way?
+    !@memcache.get(key).nil?
+  end
+  
+  def expired?(key, timeout)
+    Time.now - created(key) > timeout
+  end
+  
+  private
+
+  def created(key)
+    @memcache.get("#{key}_created_at")
+  end
+end

data/lib/api_cache/memory_store.rb

@@ -0,0 +1,33 @@
+class APICache::MemoryStore < APICache::AbstractStore
+  def initialize
+    APICache.logger.log "Using memory store"
+    @cache = {}
+    true
+  end
+
+  def set(key, value)
+    APICache.logger.log("cache: set (#{key})")
+    @cache[key] = [Time.now, value]
+    true
+  end
+
+  def get(key)
+    data = @cache[key][1]
+    APICache.logger.log("cache: #{data.nil? ? "miss" : "hit"} (#{key})")
+    data
+  end
+
+  def exists?(key)
+    !@cache[key].nil?
+  end
+  
+  def expired?(key, timeout)
+    Time.now - created(key) > timeout
+  end
+
+  private
+
+  def created(key)
+    @cache[key][0]
+  end
+end

data/lib/api_cache.rb

@@ -0,0 +1,73 @@
+class APICache
+  class NotAvailableError < RuntimeError; end
+  class Invalid <  RuntimeError; end
+  class CannotFetch < RuntimeError; end
+  
+  class << self
+    attr_accessor :cache
+    attr_accessor :api
+    attr_accessor :logger
+  end
+  
+  # Initializes the cache
+  def self.start(store = APICache::MemcacheStore, logger = APICache::Logger.new)
+    APICache.logger = logger
+    APICache.cache  = APICache::Cache.new(store)
+    APICache.api    = APICache::API.new
+  end
+  
+  # Raises an APICache::NotAvailableError if it can't get a value. You should rescue this
+  # if your application code.
+  # 
+  # Optionally call with a block. The value of the block is then used to 
+  # set the cache rather than calling the url. Use it for example if you need
+  # to make another type of request, catch custom error codes etc. To signal
+  # that the call failed just raise APICache::Invalid - the value will then 
+  # not be cached and the api will not be called again for options[:timeout] 
+  # seconds. If an old value is available in the cache then it will be used.
+  # 
+  # For example:
+  #   APICache.get("http://twitter.com/statuses/user_timeline/6869822.atom")
+  # 
+  #   APICache.get \
+  #     "http://twitter.com/statuses/user_timeline/6869822.atom",
+  #     :cache => 60, :valid => 600
+  def self.get(key, options = {}, &block)
+    options = {
+      :cache => 600,    # 10 minutes  After this time fetch new data
+      :valid => 86400,  # 1 day       Maximum time to use old data
+                        #             :forever is a valid option
+      :period => 60,    # 1 minute    Maximum frequency to call API
+      :timeout => 5     # 5 seconds   API response timeout
+    }.merge(options)
+    
+    cache_state = cache.state(key, options[:cache], options[:valid])
+    
+    if cache_state == :current
+      cache.get(key)
+    else
+      begin
+        raise APICache::CannotFetch unless api.queryable?(key, options[:period])
+        value = api.get(key, options[:timeout], &block)
+        cache.set(key, value)
+        value
+      rescue APICache::CannotFetch
+        APICache.logger.log "Failed to fetch new data from API"
+        if cache_state == :refetch
+          cache.get(key)
+        else
+          APICache.logger.log "Data not available in the cache or from API"
+          raise APICache::NotAvailableError
+        end
+      end
+    end
+  end
+end
+
+require 'api_cache/cache'
+require 'api_cache/api'
+require 'api_cache/logger'
+
+APICache.autoload 'AbstractStore', 'api_cache/abstract_store'
+APICache.autoload 'MemoryStore', 'api_cache/memory_store'
+APICache.autoload 'MemcacheStore', 'api_cache/memcache_store'

data/spec/api_cache_spec.rb

@@ -0,0 +1,73 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe APICache do
+  before :each do
+    APICache.start(APICache::MemoryStore)
+    @key = 'random_key'
+    @encoded_key = "1520171c64bfb71a95c97d310eea3492"
+    @data = 'some bit of data'
+  end
+  
+  describe "get method" do
+    
+    it "should MD5 encode the cache key" do
+      APICache::Cache.new(APICache::MemoryStore).encode(@key).should == @encoded_key
+    end
+    it "should encode the cache key before calling the store" do
+      APICache.cache.should_receive(:state).and_return(:current)
+      APICache.cache.store.should_receive(:get).with(@encoded_key).and_return(@data)
+      APICache.cache.should_receive(:encode).with(@key).and_return(@encoded_key)
+      APICache.get(@key).should != @data
+    end
+    
+    it "should fetch data from the cache if the state is :current" do
+      APICache.cache.should_receive(:state).and_return(:current)
+      APICache.cache.should_receive(:get).and_return(@data)
+      
+      APICache.get(@key).should == @data
+    end
+    
+    it "should make new request to API if the state is :refetch" do
+      APICache.cache.should_receive(:state).and_return(:refetch)
+      APICache.api.should_receive(:get).with(@key, 5).and_return(@data)
+      
+      APICache.get(@key).should == @data
+    end
+    
+    it "should return the cached value if the api cannot fetch data and state is :refetch" do
+      APICache.cache.should_receive(:state).and_return(:refetch)
+      APICache.api.should_receive(:get).with(@key, 5).and_raise(APICache::CannotFetch)
+      APICache.cache.should_receive(:get).and_return(@data)
+      
+      APICache.get(@key).should == @data
+    end
+    
+    it "should make new request to API if the state is :invalid" do
+      APICache.cache.should_receive(:state).and_return(:invalid)
+      APICache.api.should_receive(:get).with(@key, 5).and_return(@data)
+      
+      APICache.get(@key).should == @data
+    end
+    
+    it "should raise an exception if the api cannot fetch data and state is :invalid" do
+      APICache.cache.should_receive(:state).and_return(:invalid)
+      APICache.api.should_receive(:get).with(@key, 5).and_raise(APICache::CannotFetch)
+      
+      lambda { APICache.get(@key).should }.should raise_error(APICache::NotAvailableError)
+    end
+    
+    it "should make new request to API if the state is :missing" do
+      APICache.cache.should_receive(:state).and_return(:missing)
+      APICache.api.should_receive(:get).with(@key, 5).and_return(@data)
+      
+      APICache.get(@key).should == @data
+    end
+    
+    it "should raise an exception if the api cannot fetch data and state is :missing" do
+      APICache.cache.should_receive(:state).and_return(:missing)
+      APICache.api.should_receive(:get).with(@key, 5).and_raise(APICache::CannotFetch)
+      
+      lambda { APICache.get(@key).should }.should raise_error(APICache::NotAvailableError)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+require "rubygems"
+require "api_cache"
+require "spec"

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: mloughran-api_cache
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Martyn Loughran
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-02-17 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: Library to handle caching external API calls
+email: me@mloughran.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.markdown
+- VERSION.yml
+- lib/api_cache
+- lib/api_cache/abstract_store.rb
+- lib/api_cache/api.rb
+- lib/api_cache/cache.rb
+- lib/api_cache/logger.rb
+- lib/api_cache/memcache_store.rb
+- lib/api_cache/memory_store.rb
+- lib/api_cache.rb
+- spec/api_cache_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/mloughran/api_cache
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Library to handle caching external API calls
+test_files: []
+