rattlecache 0.1

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc
+test.rb
+.idea/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rattlecache.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2011 Marv Cool
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+Rattlecache is an abstraction layer between the battle.net API and Libraries (in particular the ruby gem "battlnet") that consume it.
+It supports multiple backends (in which the json objects are stored) and takes care of expiring timers for the objects.

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = ['--color', '-f progress', '-r ./spec/spec_helper.rb']
+  t.pattern = 'spec/**/*_spec.rb'
+  t.fail_on_error = false
+end

data/lib/backend_manager.rb

@@ -0,0 +1,49 @@
+module Rattlecache
+  module Backend
+
+    class InvalidBackend < Exception; end
+
+    extend self
+
+    attr_reader :backends
+    
+    @backends = {
+      :filesystem => "Filesystem"
+      #:redis => "Redis",
+      #:somOtherCoolBackend => "someOther"
+    }
+
+    def fetch(backend_name)
+      unless @backends.include?(backend_name)
+        raise InvalidBackend.new("#{backend_name.to_s} is not a valid backend!")
+      end
+
+      backend_class = @backends[backend_name]
+      return load_backend(backend_name, backend_class)
+    end
+
+    def register(identifier, klass)
+      @backends[identifier] = klass
+    end
+
+    private
+
+      def load_backend(backend_name, klass_name)
+        begin
+          klass = Rattlecache::Backend.const_get("#{klass_name}", false)
+        rescue NameError
+          begin
+            backend_file = "backends/#{backend_name.to_s}"
+            require backend_file
+            klass = Rattlecache::Backend.const_get("#{klass_name}", false)
+          rescue LoadError
+            raise InvalidBackend.new("backend #{klass_name} does not exist, and file #{backend_file} does not exist")
+          rescue NameError
+            raise InvalidBackend.new("expected #{backend_file} to define Rattlecache::Backend::#{klass_name}")
+          end
+        end
+
+        return klass.new
+      end
+  end
+end

data/lib/backends/filesystem.rb

@@ -0,0 +1,55 @@
+require 'json'
+require 'base64'
+
+module Rattlecache
+  module Backend
+    class Filesystem < Cache
+
+      def initialize(prefix = "/tmp/rattlecache/")
+        @prefix = prefix
+      end
+
+      def get(objectKey)
+        #puts "Filesystem gets you: #{objectKey}"
+        # get the file from the filesystem
+        ret = {:status => 404, :lastModified => nil, :object => nil}
+        begin
+          f = open_file(objectKey)
+          firstline = f.first
+          object = f.read
+          mtime = f.mtime
+          f.close
+          ret = {:status => 200,
+                 :object => object,
+                 :lastModified => mtime,
+                 :header => Base64.strict_decode64(firstline.chomp)
+          }
+        rescue Errno::ENOENT
+        end
+        return ret
+      end
+
+      def post(object)
+        # put the file to the filesysten
+        firstline = Base64.strict_encode64(object[:header].to_json)
+        #puts "Debug: putting headers as firstline: #{firstline}"
+        f = open_file(object[:key],"w")
+        f.puts(firstline)
+        f.puts(object[:data])
+        f.close
+        #puts "Debug: filesystem posted #{object[:key]}"
+      end
+
+      def open_file(objectKey,how="r")
+        begin
+          Dir.mkdir(@prefix) unless File.directory?(@prefix)
+          File.open(@prefix+objectKey,how)
+        rescue
+          # raise this to the caller
+          raise
+        end
+      end
+
+    end
+  end
+end

data/lib/caches/Auctionscache.rb

@@ -0,0 +1,79 @@
+module Rattlecache
+  class Auctionscache < Cache
+
+
+    # @param backend [Rattlecache::Backend]
+    # @param adapter [Battlenet::Adapter::AbstractAdapter]
+    def initialize(backend, adapter)
+      @adapter = adapter
+      @backend = backend
+    end
+
+    # @param url [String]
+    # @return [String|nil]
+    def get(url,header)
+
+      cached_data = @backend.get(sanitize(url))
+
+      # Check if any fields are requested.
+      # If yes, request without fields and look into ['lastModified']
+      # if we need to fetch new or answer with our cached result.",
+      if cached_data[:status] == 200
+        puts "Debug: Auctionscache: cache HIT for #{url}"
+        unless is_lmt_and_url_request(url)
+          puts "Debug: Auctionscache: has fields. Timethings: #{cached_data[:lastModified]} < #{get_lmt(url,header)} #{cached_data[:lastModified] < get_lmt(url,header)}"
+          if cached_data[:lastModified] < get_lmt(url,header)
+            cached_data = {:status => 404}
+          end
+        else
+          # if a guild is requested without additional fields,
+          # simply check if this file is still valid
+          if generic_needs_request?(cached_data[:header],cached_data[:lastModified])
+            cached_data = {:status => 404}
+         end
+        end
+      else
+        puts "Debug: Auctionscache: cache MISS for #{url}"
+      end
+      cached_data
+    end
+
+    # check if the supplied url is not pointing to a .json auctions file,
+    # but to a generic url for lmt and the .json url
+    def is_lmt_and_url_request(url)
+      not url.include?(".json")
+    end
+
+    # method to get the lastModified info for a specified url
+    # @param url [String] the url for which the LMT is wanted
+    # @param header [Hash] the Hash of headers for a api request
+    def get_lmt(url,header)
+      cached_data = @backend.get(sanitize(generic_auctions_url(url)))
+      puts "Debug: get_lmt() cache result for generic url: #{cached_data[:status]}"
+      # if the object was not in the cache or it is expired
+      if cached_data[:status] != 200 or generic_needs_request?(cached_data[:header],cached_data[:lastModified])
+        # request it from the api
+        request_generic(url,header)
+        # and load the new object
+        cached_data = @backend.get(sanitize(generic_auctions_url(url)))
+      end
+      # this a JS milliseconds timestamp, we only do seconds!
+      Time.at(JSON.parse(cached_data[:object])["lastModified"]/1000)
+    end
+
+    def request_generic(url,header)
+      # need to request the generic guild response
+      url = generic_auctions_url(url)
+      # request it from the API:
+      got = request_raw(url,header)
+      @backend.post({:key => sanitize(url),:header => got.header.to_hash, :data => got.body}) # and put into cache
+    end
+
+
+    def generic_auctions_url(url)
+      u = URI.parse(url)
+      u.path=(u.path.gsub("-data","/data").gsub(/\/auctions.*\.json$/,""))
+      u.to_s
+    end
+  end
+end

data/lib/caches/Fieldsrequestcache.rb

@@ -0,0 +1,73 @@
+module Rattlecache
+  class Fieldsrequestcache < Cache
+
+
+    # @param backend [Rattlecache::Backend]
+    # @param adapter [Battlenet::Adapter::AbstractAdapter]
+    def initialize(backend, adapter)
+      @adapter = adapter
+      @backend = backend
+    end
+
+    # @param url [String]
+    # @return [String|nil]
+    def get(url,header)
+
+      cached_data = @backend.get(sanitize(url))
+
+      # Check if any fields are requested.
+      # If yes, request without fields and look into ['lastModified']
+      # if we need to fetch new or answer with our cached result.",
+      if cached_data[:status] == 200
+        puts "Debug: guildcache: cache HIT for #{url}"
+        if has_fields?(url)
+          puts "Debug: guildcache: has fields. Timethings: #{cached_data[:lastModified]} < #{get_lmt(url,header)} #{cached_data[:lastModified] < get_lmt(url,header)}"
+          if cached_data[:lastModified] < get_lmt(url,header)
+            cached_data = {:status => 404}
+          end
+        else
+          # if a guild is requested without additional fields,
+          # simply check if this file is still valid
+          if generic_needs_request?(cached_data[:header],cached_data[:lastModified])
+            cached_data = {:status => 404}
+         end
+        end
+      else
+        puts "Debug: guildcache: cache MISS for #{url}"
+      end
+      cached_data
+    end
+
+    # method to get the lastModified info for a specified url
+    # @param url [String] the url for which the LMT is wanted
+    # @param header [Hash] the Hash of headers for a api request
+    def get_lmt(url,header)
+      cached_data = @backend.get(sanitize(url_without_fields(url)))
+      puts "Debug: get_lmt() cache result for generic url: #{cached_data[:status]}"
+      # if the object was not in the cache or it is expired
+      if cached_data[:status] != 200 or generic_needs_request?(cached_data[:header],cached_data[:lastModified])
+        # request it from the api
+        request_without_fields(url,header)
+        # and load the new object
+        cached_data = @backend.get(sanitize(url_without_fields(url)))
+      end
+      # this a JS milliseconds timestamp, we only do seconds!
+      Time.at(JSON.parse(cached_data[:object])["lastModified"]/1000)
+    end
+
+    def request_without_fields(url,header)
+      # need to request the generic guild response
+      url = url_without_fields(url)
+      # request it from the API:
+      got = request_raw(url,header)
+      @backend.post({:key => sanitize(url),:header => got.header.to_hash, :data => got.body}) # and put into cache
+    end
+
+    def url_without_fields(url)
+      url_obj = URI.parse(url)
+      url_obj.query=(url_obj.query.gsub(/fields=.+&|$/,"")) # strip the fields
+      url_obj.to_s
+    end
+
+  end
+end

data/lib/rattlecache.rb

@@ -0,0 +1,144 @@
+require 'backend_manager'
+require 'net/http'
+require 'digest/sha2'
+
+module Rattlecache
+
+  class Cache
+
+    # @param backend [Symbol]
+    # @param adapter [Battlenet::Adapter::AbstractAdapter]
+    def initialize(backend = :filesystem, adapter = nil)
+      puts "Debug: rattlecache adapter: #{adapter}"
+      @adapter = adapter
+      @backend = Rattlecache::Backend.fetch(backend)
+    end
+
+    @request_pragmas = {
+      "guild" => "Check if any fields are requested. If yes, request without fields and look into ['lastModified'] if we need to fetch new or answer with out cached result.",
+      "data" => "Everything data related should be considered stable. Cache it for one week.",
+      "auction" => "dont know yet...",
+      "else" => "cache it for as long as 'RETRY-AFTER'-responseheader told us. (600 sec)"
+    }
+
+    # @param url [String]
+    # @param header [Hash]
+    def get(url, header = nil)
+      @header = header
+      @header['User-Agent'] = @header['User-Agent'] << " (with rattlecache)"
+      #puts "Cache class gets you: #{objectKey}"
+
+      # what to do with this request?
+      case request_type(url)
+        when "guild"
+          puts "Debug: its a guild related request!"
+          require 'caches/Fieldsrequestcache'
+          Rattlecache::Fieldsrequestcache.new(@backend,@adapter).get(url,header)
+        when "character"
+          puts "Debug: its a character related request!"
+          require 'caches/Fieldsrequestcache'
+          Rattlecache::Fieldsrequestcache.new(@backend,@adapter).get(url,header)
+        when /auction.*/
+          puts "Debug: its a auchtion related request!"
+          require 'caches/Auctionscache'
+          Rattlecache::Auctionscache.new(@backend,@adapter).get(url,header)
+        when "item"
+          # for items it seems reasonable to cache them at least for a week
+          # a week in seconds: 60*60*24*7 = 604800
+          check_and_return(@backend.get(sanitize(url)),604800)
+        else
+          puts "Debug: its a boring request!"
+          check_and_return(@backend.get(sanitize(url)))
+      end
+    end
+
+    def check_and_return(backend_result,given_time = nil)
+      if given_time.nil?
+        if backend_result[:status] == 200 and generic_needs_request?(backend_result[:header],backend_result[:lastModified])
+          backend_result = {:status => 404}
+        end
+      else
+        if backend_result[:status] == 200 and needs_request_with_given_time?(given_time,backend_result[:lastModified])
+          backend_result = {:status => 404}
+        end
+      end
+      backend_result
+    end
+
+    # @param object [Hash]
+    def post(object)
+      #puts "Cache class puts: #{object[:key]}"
+      @backend.post({:key => sanitize(object[:key]), :header => object[:header], :data => object[:data]})
+    end
+
+    # @param headerline [Hash]
+    # @param mtime [Time]
+    # @return [TrueClass|FalseClass]
+    def generic_needs_request?(headerline,mtime)
+      header = JSON.parse(headerline)
+      #header["date"][0] is a String with CGI.rfc1123_date() encoded time,
+      # as there is no easy method to inverse this coding, I will keep using the files mtime
+      # to estimate when the data was last recieved.
+      unless header["cache-control"].nil?
+        mtime+header["cache-control"][0].split("=")[1].to_i < Time.now()
+      else
+        unless header["retry-after"].nil?
+          mtime+(header["retry-after"][0].to_i) < Time.now()
+        else
+          # if we dont find any hint, pull it again!
+          puts "Warning: Cache couldn't find any hint if this object is still valid!"
+          true
+        end
+      end
+    end
+
+    def needs_request_with_given_time?(given_time,mtime)
+      mtime+given_time < Time.now
+    end
+
+    # @param objectKey [String]
+    # @return [String]
+    def sanitize(objectKey)
+      # strip scheme, sort paramters and encode for safty
+      urlObj = URI.parse(objectKey)
+      key = urlObj.host
+      key << urlObj.path
+      key << sort_params(urlObj.query)
+      Digest::SHA256.hexdigest(key)
+    end
+
+    # @param query [String]
+    # @return [String]
+    def sort_params(query)
+      q = Hash.new
+      query.split("&").each do |parampair|
+        q[parampair.split("=")[0]] = parampair.split("=")[1]
+      end
+      s = Array.new
+      q.sort.each { |pair| s << pair.join("=")}
+      "?"+s.join("&")
+    end
+
+    # @param objectKey [String]
+    # @return [String]
+    def request_type(objectKey)
+      #[0] = "". [1]= "api". [2]="wow", [3]= what we want
+      URI.parse(objectKey).path.split("/")[3]
+    end
+
+    # @param query [String]
+    # @return [TrueClass|FalseClass]
+    def has_fields?(query)
+      not query.scan(/fields=/).empty?
+    end
+
+    # @param url [String]
+    # @param header [Hash]
+    # @return [Net::HTTPResponse]
+    def request_raw(url,header)
+      req = @adapter.get(url,header,true)
+      req.get(url,header)
+    end
+
+  end
+end

data/rattlecache.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "rattlecache"
+  s.version     = "0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Marv Cool"]
+  s.email       = "marv@hostin.is"
+  s.homepage    = "https://github.com/MrMarvin/rattlcache/wiki"
+  s.summary     = %q{A smart caching system for battlenet API Requests.}
+
+  if s.respond_to?(:add_development_dependency)
+    s.add_development_dependency "rspec"
+  end
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: rattlecache
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Marv Cool
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-23 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: 
+email: marv@hostin.is
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/backend_manager.rb
+- lib/backends/filesystem.rb
+- lib/caches/Auctionscache.rb
+- lib/caches/Fieldsrequestcache.rb
+- lib/rattlecache.rb
+- rattlecache.gemspec
+has_rdoc: true
+homepage: https://github.com/MrMarvin/rattlcache/wiki
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A smart caching system for battlenet API Requests.
+test_files: []
+