mack-caching 0.8.1 → 0.8.2

data/lib/gems/cachetastic-2.0.0/bin/cachetastic_drb_server

@@ -0,0 +1,115 @@
+#!/usr/local/bin/ruby
+require 'thread'
+require 'drb'
+require 'singleton'
+require 'optparse'
+require 'optparse/time'
+require 'ostruct'
+
+# require the necessary gems:
+require 'rubygems'
+gem 'mack-facets'
+require 'mack-facets'
+gem 'configatron'
+require 'configatron'
+gem 'cachetastic'
+require 'cachetastic'
+
+$SAFE = 1
+
+SERVER_OPTIONS = OpenStruct.new
+SERVER_OPTIONS.host = "127.0.0.1"
+SERVER_OPTIONS.port = "61676"
+SERVER_OPTIONS.config_file = nil
+SERVER_OPTIONS.verbose = false
+SERVER_OPTIONS.really_verbose = false
+
+opts = OptionParser.new do |opts|
+  
+  opts.on("-h [HOST]") do |v|
+    SERVER_OPTIONS.host = v
+  end
+  
+  opts.on("-p [PORT]", Integer) do |v|
+    SERVER_OPTIONS.port = v
+  end
+  
+  opts.on("-c [CONFIG_FILE]") do |v|
+    SERVER_OPTIONS.config_file = v
+  end
+  
+  opts.on("-v") do |v|
+    SERVER_OPTIONS.verbose = true
+  end
+  
+  opts.on("-rv") do |v|
+    SERVER_OPTIONS.verbose = true
+    SERVER_OPTIONS.really_verbose = true
+  end
+  
+end
+
+opts.parse!(ARGV)
+
+if SERVER_OPTIONS.config_file
+  configatron.configure_from_yaml(SERVER_OPTIONS.config_file)
+end
+
+module Cachetastic
+  module Drb
+    class Server
+      include Singleton
+
+      def initialize
+        DRb.start_service("druby://#{SERVER_OPTIONS.host}:#{SERVER_OPTIONS.port}", self)
+        puts "Cachetastic::Drb::Server listening for connection..."
+        DRb.thread.join
+      end # initialize
+
+      def get(ns, key)
+        puts "get: #{ns}.#{key}" if SERVER_OPTIONS.verbose
+        puts cache_for_namespace(ns).inspect if SERVER_OPTIONS.really_verbose
+        value = cache_for_namespace(ns).get(key)
+        puts "\tvalue.class: #{value.class}" if SERVER_OPTIONS.verbose
+        value
+      end
+
+      def set(ns, key, value, expiry = 0)
+        puts "set: #{ns}.#{key}: #{value}" if SERVER_OPTIONS.verbose
+        cache_for_namespace(ns).set(key, value)
+        puts cache_for_namespace(ns).inspect if SERVER_OPTIONS.really_verbose
+        value
+      end
+
+      def delete(ns, key, delay = 0)
+        puts "delete: #{ns}.#{key}" if SERVER_OPTIONS.verbose
+        cache_for_namespace(ns).delete(key, delay = 0)
+        puts cache_for_namespace(ns).inspect if SERVER_OPTIONS.really_verbose
+      end
+
+      def expire_all(ns)
+        puts "expire_all: #{ns}" if SERVER_OPTIONS.verbose
+        instance_variable_set("@store_for_#{ns}", Cachetastic::Adapters::LocalMemory.new("store_for_#{ns}"))
+      end
+
+      def ping
+        true
+      end
+
+      private
+      def cache_for_namespace(ns)
+        # puts "@store_for_#{ns}"
+        c = instance_variable_get("@store_for_#{ns}")
+        if c.nil?
+          instance_variable_set("@store_for_#{ns}", Cachetastic::Adapters::LocalMemory.new("store_for_#{ns}"))
+          c = instance_variable_get("@store_for_#{ns}")
+        end
+        c
+      end
+
+    end # Server
+  end # Drb
+end # Cachetastic
+
+# let's start `er up!
+Cachetastic::Drb::Server.instance

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/base.rb

@@ -0,0 +1,78 @@
+# This class is the interface used to develop adapters for stores.
+# Stores are where the data is actually held. These could be local memory,
+# memcached, a database, the file system, etc...
+# If you implement this API, then you should be able to plug in different
+# stores for your caches without having to change any of your code.
+# 
+# === Methods that need to be implemented:
+# * setup - used to setup the implementation of the adapter.
+# * set(key, object, expiry) - sets an object into the store using the given key and the expiry time.
+# * get(key) - returns an object from the store for a given key.
+# * delete(key, delay) - deletes an object from the store for a given key. If the store supports it, a delay can be used.
+# * expire_all - expires all objects in the store for a given cache.
+# * stats - returns statistics for the store.
+# * valid? - used to test whether or not the store is still valid. If this returns false a new instance of the adapter is created by Cachetastic::Connection
+class Cachetastic::Adapters::Base
+  
+  # attr_reader :all_options
+  # attr_reader :store_options
+  # attr_reader :servers
+  attr_reader :name
+  # attr_reader :logging
+  attr_reader :logger
+  
+  def initialize(name)
+    @name = name
+    @logger = Cachetastic::Logger.new(configuration.retrieve(:logger, ::Logger.new(STDOUT)))
+    setup
+    if self.debug?
+      self.logger.debug(self.name, :self, self.inspect)
+    end
+  end
+  
+  needs_method :setup
+  needs_method :set
+  needs_method :get
+  needs_method :delete
+  needs_method :expire_all
+  needs_method :stats
+  needs_method :valid?
+  
+  # Returns true/or falsed based on whether or not the debug setting is set to true in the
+  # configuration file. If the config setting is set, then false is returned.
+  def debug?
+    ivar_cache(:debug) do
+      configuration.retrieve(:debug, false)
+    end
+  end
+  
+  def stats
+    cache_name = self.name.to_s.camelize
+    adapter_type = self.class.to_s.gsub('Cachetastic::Adapters::', '')
+    s = "Cache: #{cache_name}\nStore Type: #{adapter_type}\n"
+    if self.servers
+      servers = self.servers.join(',')
+      s += "Servers: #{servers}"
+    end
+    puts s
+  end
+  
+  def configuration
+    Cachetastic::Adapters::Base.configuration(self.name)
+  end
+  
+  class << self
+    # Returns either the options
+    # Options need to be specified in configatrion as the methodized name of the cache with
+    # _options attached at the end.
+    # Examples:
+    #   Cachetastic::Caches::PageCache # => cachetastic_caches_page_cache_options
+    #   MyAwesomeCache # => my_awesome_cache_options
+    def configuration(name)
+      name = "#{name}_options"
+      conf = configatron.retrieve(name, configatron.cachetastic_default_options)
+      conf
+    end
+  end
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/drb.rb

@@ -0,0 +1,51 @@
+# This adapter uses Cachetastic::Drb::Server as it's backing.
+# The configuration for this should look something like this:
+#  my_awesome_cache_options:
+#    debug: false
+#    adapter: drb
+#    store_options:
+#      host: druby://127.0.0.1:61676
+class Cachetastic::Adapters::Drb < Cachetastic::Adapters::Base
+  
+  attr_accessor :drb_store
+  
+  def valid?
+    begin
+      return self.drb_store.ping
+    rescue Exception => e
+      return false
+    end
+  end
+
+  def setup
+    # self.all_options["marshall_method"] = "ruby"
+    self.drb_store = DRbObject.new_with_uri(configuration.servers)
+  end
+  
+  def expire_all
+    self.drb_store.expire_all(self.name)
+  end
+  
+  # See Cachetastic::Adapters::Base
+  def get(key)
+    Cachetastic::Caches::Base.unmarshall(self.drb_store.get(self.name, key))
+  end
+  
+  def set(key, value, expiry = 0)
+    self.drb_store.set(self.name, key, Cachetastic::Caches::Base.marshall(value), expiry)
+  end
+  
+  def delete(key, delay = 0)
+    self.drb_store.delete(self.name, key)
+  end
+  
+  def stats
+    super
+    begin
+      self.drb_store.stats if self.drb_store.respond_to? 'stats'
+    rescue Exception => e
+      puts "Calling stats on the DRb store raised this exception: #{e.message}"
+    end
+  end
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/file.rb

@@ -0,0 +1,49 @@
+# This adapter uses the file system as it's backing.
+# The configuration for this should look something like this:
+#  my_awesome_cache_options:
+#    debug: false
+#    adapter: file
+#    marshall_method: none
+#    default_expiry: <%= 24.hours %>
+#    store_options:  
+#      dir: /usr/local/caches/
+#    logging:
+#      logger_1:
+#        type: file
+#        file: log/file_store_cache.log
+class Cachetastic::Adapters::File < Cachetastic::Adapters::FileBase
+  
+  def get(key)
+    full_path = full_path_from_dir(get_key_directoy(key, false))
+    return nil unless File.exists?(full_path)
+    so = YAML::load(File.open(full_path).read)
+    if so
+      if so.invalid?
+        self.delete(key)
+        return nil
+      end
+      if so.value.is_a?(YAML::Object)
+        require so.value.class.underscore
+        so = YAML::load(File.open(full_path).read)
+      end
+      return so.value
+    end
+    return nil
+  end
+  
+  def set(key, value, expiry = 0)
+    so = Cachetastic::Adapters::StoreObject.new(key.to_s, value, expiry)
+    File.open(full_path_from_key(key), "w") do |f|
+      f.puts YAML.dump(so)
+    end
+  end
+  
+  protected
+  def store_file_name
+    return STORE_FILE_NAME
+  end
+  
+  private
+  STORE_FILE_NAME = "cache.yml"
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/file_base.rb

@@ -0,0 +1,86 @@
+require 'digest/md5'
+require 'base64'
+class Cachetastic::Adapters::FileBase < Cachetastic::Adapters::Base
+  
+  attr_reader :directory
+  attr_reader :hashed_keys
+  
+  def setup
+    @directory = File.join(self.configuration.store_options.dir, self.name.to_s)
+    FileUtils.mkdir_p(self.directory, :verbose => self.debug?)
+    @hashed_keys = {}
+  end
+  
+  def valid?
+    File.exists?(self.directory)
+  end
+  
+  def stats
+    super
+    num_files = num_directories = file_size = 0
+    everything = Dir.glob("#{self.directory}/**/*")
+    everything.reject{|x| x =~ /^\./}.each do |entry|
+      if ::File.directory?(entry)
+        num_directories +=  1
+      else
+        file_size += ::File.size(entry)
+        num_files += 1
+      end
+    end
+    puts "Number of Files: #{num_files}\nNumber of Directories: #{num_directories}\nTotal Size on Disk: #{file_size/1024.to_f} KB\n\n"
+  end
+  
+  def delete(key, delay = 0)
+    if delay <= 0
+      FileUtils.rm_rf(get_key_directoy(key), :verbose => self.debug?)
+    else
+      so = self.get(key)
+      if so
+        self.set(so.key, so.value, delay)
+      end
+    end
+  end
+  
+  def expire_all
+    FileUtils.rm_rf(self.directory, :verbose => self.debug?)
+    setup
+  end
+  
+  protected
+  def store_file_name
+    return "cachetastic.data"
+  end
+  
+  private
+  def directory_from_key(key)
+    hkey = Base64.encode64(Digest::MD5.digest(key))
+    hkey.gsub!("==\n", "")
+    i = 0
+    path = ""
+    until i >= hkey.length do
+      path = File.join(path, hkey[i..i+2])
+      i += 3
+    end
+    path
+  end
+  
+  def full_path_from_key(key)
+    full_path_from_dir(get_key_directoy(key))
+  end
+  
+  def full_path_from_dir(dir)
+    File.join(dir, store_file_name)
+  end
+  
+  def get_key_directoy(key, mkdir = true)
+    hkey = self.hashed_keys[key.to_sym]
+    if hkey.nil?
+      path = File.join(self.directory, directory_from_key(key))
+      self.hashed_keys[key.to_sym] = path
+      hkey = path
+    end
+    FileUtils.mkdir_p(hkey, :verbose => self.debug?) if mkdir
+    hkey
+  end
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/html_file.rb

@@ -0,0 +1,68 @@
+# This adapter uses the file system as it's backing.
+# Caches are stored in files called index.html.
+# At the beginning of the html the following html comments are pre-pended:
+#   <!-- cachetastic: expires_at: Fri Feb 01 20:51:45 -0500 2008 -->
+#   <!-- cachetastic: key: /channels/1-arts-humanities --> 
+# Obviously the expires_at date will be the ACTUAL expire date,
+# same goes for the key.
+# The configuration for this should look something like this:
+#  my_awesome_cache_options:
+#    debug: false
+#    adapter: html_file
+#    marshall_method: none
+#    default_expiry: <%= 24.hours %>
+#    store_options:  
+#      dir: /usr/local/caches/
+#    logging:
+#      logger_1:
+#        type: file
+#        file: log/file_store_cache.log
+require 'time'
+class Cachetastic::Adapters::HtmlFile < Cachetastic::Adapters::FileBase
+  
+  def get(key)
+    full_path = full_path_from_dir(get_key_directoy(key, false))
+    return nil unless File.exists?(full_path)
+    so = html_to_store_object(File.open(full_path).read)
+    if so
+      if so.invalid?
+        self.delete(key)
+        return nil
+      end
+      return so.value
+    end
+    return nil
+  end
+  
+  def set(key, value, expiry = 0)
+    so = Cachetastic::Adapters::StoreObject.new(key.to_s, value, expiry)
+    File.open(full_path_from_key(key), "w") do |f|
+      f.puts store_object_to_html(so)
+    end
+  end
+  
+  protected
+  def store_file_name
+    return STORE_FILE_NAME
+  end
+  
+  private
+  def html_to_store_object(html)
+    key = html.match(/<!-- cachetastic: key: (.*) -->/).captures.first
+    expires_at = html.match(/<!-- cachetastic: expires_at: (.*) -->/).captures.first
+    html.gsub!(/<!-- cachetastic: key: .* -->/, '')
+    html.gsub!(/<!-- cachetastic: expires_at: .* -->/, '')
+    so = Cachetastic::Adapters::StoreObject.new(key, html, 0)
+    so.expires_at = Time.parse(expires_at)
+    so
+  end
+  
+  def store_object_to_html(so)
+    x = so.value
+    x << "\n<!-- cachetastic: expires_at: #{so.expires_at} -->"
+    x << "\n<!-- cachetastic: key: #{so.key} -->"
+  end
+  
+  STORE_FILE_NAME = "index.html"
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/local_memory.rb

@@ -0,0 +1,68 @@
+# An adapter/store that keeps objects in local memory. This is great for development/testing,
+# but probably shouldn't be put into production. 
+# It's also a very good example of how to write a adapter.
+class Cachetastic::Adapters::LocalMemory < Cachetastic::Adapters::Base
+  
+  attr_accessor :local_store
+  
+  def valid?
+    true
+  end
+
+  def setup
+    self.local_store = {}
+  end
+  
+  def expire_all
+    self.local_store = {}
+  end
+  
+  # See Cachetastic::Adapters::Base
+  def get(key)
+    so = self.local_store[key.to_s]
+    if so
+      if so.invalid?
+        self.delete(key)
+        return nil
+      end
+      return so.value
+    end
+    return nil
+  end
+  
+  def set(key, value, expiry = 0)
+    self.local_store[key.to_s] = Cachetastic::Adapters::StoreObject.new(key.to_s, value, expiry)
+  end
+  
+  def delete(key, delay = 0)
+    if delay <= 0
+      self.local_store.delete(key.to_s)
+    else
+      so = self.get(key)
+      if so
+        self.set(so.key, so.value, delay)
+      end
+    end
+  end
+  
+  def stats
+    super
+    num_keys = self.local_store.size
+    s = "Number of Entries: #{num_keys}\n"
+    if num_keys > 0
+      expiries = []
+      keys = []
+      self.local_store.each do |key,value|
+        keys << key
+        expiries << value.expires_at
+      end
+      expiries.sort! {|x, y| x <=> y}
+      oldest_expiry = expiries.first
+      newest_expiry = expiries.last
+      s += "Oldest Entry: #{oldest_expiry}\nNewest Entry: #{newest_expiry}\nKeys: #{keys.inspect}\n"
+    end
+    puts s + "\n"
+  end
+
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/memcache.rb

@@ -0,0 +1,114 @@
+# This adapter uses Memcache as it's backing.
+# The configuration for this should look something like this:
+#  my_awesome_cache_options:
+#    debug: false
+#    adapter: memcache
+#    marshall_method: none
+#    default_expiry: <%= 24.hours %>
+#    store_options:  
+#      c_threshold: 10_000
+#      compression: true
+#      debug: false
+#      readonly: false
+#      urlencode: false
+#    logging:
+#      logger_1:
+#        type: file
+#        file: log/memcached.log
+#    servers:
+#      - 127.0.0.1:11211
+class Cachetastic::Adapters::Memcache < Cachetastic::Adapters::Base
+  
+  def setup
+    self.conn = MemCache.new(configuration.servers, configuration.store_options.to_hash.merge({:namespace => self.namespace}))
+    self.version = self.get_version(self.name)
+  end
+  
+  def set(key, value, expiry = 0, raw = false)
+    self.conn.set(key, value, expiry, raw)
+  end
+  
+  def delete(key, delay = 0)
+    self.conn.delete(key, delay)
+  end
+  
+  def get(key, raw = false)
+    self.conn.get(key, raw)
+  end
+  
+  def expire_all
+    self.increment_version(self.name)
+  end
+  
+  def inspect
+    self.conn.inspect + " <version: #{self.version}> #{self.conn.stats.inspect}"
+  end
+  
+  def valid?
+    begin
+      return (self.conn.active? && self.version == self.get_version(self.name))
+    rescue Exception => e
+      puts e.message
+      puts e.backtrace.join("\n")
+      return false
+    end
+  end
+  
+  def stats
+    super
+    begin
+      puts "Memcache stats for all caches:"
+      memc = self.conn
+      puts Kernel.pp_to_s(memc.stats)
+      paths = `sh -c 'echo $PATH'`
+      paths = paths.split(':')
+      memcached_tool_found = false
+      paths.each do |path|
+        cmd_path = File.expand_path(File.join(path,'memcached_tool'))
+        if File.exists?(cmd_path)
+           memcached_tool_found = true
+          break
+        end
+      end
+      if memcached_tool_found
+        configuration.memcache_servers.each do |server|
+          puts `memcached_tool #{server}`
+        end
+      end
+    rescue
+    end
+    puts ""
+  end
+  
+  protected
+  attr_accessor :conn
+  attr_accessor :version
+  
+  def namespace
+    v = self.get_version(self.name)
+    return "#{name}.#{v}"
+  end
+  
+  def ns_versions
+    ivar_cache do
+      ns_conn = MemCache.new(configuration.servers, configuration.store_options.to_hash.merge({:namespace => :namespace_versions}))
+    end
+  end
+  
+  def increment_version(name)
+    name = name.to_s
+    v = get_version(name)
+    self.ns_versions.set(name, v + 1)
+  end
+
+  def get_version(name)
+    name = name.to_s
+    v = self.ns_versions.get(name)
+    if v.nil?
+      self.ns_versions.set(name, 1)
+      v = 1
+    end
+    v
+  end
+  
+end

data/lib/gems/cachetastic-2.0.0/lib/cachetastic/adapters/store_object.rb

@@ -0,0 +1,28 @@
+class Cachetastic::Adapters::StoreObject #:nodoc:#
+  attr_accessor :key
+  attr_accessor :value
+  attr_accessor :expires_at
+  
+  def initialize(key, value, expiry)
+    self.key = key
+    self.value = value
+    begin
+      self.expires_at = (Time.now + (expiry == 0 ? (31536000) : expiry)) # 31536000 = one year
+    rescue RangeError => e
+      self.expires_at = Time.at(expiry)
+    end
+    # puts "now: #{Time.now}"
+    # puts "expiry: #{expiry}"
+    # puts "expires_at: #{self.expires_at}"
+  end
+  
+  def size
+    return self.value.size if self.value.respond_to?(:size)
+    -1
+  end
+  
+  def invalid?
+    Time.now >= self.expires_at
+  end
+  
+end