delta_cache 1.0

data/README.md

@@ -0,0 +1,39 @@
+# Keep track of deltas and tombstones for an array of data
+
+A cache that keeps track of deltas and tombstones for an array of data. Deltas and tombstones can be retrieved from the cache using a last-modified timestamp.
+
+## Configuration
+
+### Redis
+
+Connection
+
+    DeltaCache.connection = Redis.new(:host => "your-host-ip")
+
+### Cassandra
+
+Setup Cassandra Keyspaces and Column Families
+
+    create keyspace DeltaCache;
+    use DeltaCache;
+    create column family Cache;
+    create column family Deltas;
+
+    create keyspace DeltaCacheTest;
+    use DeltaCacheTest;
+    create column family Cache;
+    create column family Deltas;
+
+Connection
+
+    DeltaCache.connection = Cassandra.new('DeltaCache')
+
+### Namespace
+
+    DeltaCache.cache_name = "your-cache-namespace"
+
+### Logger
+
+    DeltaCache.logger = Logger.new(STDOUT, Logger::DEBUG)
+
+See emamples directory for more details.

data/delta_cache.gemspec

@@ -0,0 +1,39 @@
+
+# -*- encoding: utf-8 -*-
+$:.push('lib')
+require "delta_cache/version"
+
+Gem::Specification.new do |s|
+  s.name     = "delta_cache"
+  s.version  = DeltaCache::VERSION.dup
+  s.date     = "2012-02-01"
+  s.summary  = "A cache that keeps track of deltas and tombstones for an array of data."
+  s.email    = "john.critz@gmail.com"
+  s.homepage = ""
+  s.authors  = ['John Critz']
+
+  s.description = <<-EOF
+A cache that keeps track of deltas and tombstones for an array of data. Deltas and tombstones can be retrieved from the cache using a last-modified timestamp.
+EOF
+
+  dependencies = []
+
+  s.files         = Dir['**/*']
+  s.test_files    = Dir['test/**/*'] + Dir['spec/**/*']
+  s.executables   = Dir['bin/*'].map { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+
+  ## Make sure you can build the gem on older versions of RubyGems too:
+  s.rubygems_version = "1.8.10"
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.specification_version = 3 if s.respond_to? :specification_version
+
+  dependencies.each do |type, name, version|
+    if s.respond_to?("add_#{type}_dependency")
+      s.send("add_#{type}_dependency", name, version)
+    else
+      s.add_dependency(name, version)
+    end
+  end
+end

data/examples/callback.rb

@@ -0,0 +1,76 @@
+require 'rack'
+require './facebook'
+
+class Callback
+
+  attr_accessor :last_modified
+
+  def call(env)
+    init(env)
+    puts env.inspect
+    puts env["rack.input"].input.inspect
+    puts @req.inspect
+    puts @req.form_data?
+    puts @req.params.inspect
+
+    if is_valid_request?
+
+      if is_challenge_request?
+        [200, {'Content-Type' => 'text/plain'}, [@params["hub.challenge"]]]
+
+      else
+        changes = @params["entry"]
+        changes.each do |entry|
+
+          if is_valid_change?(entry)
+            fb_id = Integer(entry["uid"])
+
+            # update cache
+            friends_info = Facebook.get_friends(fb_id)
+            DeltaCache.new(:cache_prefix => "facebook", :key_id => fb_id).update(friends_info, fb_id)
+
+            # show changes
+            puts DeltaCache.new(
+              fb_id, :last_modified => self.last_modified
+            ).get_info(self.last_modified)
+
+            # store last modified timestamp
+            self.last_modified = DeltaCache.new(fb_id).get_last_modified
+          end
+        end
+
+        [200, {'Content-Type' => 'text/plain'}, []]
+      end
+
+    else
+      [404, {'Content-Type' => 'text/plain'}, []]
+
+    end
+  end
+
+  def init(env)
+    @req = Rack::Request.new(env)
+    @params = @req.params if @req
+  end
+
+  def is_valid_request?
+    @req && @params
+  end
+
+  def is_challenge_request?
+    return false if @params["hub.mode"].nil? || @params["hub.mode"].empty?
+    return false if @params["hub.challenge"].nil? || @params["hub.challenge"].empty?
+    return false if @params["hub.verify_token"].nil? || (token = @params["hub.verify_token"]).empty?
+    return FACEBOOK_CONFIG[:subscription_token] == token
+  end
+
+  def is_valid_change?(entry)
+    fb_id = Integer(entry["uid"])
+    fields = Array(entry["changed_fields"])
+
+    return false if fb_id.empty?
+    return false if fields.empty?
+    return fields.include?("friends")
+  end
+
+end

data/examples/facebook/callback.rb

@@ -0,0 +1,76 @@
+require 'rack'
+require './facebook'
+
+class Callback
+
+  attr_accessor :last_modified
+
+  def call(env)
+    init(env)
+    # puts env.inspect
+    # puts env["rack.input"].input.inspect
+    # puts @req.inspect
+    # puts @req.form_data?
+    # puts @req.params.inspect
+
+    if is_valid_request?
+
+      if is_challenge_request?
+        [200, {'Content-Type' => 'text/plain'}, [@params["hub.challenge"]]]
+
+      else
+        changes = @params["entry"]
+        changes.each do |entry|
+
+          if is_valid_change?(entry)
+            fb_id = Integer(entry["uid"])
+
+            # update cache
+            friends_info = Facebook.get_friends(fb_id)
+            DeltaCache::Cache.new(fb_id).update(friends_info)
+
+            # show changes
+            puts DeltaCache::Cache.new(fb_id).get_info(self.last_modified)
+
+            # store last modified timestamp
+            self.last_modified = DeltaCache::Cache.new(fb_id).get_last_modified
+          end
+        end
+
+        [200, {'Content-Type' => 'text/plain'}, []]
+      end
+
+    else
+      [404, {'Content-Type' => 'text/plain'}, []]
+
+    end
+  end
+
+  def init(env)
+    @req = Rack::Request.new(env)
+    @params = @req.params if @req
+    DeltaCache.connection = Redis.new(:host => "127.0.0.1")
+    DeltaCache.cache_name = "facebook"
+  end
+
+  def is_valid_request?
+    @req && @params
+  end
+
+  def is_challenge_request?
+    return false if @params["hub.mode"].nil? || @params["hub.mode"].empty?
+    return false if @params["hub.challenge"].nil? || @params["hub.challenge"].empty?
+    return false if @params["hub.verify_token"].nil? || (token = @params["hub.verify_token"]).empty?
+    return FACEBOOK_CONFIG[:subscription_token] == token
+  end
+
+  def is_valid_change?(entry)
+    fb_id = Integer(entry["uid"])
+    fields = Array(entry["changed_fields"])
+
+    return false if fb_id.empty?
+    return false if fields.empty?
+    return fields.include?("friends")
+  end
+
+end

data/examples/facebook/config.ru

@@ -0,0 +1,6 @@
+require './callback'
+
+use Rack::CommonLogger, STDOUT
+use Rack::ShowExceptions
+
+run Callback.new

data/examples/facebook/facebook.rb

@@ -0,0 +1,96 @@
+####
+# Enable/disable Facebook subscriptions for a Facebook application
+# so you will be notified of any changes to the users who have
+# authorized the given Facebook app.
+#
+# Usage:
+# Configure the FACEBOOK_CONFIG hash with the app information
+# and then execute the ruby script.
+#
+####
+
+require 'net/http'
+require 'uri'
+
+require 'rubygems'
+require 'fb_graph'
+require 'redis'
+
+require '../../lib/delta_cache'
+
+FACEBOOK_CONFIG = {
+  :app_id => "your-app-id",
+  :access_token => "your-fb-access-token",
+  :subscription_callback_url => "http://your-public-ip/",
+  :subscription_token => "any-token-to-validate-the-fb-request"
+}
+
+class Facebook
+
+  def self.subscribe!
+    uri = URI.parse("https://graph.facebook.com/#{FACEBOOK_CONFIG[:app_id]}/subscriptions")
+    form_data = {
+      :object => "user",
+      :fields => "name,friends",
+      :access_token => FACEBOOK_CONFIG[:access_token],
+      :callback_url => FACEBOOK_CONFIG[:subscription_callback_url],
+      :verify_token => FACEBOOK_CONFIG[:subscription_token]
+    }
+
+    HTTP.new(uri).post(form_data)
+  end
+
+  def self.unsubscribe!
+    uri = URI.parse("https://graph.facebook.com/#{FACEBOOK_CONFIG[:app_id]}/subscriptions")
+    form_data = {
+      :object => "user",
+      :access_token => FACEBOOK_CONFIG[:access_token]
+    }
+
+    HTTP.new(uri).delete(form_data)
+  end
+
+  def self.get_friends(fb_id)
+    FbGraph::User.fetch(fb_id, :access_token => FACEBOOK_CONFIG[:access_token])
+  end
+
+  class HTTP
+
+    attr_accessor :uri, :http
+
+    def initialize(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.read_timeout = 120
+
+      self.uri = uri
+      self.http = http
+    end
+
+    def post(form_data)
+      request = Net::HTTP::Post.new(self.uri.request_uri)
+      request.set_form_data(form_data)
+      response = self.http.request(request)
+
+      puts "\nForm Data"
+      puts form_data.inspect
+      puts "--"
+
+      puts "\nResponse"
+      puts response
+      puts "--"
+    end
+
+    def delete(form_data)
+      request = Net::HTTP::Delete.new(self.uri.request_uri)
+      request.set_form_data(form_data)
+      response = self.http.request(request)
+
+      puts "\nResponse"
+      puts response
+      puts "--"
+    end
+
+  end
+
+end

data/examples/notifications/notifications.rb

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'redis'
+require 'json'
+
+require File.expand_path('../../lib/delta_cache.rb')
+
+DeltaCache.connection = Redis.new(:host => "127.0.0.1")
+DeltaCache.cache_name = "notifications"
+
+class Notifications
+
+  attr_accessor :cache
+
+  def initialize(cache_id)
+    self.cache = DeltaCache::Cache.new(cache_id)
+
+    ## flush and initialize cache
+    cache.flush!
+
+    puts "\nInitializing cache..."
+    notifications = [
+      {:name => "John", :message => "Be my friend."},
+    ]
+    update_and_print(notifications)
+  end
+
+  def add_to_cache
+    puts "\nAdding Steve and Bill to cache..."
+
+    lm = cache.get_last_modified
+    puts "\n    Deltas since: #{lm}"
+    notifications = [
+      {:name => "John", :message => "Be my friend."},
+      {:name => "Steve", :message => "Be my friend."},
+      {:name => "Bill", :message => "Be my friend."}
+    ]
+
+    update_and_print(notifications)
+  end
+
+  def delete_from_cache
+    puts "\nDeleting John from cache..."
+
+    lm = cache.get_last_modified
+    puts "\n    Tombstones since: #{lm}"
+    notifications = [
+      {:name => "Steve", :message => "Be my friend."},
+      {:name => "Bill", :message => "Be my friend."}
+    ]
+
+    update_and_print(notifications)
+  end
+
+  def update_and_print(data)
+    lm = cache.get_last_modified
+    cache.update(data)
+    puts "    " + cache.get_info(lm).inspect
+  end
+
+end
+
+n = Notifications.new(1)
+sleep 2
+
+n.add_to_cache
+sleep 2
+
+n.delete_from_cache

data/lib/delta_cache/db/cassandra.rb

@@ -0,0 +1,139 @@
+# Stores friend cache data in a Cassandra keyspace. The advantage here is
+# the cache can grow larger than the memory on one machine. The disadvantage
+# is that you have to set up multiple machines and make some decisions about
+# acceptable thresholds around consistency and atomicity.
+#
+# This module requires initialization, like so:
+#
+#   DeltaCache.connection = Cassandra.new('DeltaCache')
+#   DeltaCache.logger = Logger.new(STDOUT, Logger::DEBUG)
+#
+# You only need to set the logger if you need to debug.
+#
+# ## Storage
+#
+# This adapter uses two column families: `:Cache` and `:Deltas`.
+#
+# The `:Cache` CF stores each cached object as a JSON blob. Each row is keyed
+# by the SHA1 hash of the blob. The blob itself is stored in a column named
+# `blob`. These rows won't ever get too wide, so this CF is a good candidate
+# for row caching by Cassandra.
+#
+# The `:Deltas` CF stores changes to objects cached for each user. There are
+# two rows per user; one tracks newly cached objects and the other tracks
+# cached objects that have been removed. Each row contains a pair of columns
+# per cached object. One maps a timestamp to a cached object and is used for
+# caching the most recent changes to a user's cached objects. The other maps
+# cached objects back to a timestamp and is used to find delta entries to
+# remove if a cached object is marked as removed.
+
+class DeltaCache::CassandraDB
+
+  MAX_DELTAS = 10_000
+
+  attr_accessor :cache_id
+
+  def initialize(cache_id)
+    self.cache_id = cache_id
+  end
+
+  def connection
+    DeltaCache.connection
+  end
+
+  def logger
+    DeltaCache.logger
+  end
+
+  def cache_name
+    DeltaCache.cache_name
+  end
+
+  # Given a key ID, generate a key into `:Deltas` for cached objects.
+  def info_key
+    ["cache", cache_name, cache_id].join(":")
+  end
+
+  # Given a key ID, generate a key into `:Deltas` for deleted cache objects.
+  def deleted_info_key
+    ["deleted", cache_name, cache_id].join(":")
+  end
+
+  # Store a cached object.
+  def set(data)
+    data = data.to_json
+    cache_key = Digest::SHA1.hexdigest(data)
+
+    log("insert :Cache, #{cache_key} -> {'blob' => #{data.inspect}}")
+    connection.insert(:Cache, cache_key, { "blob" => data })
+    cache_key
+  end
+
+  # Add a cached object to a delta timeline.
+  def add(key, timestamp, value)
+    columns = { timestamp_name(timestamp) => value, cache_name(value) => timestamp.to_f.to_s }
+
+    log("insert :Deltas, #{key} -> #{columns.inspect}")
+    connection.insert(:Deltas, key, columns)
+  end
+
+  # Remove a cached object from a delta timeline.
+  def rem(key, value)
+    log("get :Deltas, #{key}, #{cache_name(value)}")
+    timestamp = connection.get(:Deltas, key, cache_name(value))
+
+    log("remove :Deltas, #{key} -> #{timestamp_name(timestamp)}")
+    connection.remove(:Deltas, key, timestamp_name(timestamp))
+
+    log("remove :Deltas, #{key} -> #{cache_name(value)}")
+    connection.remove(:Deltas, key, cache_name(value))
+  end
+
+  # Fetch multiple cached objects.
+  def get(keys)
+    log("get :Cache, #{keys.inspect}")
+    connection.multi_get(:Cache, Array(keys)).values.map { |v| v['blob'] }
+  end
+
+  # Find all the cached objects referenced by a delta timeline.
+  def get_rev_range(key, most_recent, last_modified)
+    log("get :Deltas, #{key}, :reversed => true, :start => #{timestamp_name(most_recent)}, :finish => #{timestamp_name(last_modified)}")
+    connection.get(:Deltas, key, :reversed => true, :start => timestamp_name(most_recent), :finish => timestamp_name(last_modified), :count => MAX_DELTAS).values
+  end
+
+  # Get a mapping of cached objects to delta timestamps for a given user.
+  def get_timestamp(key, value)
+    log("get :Deltas, #{key}, #{cache_name(value)}")
+    score = connection.get(:Deltas, key, cache_name(value))
+    Time.at(score.to_f).utc
+  end
+
+  # Remove an object from the cache.
+  def del(key)
+    log("remove :Cache, #{key}")
+    connection.remove(:Cache, key)
+  end
+
+  # Check if a delta cache exists for a user.
+  def exists(key)
+    log("exists? :Deltas, #{key}")
+    connection.exists?(:Deltas, key)
+  end
+
+  # Given a timestamp, generate a column name for use in range queries on `:Deltas`.
+  def timestamp_name(timestamp)
+    ["ts", timestamp.to_f].join(":")
+  end
+
+  # Given the SHA1 for a cached object, generate a key into `:Cache`.
+  def cache_name(value)
+    ["cache", value].join(":")
+  end
+
+  # Private: Log a message, if a logger is set.
+  def log(msg)
+    return if logger.nil?
+    logger.debug(msg)
+  end
+
+end

data/lib/delta_cache/db/redis.rb

@@ -0,0 +1,79 @@
+class DeltaCache::RedisDB
+
+  attr_accessor :cache_id
+
+  def initialize(cache_id)
+    self.cache_id = cache_id
+  end
+
+  def connection
+    DeltaCache.connection
+  end
+
+  def cache_name
+    DeltaCache.cache_name
+  end
+
+  def info_key
+    [
+      "delta_cache",
+      "info",
+      cache_name,
+      cache_id
+      ].join(":")
+  end
+
+  def deleted_info_key
+    [
+      "delta_cache",
+      "deleted_info",
+      cache_name,
+      cache_id
+      ].join(":")
+  end
+
+  # set parent key that holds the data
+  def set(data)
+    data = data.to_json
+    cache_key = Digest::SHA1.hexdigest(data)
+    connection.set(cache_key, data)
+    cache_key
+  end
+
+  # add key to a set
+  def add(key, timestamp, value)
+    connection.zadd(key, timestamp.to_i, value)
+  end
+
+  # remove key from a set
+  def rem(key, value)
+    connection.zrem(key, value)
+  end
+
+  # get a list of keys
+  def get(keys)
+    connection.mget(*keys)
+  end
+
+  # get members from a set in reverse order
+  def get_rev_range(key, end_pos, start_pos)
+    connection.zrevrangebyscore(key, end_pos.to_i, start_pos.to_i)
+  end
+
+  # get the 'score' of the member in a set
+  def get_timestamp(key, value)
+    score = connection.zscore(key, value)
+    Time.at(score.to_i).utc
+  end
+
+  # delete a key
+  def del(key)
+    connection.del(key)
+  end
+
+  # does a key exist
+  def exists(key)
+    connection.exists(key)
+  end
+
+end

data/lib/delta_cache/version.rb

@@ -0,0 +1,3 @@
+module DeltaCache
+  VERSION = '1.0'
+end

data/lib/delta_cache.rb

@@ -0,0 +1,136 @@
+module DeltaCache
+
+  require 'time'
+  require 'digest'
+
+  require 'delta_cache/db/redis'
+  require 'delta_cache/db/cassandra'
+
+  class << self
+    attr_accessor :connection, :logger, :cache_name
+  end
+
+  class Cache
+
+    attr_accessor :db, :cache_id
+
+    def initialize(cache_id)
+      self.cache_id = cache_id
+
+      raise "DeltaCache.connection is not defined." unless DeltaCache.connection
+      raise "DeltaCache.cache_name is not defined." unless DeltaCache.cache_name
+
+      class_name = DeltaCache.connection.class.name
+      if class_name =~ /redis/i
+        self.db = DeltaCache::RedisDB.new(cache_id)
+      elsif class_name =~ /cassandra/i
+        self.db = DeltaCache::CassandraDB.new(cache_id)
+      end
+    end
+
+    def info_key
+      db.info_key
+    end
+
+    def deleted_info_key
+      db.deleted_info_key
+    end
+
+    def exists?
+      db.exists(info_key)
+    end
+
+    # this will remove all cache for the given id
+    def flush!
+      db.del(info_key)
+      db.del(deleted_info_key)
+    end
+
+    def update(info)
+      cache_new(info)
+      # stores tombstones for deleted records
+      cache_deleted(info)
+
+      return true
+    end
+
+    def cache_new(info)
+      new_info = (info - get_info(nil, false))
+      new_info.each do |info|
+        info_id = set_info(info)
+        db.rem(deleted_info_key, info_id)
+        db.add(info_key, timestamp, info_id)
+      end
+    end
+
+    def cache_deleted(info)
+      deleted_info = (get_info(nil, false) - info)
+      deleted_info.each do |info|
+        info_id = set_info(info)
+        db.rem(info_key, info_id)
+        db.add(deleted_info_key, timestamp, info_id)
+      end
+    end
+
+    def timestamp(time=nil)
+      Time.parse((time || Time.now).to_s).utc
+    end
+
+    def set_info(info)
+      db.set(info)
+    end
+
+    def get_info(last_modified=nil, show_deleted_flag=true)
+      return [] unless exists?
+
+      last_modified_time = if last_modified.nil?
+        Time.at(0)
+      else
+        timestamp(last_modified)
+      end
+
+      info = []
+
+      info_ids = db.get_rev_range(info_key, timestamp, last_modified_time + 1)
+      if info_ids.any?
+        info += db.get(info_ids).compact.map do |f|
+          hash = JSON.parse(f)
+          hash.merge!(:deleted => false) if show_deleted_flag
+          hash
+        end
+      end
+
+      if last_modified_time.to_i > 0
+        info_ids = db.get_rev_range(deleted_info_key, timestamp, last_modified_time + 1)
+        if info_ids.any?
+          info += db.get(info_ids).compact.map do |f|
+            hash = JSON.parse(f)
+            hash.merge!(:deleted => true) if show_deleted_flag
+            hash
+          end
+        end
+      end
+
+      info.map do |hash|
+        new_hash = Hash.new
+        hash.each do |k, v|
+          new_hash[k.to_sym] = v
+        end
+        new_hash
+      end
+    end
+
+    def get_last_modified
+      info_ids = db.get_rev_range(info_key, timestamp, Time.at(0))
+      deleted_info_ids = db.get_rev_range(deleted_info_key, timestamp, Time.at(0))
+
+      last_modified = [ db.get_timestamp(info_key, info_ids.first),
+        db.get_timestamp(deleted_info_key, deleted_info_ids.first) ].compact
+      return Time.now.utc.httpdate unless last_modified.any?
+
+      Time.at(last_modified.max.to_i).utc.httpdate
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: delta_cache
+version: !ruby/object:Gem::Version
+  version: '1.0'
+  prerelease: 
+platform: ruby
+authors:
+- John Critz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-01 00:00:00.000000000Z
+dependencies: []
+description: ! 'A cache that keeps track of deltas and tombstones for an array of
+  data. Deltas and tombstones can be retrieved from the cache using a last-modified
+  timestamp.
+
+'
+email: john.critz@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- delta_cache-1.0.gem
+- delta_cache.gemspec
+- examples/callback.rb
+- examples/facebook/callback.rb
+- examples/facebook/config.ru
+- examples/facebook/facebook.rb
+- examples/notifications/notifications.rb
+- lib/delta_cache/db/cassandra.rb
+- lib/delta_cache/db/redis.rb
+- lib/delta_cache/version.rb
+- lib/delta_cache.rb
+- README.md
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A cache that keeps track of deltas and tombstones for an array of data.
+test_files: []