redtastic 0.2.1 → 0.2.2

data/lib/redtastic/model.rb

@@ -0,0 +1,285 @@
+module Redtastic
+  class Model
+    class << self
+      # Recording
+
+      def increment(params)
+        key_data = fill_keys_for_update(params)
+        if @_type == :unique
+          argv = []
+          argv << params[:unique_id]
+          Redtastic::ScriptManager.msadd(key_data[0], argv)
+        else
+          Redtastic::ScriptManager.hmincrby(key_data[0], key_data[1].unshift(1))
+        end
+      end
+
+      def decrement(params)
+        key_data = fill_keys_for_update(params)
+        if @_type == :unique
+          argv = []
+          argv << params[:unique_id]
+          Redtastic::ScriptManager.msrem(key_data[0], argv)
+        else
+          Redtastic::ScriptManager.hmincrby(key_data[0], key_data[1].unshift(-1))
+        end
+      end
+
+      # Retrieving
+
+      def find(params)
+        keys = []
+        argv = []
+
+        # Construct the key's timestamp from inputed date parameters
+        timestamp = ''
+        timestamp += "#{params[:year]}"
+        timestamp += "-#{zeros(params[:month])}" if params[:month].present?
+        timestamp += "-W#{params[:week]}"        if params[:week].present?
+        timestamp += "-#{zeros(params[:day])}"   if params[:day].present?
+        params.merge!(timestamp: timestamp)
+
+        # Handle multiple ids
+        ids = param_to_array(params[:id])
+
+        ids.each do |id|
+          params[:id] = id
+          keys << key(params)
+          argv << index(id)
+        end
+
+        if @_type == :unique
+          unique_argv = []
+          unique_argv << params[:unique_id]
+          result = Redtastic::ScriptManager.msismember(keys, unique_argv)
+        else
+          result = Redtastic::ScriptManager.hmfind(keys, argv)
+        end
+
+        # If only for a single id, just return the value rather than an array
+        if result.size == 1
+          result[0]
+        else
+          result
+        end
+      end
+
+      def aggregate(params)
+        key_data  = fill_keys_and_dates(params)
+        keys      = key_data[0]
+
+        # If interval is present, we return a hash including the total as well as a data point for each interval.
+        # Example: Visits.aggregate(start_date: 2014-01-05, end_date: 2013-01-06, id: 1, interval: :days)
+        # {
+        #    visits: 2
+        #    days: [
+        #      {
+        #        created_at: 2014-01-05,
+        #        visits: 1
+        #      },
+        #      {
+        #        created_at: 2014-01-06,
+        #        visits: 1
+        #      }
+        #    ]
+        # }
+        if params[:interval].present? && @_resolution.present?
+          if @_type == :unique
+            argv = []
+            argv << key_data[1].shift # Only need the # of business ids (which is 1st element) from key_data[1]
+            argv << temp_key
+            if params[:attributes].present?
+              attributes = param_to_array(params[:attributes])
+              attributes.each do |attribute|
+                keys << attribute_key(attribute)
+                argv << 1
+              end
+            end
+            data_points = Redtastic::ScriptManager.union_data_points_for_keys(keys, argv)
+          else
+            data_points = Redtastic::ScriptManager.data_points_for_keys(keys, key_data[1])
+          end
+
+          result = HashWithIndifferentAccess.new
+          dates  = key_data[2]
+          # The data_points_for_keys lua script returns an array of all the data points, with one exception:
+          # the value at index 0 is the total across all the data points, so we pop it off of the data points array.
+          result[model_name]         = data_points.shift
+          result[params[:interval]]  = []
+
+          data_points.each_with_index do |data_point, index|
+            point_hash                 = HashWithIndifferentAccess.new
+            point_hash[model_name]     = data_point
+            point_hash[:date]          = dates[index]
+            result[params[:interval]]  << point_hash
+          end
+          result
+        else
+          # If interval is not present, we just return the total as an integer
+          if @_type == :unique
+            argv = []
+            argv << temp_key
+            if params[:attributes].present?
+              attributes = param_to_array(params[:attributes])
+              attributes.each do |attribute|
+                keys << attribute_key(attribute)
+                argv << 1
+              end
+            end
+            Redtastic::ScriptManager.msunion(keys, argv)
+          else
+            key_data[1].shift # Remove the number of ids from the argv array (don't need it in the sum method)
+            Redtastic::ScriptManager.sum(keys, key_data[1]).to_i
+          end
+        end
+      end
+
+      private
+
+        def type(type_name)
+          types = [:counter, :unique, :mosaic]
+          fail "#{type_name} is not a valid type" unless types.include?(type_name)
+          @_type = type_name
+        end
+
+        def resolution(resolution_name)
+          resolutions = [:days, :weeks, :months, :years]
+          fail "#{resolution_name} is not a valid resolution" unless resolutions.include?(resolution_name)
+          @_resolution = resolution_name
+        end
+
+        def fill_keys_for_update(params)
+          keys = []
+          argv = []
+
+          # Handle multiple keys
+          ids = param_to_array(params[:id])
+
+          ids.each do |id|
+            params[:id] = id
+            if params[:timestamp].present?
+              # This is for an update, so we want to build a key for each resolution that is applicable to the model
+              scoped_resolutions.each do |resolution|
+                keys << key(params, resolution)
+                argv << index(id)
+              end
+            else
+              keys << key(params)
+              argv << index(id)
+            end
+          end
+          [keys, argv]
+        end
+
+        def fill_keys_and_dates(params)
+          keys  = []
+          dates = []
+          argv  = []
+          ids   = param_to_array(params[:id])
+
+          argv << ids.size
+          start_date = Date.parse(params[:start_date]) if params[:start_date].is_a?(String)
+          end_date   = Date.parse(params[:end_date])   if params[:end_date].is_a?(String)
+
+          if params[:interval].present?
+            interval = params[:interval]
+          else
+            interval = @_resolution
+          end
+
+          current_date = start_date
+          while current_date <= end_date
+            params[:timestamp] = current_date
+            dates << formatted_timestamp(current_date, interval)
+            ids.each do |id|
+              params[:id] = id
+              keys << key(params, interval)
+              argv << index(id)
+            end
+            current_date = current_date.advance(interval => +1)
+          end
+          [keys, argv, dates]
+        end
+
+        def key(params, interval = nil)
+          key = ''
+          key += "#{Redtastic::Connection.namespace}:" if Redtastic::Connection.namespace.present?
+          key += "#{model_name}"
+          if params[:timestamp].present?
+            timestamp = params[:timestamp]
+            timestamp = formatted_timestamp(params[:timestamp], interval) if interval.present?
+            key += ":#{timestamp}"
+          end
+          if @_type == :counter
+            key += ":#{bucket(params[:id])}"
+          else
+            key += ":#{params[:id]}" if params[:id].present?
+          end
+          key
+        end
+
+        def formatted_timestamp(timestamp, interval)
+          timestamp = Date.parse(timestamp) if timestamp.is_a?(String)
+          case interval
+          when :days
+            timestamp.strftime('%Y-%m-%d')
+          when :weeks
+            week_number = timestamp.cweek
+            result = timestamp.strftime('%Y')
+            result + "-W#{week_number}"
+          when :months
+            timestamp.strftime('%Y-%m')
+          when :years
+            timestamp.strftime('%Y')
+          end
+        end
+
+        def bucket(id)
+          @_type == :counter ? id / 1000 : id
+        end
+
+        def index(id)
+          id % 1000 if id.is_a?(Integer)
+        end
+
+        def zeros(number)
+          "0#{number}" if number < 10
+        end
+
+        def model_name
+          name.underscore
+        end
+
+        def scoped_resolutions
+          case @_resolution
+          when :days
+            [:days, :weeks, :months, :years]
+          when :weeks
+            [:weeks, :months, :years]
+          when :months
+            [:months, :years]
+          when :years
+            [:years]
+          else
+            []
+          end
+        end
+
+        def param_to_array(param)
+          result = []
+          param.is_a?(Array) ? result = param : result << param
+        end
+
+        def temp_key
+          seed = Array.new(8) { [*'a'..'z'].sample }.join
+          "temp:#{seed}"
+        end
+
+        def attribute_key(attribute)
+          key = ''
+          key += "#{Redtastic::Connection.namespace}:" if Redtastic::Connection.namespace.present?
+          key + attribute.to_s
+        end
+    end
+  end
+end

data/lib/redtastic/script_manager.rb

@@ -0,0 +1,32 @@
+module Redtastic
+  class ScriptManager
+    class << self
+      def load_scripts(script_path)
+        @stored_methods = HashWithIndifferentAccess.new unless @stored_methods.is_a?(Hash)
+        Dir["#{script_path}/*.lua"].map do |file|
+          method = File.basename(file, '.*')
+          unless @stored_methods.key?(method)
+            @stored_methods[method] = Redtastic::Connection.redis.script(:load, `cat #{file}`)
+          end
+        end
+      end
+
+      def method_missing(method_name, *args)
+        if @stored_methods.is_a?(Hash) && @stored_methods.key?(method_name)
+          Redtastic::Connection.redis.evalsha(@stored_methods[method_name], *args)
+        else
+          fail("Could not find script: #{method_name}.lua")
+        end
+      end
+
+      def flush_scripts
+        @stored_methods = nil
+        Redtastic::Connection.redis.script(:flush)
+      end
+
+      def to_ary
+        nil
+      end
+    end
+  end
+end

data/lib/redtastic/scripts/data_points_for_keys.lua

@@ -0,0 +1,37 @@
+-- Returns total combined sum accross all ids and keys, as well as the combined sum accross all ids for each
+-- specified data point interval
+
+local result            = {}
+local sum               = 0
+local number_of_ids     = tonumber(ARGV[1])
+local data_point_index  = 2 -- Initialized to 2 since position 1 in result is reserved for the total sum
+local count             = 1 -- Used to track whether we should should move to the next data point
+
+for index, key in ipairs(KEYS) do
+  -- Get the value associated with the KEY + INDEX pair
+  local value = tonumber(redis.call('HGET', key, ARGV[index+1]))
+
+  -- Initialize the total value for the data point if it hasn't been initialized already
+  if result[data_point_index] == nil then
+    result[data_point_index] = 0
+  end
+
+  if value then
+    sum = sum + value
+    result[data_point_index] = result[data_point_index] + value
+  end
+
+  -- Check if we've accounted for each id for the current data point
+  -- If true, then move to the next data point
+  if count == number_of_ids then
+    count = 1
+    data_point_index = data_point_index + 1
+  else
+    count = count + 1
+  end
+end
+
+-- Position 1 in the result array is reserved for the sum
+result[1] = sum
+
+return result

data/lib/redtastic/scripts/hmfind.lua

@@ -0,0 +1,14 @@
+-- Returns an array of values for each corresponding key/index pair passed into KEYS & ARGV
+
+local result = {}
+
+for index, key in ipairs(KEYS) do
+  local value = tonumber(redis.call('HGET', key, ARGV[index]))
+  if value then
+    result[index] = value
+  else
+    result[index] = 0
+  end
+end
+
+return result

data/lib/redtastic/scripts/hmincrby.lua

@@ -0,0 +1,9 @@
+-- Increments each corresponding key/index pair passed into KEYS & ARGV
+-- SPECIAL: the first index in ARGV needs to be set to the value to increment by
+
+local incrby = ARGV[1]
+
+for index, key in ipairs(KEYS) do
+  -- For each key / id increment by the passed in value
+  redis.call('HINCRBY', key, ARGV[index+1], incrby)
+end

data/lib/redtastic/scripts/msadd.lua

@@ -0,0 +1,3 @@
+for index, key in ipairs(KEYS) do
+  redis.call('sadd', key, ARGV[1])
+end

data/lib/redtastic/scripts/msismember.lua

@@ -0,0 +1,7 @@
+local result = {}
+
+for index, key in ipairs(KEYS) do
+  result[index] = redis.call('sismember', key, ARGV[1])
+end
+
+return result

data/lib/redtastic/scripts/msrem.lua

@@ -0,0 +1,3 @@
+for index, key in ipairs(KEYS) do
+  redis.call('srem', key, ARGV[1])
+end

data/lib/redtastic/scripts/msunion.lua

@@ -0,0 +1,28 @@
+-- SPECIAL: ARGV[1] -> tempkey
+-- SPECIAL: All ARGV[n], where n > 1, signify an attribute
+-- SPECIAL: last n keys, where n is number of attributes, is keys associated with attributes
+
+local num_attributes = table.getn(ARGV) - 1
+local attribute_keys = {}
+
+-- Remove keys associated with attributes and add them to our own attributes_keys array
+for i=1,num_attributes do
+  table.insert(attribute_keys, table.remove(KEYS))
+end
+
+-- union all of the keys
+redis.call('sunionstore', ARGV[1], unpack(KEYS))
+
+-- If attributes are present, we want to get the intersect of the result + any attributes and store in ARGV[1]
+if num_attributes > 0 then
+  table.insert(attribute_keys, ARGV[1])
+  redis.call('sinterstore', ARGV[1], unpack(attribute_keys))
+end
+
+-- get the cardinality of the resulting set
+local count = redis.call('scard', ARGV[1])
+
+-- delete the temp key
+redis.call('del', ARGV[1])
+
+return count

data/lib/redtastic/scripts/sum.lua

@@ -0,0 +1,13 @@
+-- Returns the total sum of the values of each key/index pair passed into KEYS & ARGV
+
+local sum = 0
+
+for index, key in ipairs(KEYS) do
+  -- For each key, read the value and add it to the total
+  local value = tonumber(redis.call('HGET', key, ARGV[index]))
+  if value then
+    sum = sum + value
+  end
+end
+
+return sum

data/lib/redtastic/scripts/union_data_points_for_keys.lua

@@ -0,0 +1,51 @@
+-- SPECIAL: ARGV[1] -> number_of_ids
+-- SPECIAL: ARGV[2] -> temp_key
+-- SPECIAL: All ARGV[n], where n > 2, signify an attribute
+-- SPECIAL: last n keys, where n is number of attributes, is keys associated with attributes
+
+local result = {}
+local number_of_ids = tonumber(ARGV[1])
+local data_point_index = 2
+local count = 1
+local num_attributes = table.getn(ARGV) - 2
+local attribute_keys = {}
+
+-- Remove keys associated with attributes and add them to our own attributes_keys array
+for i=1,num_attributes do
+  table.insert(attribute_keys, table.remove(KEYS))
+end
+
+-- union all of the keys
+redis.call('sunionstore', ARGV[2], unpack(KEYS))
+
+-- If attribute are present, we want to get the intersect of the result + any attributes and store in ARGV[2]
+if num_attributes > 0 then
+  redis.call('sinterstore', ARGV[2], ARGV[2], unpack(attribute_keys))
+end
+
+-- get the cardinality of the resulting set
+result[1] = redis.call('scard', ARGV[2])
+
+-- This loop returns the union of all the keys (for any number of ids) for each data point interval
+local index = 1
+while KEYS[index] do
+  local keys = {}
+  for i=0,(number_of_ids-1) do
+    keys[i+1] = KEYS[index+i]
+  end
+  redis.call('sunionstore', ARGV[2], unpack(keys))
+
+  -- If attributes are present, we want to get the interset of this data points result + any attributes
+  if num_attributes > 0 then
+    redis.call('sinterstore', ARGV[2], ARGV[2], unpack(attribute_keys))
+  end
+
+  result[data_point_index] = redis.call('scard', ARGV[2])
+  data_point_index = data_point_index + 1
+  index = index + number_of_ids
+end
+
+-- delete the temp key
+redis.call('del', ARGV[2])
+
+return result

data/lib/redtastic/version.rb

@@ -0,0 +1,3 @@
+module Redtastic
+  VERSION = '0.2.2'
+end

data/redtastic.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/redtastic/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = 'redtastic'
+  gem.version       = Redtastic::VERSION
+  gem.date          = '2013-01-15'
+  gem.authors       = ['Joe DiVita']
+  gem.email         = ['joediv31@gmail.com']
+  gem.description   = %q{ A simple, Redis-backed interface for storing, retrieving, and aggregating analytics }
+  gem.summary       = %q{ A simple, Redis-backed interface for storing, retrieving, and aggregating analytics }
+  gem.homepage      = 'https://github.com/bellycard/redtastic'
+  gem.files         = `git ls-files`.split($\)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'redis'
+  gem.add_dependency 'activesupport'
+
+  gem.add_development_dependency 'dotenv'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'git'
+  gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'coveralls'
+end

data/spec/connection_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Redtastic::Connection do
+  before do
+    # Reset any connections
+    Redtastic::Connection.redis = nil
+    Redtastic::Connection.namespace = nil
+  end
+
+  describe '#establish_connection' do
+    before do
+      Redtastic::ScriptManager.stub(:load_scripts)
+      redis = Redis.new(host: 'foo', port: 9000)
+      Redtastic::Connection.establish_connection(redis, 'bar')
+    end
+
+    it 'properly sets the redis connection' do
+      expect(Redtastic::Connection.redis.client.host).to eq('foo')
+      expect(Redtastic::Connection.redis.client.port).to eq(9000)
+    end
+
+    it 'properly sets the namespace' do
+      expect(Redtastic::Connection.namespace).to eq('bar')
+    end
+  end
+
+  context 'when setting options one at a time' do
+    before do
+      Redtastic::Connection.redis = Redis.new(host: 'foo', port: 1111)
+      Redtastic::Connection.namespace = 'bar'
+    end
+
+    it 'properly sets the redis connection' do
+      expect(Redtastic::Connection.redis.client.host).to eq('foo')
+      expect(Redtastic::Connection.redis.client.port).to eq(1111)
+    end
+
+    it 'properly sets the namespace' do
+      expect(Redtastic::Connection.namespace).to eq('bar')
+    end
+  end
+end