minuteman 1.0.3 → 2.0.0.pre

data/lib/minuteman/analyzable.rb

@@ -0,0 +1,96 @@
+require 'msgpack'
+
+module Minuteman
+  module Analyzable
+    module ErrorPatterns
+      DUPLICATE = /(UniqueIndexViolation: (\w+))/.freeze
+      NOSCRIPT = /^NOSCRIPT/.freeze
+    end
+
+    def &(event)
+      operation("AND", [self, event])
+    end
+
+    def |(event)
+      operation("OR", [self, event])
+    end
+    alias_method :+, :|
+
+    def ^(event)
+      operation("XOR", [self, event])
+    end
+
+    def -@()
+      operation("NOT", [self])
+    end
+    alias :~@ :-@
+
+    def -(event)
+      operation("MINUS", [self, event])
+    end
+
+    def count
+      Minuteman.config.redis.call("BITCOUNT", key)
+    end
+
+    def include?(user)
+      Minuteman.config.redis.call("GETBIT", key, user.id) == 1
+    end
+
+    private
+
+    def key_exists?(key)
+      Minuteman.config.redis.call("EXISTS", key) == 1
+    end
+
+    def operation(action, events = [])
+      base_key = Minuteman.config.operations_prefix
+
+      destination_key = if action == "NOT"
+                          "#{base_key}#{events[0].id}:#{action}"
+                        else
+                          src, dst = events[0].id, events[1].id
+                          "#{base_key}#{src}:#{action}:#{dst}"
+                        end
+
+      if key_exists?(destination_key)
+        return Minuteman::Result.new(destination_key)
+      end
+
+      script(Minuteman::LUA_OPERATIONS, 0, action.upcase.to_msgpack,
+             events.map(&:key).to_msgpack, destination_key.to_msgpack)
+
+      Minuteman::Result.new(destination_key)
+    end
+
+    # Stolen from Ohm
+    def script(file, *args)
+      begin
+        cache = Minuteman::LUA_CACHE[Minuteman.config.redis.url]
+
+        if cache.key?(file)
+          sha = cache[file]
+        else
+          src = File.read(file)
+          sha = Minuteman.config.redis.call("SCRIPT", "LOAD", src)
+
+          cache[file] = sha
+        end
+
+        Minuteman.config.redis.call!("EVALSHA", sha, *args)
+
+      rescue RuntimeError
+        case $!.message
+        when ErrorPatterns::NOSCRIPT
+          Minuteman::LUA_CACHE[Minuteman.config.redis.url].clear
+          retry
+        when ErrorPatterns::DUPLICATE
+          raise UniqueIndexViolation, $1
+        else
+          raise $!
+        end
+      end
+    end
+
+  end
+end

data/lib/minuteman/analyzer.rb

@@ -0,0 +1,21 @@
+require 'minuteman/event'
+
+module Minuteman
+  class Analyzer
+    def initialize(action, klass = Minuteman::Event, user = nil)
+      Minuteman.patterns.keys.each do |method|
+        define_singleton_method(method) do |time = Time.now.utc|
+          if !Minuteman.patterns.include?(method)
+            raise MissingPattern.new(method)
+          end
+
+          key = Minuteman.patterns[method].call(time)
+          search = { type: action, time: key }
+          search[:user_id] = user.id if !user.nil?
+
+          klass.find_or_create(search)
+        end
+      end
+    end
+  end
+end

data/lib/minuteman/configuration.rb

@@ -0,0 +1,25 @@
+module Minuteman
+  class Configuration
+    attr_accessor :redis, :patterns, :prefix, :parallel, :operations_prefix
+
+    def initialize
+      @redis = Ohm.redis
+      @prefix = "Minuteman".freeze
+      @operations_prefix = "#{@prefix}::Operations:"
+      @parallel = false
+
+      @patterns = {
+        year:   -> (time) { time.strftime("%Y") },
+        month:  -> (time) { time.strftime("%Y-%m") },
+        day:    -> (time) { time.strftime("%Y-%m-%d") },
+        hour:   -> (time) { time.strftime("%Y-%m-%d %H") },
+        minute: -> (time) { time.strftime("%Y-%m-%d %H:%m") },
+      }
+    end
+
+    def redis=(redis)
+      @redis = redis
+      Ohm.redis = redis
+    end
+  end
+end

data/lib/minuteman/counter.rb

@@ -0,0 +1,21 @@
+require 'minuteman/model'
+
+module Minuteman
+  class Counter < Minuteman::Model
+    class User < Counter
+      attribute :user_id
+
+      def key
+        "#{super}:#{user_id}"
+      end
+    end
+
+    def incr
+      Minuteman.config.redis.call("INCR", key)
+    end
+
+    def count
+      Minuteman.config.redis.call("GET", key).to_i
+    end
+  end
+end

data/lib/minuteman/event.rb

@@ -0,0 +1,12 @@
+require 'minuteman/model'
+require 'minuteman/analyzable'
+
+module Minuteman
+  class Event < Minuteman::Model
+    include Minuteman::Analyzable
+
+    def setbit(int)
+      Minuteman.config.redis.call("SETBIT", key, int, 1)
+    end
+  end
+end

data/lib/minuteman/lua/operations.lua

@@ -0,0 +1,37 @@
+---
+redis.log(redis.LOG_NOTICE, 'Minuteman')
+
+local action = cmsgpack.unpack(ARGV[1])
+local keys = cmsgpack.unpack(ARGV[2])
+local dest = cmsgpack.unpack(ARGV[3])
+
+local function operate(action, keys)
+  if type(keys) == "string" then keys = { keys } end
+
+  redis.call("BITOP", action, dest, unpack(keys) )
+
+  return dest
+end
+
+local function AND(keys) return operate("AND", keys) end
+local function OR(keys)  return operate("OR",  keys) end
+local function XOR(keys) return operate("XOR", keys) end
+local function NOT(keys) return operate("NOT", keys) end
+
+local function MINUS(keys)
+  local items = keys
+  local src = table.remove(items, 1)
+  local and_op = AND(keys)
+
+  return XOR({ src, and_op })
+end
+
+local function operation(action, keys)
+  if action == "MINUS" then
+    return MINUS(keys)
+  else
+    return operate(action, keys)
+  end
+end
+
+return operation(action, keys)

data/lib/minuteman/model.rb

@@ -0,0 +1,36 @@
+require 'ohm'
+
+module Minuteman
+  class Model < ::Ohm::Model
+    attribute :type
+    attribute :time
+
+    def self.find(*args)
+      looked_up = "#{self.name}::#{args.first[:type]}:#{args.first[:time]}:id"
+      potential_id = Minuteman.config.redis.call("GET", looked_up)
+
+      return nil if !potential_id
+
+      event = self[potential_id]
+      event.type = args.first[:type]
+      event.time = args.first[:time]
+
+      event
+    end
+
+    def self.find_or_create(*args)
+      find(*args) || create(*args)
+    end
+
+    def self.create(*args)
+      event = super(*args)
+      Minuteman.config.redis.call("SET", "#{event.key}:id", event.id)
+      event
+    end
+
+    def key
+      "#{self.class.name}::#{type}:#{time}"
+    end
+
+  end
+end

data/lib/minuteman/result.rb

@@ -0,0 +1,12 @@
+require 'minuteman/configuration'
+require 'minuteman/analyzable'
+
+module Minuteman
+  Result = Struct.new(:key) do
+    include Minuteman::Analyzable
+
+    def id
+      @_id ||= "(#{key.gsub(Minuteman.config.operations_prefix, "")})"
+    end
+  end
+end

data/lib/minuteman/trigger.rb

@@ -0,0 +1,4 @@
+module Minuteman
+  module Trigger
+  end
+end

data/lib/minuteman/user.rb

@@ -0,0 +1,38 @@
+require 'ohm'
+require 'securerandom'
+
+module Minuteman
+  class User < ::Ohm::Model
+    attribute :uid
+    attribute :identifier
+
+    unique :uid
+    unique :identifier
+
+    def save
+      self.uid ||= SecureRandom.uuid
+      super
+    end
+
+    def track(action, time = Time.now.utc)
+      Minuteman.track(action, self, time)
+    end
+
+    def add(action, time = Time.now.utc)
+      Minuteman.add(action, time, self)
+    end
+
+    def count(action, time = Time.now.utc)
+      Minuteman::Analyzer.new(action, Minuteman::Counter::User, self)
+    end
+
+    def promote(identifier)
+      self.identifier = identifier
+      save
+    end
+
+    def self.[](identifier_or_uuid)
+      with(:uid, identifier_or_uuid) || with(:identifier, identifier_or_uuid)
+    end
+  end
+end

data/minuteman.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = "minuteman"
-  s.version           = "1.0.3"
+  s.version           = "2.0.0.pre"
   s.summary           = "Bit Analytics"
   s.description       = "Fast and furious tracking system using Redis bitwise operations"
   s.authors           = ["elcuervo"]
@@ -10,9 +10,8 @@ Gem::Specification.new do |s|
   s.files             = `git ls-files`.split("\n")
   s.test_files        = `git ls-files test`.split("\n")
 
-  s.add_dependency("redis", "~> 3.0.3")
+  s.add_dependency("redic", "~> 1.5.0")
+  s.add_dependency("ohm",   "~> 2.3.0")
 
-  s.add_development_dependency("minitest",        "~> 4.3.0")
-  s.add_development_dependency("minitest-given",  "~> 3.0.0")
-  s.add_development_dependency("redis-namespace", "~> 1.2.1")
+  s.add_development_dependency("cutest", "~> 1.2.2")
 end

data/test/helper.rb

@@ -0,0 +1,2 @@
+require 'cutest'
+require 'minuteman'

data/test/minuteman_bench.rb

@@ -0,0 +1,72 @@
+require 'minuteman'
+require 'benchmark/ips'
+
+Minuteman.configure do |config|
+  config.redis = Redic.new("redis://127.0.0.1:6379/1")
+  config.parallel = true
+end
+
+Minuteman.config.redis.call("FLUSHDB")
+
+users = Array.new(5000) { Minuteman::User.create }
+users_one = users.sample(5000)
+users_two = users.sample(2000)
+users_three = users.sample(1000)
+
+Minuteman.track("first:page", users_one)
+Minuteman.track("second:page", users_two)
+Minuteman.track("third:page", users_three)
+
+Benchmark.ips do |x|
+  x.report("tracking users") { Minuteman.track("first:page", users_three) }
+
+  x.report("tracking annoymous users") {
+    Minuteman.track("first:page")
+  }
+
+  x.report("operation: AND") {
+    Minuteman("first:page").day & Minuteman("second:page").day
+  }
+
+  x.report("operation: OR") {
+    Minuteman("first:page").day | Minuteman("second:page").day
+  }
+
+  x.report("operation: XOR") {
+    Minuteman("first:page").day ^ Minuteman("second:page").day
+  }
+
+  x.report("operation: NOT") {
+    -Minuteman("second:page").day
+  }
+
+  x.report("operation: MINUS") {
+    Minuteman("first:page").day + Minuteman("second:page").day
+  }
+
+  x.report("complex operations") {
+    (
+      Minuteman("first:page").day + Minuteman("second:page").day
+    ) - Minuteman("third:page").day
+  }
+
+  x.report("adding to the counter") {
+    Minuteman.add("first:counter")
+  }
+
+  x.report("checking the counter") {
+    Counterman("first:counter").month.count
+  }
+
+  x.report("tracking through a user") {
+    users.sample.track("some:event")
+  }
+
+  x.report("counting through a user") {
+    users.sample.add("some:event")
+  }
+
+  x.report("checking the counter through a user") {
+    users.sample.count("some:event").day.count
+  }
+end

data/test/minuteman_test.rb

@@ -0,0 +1,204 @@
+require 'helper'
+
+@patterns = Minuteman.patterns
+
+prepare do
+  Minuteman.configure do |config|
+    config.redis = Redic.new("redis://127.0.0.1:6379/1")
+  end
+end
+
+setup do
+  Minuteman.config.redis.call("FLUSHDB")
+
+  Minuteman.configure do |config|
+    config.patterns = @patterns
+  end
+end
+
+test "a connection" do
+  assert_equal Minuteman.config.redis.class, Redic
+end
+
+test "models in minuteman namespace" do
+  assert_equal Minuteman::User.create.key, "Minuteman::User:1"
+end
+
+test "an anonymous user" do
+  user = Minuteman::User.create
+
+  assert user.is_a?(Minuteman::User)
+  assert !!user.uid
+  assert !user.identifier
+  assert user.id
+end
+
+test "access a user with and id or an uuid" do
+  user = Minuteman::User.create(identifier: 5)
+
+  assert Minuteman::User[user.uid].is_a?(Minuteman::User)
+  assert Minuteman::User[user.identifier].is_a?(Minuteman::User)
+end
+
+test "track an anonymous user" do
+  user = Minuteman.track("anonymous:user")
+  assert user.uid
+end
+
+test "track an user" do
+  user = Minuteman::User.create
+
+  assert Minuteman.track("login:successful", user)
+
+  analyzer = Minuteman.analyze("login:successful")
+  assert analyzer.day(Time.now.utc).count == 1
+end
+
+test "tracks an anonymous user and the promotes it to a real one" do
+  user = Minuteman.track("enter:website")
+  assert user.identifier == nil
+
+  user.promote(42)
+
+  assert user.identifier == 42
+  assert Minuteman::User[42].uid == user.uid
+  assert Minuteman("enter:website").day.include?(user)
+end
+
+test "create your own storage patterns and access analyzer" do
+  Minuteman.configure do |config|
+    config.patterns = {
+      dia: -> (time) { time.strftime("%Y-%m-%d") }
+    }
+  end
+
+  Minuteman.track("logeo:exitoso")
+  assert Minuteman("logeo:exitoso").dia.count == 1
+end
+
+test "use the method shortcut" do
+  5.times { Minuteman.track("enter:website") }
+
+  assert Minuteman("enter:website").day.count == 5
+end
+
+scope "operations" do
+  setup do
+    Minuteman.config.redis.call("FLUSHDB")
+
+    @users = Array.new(3) { Minuteman::User.create }
+    @users.each do |user|
+      Minuteman.track("landing_page:new", @users)
+    end
+
+    Minuteman.track("buy:product", @users[0])
+    Minuteman.track("buy:product", @users[2])
+  end
+
+  test "AND" do
+    and_op = Minuteman("landing_page:new").day & Minuteman("buy:product").day
+    assert and_op.count == 2
+  end
+
+  test "OR" do
+    or_op = Minuteman("landing_page:new").day | Minuteman("buy:product").day
+    assert or_op.count == 3
+  end
+
+  test "XOR" do
+    xor_op = Minuteman("landing_page:new").day ^ Minuteman("buy:product").day
+    assert xor_op.count == 1
+  end
+
+  test "NOT" do
+    assert Minuteman("buy:product").day.include?(@users[2])
+
+    not_op = -Minuteman("buy:product").day
+    assert !not_op.include?(@users[2])
+  end
+
+  test "MINUS" do
+    assert Minuteman("landing_page:new").day.include?(@users[2])
+    assert Minuteman("buy:product").day.include?(@users[2])
+
+    minus_op = Minuteman("landing_page:new").day - Minuteman("buy:product").day
+
+    assert !minus_op.include?(@users[2])
+    assert minus_op.include?(@users[1])
+  end
+end
+
+scope "complex operations" do
+  setup do
+    Minuteman.config.redis.call("FLUSHDB")
+    @users = Array.new(6) { Minuteman::User.create }
+
+    [ @users[0], @users[1], @users[2] ].each do |u|
+      Minuteman.track("promo:email", u)
+    end
+
+    [ @users[3], @users[4], @users[5] ].each do |u|
+      Minuteman.track("promo:facebook", u)
+    end
+
+    [ @users[1], @users[4], @users[6] ].each do |u|
+      Minuteman.track("user:new", u)
+    end
+  end
+
+  test "verbose" do
+    got_promos = Minuteman("promo:email").day + Minuteman("promo:facebook").day
+
+    @users[0..5].each do |u|
+      assert got_promos.include?(u)
+    end
+
+    new_users = Minuteman("user:new").day
+    query = got_promos & new_users
+
+    [ @users[1], @users[4] ].each do |u|
+      assert query.include?(u)
+    end
+
+    assert query.count == 2
+  end
+
+  test "readable" do
+    query = (
+      Minuteman("promo:email").day + Minuteman("promo:facebook").day
+    ) & Minuteman("user:new").day
+
+    assert query.count == 2
+  end
+end
+
+test "count a given event" do
+  10.times { Minuteman.add("enter:new_landing") }
+
+  assert Counterman("enter:new_landing").day.count == 10
+end
+
+test "count events on some dates" do
+  day = Time.new(2015, 10, 15)
+  next_day = Time.new(2015, 10, 16)
+
+  5.times { Minuteman.add("drink:beer", day) }
+  2.times { Minuteman.add("drink:beer", next_day) }
+
+  assert Counterman("drink:beer").month(day).count == 7
+  assert Counterman("drink:beer").day(day).count == 5
+end
+
+scope "do actions through a user" do
+  test "track an event" do
+    user = Minuteman::User.create
+    user.track("login:page")
+
+    3.times { user.add("login:attempts") }
+    2.times { Minuteman.add("login:attempts") }
+
+    assert Minuteman("login:page").day.include?(user)
+    assert Counterman("login:attempts").day.count == 5
+    assert user.count("login:attempts").day.count == 3
+  end
+end