all_seeing_eye 0.0.1

data/lib/all_seeing_eye.rb

@@ -0,0 +1,26 @@
+class AllSeeingEye  
+  def self.configuration(location = nil)
+    location ||= 'config'
+    @@configuration ||= if defined?(Rails)
+      YAML::load_file("#{Rails.root}/#{location}/all_seeing_eye.yml")
+    else
+      YAML::load_file("./#{location}/all_seeing_eye.yml")
+    end
+    @@configuration
+  end
+  
+  def self.redis(options = {})
+    options = {:host => 'localhost', :port => 6379}.merge(options)
+    @@redis ||= Redis.new(:host => options[:host], :port => options[:port])
+  end
+end
+
+require 'redis'
+require 'yaml'
+require 'chronic'
+require 'rbtree'
+require 'all_seeing_eye/model'
+require 'all_seeing_eye/models/request'
+require 'all_seeing_eye/integrations/rails2' if defined?(Rails)
+require 'active_support'
+require 'active_support/all'

data/test/base_test.rb

@@ -0,0 +1,11 @@
+require 'test_helper'
+
+context 'with the base class' do
+  test 'load the configuration' do
+    assert AllSeeingEye.configuration['model']
+  end
+  
+  test 'connect to Redis' do
+    assert_equal '1', AllSeeingEye.redis.info["connected_clients"]
+  end
+end

data/test/fixtures/all_seeing_eye.yml

@@ -0,0 +1,21 @@
+timeout: 0.01
+model:
+  ip:
+    object: request
+    method: .remote_ip
+  uri:
+    object: request
+    method: .request_uri
+  browser:
+    object: request
+    method: .env["HTTP_USER_AGENT"]
+  controller:
+    object: params
+    method: '[:controller]'
+    params: 
+  action:
+    object: params
+    method: '[:action]'
+  status:
+    object: response
+    method: .status

data/test/fixtures/redis-test.conf

@@ -0,0 +1,115 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes

data/test/model_test.rb

@@ -0,0 +1,306 @@
+require 'test_helper'
+
+context 'class methods without setup' do
+  teardown do
+    AllSeeingEye.redis.flushall
+  end
+  
+  test 'models know their own name' do
+    assert_equal 'model', AllSeeingEye::Model.model_name
+  end
+  
+  test 'models correctly find their next ID' do
+    id = AllSeeingEye::Model.next_id
+    (id + 1..(id + 11)).each {|n|assert_equal AllSeeingEye::Model.next_id, n}
+  end
+  
+  test 'models should get fields' do
+    assert_equal AllSeeingEye::Model.fields,
+      {"uri"=>{"method"=>".request_uri", "object"=>"request"}, "action"=>{"method"=>"[:action]", "object"=>"params"}, "ip"=>{"method"=>".remote_ip", "object"=>"request"}, "controller"=>{"method"=>"[:controller]", "params"=>nil, "object"=>"params"}, "browser"=>{"method"=>".env[\"HTTP_USER_AGENT\"]", "object"=>"request"}, "status"=>{"method"=>".status", "object"=>"response"}}
+  end
+  
+  test 'models should get field names' do
+    assert_equal ["uri", "action", "ip", "controller", "browser", "status", "created_at", "time_spent"],
+      AllSeeingEye::Model.field_names
+  end
+  
+  test 'return a new model object' do
+    time = Time.now
+    obj = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => time)
+    
+    assert_equal '/test/', obj.uri
+    assert_equal '127.0.0.1', obj.ip
+    assert_equal time, obj.created_at
+    assert_equal nil, obj.action
+  end
+  
+  test 'create a model object' do
+    obj = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1')
+    
+    assert_equal '/test/', obj.uri
+    assert_equal '127.0.0.1', obj.ip
+    assert_equal Marshal.dump(obj), AllSeeingEye.redis["allseeingeye:model:#{AllSeeingEye::Model.next_id - 1}"]
+    assert_equal '1', AllSeeingEye.redis.zscore("allseeingeye:model:fields:uri", '/test/')
+    assert_equal '1', AllSeeingEye.redis.zscore("allseeingeye:model:fields:ip", '127.0.0.1')
+    assert_equal [obj.id.to_s], AllSeeingEye.redis.sort("allseeingeye:model:fields:uri:/test/", :by => 'nosort')
+    assert_equal [obj.id.to_s], AllSeeingEye.redis.sort("allseeingeye:model:fields:ip:127.0.0.1", :by => 'nosort')
+  end
+  
+  test 'unmarshal an object with load' do
+    time = Time.now
+    raw = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => time)
+    dump = Marshal.dump(raw)
+    obj = AllSeeingEye::Model.load(dump)
+    
+    assert_equal '/test/', obj.uri
+    assert_equal '127.0.0.1', obj.ip
+    assert_equal time, obj.created_at
+    assert_equal raw.id, obj.id
+  end
+  
+  test 'find an object by its ID' do
+    time = Time.now
+    obj = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => time)
+    found = AllSeeingEye::Model.find(obj.id)
+    
+    [:uri, :ip, :created_at].each {|f| assert_equal obj.send(f), found.send(f)}
+  end
+  
+  test 'find all objects' do
+    obj1 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    obj2 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    obj3 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    
+    assert_equal [obj1.id, obj2.id, obj3.id], AllSeeingEye::Model.all.collect(&:id)
+  end
+  
+  test 'find the count of all objects' do
+    obj1 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    obj2 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    obj3 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => Time.now)
+    
+    assert_equal 3, AllSeeingEye::Model.count
+  end
+end
+
+context 'class methods with setup' do
+  setup do
+    @time = Time.now
+    
+    @obj1 = AllSeeingEye::Model.create(:uri => '/', :ip => '127.0.0.1', :created_at => @time - 10 * 60)
+    @obj2 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => @time - 5 * 60)
+    @obj3 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => @time - 3 * 60)
+    @obj4 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '192.168.0.1', :created_at => @time - 1 * 60)
+    @obj5 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '127.0.0.1', :created_at => @time + 2 * 60)
+    @obj6 = AllSeeingEye::Model.create(:uri => '/test/', :ip => '192.168.0.1', :created_at => @time + 2 * 60)
+  end
+  
+  teardown do
+    AllSeeingEye.redis.flushall
+  end
+  
+  test 'find all objects in a date range with a start and a stop' do
+    results = AllSeeingEye::Model.all(:start => @time - 11 * 60, :stop => @time - 4 * 60)
+    
+    assert_equal 2, results.count
+    assert_equal @obj1.id, results[0].id
+    assert_equal @obj2.id, results[1].id
+  end
+  
+  test 'find all objects in a date range with only a start' do
+    results = AllSeeingEye::Model.all(:start => @time)
+    
+    assert_equal 2, results.count
+    assert_equal @obj5.id, results[0].id
+    assert_equal @obj6.id, results[1].id
+  end
+  
+  test 'find all objects in a date range with only a stop' do
+    results = AllSeeingEye::Model.all(:stop => @time)
+    
+    assert_equal 4, results.count
+    assert_equal @obj1.id, results[0].id
+    assert_equal @obj2.id, results[1].id   
+    assert_equal @obj3.id, results[2].id
+    assert_equal @obj4.id, results[3].id   
+  end
+  
+  test 'find model objects by a string field' do
+    results = AllSeeingEye::Model.find_by_field('uri', :value => '/test/')
+
+    assert_equal 5, results.count
+    assert results.collect(&:id).include?(@obj2.id)
+    assert results.collect(&:id).include?(@obj3.id)
+    assert results.collect(&:id).include?(@obj4.id)
+    assert results.collect(&:id).include?(@obj5.id)
+    assert results.collect(&:id).include?(@obj6.id)
+  end
+  
+  test 'find model objects by a string field with a start' do
+    results = AllSeeingEye::Model.find_by_field('uri', :value => '/test/', :start => @time - 4 * 60)
+    
+    assert_equal 4, results.count
+    assert results.collect(&:id).include?(@obj3.id)
+    assert results.collect(&:id).include?(@obj4.id)
+    assert results.collect(&:id).include?(@obj5.id)
+    assert results.collect(&:id).include?(@obj6.id)
+  end
+  
+  test 'find model objects by a string field with a stop' do
+    results = AllSeeingEye::Model.find_by_field('uri', :value => '/test/', :stop => @time)
+    
+    assert_equal 3, results.count
+    assert results.collect(&:id).include?(@obj2.id)
+    assert results.collect(&:id).include?(@obj3.id)
+    assert results.collect(&:id).include?(@obj4.id)
+  end
+  
+  test 'find model objects by a string field with a start and a stop' do
+    results = AllSeeingEye::Model.find_by_field('uri', :value => '/test/', :start => @time - 4 * 60, :stop => @time)
+    
+    assert_equal 2, results.count
+    assert results.collect(&:id).include?(@obj3.id)
+    assert results.collect(&:id).include?(@obj4.id)
+  end
+  
+  test 'find model objects by a date field' do
+    results = AllSeeingEye::Model.find_by_field('created_at', :start => @obj1.created_at, :stop => @obj4.created_at)
+    
+    assert_equal 4, results.count
+    assert_equal results.first.id, @obj1.id
+    assert_equal results.last.id, @obj4.id
+  end
+  
+  test 'count model objects' do
+    assert_equal 2, AllSeeingEye::Model.count(:start => @time - 11 * 60, :stop => @time - 4 * 60)
+    assert_equal 2, AllSeeingEye::Model.count(:start => @time + 2 * 60, :stop => @time + 2 * 60)
+    assert_equal 4, AllSeeingEye::Model.count(:stop => @time)
+    assert_equal 2, AllSeeingEye::Model.count(:start => @time)
+    assert_equal 6, AllSeeingEye::Model.count
+  end
+  
+  test 'return the first object' do
+    assert_equal @obj1.id, AllSeeingEye::Model.first.id
+  end
+  
+  test 'return the last object' do
+    assert_equal @obj6.id, AllSeeingEye::Model.last.id
+  end
+  
+  test 'return count in a field' do
+    counts = AllSeeingEye::Model.count_by_field('uri')
+    
+    assert_equal '/test/', counts.first.first
+    assert_equal 5, counts.first.last
+    assert_equal '/', counts.last.first
+    assert_equal 1, counts.last.last
+  end
+  
+  test 'search with Chronic' do
+    results = AllSeeingEye::Model.search('last friday to five minutes ago')
+    
+    assert_equal 2, results.size
+    assert_equal @obj1.id, results[0].id
+    assert_equal @obj2.id, results[1].id
+  end
+  
+  test 'search by a specific field' do
+    results = AllSeeingEye::Model.search('uri:/test/')
+    
+    assert_equal 5, results.count
+    assert results.collect(&:id).include?(@obj2.id)
+    assert results.collect(&:id).include?(@obj3.id)
+    assert results.collect(&:id).include?(@obj4.id)
+    assert results.collect(&:id).include?(@obj5.id)
+    assert results.collect(&:id).include?(@obj6.id)
+  end
+  
+  test 'search without a specific field' do
+    results = AllSeeingEye::Model.search('192.168.0.1')
+    
+    assert_equal 2, results.count
+    assert results.collect(&:id).include?(@obj4.id)
+    assert results.collect(&:id).include?(@obj6.id)
+  end
+end
+
+context 'many objects' do
+  setup do
+    @time = Time.now
+    
+    2000.times do |n|
+      instance_variable_set("@obj#{n}".to_sym, AllSeeingEye::Model.create(:uri => '/', :ip => '127.0.0.1', :created_at => @time - (n * (1999 - n)).minutes))
+    end
+  end
+  
+  test 'conglomerate an array successfully' do
+    results = AllSeeingEye::Model.all
+    counts = AllSeeingEye::Model.conglomerate(results)
+    
+    assert_equal 500, counts.count
+    assert_equal 2000, counts.inject(0) {|sum, k| sum + k.last}
+    assert_equal @obj999.created_at.to_i, counts.first.first
+  end
+end
+
+context 'instance methods' do  
+  teardown do
+    AllSeeingEye.redis.flushall
+  end
+  
+  test 'assign all attr_accessors from fields at initialization' do
+    obj = AllSeeingEye::Model.new
+    
+    AllSeeingEye::Model.field_names.each {|f| assert obj.respond_to?(f.to_sym); assert obj.respond_to?("#{f}=".to_sym)}
+  end
+  
+  test 'assign an object an ID if it does not have one' do
+    obj = AllSeeingEye::Model.new
+    
+    assert_equal nil, obj.instance_variable_get(:@id)
+    assert_equal 1, obj.id
+    assert_equal 1, obj.instance_variable_get(:@id)
+  end
+  
+  test 'save a new model object' do
+    obj = AllSeeingEye::Model.new(:uri => '/test/', :ip => '127.0.0.1')
+    
+    assert obj.save
+    assert_equal '/test/', obj.uri
+    assert_equal '127.0.0.1', obj.ip
+    assert_equal Marshal.dump(obj), AllSeeingEye.redis["allseeingeye:model:#{AllSeeingEye::Model.next_id - 1}"]
+    assert_equal '1', AllSeeingEye.redis.zscore("allseeingeye:model:fields:uri", '/test/')
+    assert_equal '1', AllSeeingEye.redis.zscore("allseeingeye:model:fields:ip", '127.0.0.1')
+    assert_equal [obj.id.to_s], AllSeeingEye.redis.sort("allseeingeye:model:fields:uri:/test/", :by => 'nosort')
+  end
+  
+  test 'save a new model object with date attributes' do
+    time = Time.now
+    obj = AllSeeingEye::Model.new(:created_at => time)
+    
+    assert obj.save
+    assert_equal time, obj.created_at
+    assert_equal Marshal.dump(obj), AllSeeingEye.redis["allseeingeye:model:#{AllSeeingEye::Model.next_id - 1}"]
+    assert_equal time.to_i.to_s, AllSeeingEye.redis.zscore("allseeingeye:model:fields:created_at", obj.id)
+  end
+  
+  test 'ignore empty and nil attributes in model object creation' do
+    time = Time.now
+    obj = AllSeeingEye::Model.new(:created_at => time, :uri => '/test/', :ip => nil, :action => '')
+    
+    assert obj.save
+    assert_equal time, obj.created_at
+    assert_equal '/test/', obj.uri
+    assert_equal nil, obj.ip
+    assert_equal '', obj.action
+    assert_equal Marshal.dump(obj), AllSeeingEye.redis["allseeingeye:model:#{AllSeeingEye::Model.next_id - 1}"]
+    assert_equal time.to_i.to_s, AllSeeingEye.redis.zscore("allseeingeye:model:fields:created_at", obj.id)
+    assert_equal '1', AllSeeingEye.redis.zscore("allseeingeye:model:fields:uri", '/test/')
+    assert_equal nil, AllSeeingEye.redis.zscore("allseeingeye:model:fields:ip", nil)
+    assert_equal nil, AllSeeingEye.redis.zscore("allseeingeye:model:fields:action", '')
+    assert_equal [obj.id.to_s], AllSeeingEye.redis.sort("allseeingeye:model:fields:uri:/test/", :by => 'nosort')
+    assert_equal [], AllSeeingEye.redis.sort("allseeingeye:model:fields:ip:", :by => 'nosort')
+    assert_equal [], AllSeeingEye.redis.sort("allseeingeye:model:fields:action:", :by => 'nosort')
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,57 @@
+require 'rubygems'
+
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+require 'test/unit'
+require 'yaml'
+require 'all_seeing_eye'
+
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** try running `sudo rake install`"
+  abort ''
+end
+
+at_exit do
+  next if $!
+
+  if defined?(MiniTest)
+    exit_code = MiniTest::Unit.new.run(ARGV)
+  else
+    exit_code = Test::Unit::AutoRunner.run
+  end
+
+  pid = `ps -A -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+  exit exit_code
+end
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/fixtures/redis-test.conf`
+AllSeeingEye.redis(:host => 'localhost', :port => 9736)
+
+AllSeeingEye.configuration('test/fixtures')
+
+##
+# test/spec/mini 3
+# http://gist.github.com/25455
+# chris@ozmm.org
+#
+def context(*args, &block)
+  return super unless (name = args.first) && block
+  require 'test/unit'
+  klass = Class.new(defined?(ActiveSupport::TestCase) ? ActiveSupport::TestCase : Test::Unit::TestCase) do
+    def self.test(name, &block)
+      define_method("test_#{name.gsub(/\W/,'_')}", &block) if block
+    end
+    def self.xtest(*args) end
+    def self.setup(&block) define_method(:setup, &block) end
+    def self.teardown(&block) define_method(:teardown, &block) end
+  end
+  (class << klass; self end).send(:define_method, :name) { name.gsub(/\W/,'_') }
+  klass.class_eval &block
+  # XXX: In 1.8.x, not all tests will run unless anonymous classes are kept in scope.
+  ($test_classes ||= []) << klass
+end