wackamole 0.0.3 → 0.0.4

data/lib/wackamole.rb

@@ -1,7 +1,7 @@
 module Wackamole
 
   # :stopdoc:
-  VERSION = '0.0.3'                                                          unless defined? Wackamole::VERSION
+  VERSION = '0.0.4'                                                          unless defined? Wackamole::VERSION
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR unless defined? Wackamole::LIBPATH
   PATH    = ::File.dirname(LIBPATH) + ::File::SEPARATOR                      unless defined? Wackamole::PATH
   # :startdoc:

data/lib/wackamole/models/control.rb

@@ -3,6 +3,22 @@ require 'logger'
 
 module Wackamole
   class Control
+
+    # -----------------------------------------------------------------------
+    # Initialize app by reading off mongo configuration parameters if necessary
+    def self.init_config( config_file, env )
+      begin
+        config  = YAML.load_file( config_file )
+        @config = config[env]
+        raise "Invalid environment `#{env}" unless @config
+        raise "Unable to find host in - #{@config.inspect}" unless @config.has_key?('host')
+        raise "Unable to find port in - #{@config.inspect}" unless @config.has_key?('port')            
+      rescue => boom
+        @config = nil          
+        raise "Hoy! An error occur loading the config file `#{config_file} -- #{boom}"
+      end
+      @config
+    end
     
     # -------------------------------------------------------------------------
     # Defines mole db identity      
@@ -76,25 +92,6 @@ module Wackamole
         @db = nil
         db( db_name )
       end
-
-      # -----------------------------------------------------------------------
-      # Initialize app by reading off mongo configuration parameters if necessary
-# BOZO !!      
-# File.join( ENV['HOME'], %w[.wackamole wackamole.yml] )
-# 
-      def self.init_config( config_file, env )
-        begin
-          config      = YAML.load_file( config_file )
-          @config     = config[env]
-          raise "Invalid environment `#{env}" unless @config
-          raise "Unable to find host in - #{@config.inspect}" unless @config.has_key?('host')
-          raise "Unable to find port in - #{@config.inspect}" unless @config.has_key?('port')            
-        rescue => boom
-          @config = nil          
-          raise "Hoy! An error occur loading the config file `#{config_file} -- #{boom}"
-        end
-        @config
-      end
       
       def self.config
         raise "You must call init_config before using this object" unless @config
@@ -119,13 +116,39 @@ module Wackamole
         return @db if @db and @db.name == db_name
         raise "No database specified" unless db_name
         @db = connection.db( db_name, opts )
-        ensure_indexes
-        @db
       end  
     
       # -----------------------------------------------------------------------
       # Make sure the right indexes are set
-      def self.ensure_indexes
+      def self.ensure_indexes!( set=true, drop_first=false, show_info=false, clear_only=false )
+        mole_databases.each do |db_name|
+          db = db( db_name )
+          
+          puts "-"*100
+          puts "Database - #{db_name}"
+          
+          if drop_first or clear_only
+            puts "[WARNING] Dropping indexes on all Rackamole collections"
+            %w(logs features users).each do |c|            
+              db[c].drop_indexes
+            end            
+          end
+          
+          if set and !clear_only
+            puts "[INFO] Creating indexes on all Rackamole collections..."
+            Wackamole::Log.ensure_indexes!
+            Wackamole::Feature.ensure_indexes!
+            Wackamole::User.ensure_indexes!
+          end
+                       
+          if show_info
+            %w(logs features users).each do |c|
+              indexes = db[c].index_information
+              puts "\tIndexes on collection `#{c}"
+              indexes.keys.sort.each { |k| puts "\t\tIndex: %-30s -- %s" % [k,indexes[k].join( ", ")] }
+            end
+          end
+        end
       end
   end
 end

data/lib/wackamole/models/log.rb

@@ -29,16 +29,18 @@ module Wackamole
       indexes       = logs_cltn.index_information
       created_count = 0
             
-      [:fid, :uid, :did, :tid].each do |name|
+      [:typ, :fid].each do |name|
         unless indexes.has_key?( "#{name}_1" )
           logs_cltn.create_index( name )
           created_count += 1
         end
       end
-      unless indexes.has_key?( 'did_-1_tid_-1' )
-        logs_cltn.create_index( [ [:did, Mongo::DESCENDING], [:tid, Mongo::DESCENDING] ] ) 
-        created_count += 1
+      unless indexes.key?( "did_-1_tid_-1_type_-1" )
+        logs_cltn.create_index( [[:did, Mongo::DESCENDING], [:tid, Mongo::DESCENDING], [:type, Mongo::DESCENDING] ] )
       end
+      unless indexes.key?( "fid_-1_did_-1" )
+        logs_cltn.create_index( [[:fid, Mongo::DESCENDING], [:did, Mongo::DESCENDING], [:type, Mongo::DESCENDING] ] )
+      end              
       created_count
     end
   end    

data/lib/wackamole/models/mission.rb

@@ -4,73 +4,57 @@ module Wackamole
   class Mission
     extend ::SingleForwardable
 
-    def self.rollups_cltn() @rollups ||= Wackamole::Control.collection( 'rollups', 'wackamole_mdb', :strict => false ); end  
-  
-    def_delegators :rollups_cltn, :find, :find_one
-  
-    # -------------------------------------------------------------------------
-    # Allows user to clear out perf or fault state til the next tick...  
-    def self.reset!( app, env, type )
-      rollups_cltn.update( { :app => app }, { '$set' => { "envs.#{env}.#{type}" => 0 } } )
+    # -----------------------------------------------------------------------
+    # Pick up moled application pulse
+    def self.pulse( last_tick )
+      to_date   = count_logs
+      today     = count_logs( last_tick, true )
+      last_tick = count_logs( last_tick )
+      { :to_date => to_date, :today => today, :last_tick => last_tick }
     end
-      
-    # -------------------------------------------------------------------------      
-    # Clean up rollups. Check if mole_db is still around
-    def self.clean_up!
-      databases   = Wackamole::Control.mole_databases
-      con         = Wackamole::Control.connection      
-      rollups     = rollups_cltn.find( {} )
-      delete_list = []
-      rollups.each do |rollup|
-        app       = rollup['app']
-        envs_info = rollup[envs].keys
-        envs_info.each do |env|
-          db_name = Wackamole::Control.to_mole_db( app, env )
-          delete_list << env unless databases.include?( db_name )
-        end
-        # if app is no longer around blow away the rollup
-        if delete_list.size == envs_info.size
-          rollups_cltn.remove( { :_id => rollup['_id'] } )
-        else
-          delete_list.each do |env| 
-            rollup[envs].delete( env )
-          end
-          rollups_cltn.save( rollup, :safe => true ) unless delete_list.empty?
-        end
+    
+    # -----------------------------------------------------------------------      
+    # generates mole logs conditons
+    def self.gen_conds( now, single_day )
+      conds = {}
+      if now
+        date_id     = now.to_date_id.to_s
+        time_id     = now.to_time_id
+        conds[:did] = date_id
+        conds[:tid] = {'$gte' => time_id} unless single_day
       end
+      conds
     end
     
-    # -------------------------------------------------------------------------
-    # Retrieve reports if any...  
-    # BOZO !! Handle case where report is no longer valid - ie no mole db
-    def self.rollups( now, reset )
-      clean_up!
-      rollups = comb_applications( now, reset )
-      rollups.each_pair do |app_name, env_info|
-        env_info[envs].each_pair do |env, info|
-          info.each_pair do |mole_type, count|
-            rollup    = rollups_cltn.find_one( { :app => app_name } )
-            type_name = to_type_name( mole_type )
-            if rollup
-              rollup_info = rollup[envs]
-              if rollup_info and rollup_info[env]
-                (rollup_info[env][type_name] and !reset) ? rollup_info[env][type_name] += count : rollup_info[env][type_name] = count
-              elsif rollup_info
-                rollup_info[env] = { type_name => count }
-              # else
-              #   rollup_info[envs] = { env => { type_name => count } }
-              end
-              rollups_cltn.save( rollup, :safe => true )
-            else
-              row = { :app => app_name, envs => { env => { type_name => count } } }
-              rollups_cltn.insert( row, :safe => true )
-            end
+    # -----------------------------------------------------------------------
+    # Compute mole counts for each moled apps
+    def self.count_logs( now=nil, single_day=false )
+      counts = {}
+      conds  = gen_conds( now, single_day )
+      # elapsed = Benchmark.realtime do
+        Wackamole::Control.mole_databases.each do |db_name|
+          db            = Wackamole::Control.db( db_name )
+          app_name, env = Wackamole::Control.extract_app_info( db_name )
+          logs_cltn     = db['logs']
+          
+          totals = { Rackamole.feature => 0, Rackamole.perf => 0, Rackamole.fault => 0 }
+          if counts[app_name]
+            counts[app_name][env] = totals
+          else
+            counts[app_name] = { env => totals }
+          end
+          row = counts[app_name][env]
+          [Rackamole.feature, Rackamole.perf, Rackamole.fault].each do |t|
+            conds[:typ] = t
+            logs  = logs_cltn.find( conds, :fields => [:_id] )
+            row[t] = logs.count
           end
         end
-      end
-      find( {}, :sort => [ [:app, Mongo::ASCENDING], [:env, Mongo::ASCENDING] ] ).to_a
+      # end
+      # puts "Computing counts %d -- %5.4f" % [counts.size, elapsed]
+      counts       
     end
-  
+      
     # =========================================================================
     private
 
@@ -88,77 +72,5 @@ module Wackamole
             raise "Invalid mole log type `#{type}"
         end
       end
-  
-      # ---------------------------------------------------------------------------
-      # Check moled apps status - reports any perf/excep that occurred since the 
-      # last check
-      def self.comb_applications( now, reset )     
-        report = {}
-        Wackamole::Control.mole_databases.each do |db_name|
-          db = Wackamole::Control.db( db_name )
-                         
-          app_name, env = Wackamole::Control.extract_app_info( db_name )        
-          totals        = analyse_logs( db, now, reset )
-        
-          if report[app_name]
-            report[app_name][envs][env] = totals
-          else
-            report[app_name] = { envs => { env => totals } }
-          end
-        end
-        report
-      end
-    
-      # -------------------------------------------------------------------------
-      # Report on possible application issues
-      def self.amend_report( report, app_name, env, log )
-        type = log['typ']
-        
-        if report[app_name]
-          env_info = report[app_name][envs]
-          if env_info[env]
-            env_info[env][type] ? env_info[env][type] += 1 : env_info[env][type] = 1
-          else
-            env_info[ env ] = { type => 1 }
-          end
-        else
-          report[app_name] = { envs => { env => { type => 1 } }  } 
-        end
-      end    
-
-      # -------------------------------------------------------------------------
-      # envs key
-      def self.envs() @envs ||= 'envs'; end
-        
-      # -------------------------------------------------------------------------        
-      # computes counts for each mole types
-      def self.analyse_logs( db, now, reset )
-        check_types = [Rackamole.perf, Rackamole.fault, Rackamole.feature]
-        date_id     = now.to_date_id.to_s
-        time_id     = now.to_time_id
-      
-        if reset
-          conds = {
-            :did => date_id,
-            :typ => { '$in'  => check_types }
-          }
-        else
-          conds = {
-            :did => { '$gte' => date_id },
-            :tid => { '$gte' => time_id },
-            :typ => { '$in'  => check_types }
-          }
-        end        
-
-        logs   = db['logs'].find( conds, :fields => ['typ', 'rti', 'fault', 'fid'] )
-        totals = 
-        { 
-          Rackamole.feature  => 0, 
-          Rackamole.perf     => 0, 
-          Rackamole.fault    => 0 
-        }
-        logs.each { |log| totals[log['typ']] += 1 }
-        totals
-      end
    end   
 end

data/lib/wackamole/models/mole_info.rb

@@ -12,7 +12,9 @@ module Wackamole
     # TODO - PERF - try just using cursor vs to_a
     def self.collect_dashboard_info( now )    
       info    = {}
-      day_logs = logs_cltn.find( { :did => now.to_date_id.to_s }, :fields => [:typ, :fid, :tid, :did, :uid] ).to_a
+      day_logs = logs_cltn.find( { :did => now.to_date_id.to_s }, 
+        :fields => [:typ, :fid, :tid, :did, :uid], 
+        :sort => [ [:tid => Mongo::ASCENDING] ] ).to_a
         
       # Fetch user count for this hour
       users = day_logs.inject( Set.new ) do |set,log| 
@@ -44,27 +46,31 @@ module Wackamole
       end
     
       # Count all logs per hourly time period
-      times         = (0...24).to_a
-      time_info     = times.inject(OrderedHash.new) { |res,time| res[time] = { :user => 0, :feature => 0, :perf => 0, :fault => 0 };res }    
+      hours         = (0...24).to_a
+      hour_info     = hours.inject(OrderedHash.new) { |res,hour| res[hour] = { :user => 0, :feature => 0, :perf => 0, :fault => 0 };res }    
       user_per_hour = {}
       day_logs.each do |log|
         date_tokens = log['did'].match( /(\d{4})(\d{2})(\d{2})/ ).captures
-        time_tokens = log['tid'].match( /(\d{2})(\d{2})(\d{2})/ ).captures    
+        time_tokens = log['tid'].match( /(\d{2})(\d{2})(\d{2})/ ).captures
+        
         utc         = Time.utc( date_tokens[0], date_tokens[1], date_tokens[2], time_tokens[0], time_tokens[1], time_tokens[2] )
-        time        = utc.getlocal.hour
-        if user_per_hour[time]
-          unless user_per_hour[time].include? log['uid']
-            time_info[time][:user] += 1
-            user_per_hour[time] << log['uid']
+        local       = utc.localtime
+        hour        = local.hour
+        current_day = local.day
+        next if local.day != current_day
+        if user_per_hour[hour]
+          unless user_per_hour[hour].include? log['uid']
+            hour_info[hour][:user] += 1
+            user_per_hour[hour] << log['uid']
           end
         else
-          user_per_hour[time] = [ log['uid'] ]
-          time_info[time][:user] += 1
+          user_per_hour[hour] = [ log['uid'] ]
+          hour_info[hour][:user] += 1
         end
         case log['typ']
-          when Rackamole.feature : time_info[time][:feature] += 1 if features.add?( log['fid'])
-          when Rackamole.perf    : time_info[time][:perf]    += 1
-          when Rackamole.fault   : time_info[time][:fault]   += 1
+          when Rackamole.feature : hour_info[hour][:feature] += 1 if features.add?( log['fid'])
+          when Rackamole.perf    : hour_info[hour][:perf]    += 1
+          when Rackamole.fault   : hour_info[hour][:fault]   += 1
         end
       end
         
@@ -73,7 +79,7 @@ module Wackamole
       info[:fault_series]   = []
       info[:perf_series]    = []
       info[:feature_series] = []
-      time_info.values.map do |hash| 
+      hour_info.values.map do |hash| 
         info[:user_series]    << hash[:user]
         info[:fault_series]   << hash[:fault]
         info[:perf_series]    << hash[:perf]

data/lib/wackamole/models/search_filter.rb

@@ -39,11 +39,17 @@ module Wackamole
       @types ||= %w[All Feature Perf Fault]
     end
   
+    # ---------------------------------------------------------------------------
+    # Set filter type  
+    def mole_type( type )
+      self.type = to_filter_type( type )
+    end
+    
     # ---------------------------------------------------------------------------
     # Find all features  
     def features
       rows = Feature.features_cltn.find().to_a
-      features = rows.map { |f| [context_for(f), f['_id']] }   
+      features = rows.map { |f| [context_for(f), f['_id'].to_s] }   
       features.sort! { |a,b| a.first <=> b.first }
       features.insert( 0, ["All", -1] )
     end
@@ -117,7 +123,7 @@ module Wackamole
       end
     
       if browser_type != 'All'
-        conds[:bro] = browser_type
+        conds["bro.name"] = browser_type
       end
     
       # filter mole_features
@@ -135,7 +141,7 @@ module Wackamole
         if key
           if key == "user"
             users = Wackamole::User.users_cltn.find( { :una => Regexp.new( tokens.first ) }, :fields => ['_id'] )
-            conds[field_map( key )] = { '$in' => users.collect{ |u| u['_id'] } }
+            conds[field_map( 'user_id' )] = { '$in' => users.collect{ |u| u['_id'] } }
           elsif tokens.size == 2
             conds["#{field_map(key)}.#{tokens.first}"] = Regexp.new( tokens.last )
           elsif tokens.size == 1
@@ -154,16 +160,24 @@ module Wackamole
       # ---------------------------------------------------------------------------
       # Search filter key name map
       def field_map( key )
-        case key.to_sym
-          when :user    : :uid
-          when :method  : :met
-          when :host    : :hos
-          when :session : :ses
-          when :params  : :par
-          else            key
-        end
+        field = Rackamole::Store::MongoDb.field_map[key.to_sym]
+        raise "Unable to map attribute `#{key}" unless field
+        field
       end
   
+      def to_filter_type( type )
+        case type
+          when Rackamole.feature
+            'Feature'
+          when Rackamole.perf
+            'Perf'
+          when Rackamole.fault
+            'Fault'
+          else         
+            raise "Invalid rackamole type `#{type}"
+        end        
+      end
+      
       # Map named type to fixnum value
       def map_mole_type( type )
         case type