appstats 0.0.13 → 0.0.14

data/.gitignore

@@ -4,3 +4,5 @@ pkg/*
 db/migrate
 doc
 config
+config/appstats.yml
+log

data/Gemfile.lock

@@ -1,7 +1,9 @@
 PATH
   remote: .
   specs:
-    appstats (0.0.13)
+    appstats (0.0.14)
+      daemons
+      net-scp
       rails (>= 2.3.0)
 
 GEM
@@ -37,17 +39,21 @@ GEM
     activesupport (3.0.3)
     arel (2.0.7)
     builder (2.1.2)
+    daemons (1.1.0)
     diff-lcs (1.1.2)
     erubis (2.6.6)
       abstract (>= 1.0.0)
     i18n (0.5.0)
-    mail (2.2.14)
+    mail (2.2.15)
       activesupport (>= 2.3.6)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
     mime-types (1.16)
     mysql (2.8.1)
+    net-scp (1.0.4)
+      net-ssh (>= 1.99.1)
+    net-ssh (2.1.0)
     polyglot (0.3.1)
     rack (1.2.1)
     rack-mount (0.6.13)

data/appstats.gemspec

@@ -15,7 +15,8 @@ Gem::Specification.new do |s|
   # Models are to be used in Rails 3 environment, but the logger can work with Rails 2 apps
   # But, for testing appstats itself, you will need Rails 3
   s.add_dependency('rails','>=2.3.0')
-  # s.add_dependency('rails','3.0.3')
+  s.add_dependency('daemons')
+  s.add_dependency('net-scp')
   
   s.add_development_dependency('rspec')
   s.add_development_dependency('ZenTest')

data/db/config.yml

@@ -14,4 +14,4 @@ test:
   username: root
   password: 
   host: localhost
-  encoding: utf8
+  encoding: utf8

data/db/migrations/20110203151136_create_appstats_contexts.rb

@@ -0,0 +1,16 @@
+class CreateAppstatsContexts < ActiveRecord::Migration
+  def self.up
+    create_table :appstats_contexts do |t|
+      t.string :context_key
+      t.string :context_value
+      t.integer :context_int
+      t.float :context_float
+      t.integer :appstats_entry_id
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :appstats_contexts
+  end
+end

data/db/migrations/20110203151635_rework_appstats_entries.rb

@@ -0,0 +1,21 @@
+class ReworkAppstatsEntries < ActiveRecord::Migration
+  def self.up
+    drop_table :appstats_entries
+    create_table :appstats_entries do |t|
+      t.string :action
+      t.datetime :occurred_at
+      t.text :raw_entry
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :appstats_entries
+    create_table :appstats_entries do |t|
+      t.string :entry_type
+      t.string :name, :null => false
+      t.string :description
+      t.timestamps
+    end
+  end
+end

data/db/migrations/20110204183259_create_log_collectors.rb

@@ -0,0 +1,16 @@
+class CreateLogCollectors < ActiveRecord::Migration
+  def self.up
+    create_table :appstats_log_collectors do |t|
+      t.string :host
+      t.string :filename
+      t.string :status
+      t.timestamps
+    end
+    add_column :appstats_entries, :appstats_log_collector_id, :integer
+  end
+
+  def self.down
+    drop_table :appstats_log_collectors
+    remove_column :appstats_entries, :appstats_log_collector_id
+  end
+end

data/db/schema.rb

@@ -10,12 +10,31 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110105221941) do
+ActiveRecord::Schema.define(:version => 20110204183259) do
+
+  create_table "appstats_contexts", :force => true do |t|
+    t.string   "context_key"
+    t.string   "context_value"
+    t.integer  "context_int"
+    t.float    "context_float"
+    t.integer  "appstats_entry_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
 
   create_table "appstats_entries", :force => true do |t|
-    t.string   "entry_type"
-    t.string   "name",        :null => false
-    t.string   "description"
+    t.string   "action"
+    t.datetime "occurred_at"
+    t.text     "raw_entry"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "appstats_log_collector_id"
+  end
+
+  create_table "appstats_log_collectors", :force => true do |t|
+    t.string   "host"
+    t.string   "filename"
+    t.string   "status"
     t.datetime "created_at"
     t.datetime "updated_at"
   end

data/lib/appstats.rb

@@ -2,10 +2,22 @@ require 'rubygems'
 require 'active_record'
 require "#{File.dirname(__FILE__)}/appstats/code_injections"
 require "#{File.dirname(__FILE__)}/appstats/entry"
+require "#{File.dirname(__FILE__)}/appstats/context"
 require "#{File.dirname(__FILE__)}/appstats/tasks"
-require "#{File.dirname(__FILE__)}/appstats/version"
 require "#{File.dirname(__FILE__)}/appstats/logger"
+require "#{File.dirname(__FILE__)}/appstats/log_collector"
+
+# required in the appstats.gemspec
+# require "#{File.dirname(__FILE__)}/appstats/version"
 
 module Appstats
-  # Your code goes here...
+
+  def self.log(type,message)
+    if $logger.nil?
+      # puts "LOCAL LOG #{type}: #{message}"
+    else
+      $logger.send(type,message)  
+    end
+  end
+  
 end

data/lib/appstats/context.rb

@@ -0,0 +1,26 @@
+
+module Appstats
+  class Context < ActiveRecord::Base
+    set_table_name "appstats_contexts"
+    belongs_to :entry, :foreign_key => "appstats_entry_id"
+    attr_accessible :context_key, :context_value
+  
+    def context_value=(value)
+      self[:context_value] = value
+      self[:context_int] = nil
+      self[:context_float] = nil
+      return if value.nil?
+      as_int = value.to_i
+      as_float = value.to_f
+      self[:context_int] = as_int if as_int.to_s == value
+      self[:context_float] = as_float if as_float.to_s == value || !self[:context_int].nil?
+    end
+  
+    def to_s
+      return "No Context" if context_key.nil? || context_key == ''
+      "#{context_key}[]" if context_value.nil?
+      "#{context_key}[#{context_value}]"
+    end
+  
+  end
+end

data/lib/appstats/entry.rb

@@ -3,10 +3,39 @@ module Appstats
   class Entry < ActiveRecord::Base
     set_table_name "appstats_entries"
     
-    attr_accessible :entry_type, :name, :description
+    has_many :contexts, :table_name => 'appstats_contexts', :foreign_key => 'appstats_entry_id', :order => 'context_key'
+    belongs_to :log_collector, :foreign_key => "appstats_log_collector_id"
+    
+    attr_accessible :action, :occurred_at, :raw_entry
   
     def to_s
-      "Entry [type],[name],[description]"
+      return "No Entry" if action.nil? || action == ''
+      return action if occurred_at.nil?
+      "#{action} at #{occurred_at.strftime('%Y-%m-%d %H:%M:%S')}"
+    end
+    
+    def self.load_from_logger_file(filename)
+      return false if filename.nil?
+      return false unless File.exists?(filename)
+      File.open(filename,"r").readlines.each do |line|
+        load_from_logger_entry(line.strip)
+      end
+      true
+    end
+    
+    def self.load_from_logger_entry(action_and_contexts)
+      return false if action_and_contexts.nil? || action_and_contexts == ''
+      hash = Logger.entry_to_hash(action_and_contexts)
+      entry = Appstats::Entry.new(:action => hash[:action], :raw_entry => action_and_contexts)
+      entry.occurred_at = Time.parse(hash[:timestamp]) unless hash[:timestamp].nil?
+      hash.each do |key,value|
+        next if key == :action
+        next if key == :timestamp
+        context = Appstats::Context.create(:context_key => key, :context_value => value)
+        entry.contexts<< context
+      end
+      entry.save
+      entry
     end
   
   end

data/lib/appstats/log_collector.rb

@@ -0,0 +1,124 @@
+require 'net/ssh'
+require 'net/scp'
+
+module Appstats
+  class LogCollector < ActiveRecord::Base
+    set_table_name "appstats_log_collectors"
+  
+    attr_accessible :host, :filename, :status
+
+    def local_filename
+      File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
+    end
+
+    def self.find_remote_files(remote_login,path,log_template)
+      begin
+        Appstats.log(:info,"Looking for logs in [#{remote_login[:user]}@#{remote_login[:host]}:#{path}] labelled [#{log_template}]")
+        Net::SSH.start(remote_login[:host], remote_login[:user], :password => remote_login[:password] ) do |ssh|
+         all_files = ssh.exec!("cd #{path} && ls | grep #{log_template}").split
+         load_remote_files(remote_login,path,all_files)
+        end
+      rescue Exception => e
+        Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#find_remote_files")
+        Appstats.log(:error,e.message)
+        0
+      end
+    end
+    
+    def self.load_remote_files(remote_login,path,all_files)
+      if all_files.empty?
+        Appstats.log(:info,"No remote logs to load.")
+        return 0
+      end
+
+      count = 0
+      Appstats.log(:info, "About to analyze #{all_files.size} file(s).")
+      all_files.each do |log_name|
+        filename = File.join(path,log_name)
+        if LogCollector.find_by_host_and_filename(remote_login[:host],filename).nil?
+          log_collector = LogCollector.create(:host => remote_login[:host], :filename => filename, :status => "unprocessed")
+          Appstats.log(:info, "  - #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
+          count += 1
+        else
+          Appstats.log(:info, "  - ALREADY LOADED #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
+        end
+      end
+      Appstats.log(:info, "Loaded #{count} file(s).")
+      count
+    end
+    
+    def self.download_remote_files(raw_logins)
+      all = LogCollector.where("status = 'unprocessed'").all
+      if all.empty?
+        Appstats.log(:info,"No remote logs to download.")
+        return 0
+      end
+
+      normalized_logins = {}
+      raw_logins.each do |login|
+        normalized_logins[login[:host]] = login
+      end
+      count = 0
+      
+      Appstats.log(:info,"About to download #{all.size} file(s).")
+      all.each do |log_collector|
+        host = log_collector.host
+        user = normalized_logins[host][:user]
+        password = normalized_logins[host][:password]
+        begin
+          Net::SCP.start( host, user, :password => password ) do |scp|
+            scp.download!( log_collector.filename, log_collector.local_filename )
+          end
+        rescue Exception => e
+          Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#download_remote_files")
+          Appstats.log(:error,e.message)
+        end
+        if File.exists?(log_collector.local_filename)
+          Appstats.log(:info,"  - #{user}@#{host}:#{log_collector.filename} > #{log_collector.local_filename}")
+          log_collector.status = 'downloaded'
+          count += 1  
+        else
+          Appstats.log(:error, "File #{log_collector.local_filename} did not download.")
+          log_collector.status = 'failed_download'
+        end
+        log_collector.save
+      end
+      Appstats.log(:info,"Downloaded #{count} file(s).")
+      count
+    end
+    
+    def self.process_local_files
+      all = LogCollector.where("status = 'downloaded'").all
+      if all.empty?
+        Appstats.log(:info,"No local logs to process.")
+        return 0
+      end
+      Appstats.log(:info,"About to process #{all.size} file(s).")
+      count = 0
+      total_entries = 0
+      all.each do |log_collector|
+        current_entries = 0
+        begin
+          File.open(log_collector.local_filename,"r").readlines.each do |line|
+            entry = Entry.load_from_logger_entry(line.strip)
+            entry.log_collector = log_collector
+            entry.save
+            current_entries += 1
+            total_entries += 1
+          end
+          Appstats.log(:info,"  - #{current_entries} entr(ies) in #{log_collector.local_filename}.")
+          log_collector.status = "processed"
+          log_collector.save
+          count += 1
+        rescue Exception => e
+          Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#process_local_files")
+          Appstats.log(:error,e.message)
+        end
+      end
+      Appstats.log(:info,"Processed #{count} file(s) with #{total_entries} entr(ies).")
+      count
+    end
+  
+  end
+  
+end

data/lib/appstats/version.rb

@@ -1,3 +1,3 @@
 module Appstats
-  VERSION = "0.0.13"
+  VERSION = "0.0.14"
 end

data/lib/daemons/appstats_log_collector.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+$running = true
+Signal.trap("TERM") do 
+  $running = false
+end
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage #{File.basename(__FILE__)} [options]"
+  options[:config] = "./appstats.config"
+  opts.on( '--config FILE', 'Contains information about the databsae connection and the files to read' ) do |file|
+    options[:config] = file
+  end
+  options[:logfile] = "./appstats.log"
+  opts.on( '--logfile FILE', 'Write log to file' ) do |file|
+    options[:logfile] = file
+  end
+end
+optparse.parse!
+
+require 'logger'
+$logger = Logger.new(options[:logfile])
+unless File.exists?(options[:config])
+  Appstats.log(:info,"Cannot find config file [#{options[:config]}]")
+  exit(1)
+end
+
+appstats_config = YAML::load(File.open(options[:config]))
+ActiveRecord::Base.establish_connection(appstats_config['database'])
+require File.join(File.dirname(__FILE__),"..","appstats")
+
+Appstats.log(:info,"Started Appstats Log Collector")
+while($running) do
+  appstats_config["remote_servers"].each do |remote_server|
+    Appstats::LogCollector.find_remote_files(remote_server,remote_server[:path],remote_server[:template])
+  end
+  Appstats::LogCollector.download_remote_files(appstats_config["remote_servers"])
+  Appstats::LogCollector.process_local_files
+  a_day_in_seconds = 60*60*24
+  sleep a_day_in_seconds
+end

data/lib/daemons/appstats_log_collector_ctl

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require "daemons"
+require 'yaml'
+require 'erb'
+
+gem 'activesupport', '>=3.0.0.beta4'
+gem 'activerecord', '>=3.0.0.beta4'
+require 'active_support'
+require 'active_record'
+
+# For some reason, ActiveSupport 3.0.0 doesn't load.
+# Load needed extension directly for now.
+require "active_support/core_ext/object"
+require "active_support/core_ext/hash"
+
+options = YAML.load(
+  ERB.new(
+    IO.read(
+      File.dirname(__FILE__) + "/../../config/daemons.yml"
+  )).result).with_indifferent_access
+    
+options[:dir_mode] = options[:dir_mode].to_sym
+
+Daemons.run File.dirname(__FILE__) + "/appstats_log_collector.rb", options