appstats 0.0.13 → 0.0.14

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore CHANGED
@@ -4,3 +4,5 @@ pkg/*
4
4
  db/migrate
5
5
  doc
6
6
  config
7
+ config/appstats.yml
8
+ log
@@ -1,7 +1,9 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- appstats (0.0.13)
4
+ appstats (0.0.14)
5
+ daemons
6
+ net-scp
5
7
  rails (>= 2.3.0)
6
8
 
7
9
  GEM
@@ -37,17 +39,21 @@ GEM
37
39
  activesupport (3.0.3)
38
40
  arel (2.0.7)
39
41
  builder (2.1.2)
42
+ daemons (1.1.0)
40
43
  diff-lcs (1.1.2)
41
44
  erubis (2.6.6)
42
45
  abstract (>= 1.0.0)
43
46
  i18n (0.5.0)
44
- mail (2.2.14)
47
+ mail (2.2.15)
45
48
  activesupport (>= 2.3.6)
46
49
  i18n (>= 0.4.0)
47
50
  mime-types (~> 1.16)
48
51
  treetop (~> 1.4.8)
49
52
  mime-types (1.16)
50
53
  mysql (2.8.1)
54
+ net-scp (1.0.4)
55
+ net-ssh (>= 1.99.1)
56
+ net-ssh (2.1.0)
51
57
  polyglot (0.3.1)
52
58
  rack (1.2.1)
53
59
  rack-mount (0.6.13)
@@ -15,7 +15,8 @@ Gem::Specification.new do |s|
15
15
  # Models are to be used in Rails 3 environment, but the logger can work with Rails 2 apps
16
16
  # But, for testing appstats itself, you will need Rails 3
17
17
  s.add_dependency('rails','>=2.3.0')
18
- # s.add_dependency('rails','3.0.3')
18
+ s.add_dependency('daemons')
19
+ s.add_dependency('net-scp')
19
20
 
20
21
  s.add_development_dependency('rspec')
21
22
  s.add_development_dependency('ZenTest')
@@ -14,4 +14,4 @@ test:
14
14
  username: root
15
15
  password:
16
16
  host: localhost
17
- encoding: utf8
17
+ encoding: utf8
@@ -0,0 +1,16 @@
1
+ class CreateAppstatsContexts < ActiveRecord::Migration
2
+ def self.up
3
+ create_table :appstats_contexts do |t|
4
+ t.string :context_key
5
+ t.string :context_value
6
+ t.integer :context_int
7
+ t.float :context_float
8
+ t.integer :appstats_entry_id
9
+ t.timestamps
10
+ end
11
+ end
12
+
13
+ def self.down
14
+ drop_table :appstats_contexts
15
+ end
16
+ end
@@ -0,0 +1,21 @@
1
+ class ReworkAppstatsEntries < ActiveRecord::Migration
2
+ def self.up
3
+ drop_table :appstats_entries
4
+ create_table :appstats_entries do |t|
5
+ t.string :action
6
+ t.datetime :occurred_at
7
+ t.text :raw_entry
8
+ t.timestamps
9
+ end
10
+ end
11
+
12
+ def self.down
13
+ drop_table :appstats_entries
14
+ create_table :appstats_entries do |t|
15
+ t.string :entry_type
16
+ t.string :name, :null => false
17
+ t.string :description
18
+ t.timestamps
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,16 @@
1
+ class CreateLogCollectors < ActiveRecord::Migration
2
+ def self.up
3
+ create_table :appstats_log_collectors do |t|
4
+ t.string :host
5
+ t.string :filename
6
+ t.string :status
7
+ t.timestamps
8
+ end
9
+ add_column :appstats_entries, :appstats_log_collector_id, :integer
10
+ end
11
+
12
+ def self.down
13
+ drop_table :appstats_log_collectors
14
+ remove_column :appstats_entries, :appstats_log_collector_id
15
+ end
16
+ end
@@ -10,12 +10,31 @@
10
10
  #
11
11
  # It's strongly recommended to check this file into your version control system.
12
12
 
13
- ActiveRecord::Schema.define(:version => 20110105221941) do
13
+ ActiveRecord::Schema.define(:version => 20110204183259) do
14
+
15
+ create_table "appstats_contexts", :force => true do |t|
16
+ t.string "context_key"
17
+ t.string "context_value"
18
+ t.integer "context_int"
19
+ t.float "context_float"
20
+ t.integer "appstats_entry_id"
21
+ t.datetime "created_at"
22
+ t.datetime "updated_at"
23
+ end
14
24
 
15
25
  create_table "appstats_entries", :force => true do |t|
16
- t.string "entry_type"
17
- t.string "name", :null => false
18
- t.string "description"
26
+ t.string "action"
27
+ t.datetime "occurred_at"
28
+ t.text "raw_entry"
29
+ t.datetime "created_at"
30
+ t.datetime "updated_at"
31
+ t.integer "appstats_log_collector_id"
32
+ end
33
+
34
+ create_table "appstats_log_collectors", :force => true do |t|
35
+ t.string "host"
36
+ t.string "filename"
37
+ t.string "status"
19
38
  t.datetime "created_at"
20
39
  t.datetime "updated_at"
21
40
  end
@@ -2,10 +2,22 @@ require 'rubygems'
2
2
  require 'active_record'
3
3
  require "#{File.dirname(__FILE__)}/appstats/code_injections"
4
4
  require "#{File.dirname(__FILE__)}/appstats/entry"
5
+ require "#{File.dirname(__FILE__)}/appstats/context"
5
6
  require "#{File.dirname(__FILE__)}/appstats/tasks"
6
- require "#{File.dirname(__FILE__)}/appstats/version"
7
7
  require "#{File.dirname(__FILE__)}/appstats/logger"
8
+ require "#{File.dirname(__FILE__)}/appstats/log_collector"
9
+
10
+ # required in the appstats.gemspec
11
+ # require "#{File.dirname(__FILE__)}/appstats/version"
8
12
 
9
13
  module Appstats
10
- # Your code goes here...
14
+
15
+ def self.log(type,message)
16
+ if $logger.nil?
17
+ # puts "LOCAL LOG #{type}: #{message}"
18
+ else
19
+ $logger.send(type,message)
20
+ end
21
+ end
22
+
11
23
  end
@@ -0,0 +1,26 @@
1
+
2
+ module Appstats
3
+ class Context < ActiveRecord::Base
4
+ set_table_name "appstats_contexts"
5
+ belongs_to :entry, :foreign_key => "appstats_entry_id"
6
+ attr_accessible :context_key, :context_value
7
+
8
+ def context_value=(value)
9
+ self[:context_value] = value
10
+ self[:context_int] = nil
11
+ self[:context_float] = nil
12
+ return if value.nil?
13
+ as_int = value.to_i
14
+ as_float = value.to_f
15
+ self[:context_int] = as_int if as_int.to_s == value
16
+ self[:context_float] = as_float if as_float.to_s == value || !self[:context_int].nil?
17
+ end
18
+
19
+ def to_s
20
+ return "No Context" if context_key.nil? || context_key == ''
21
+ "#{context_key}[]" if context_value.nil?
22
+ "#{context_key}[#{context_value}]"
23
+ end
24
+
25
+ end
26
+ end
@@ -3,10 +3,39 @@ module Appstats
3
3
  class Entry < ActiveRecord::Base
4
4
  set_table_name "appstats_entries"
5
5
 
6
- attr_accessible :entry_type, :name, :description
6
+ has_many :contexts, :table_name => 'appstats_contexts', :foreign_key => 'appstats_entry_id', :order => 'context_key'
7
+ belongs_to :log_collector, :foreign_key => "appstats_log_collector_id"
8
+
9
+ attr_accessible :action, :occurred_at, :raw_entry
7
10
 
8
11
  def to_s
9
- "Entry [type],[name],[description]"
12
+ return "No Entry" if action.nil? || action == ''
13
+ return action if occurred_at.nil?
14
+ "#{action} at #{occurred_at.strftime('%Y-%m-%d %H:%M:%S')}"
15
+ end
16
+
17
+ def self.load_from_logger_file(filename)
18
+ return false if filename.nil?
19
+ return false unless File.exists?(filename)
20
+ File.open(filename,"r").readlines.each do |line|
21
+ load_from_logger_entry(line.strip)
22
+ end
23
+ true
24
+ end
25
+
26
+ def self.load_from_logger_entry(action_and_contexts)
27
+ return false if action_and_contexts.nil? || action_and_contexts == ''
28
+ hash = Logger.entry_to_hash(action_and_contexts)
29
+ entry = Appstats::Entry.new(:action => hash[:action], :raw_entry => action_and_contexts)
30
+ entry.occurred_at = Time.parse(hash[:timestamp]) unless hash[:timestamp].nil?
31
+ hash.each do |key,value|
32
+ next if key == :action
33
+ next if key == :timestamp
34
+ context = Appstats::Context.create(:context_key => key, :context_value => value)
35
+ entry.contexts<< context
36
+ end
37
+ entry.save
38
+ entry
10
39
  end
11
40
 
12
41
  end
@@ -0,0 +1,124 @@
1
+ require 'net/ssh'
2
+ require 'net/scp'
3
+
4
+ module Appstats
5
+ class LogCollector < ActiveRecord::Base
6
+ set_table_name "appstats_log_collectors"
7
+
8
+ attr_accessible :host, :filename, :status
9
+
10
+ def local_filename
11
+ File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
12
+ end
13
+
14
+ def self.find_remote_files(remote_login,path,log_template)
15
+ begin
16
+ Appstats.log(:info,"Looking for logs in [#{remote_login[:user]}@#{remote_login[:host]}:#{path}] labelled [#{log_template}]")
17
+ Net::SSH.start(remote_login[:host], remote_login[:user], :password => remote_login[:password] ) do |ssh|
18
+ all_files = ssh.exec!("cd #{path} && ls | grep #{log_template}").split
19
+ load_remote_files(remote_login,path,all_files)
20
+ end
21
+ rescue Exception => e
22
+ Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#find_remote_files")
23
+ Appstats.log(:error,e.message)
24
+ 0
25
+ end
26
+ end
27
+
28
+ def self.load_remote_files(remote_login,path,all_files)
29
+ if all_files.empty?
30
+ Appstats.log(:info,"No remote logs to load.")
31
+ return 0
32
+ end
33
+
34
+ count = 0
35
+ Appstats.log(:info, "About to analyze #{all_files.size} file(s).")
36
+ all_files.each do |log_name|
37
+ filename = File.join(path,log_name)
38
+ if LogCollector.find_by_host_and_filename(remote_login[:host],filename).nil?
39
+ log_collector = LogCollector.create(:host => remote_login[:host], :filename => filename, :status => "unprocessed")
40
+ Appstats.log(:info, " - #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
41
+ count += 1
42
+ else
43
+ Appstats.log(:info, " - ALREADY LOADED #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
44
+ end
45
+ end
46
+ Appstats.log(:info, "Loaded #{count} file(s).")
47
+ count
48
+ end
49
+
50
+ def self.download_remote_files(raw_logins)
51
+ all = LogCollector.where("status = 'unprocessed'").all
52
+ if all.empty?
53
+ Appstats.log(:info,"No remote logs to download.")
54
+ return 0
55
+ end
56
+
57
+ normalized_logins = {}
58
+ raw_logins.each do |login|
59
+ normalized_logins[login[:host]] = login
60
+ end
61
+ count = 0
62
+
63
+ Appstats.log(:info,"About to download #{all.size} file(s).")
64
+ all.each do |log_collector|
65
+ host = log_collector.host
66
+ user = normalized_logins[host][:user]
67
+ password = normalized_logins[host][:password]
68
+ begin
69
+ Net::SCP.start( host, user, :password => password ) do |scp|
70
+ scp.download!( log_collector.filename, log_collector.local_filename )
71
+ end
72
+ rescue Exception => e
73
+ Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#download_remote_files")
74
+ Appstats.log(:error,e.message)
75
+ end
76
+ if File.exists?(log_collector.local_filename)
77
+ Appstats.log(:info," - #{user}@#{host}:#{log_collector.filename} > #{log_collector.local_filename}")
78
+ log_collector.status = 'downloaded'
79
+ count += 1
80
+ else
81
+ Appstats.log(:error, "File #{log_collector.local_filename} did not download.")
82
+ log_collector.status = 'failed_download'
83
+ end
84
+ log_collector.save
85
+ end
86
+ Appstats.log(:info,"Downloaded #{count} file(s).")
87
+ count
88
+ end
89
+
90
+ def self.process_local_files
91
+ all = LogCollector.where("status = 'downloaded'").all
92
+ if all.empty?
93
+ Appstats.log(:info,"No local logs to process.")
94
+ return 0
95
+ end
96
+ Appstats.log(:info,"About to process #{all.size} file(s).")
97
+ count = 0
98
+ total_entries = 0
99
+ all.each do |log_collector|
100
+ current_entries = 0
101
+ begin
102
+ File.open(log_collector.local_filename,"r").readlines.each do |line|
103
+ entry = Entry.load_from_logger_entry(line.strip)
104
+ entry.log_collector = log_collector
105
+ entry.save
106
+ current_entries += 1
107
+ total_entries += 1
108
+ end
109
+ Appstats.log(:info," - #{current_entries} entr(ies) in #{log_collector.local_filename}.")
110
+ log_collector.status = "processed"
111
+ log_collector.save
112
+ count += 1
113
+ rescue Exception => e
114
+ Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#process_local_files")
115
+ Appstats.log(:error,e.message)
116
+ end
117
+ end
118
+ Appstats.log(:info,"Processed #{count} file(s) with #{total_entries} entr(ies).")
119
+ count
120
+ end
121
+
122
+ end
123
+
124
+ end
@@ -1,3 +1,3 @@
1
1
  module Appstats
2
- VERSION = "0.0.13"
2
+ VERSION = "0.0.14"
3
3
  end
File without changes
@@ -0,0 +1,42 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $running = true
4
+ Signal.trap("TERM") do
5
+ $running = false
6
+ end
7
+
8
+ options = {}
9
+ optparse = OptionParser.new do |opts|
10
+ opts.banner = "Usage #{File.basename(__FILE__)} [options]"
11
+ options[:config] = "./appstats.config"
12
+ opts.on( '--config FILE', 'Contains information about the databsae connection and the files to read' ) do |file|
13
+ options[:config] = file
14
+ end
15
+ options[:logfile] = "./appstats.log"
16
+ opts.on( '--logfile FILE', 'Write log to file' ) do |file|
17
+ options[:logfile] = file
18
+ end
19
+ end
20
+ optparse.parse!
21
+
22
+ require 'logger'
23
+ $logger = Logger.new(options[:logfile])
24
+ unless File.exists?(options[:config])
25
+ Appstats.log(:info,"Cannot find config file [#{options[:config]}]")
26
+ exit(1)
27
+ end
28
+
29
+ appstats_config = YAML::load(File.open(options[:config]))
30
+ ActiveRecord::Base.establish_connection(appstats_config['database'])
31
+ require File.join(File.dirname(__FILE__),"..","appstats")
32
+
33
+ Appstats.log(:info,"Started Appstats Log Collector")
34
+ while($running) do
35
+ appstats_config["remote_servers"].each do |remote_server|
36
+ Appstats::LogCollector.find_remote_files(remote_server,remote_server[:path],remote_server[:template])
37
+ end
38
+ Appstats::LogCollector.download_remote_files(appstats_config["remote_servers"])
39
+ Appstats::LogCollector.process_local_files
40
+ a_day_in_seconds = 60*60*24
41
+ sleep a_day_in_seconds
42
+ end
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env ruby
2
+ require 'rubygems'
3
+ require "daemons"
4
+ require 'yaml'
5
+ require 'erb'
6
+
7
+ gem 'activesupport', '>=3.0.0.beta4'
8
+ gem 'activerecord', '>=3.0.0.beta4'
9
+ require 'active_support'
10
+ require 'active_record'
11
+
12
+ # For some reason, ActiveSupport 3.0.0 doesn't load.
13
+ # Load needed extension directly for now.
14
+ require "active_support/core_ext/object"
15
+ require "active_support/core_ext/hash"
16
+
17
+ options = YAML.load(
18
+ ERB.new(
19
+ IO.read(
20
+ File.dirname(__FILE__) + "/../../config/daemons.yml"
21
+ )).result).with_indifferent_access
22
+
23
+ options[:dir_mode] = options[:dir_mode].to_sym
24
+
25
+ Daemons.run File.dirname(__FILE__) + "/appstats_log_collector.rb", options