appstats 0.0.13 → 0.0.14
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -0
- data/Gemfile.lock +8 -2
- data/appstats.gemspec +2 -1
- data/db/config.yml +1 -1
- data/db/migrations/20110203151136_create_appstats_contexts.rb +16 -0
- data/db/migrations/20110203151635_rework_appstats_entries.rb +21 -0
- data/db/migrations/20110204183259_create_log_collectors.rb +16 -0
- data/db/schema.rb +23 -4
- data/lib/appstats.rb +14 -2
- data/lib/appstats/context.rb +26 -0
- data/lib/appstats/entry.rb +31 -2
- data/lib/appstats/log_collector.rb +124 -0
- data/lib/appstats/version.rb +1 -1
- data/lib/daemons/appstats_log_collector +0 -0
- data/lib/daemons/appstats_log_collector.rb +42 -0
- data/lib/daemons/appstats_log_collector_ctl +25 -0
- data/spec/context_spec.rb +123 -0
- data/spec/entry_spec.rb +162 -17
- data/spec/log_collector_spec.rb +284 -0
- data/spec/logger_spec.rb +16 -17
- metadata +50 -10
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
appstats (0.0.
|
4
|
+
appstats (0.0.14)
|
5
|
+
daemons
|
6
|
+
net-scp
|
5
7
|
rails (>= 2.3.0)
|
6
8
|
|
7
9
|
GEM
|
@@ -37,17 +39,21 @@ GEM
|
|
37
39
|
activesupport (3.0.3)
|
38
40
|
arel (2.0.7)
|
39
41
|
builder (2.1.2)
|
42
|
+
daemons (1.1.0)
|
40
43
|
diff-lcs (1.1.2)
|
41
44
|
erubis (2.6.6)
|
42
45
|
abstract (>= 1.0.0)
|
43
46
|
i18n (0.5.0)
|
44
|
-
mail (2.2.
|
47
|
+
mail (2.2.15)
|
45
48
|
activesupport (>= 2.3.6)
|
46
49
|
i18n (>= 0.4.0)
|
47
50
|
mime-types (~> 1.16)
|
48
51
|
treetop (~> 1.4.8)
|
49
52
|
mime-types (1.16)
|
50
53
|
mysql (2.8.1)
|
54
|
+
net-scp (1.0.4)
|
55
|
+
net-ssh (>= 1.99.1)
|
56
|
+
net-ssh (2.1.0)
|
51
57
|
polyglot (0.3.1)
|
52
58
|
rack (1.2.1)
|
53
59
|
rack-mount (0.6.13)
|
data/appstats.gemspec
CHANGED
@@ -15,7 +15,8 @@ Gem::Specification.new do |s|
|
|
15
15
|
# Models are to be used in Rails 3 environment, but the logger can work with Rails 2 apps
|
16
16
|
# But, for testing appstats itself, you will need Rails 3
|
17
17
|
s.add_dependency('rails','>=2.3.0')
|
18
|
-
|
18
|
+
s.add_dependency('daemons')
|
19
|
+
s.add_dependency('net-scp')
|
19
20
|
|
20
21
|
s.add_development_dependency('rspec')
|
21
22
|
s.add_development_dependency('ZenTest')
|
data/db/config.yml
CHANGED
@@ -0,0 +1,16 @@
|
|
1
|
+
class CreateAppstatsContexts < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :appstats_contexts do |t|
|
4
|
+
t.string :context_key
|
5
|
+
t.string :context_value
|
6
|
+
t.integer :context_int
|
7
|
+
t.float :context_float
|
8
|
+
t.integer :appstats_entry_id
|
9
|
+
t.timestamps
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.down
|
14
|
+
drop_table :appstats_contexts
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
class ReworkAppstatsEntries < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
drop_table :appstats_entries
|
4
|
+
create_table :appstats_entries do |t|
|
5
|
+
t.string :action
|
6
|
+
t.datetime :occurred_at
|
7
|
+
t.text :raw_entry
|
8
|
+
t.timestamps
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.down
|
13
|
+
drop_table :appstats_entries
|
14
|
+
create_table :appstats_entries do |t|
|
15
|
+
t.string :entry_type
|
16
|
+
t.string :name, :null => false
|
17
|
+
t.string :description
|
18
|
+
t.timestamps
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class CreateLogCollectors < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :appstats_log_collectors do |t|
|
4
|
+
t.string :host
|
5
|
+
t.string :filename
|
6
|
+
t.string :status
|
7
|
+
t.timestamps
|
8
|
+
end
|
9
|
+
add_column :appstats_entries, :appstats_log_collector_id, :integer
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.down
|
13
|
+
drop_table :appstats_log_collectors
|
14
|
+
remove_column :appstats_entries, :appstats_log_collector_id
|
15
|
+
end
|
16
|
+
end
|
data/db/schema.rb
CHANGED
@@ -10,12 +10,31 @@
|
|
10
10
|
#
|
11
11
|
# It's strongly recommended to check this file into your version control system.
|
12
12
|
|
13
|
-
ActiveRecord::Schema.define(:version =>
|
13
|
+
ActiveRecord::Schema.define(:version => 20110204183259) do
|
14
|
+
|
15
|
+
create_table "appstats_contexts", :force => true do |t|
|
16
|
+
t.string "context_key"
|
17
|
+
t.string "context_value"
|
18
|
+
t.integer "context_int"
|
19
|
+
t.float "context_float"
|
20
|
+
t.integer "appstats_entry_id"
|
21
|
+
t.datetime "created_at"
|
22
|
+
t.datetime "updated_at"
|
23
|
+
end
|
14
24
|
|
15
25
|
create_table "appstats_entries", :force => true do |t|
|
16
|
-
t.string "
|
17
|
-
t.
|
18
|
-
t.
|
26
|
+
t.string "action"
|
27
|
+
t.datetime "occurred_at"
|
28
|
+
t.text "raw_entry"
|
29
|
+
t.datetime "created_at"
|
30
|
+
t.datetime "updated_at"
|
31
|
+
t.integer "appstats_log_collector_id"
|
32
|
+
end
|
33
|
+
|
34
|
+
create_table "appstats_log_collectors", :force => true do |t|
|
35
|
+
t.string "host"
|
36
|
+
t.string "filename"
|
37
|
+
t.string "status"
|
19
38
|
t.datetime "created_at"
|
20
39
|
t.datetime "updated_at"
|
21
40
|
end
|
data/lib/appstats.rb
CHANGED
@@ -2,10 +2,22 @@ require 'rubygems'
|
|
2
2
|
require 'active_record'
|
3
3
|
require "#{File.dirname(__FILE__)}/appstats/code_injections"
|
4
4
|
require "#{File.dirname(__FILE__)}/appstats/entry"
|
5
|
+
require "#{File.dirname(__FILE__)}/appstats/context"
|
5
6
|
require "#{File.dirname(__FILE__)}/appstats/tasks"
|
6
|
-
require "#{File.dirname(__FILE__)}/appstats/version"
|
7
7
|
require "#{File.dirname(__FILE__)}/appstats/logger"
|
8
|
+
require "#{File.dirname(__FILE__)}/appstats/log_collector"
|
9
|
+
|
10
|
+
# required in the appstats.gemspec
|
11
|
+
# require "#{File.dirname(__FILE__)}/appstats/version"
|
8
12
|
|
9
13
|
module Appstats
|
10
|
-
|
14
|
+
|
15
|
+
def self.log(type,message)
|
16
|
+
if $logger.nil?
|
17
|
+
# puts "LOCAL LOG #{type}: #{message}"
|
18
|
+
else
|
19
|
+
$logger.send(type,message)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
11
23
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
module Appstats
|
3
|
+
class Context < ActiveRecord::Base
|
4
|
+
set_table_name "appstats_contexts"
|
5
|
+
belongs_to :entry, :foreign_key => "appstats_entry_id"
|
6
|
+
attr_accessible :context_key, :context_value
|
7
|
+
|
8
|
+
def context_value=(value)
|
9
|
+
self[:context_value] = value
|
10
|
+
self[:context_int] = nil
|
11
|
+
self[:context_float] = nil
|
12
|
+
return if value.nil?
|
13
|
+
as_int = value.to_i
|
14
|
+
as_float = value.to_f
|
15
|
+
self[:context_int] = as_int if as_int.to_s == value
|
16
|
+
self[:context_float] = as_float if as_float.to_s == value || !self[:context_int].nil?
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
return "No Context" if context_key.nil? || context_key == ''
|
21
|
+
"#{context_key}[]" if context_value.nil?
|
22
|
+
"#{context_key}[#{context_value}]"
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
data/lib/appstats/entry.rb
CHANGED
@@ -3,10 +3,39 @@ module Appstats
|
|
3
3
|
class Entry < ActiveRecord::Base
|
4
4
|
set_table_name "appstats_entries"
|
5
5
|
|
6
|
-
|
6
|
+
has_many :contexts, :table_name => 'appstats_contexts', :foreign_key => 'appstats_entry_id', :order => 'context_key'
|
7
|
+
belongs_to :log_collector, :foreign_key => "appstats_log_collector_id"
|
8
|
+
|
9
|
+
attr_accessible :action, :occurred_at, :raw_entry
|
7
10
|
|
8
11
|
def to_s
|
9
|
-
"Entry
|
12
|
+
return "No Entry" if action.nil? || action == ''
|
13
|
+
return action if occurred_at.nil?
|
14
|
+
"#{action} at #{occurred_at.strftime('%Y-%m-%d %H:%M:%S')}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.load_from_logger_file(filename)
|
18
|
+
return false if filename.nil?
|
19
|
+
return false unless File.exists?(filename)
|
20
|
+
File.open(filename,"r").readlines.each do |line|
|
21
|
+
load_from_logger_entry(line.strip)
|
22
|
+
end
|
23
|
+
true
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.load_from_logger_entry(action_and_contexts)
|
27
|
+
return false if action_and_contexts.nil? || action_and_contexts == ''
|
28
|
+
hash = Logger.entry_to_hash(action_and_contexts)
|
29
|
+
entry = Appstats::Entry.new(:action => hash[:action], :raw_entry => action_and_contexts)
|
30
|
+
entry.occurred_at = Time.parse(hash[:timestamp]) unless hash[:timestamp].nil?
|
31
|
+
hash.each do |key,value|
|
32
|
+
next if key == :action
|
33
|
+
next if key == :timestamp
|
34
|
+
context = Appstats::Context.create(:context_key => key, :context_value => value)
|
35
|
+
entry.contexts<< context
|
36
|
+
end
|
37
|
+
entry.save
|
38
|
+
entry
|
10
39
|
end
|
11
40
|
|
12
41
|
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'net/ssh'
|
2
|
+
require 'net/scp'
|
3
|
+
|
4
|
+
module Appstats
|
5
|
+
class LogCollector < ActiveRecord::Base
|
6
|
+
set_table_name "appstats_log_collectors"
|
7
|
+
|
8
|
+
attr_accessible :host, :filename, :status
|
9
|
+
|
10
|
+
def local_filename
|
11
|
+
File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.find_remote_files(remote_login,path,log_template)
|
15
|
+
begin
|
16
|
+
Appstats.log(:info,"Looking for logs in [#{remote_login[:user]}@#{remote_login[:host]}:#{path}] labelled [#{log_template}]")
|
17
|
+
Net::SSH.start(remote_login[:host], remote_login[:user], :password => remote_login[:password] ) do |ssh|
|
18
|
+
all_files = ssh.exec!("cd #{path} && ls | grep #{log_template}").split
|
19
|
+
load_remote_files(remote_login,path,all_files)
|
20
|
+
end
|
21
|
+
rescue Exception => e
|
22
|
+
Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#find_remote_files")
|
23
|
+
Appstats.log(:error,e.message)
|
24
|
+
0
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.load_remote_files(remote_login,path,all_files)
|
29
|
+
if all_files.empty?
|
30
|
+
Appstats.log(:info,"No remote logs to load.")
|
31
|
+
return 0
|
32
|
+
end
|
33
|
+
|
34
|
+
count = 0
|
35
|
+
Appstats.log(:info, "About to analyze #{all_files.size} file(s).")
|
36
|
+
all_files.each do |log_name|
|
37
|
+
filename = File.join(path,log_name)
|
38
|
+
if LogCollector.find_by_host_and_filename(remote_login[:host],filename).nil?
|
39
|
+
log_collector = LogCollector.create(:host => remote_login[:host], :filename => filename, :status => "unprocessed")
|
40
|
+
Appstats.log(:info, " - #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
|
41
|
+
count += 1
|
42
|
+
else
|
43
|
+
Appstats.log(:info, " - ALREADY LOADED #{remote_login[:user]}@#{remote_login[:host]}:#{filename}")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
Appstats.log(:info, "Loaded #{count} file(s).")
|
47
|
+
count
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.download_remote_files(raw_logins)
|
51
|
+
all = LogCollector.where("status = 'unprocessed'").all
|
52
|
+
if all.empty?
|
53
|
+
Appstats.log(:info,"No remote logs to download.")
|
54
|
+
return 0
|
55
|
+
end
|
56
|
+
|
57
|
+
normalized_logins = {}
|
58
|
+
raw_logins.each do |login|
|
59
|
+
normalized_logins[login[:host]] = login
|
60
|
+
end
|
61
|
+
count = 0
|
62
|
+
|
63
|
+
Appstats.log(:info,"About to download #{all.size} file(s).")
|
64
|
+
all.each do |log_collector|
|
65
|
+
host = log_collector.host
|
66
|
+
user = normalized_logins[host][:user]
|
67
|
+
password = normalized_logins[host][:password]
|
68
|
+
begin
|
69
|
+
Net::SCP.start( host, user, :password => password ) do |scp|
|
70
|
+
scp.download!( log_collector.filename, log_collector.local_filename )
|
71
|
+
end
|
72
|
+
rescue Exception => e
|
73
|
+
Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#download_remote_files")
|
74
|
+
Appstats.log(:error,e.message)
|
75
|
+
end
|
76
|
+
if File.exists?(log_collector.local_filename)
|
77
|
+
Appstats.log(:info," - #{user}@#{host}:#{log_collector.filename} > #{log_collector.local_filename}")
|
78
|
+
log_collector.status = 'downloaded'
|
79
|
+
count += 1
|
80
|
+
else
|
81
|
+
Appstats.log(:error, "File #{log_collector.local_filename} did not download.")
|
82
|
+
log_collector.status = 'failed_download'
|
83
|
+
end
|
84
|
+
log_collector.save
|
85
|
+
end
|
86
|
+
Appstats.log(:info,"Downloaded #{count} file(s).")
|
87
|
+
count
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.process_local_files
|
91
|
+
all = LogCollector.where("status = 'downloaded'").all
|
92
|
+
if all.empty?
|
93
|
+
Appstats.log(:info,"No local logs to process.")
|
94
|
+
return 0
|
95
|
+
end
|
96
|
+
Appstats.log(:info,"About to process #{all.size} file(s).")
|
97
|
+
count = 0
|
98
|
+
total_entries = 0
|
99
|
+
all.each do |log_collector|
|
100
|
+
current_entries = 0
|
101
|
+
begin
|
102
|
+
File.open(log_collector.local_filename,"r").readlines.each do |line|
|
103
|
+
entry = Entry.load_from_logger_entry(line.strip)
|
104
|
+
entry.log_collector = log_collector
|
105
|
+
entry.save
|
106
|
+
current_entries += 1
|
107
|
+
total_entries += 1
|
108
|
+
end
|
109
|
+
Appstats.log(:info," - #{current_entries} entr(ies) in #{log_collector.local_filename}.")
|
110
|
+
log_collector.status = "processed"
|
111
|
+
log_collector.save
|
112
|
+
count += 1
|
113
|
+
rescue Exception => e
|
114
|
+
Appstats.log(:error,"Something bad occurred during Appstats::LogCollector#process_local_files")
|
115
|
+
Appstats.log(:error,e.message)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
Appstats.log(:info,"Processed #{count} file(s) with #{total_entries} entr(ies).")
|
119
|
+
count
|
120
|
+
end
|
121
|
+
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
data/lib/appstats/version.rb
CHANGED
File without changes
|
@@ -0,0 +1,42 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$running = true
|
4
|
+
Signal.trap("TERM") do
|
5
|
+
$running = false
|
6
|
+
end
|
7
|
+
|
8
|
+
options = {}
|
9
|
+
optparse = OptionParser.new do |opts|
|
10
|
+
opts.banner = "Usage #{File.basename(__FILE__)} [options]"
|
11
|
+
options[:config] = "./appstats.config"
|
12
|
+
opts.on( '--config FILE', 'Contains information about the databsae connection and the files to read' ) do |file|
|
13
|
+
options[:config] = file
|
14
|
+
end
|
15
|
+
options[:logfile] = "./appstats.log"
|
16
|
+
opts.on( '--logfile FILE', 'Write log to file' ) do |file|
|
17
|
+
options[:logfile] = file
|
18
|
+
end
|
19
|
+
end
|
20
|
+
optparse.parse!
|
21
|
+
|
22
|
+
require 'logger'
|
23
|
+
$logger = Logger.new(options[:logfile])
|
24
|
+
unless File.exists?(options[:config])
|
25
|
+
Appstats.log(:info,"Cannot find config file [#{options[:config]}]")
|
26
|
+
exit(1)
|
27
|
+
end
|
28
|
+
|
29
|
+
appstats_config = YAML::load(File.open(options[:config]))
|
30
|
+
ActiveRecord::Base.establish_connection(appstats_config['database'])
|
31
|
+
require File.join(File.dirname(__FILE__),"..","appstats")
|
32
|
+
|
33
|
+
Appstats.log(:info,"Started Appstats Log Collector")
|
34
|
+
while($running) do
|
35
|
+
appstats_config["remote_servers"].each do |remote_server|
|
36
|
+
Appstats::LogCollector.find_remote_files(remote_server,remote_server[:path],remote_server[:template])
|
37
|
+
end
|
38
|
+
Appstats::LogCollector.download_remote_files(appstats_config["remote_servers"])
|
39
|
+
Appstats::LogCollector.process_local_files
|
40
|
+
a_day_in_seconds = 60*60*24
|
41
|
+
sleep a_day_in_seconds
|
42
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require 'rubygems'
|
3
|
+
require "daemons"
|
4
|
+
require 'yaml'
|
5
|
+
require 'erb'
|
6
|
+
|
7
|
+
gem 'activesupport', '>=3.0.0.beta4'
|
8
|
+
gem 'activerecord', '>=3.0.0.beta4'
|
9
|
+
require 'active_support'
|
10
|
+
require 'active_record'
|
11
|
+
|
12
|
+
# For some reason, ActiveSupport 3.0.0 doesn't load.
|
13
|
+
# Load needed extension directly for now.
|
14
|
+
require "active_support/core_ext/object"
|
15
|
+
require "active_support/core_ext/hash"
|
16
|
+
|
17
|
+
options = YAML.load(
|
18
|
+
ERB.new(
|
19
|
+
IO.read(
|
20
|
+
File.dirname(__FILE__) + "/../../config/daemons.yml"
|
21
|
+
)).result).with_indifferent_access
|
22
|
+
|
23
|
+
options[:dir_mode] = options[:dir_mode].to_sym
|
24
|
+
|
25
|
+
Daemons.run File.dirname(__FILE__) + "/appstats_log_collector.rb", options
|