appstats 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +1 -1
- data/db/migrations/20110215155830_create_appstats_hosts.rb +13 -0
- data/db/schema.rb +8 -1
- data/lib/appstats.rb +2 -0
- data/lib/appstats/host.rb +18 -0
- data/lib/appstats/parser.rb +239 -0
- data/lib/appstats/query.rb +94 -35
- data/lib/appstats/version.rb +1 -1
- data/spec/entry_spec.rb +4 -4
- data/spec/host_spec.rb +51 -0
- data/spec/logger_spec.rb +18 -18
- data/spec/parser_spec.rb +329 -0
- data/spec/query_spec.rb +172 -23
- metadata +11 -4
data/Gemfile.lock
CHANGED
data/db/schema.rb
CHANGED
@@ -10,7 +10,7 @@
|
|
10
10
|
#
|
11
11
|
# It's strongly recommended to check this file into your version control system.
|
12
12
|
|
13
|
-
ActiveRecord::Schema.define(:version =>
|
13
|
+
ActiveRecord::Schema.define(:version => 20110215155830) do
|
14
14
|
|
15
15
|
create_table "appstats_actions", :force => true do |t|
|
16
16
|
t.string "name"
|
@@ -56,6 +56,13 @@ ActiveRecord::Schema.define(:version => 20110210225606) do
|
|
56
56
|
add_index "appstats_entries", ["year", "month"], :name => "index_entries_by_month"
|
57
57
|
add_index "appstats_entries", ["year"], :name => "index_entries_by_year"
|
58
58
|
|
59
|
+
create_table "appstats_hosts", :force => true do |t|
|
60
|
+
t.string "name"
|
61
|
+
t.string "status"
|
62
|
+
t.datetime "created_at"
|
63
|
+
t.datetime "updated_at"
|
64
|
+
end
|
65
|
+
|
59
66
|
create_table "appstats_log_collectors", :force => true do |t|
|
60
67
|
t.string "host"
|
61
68
|
t.string "filename"
|
data/lib/appstats.rb
CHANGED
@@ -12,6 +12,8 @@ require "#{File.dirname(__FILE__)}/appstats/logger"
|
|
12
12
|
require "#{File.dirname(__FILE__)}/appstats/log_collector"
|
13
13
|
require "#{File.dirname(__FILE__)}/appstats/query"
|
14
14
|
require "#{File.dirname(__FILE__)}/appstats/result"
|
15
|
+
require "#{File.dirname(__FILE__)}/appstats/host"
|
16
|
+
require "#{File.dirname(__FILE__)}/appstats/parser"
|
15
17
|
require "#{File.dirname(__FILE__)}/appstats/test_object"
|
16
18
|
|
17
19
|
# required in the appstats.gemspec
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Appstats
|
2
|
+
class Host < ActiveRecord::Base
|
3
|
+
set_table_name "appstats_hosts"
|
4
|
+
|
5
|
+
attr_accessible :name, :status
|
6
|
+
|
7
|
+
def self.update_hosts
|
8
|
+
sql = "select distinct(host) from appstats_log_collectors where host not in (select name from appstats_hosts)"
|
9
|
+
count = 0
|
10
|
+
ActiveRecord::Base.connection.execute(sql).each do |row|
|
11
|
+
Appstats::Host.create(:name => row[0], :status => 'derived')
|
12
|
+
count += 1
|
13
|
+
end
|
14
|
+
count
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,239 @@
|
|
1
|
+
|
2
|
+
module Appstats
|
3
|
+
class Parser
|
4
|
+
|
5
|
+
attr_reader :raw_rules, :rules, :repeating, :raw_tokenize, :tokenize, :tokenize_no_spaces, :tokenize_regex, :tokenize_regex_no_spaces, :results, :raw_results, :constants
|
6
|
+
|
7
|
+
def initialize(data = {})
|
8
|
+
@raw_rules = data[:rules]
|
9
|
+
@raw_tokenize = data[:tokenize]
|
10
|
+
@repeating = data[:repeating] == true
|
11
|
+
@results = {}
|
12
|
+
@raw_results = []
|
13
|
+
update_tokens
|
14
|
+
update_rules
|
15
|
+
end
|
16
|
+
|
17
|
+
def parse(input)
|
18
|
+
@results = {}
|
19
|
+
@raw_results = []
|
20
|
+
return false if input.nil?
|
21
|
+
return false if @rules.size == 0
|
22
|
+
|
23
|
+
@rule_index = 0
|
24
|
+
@max_rule_index = @rules.size - 1
|
25
|
+
@previous_text_so_far = input.strip
|
26
|
+
@text_so_far = @previous_text_so_far
|
27
|
+
@remaining_constants = @constants.dup
|
28
|
+
|
29
|
+
while !@text_so_far.blank?
|
30
|
+
process_constant_if_present
|
31
|
+
break if @rule_index > @max_rule_index && !@repeating
|
32
|
+
@rule_index = 0 if @rule_index > @max_rule_index
|
33
|
+
|
34
|
+
rule = @rules[@rule_index]
|
35
|
+
@rule_index += 1
|
36
|
+
|
37
|
+
if rule.kind_of?(Hash)
|
38
|
+
if rule[:stop] == :constant
|
39
|
+
was_found = false
|
40
|
+
@remaining_constants.each_with_index do |k,index|
|
41
|
+
p = parse_word(@text_so_far,k,true)
|
42
|
+
if p[0].nil?
|
43
|
+
unset_rules_until(k)
|
44
|
+
else
|
45
|
+
(index-1).downto(0) do |i|
|
46
|
+
@remaining_constants.delete_at(i)
|
47
|
+
end
|
48
|
+
add_results(rule[:rule],p[0])
|
49
|
+
@text_so_far = p[1]
|
50
|
+
was_found = true
|
51
|
+
break
|
52
|
+
end
|
53
|
+
end
|
54
|
+
unless was_found
|
55
|
+
add_results(rule[:rule],@text_so_far)
|
56
|
+
@text_so_far = nil
|
57
|
+
end
|
58
|
+
else
|
59
|
+
p = parse_word(@text_so_far,rule[:stop],false)
|
60
|
+
add_results(rule[:rule],p[0])
|
61
|
+
@text_so_far = p[1]
|
62
|
+
end
|
63
|
+
end
|
64
|
+
break if @previous_text_so_far == @text_so_far
|
65
|
+
@previous_text_so_far = @text_so_far
|
66
|
+
end
|
67
|
+
remove_tokens_at_start(@text_so_far)
|
68
|
+
unset_rules_until(nil)
|
69
|
+
true
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.parse_constant(current_text,constant)
|
73
|
+
answer = [nil,nil]
|
74
|
+
return answer if current_text.blank? || constant.nil?
|
75
|
+
current_text.strip!
|
76
|
+
m = current_text.match(/^(#{constant})(.*)$/im)
|
77
|
+
answer[0] = m[1] unless m.nil?
|
78
|
+
answer[1] = m.nil? ? current_text : m[2]
|
79
|
+
clean_parsed_word(answer)
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.merge_regex_filter(a,b)
|
83
|
+
return "" if a.blank? && b.blank?
|
84
|
+
return "(#{a})" if b.blank?
|
85
|
+
return "(#{b})" if a.blank?
|
86
|
+
"(#{a}|#{b})"
|
87
|
+
end
|
88
|
+
|
89
|
+
def parse_word(current_text,stop_on,strict = false)
|
90
|
+
answer = [nil,nil]
|
91
|
+
return answer if current_text.blank? || stop_on.nil?
|
92
|
+
current_text.strip!
|
93
|
+
|
94
|
+
current_text = remove_tokens_at_start(current_text)
|
95
|
+
|
96
|
+
if stop_on == :end
|
97
|
+
filter = Parser.merge_regex_filter(nil,@tokenize_regex)
|
98
|
+
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
99
|
+
if m.nil? || m[1].blank?
|
100
|
+
answer[0] = current_text
|
101
|
+
else
|
102
|
+
answer[0] = m[1]
|
103
|
+
answer[1] = m[2]
|
104
|
+
end
|
105
|
+
elsif stop_on == :space
|
106
|
+
filter = Parser.merge_regex_filter('\s',@tokenize_regex)
|
107
|
+
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
108
|
+
if m.nil?
|
109
|
+
answer[0] = current_text
|
110
|
+
else
|
111
|
+
answer[0] = m[1]
|
112
|
+
answer[1] = m[2]
|
113
|
+
end
|
114
|
+
else
|
115
|
+
filter = Parser.merge_regex_filter(stop_on,@tokenize_regex)
|
116
|
+
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
117
|
+
if strict
|
118
|
+
answer[0] = m[1] unless m.nil?
|
119
|
+
answer[1] = m.nil? ? current_text : m[2]
|
120
|
+
else
|
121
|
+
answer[0] = m.nil? ? current_text : m[1]
|
122
|
+
answer[1] = m[2] unless m.nil?
|
123
|
+
end
|
124
|
+
end
|
125
|
+
Parser.clean_parsed_word(answer)
|
126
|
+
end
|
127
|
+
|
128
|
+
private
|
129
|
+
|
130
|
+
def self.clean_parsed_word(answer)
|
131
|
+
answer[0].strip! unless answer[0].nil?
|
132
|
+
answer[1].strip! unless answer[1].nil?
|
133
|
+
answer[0] = nil if answer[0].blank?
|
134
|
+
answer[1] = nil if answer[1].blank?
|
135
|
+
answer
|
136
|
+
end
|
137
|
+
|
138
|
+
def process_constant_if_present
|
139
|
+
while process_tokens_if_present; end
|
140
|
+
to_delete = nil
|
141
|
+
@remaining_constants.each do |k|
|
142
|
+
p = Parser.parse_constant(@text_so_far,k)
|
143
|
+
next if p[0].nil?
|
144
|
+
to_delete = k
|
145
|
+
unset_rules_until(k)
|
146
|
+
add_constant(p[0])
|
147
|
+
@text_so_far = p[1]
|
148
|
+
end
|
149
|
+
@remaining_constants.delete(to_delete) unless to_delete.nil?
|
150
|
+
end
|
151
|
+
|
152
|
+
def process_tokens_if_present
|
153
|
+
found = false
|
154
|
+
@tokenize.each do |k|
|
155
|
+
p = Parser.parse_constant(@text_so_far,k)
|
156
|
+
next if p[0].nil?
|
157
|
+
add_constant(p[0])
|
158
|
+
@text_so_far = p[1]
|
159
|
+
found = true
|
160
|
+
end
|
161
|
+
found
|
162
|
+
end
|
163
|
+
|
164
|
+
def unset_rules_until(k)
|
165
|
+
@rules[@rule_index..-1].each do |rule|
|
166
|
+
@rule_index += 1
|
167
|
+
break if rule.eql?(k)
|
168
|
+
add_results(rule[:rule],nil) if rule.kind_of?(Hash)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def update_tokens
|
173
|
+
@tokenize = []
|
174
|
+
@tokenize_no_spaces = []
|
175
|
+
@tokenize_regex = nil
|
176
|
+
@tokenize_regex_no_spaces = nil
|
177
|
+
return if @raw_tokenize.blank?
|
178
|
+
@raw_tokenize.split(" ").each do |token|
|
179
|
+
current_token = token.upcase
|
180
|
+
current_token.gsub!("(",'\(')
|
181
|
+
current_token.gsub!(")",'\)')
|
182
|
+
current_token.gsub!("|",'\|')
|
183
|
+
@tokenize_no_spaces<< current_token
|
184
|
+
current_token = "\\s+#{current_token}" unless current_token.match(/.*[a-z].*/i).nil?
|
185
|
+
@tokenize<< current_token
|
186
|
+
end
|
187
|
+
@tokenize_regex_no_spaces = @tokenize_no_spaces.join("|")
|
188
|
+
@tokenize_regex = @tokenize.join("|")
|
189
|
+
end
|
190
|
+
|
191
|
+
def update_rules
|
192
|
+
@rules = []
|
193
|
+
@constants = []
|
194
|
+
current_rule = nil
|
195
|
+
return if @raw_rules.blank?
|
196
|
+
@raw_rules.split(" ").each do |rule|
|
197
|
+
|
198
|
+
if rule.starts_with?(":") && rule.size > 1
|
199
|
+
current_rule = { :rule => rule[1..-1].to_sym, :stop => :end }
|
200
|
+
previous_stop_on = :space
|
201
|
+
else
|
202
|
+
current_rule = rule.upcase
|
203
|
+
@constants<< current_rule
|
204
|
+
previous_stop_on = :constant
|
205
|
+
end
|
206
|
+
|
207
|
+
if @rules.last.kind_of?(Hash)
|
208
|
+
@rules.last[:stop] = previous_stop_on
|
209
|
+
end
|
210
|
+
|
211
|
+
@rules<< current_rule
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
def add_constant(value)
|
216
|
+
@raw_results<< value
|
217
|
+
end
|
218
|
+
|
219
|
+
def add_results(rule_name,value)
|
220
|
+
@raw_results<< { rule_name => value }
|
221
|
+
@results[rule_name] = value
|
222
|
+
end
|
223
|
+
|
224
|
+
def remove_tokens_at_start(current_text)
|
225
|
+
return current_text if current_text.blank?
|
226
|
+
current_text.blank?
|
227
|
+
loop do
|
228
|
+
break if @tokenize_regex.blank?
|
229
|
+
m = current_text.match(/^(#{@tokenize_regex_no_spaces})(.*)$/im)
|
230
|
+
break if m.nil? || m[1].blank?
|
231
|
+
add_constant(m[1])
|
232
|
+
current_text = m[2]
|
233
|
+
current_text.strip! unless current_text.nil?
|
234
|
+
end
|
235
|
+
current_text
|
236
|
+
end
|
237
|
+
|
238
|
+
end
|
239
|
+
end
|
data/lib/appstats/query.rb
CHANGED
@@ -2,8 +2,9 @@
|
|
2
2
|
module Appstats
|
3
3
|
class Query
|
4
4
|
|
5
|
+
@@nill_query = "select 0 from appstats_entries LIMIT 1"
|
5
6
|
@@default = "1=1"
|
6
|
-
attr_accessor :query, :action, :host, :date_range, :query_to_sql
|
7
|
+
attr_accessor :query, :action, :host, :date_range, :query_to_sql, :contexts
|
7
8
|
|
8
9
|
def initialize(data = {})
|
9
10
|
self.query=(data[:query])
|
@@ -28,17 +29,60 @@ module Appstats
|
|
28
29
|
return @@default if m.nil?
|
29
30
|
host = m[1]
|
30
31
|
return @@default if host == '' or host.nil?
|
31
|
-
"EXISTS (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = id and host = '#{host}' )"
|
32
|
+
"EXISTS (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = appstats_log_collectors.id and host = '#{host}' )"
|
32
33
|
end
|
33
34
|
|
34
|
-
def self.
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
def self.contexts_filter_to_sql(raw_input)
|
36
|
+
context_parser = Appstats::Parser.new(:rules => ":context", :repeating => true, :tokenize => "|| && = <= >= <> != ( )")
|
37
|
+
return @@default if (raw_input.blank? || !context_parser.parse(raw_input))
|
38
|
+
sql = "EXISTS (select * from appstats_contexts where appstats_entries.id = appstats_contexts.appstats_entry_id and ("
|
39
|
+
|
40
|
+
status = :next
|
41
|
+
comparator = "="
|
42
|
+
context_parser.raw_results.each do |entry|
|
43
|
+
if entry.kind_of?(String)
|
44
|
+
sqlentry = sqlize(entry)
|
45
|
+
if Query.comparator?(entry) && status == :waiting_comparator
|
46
|
+
comparator = sqlize(entry)
|
47
|
+
status = :waiting_operand
|
48
|
+
else
|
49
|
+
sql += ")" if status == :waiting_comparator
|
50
|
+
sql += " #{sqlentry}"
|
51
|
+
status = :next
|
52
|
+
end
|
53
|
+
next
|
54
|
+
end
|
55
|
+
if status == :next
|
56
|
+
status = :waiting_comparator
|
57
|
+
sql += " (context_key='#{sqlclean(entry[:context])}'"
|
58
|
+
else
|
59
|
+
status = :next
|
60
|
+
sql += " and context_value#{comparator}'#{sqlclean(entry[:context])}')"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
sql += ")" if status == :waiting_comparator
|
64
|
+
sql += "))"
|
65
|
+
sql
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.sqlize(input)
|
69
|
+
return "and" if input == "&&"
|
70
|
+
return "or" if input == "||"
|
71
|
+
return "<>" if input == "!="
|
72
|
+
input
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.sqlclean(raw_input)
|
76
|
+
return raw_input if raw_input.blank?
|
77
|
+
m = raw_input.match(/^['"](.*)['"]$/)
|
78
|
+
input = m.nil? ? raw_input : m[1]
|
79
|
+
input = input.gsub(/\\/, '\&\&').gsub(/'/, "''")
|
80
|
+
input
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.comparator?(raw_input)
|
84
|
+
return false if raw_input.nil?
|
85
|
+
["=","!=","<>",">","<",">=","<="].include?(raw_input)
|
42
86
|
end
|
43
87
|
|
44
88
|
private
|
@@ -49,35 +93,50 @@ module Appstats
|
|
49
93
|
end
|
50
94
|
|
51
95
|
def parse_query
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
date_range_text = m_on_server.nil? ? current_query : m_on_server[1]
|
69
|
-
if date_range_text.size > 0
|
70
|
-
@date_range = DateRange.parse(date_range_text)
|
96
|
+
reset_query
|
97
|
+
return nil_query if @query.nil?
|
98
|
+
current_query = fix_legacy_structures(@query)
|
99
|
+
|
100
|
+
parser = Appstats::Parser.new(:rules => ":operation :action :date on :host where :contexts")
|
101
|
+
return nil_query unless parser.parse(current_query)
|
102
|
+
|
103
|
+
@operation = parser.results[:operation]
|
104
|
+
@action = normalize_action_name(parser.results[:action])
|
105
|
+
@date_range = DateRange.parse(parser.results[:date])
|
106
|
+
@host = parser.results[:host]
|
107
|
+
@contexts = parser.results[:contexts]
|
108
|
+
|
109
|
+
if @operation == "#"
|
110
|
+
@query_to_sql = "select count(*) from appstats_entries"
|
111
|
+
@query_to_sql += " where action = '#{@action}'" unless @action.blank?
|
71
112
|
@query_to_sql += " and #{@date_range.to_sql}" unless @date_range.to_sql == "1=1"
|
113
|
+
@query_to_sql += " and #{Query.host_filter_to_sql(@host)}" unless @host.nil?
|
114
|
+
@query_to_sql += " and #{Query.contexts_filter_to_sql(@contexts)}" unless @contexts.nil?
|
72
115
|
end
|
73
|
-
return @query_to_sql if m_on_server.nil?
|
74
|
-
|
75
|
-
@host = m_on_server[2]
|
76
|
-
@query_to_sql += " and exists (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = appstats_log_collectors.id and host = '#{@host}')"
|
77
116
|
|
78
117
|
@query_to_sql
|
79
|
-
end
|
80
|
-
|
118
|
+
end
|
119
|
+
|
120
|
+
def fix_legacy_structures(raw_input)
|
121
|
+
query = raw_input.gsub(/on\s*server/,"on")
|
122
|
+
query
|
123
|
+
end
|
124
|
+
|
125
|
+
def sql_for_conext(context_name,contact_value)
|
126
|
+
"EXISTS(select * from appstats_contexts where appstats_contexts.appstats_entry_id=appstats_entries.id and context_key='#{context_name}' and context_value='#{contact_value}' )"
|
127
|
+
end
|
128
|
+
|
129
|
+
def nil_query
|
130
|
+
@query_to_sql = @@nill_query
|
131
|
+
@query_to_sql
|
132
|
+
end
|
133
|
+
|
134
|
+
def reset_query
|
135
|
+
@action = nil
|
136
|
+
@host = nil
|
137
|
+
nil_query
|
138
|
+
@date_range = DateRange.new
|
139
|
+
end
|
81
140
|
|
82
141
|
end
|
83
142
|
end
|