appstats 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +1 -1
- data/lib/appstats/parser.rb +28 -14
- data/lib/appstats/version.rb +1 -1
- data/spec/entry_spec.rb +4 -4
- data/spec/logger_spec.rb +18 -18
- data/spec/parser_spec.rb +45 -10
- metadata +3 -3
data/Gemfile.lock
CHANGED
data/lib/appstats/parser.rb
CHANGED
@@ -2,7 +2,9 @@
|
|
2
2
|
module Appstats
|
3
3
|
class Parser
|
4
4
|
|
5
|
-
attr_reader :raw_rules, :rules, :repeating, :raw_tokenize, :
|
5
|
+
attr_reader :raw_rules, :rules, :repeating, :raw_tokenize, :results, :raw_results,
|
6
|
+
:tokenize, :tokenize_no_spaces, :tokenize_regex, :tokenize_regex_no_spaces,
|
7
|
+
:constants, :constants_no_spaces
|
6
8
|
|
7
9
|
def initialize(data = {})
|
8
10
|
@raw_rules = data[:rules]
|
@@ -25,6 +27,7 @@ module Appstats
|
|
25
27
|
@previous_text_so_far = input.strip
|
26
28
|
@text_so_far = @previous_text_so_far
|
27
29
|
@remaining_constants = @constants.dup
|
30
|
+
@remaining_constants_no_spaces = @constants_no_spaces.dup
|
28
31
|
|
29
32
|
while !@text_so_far.blank?
|
30
33
|
process_constant_if_present
|
@@ -37,12 +40,13 @@ module Appstats
|
|
37
40
|
if rule.kind_of?(Hash)
|
38
41
|
if rule[:stop] == :constant
|
39
42
|
was_found = false
|
40
|
-
@
|
43
|
+
@remaining_constants_no_spaces.each_with_index do |k,index|
|
41
44
|
p = parse_word(@text_so_far,k,true)
|
42
45
|
if p[0].nil?
|
43
46
|
unset_rules_until(k)
|
44
47
|
else
|
45
48
|
(index-1).downto(0) do |i|
|
49
|
+
@remaining_constants_no_spaces.delete_at(i)
|
46
50
|
@remaining_constants.delete_at(i)
|
47
51
|
end
|
48
52
|
add_results(rule[:rule],p[0])
|
@@ -79,11 +83,10 @@ module Appstats
|
|
79
83
|
clean_parsed_word(answer)
|
80
84
|
end
|
81
85
|
|
82
|
-
def self.merge_regex_filter(
|
83
|
-
|
84
|
-
return "
|
85
|
-
|
86
|
-
"(#{a}|#{b})"
|
86
|
+
def self.merge_regex_filter(inputs = [])
|
87
|
+
inputs.collect! { |x| x unless x.blank? }.compact!
|
88
|
+
return "" if inputs.empty?
|
89
|
+
"(#{inputs.join('|')})"
|
87
90
|
end
|
88
91
|
|
89
92
|
def parse_word(current_text,stop_on,strict = false)
|
@@ -94,7 +97,7 @@ module Appstats
|
|
94
97
|
current_text = remove_tokens_at_start(current_text)
|
95
98
|
|
96
99
|
if stop_on == :end
|
97
|
-
filter = Parser.merge_regex_filter(nil,@tokenize_regex)
|
100
|
+
filter = Parser.merge_regex_filter([nil,@tokenize_regex])
|
98
101
|
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
99
102
|
if m.nil? || m[1].blank?
|
100
103
|
answer[0] = current_text
|
@@ -103,7 +106,7 @@ module Appstats
|
|
103
106
|
answer[1] = m[2]
|
104
107
|
end
|
105
108
|
elsif stop_on == :space
|
106
|
-
filter = Parser.merge_regex_filter('\s',@tokenize_regex)
|
109
|
+
filter = Parser.merge_regex_filter(['\s',@tokenize_regex,remaining_constants_regex])
|
107
110
|
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
108
111
|
if m.nil?
|
109
112
|
answer[0] = current_text
|
@@ -112,7 +115,7 @@ module Appstats
|
|
112
115
|
answer[1] = m[2]
|
113
116
|
end
|
114
117
|
else
|
115
|
-
filter = Parser.merge_regex_filter(stop_on,@tokenize_regex)
|
118
|
+
filter = Parser.merge_regex_filter([stop_on,@tokenize_regex,remaining_constants_regex])
|
116
119
|
m = current_text.match(/^(.*?)(#{filter}.*)$/im)
|
117
120
|
if strict
|
118
121
|
answer[0] = m[1] unless m.nil?
|
@@ -138,7 +141,7 @@ module Appstats
|
|
138
141
|
def process_constant_if_present
|
139
142
|
while process_tokens_if_present; end
|
140
143
|
to_delete = nil
|
141
|
-
@
|
144
|
+
@remaining_constants_no_spaces.each do |k|
|
142
145
|
p = Parser.parse_constant(@text_so_far,k)
|
143
146
|
next if p[0].nil?
|
144
147
|
to_delete = k
|
@@ -147,6 +150,7 @@ module Appstats
|
|
147
150
|
@text_so_far = p[1]
|
148
151
|
end
|
149
152
|
@remaining_constants.delete(to_delete) unless to_delete.nil?
|
153
|
+
@remaining_constants_no_spaces.delete(to_delete) unless to_delete.nil?
|
150
154
|
end
|
151
155
|
|
152
156
|
def process_tokens_if_present
|
@@ -191,15 +195,20 @@ module Appstats
|
|
191
195
|
def update_rules
|
192
196
|
@rules = []
|
193
197
|
@constants = []
|
198
|
+
@constants_no_spaces = []
|
194
199
|
current_rule = nil
|
195
200
|
return if @raw_rules.blank?
|
196
201
|
@raw_rules.split(" ").each do |rule|
|
202
|
+
current_rule_no_spaces = nil
|
197
203
|
|
198
204
|
if rule.starts_with?(":") && rule.size > 1
|
199
|
-
|
205
|
+
current_rule_no_spaces = { :rule => rule[1..-1].to_sym, :stop => :end }
|
200
206
|
previous_stop_on = :space
|
201
207
|
else
|
202
208
|
current_rule = rule.upcase
|
209
|
+
current_rule_no_spaces = current_rule
|
210
|
+
@constants_no_spaces<< current_rule_no_spaces
|
211
|
+
current_rule = "\\s+#{current_rule}" unless current_rule.match(/.*[a-z].*/i).nil?
|
203
212
|
@constants<< current_rule
|
204
213
|
previous_stop_on = :constant
|
205
214
|
end
|
@@ -208,7 +217,7 @@ module Appstats
|
|
208
217
|
@rules.last[:stop] = previous_stop_on
|
209
218
|
end
|
210
219
|
|
211
|
-
@rules<<
|
220
|
+
@rules<< current_rule_no_spaces
|
212
221
|
end
|
213
222
|
end
|
214
223
|
|
@@ -234,6 +243,11 @@ module Appstats
|
|
234
243
|
end
|
235
244
|
current_text
|
236
245
|
end
|
237
|
-
|
246
|
+
|
247
|
+
def remaining_constants_regex
|
248
|
+
return "" if @remaining_constants.nil?
|
249
|
+
@remaining_constants.join("|")
|
250
|
+
end
|
251
|
+
|
238
252
|
end
|
239
253
|
end
|
data/lib/appstats/version.rb
CHANGED
data/spec/entry_spec.rb
CHANGED
@@ -188,18 +188,18 @@ module Appstats
|
|
188
188
|
end
|
189
189
|
|
190
190
|
it "should understand an entry without contexts" do
|
191
|
-
entry = Entry.create_from_logger_string("0.9.
|
191
|
+
entry = Entry.create_from_logger_string("0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
|
192
192
|
Entry.count.should == @before_count + 1
|
193
193
|
entry.action.should == "address_search"
|
194
|
-
entry.raw_entry.should == "0.9.
|
194
|
+
entry.raw_entry.should == "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
|
195
195
|
entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
|
196
196
|
end
|
197
197
|
|
198
198
|
it "should understand contexts" do
|
199
|
-
entry = Entry.create_from_logger_string("0.9.
|
199
|
+
entry = Entry.create_from_logger_string("0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
|
200
200
|
Entry.count.should == @before_count + 1
|
201
201
|
entry.action.should == "address_filter"
|
202
|
-
entry.raw_entry.should == "0.9.
|
202
|
+
entry.raw_entry.should == "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
|
203
203
|
entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
|
204
204
|
entry.contexts.size.should == 2
|
205
205
|
entry.contexts[0].context_key = "app_name"
|
data/spec/logger_spec.rb
CHANGED
@@ -115,7 +115,7 @@ module Appstats
|
|
115
115
|
|
116
116
|
it "should accept numbers" do
|
117
117
|
Appstats::Logger.entry(5, :blah => 6)
|
118
|
-
Appstats::Logger.raw_read.should == ["0.9.
|
118
|
+
Appstats::Logger.raw_read.should == ["0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
|
119
119
|
end
|
120
120
|
|
121
121
|
end
|
@@ -124,7 +124,7 @@ module Appstats
|
|
124
124
|
|
125
125
|
it "should look similar to regular entry" do
|
126
126
|
Appstats::Logger.exception_entry(RuntimeError.new("blah"),:on => "login")
|
127
|
-
Appstats::Logger.raw_read.should == ["0.9.
|
127
|
+
Appstats::Logger.raw_read.should == ["0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
|
128
128
|
end
|
129
129
|
|
130
130
|
end
|
@@ -141,29 +141,29 @@ module Appstats
|
|
141
141
|
|
142
142
|
it "should handle a statistics entry" do
|
143
143
|
expected = { :action => "address_search", :timestamp => "2010-09-21 23:15:20" }
|
144
|
-
actual = Appstats::Logger.entry_to_hash("0.9.
|
144
|
+
actual = Appstats::Logger.entry_to_hash("0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
|
145
145
|
actual.should == expected
|
146
146
|
end
|
147
147
|
|
148
148
|
it "should handle contexts" do
|
149
149
|
expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
|
150
|
-
actual = Appstats::Logger.entry_to_hash("0.9.
|
150
|
+
actual = Appstats::Logger.entry_to_hash("0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
|
151
151
|
actual.should == expected
|
152
152
|
end
|
153
153
|
|
154
154
|
it "should handle actions with the delimiter (and change the delimiter)" do
|
155
155
|
expected = { :action => "address:=search-n", :timestamp => "2010-09-21 23:15:20" }
|
156
|
-
actual = Appstats::Logger.entry_to_hash("0.9.
|
156
|
+
actual = Appstats::Logger.entry_to_hash("0.9.1 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
|
157
157
|
actual.should == expected
|
158
158
|
|
159
159
|
expected = { :action => "address::search==--n", :timestamp => "2010-09-21 23:15:20" }
|
160
|
-
actual = Appstats::Logger.entry_to_hash("0.9.
|
160
|
+
actual = Appstats::Logger.entry_to_hash("0.9.1 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
|
161
161
|
actual.should == expected
|
162
162
|
end
|
163
163
|
|
164
164
|
it "should handle contexts with the delimiter (and change the delimiter)" do
|
165
165
|
expected = { :action => "address", :timestamp => "2010-09-21 23:15:20", :server => "market:eval=-n" }
|
166
|
-
actual = Appstats::Logger.entry_to_hash("0.9.
|
166
|
+
actual = Appstats::Logger.entry_to_hash("0.9.1 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
|
167
167
|
actual.should == expected
|
168
168
|
end
|
169
169
|
|
@@ -172,66 +172,66 @@ module Appstats
|
|
172
172
|
describe "#entry_to_s" do
|
173
173
|
|
174
174
|
it "should handle a statistics entry" do
|
175
|
-
expected = "0.9.
|
175
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
|
176
176
|
actual = Appstats::Logger.entry_to_s("address_search")
|
177
177
|
actual.should == expected
|
178
178
|
end
|
179
179
|
|
180
180
|
it "should handle numbers" do
|
181
|
-
expected = "0.9.
|
181
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
|
182
182
|
actual = Appstats::Logger.entry_to_s(1,:note => 2.2)
|
183
183
|
actual.should == expected
|
184
184
|
end
|
185
185
|
|
186
186
|
it "should handle default contexts" do
|
187
187
|
Appstats::Logger.default_contexts[:app_name] = "market"
|
188
|
-
expected = "0.9.
|
188
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
|
189
189
|
actual = Appstats::Logger.entry_to_s("address_search")
|
190
190
|
actual.should == expected
|
191
191
|
end
|
192
192
|
|
193
193
|
it "should handle contexts (and sort them by symbol)" do
|
194
|
-
expected = "0.9.
|
194
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
|
195
195
|
actual = Appstats::Logger.entry_to_s("address_filter", { :server => "Live", :app_name => 'Market' })
|
196
196
|
actual.should == expected
|
197
197
|
end
|
198
198
|
|
199
199
|
it "should handle actions with the delimiter (and change the delimiter)" do
|
200
|
-
expected = "0.9.
|
200
|
+
expected = "0.9.1 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
|
201
201
|
actual = Appstats::Logger.entry_to_s("address:=search-n")
|
202
202
|
actual.should == expected
|
203
203
|
|
204
|
-
expected = "0.9.
|
204
|
+
expected = "0.9.1 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
|
205
205
|
actual = Appstats::Logger.entry_to_s("address::search==--n")
|
206
206
|
actual.should == expected
|
207
207
|
end
|
208
208
|
|
209
209
|
it "should handle contexts with the delimiter (and change the delimiter)" do
|
210
|
-
expected = "0.9.
|
210
|
+
expected = "0.9.1 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
|
211
211
|
actual = Appstats::Logger.entry_to_s("address", :server => 'market:eval=-n')
|
212
212
|
actual.should == expected
|
213
213
|
end
|
214
214
|
|
215
215
|
it "should ignore spaces" do
|
216
|
-
expected = "0.9.
|
216
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
|
217
217
|
actual = Appstats::Logger.entry_to_s("address search")
|
218
218
|
actual.should == expected
|
219
219
|
end
|
220
220
|
|
221
221
|
it "should convert newlines in action" do
|
222
|
-
expected = "0.9.
|
222
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
|
223
223
|
actual = Appstats::Logger.entry_to_s("address_\nsearch")
|
224
224
|
actual.should == expected
|
225
225
|
end
|
226
226
|
|
227
227
|
it "should convert newlines in context" do
|
228
|
-
expected = "0.9.
|
228
|
+
expected = "0.9.1 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
|
229
229
|
actual = Appstats::Logger.entry_to_s("address_search",:blah => "some\nlong\nstatement")
|
230
230
|
actual.should == expected
|
231
231
|
end
|
232
232
|
|
233
233
|
it "should convert newlines based on the delimiter" do
|
234
|
-
expected = "0.9.
|
234
|
+
expected = "0.9.1 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
|
235
235
|
actual = Appstats::Logger.entry_to_s("address:=\nsearch-n")
|
236
236
|
actual.should == expected
|
237
237
|
end
|
data/spec/parser_spec.rb
CHANGED
@@ -17,6 +17,7 @@ module Appstats
|
|
17
17
|
@parser.rules.should == []
|
18
18
|
@parser.tokenize.should == []
|
19
19
|
@parser.constants.should == []
|
20
|
+
@parser.constants_no_spaces.should == []
|
20
21
|
end
|
21
22
|
|
22
23
|
it "should set rules from constructor" do
|
@@ -27,7 +28,8 @@ module Appstats
|
|
27
28
|
parser.rules.should == [ { :rule => :name, :stop => :constant }, "OR", { :rule => :bust, :stop => :end} ]
|
28
29
|
parser.tokenize.should == ["\\s+A","\\s+BB","\\s+C"]
|
29
30
|
parser.tokenize_no_spaces.should == ["A","BB","C"]
|
30
|
-
parser.constants.should == ["OR"]
|
31
|
+
parser.constants.should == ["\\s+OR"]
|
32
|
+
parser.constants_no_spaces.should == ["OR"]
|
31
33
|
end
|
32
34
|
|
33
35
|
it "should espace tokens as required" do
|
@@ -85,14 +87,34 @@ module Appstats
|
|
85
87
|
end
|
86
88
|
|
87
89
|
it "should upper case constants" do
|
88
|
-
Parser.new(:rules => "blah").constants.should == ["BLAH"]
|
90
|
+
Parser.new(:rules => "blah").constants.should == ["\\s+BLAH"]
|
89
91
|
end
|
90
92
|
|
91
93
|
it "should deal with multiple constants" do
|
92
|
-
Parser.new(:rules => ":name = :blah and :moreblah").constants.should == ["=","AND"]
|
94
|
+
Parser.new(:rules => ":name = :blah and :moreblah").constants.should == ["=","\\s+AND"]
|
93
95
|
end
|
94
96
|
|
95
97
|
end
|
98
|
+
|
99
|
+
describe "#constants_no_spaces" do
|
100
|
+
|
101
|
+
it "should be empty if only variables" do
|
102
|
+
Parser.new(:rules => ":name :blah").constants_no_spaces.should == [ ]
|
103
|
+
end
|
104
|
+
|
105
|
+
it "should track all constants" do
|
106
|
+
Parser.new(:rules => ":name : :date").constants_no_spaces.should == [ ":" ]
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should upper case constants" do
|
110
|
+
Parser.new(:rules => "blah").constants_no_spaces.should == ["BLAH"]
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should deal with multiple constants" do
|
114
|
+
Parser.new(:rules => ":name = :blah and :moreblah").constants_no_spaces.should == ["=","AND"]
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
96
118
|
|
97
119
|
describe "#parse_constant" do
|
98
120
|
|
@@ -214,19 +236,24 @@ module Appstats
|
|
214
236
|
describe "#merge_regex_filter" do
|
215
237
|
|
216
238
|
it "should handle nil" do
|
217
|
-
Parser.merge_regex_filter(nil,nil).should == ""
|
218
|
-
Parser.merge_regex_filter('','').should == ""
|
239
|
+
Parser.merge_regex_filter([nil,nil]).should == ""
|
240
|
+
Parser.merge_regex_filter(['','']).should == ""
|
219
241
|
end
|
220
242
|
|
221
243
|
it "should handle nil on one side" do
|
222
|
-
Parser.merge_regex_filter('\s',nil).should == '(\s)'
|
223
|
-
Parser.merge_regex_filter(nil,'\s').should == '(\s)'
|
224
|
-
Parser.merge_regex_filter('\s','').should == '(\s)'
|
225
|
-
Parser.merge_regex_filter('','\s').should == '(\s)'
|
244
|
+
Parser.merge_regex_filter(['\s',nil]).should == '(\s)'
|
245
|
+
Parser.merge_regex_filter([nil,'\s']).should == '(\s)'
|
246
|
+
Parser.merge_regex_filter(['\s','']).should == '(\s)'
|
247
|
+
Parser.merge_regex_filter(['','\s']).should == '(\s)'
|
226
248
|
end
|
227
249
|
|
228
250
|
it "should handle both sides" do
|
229
|
-
Parser.merge_regex_filter('\s','a|b').should == '(\s|a|b)'
|
251
|
+
Parser.merge_regex_filter(['\s','a|b']).should == '(\s|a|b)'
|
252
|
+
end
|
253
|
+
|
254
|
+
it "should handle three inputs" do
|
255
|
+
Parser.merge_regex_filter(['\s','a|b','dd']).should == '(\s|a|b|dd)'
|
256
|
+
Parser.merge_regex_filter(['\s','',nil]).should == '(\s)'
|
230
257
|
end
|
231
258
|
|
232
259
|
|
@@ -321,6 +348,14 @@ module Appstats
|
|
321
348
|
parser.results.should == {:operation => "#", :action => "logins", :date => "between 2010-01-15 and 2010-01-31", :host => "your.localnet", :contexts => nil }
|
322
349
|
end
|
323
350
|
|
351
|
+
it "should handle last week" do
|
352
|
+
|
353
|
+
parser = Appstats::Parser.new(:rules => ":operation :action :date on :host where :contexts")
|
354
|
+
parser.parse("# logins last week where service_provider = Cox Communications").should == true
|
355
|
+
parser.results.should == {:operation => "#", :action => "logins", :date => "last week", :host => nil, :contexts => "service_provider = Cox Communications" }
|
356
|
+
|
357
|
+
end
|
358
|
+
|
324
359
|
end
|
325
360
|
|
326
361
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: appstats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 57
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 9
|
9
|
-
-
|
10
|
-
version: 0.9.
|
9
|
+
- 1
|
10
|
+
version: 0.9.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Andrew Forward
|