wvanbergen-request-log-analyzer 1.0.2 → 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -3,6 +3,17 @@ require 'activerecord'
3
3
 
4
4
  module RequestLogAnalyzer::Aggregator
5
5
 
6
+ # The database aggregator will create an SQLite3 database with all parsed request information.
7
+ #
8
+ # The prepare method will create a database schema according to the file format definitions.
9
+ # It will also create ActiveRecord::Base subclasses to interact with the created tables.
10
+ # Then, the aggregate method will be called for every parsed request. The information of
11
+ # these requests is inserted into the tables using the ActiveRecord classes.
12
+ #
13
+ # A requests table will be created, in which a record is inserted for every parsed request.
14
+ # For every line type, a separate table will be created with a request_id field to point to
15
+ # the request record, and a field for every parsed value. Finally, a warnings table will be
16
+ # created to log all parse warnings.
6
17
  class Database < Base
7
18
 
8
19
  # Establishes a connection to the database and creates the necessary database schema for the
@@ -54,7 +65,7 @@ module RequestLogAnalyzer::Aggregator
54
65
  # This function creates a database table for a given line definition.
55
66
  # It will create a field for every capture in the line, and adds a lineno field to indicate at
56
67
  # what line in the original file the line was found, and a request_id to link lines related
57
- # to the same request.
68
+ # to the same request. It will also create an index in the request_id field to speed up queries.
58
69
  def create_database_table(name, definition)
59
70
  ActiveRecord::Migration.verbose = options[:debug]
60
71
  ActiveRecord::Migration.create_table("#{name}_lines") do |t|
@@ -64,6 +75,7 @@ module RequestLogAnalyzer::Aggregator
64
75
  t.column(capture[:name], column_type(capture))
65
76
  end
66
77
  end
78
+ ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
67
79
  end
68
80
 
69
81
  # Creates an ActiveRecord class for a given line definition.
@@ -181,7 +181,7 @@ module RequestLogAnalyzer
181
181
 
182
182
  begin
183
183
  @source.requests do |request|
184
- #@filters.each { |filter| request = filter.filter(request) }
184
+ @filters.each { |filter| request = filter.filter(request) }
185
185
  @aggregators.each { |agg| agg.aggregate(request) } if request
186
186
  end
187
187
  rescue Interrupt => e
@@ -198,4 +198,4 @@ module RequestLogAnalyzer
198
198
  end
199
199
 
200
200
  end
201
- end
201
+ end
@@ -0,0 +1,43 @@
1
+ module RequestLogAnalyzer::Filter
2
+
3
+ # Filter to select or reject a specific field
4
+ # Options
5
+ # * <tt>:mode</tt> :reject or :accept.
6
+ # * <tt>:field</tt> Specific field to accept or reject.
7
+ # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
8
+ class Anonimize < Base
9
+
10
+ def prepare
11
+ end
12
+
13
+ def generate_random_ip
14
+ "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
15
+ end
16
+
17
+ def anonymize_url(value)
18
+ return value.sub(/^https?\:\/\/[A-z0-9\.-]+\//, "http://example.com/")
19
+ end
20
+
21
+ def fuzz(value)
22
+ value * ((75 + rand(50)) / 100.0)
23
+ end
24
+
25
+ def filter(request)
26
+ return nil unless request
27
+
28
+ request.attributes.each do |key, value|
29
+ if key == :ip
30
+ request.attributes[key] = generate_random_ip
31
+ elsif key == :url
32
+ request.attributes[key] = anonymize_url(value)
33
+ elsif [ :duration, :view, :db, :type, :after_filters_time, :before_filters_time,
34
+ :action_time].include?(key)
35
+ request.attributes[key] = fuzz(value)
36
+ end
37
+ end
38
+
39
+ return request
40
+ end
41
+ end
42
+
43
+ end
@@ -22,6 +22,11 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
22
22
  ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
23
23
  ActiveRecord::Migration.should_receive(:create_table).with("test_lines")
24
24
  ActiveRecord::Migration.should_receive(:create_table).with("last_lines")
25
+
26
+ ActiveRecord::Migration.should_receive(:add_index).with("first_lines", [:request_id])
27
+ ActiveRecord::Migration.should_receive(:add_index).with("test_lines", [:request_id])
28
+ ActiveRecord::Migration.should_receive(:add_index).with("last_lines", [:request_id])
29
+
25
30
  @database_inserter.prepare
26
31
  end
27
32
 
data/spec/filter_spec.rb CHANGED
@@ -1,6 +1,7 @@
1
1
  require File.dirname(__FILE__) + '/spec_helper'
2
2
  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
3
3
  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
4
+ require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/anonimize'
4
5
 
5
6
  describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
6
7
  include RequestLogAnalyzerSpecHelper
@@ -134,4 +135,26 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
134
135
  it "should accept a request if the value is not the first value" do
135
136
  @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
136
137
  end
138
+ end
139
+
140
+ describe RequestLogAnalyzer::Filter::Anonimize, 'anonimize request' do
141
+ include RequestLogAnalyzerSpecHelper
142
+
143
+ before(:each) do
144
+ @filter = RequestLogAnalyzer::Filter::Anonimize.new(spec_format)
145
+ @filter.prepare
146
+ end
147
+
148
+ it "should anonimize ip" do
149
+ @filter.filter(request(:ip => '123.123.123.123'))[:ip].should_not eql('123.123.123.123')
150
+ end
151
+
152
+ it "should anonimize url" do
153
+ @filter.filter(request(:url => 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees')
154
+ end
155
+
156
+ it "should anonimize url" do
157
+ @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
158
+ end
159
+
137
160
  end
@@ -1,6 +1,8 @@
1
1
  require 'rubygems'
2
+ require 'rubyforge'
2
3
  require 'rake'
3
4
  require 'rake/tasklib'
5
+ require 'rake/gempackagetask'
4
6
  require 'date'
5
7
 
6
8
  module Rake
@@ -25,22 +27,14 @@ module Rake
25
27
  desc "Updates the file lists for this gem"
26
28
  task(:manifest) { manifest_task }
27
29
 
28
- desc "Builds a ruby gem for #{@name}"
29
- task(:build => [:manifest]) { build_task }
30
-
31
- desc "Installs the ruby gem for #{@name} locally"
32
- task(:install => [:build]) { install_task }
33
-
34
- desc "Uninstalls the ruby gem for #{@name} locally"
35
- task(:uninstall) { uninstall_task }
36
-
37
30
  desc "Releases a new version of #{@name}"
38
- task(:release) { release_task }
39
- end
31
+ task(:release => :package) { release_task }
32
+
33
+ Rake::GemPackageTask.new(@specification) do |pkg|
34
+ end
35
+ end
40
36
  end
41
37
 
42
-
43
-
44
38
  protected
45
39
 
46
40
  def reload_gemspec!
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wvanbergen-request-log-analyzer
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2009-01-13 00:00:00 -08:00
13
+ date: 2009-01-14 00:00:00 -08:00
14
14
  default_executable: request-log-analyzer
15
15
  dependencies: []
16
16
 
@@ -48,6 +48,7 @@ files:
48
48
  - lib/request_log_analyzer/file_format/merb.rb
49
49
  - lib/request_log_analyzer/file_format/rails.rb
50
50
  - lib/request_log_analyzer/filter
51
+ - lib/request_log_analyzer/filter/anonimize.rb
51
52
  - lib/request_log_analyzer/filter/base.rb
52
53
  - lib/request_log_analyzer/filter/field.rb
53
54
  - lib/request_log_analyzer/filter/timespan.rb
@@ -116,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
116
117
  version:
117
118
  requirements: []
118
119
 
119
- rubyforge_project:
120
+ rubyforge_project: r-l-a
120
121
  rubygems_version: 1.2.0
121
122
  signing_key:
122
123
  specification_version: 2