wvanbergen-request-log-analyzer 1.0.2 → 1.0.3

data/lib/request_log_analyzer/aggregator/database.rb

@@ -3,6 +3,17 @@ require 'activerecord'
 
 module RequestLogAnalyzer::Aggregator
 
+  # The database aggregator will create an SQLite3 database with all parsed request information.
+  #
+  # The prepare method will create a database schema according to the file format definitions.
+  # It will also create ActiveRecord::Base subclasses to interact with the created tables. 
+  # Then, the aggregate method will be called for every parsed request. The information of
+  # these requests is inserted into the tables using the ActiveRecord classes.
+  #
+  # A requests table will be created, in which a record is inserted for every parsed request.
+  # For every line type, a separate table will be created with a request_id field to point to
+  # the request record, and a field for every parsed value. Finally, a warnings table will be
+  # created to log all parse warnings.
   class Database < Base
 
     # Establishes a connection to the database and creates the necessary database schema for the
@@ -54,7 +65,7 @@ module RequestLogAnalyzer::Aggregator
     # This function creates a database table for a given line definition.
     # It will create a field for every capture in the line, and adds a lineno field to indicate at
     # what line in the original file the line was found, and a request_id to link lines related
-    # to the same request.
+    # to the same request. It will also create an index in the request_id field to speed up queries.
     def create_database_table(name, definition)
       ActiveRecord::Migration.verbose = options[:debug]
       ActiveRecord::Migration.create_table("#{name}_lines") do |t|
@@ -64,6 +75,7 @@ module RequestLogAnalyzer::Aggregator
           t.column(capture[:name], column_type(capture))
         end
       end
+      ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
     end
     
     # Creates an ActiveRecord class for a given line definition.

data/lib/request_log_analyzer/controller.rb

@@ -181,7 +181,7 @@ module RequestLogAnalyzer
       
       begin
         @source.requests do |request|
-          #@filters.each { |filter| request = filter.filter(request) }
+          @filters.each { |filter| request = filter.filter(request) }
           @aggregators.each { |agg| agg.aggregate(request) } if request
         end
       rescue Interrupt => e
@@ -198,4 +198,4 @@ module RequestLogAnalyzer
     end
     
   end
-end
+end

data/lib/request_log_analyzer/filter/anonimize.rb

@@ -0,0 +1,43 @@
+module RequestLogAnalyzer::Filter
+  
+  # Filter to select or reject a specific field
+  # Options
+  # * <tt>:mode</tt> :reject or :accept.
+  # * <tt>:field</tt> Specific field to accept or reject.
+  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
+  class Anonimize < Base
+   
+    def prepare
+    end
+
+    def generate_random_ip
+      "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
+    end
+
+    def anonymize_url(value)
+      return value.sub(/^https?\:\/\/[A-z0-9\.-]+\//, "http://example.com/")
+    end
+    
+    def fuzz(value)
+      value * ((75 + rand(50)) / 100.0)
+    end
+
+    def filter(request)
+      return nil unless request
+      
+      request.attributes.each do |key, value|
+        if key == :ip
+          request.attributes[key] = generate_random_ip
+        elsif key == :url
+          request.attributes[key] = anonymize_url(value)
+        elsif [ :duration, :view, :db, :type, :after_filters_time, :before_filters_time,
+                :action_time].include?(key)
+          request.attributes[key] = fuzz(value)
+        end
+      end
+      
+      return request
+    end 
+  end
+  
+end

data/spec/database_inserter_spec.rb

@@ -22,6 +22,11 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
     ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
     ActiveRecord::Migration.should_receive(:create_table).with("test_lines")        
     ActiveRecord::Migration.should_receive(:create_table).with("last_lines")    
+
+    ActiveRecord::Migration.should_receive(:add_index).with("first_lines", [:request_id])    
+    ActiveRecord::Migration.should_receive(:add_index).with("test_lines", [:request_id])
+    ActiveRecord::Migration.should_receive(:add_index).with("last_lines", [:request_id])
+
     @database_inserter.prepare
   end
   

data/spec/filter_spec.rb

@@ -1,6 +1,7 @@
 require File.dirname(__FILE__) + '/spec_helper'
 require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
 require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/anonimize'
 
 describe RequestLogAnalyzer::Filter::Timespan, 'both before and after'  do
   include RequestLogAnalyzerSpecHelper
@@ -134,4 +135,26 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
   it "should accept a request if the value is not the first value" do
     @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
   end  
+end
+
+describe RequestLogAnalyzer::Filter::Anonimize, 'anonimize request' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Anonimize.new(spec_format)
+    @filter.prepare
+  end
+  
+  it "should anonimize ip" do
+    @filter.filter(request(:ip => '123.123.123.123'))[:ip].should_not eql('123.123.123.123')
+  end
+
+  it "should anonimize url" do
+    @filter.filter(request(:url => 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees')
+  end
+
+  it "should anonimize url" do
+    @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
+  end
+  
 end

data/tasks/github-gem.rake

@@ -1,6 +1,8 @@
 require 'rubygems'
+require 'rubyforge'
 require 'rake'
 require 'rake/tasklib'
+require 'rake/gempackagetask'
 require 'date'
 
 module Rake 
@@ -25,22 +27,14 @@ module Rake
         desc "Updates the file lists for this gem"
         task(:manifest) { manifest_task }
         
-        desc "Builds a ruby gem for #{@name}"
-        task(:build => [:manifest]) { build_task }
-        
-        desc "Installs the ruby gem for #{@name} locally"
-        task(:install => [:build]) { install_task }
-        
-        desc "Uninstalls the ruby gem for #{@name} locally"
-        task(:uninstall) { uninstall_task }             
-        
         desc "Releases a new version of #{@name}"
-        task(:release) { release_task } 
-      end
+        task(:release => :package) { release_task } 
+        
+        Rake::GemPackageTask.new(@specification) do |pkg|
+        end
+      end    
     end
     
-
-    
     protected 
 
     def reload_gemspec!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-13 00:00:00 -08:00
+date: 2009-01-14 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -48,6 +48,7 @@ files:
 - lib/request_log_analyzer/file_format/merb.rb
 - lib/request_log_analyzer/file_format/rails.rb
 - lib/request_log_analyzer/filter
+- lib/request_log_analyzer/filter/anonimize.rb
 - lib/request_log_analyzer/filter/base.rb
 - lib/request_log_analyzer/filter/field.rb
 - lib/request_log_analyzer/filter/timespan.rb
@@ -116,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: 
+rubyforge_project: r-l-a
 rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2