wvanbergen-request-log-analyzer 0.1.0

data/lib/rails_analyzer/log_parser.rb

@@ -0,0 +1,83 @@
+require 'date'
+
+module RailsAnalyzer
+  # Parse a rails log file
+  class LogParser 
+
+    LOG_LINES = {
+      # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
+      :started => {
+        :teaser => /Processing/,
+        :regexp => /Processing (\w+)#(\w+) \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
+        :params => { :controller => 1, :action => 2, :ip => 3, :timestamp => 4, :method => 5}
+      },
+      # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+      :failed => {
+        :teaser => /Error/,
+        :regexp => /(\w+)(Error|Invalid) \((.*)\)\:(.*)/,
+        :params => { :error => 1, :exception_string => 3, :stack_trace => 4 }
+      },
+      # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+      :completed => {
+        :teaser => /Completed/,
+        :regexp => /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/,
+        :params => { :url => 7, :status => [6, :to_i], :duration => [1, :to_f], :rendering => [3, :to_f], :db => [5, :to_f] }
+      }
+    }
+    
+    # LogParser initializer
+    # <tt>file</tt> The fileobject this LogParser wil operate on.
+    def initialize(file, options = {})
+      @file_name = file
+      @options = options
+      @file_size = File.size(@file_name)
+    end
+    
+    def progress(&block)
+      @progress_handler = block
+    end
+
+    # Output a warning
+    # <tt>message</tt> The warning message (object)
+    def warn(message)
+      puts " -> " + message.to_s
+    end  
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def each(*line_types, &block)
+      # parse everything by default 
+      line_types = LOG_LINES.keys if line_types.empty?
+
+      File.open(@file_name) do |file|
+        
+        file.each_line do |line|
+          
+          @progress_handler.call(file.pos, @file_size) if @progress_handler
+          
+          line_types.each do |line_type|
+            if LOG_LINES[line_type][:teaser] =~ line
+              if LOG_LINES[line_type][:regexp] =~ line
+                request = { :type => line_type, :line => file.lineno }
+                LOG_LINES[line_type][:params].each do |key, value|
+                  request[key] = case value
+                    when Numeric; $~[value]
+                    when Array;   $~[value.first].send(value.last)
+                    else; nil
+                  end
+                  
+                end
+                yield(request) if block_given?
+              else
+                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
+              end
+            end            
+          end
+        end
+        @progress_handler.call(:finished, @file_size) if @progress_handler
+      end      
+    end
+  end
+end

data/lib/rails_analyzer/record_inserter.rb

@@ -0,0 +1,161 @@
+require 'rubygems'
+require 'sqlite3'
+
+module RailsAnalyzer
+  
+  # Set of functions that can be used to easily log requests into a SQLite3 Database.
+  class RecordInserter
+    
+    attr_reader :database
+    attr_reader :current_request
+    attr_reader :warning_count
+    
+    # Initializer
+    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
+    def initialize(db_file)
+      @database = SQLite3::Database.new(db_file)
+      @insert_statements = nil
+      @warning_count = 0
+      create_tables_if_needed!
+    end
+        
+    # Calculate the database durations of the requests currenty in the database.
+    # Used if a logfile does contain any database durations.
+    def calculate_db_durations!
+      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
+    end
+    
+    # Insert a batch of loglines into the database.
+    # Function prepares insert statements, yeilds and then closes and commits.
+    def insert_batch(&block)
+      @database.transaction
+      prepare_statements!
+      block.call(self)
+      close_prepared_statements!
+      @database.commit
+    rescue Exception => e
+      puts e.message
+      @database.rollback
+    end
+    
+    def insert_warning(line, warning)
+      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
+      @warning_count += 1
+    end
+        
+    # Insert a request into the database.
+    # <tt>request</tt> The request to insert.
+    # <tt>close_statements</tt> Close prepared statements (default false)
+    def insert(request, close_statements = false)
+      unless @insert_statements
+        prepare_statements! 
+        close_statements = true
+      end
+        
+      if request[:type] && @insert_statements.has_key?(request[:type]) 
+        if request[:type] == :started
+          insert_warning(request[:line], "Unclosed request encountered on line #{request[:line]} (request started on line #{@current_request})") unless @current_request.nil?
+          @current_request = request[:line]
+        elsif [:failed, :completed].include?(request[:type])
+          @current_request = nil
+        end
+        
+        begin
+          @insert_statements[request.delete(:type)].execute(request) 
+        rescue SQLite3::Exception => e
+          insert_warning(request[:line], "Could not save log line to database: " + e.message.to_s)
+        end        
+      else
+        insert_warning(request[:line], "Ignored unknown statement type")
+      end
+      
+      close_prepared_statements! if close_statements
+    end
+    
+    # Insert a batch of files into the database.
+    # <tt>db_file</tt> The filename of the database file to use.
+    # Returns the created database.
+    def self.insert_batch_into(db_file, options = {}, &block)
+      db = RecordInserter.new(db_file)
+      db.insert_batch(&block)
+      return db
+    end    
+    
+    def count(type)
+      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
+    end
+        
+    protected
+    
+    # Prepare insert statements.
+    def prepare_statements!
+      @insert_statements = {
+        :started => @database.prepare("
+            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
+                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
+                                  
+        :failed => @database.prepare("
+            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
+                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
+                                 
+        :completed => @database.prepare("
+            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
+                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
+      }
+    end
+    
+    # Close all prepared statments
+    def close_prepared_statements!
+      @insert_statements.each { |key, stmt| stmt.close }
+    end
+
+    # Create the needed database tables if they don't exist.
+    def create_tables_if_needed!
+      
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS started_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,
+          timestamp DATETIME NOT NULL, 
+          controller VARCHAR(255) NOT NULL, 
+          action VARCHAR(255) NOT NULL,
+          method VARCHAR(6) NOT NULL,          
+          ip VARCHAR(6) NOT NULL
+        )
+      ");
+
+      @database.execute("
+          CREATE TABLE IF NOT EXISTS failed_requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            line INTEGER NOT NULL,            
+            started_request_id INTEGER,        
+            error VARCHAR(255),    
+            exception_string VARCHAR(255),
+            stack_trace TEXT
+          )      
+      ");
+
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS completed_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,          
+          started_request_id INTEGER,
+          url VARCHAR(255) NOT NULL,
+          hashed_url VARCHAR(255),
+          status INTEGER NOT NULL,
+          duration FLOAT,
+          rendering_time FLOAT,
+          database_time FLOAT
+        )
+      ");    
+      
+      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,       
+          warning VARCHAR(255) NOT NULL
+        )
+      ");
+    end
+
+  end
+end

data/lib/rails_analyzer/summarizer.rb

@@ -0,0 +1,121 @@
+module RailsAnalyzer
+  
+  # Functions to summarize an array of requets.
+  # Can calculate request counts, duratations, mean times etc. of all the requests given.
+  class Summarizer
+
+    attr_reader :actions
+    attr_reader :errors
+    attr_reader :request_count
+    attr_reader :request_time_graph
+    attr_reader :first_request_at
+    attr_reader :last_request_at
+
+    attr_accessor :blocker_duration
+    DEFAULT_BLOCKER_DURATION = 1.0
+   
+    # Initializer. Sets global variables
+    # Options
+    # * <tt>:calculate_database</tt> Calculate the database times if they are not explicitly logged.
+    def initialize(options = {})
+      @actions  = {}
+      @blockers = {}
+      @errors   = {}
+      @request_count = 0
+      @blocker_duration = DEFAULT_BLOCKER_DURATION
+      @calculate_database = options[:calculate_database]
+      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+    end
+   
+    # Check if any of the request parsed had a timestamp.
+    def has_timestamps?
+      @first_request_at
+    end
+   
+    # Parse a request string into a hash containing all keys found in the string.
+    # Yields the hash found to the block operator.
+    # <tt>request</tt> The request string to parse.
+    # <tt>&block</tt> Block operator
+    def group(request, &block)
+      request[:duration] ||= 0
+      
+      case request[:type]   
+        when :started      
+          @first_request_at ||= request[:timestamp] # assume time-based order of file
+          @last_request_at  = request[:timestamp]   # assume time-based order of file
+          @request_time_graph[request[:timestamp][11..12].to_i] +=1
+
+        when :completed
+          @request_count += 1 
+          hash = block_given? ? yield(request) : request.hash
+
+          @actions[hash] ||= {:count => 0, :total_time => 0.0, :total_db_time => 0.0, :total_rendering_time => 0.0, 
+                                :min_time => request[:duration], :max_time => request[:duration] }
+                            
+          @actions[hash][:count] += 1
+          @actions[hash][:total_time] += request[:duration]
+          @actions[hash][:total_db_time] += request[:db] if request[:db]
+          @actions[hash][:total_db_time] += request[:duration] - request[:rendering] if @calculate_database
+
+          @actions[hash][:total_rendering_time] += request[:rendering] if request[:rendering]
+      
+          @actions[hash][:min_time] = [@actions[hash][:min_time], request[:duration]].min
+          @actions[hash][:max_time] = [@actions[hash][:min_time], request[:duration]].max
+          @actions[hash][:mean_time] = @actions[hash][:total_time] / @actions[hash][:count].to_f
+      
+          @actions[hash][:mean_db_time] = @actions[hash][:total_db_time] / @actions[hash][:count].to_f      
+          @actions[hash][:mean_rendering_time] = @actions[hash][:total_rendering_time] / @actions[hash][:count].to_f            
+      
+          if request[:duration] > @blocker_duration
+            @blockers[hash] ||= { :count => 0, :total_time => 0.0 }
+            @blockers[hash][:count]      += 1
+            @blockers[hash][:total_time] += request[:duration]
+          end
+
+        when :failed
+          hash = request[:error]
+          @errors[hash] ||= {:count => 0, :exception_strings => {}}
+          @errors[hash][:count] +=1
+          
+          @errors[hash][:exception_strings][request[:exception_string]] ||= 0
+          @errors[hash][:exception_strings][request[:exception_string]] += 1
+      end
+    end
+    
+    # Return a list of requests sorted on a specific action field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
+    def sort_actions_by(field, min_count = nil)
+      actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
+      actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_blockers_by(field, min_count = @blocker_duration)
+      blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
+      blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_errors_by(field, min_count = nil)
+      errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
+      errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+
+    # Calculate the duration of a request
+    # Returns a DateTime object if possible, 0 otherwise.
+    def duration
+      (@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).round : 0
+    end
+    
+    # Check if the request time graph usable data.
+    def request_time_graph?
+      @request_time_graph.uniq != [0] && duration > 0
+    end
+
+  end
+end 

data/lib/ruby-progressbar/progressbar.en.rd

@@ -0,0 +1,103 @@
+=begin
+index:Ej
+
+= Ruby/ProgressBar: A Text Progress Bar Library for Ruby
+
+Last Modified: 2005-05-22 00:28:04
+
+--
+
+Ruby/ProgressBar is a text progress bar library for Ruby.
+It can indicate progress with percentage, a progress bar,
+and estimated remaining time.
+
+The latest version of Ruby/ProgressBar is available at 
+((<URL:http://namazu.org/~satoru/ruby-progressbar/>))
+.
+
+== Examples
+
+  % irb --simple-prompt -r progressbar
+  >> pbar = ProgressBar.new("test", 100)
+  => (ProgressBar: 0/100)
+  >> 100.times {sleep(0.1); pbar.inc}; pbar.finish
+  test:          100% |oooooooooooooooooooooooooooooooooooooooo| Time: 00:00:10
+  => nil
+
+  >> pbar = ProgressBar.new("test", 100)
+  => (ProgressBar: 0/100)
+  >> (1..100).each{|x| sleep(0.1); pbar.set(x)}; pbar.finish
+  test:           67% |oooooooooooooooooooooooooo              | ETA:  00:00:03
+
+== API
+
+--- ProgressBar#new (title, total, out = STDERR)
+    Display the initial progress bar and return a
+    ProgressBar object.  ((|title|)) specifies the title,
+    and ((|total|)) specifies the total cost of processing.
+    Optional parameter ((|out|)) specifies the output IO.
+
+    The display of the progress bar is updated when one or
+    more percent is proceeded or one or more seconds are
+    elapsed from the previous display.
+
+--- ProgressBar#inc (step = 1)
+    Increase the internal counter by ((|step|)) and update
+    the display of the progress bar. Display the estimated
+    remaining time on the right side of the bar. The counter
+    does not go beyond the ((|total|)).
+
+--- ProgressBar#set (count)
+    Set the internal counter to ((|count|)) and update the
+    display of the progress bar. Display the estimated
+    remaining time on the right side of the bar.  Raise if
+    ((|count|)) is a negative number or a number more than
+    the ((|total|)).
+
+--- ProgressBar#finish
+    Stop the progress bar and update the display of progress
+    bar. Display the elapsed time on the right side of the bar.
+    The progress bar always stops at 100 % by the method.
+
+--- ProgressBar#halt
+    Stop the progress bar and update the display of progress
+    bar. Display the elapsed time on the right side of the bar.
+    The progress bar stops at the current percentage by the method.
+
+--- ProgressBar#format=
+    Set the format for displaying a progress bar.
+    Default: "%-14s %3d%% %s %s".
+
+--- ProgressBar#format_arguments=
+    Set the methods for displaying a progress bar.
+    Default: [:title, :percentage, :bar, :stat].
+
+--- ProgressBar#file_transfer_mode
+    Use  :stat_for_file_transfer instead of :stat to display
+    transfered bytes and transfer rate.
+
+
+ReverseProgressBar class is also available.  The
+functionality is identical to ProgressBar but the direction
+of the progress bar is just opposite.
+
+== Limitations
+
+Since the progress is calculated by the proportion to the
+total cost of processing, Ruby/ProgressBar cannot be used if
+the total cost of processing is unknown in advance.
+Moreover, the estimation of remaining time cannot be
+accurately performed if the progress does not flow uniformly.
+
+== Download
+
+Ruby/ProgressBar is a free software with ABSOLUTELY NO WARRANTY
+under the terms of Ruby's license.
+
+  * ((<URL:http://namazu.org/~satoru/ruby-progressbar/ruby-progressbar-0.9.tar.gz>))
+  * ((<URL:http://cvs.namazu.org/ruby-progressbar/>))
+
+--
+
+- ((<Satoru Takabayashi|URL:http://namazu.org/~satoru/>)) -
+=end

data/lib/ruby-progressbar/progressbar.ja.rd

@@ -0,0 +1,100 @@
+=begin
+index:eJ
+
+= Ruby/ProgressBar: �ץ����쥹�С���ƥ����Ȥ�ɽ������ Ruby�ѤΥ饤�֥��
+
+�ǽ�������: 2005-05-22 00:28:53
+
+
+--
+
+Ruby/ProgressBar �ϥץ����쥹�С���ƥ����Ȥ�ɽ������ Ruby��
+�Υ饤�֥��Ǥ��������ο�Ľ������ѡ�����ȡ��ץ����쥹�С���
+����ӿ���Ĥ���֤Ȥ���ɽ�����ޤ���
+
+�ǿ��Ǥ�
+((<URL:http://namazu.org/~satoru/ruby-progressbar/>))
+���������ǽ�Ǥ�
+
+== ������
+
+  % irb --simple-prompt -r progressbar
+  >> pbar = ProgressBar.new("test", 100)
+  => (ProgressBar: 0/100)
+  >> 100.times {sleep(0.1); pbar.inc}; pbar.finish
+  test:          100% |oooooooooooooooooooooooooooooooooooooooo| Time: 00:00:10
+  => nil
+
+  >> pbar = ProgressBar.new("test", 100)
+  => (ProgressBar: 0/100)
+  >> (1..100).each{|x| sleep(0.1); pbar.set(x)}; pbar.finish
+  test:           67% |oooooooooooooooooooooooooo              | ETA:  00:00:03
+
+== API
+
+--- ProgressBar#new (title, total, out = STDERR)
+    �ץ����쥹�С��ν�����֤�ɽ������������ ProgressBar����
+    �������Ȥ��֤���((|title|)) �Ǹ��Ф���((|total|)) �ǽ�
+    �������פ�((|out|)) �ǽ������ IO �����ꤹ�롣
+
+    �ץ����쥹�С���ɽ���ϡ������ɽ�������Ľ�� 1%�ʾ夢��
+    ���Ȥ������뤤�� 1�ðʾ�вᤷ�����˹�������ޤ���
+
+--- ProgressBar#inc (step = 1)
+    �����Υ����󥿤� ((|step|)) �������ʤ�ơ��ץ����쥹�С�
+    ��ɽ���򹹿����롣�С��α�¦�ˤϿ���Ĥ���֤�ɽ�����롣
+    �����󥿤� ((|total|)) ��ۤ��ƿʤळ�ȤϤʤ���
+
+--- ProgressBar#set (count)
+    �����󥿤��ͤ� ((|count|)) �����ꤷ���ץ����쥹�С���
+    ɽ���򹹿����롣�С��α�¦�ˤϿ���Ĥ���֤�ɽ�����롣
+    ((|count|)) �˥ޥ��ʥ����ͤ��뤤�� ((|total|)) ����礭
+    ���ͤ��Ϥ����㳰��ȯ�����롣
+
+--- ProgressBar#finish
+    �ץ����쥹�С�����ߤ����ץ����쥹�С���ɽ���򹹿����롣
+    �ץ����쥹�С��α�¦�ˤϷв���֤�ɽ�����롣
+    ���ΤȤ����ץ����쥹�С��� 100% �ǽ�λ���롣
+
+--- ProgressBar#halt
+    �ץ����쥹�С�����ߤ����ץ����쥹�С���ɽ���򹹿����롣
+    �ץ����쥹�С��α�¦�ˤϷв���֤�ɽ�����롣
+    ���ΤȤ����ץ����쥹�С��Ϥ��λ����Υѡ�����ơ����ǽ�λ���롣
+
+--- ProgressBar#format=
+    �ץ����쥹�С�ɽ���Υե����ޥåȤ����ꤹ�롣
+    ̤�ѹ����� "%-14s %3d%% %s %s"
+
+--- ProgressBar#format_arguments=
+    �ץ����쥹�С�ɽ���˻Ȥ��ؿ������ꤹ�롣
+    ̤�ѹ����� [:title, :percentage, :bar, :stat]
+    �ե�����ž�����ˤ� :stat ���Ѥ��� :stat_for_file_transfer
+    ��Ȥ���ž���Х��ȿ���ž��®�٤�ɽ���Ǥ��롣
+
+--- ProgressBar#file_transfer_mode
+    �ץ����쥹�С�ɽ���� :stat ���Ѥ��� :stat_for_file_transfer
+    ��Ȥ���ž���Х��ȿ���ž��®�٤�ɽ�����롣
+
+
+ReverseProgressBar �Ȥ������饹���󶡤���ޤ�����ǽ��
+ProgressBar �Ȥޤä���Ʊ���Ǥ������ץ����쥹�С��οʹ�������
+�դˤʤäƤ��ޤ���
+
+== ���»���
+
+��Ľ��������������פ��Ф�����Ȥ��Ʒ׻����뤿�ᡢ��������
+�פ������ˤ狼��ʤ������ǤϻȤ��ޤ��󡣤ޤ�����Ľ��ή�줬��
+��Ǥʤ��Ȥ��ˤϻĤ���֤ο�����������Ԥ��ޤ���
+
+== �����������
+
+Ruby �Υ饤���󥹤˽��ä��ե꡼���եȥ������Ȥ��Ƹ������ޤ���
+������̵�ݾڤǤ���
+
+  * ((<URL:http://namazu.org/~satoru/ruby-progressbar/ruby-progressbar-0.9.tar.gz>))
+  * ((<URL:http://cvs.namazu.org/ruby-progressbar/>))
+
+--
+
+- ((<Satoru Takabayashi|URL:http://namazu.org/~satoru/>)) -
+=end