ngmoco-request-log-analyzer 1.4.2

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -0,0 +1,71 @@
+module RequestLogAnalyzer::FileFormat
+
+  # FileFormat for Amazon S3 access logs.
+  #
+  # Access logs are disabled by default on Amazon S3. To enable logging, see
+  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
+  class AmazonS3 < Base
+
+    line_definition :access do |line|
+      line.header = true
+      line.footer = true
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
+      line.captures << { :name => :bucket_owner,    :type => :string } <<
+                       { :name => :bucket,          :type => :string } <<
+                       { :name => :timestamp,       :type => :timestamp } <<
+                       { :name => :remote_ip,       :type => :string } <<
+                       { :name => :requester,       :type => :string } <<
+                       { :name => :request_id,      :type => :string } <<
+                       { :name => :operation,       :type => :string } <<
+                       { :name => :key,             :type => :nillable_string } <<
+                       { :name => :request_uri,     :type => :string } <<
+                       { :name => :http_status,     :type => :integer } <<
+                       { :name => :error_code,      :type => :nillable_string } <<
+                       { :name => :bytes_sent,      :type => :traffic,  :unit => :byte } <<
+                       { :name => :object_size,     :type => :traffic,  :unit => :byte } <<
+                       { :name => :total_time,      :type => :duration, :unit => :msec } <<
+                       { :name => :turnaround_time, :type => :duration, :unit => :msec } <<
+                       { :name => :referer,         :type => :referer } <<
+                       { :name => :user_agent,      :type => :user_agent }
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
+      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
+      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
+      analyze.frequency :category => :http_status, :title => 'HTTP status codes'
+      analyze.frequency :category => :error_code, :title => 'Error codes'
+    end
+
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # Make sure that the string '-' is parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value == '-' ? nil : value
+      end
+
+      # Can be implemented in subclasses for improved categorizations
+      def convert_referer(value, definition)
+        value == '-' ? nil : value
+      end
+
+      # Can be implemented in subclasses for improved categorizations
+      def convert_user_agent(value, definition)
+        value == '-' ? nil : value
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/file_format/apache.rb

@@ -0,0 +1,141 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The Apache file format is able to log Apache access.log files.
+  #
+  # The access.log can be configured in Apache to have many different formats. In theory, this
+  # FileFormat can handle any format, but it must be aware of the log formatting that is used
+  # by sending the formatting string as parameter to the create method, e.g.:
+  #
+  #     RequestLogAnalyzer::FileFormat::Apache.create('%h %l %u %t "%r" %>s %b')
+  #
+  # It also supports the predefined Apache log formats "common" and "combined". The line
+  # definition and the report definition will be constructed using this file format string.
+  # From the command line, you can provide the format string using the <tt>--apache-format</tt>
+  # command line option.
+  class Apache < Base
+
+    # A hash of predefined Apache log formats
+    LOG_FORMAT_DEFAULTS = {
+      :common   => '%h %l %u %t "%r" %>s %b',
+      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
+      :rack     => '%h %l %u %t "%r" %>s %b %T',
+      :referer  => '%{Referer}i -> %U',
+      :agent    => '%{User-agent}i'
+    }
+
+    # A hash that defines how the log format directives should be parsed.
+    LOG_DIRECTIVES = {
+      '%' => { :regexp => '%', :captures => [] },
+      'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
+      'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
+      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
+      'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
+      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
+      'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
+      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
+      't' => { :regexp => '\[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
+      's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
+      'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
+      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
+      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
+      'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
+               'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
+             }
+    }
+
+    # Creates the Apache log format language based on a Apache log format string.
+    # It will set up the line definition and the report trackers according to the Apache access log format,
+    # which should be passed as first argument. By default, is uses the 'combined' log format.
+    def self.create(*args)
+      access_line = access_line_definition(args.first)
+      trackers = report_trackers(access_line) + report_definer.trackers
+      self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
+    end
+
+    # Creates the access log line definition based on the Apache log format string
+    def self.access_line_definition(format_string)
+      format_string ||= :combined
+      format_string   = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
+
+      line_regexp = ''
+      captures    = []
+      format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
+
+        line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
+
+        if variable
+          # Check if we recognize the log directive
+          directive = LOG_DIRECTIVES[variable]
+          directive = directive[arg] if directive && arg
+
+          if directive
+            line_regexp << directive[:regexp]   # Parse the value of the directive
+            captures    += directive[:captures] # Add the directive's information to the captures
+          else
+            puts "%#{directive} log directiven not yet supported, field is ignored."
+            line_regexp << '.*' # Just accept any input for this literal
+          end
+        end
+      end
+
+      # Return a new line definition object
+      return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
+                                        :captures => captures, :header => true, :footer => true)
+    end
+
+    # Sets up the report trackers according to the fields captured by the access line definition.
+    def self.report_trackers(line_definition)
+      analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+
+      analyze.timespan      if line_definition.captures?(:timestamp)
+      analyze.hourly_spread if line_definition.captures?(:timestamp)
+
+      analyze.frequency :category => :http_method, :title => "HTTP methods"  if line_definition.captures?(:http_method)
+      analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
+      analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs"    if line_definition.captures?(:path)
+
+      analyze.frequency :category => :user_agent, :title => "User agents"    if line_definition.captures?(:user_agent)
+      analyze.frequency :category => :referer,    :title => "Referers"       if line_definition.captures?(:referer)
+
+      analyze.duration :duration => :duration,  :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
+      analyze.traffic  :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic'          if line_definition.captures?(:bytes_sent)
+
+      return analyze.trackers
+    end
+
+    # Define a custom Request class for the Apache file format to speed up timestamp handling.
+    class Request < RequestLogAnalyzer::Request
+
+      def category
+        first(:path)
+      end
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # This function can be overridden to rewrite the path for better categorization in the
+      # reports.
+      def convert_path(value, definition)
+        value
+      end
+
+      # This function can be overridden to simplify the user agent string for better
+      # categorization in the reports
+      def convert_user_agent(value, definition)
+        value # TODO
+      end
+
+      # Make sure that the string '-' is parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value == '-' ? nil : value
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -0,0 +1,67 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The Merb file format parses the request header with the timestamp, the params line
+  # with the most important request information and the durations line which contains
+  # the different request durations that can be used for analysis.
+  class Merb < Base
+
+    # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+    line_definition :started do |line|
+      line.header = true
+      line.teaser = /Started request handling\:/
+      line.regexp = /Started request handling\:\ (.+)/
+      line.captures << { :name => :timestamp, :type => :timestamp }
+    end
+
+    # ~ Params: {"action"=>"create", "controller"=>"session"}
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+    line_definition :params do |line|
+      line.teaser = /Params\:\ /
+      line.regexp = /Params\:\ (\{.+\})/
+      line.captures << { :name => :params, :type => :eval, :provides => {
+            :namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
+    end
+
+    # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+    line_definition :completed do |line|
+      line.footer = true
+      # line.teaser = Regexp.new(Regexp.quote('~ {:'))
+      line.regexp = /(\{.*\:dispatch_time\s*=>\s*\d+\.\d+.*\})/
+      line.captures << { :name => :times_hash, :type => :eval, :provides => {
+            :dispatch_time => :duration, :after_filters_time => :duration,
+            :before_filters_time => :duration, :action_time => :duration } }
+    end
+
+    REQUEST_CATEGORIZER = Proc.new do |request|
+      category = "#{request[:controller]}##{request[:action]}"
+      category = "#{request[:namespace]}::#{category}" if request[:namespace]
+      category = "#{category}.#{request[:format]}"     if request[:format]
+      category
+    end
+
+    report do |analyze|
+
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => "Top 20 by hits"
+      analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
+
+      # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
+      # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
+      # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
+    end
+
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Speed up timestamp conversion
+      def convert_timestamp(value, definition)
+        "#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format/rack.rb

@@ -0,0 +1,11 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Rack < Apache
+
+    def self.create(*args)
+      super(:rack, *args)
+    end
+
+  end
+
+end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -0,0 +1,176 @@
+module RequestLogAnalyzer::FileFormat
+
+  # Default FileFormat class for Rails logs.
+  #
+  # Instances will be created dynamically based on the lines you want it to parse. You can
+  # specify what lines should be included in the parser by providing a list to the create
+  # method as first argument.
+  class Rails < Base
+
+    # Creates a Rails FileFormat instance.
+    #
+    # The lines that will be parsed can be defined by the argument to this function, 
+    # which should be an array of line names, or a list of line names as comma separated
+    # string. The resulting report depends on the lines that will be parsed. You can 
+    # also provide s string that describes a common set of lines, like "production",
+    # "development" or "production".
+    def self.create(lines = 'production')
+      definitions_hash = line_definer.line_definitions.clone
+      
+      lines = lines.to_s.split(',') if lines.kind_of?(String)
+      lines = [lines.to_s]          if lines.kind_of?(Symbol)
+      
+      lines.each do |line|
+        line = line.to_sym
+        if LINE_COLLECTIONS.has_key?(line)
+          LINE_COLLECTIONS[line].each { |l| definitions_hash[l] = LINE_DEFINITIONS[l] }
+        elsif LINE_DEFINITIONS.has_key?(line)
+          definitions_hash[line] = LINE_DEFINITIONS[line]
+        else
+          raise "Unrecognized Rails log line name: #{line.inspect}!"
+        end
+      end
+
+      return self.new(definitions_hash, report_trackers(definitions_hash))
+    end
+    
+    # Creates trackers based on the specified line definitions.
+    #
+    # The more lines that will be parsed, the more information will appear in the report.
+    def self.report_trackers(lines)
+      analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+      
+      analyze.timespan
+      analyze.hourly_spread
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Most requested'
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      
+      if lines.has_key?(:cache_hit)
+        analyze.frequency(:category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, 
+              :title => 'Rails action cache hits')
+      end
+      
+      analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
+        :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+      
+      if lines.has_key?(:failure)
+        analyze.frequency :error, :title => 'Failed requests', :line_type => :failure
+      end
+
+      if lines.has_key?(:rendered)
+        analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true, :title => 'Partial rendering duration'
+      end
+
+      if lines.has_key?(:query_executed)
+        analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true, :title => 'Query duration'
+      end
+      
+      return analyze.trackers + report_definer.trackers
+    end
+
+    # Rails < 2.1 completed line example
+    # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+    RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+
+    # Rails > 2.1 completed line example
+    # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
+    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+
+    # A hash of definitions for all common lines in Rails logs.
+    LINE_DEFINITIONS = {
+      :processing => RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,
+            :teaser   => /Processing /,
+            :regexp   => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
+            :captures => [{ :name => :controller, :type  => :string },
+                          { :name => :action,     :type  => :string },
+                          { :name => :format,     :type  => :string, :default => 'html' },
+                          { :name => :ip,         :type  => :string },
+                          { :name => :timestamp,  :type  => :timestamp },
+                          { :name => :method,     :type  => :string }]),
+
+      :completed => RequestLogAnalyzer::LineDefinition.new(:completed, :footer => true,
+            :teaser   => /Completed in /,
+            :regexp   => Regexp.union(RAILS_21_COMPLETED, RAILS_22_COMPLETED),
+            :captures => [{ :name => :duration, :type => :duration, :unit => :sec },   # First old variant capture
+                          { :name => :view,     :type => :duration, :unit => :sec },
+                          { :name => :db,       :type => :duration, :unit => :sec },
+                          { :name => :status,   :type => :integer },
+                          { :name => :url,      :type => :string },                    # Last old variant capture
+                          { :name => :duration, :type => :duration, :unit => :msec },  # First new variant capture
+                          { :name => :view,     :type => :duration, :unit => :msec },
+                          { :name => :db,       :type => :duration, :unit => :msec },
+                          { :name => :status,   :type => :integer },
+                          { :name => :url,      :type => :string }]),                  # Last new variant capture
+
+      :failure => RequestLogAnalyzer::LineDefinition.new(:failure, :footer => true,
+          :teaser   => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:/,
+          :regexp   => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/,
+          :captures => [{ :name => :error,       :type => :string },
+                        { :name => :message,     :type => :string },
+                        { :name => :line,        :type => :integer },
+                        { :name => :file,        :type => :string }]),
+
+      :cache_hit => RequestLogAnalyzer::LineDefinition.new(:cache_hit,
+          :regexp => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter/),
+
+      :parameters => RequestLogAnalyzer::LineDefinition.new(:parameters,
+          :teaser   => /  Parameters:/,
+          :regexp   => /  Parameters:\s+(\{.*\})/,
+          :captures => [{ :name => :params, :type => :eval }]),
+          
+      :rendered => RequestLogAnalyzer::LineDefinition.new(:rendered,
+          :teaser   => /Rendered /,
+          :regexp   => /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
+          :captures => [{ :name => :render_file,     :type  => :string },
+                        { :name => :render_duration, :type  => :duration, :unit => :msec }]),
+
+      :query_executed => RequestLogAnalyzer::LineDefinition.new(:query_executed,
+          :regexp   => /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/,
+          :captures => [{ :name => :query_class,    :type  => :string },
+                        { :name => :query_duration, :type  => :duration, :unit => :msec },
+                        { :name => :query_sql,      :type  => :sql }]),
+
+      :query_cached => RequestLogAnalyzer::LineDefinition.new(:query_cached,
+          :regexp   => /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/,
+          :captures => [{ :name => :cached_duration, :type  => :duration, :unit => :msec },
+                        { :name => :cached_sql,      :type  => :sql }])
+    }
+
+    # Definitions of common combinations of lines that can be parsed
+    LINE_COLLECTIONS = {
+      :minimal     => [:processing, :completed],
+      :production  => [:processing, :completed, :failure, :cache_hit],
+      :development => [:processing, :completed, :failure, :rendered, :query_executed, :query_cached],
+      :all         => LINE_DEFINITIONS.keys
+    }
+
+
+    # Simple function to categorize Rails requests using controller/actions/format and method.
+    REQUEST_CATEGORIZER = Proc.new do |request|
+      "#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
+    end
+
+    # Define a custom Request class for the Rails file format to speed up timestamp handling
+    # and to ensure that a format is always set.
+    class Request < RequestLogAnalyzer::Request
+
+      # Do not use DateTime.parse
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/, '')[0...14].to_i
+      end
+
+      # Sanitizes SQL queries so that they can be grouped
+      def convert_sql(sql, definition)
+        sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
+      end
+    end
+
+  end
+
+end