ndr_error 1.0.0

data/lib/ndr_error/backtrace_compression.rb

@@ -0,0 +1,73 @@
+module NdrError
+  # Mixin to help compressing/expanding stored backtraces.
+  module BacktraceCompression
+    # Attempts to return just the "application" trace, so we can
+    # highlight "our" code vs. 3rd-party libraries.
+    def application_trace
+      app_trace = backtrace.select do |line|
+        app = line =~ %r{/(#{Rails.root.basename}|current|releases)/}
+        ndr = line =~ %r{/gems/ndr_}
+
+        app || ndr
+      end
+
+      app_trace = backtrace.reject { |line| line =~ %r{/gems/} } if app_trace.blank?
+      app_trace
+    end
+
+    # Returns the backtrace as an array.
+    def backtrace
+      inflate_backtrace(lines)
+    end
+
+    # Stores as much of the backtrace as is possible in a string.
+    # Splits out any nested lines, e.g. TemplateError backtrace.
+    def backtrace=(trace)
+      array = trace ? trace.dup : []
+      array = array.map { |line| line.split("\n") }.flatten
+      dump  = deflate_backtrace(array)
+
+      while dump.length >= 4000
+        array.pop
+        dump = deflate_backtrace(array)
+      end
+
+      self.lines = dump
+    end
+
+    private
+
+    # Helper for wrapping the lines column, enables
+    # the backtrace to be returned as an array rather
+    # that as the stored string. If compression is enabled,
+    # will decompress the string before returing the array.
+    def inflate_backtrace(data)
+      string = data || ''
+
+      if NdrError.compress_backtrace
+        data   = Base64.decode64(string)
+        begin
+          string = Zlib::Inflate.inflate(data)
+        rescue
+          Rails.logger.warn('NdrError: failed to inflate backtrace!')
+        end
+      end
+
+      string.split("\n")
+    end
+
+    # Type conversion helper, so backtrace can be assigned
+    # as an array and saved as a string. If compression is
+    # enabled, this method will compress the string too.
+    def deflate_backtrace(lines)
+      string = lines.join("\n")
+
+      if NdrError.compress_backtrace
+        data   = Zlib::Deflate.deflate(string)
+        string = Base64.encode64(data)
+      end
+
+      string
+    end
+  end
+end

data/lib/ndr_error/engine.rb

@@ -0,0 +1,36 @@
+require 'active_support/all' # TODO: want duration
+
+require 'will_paginate'
+require 'will_paginate/array'
+
+require 'jquery-rails'
+
+module NdrError
+  # Engine configuration goes here; hook in to the host app.
+  class Engine < ::Rails::Engine
+    isolate_namespace NdrError
+
+    # Hook into host app's asset pipeline
+    initializer 'ndr_error.assets.precompile' do |app|
+      app.config.assets.precompile += %w(
+        ndr_error/ndr_error.css
+        ndr_error/ndr_error.js
+      )
+    end
+
+    # Extract context filtering from the host application
+    initializer 'ndr_error.set_default_filtering' do |app|
+      NdrError.filtered_parameters.concat app.config.filter_parameters
+    end
+
+    # Ensure helpers remain visible in development...
+    # See:
+    #
+    #   * http://stackoverflow.com/questions/{9809787,26645033,12191822}
+    #   * https://robots.thoughtbot.com/tips-for-writing-your-own-rails-engine
+    #
+    config.to_prepare do
+      ApplicationController.helper(ApplicationHelper)
+    end
+  end
+end

data/lib/ndr_error/finder.rb

@@ -0,0 +1,47 @@
+module NdrError
+  # Module to help with searching through fingerprints / logs.
+  module Finder
+    def search(keywords)
+      scope = Fingerprint.includes(:error_logs)
+      return scope unless keywords && keywords.any?
+
+      # Fetch the collection of fingerprint matches:
+      records = scope.filter_by_keywords(keywords).to_a
+
+      # Add to that fingerprints with log records that matched the search:
+      log_print_ids = Log.filter_by_keywords(keywords).pluck(:error_fingerprintid)
+      records.concat scope.find(log_print_ids)
+
+      order(records)
+    end
+
+    # Proxy to paginate fingerprint results, filtering them
+    # if search keywords have been supplied.
+    def paginate(keywords, page)
+      search(keywords).paginate(page: page, per_page: Fingerprint.per_page)
+    end
+
+    # Sends finds through to the fingerprint resource.
+    def find(id)
+      Fingerprint.find(id)
+    end
+
+    private
+
+    # Intelligent search order:
+    #
+    #   Bring records that have matched multiple times
+    #   to the front, but attempt to maintain the
+    #   created_at DESC order within groups.
+    #
+    def order(records)
+      grouped = records.group_by { |record| records.count(record) }.each do |_count, group|
+        group.sort! do |a, b|
+          [b.updated_at, b.error_fingerprintid] <=> [a.updated_at, a.error_fingerprintid]
+        end
+      end
+
+      grouped.values.flatten.uniq
+    end
+  end
+end

data/lib/ndr_error/fuzzing.rb

@@ -0,0 +1,51 @@
+module NdrError
+  # Mixin to help with fuzzing of exception messages/traces.
+  module Fuzzing
+    def fuzz(description, backtrace)
+      Digest::MD5.hexdigest(fuzz_description(description) + fuzz_backtrace(backtrace))
+    end
+
+    private
+
+    # Prepare a fuzzed description:
+    def fuzz_description(description)
+      # Apply the fuzzers sequentially:
+      NdrError.description_fuzzers.inject(description) { |a, e| e.call(a) }
+    end
+
+    # Fuzz a backtrace:
+    #   * independent of deployment paths
+    #   * independent of line numbers
+    def fuzz_backtrace(backtrace)
+      backtrace.map { |line| fuzz_line(line) }.join("\n")
+    end
+
+    def fuzz_line(line)
+      line = line.sub(Rails.root.to_s, 'Rails.root')
+
+      # Remove gem version numbers from backtrace
+      line.sub!(Regexp.new('/gems/([^/]*-)[0-9][0-9.]*/'), '/gems/\1/')
+
+      # Remove GEM_PATH / LOAD_PATH differences:
+      sub_paths!(line)
+
+      # Remove Rails compiled callback & template identifiers:
+      line.gsub!(Regexp.new('___?[0-9][0-9_]*[0-9]'), '__COMPILED_ID')
+
+      # Remove line numbers:
+      line.gsub!(/:(\d+):/, '')
+
+      line
+    end
+
+    def sub_paths!(line)
+      Gem.path.each do |path|
+        line.sub!(Regexp.new('^' + Regexp.escape(path)), 'Gem.path')
+      end
+
+      $LOAD_PATH.each do |path|
+        line.sub!(Regexp.new('^' + Regexp.escape(path)), '$LOAD_PATH')
+      end
+    end
+  end
+end

data/lib/ndr_error/logging.rb

@@ -0,0 +1,36 @@
+module NdrError
+  # Module to contain helpers for logging
+  module Logging
+    # Which attributes can be populated when manually logging an exception:
+    ANCILLARY_ATTRS_WHITELIST = [:user_id, :user_roles, :svn_revision]
+
+    # Log the given `exception`.
+    def log(exception, ancillary_data, request_object)
+      log = initialize_log(ancillary_data)
+      log.register_exception(exception)
+      log.register_request(request_object)
+
+      print = Fingerprint.find_or_create_by_id(log.md5_digest)
+      error = print.store_log(log)
+
+      [print, error]
+    end
+
+    private
+
+    # Manual attribute whitelisting:
+    def initialize_log(ancillary_data)
+      Log.new.tap do |log|
+        ancillary_data.symbolize_keys.each do |key, value|
+          fail "Mass-assigning #{key} is forbidden!" unless ANCILLARY_ATTRS_WHITELIST.include?(key)
+
+          if ActiveRecord::Base.respond_to?(:protected_attributes)
+            log.assign_attributes({ key => value }, without_protection: true)
+          else
+            log.assign_attributes(key => value)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ndr_error/middleware/public_exceptions.rb

@@ -0,0 +1,41 @@
+module NdrError
+  module Middleware
+    # Middleware for logging exceptions, can be used as exception_app for Rails.
+    class PublicExceptions < ::ActionDispatch::PublicExceptions
+      def call(env)
+        rescuing_everything do
+          request   = ActionDispatch::Request.new(env)
+          exception = env['action_dispatch.exception']
+
+          # "falsey" callback return value allows logging to be skipped
+          log_exception(request, exception) if run_exception_callback(request, exception)
+        end
+
+        super # Invoke the PublicExceptions behaviour
+      end
+
+      private
+
+      def log_exception(request, exception)
+        parameters = NdrError.log_parameters.call(request)
+        _fingerprint, _log = NdrError.log(exception, parameters, request)
+      end
+
+      def run_exception_callback(request, exception)
+        NdrError.exception_app_callback.call(request, exception)
+      end
+
+      # Unhandled exceptions with logging could terminate the web server!
+      def rescuing_everything
+        yield
+      rescue Exception => exception # rubocop:disable Lint/RescueException
+        # "Log the exception caused by logging an exception..."
+        Rails.logger.warn <<-MSG.strip_heredoc
+          NdrError failed to log an exception!
+            logging error class:   #{exception.class}
+            logging error message: #{exception.message}
+        MSG
+      end
+    end
+  end
+end

data/lib/ndr_error/uuid_builder.rb

@@ -0,0 +1,46 @@
+module NdrError
+  # Mixin to assist with building of UUIDs.
+  module UuidBuilder
+    # Populate the primary key column. Taking inspiration
+    # from MongoDB, hashes time, machine identifier etc
+    # so can be used for ordering at a pinch.
+    def construct_uuid
+      parts = [
+        [Time.current.to_i].pack('N'),                # 4 bytes of current time seconds
+        Digest::MD5.digest(Socket.gethostname)[0, 7], # 7 bytes of machine identifier
+        [Process.pid % 0xFFFF].pack('n'),             # 2 bytes of pid (looping at 2^16)
+        [increment_value].pack('N')[1, 3]             # 3 bytes of incrementing counter
+      ]
+
+      format pad_parts(parts)
+    end
+
+    private
+
+    # Pad out the components as hex:
+    def pad_parts(parts)
+      parts.join.unpack('C16').map do |piece|
+        piece.to_s(16).tap { |hex| hex.prepend('0') if hex.size == 1 }
+      end.join
+    end
+
+    def format(string)
+      string.sub(/(.{8})(.{14})(.{4})(.{6})/) do
+        time, machine, pid, counter = Regexp.last_match[1..4]
+        "#{time}-#{machine}-0#{pid}-#{counter}"
+      end
+    end
+
+    # rubocop:disable Style/ClassVars
+    @@_lock  = Mutex.new
+    @@_index = 0
+
+    # Used in UUID generation.
+    def increment_value
+      @@_lock.synchronize do
+        @@_index = (@@_index + 1) % 0xFFFFFF
+      end
+    end
+    # rubocop:enable Style/ClassVars
+  end
+end

data/lib/ndr_error/version.rb

@@ -0,0 +1,4 @@
+# Contains the version of NdrError. Sourced by the gemspec.
+module NdrError
+  VERSION = '1.0.0'
+end

data/lib/ndr_error.rb

@@ -0,0 +1,87 @@
+require 'ndr_error/engine'
+
+require 'ndr_error/backtrace_compression'
+require 'ndr_error/finder'
+require 'ndr_error/fuzzing'
+require 'ndr_error/logging'
+require 'ndr_error/uuid_builder'
+
+# Configuration for NdrError + convienence methods.
+module NdrError
+  extend Finder
+  extend Logging
+
+  # Callable object, that by default is called by the NdrError::Middleware::PublicExceptions
+  # middleware (with the request and exception objects) when an exception is handled.
+  # Returning a "falsey" value will prevent the exception from being logged.
+  mattr_accessor :exception_app_callback
+  self.exception_app_callback = ->(_request, _exception) { true }
+
+  # Callable object (called with controller context) that is used to check
+  # if the current user is authenticated with the host app.
+  mattr_accessor :check_current_user_authentication
+  self.check_current_user_authentication = nil # Must be configured by the host app
+
+  # Callable object (called with controller context) that is used to check
+  # update/destroy permissions of the current session. Defaults to true.
+  mattr_accessor :check_current_user_permissions
+  self.check_current_user_permissions = ->(_context) { true }
+
+  # The name of the error_log column that corresponds to the user.
+  mattr_accessor :user_column
+  self.user_column = nil # Must be configured by the host app
+
+  # A callable object that returns a hash of current context (e.g. user).
+  # The host app must at least set the :user_column to a value. If bundled
+  # NdrError::Middleware::PublicExceptions middleware is used, it is called
+  # with the Rack request object.
+  mattr_accessor :log_parameters
+  self.log_parameters = ->(_request) { Hash.new }
+
+  # Request parameters that we should not be capturing as part of the error context.
+  mattr_accessor :filtered_parameters
+  self.filtered_parameters = [] # Populated by a railtie
+
+  # Callable object, which will give the app server hostname:
+  mattr_accessor :hostname_identifier
+  self.hostname_identifier = -> { 'unknown host' }
+
+  # Callable object, which will give information about the current data
+  mattr_accessor :database_identifier
+  self.database_identifier = -> { 'unknown database' }
+
+  # Callable object, which will return the current database time.
+  mattr_accessor :database_time_checker
+  self.database_time_checker = -> { nil }
+
+  # The number of logs that are kept before only a counter is incremented
+  mattr_accessor :fingerprint_threshold
+  self.fingerprint_threshold = 100
+
+  # How long to logs remain soft-deleted, before they are cleaned up?
+  mattr_accessor :log_grace_period
+  self.log_grace_period = 90.days
+
+  # Should the captured backtrace be compressed?
+  mattr_accessor :compress_backtrace
+  self.compress_backtrace = true
+
+  # Callable objects, which fuzz an Exception's message for hashing.
+  mattr_accessor :description_fuzzers
+  self.description_fuzzers = [
+    ->(desc) { desc.sub('...[truncated]', '') }, # column length truncation
+    ->(desc) { desc.gsub(/(undefined method `[^']*' for )(.*)/, '\1') }, # obj details
+    ->(desc) { desc.gsub(/([0-9a-fA-F]*\d?[0-9a-fA-F]*)/, '') } # remove hex
+  ]
+
+  # Optionally, a regular expression against which the URL of
+  # fingerprint tickets can be validated.
+  mattr_accessor :ticket_url_format
+  self.ticket_url_format = nil
+
+  # Help Rails find the table of any
+  # namespaced modules.
+  def self.table_name_prefix
+    'error_'
+  end
+end

data/lib/tasks/ndr_error_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :ndr_error do
+#   # Task goes here
+# end

data/test/dummy/README.rdoc

@@ -0,0 +1,261 @@
+== Welcome to Rails
+
+Rails is a web-application framework that includes everything needed to create
+database-backed web applications according to the Model-View-Control pattern.
+
+This pattern splits the view (also called the presentation) into "dumb"
+templates that are primarily responsible for inserting pre-built data in between
+HTML tags. The model contains the "smart" domain objects (such as Account,
+Product, Person, Post) that holds all the business logic and knows how to
+persist themselves to a database. The controller handles the incoming requests
+(such as Save New Account, Update Product, Show Post) by manipulating the model
+and directing data to the view.
+
+In Rails, the model is handled by what's called an object-relational mapping
+layer entitled Active Record. This layer allows you to present the data from
+database rows as objects and embellish these data objects with business logic
+methods. You can read more about Active Record in
+link:files/vendor/rails/activerecord/README.html.
+
+The controller and view are handled by the Action Pack, which handles both
+layers by its two parts: Action View and Action Controller. These two layers
+are bundled in a single package due to their heavy interdependence. This is
+unlike the relationship between the Active Record and Action Pack that is much
+more separate. Each of these packages can be used independently outside of
+Rails. You can read more about Action Pack in
+link:files/vendor/rails/actionpack/README.html.
+
+
+== Getting Started
+
+1. At the command prompt, create a new Rails application:
+       <tt>rails new myapp</tt> (where <tt>myapp</tt> is the application name)
+
+2. Change directory to <tt>myapp</tt> and start the web server:
+       <tt>cd myapp; rails server</tt> (run with --help for options)
+
+3. Go to http://localhost:3000/ and you'll see:
+       "Welcome aboard: You're riding Ruby on Rails!"
+
+4. Follow the guidelines to start developing your application. You can find
+the following resources handy:
+
+* The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html
+* Ruby on Rails Tutorial Book: http://www.railstutorial.org/
+
+
+== Debugging Rails
+
+Sometimes your application goes wrong. Fortunately there are a lot of tools that
+will help you debug it and get it back on the rails.
+
+First area to check is the application log files. Have "tail -f" commands
+running on the server.log and development.log. Rails will automatically display
+debugging and runtime information to these files. Debugging info will also be
+shown in the browser on requests from 127.0.0.1.
+
+You can also log your own messages directly into the log file from your code
+using the Ruby logger class from inside your controllers. Example:
+
+  class WeblogController < ActionController::Base
+    def destroy
+      @weblog = Weblog.find(params[:id])
+      @weblog.destroy
+      logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!")
+    end
+  end
+
+The result will be a message in your log file along the lines of:
+
+  Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1!
+
+More information on how to use the logger is at http://www.ruby-doc.org/core/
+
+Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are
+several books available online as well:
+
+* Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe)
+* Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide)
+
+These two books will bring you up to speed on the Ruby language and also on
+programming in general.
+
+
+== Debugger
+
+Debugger support is available through the debugger command when you start your
+Mongrel or WEBrick server with --debugger. This means that you can break out of
+execution at any point in the code, investigate and change the model, and then,
+resume execution! You need to install ruby-debug to run the server in debugging
+mode. With gems, use <tt>sudo gem install ruby-debug</tt>. Example:
+
+  class WeblogController < ActionController::Base
+    def index
+      @posts = Post.all
+      debugger
+    end
+  end
+
+So the controller will accept the action, run the first line, then present you
+with a IRB prompt in the server window. Here you can do things like:
+
+  >> @posts.inspect
+  => "[#<Post:0x14a6be8
+          @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>,
+       #<Post:0x14a6620
+          @attributes={"title"=>"Rails", "body"=>"Only ten..", "id"=>"2"}>]"
+  >> @posts.first.title = "hello from a debugger"
+  => "hello from a debugger"
+
+...and even better, you can examine how your runtime objects actually work:
+
+  >> f = @posts.first
+  => #<Post:0x13630c4 @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>
+  >> f.
+  Display all 152 possibilities? (y or n)
+
+Finally, when you're ready to resume execution, you can enter "cont".
+
+
+== Console
+
+The console is a Ruby shell, which allows you to interact with your
+application's domain model. Here you'll have all parts of the application
+configured, just like it is when the application is running. You can inspect
+domain models, change values, and save to the database. Starting the script
+without arguments will launch it in the development environment.
+
+To start the console, run <tt>rails console</tt> from the application
+directory.
+
+Options:
+
+* Passing the <tt>-s, --sandbox</tt> argument will rollback any modifications
+  made to the database.
+* Passing an environment name as an argument will load the corresponding
+  environment. Example: <tt>rails console production</tt>.
+
+To reload your controllers and models after launching the console run
+<tt>reload!</tt>
+
+More information about irb can be found at:
+link:http://www.rubycentral.org/pickaxe/irb.html
+
+
+== dbconsole
+
+You can go to the command line of your database directly through <tt>rails
+dbconsole</tt>. You would be connected to the database with the credentials
+defined in database.yml. Starting the script without arguments will connect you
+to the development database. Passing an argument will connect you to a different
+database, like <tt>rails dbconsole production</tt>. Currently works for MySQL,
+PostgreSQL and SQLite 3.
+
+== Description of Contents
+
+The default directory structure of a generated Ruby on Rails application:
+
+  |-- app
+  |   |-- assets
+  |   |   |-- images
+  |   |   |-- javascripts
+  |   |   `-- stylesheets
+  |   |-- controllers
+  |   |-- helpers
+  |   |-- mailers
+  |   |-- models
+  |   `-- views
+  |       `-- layouts
+  |-- config
+  |   |-- environments
+  |   |-- initializers
+  |   `-- locales
+  |-- db
+  |-- doc
+  |-- lib
+  |   |-- assets
+  |   `-- tasks
+  |-- log
+  |-- public
+  |-- script
+  |-- test
+  |   |-- fixtures
+  |   |-- functional
+  |   |-- integration
+  |   |-- performance
+  |   `-- unit
+  |-- tmp
+  |   `-- cache
+  |       `-- assets
+  `-- vendor
+      |-- assets
+      |   |-- javascripts
+      |   `-- stylesheets
+      `-- plugins
+
+app
+  Holds all the code that's specific to this particular application.
+
+app/assets
+  Contains subdirectories for images, stylesheets, and JavaScript files.
+
+app/controllers
+  Holds controllers that should be named like weblogs_controller.rb for
+  automated URL mapping. All controllers should descend from
+  ApplicationController which itself descends from ActionController::Base.
+
+app/models
+  Holds models that should be named like post.rb. Models descend from
+  ActiveRecord::Base by default.
+
+app/views
+  Holds the template files for the view that should be named like
+  weblogs/index.html.erb for the WeblogsController#index action. All views use
+  eRuby syntax by default.
+
+app/views/layouts
+  Holds the template files for layouts to be used with views. This models the
+  common header/footer method of wrapping views. In your views, define a layout
+  using the <tt>layout :default</tt> and create a file named default.html.erb.
+  Inside default.html.erb, call <% yield %> to render the view using this
+  layout.
+
+app/helpers
+  Holds view helpers that should be named like weblogs_helper.rb. These are
+  generated for you automatically when using generators for controllers.
+  Helpers can be used to wrap functionality for your views into methods.
+
+config
+  Configuration files for the Rails environment, the routing map, the database,
+  and other dependencies.
+
+db
+  Contains the database schema in schema.rb. db/migrate contains all the
+  sequence of Migrations for your schema.
+
+doc
+  This directory is where your application documentation will be stored when
+  generated using <tt>rake doc:app</tt>
+
+lib
+  Application specific libraries. Basically, any kind of custom code that
+  doesn't belong under controllers, models, or helpers. This directory is in
+  the load path.
+
+public
+  The directory available for the web server. Also contains the dispatchers and the
+  default HTML files. This should be set as the DOCUMENT_ROOT of your web
+  server.
+
+script
+  Helper scripts for automation and generation.
+
+test
+  Unit and functional tests along with fixtures. When using the rails generate
+  command, template test files will be generated for you and placed in this
+  directory.
+
+vendor
+  External libraries that the application depends on. Also includes the plugins
+  subdirectory. If the app has frozen rails, those gems also go here, under
+  vendor/rails/. This directory is in the load path.