sidekiq-logstash 0.2.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6fe579d57bfb9c6b9852fb75ddb7717c76347491
+  data.tar.gz: c92117609504b57e5802d0b0657ac7117bb39e85
+SHA512:
+  metadata.gz: 89832527fb8baa0f0bcda0d9e95b4ef888617a307abe696c1baa7c789d3df4eb54fe59fcdeefbd68972d20102c82c26289ba84965ed2063a4276e2f0e8524859
+  data.tar.gz: 718580d0fa6d681a3600733be5a6678b3eca8903196a6c84d09570b4a5b68ba01a7ca0d8acf6bb7cbbda3638ebd345107ae1a9878be7cf7878b062ccfb85b165

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea
+*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sidekiq-logstash.gemspec
+gemspec

data/README.md

@@ -0,0 +1,86 @@
+# Sidekiq::Logstash
+
+[<img src="https://img.shields.io/badge/version-0.2.6-green.svg" alt="version" />](https://github.com/iMacTia/sidekiq-logstash) [<img src="https://travis-ci.org/iMacTia/sidekiq-logstash.svg?branch=master" alt="version" />](https://travis-ci.org/iMacTia/sidekiq-logstash)
+
+Sidekiq::Logstash turns your [Sidekiq](https://github.com/mperham/sidekiq) log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
+
+```json
+{
+  "class"           : "MyWorker",
+  "args"            : ["first_param","second_param"],
+  "retry"           : true,
+  "queue"           : "default",
+  "status"          : "fail",
+  "jid"             : "fd71783c0afa3f5e0958f3e9",
+  "created_at"      : "2016-07-02T14:03:26.423Z",
+  "enqueued_at"     : "2016-07-02T14:03:26.425Z",
+  "retried_at"      : "2016-07-02T16:28:42.195Z",
+  "failed_at"       : "2016-07-02T13:04:58.298Z",
+  "retried_at"      : "2016-07-02T14:04:11.051Z",
+  "retry_count"     : 1,
+  "pid"             : 70354,
+  "duration"        : 0.306,
+  "error_message"   : "An error message that occurred during job execution.",
+  "error_backtrace" : "...",
+  "@timestamp"      : "2016-07-02T14:03:27.259Z",
+  "@version"        : "1"
+}
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-logstash'
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install sidekiq-logstash
+```
+
+## Usage
+
+Simply add the following to your sidekiq configuration (in Rails, this will be `initializers/sidekiq.rb`)
+
+```ruby
+Sidekiq::Logstash.setup
+```
+
+I suggest you add it on top of it, before any other `Sidekiq.configure_server` initialization, in order to avoid unformatted logging.
+
+## Configuration
+
+Sidekiq::Logstash allows you to provide custom configuration
+
+```ruby
+Sidekiq::Logstash.configure do |config|
+  # filter_args will allow you to filter the job arguments removing
+  # it works just like rails params filtering (http://guides.rubyonrails.org/action_controller_overview.html#parameters-filtering)
+  config.filter_args << 'foo'
+  
+  # custom_option is a Proc that will be called before logging the payload, allowing you to add fields to it
+  config.custom_options = lambda do |payload|
+    payload['my_custom_field'] = 'my_custom_value'
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/iMacTia/sidekiq-logstash.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'sidekiq/logstash'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/test_console

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'sidekiq/logstash'
+require_relative '../spec/spec_helper'
+FactoryGirl.find_definitions
+
+require 'irb'
+IRB.start

data/lib/sidekiq/logging/argument_filter.rb

@@ -0,0 +1,77 @@
+# This implementation is taken directly from https://github.com/rails/rails/blob/52ce6ece8c8f74064bb64e0a0b1ddd83092718e1/actionpack/lib/action_dispatch/http/parameter_filter.rb
+# Adding actionpack to the gem dependencies would have been too heavy, so here is just what we need.
+
+module Sidekiq
+  module Logging
+    class ArgumentFilter
+      FILTERED = '[FILTERED]'.freeze
+
+      def initialize(filters = [])
+        @filters = filters
+      end
+
+      def filter(args)
+        compiled_filter.call(args)
+      end
+
+      private
+
+      def compiled_filter
+        @compiled_filter ||= CompiledFilter.compile(@filters)
+      end
+
+      class CompiledFilter # :nodoc:
+        def self.compile(filters)
+          return lambda { |args| args.dup } if filters.empty?
+          strings, regexps, blocks = [], [], []
+          filters.each do |item|
+            case item
+              when Proc
+                blocks << item
+              when Regexp
+                regexps << item
+              else
+                strings << Regexp.escape(item.to_s)
+            end
+          end
+          deep_regexps, regexps = regexps.partition { |r| r.to_s.include?("\\.".freeze) }
+          deep_strings, strings = strings.partition { |s| s.include?("\\.".freeze) }
+          regexps << Regexp.new(strings.join('|'.freeze), true) unless strings.empty?
+          deep_regexps << Regexp.new(deep_strings.join('|'.freeze), true) unless deep_strings.empty?
+          new regexps, deep_regexps, blocks
+        end
+
+        attr_reader :regexps, :deep_regexps, :blocks
+
+        def initialize(regexps, deep_regexps, blocks)
+          @regexps = regexps
+          @deep_regexps = deep_regexps.any? ? deep_regexps : nil
+          @blocks  = blocks
+        end
+
+        def call(original_args, parents = [])
+          filtered_args = {}
+          original_args.each do |key, value|
+            parents.push(key) if deep_regexps
+            if regexps.any? { |r| key =~ r }
+              value = FILTERED
+            elsif deep_regexps && (joined = parents.join('.')) && deep_regexps.any? { |r| joined =~ r }
+              value = FILTERED
+            elsif value.is_a?(Hash)
+              value = call(value, parents)
+            elsif value.is_a?(Array)
+              value = value.map { |v| v.is_a?(Hash) ? call(v, parents) : v }
+            elsif blocks.any?
+              key = key.dup rescue key
+              value = value.dup rescue value
+              blocks.each { |b| b.call(key, value) }
+            end
+            parents.pop if deep_regexps
+            filtered_args[key] = value
+          end
+          filtered_args
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/logging/logstash_formatter.rb

@@ -0,0 +1,28 @@
+require 'logstash-event'
+
+module Sidekiq
+  module Logging
+    class LogstashFormatter
+      def call(severity, time, progname, data)
+        if data.is_a? Hash
+          json_data = data
+        else
+          json_data = {
+              severity: severity,
+              message: data
+          }
+        end
+
+        # Merge custom_options to provide customization
+        custom_options.call(json_data) if custom_options rescue nil
+        event = LogStash::Event.new(json_data)
+
+        "#{event.to_json}\n"
+      end
+
+      def custom_options
+        Sidekiq::Logstash.configuration.custom_options
+      end
+    end
+  end
+end

data/lib/sidekiq/logstash/configuration.rb

@@ -0,0 +1,17 @@
+module Sidekiq
+  module Logstash
+    class Configuration
+      attr_accessor :custom_options, :filter_args
+
+      def initialize
+        @filter_args = []
+      end
+
+      # Added to ensure custom_options is a Proc
+      def custom_options=(proc)
+        raise ArgumentError, 'Argument must be a Proc.' unless proc.is_a?(Proc)
+        @custom_options = proc
+      end
+    end
+  end
+end

data/lib/sidekiq/logstash/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Logstash
+    VERSION = '0.2.6'
+  end
+end

data/lib/sidekiq/logstash.rb

@@ -0,0 +1,34 @@
+require 'sidekiq/logstash/configuration'
+require 'sidekiq/logstash/version'
+require 'sidekiq/middleware/server/logstah_logging'
+require 'sidekiq/logging/logstash_formatter'
+require 'sidekiq/logging/argument_filter'
+
+module Sidekiq
+  module Logstash
+    def self.configuration
+      @configuration ||= Configuration.new
+    end
+
+    def self.configure
+      yield(configuration)
+    end
+
+    def self.setup(opts = {})
+      # Calls Sidekiq.configure_server to inject logics
+      Sidekiq.configure_server do |config|
+        # Remove default Sidekiq error_handler that logs errors
+        config.error_handlers.delete_if {|h| h.is_a?(Sidekiq::ExceptionHandler::Logger) }
+
+        # Add logstash support
+        config.server_middleware do |chain|
+          chain.add Sidekiq::Middleware::Server::LogstashLogging
+          chain.remove Sidekiq::Middleware::Server::Logging
+        end
+
+        # Set custom formatter for Sidekiq logger
+        config.logger.formatter = Sidekiq::Logging::LogstashFormatter.new
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/logstah_logging.rb

@@ -0,0 +1,80 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class LogstashLogging
+        def call(_, job, _)
+          started_at = Time.now.utc
+          yield
+          Sidekiq.logger.info log_job(job, started_at)
+        rescue => exc
+          begin
+            Sidekiq.logger.warn log_job(job, started_at, exc)
+          rescue => ex
+            Sidekiq.logger.error 'Error logging the job execution!'
+            Sidekiq.logger.error "Job: #{job}"
+            Sidekiq.logger.error "Job Exception: #{exc}"
+            Sidekiq.logger.error "Log Exception: #{ex}"
+          end
+          raise
+        end
+
+        def log_job(payload, started_at, exc = nil)
+          # Create a copy of the payload using JSON
+          # This should always be possible since Sidekiq store it in Redis
+          payload = JSON.parse(JSON.unparse(payload))
+
+          # Convert timestamps into Time instances
+          %w( created_at enqueued_at retried_at failed_at completed_at ).each do |key|
+            payload[key] = parse_time(payload[key]) if payload[key]
+          end
+
+          # Add process id params
+          payload['pid'] = ::Process.pid
+          payload['duration'] = elapsed(started_at)
+
+          message = "#{payload['class']} JID-#{payload['jid']}"
+
+          if exc
+            payload['message'] = "#{message}: fail: #{payload['duration']} sec"
+            payload['job_status'] = 'fail'
+            payload['error_message'] = exc.message
+            payload['error'] = exc.class
+            payload['error_backtrace'] = %('#{exc.backtrace.join("\n")}')
+          else
+            payload['message'] = "#{message}: done: #{payload['duration']} sec"
+            payload['job_status'] = 'done'
+            payload['completed_at'] = Time.now.utc
+          end
+
+          # Filter sensitive parameters
+          unless filter_args.empty?
+            args_filter = Sidekiq::Logging::ArgumentFilter.new(filter_args)
+            payload['args'] = args_filter.filter({ args: payload['args'] })[:args]
+          end
+
+          # Needs to map all args to strings for ElasticSearch compatibility
+          payload['args'].map!(&:to_s)
+
+          payload
+        end
+
+        def elapsed(start)
+          (Time.now.utc - start).round(3)
+        end
+
+        def parse_time(timestamp)
+          return timestamp if timestamp.is_a? Time
+          timestamp.is_a?(Float) ?
+              Time.at(timestamp).utc :
+              Time.parse(timestamp)
+        rescue
+          timestamp
+        end
+
+        def filter_args
+          Sidekiq::Logstash.configuration.filter_args
+        end
+      end
+    end
+  end
+end

data/sidekiq-logstash.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/logstash/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sidekiq-logstash'
+  spec.version       = Sidekiq::Logstash::VERSION
+  spec.authors       = ['Mattia Giuffrida']
+  spec.email         = ['giuffrida.mattia@gmail.com']
+
+  spec.summary       = %q{Logstash plugin for Sidekiq}
+  spec.description   = <<-DESC
+Sidekiq::Logstash turns your Sidekiq log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
+DESC
+  spec.homepage      = 'https://github.com/iMacTia/sidekiq-logstash'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'logstash-event', '~> 1.2'
+  spec.add_runtime_dependency 'sidekiq', '~> 4.0'
+
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'factory_girl', '~> 4.0'
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-logstash
+version: !ruby/object:Gem::Version
+  version: 0.2.6
+platform: ruby
+authors:
+- Mattia Giuffrida
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2017-03-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-event
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  name: sidekiq
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  name: factory_girl
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: 'Sidekiq::Logstash turns your Sidekiq log into an organised, aggregated,
+  JSON-syntax log ready to be sent to a logstash server.
+
+  '
+email:
+- giuffrida.mattia@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bin/test_console
+- lib/sidekiq/logging/argument_filter.rb
+- lib/sidekiq/logging/logstash_formatter.rb
+- lib/sidekiq/logstash.rb
+- lib/sidekiq/logstash/configuration.rb
+- lib/sidekiq/logstash/version.rb
+- lib/sidekiq/middleware/server/logstah_logging.rb
+- sidekiq-logstash.gemspec
+homepage: https://github.com/iMacTia/sidekiq-logstash
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.10
+signing_key:
+specification_version: 4
+summary: Logstash plugin for Sidekiq
+test_files: []