loga 2.5.1 → 2.6.0

data/spec/fixtures/rails61.rb

@@ -0,0 +1,80 @@
+require 'action_controller/railtie'
+require 'action_mailer/railtie'
+
+Bundler.require(*Rails.groups)
+
+STREAM = StringIO.new unless defined?(STREAM)
+
+class Dummy < Rails::Application
+  config.eager_load = true
+  config.filter_parameters += [:password]
+  config.secret_key_base = '2624599ca9ab3cf3823626240138a128118a87683bf03ab8f155844c33b3cd8cbbfa3ef5e29db6f5bd182f8bd4776209d9577cfb46ac51bfd232b00ab0136b24'
+  config.session_store :cookie_store, key: '_rails61_session'
+
+  config.log_tags = [:uuid, 'TEST_TAG']
+  config.loga = {
+    device: STREAM,
+    host: 'bird.example.com',
+    service_name: 'hello_world_app',
+    service_version: '1.0',
+  }
+  config.action_mailer.delivery_method = :test
+end
+
+class ApplicationController < ActionController::Base
+  include Rails.application.routes.url_helpers
+  protect_from_forgery with: :null_session
+
+  def ok
+    render plain: 'Hello Rails'
+  end
+
+  def error
+    nil.name
+  end
+
+  def show
+    render json: params
+  end
+
+  def create
+    render json: params
+  end
+
+  def new
+    redirect_to :ok
+  end
+
+  def update
+    @id = params[:id]
+    render '/user'
+  end
+end
+
+class FakeMailer < ActionMailer::Base
+  default from: 'notifications@example.com'
+
+  def self.send_email
+    basic_mail.deliver_now
+  end
+
+  def basic_mail
+    mail(
+      to: 'user@example.com',
+      subject: 'Welcome to My Awesome Site',
+      body: 'Banana muffin',
+      content_type: 'text/html',
+    )
+  end
+end
+
+Dummy.routes.append do
+  get 'ok'        => 'application#ok'
+  get 'error'     => 'application#error'
+  get 'show'      => 'application#show'
+  post 'users'    => 'application#create'
+  get 'new'       => 'application#new'
+  put 'users/:id' => 'application#update'
+end
+
+Dummy.initialize!

data/spec/fixtures/rails70.rb

@@ -0,0 +1,80 @@
+require 'action_controller/railtie'
+require 'action_mailer/railtie'
+
+Bundler.require(*Rails.groups)
+
+STREAM = StringIO.new unless defined?(STREAM)
+
+class Dummy < Rails::Application
+  config.eager_load = true
+  config.filter_parameters += [:password]
+  config.secret_key_base = '2624599ca9ab3cf3823626240138a128118a87683bf03ab8f155844c33b3cd8cbbfa3ef5e29db6f5bd182f8bd4776209d9577cfb46ac51bfd232b00ab0136b24'
+  config.session_store :cookie_store, key: '_rails70_session'
+
+  config.log_tags = [:uuid, 'TEST_TAG']
+  config.loga = {
+    device: STREAM,
+    host: 'bird.example.com',
+    service_name: 'hello_world_app',
+    service_version: '1.0',
+  }
+  config.action_mailer.delivery_method = :test
+end
+
+class ApplicationController < ActionController::Base
+  include Rails.application.routes.url_helpers
+  protect_from_forgery with: :null_session
+
+  def ok
+    render plain: 'Hello Rails'
+  end
+
+  def error
+    nil.name
+  end
+
+  def show
+    render json: params
+  end
+
+  def create
+    render json: params
+  end
+
+  def new
+    redirect_to :ok
+  end
+
+  def update
+    @id = params[:id]
+    render '/user'
+  end
+end
+
+class FakeMailer < ActionMailer::Base
+  default from: 'notifications@example.com'
+
+  def self.send_email
+    basic_mail.deliver_now
+  end
+
+  def basic_mail
+    mail(
+      to: 'user@example.com',
+      subject: 'Welcome to My Awesome Site',
+      body: 'Banana muffin',
+      content_type: 'text/html',
+    )
+  end
+end
+
+Dummy.routes.append do
+  get 'ok'        => 'application#ok'
+  get 'error'     => 'application#error'
+  get 'show'      => 'application#show'
+  post 'users'    => 'application#create'
+  get 'new'       => 'application#new'
+  put 'users/:id' => 'application#update'
+end
+
+Dummy.initialize!

data/spec/integration/sidekiq5_spec.rb

@@ -0,0 +1,143 @@
+require 'spec_helper'
+require 'timecop'
+require 'fakeredis'
+
+dummy_redis_config = ConnectionPool.new(size: 5) { Redis.new }
+
+Sidekiq.configure_client do |config|
+  config.redis = dummy_redis_config
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = dummy_redis_config
+end
+
+class MySidekiqWorker
+  include Sidekiq::Worker
+
+  def perform(_name); end
+end
+
+describe 'Sidekiq client logger' do
+  let(:mgr) do
+    Class.new do
+      attr_reader :latest_error, :mutex, :cond
+
+      def initialize
+        @mutex = ::Mutex.new
+        @cond = ::ConditionVariable.new
+      end
+
+      def processor_died(_inst, err)
+        @latest_error = err
+
+        @mutex.synchronize { @cond.signal }
+      end
+
+      def processor_stopped(_inst)
+        @mutex.synchronize { @cond.signal }
+      end
+
+      def options
+        {
+          concurrency: 3,
+          queues: ['default'],
+          job_logger: Loga::Sidekiq5::JobLogger,
+        }
+      end
+    end
+  end
+
+  let(:target) { StringIO.new }
+
+  def read_json_log(line:)
+    target.rewind
+    JSON.parse(target.each_line.drop(line - 1).first)
+  end
+
+  before do
+    Redis.current.flushall
+
+    Loga.reset
+
+    Loga.configure(
+      service_name: 'hello_world_app',
+      service_version: '1.0',
+      device: target,
+      format: :gelf,
+    )
+  end
+
+  it 'has the proper job logger' do
+    expect(Sidekiq.options[:job_logger]).to eq Loga::Sidekiq5::JobLogger
+  end
+
+  it 'has the proper logger for Sidekiq.logger' do
+    expect(Sidekiq.logger).to eq Loga.logger
+  end
+
+  it 'pushes a new element in the default queue' do
+    MySidekiqWorker.perform_async('Bob')
+
+    last_element = JSON.parse(Redis.current.lpop('queue:default'))
+
+    aggregate_failures do
+      expect(last_element['class']).to eq 'MySidekiqWorker'
+      expect(last_element['args']).to eq ['Bob']
+      expect(last_element['retry']).to eq true
+      expect(last_element['queue']).to eq 'default'
+    end
+  end
+
+  it 'has the proper logger Sidekiq::Logging.logger' do
+    expect(Sidekiq::Logging.logger).to eq Loga.logger
+  end
+
+  # https://github.com/mperham/sidekiq/blob/97363210b47a4f8a1d8c1233aaa059d6643f5040/test/test_actors.rb#L57-L79
+
+  it 'logs the job processing event' do
+    MySidekiqWorker.perform_async('Bob')
+
+    require 'sidekiq/processor'
+
+    Sidekiq::Processor.new(mgr.new).start
+    sleep 0.5
+
+    json_line = read_json_log(line: 1)
+
+    aggregate_failures do
+      expect(json_line).to include(
+        '_queue'=> 'default',
+        '_retry'=> true,
+        '_params'=> ['Bob'],
+        '_class'=> 'MySidekiqWorker',
+        '_type'=> 'sidekiq',
+        '_service.name'=> 'hello_world_app',
+        '_service.version'=> '1.0',
+        '_tags'=> '',
+        'level'=> 6,
+        'version'=> '1.1',
+      )
+
+      %w[_created_at _enqueued_at _jid _duration timestamp host].each do |key|
+        expect(json_line).to have_key(key)
+      end
+
+      expect(json_line['short_message']).to match(/MySidekiqWorker with jid:*/)
+    end
+
+    # This was a bug - the duration was constantly incresing based on when
+    # the logger was created. https://github.com/FundingCircle/loga/pull/117
+    #
+    # Test that after sleeping for few seconds the duration is still under 500ms
+    sleep 1
+
+    MySidekiqWorker.perform_async('Bob')
+
+    sleep 1
+
+    json_line = read_json_log(line: 2)
+
+    expect(json_line['_duration']).to be < 500
+  end
+end

data/spec/integration/sidekiq6_spec.rb

@@ -0,0 +1,164 @@
+require 'spec_helper'
+require 'timecop'
+require 'fakeredis'
+
+dummy_redis_config = ConnectionPool.new(size: 5) { Redis.new }
+
+Sidekiq.configure_client do |config|
+  config.redis = dummy_redis_config
+end
+
+Sidekiq.configure_server do |config|
+  config.redis = dummy_redis_config
+end
+
+class MySidekiqWorker
+  include Sidekiq::Worker
+
+  def perform(_name)
+    logger.info('Hello from MySidekiqWorker')
+  end
+end
+
+describe 'Sidekiq client logger' do
+  let(:mgr) do
+    # https://github.com/mperham/sidekiq/blob/v6.1.2/test/test_actors.rb#L58-L82
+    Class.new do
+      attr_reader :latest_error, :mutex, :cond
+
+      def initialize
+        @mutex = ::Mutex.new
+        @cond = ::ConditionVariable.new
+      end
+
+      def processor_died(_inst, err)
+        @latest_error = err
+
+        @mutex.synchronize { @cond.signal }
+      end
+
+      def processor_stopped(_inst)
+        @mutex.synchronize { @cond.signal }
+      end
+
+      def options
+        {
+          concurrency: 3,
+          queues: ['default'],
+          job_logger: Loga::Sidekiq6::JobLogger,
+        }.tap { |opts| opts[:fetch] = ::Sidekiq::BasicFetch.new(opts) }
+      end
+    end
+  end
+
+  let(:target) { StringIO.new }
+
+  def read_json_log(line:)
+    target.rewind
+    JSON.parse(target.each_line.drop(line - 1).first)
+  end
+
+  before do
+    Redis.current.flushall
+
+    Loga.reset
+
+    Loga.configure(
+      service_name: 'hello_world_app',
+      service_version: '1.0',
+      device: target,
+      format: :gelf,
+    )
+  end
+
+  it 'has the proper job logger' do
+    expect(Sidekiq.options[:job_logger]).to eq Loga::Sidekiq6::JobLogger
+  end
+
+  it 'has the proper logger for Sidekiq.logger' do
+    expect(Sidekiq.logger).to eq Loga.logger
+  end
+
+  it 'pushes a new element in the default queue' do
+    MySidekiqWorker.perform_async('Bob')
+
+    last_element = JSON.parse(Redis.current.lpop('queue:default'))
+
+    aggregate_failures do
+      expect(last_element['class']).to eq 'MySidekiqWorker'
+      expect(last_element['args']).to eq ['Bob']
+      expect(last_element['retry']).to eq true
+      expect(last_element['queue']).to eq 'default'
+    end
+  end
+
+  def test_log_from_worker(json_line)
+    aggregate_failures do
+      expect(json_line).to include(
+        '_class' => 'MySidekiqWorker',
+        '_service.name' => 'hello_world_app',
+        '_service.version' => '1.0',
+        '_tags' => '',
+        'level' => 6,
+        'version' => '1.1',
+        'short_message' => 'Hello from MySidekiqWorker',
+      )
+
+      %w[_jid timestamp host].each do |key|
+        expect(json_line).to have_key(key)
+      end
+
+      expect(json_line).not_to include('_duration')
+    end
+  end
+
+  def test_job_end_log(json_line) # rubocop:disable Metrics/MethodLength
+    aggregate_failures do
+      expect(json_line).to include(
+        '_queue' => 'default',
+        '_retry' => true,
+        '_params' => ['Bob'],
+        '_class' => 'MySidekiqWorker',
+        '_type' => 'sidekiq',
+        '_service.name' => 'hello_world_app',
+        '_service.version' => '1.0',
+        '_tags' => '',
+        'level' => 6,
+        'version' => '1.1',
+      )
+
+      %w[_created_at _enqueued_at _jid _duration timestamp host].each do |key|
+        expect(json_line).to have_key(key)
+      end
+
+      expect(json_line['_duration']).to be < 500
+      expect(json_line['short_message']).to match(/MySidekiqWorker with jid:*/)
+    end
+  end
+
+  it 'logs the job processing event' do
+    MySidekiqWorker.perform_async('Bob')
+
+    require 'sidekiq/processor'
+
+    sidekiq_manager = mgr.new
+    Sidekiq::Processor.new(sidekiq_manager, sidekiq_manager.options).start
+    sleep 0.5
+
+    test_log_from_worker(read_json_log(line: 1))
+    test_job_end_log(read_json_log(line: 2))
+
+    # This was a bug - the duration was constantly incresing based on when
+    # the logger was created. https://github.com/FundingCircle/loga/pull/117
+    #
+    # Test that after sleeping for few seconds the duration is still under 500ms
+    sleep 1
+
+    MySidekiqWorker.perform_async('Bob')
+
+    sleep 1
+
+    test_log_from_worker(read_json_log(line: 3))
+    test_job_end_log(read_json_log(line: 4))
+  end
+end

data/spec/integration/sinatra_spec.rb

@@ -68,7 +68,6 @@ RSpec.describe 'Structured logging with Sinatra', :with_hostname, :timecop do
     end
     let(:data) do
       {
-        'status' => 200,
         'method' => 'GET',
         'path'   => '/ok',
         'params' => { 'username'=>'yoshi' },
@@ -76,6 +75,7 @@ RSpec.describe 'Structured logging with Sinatra', :with_hostname, :timecop do
         'request_ip' => '127.0.0.1',
         'user_agent' => nil,
         'duration'   => 0,
+        'status' => 200,
       }
     end
     let(:data_as_text)  { "data=#{{ request: data }.inspect}" }

data/spec/loga/{sidekiq → sidekiq5}/job_logger_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
+require 'loga/sidekiq5/job_logger'
 
-RSpec.describe Loga::Sidekiq::JobLogger do
+RSpec.describe Loga::Sidekiq5::JobLogger do
   subject(:job_logger) { described_class.new }
 
   let(:target) { StringIO.new }

data/spec/loga/sidekiq6/job_logger_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+require 'loga/sidekiq6/job_logger'
+
+RSpec.describe Loga::Sidekiq6::JobLogger do
+  subject(:job_logger) { described_class.new }
+
+  let(:target) { StringIO.new }
+
+  let(:json_line) do
+    target.rewind
+    JSON.parse(target.read.split("\n").last)
+  end
+
+  before do
+    Loga.reset
+
+    Loga.configure(
+      service_name: 'hello_world_app',
+      service_version: '1.0',
+      device: target,
+      format: :gelf,
+    )
+  end
+
+  # https://github.com/mperham/sidekiq/blob/v6.1.2/lib/sidekiq/job_logger.rb
+  it 'inherits from ::Sidekiq::JobLogger' do
+    expect(subject).to be_a(::Sidekiq::JobLogger)
+  end
+
+  describe '#call' do
+    context 'when the job passess successfully' do
+      let(:item_data) do
+        {
+          'class' => 'HardWorker',
+          'args' => ['asd'],
+          'retry' => true,
+          'queue' => 'default',
+          'jid' => '591f6f66ee0d218fb451dfb6',
+          'created_at' => 1_528_799_582.904939,
+          'enqueued_at' => 1_528_799_582.9049861,
+        }
+      end
+
+      it 'has the the required attributes on call' do
+        job_logger.call(item_data, 'queue') do
+          # something
+        end
+
+        expected_body = {
+          'version' => '1.1',
+          'level' => 6,
+          '_type' => 'sidekiq',
+          '_created_at' => 1_528_799_582.904939,
+          '_enqueued_at' => 1_528_799_582.9049861,
+          '_jid' => '591f6f66ee0d218fb451dfb6',
+          '_retry' => true,
+          '_queue' => 'default',
+          '_service.name' => 'hello_world_app',
+          '_class' => 'HardWorker',
+          '_service.version' => '1.0',
+          '_tags' => '',
+          '_params' => ['asd'],
+        }
+
+        aggregate_failures do
+          expect(json_line).to include(expected_body)
+          expect(json_line['timestamp']).to be_a(Float)
+          expect(json_line['host']).to be_a(String)
+          expect(json_line['short_message']).to match(/HardWorker with jid:*/)
+        end
+      end
+    end
+
+    context 'when the job fails' do
+      let(:item_data) do
+        {
+          'class' => 'HardWorker',
+          'args' => ['asd'],
+          'retry' => true,
+          'queue' => 'default',
+          'jid' => '591f6f66ee0d218fb451dfb6',
+          'created_at' => 1_528_799_582.904939,
+          'enqueued_at' => 1_528_799_582.9049861,
+        }
+      end
+
+      it 'has the the required attributes on call' do
+        failed_job = lambda do
+          job_logger.call(item_data, 'queue') do
+            raise StandardError
+          end
+        end
+
+        expected_body = {
+          'version' => '1.1',
+          'level' => 4,
+          '_type' => 'sidekiq',
+          '_created_at' => 1_528_799_582.904939,
+          '_enqueued_at' => 1_528_799_582.9049861,
+          '_jid' => '591f6f66ee0d218fb451dfb6',
+          '_retry' => true,
+          '_queue' => 'default',
+          '_service.name' => 'hello_world_app',
+          '_class' => 'HardWorker',
+          '_service.version' => '1.0',
+          '_tags' => '',
+          '_params' => ['asd'],
+          '_exception' => 'StandardError',
+        }
+
+        aggregate_failures do
+          expect(&failed_job).to raise_error(StandardError)
+          expect(json_line).to include(expected_body)
+          expect(json_line['timestamp']).to be_a(Float)
+          expect(json_line['host']).to be_a(String)
+          expect(json_line['short_message']).to match(/HardWorker with jid:*/)
+        end
+      end
+    end
+  end
+end

data/spec/loga/sidekiq_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 RSpec.describe Loga::Sidekiq do
   describe '.configure_logging' do
-    context 'when sidekiq version is 5.1' do
+    context 'when sidekiq is defined' do
       it 'gets invoked on Loga.configure' do
         allow(described_class).to receive(:configure_logging)
 
@@ -18,7 +18,7 @@ RSpec.describe Loga::Sidekiq do
         expect(described_class).to have_received(:configure_logging)
       end
 
-      it 'assigns our custom sidekiq job logger' do
+      it 'assigns our custom sidekiq job logger depending on the sidekiq version' do
         Loga.reset
 
         Loga.configure(
@@ -28,7 +28,14 @@ RSpec.describe Loga::Sidekiq do
           format: :gelf,
         )
 
-        expect(::Sidekiq.options[:job_logger]).to eq(Loga::Sidekiq::JobLogger)
+        m = ENV['BUNDLE_GEMFILE'].match(/sidekiq(?<version>\d+)/)
+
+        case m['version']
+        when '51'
+          expect(::Sidekiq.options[:job_logger]).to eq(Loga::Sidekiq5::JobLogger)
+        when '6'
+          expect(::Sidekiq.options[:job_logger]).to eq(Loga::Sidekiq6::JobLogger)
+        end
       end
     end
 

data/spec/spec_helper.rb

@@ -29,14 +29,23 @@ when /sinatra/
 when /unit/
   rspec_pattern = 'unit/**/*_spec.rb'
   require 'loga'
-when /sidekiq/
-  sidekiq_specs = [
-    'integration/sidekiq_spec.rb',
-    'spec/loga/sidekiq/**/*_spec.rb',
-    'spec/loga/sidekiq_spec.rb',
-  ]
-
-  rspec_pattern = sidekiq_specs.join(',')
+when /sidekiq(?<version>\d+)/
+  case $LAST_MATCH_INFO['version']
+  when '51'
+    rspec_pattern = [
+      'spec/integration/sidekiq5_spec.rb',
+      'spec/loga/sidekiq5/**/*_spec.rb',
+      'spec/loga/sidekiq_spec.rb',
+    ].join(',')
+  when '6'
+    rspec_pattern = [
+      'spec/integration/sidekiq6_spec.rb',
+      'spec/loga/sidekiq6/**/*_spec.rb',
+      'spec/loga/sidekiq_spec.rb',
+    ].join(',')
+  else
+    raise 'FIXME: Unknown sidekiq - update this file.'
+  end
 
   require 'sidekiq'
   require 'sidekiq/cli'