contextualized_logs 0.0.1.pre.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2999bdeabefea3e6d361b02314d3746a98f2205e2af9b1ca48abd66cbeb2338d
+  data.tar.gz: 6dd09536859433929a51c184571af703611c69d10beddf6e7ba78280905288a6
+SHA512:
+  metadata.gz: 119a4ec8c01cf01261b122bcd9f0a8526e0c888ab25444ae0f20487e1528f282e8f58592c424a8b78f1b0fa198265be1168aa0304348ad185c26d10419d2ea98
+  data.tar.gz: bf2d43f375f0b1b1800b7febc9bcfcb76827ec4143a49487059d307e4dd2f5bf651f4434a6230007acdb4e0420aa29a5000e9e2f819e61fcf1fd63011a419341

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2020 Hugues Bernet-Rollande
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,480 @@
+# ContextualizedLogs
+
+Online logging solution (like [Datadog](https://www.datadoghq.com)) have drastically transform the way we log.
+
+An app will nowdays logs dozen (hundred) of logs per requests.
+
+The issue is often to correlate this logs, with the initiating request (or job) and add shared metadata on this logs.
+
+Here come `ContextualizedLogs`.
+
+The main idea is to enhance your logs from your controller (including `ContextualizedController`, which use a before action), which will add the params to your logs (and some metadata about the request itself, like `request.uuid`).
+
+This metadata are stored in a `ActiveSupport::CurrentAttributes` which is a singleton (reset per request).
+
+Each subsequent logs in this thread (request) will also be enriched with this metadata, making it easier to find all the logs associated with a request (`uuid`, `ip`, `params.xxx`).
+
+On top of this, logs can also be enriched by the ActiveRecord model they use (`create` or `find`) (models including `ContextualizedModel`). So any time a contextualized model is created or find, some metadata related to the model (`id`, ...) will also be added to the logs.
+
+Allowing you to find all logs which "touched" this models.
+
+All logs are (by default in json format)
+
+```ruby
+class MyController < ApplicationController
+  include ContextualizedLogs::ContextualizedController
+end
+```
+
+```
+curl --referer 'referer' --user-agent 'user_agent' -H "Origin: http://localhost" http://localhost/my_controller?param=a
+
+# development.log
+{
+  syslog: {
+    env: 'development',
+    host: 'localhost'
+  },
+  type: 'INFO',
+  time: '2020-04-24T19:52:51.452+02:00',
+  log_type: 'log',
+  resource_name: 'mycontroller_show',
+  http: {
+    referer: 'referer',
+    request_id: 'xxxx-xxxx-xxxx-xxxx'
+    useragent: 'user_agent',
+    origin: 'http://localhost'
+  },
+  network: {
+    client: {
+      ip: '127.0.0.1',
+      remote_addr: '127.0.0.1',
+      remote_ip: '127.0.0.1',
+      x_forwarded_for: '127.0.0.1'
+    }
+  }
+}
+```
+
+```ruby
+class User < ActiveRecord::Base
+  include ContextualizedLogs::ContextualizedModel
+  contextualizable { user_ids: :id }
+end
+
+class UserController < ApplicationController
+  include ContextualizedLogs::ContextualizedController
+  contextualized_model true
+
+  def show
+    User.find(params[:id])
+  end
+end
+```
+
+```
+curl http://localhost/users/1
+
+# development.log
+{
+  syslog: {
+    env: 'development',
+    host: 'localhost'
+  },
+  type: 'INFO',
+  time: '2020-04-24T19:52:51.452+02:00',
+  log_type: 'log',
+  context_values: {
+    user_ids: [1]
+  },
+  resource_name: 'mycontroller_show',
+  http: {
+    request_id: 'xxxx-xxxx-xxxx-xxxx'
+  },
+  network: {
+    client: {
+      ip: '127.0.0.1',
+      remote_addr: '127.0.0.1',
+      remote_ip: '127.0.0.1',
+      x_forwarded_for: '127.0.0.1'
+    }
+  }
+}
+```
+
+```ruby
+class User < ActiveRecord::Base
+  include ContextualizedLogs::ContextualizedModel
+  contextualizable { user_ids: :id }
+end
+
+class UserTracker < ActiveRecord::Base
+  include ContextualizedLogs::ContextualizedModel
+
+  belongs_to :user
+
+  contextualizable { user_tracker_ids: :id }
+end
+
+class UserController < ApplicationController
+  include ContextualizedLogs::ContextualizedController
+  contextualized_model true
+
+  def show
+    user_id = params[:id]
+    User.find(user_id)
+    UserTrackerWorker.perform_async(user_id, 'show')
+  end
+end
+
+class UserTrackerWorker
+  include Sidekiq::Worker
+  include ContextualizedLogs::ContextualizedWorker
+  contextualized_worker true
+  contextualized_model true
+  def self.contextualize_args(args)
+    { user_id: args.first, action: args.last }
+  end
+
+  def perform(user_id, action)
+    UserTracker.create(user_id: user_id, action: action)
+  end
+end
+```
+
+```
+curl http://localhost/users/1
+
+# development.log
+{
+  syslog: {
+    env: 'development',
+    host: 'localhost'
+  },
+  type: 'INFO',
+  time: '2020-04-24T19:52:51.452+02:00',
+  log_type: 'log',
+  context_values: {
+    user_ids: [1]
+  },
+  enqueued_jobs_ids: ['1234-xxxx-xxxx-xxxx']
+  resource_name: 'mycontroller_show',
+  http: {
+    request_id: 'xxxx-xxxx-xxxx-xxxx'
+  },
+  network: {
+    client: {
+      ip: '127.0.0.1',
+      remote_addr: '127.0.0.1',
+      remote_ip: '127.0.0.1',
+      x_forwarded_for: '127.0.0.1'
+    }
+  }
+}
+{
+  syslog: {
+    env: 'development',
+    host: 'localhost'
+  },
+  type: 'INFO',
+  time: '2020-04-24T19:52:51.452+02:00',
+  log_type: 'log',
+  message: 'sidekiq: completing job UserWorker: 1234-xxxx-xxxx-xxxx, on queue default',
+  job: {
+    worker: 'UserWorker',
+    id: '1234-xxxx-xxxx-xxxx',
+    args: {
+      user_id: 1,
+      action: 'show'
+    }
+  }
+  context_values: {
+    user_ids: [1],
+    user_tracker_ids: [1]
+  },
+  enqueued_jobs_ids: ['xxxx-xxxx-xxxx-xxxx']
+  resource_name: 'mycontroller_show',
+  http: {
+    request_id: 'xxxx-xxxx-xxxx-xxxx'
+  },
+  network: {
+    client: {
+      ip: '127.0.0.1',
+      remote_addr: '127.0.0.1',
+      remote_ip: '127.0.0.1',
+      x_forwarded_for: '127.0.0.1'
+    }
+  }
+}
+{
+  syslog: {
+    env: 'development',
+    host: 'localhost'
+  },
+  type: 'INFO',
+  time: '2020-04-24T19:52:51.452+02:00',
+  log_type: 'log',
+  message: 'sidekiq: completing job UserWorker: 1234-xxxx-xxxx-xxxx, on queue default',
+  job: {
+    worker: 'UserWorker',
+    id: '1234-xxxx-xxxx-xxxx',
+    args: {
+      user_id: 1,
+      action: 'show'
+    }
+  }
+  context_values: {
+    user_ids: [1],
+    user_tracker_ids: [1]
+  },
+  enqueued_jobs_ids: ['xxxx-xxxx-xxxx-xxxx']
+  resource_name: 'mycontroller_show',
+  http: {
+    request_id: 'xxxx-xxxx-xxxx-xxxx'
+  },
+  network: {
+    client: {
+      ip: '127.0.0.1',
+      remote_addr: '127.0.0.1',
+      remote_ip: '127.0.0.1',
+      x_forwarded_for: '127.0.0.1'
+    }
+  }
+}
+```
+
+## Demo
+
+### start rails
+ ```
+ bin/setup
+ bin/rails server
+ ```
+
+### start sidekiq
+ ```
+ bundle exec sidekiq
+ ```
+
+### tail logs
+ ```
+ tail -f log/development
+ ```
+
+### do some requests
+ ```
+curl -X POST -d '{"value": "value"}' -H 'Content-Type: application/json' "http://localhost:3000/model"
+curl "http://localhost:3000/model/1"
+curl "http://localhost:3000/model"
+curl -X DELETE "http://localhost:3000/model/1"
+```
+
+## Usage
+
+### ContextualizedLogger
+
+In order to enrich your logs, you needs to use (subclass of `ActiveSupport::Logger`) `ContextualizedLogger`
+
+> ContextualizedLogger logs by default some request metadata following Datadog naming convention
+> https://docs.hq.com/logs/processing/attributes_naming_convention/#source-code
+
+```
+Rails.application.configure do
+  config.logger = ContextualizedLogs::ContextualizedLogger.new("log/#{Rails.env}.log")
+end
+````
+
+### ContextualizedController
+
+```
+class Controller < ApplicationController
+  include ContextualizedLogs::ContextualizedController
+end
+```
+
+**All** (from the controller or any service, model, ... it used) this controller logs will now be enriched with some controller related metadata.
+
+### ContextualizedModel
+
+```
+class Model < ActiveRecord::Base
+  include ContextualizedLogs::ContextualizedModel
+
+  # cherry picking which model value/column should be added to CurrentContext metadata
+  contextualizable keys: {model_ids: :id}
+end
+```
+
+If `ContextualizedLogs::CurrentContext.contextualized_model_enabled` is enable on the current tread, any Model which is created or find will add `{ context_values: { model_ids: ids } }`.
+So if you fetch model (`id == 1`), and create model (`id == 2`), your logs will now contain `{ context_values: { model_ids: [1, 2] } }`.
+
+### ContextualizedWorker
+
+```
+class Worker
+  include ContextualizedLogs::ContextualizedWorker
+  contextualized_worker true # enable logging of job enqueuing, performing, completing and failure
+  contextualized_model true # enable logging of any (contextualized) model found or created while performing job
+
+  # enable adding jobs args (cherry picked) to log metadata (CurrentContext) to be logged alongs any job logs
+  def self.contextualize_args(args)
+    { first: args.first }
+  end
+end
+```
+
+If `ContextualizedLogs::CurrentContext.contextualized_model_enabled` is enable on the current tread, any Model which is created or find will add `{ context_values: { model_ids: ids } }`.
+So if you fetch model (`id == 1`), and create model (`id == 2`), your logs will now contain `{ context_values: { model_ids: [1, 2] } }`.
+
+#### Metadata Customization
+
+If you wish to logs different predefined metadata (`request.uuid`, `request.ip`, ...)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'contextualized_logs'
+```
+
+And then execute:
+
+$ bundle install
+
+
+## Roadmap
+
+- [x] contextualized logger
+- [x] contextualized controller
+- [x] contextualized model
+- [x] contextualized worker
+- [ ] lograge
+
+## Specs
+
+```
+$ rake
+The dependency tzinfo-data (>= 0) will be unused by any of the platforms Bundler is installing for. Bundler is installing for ruby but the dependency is only for x86-mingw32, x86-mswin32, x64-mingw32, java. To add those platforms to the bundle, run `bundle lock --add-platform x86-mingw32 x86-mswin32 x64-mingw32 java`.
+/Users/hugues/.rvm/rubies/ruby-2.5.1/bin/ruby -I/Users/hugues/.rvm/gems/ruby-2.5.1/gems/rspec-core-3.8.2/lib:/Users/hugues/.rvm/gems/ruby-2.5.1/gems/rspec-support-3.8.3/lib /Users/hugues/.rvm/gems/ruby-2.5.1/gems/rspec-core-3.8.2/exe/rspec --pattern spec/\*\*\{,/\*/\*\*\}/\*_spec.rb
+
+DummyController
+  should set request details
+  should NOT set enable model context values
+  should set resource_name
+  should set request details
+
+ContextualizedModelDummyController
+  should set request details
+  should set enable model context values
+
+ContextualizedLogs::ContextualizedLogger
+  format log
+  includes stack
+  format exception
+  inject context
+  dump
+    respect severity debug (default)
+    dump message
+    dump exception
+
+ContextualizedLogs::ContextualizedModel
+  .contextualizable
+    set contextualizable keys
+  .contextualize
+    with contextualized_model_enabled == true
+      set contextualizable values
+    with contextualized_model_enabled == false
+      set contextualizable values
+  with CurrentContext.contextualized_model_enabled == true
+    behaves like after_create context
+      .after_create
+        set context
+    behaves like after_find context
+      .after_find
+        does
+  with CurrentContext.contextualized_model_enabled == false
+    behaves like after_create context
+      .after_create
+        set context
+    behaves like after_find context
+      .after_find
+        does
+
+ContextualizedLogs::Sidekiq::Middleware::Client::InjectCurrentContext
+  ContextualizedWorker
+    with uncontextualized worker
+      DOES NOT change job context
+      DOES NOT log job enqueued
+      DOES NOT enable model context values
+      behaves like it client yield
+        should eq true
+    with contextualized worker
+      DOES change job context
+      DOES log job enqueued
+      behaves like it client yield
+        should eq true
+      with contextualized model
+        DOES enable model context values
+
+ContextualizedLogs::Sidekiq::Middleware::Server::RestoreCurrentContext
+  with uncontextualized worker
+    DOES NOT log job
+    DOES NOT log job failure
+    behaves like it server yield
+      should eq true
+    behaves like enable model context values
+      enable model context values
+  with contextualized worker
+    behaves like it server yield
+      should eq true
+    behaves like log job failure
+      log job failure
+    behaves like log with context
+      log with context
+    behaves like enable model context values
+      enable model context values
+  with contextualized model worker
+    behaves like it server yield
+      should eq true
+    behaves like log job failure
+      log job failure
+    behaves like log with context
+      log with context
+    behaves like enable model context values
+      enable model context values
+  with contextualized model worker
+    log with args
+    behaves like it server yield
+      should eq true
+    behaves like log job failure
+      log job failure
+    behaves like log with context
+      log with context
+    behaves like enable model context values
+      enable model context values
+
+ContextualizedLogs
+  has a version number
+
+Finished in 0.73283 seconds (files took 1.34 seconds to load)
+46 examples, 0 failures
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/huguesbr/contextualized_logs. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the sidekiq_lockable_job project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/huguesbr/sidekiq_lockable_job/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,28 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ContextualizedLogs'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'bundler/gem_tasks'
+require "rspec/core/rake_task"
+
+# RSpec::Core::RakeTask.new(:spec)
+#
+task :default => :spec
+# # Add your own tasks in files placed in lib/tasks ending in .rake,
+# # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+#
+require_relative 'config/application'
+#
+Rails.application.load_tasks

data/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../javascripts .js
+//= link_directory ../stylesheets .css

data/app/assets/javascripts/application.js

@@ -0,0 +1,16 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, or any plugin's
+// vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require turbolinks
+//= require_tree .

data/app/assets/javascripts/cable.js

@@ -0,0 +1,13 @@
+// Action Cable provides the framework to deal with WebSockets in Rails.
+// You can generate new channels where WebSocket features live using the `rails generate channel` command.
+//
+//= require action_cable
+//= require_self
+//= require_tree ./channels
+
+(function() {
+  this.App || (this.App = {});
+
+  App.cable = ActionCable.createConsumer();
+
+}).call(this);

data/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, or any plugin's
+ * vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::API
+end

data/app/controllers/model_controller.rb

@@ -0,0 +1,26 @@
+class ModelController < ApplicationController
+  include ContextualizedLogs::ContextualizedController
+  contextualized_model true
+
+  def index
+    models = Model.all.map { |model| { id: model.id, value: model.value} }
+    render json: { models: models }, status: 200
+  end
+
+  def show
+    model = Model.find(params[:id])
+    ModelWorker.perform_async(model.id, 'show')
+    render json: { models: { id: model.id, value: model.value} }, status: 200
+  end
+
+  def destroy
+    Model.find(params[:id]).destroy!
+    render json: { success: 'true' }, status: 200
+  end
+
+  def create
+    model = Model.create(value: params[:value])
+    ModelWorker.perform_async(model.id, 'create')
+    render json: { models: { id: model.id, value: model.value} }, status: 201
+  end
+end

data/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/app/jobs/application_job.rb

@@ -0,0 +1,2 @@
+class ApplicationJob < ActiveJob::Base
+end

data/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: 'from@example.com'
+  layout 'mailer'
+end

data/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/app/models/model.rb

@@ -0,0 +1,4 @@
+class Model < ActiveRecord::Base
+  include ContextualizedLogs::ContextualizedModel
+  contextualizable keys: { values: :value }
+end

data/app/views/layouts/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Demo</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all', 'data-turbolinks-track': 'reload' %>
+    <%= javascript_include_tag 'application', 'data-turbolinks-track': 'reload' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>