nexus_semantic_logger 1.0.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f41cfcb762aa97a9c9e03abe6ec14570b5a9a90121c0df24b620105541be5c6
-  data.tar.gz: b9eb8d732937fa6ee5dcf6a64a616dbdb2e4800b734156db001807cde9735c7a
+  metadata.gz: 6d90e3661ba7e5c82b2e1777d465df8f88617a3ad8287f8d0b9bdd461f0729a6
+  data.tar.gz: 1217216fdf855b72558510e52a50f0308c10b3e07ccbc80369fb228f0fe51040
 SHA512:
-  metadata.gz: 6505a0e3c84b133a8bb2fc3151959645dbf536977c5b6eb224d99f42c77066923c59e1f4d860f09f362b3b025e9d2cf97e5e1eb22cc8014230a3d8abd2d2144c
-  data.tar.gz: e1ff91e0a80205c43b3b5f32c296d122ab77ec40a9a7541d85bcb9421762006cdb2050cd8b937b9341e32ec07ecd9accfae74a3de68d8507e52bf8535024c754
+  metadata.gz: a4261cc18604aade1ebd48efcd2cbb680ee26da22c366bcb544c786d42bffdad1a57ec2249f53347f1bcb5dadd2b0c13ff20f8699a7e30131e1e85fb7f1d2244
+  data.tar.gz: f2ea77247cc5e769e1dc7706595b85dc62179ceef8d05f11d78904632cd6cb39c837d64e401e278edf52333a75552dc6de41634f8b6862ac26155d528de9397b

data/.rubocop.yml

@@ -1,6 +1,10 @@
 inherit_gem:
   rubocop-shopify: rubocop.yml
 
+Style/ClassVars:
+  Enabled: false
+Style/ConditionalAssignment:
+  Enabled: false
 Style/GlobalVars:
   Enabled: false
 Style/FrozenStringLiteralComment:

data/README.md

@@ -1,3 +1,57 @@
 # nexus_semantic_logger
 
 Configures a [semantic_logger](https://rubygems.org/gems/rails_semantic_logger) as required for NexusMods components.
+
+## Telemetry
+
+As well as providing a semantic logger, this gem handles datadog telemetry associated with the logging approach:
+
+* logs
+* traces
+* metrics
+   * statsd is automatically attached to datadog runtime metrics and may also be used for custom metrics.
+
+### Sending metrics
+
+Ensure the metric name is in the format: `nexus.{component}.{major}.{minor}`
+
+Where _major and minor_ are specific to the component logic e.g. `nexus.uploads.clamscan.pass`
+
+For example, to increment a count:
+
+```
+NexusSemanticLogger.metrics.increment('nexus.users.registration.complete')
+```
+
+# Local gem development
+
+Steps to run this gem from local sources in one the nexus 'staged build' rails components:
+
+## Copy gem sources to component
+
+```
+cd ~/legacy/users
+cp -r ../nexus_semantic_logger .
+```
+
+## Adjust component Dockerfile to include gem sources
+
+Within stage 1, append a COPY after the Gemfile copy:
+
+```
+COPY --chown=nexus:nexus Gemfile* ./
+COPY --chown=nexus:nexus nexus_semantic_logger/ ./nexus_semantic_logger/
+```
+
+Within stage 2, append a COPY after the bundle copy:
+
+```
+COPY --from=stage1 /usr/local/bundle /usr/local/bundle
+COPY --from=stage1 /app/nexus_semantic_logger/ /app/nexus_semantic_logger/
+```
+
+## Adjust Gemfile to use local path
+
+```
+gem 'nexus_semantic_logger', :path => "/app/nexus_semantic_logger"
+

data/lib/nexus_semantic_logger/appender_filter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+require 'rails_semantic_logger'
+
+module NexusSemanticLogger
+  class AppenderFilter
+    def self.filter_lambda
+      -> (log) {
+        # log API see https://logger.rocketjob.io/log_struct.html
+        # and lib/semantic_logger/levels.rb
+        # level: :trace=>0, :debug=>1, :info=>2, :warn=>3, :error=>4, :fatal=>5
+        current_log_level = SemanticLogger::Levels.index(env_names_default_level)
+
+        # Names allow overriding the level that will be appended,
+        # else the global level is used to determine appending.
+        append = false
+        if log.name.in?(env_names_trace)
+          append = true
+        elsif log.name.in?(env_names_debug)
+          append = log.level_index >= 1
+        elsif log.name.in?(env_names_info)
+          append = log.level_index >= 2
+        elsif log.name.in?(env_names_warn)
+          append = log.level_index >= 3
+        elsif log.name.in?(env_names_error)
+          append = log.level_index >= 4
+        elsif log.name.in?(env_names_fatal)
+          append = log.level_index >= 5
+        else
+          append = log.level_index >= current_log_level
+        end
+        append
+      }
+    end
+
+    def self.env_names_default_level
+      @@names_default_level ||= ENV.fetch('LOG_NAMES_DEFAULT_LEVEL', Rails.application.config.log_level)
+    end
+
+    def self.env_names_trace
+      @@names_trace ||= fetch_env_names('LOG_NAMES_TRACE')
+    end
+
+    def self.env_names_debug
+      @@names_debug ||= fetch_env_names('LOG_NAMES_DEBUG')
+    end
+
+    def self.env_names_info
+      @@names_info ||= fetch_env_names('LOG_NAMES_INFO')
+    end
+
+    def self.env_names_warn
+      @@names_warn ||= fetch_env_names('LOG_NAMES_WARN')
+    end
+
+    def self.env_names_error
+      @@names_error ||= fetch_env_names('LOG_NAMES_ERROR')
+    end
+
+    def self.env_names_fatal
+      @@names_fatal ||= fetch_env_names('LOG_NAMES_FATAL')
+    end
+
+    def self.flush
+      @@names_default_level = nil
+      @@names_trace = nil
+      @@names_debug = nil
+      @@names_info = nil
+      @@names_warn = nil
+      @@names_error = nil
+      @@names_fatal = nil
+    end
+
+    def self.fetch_env_names(var)
+      ENV.fetch(var, '').split(',').to_set
+    end
+  end
+end

data/lib/nexus_semantic_logger/application.rb

@@ -33,7 +33,10 @@ module NexusSemanticLogger
       # Default logging is stdout in datadog compatible JSON.
       config.rails_semantic_logger.format = NexusSemanticLogger::DatadogFormatter.new(service)
       config.rails_semantic_logger.add_file_appender = false
-      config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format)
+      dd_appender = config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format)
+      dd_appender.filter = NexusSemanticLogger::AppenderFilter.filter_lambda
+
+      NexusSemanticLogger::DatadogTracer.new(service)
 
       logger.info('SemanticLogger initialised.', level: config.log_level)
     end
@@ -44,13 +47,19 @@ module NexusSemanticLogger
 
       # Change default logging to coloured logging on stdout.
       config.semantic_logger.clear_appenders!
-      config.semantic_logger.add_appender(io: $stdout, formatter: :color)
-      if ENV['DD_AGENT_HOST'].present? && ENV['DD_AGENT_PORT'].present?
+      color_appender = config.semantic_logger.add_appender(io: $stdout, formatter: :color)
+      color_appender.filter = NexusSemanticLogger::AppenderFilter.filter_lambda
+
+      if ENV['DD_AGENT_HOST'].present? && ENV['DD_AGENT_LOGGING_PORT'].present?
         # Development logs can be sent to datadog via a TCP logging endpoint on a local agent.
         # Each port is assigned a particular service.
         # See https://logger.rocketjob.io/appenders.html
-        config.semantic_logger.add_appender(appender: :tcp, server: "#{ENV['DD_AGENT_HOST']}:#{ENV['DD_AGENT_PORT']}",
-                                            formatter: config.rails_semantic_logger.format)
+        dd_appender = config.semantic_logger.add_appender(
+          appender: :tcp,
+          server: "#{ENV['DD_AGENT_HOST']}:#{ENV['DD_AGENT_LOGGING_PORT']}",
+          formatter: config.rails_semantic_logger.format
+        )
+        dd_appender.filter = NexusSemanticLogger::AppenderFilter.filter_lambda
       end
 
       logger.info('SemanticLogger initialised in development.', level: config.log_level)
@@ -59,7 +68,8 @@ module NexusSemanticLogger
     def self.test(config)
       # Use human readable coloured output for logs when running tests.
       config.semantic_logger.clear_appenders!
-      config.semantic_logger.add_appender(io: $stdout, formatter: :color)
+      color_appender = config.semantic_logger.add_appender(io: $stdout, formatter: :color)
+      color_appender.filter = NexusSemanticLogger::AppenderFilter.filter_lambda
     end
   end
 end

data/lib/nexus_semantic_logger/datadog_singleton.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+require 'singleton'
+
+module NexusSemanticLogger
+  # Application wide location to get datadog objects.
+  # dogstatsd-ruby maintains its own queue and thread for flushing, so the client code should never create its
+  # own statsd instance.
+  class DatadogSingleton
+    include Singleton
+    attr_accessor :statsd, :tags
+
+    def flush
+      statsd&.flush(sync: Rails.env.development?) # Force flush sync in development, speed up checks.
+    end
+
+    def increment(metric_name)
+      statsd&.increment(metric_name, tags: tags)
+      flush
+    end
+  end
+end

data/lib/nexus_semantic_logger/datadog_tracer.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+require 'datadog/statsd'
+require 'ddtrace'
+
+module NexusSemanticLogger
+  class DatadogTracer
+    def initialize(service)
+      Datadog.configure do |c|
+        if ENV['DD_AGENT_HOST'].present?
+          # To enable runtime metrics collection, set `true`. Defaults to `false`
+          # You can also set DD_RUNTIME_METRICS_ENABLED=true to configure this.
+          c.runtime_metrics.enabled = true
+
+          # Configure DogStatsD instance for sending runtime metrics.
+          # By default, runtime metrics from the application are sent to the Datadog Agent with DogStatsD on port 8125.
+          datadog_singleton = DatadogSingleton.instance
+          datadog_singleton.statsd = Datadog::Statsd.new(ENV['DD_AGENT_HOST'], 8125)
+          datadog_singleton.tags = ["env:#{Rails.env}", "service:#{service}"]
+          c.runtime_metrics.statsd = datadog_singleton.statsd
+
+          # Tracer requires configuration to a datadog agent via DD_AGENT_HOST.
+          dd_force_tracer_val = ENV.fetch('DD_FORCE_TRACER', false)
+          dd_force_tracer = dd_force_tracer_val.present? && dd_force_tracer_val.to_s == 'true'
+          c.tracer(enabled: Rails.env.production? || dd_force_tracer, env: Rails.env)
+        end
+
+        c.use(:rails, service_name: service)
+
+        c.logger.level = Logger::WARN # ddtrace info logging is too verbose.
+      end
+    end
+  end
+end

data/lib/nexus_semantic_logger/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 module NexusSemanticLogger
-  # Leave this as 1.0.2 in order for CI process to replace with the tagged version.
-  VERSION = '1.0.2'
+  # Leave this as 1.3.0 in order for CI process to replace with the tagged version.
+  VERSION = '1.3.0'
 end

data/lib/nexus_semantic_logger.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
+require 'nexus_semantic_logger/appender_filter'
 require 'nexus_semantic_logger/application'
 require 'nexus_semantic_logger/datadog_formatter'
+require 'nexus_semantic_logger/datadog_singleton'
+require 'nexus_semantic_logger/datadog_tracer'
 
 module NexusSemanticLogger
+  # Get application wide object for sending metrics.
+  def metrics
+    DatadogSingleton.instance
+  end
 end

data/nexus_semantic_logger.gemspec

@@ -16,5 +16,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency('amazing_print', '~> 1.4.0')
   spec.add_dependency('rails_semantic_logger', '~> 4.10.0')
   spec.add_dependency('net_tcp_client', '~> 2.2.0') # For TCP logging.
+  spec.add_dependency('dogstatsd-ruby', '~> 5.4.0') # For custom application metrics.
   spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nexus_semantic_logger
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.3.0
 platform: ruby
 authors:
 - Johnathon Harris
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-05 00:00:00.000000000 Z
+date: 2022-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: dogstatsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.4.0
 description: 
 email: john.harris@nexusmods.com
 executables: []
@@ -63,8 +77,11 @@ files:
 - Gemfile
 - README.md
 - lib/nexus_semantic_logger.rb
+- lib/nexus_semantic_logger/appender_filter.rb
 - lib/nexus_semantic_logger/application.rb
 - lib/nexus_semantic_logger/datadog_formatter.rb
+- lib/nexus_semantic_logger/datadog_singleton.rb
+- lib/nexus_semantic_logger/datadog_tracer.rb
 - lib/nexus_semantic_logger/version.rb
 - nexus_semantic_logger.gemspec
 homepage: 
@@ -85,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.17
+rubygems_version: 3.3.21
 signing_key: 
 specification_version: 4
 summary: semantic_logger usage for nexus