skylight 4.3.2 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2bf022bf2ba25bb9f9769526a93435b154a91f313862d068fd7b976d50d1bbb
-  data.tar.gz: b7e87b5ffd0f1ac492156103cb4c6dadb22e3b245b73ceaaa46ad2b3f1cce3a6
+  metadata.gz: 76c9ece1445d480897ecc306024fe939dacb94c14a02e68f7e33f5d722955284
+  data.tar.gz: 839b19ed8b3846003b3a54e5d0f3e327b0f969f2672b267dcaf4d1aa27df6254
 SHA512:
-  metadata.gz: b8fb515ffaf327347fba911035032a1827d2f33f96b654da6be32a3296c97ca4a2f863e43215904e7bffa0c8c1addbfb2e20de78c78fab24d79f2d11856737af
-  data.tar.gz: 0055467bc67cb145b1cbf2f30caa77d52a266e74fcc5f48b851ceffbb8736df2bfe7fd92a45f374547d06e3b4d5257e1738d1a7651374a19336a7b0ca7910377
+  metadata.gz: 2cbbcfd4a159b6abfa976f9d8ad4a87317973955d058408b681b1564d6d54bacadcb0e60c4bbac9ea03383dfed6d9355ac953a0b636c50a60ff99e52bcb3c5a2
+  data.tar.gz: b338f0d0a22d0b904600d050f217758d3b07c28e66c88c6f5eb961cd8701750b58a7175e3e9e31dad604c6c58c2d1b22197517c239192f41b9579c5c08a9ca5b

data/CHANGELOG.md

@@ -1,3 +1,38 @@
+## 5.0.1 (March 11, 2021)
+* [IMPROVEMENT] Use argument-forwarding (...) where available in custom instrumentation
+
+## 5.0.0 (March 5, 2021)
+
+* [FEATURE] Add normalizer for Shrine events (thanks @janko!)
+* [FEATURE] Source Locations detection and reporting is now enabled by default (can be disabled with `SKYLIGHT_ENABLE_SOURCE_LOCATIONS=false`)
+* [FEATURE] Configuration for the Source Locations caches via `SYLIGHT_SOURCE_LOCATION_CACHE_SIZE`
+
+* [IMPROVEMENT] Improve keyword argument handling in Skylight::Helpers (thanks @lukebooth!)
+* [IMPROVEMENT] Replace a Kernel.puts with Skylight.log (thanks @johnnyshields!)
+* [IMPROVEMENT] Various updates to the SQL lexer
+* [IMPROVEMENT] Reduce volume of log messages sent to the native logger in debug level
+* [IMPROVEMENT] Optimizations for the Source Locations extension
+* [IMPROVEMENT] Improved Delayed::Job probe
+* [IMPROVEMENT] Maintain method visibility when instrumenting with `instrument_method`
+* [IMPROVEMENT] Update probes to use `Module#prepend` where possible
+* [IMPROVEMENT] New tokio-based skylightd
+* [IMPROVEMENT] Support `render_layout` notifications in Rails 6.1
+* [IMPROVEMENT] Support `ActionMailer::MailDeliveryJob` in Rails 6.1
+* [IMPROVEMENT] Better logging config. `SKYLIGHT_NATIVE_LOG_LEVEL` now defaults to `warn`.
+
+* [BREAKING] Rename `environment` keyword argument to `priority_key`. Note `env` has not changed.
+* [BREAKING] Drop support for Ruby 2.4
+* [BREAKING] Merge skylight-core into skylight. All classes previously namespaced under `Skylight::Core` have been moved to `Skylight`.
+* [BREAKING] Remove `Skylight::Util::Inflector`
+* [BREAKING] Drop support for Rails 4
+* [BREAKING] Drop support for Ruby 2.3
+
+* [BUGFIX] Fix issue with missing metadata in MongoDB probe
+* [BUGFIX] Resolve an inability to parse certain SQL queries containing arrays
+* [BUGFIX] Allow multiple probes to be registered under the same key
+* [BUGFIX] Do not refer to Redis constant until the probe is installed
+* [BUGFIX] Fix nested calls to `Normalizers::Faraday::Request.disable`
+
 ## 4.3.2 (December 14, 2020)
 * [BUGFIX] Backport an ActionView fix from Skylight 5 (makes Skylight 4 compatible with Rails 6.1)
 
@@ -9,9 +44,6 @@
 * [IMPROVEMENT] Update Grape normalizer for version 1.3.1
 * [BUGFIX] Fix an issue where GraphQL normalizers could fail to load in non-Rails contexts
 
-## 4.2.3 (March 10, 2020)
-* [BUGFIX] Fix an issue where the GraphQL probe may not always be installed
-
 ## 4.2.2 (February 25, 2020)
 * Support graphql-ruby version 1.10
 

data/CONTRIBUTING.md

@@ -16,14 +16,8 @@ If you prefer to run tests in your own environment, you may do so as follows:
 
 ```shell
 # Select a gemfile and bundle install
-export BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.rails-5.2.x
+export BUNDLE_GEMFILE=$PWD/gemfiles/rails-5.2.x/Gemfile
 bundle install
-
-# Run the skylight-core test suite (takes a few seconds)
-pushd skylight-core
-bundle exec rspec
-popd
-
-# Run the main test suite (takes 5-10 minutes)
+# Run the test suite (takes 5-10 minutes)
 bundle exec rspec
 ```

data/ext/extconf.rb

@@ -32,7 +32,8 @@ SKYLIGHT_FETCH_LIB  = !ENV.key?("SKYLIGHT_FETCH_LIB") || ENV["SKYLIGHT_FETCH_LIB
 
 # Directory where skylight.h exists
 SKYLIGHT_HDR_PATH = ENV["SKYLIGHT_HDR_PATH"] || ENV["SKYLIGHT_LIB_PATH"] || "."
-SKYLIGHT_LIB_PATH = ENV["SKYLIGHT_LIB_PATH"] || File.expand_path("../../lib/skylight/native/#{Platform.tuple}", __FILE__)
+SKYLIGHT_LIB_PATH = ENV["SKYLIGHT_LIB_PATH"] ||
+                    File.expand_path("../../lib/skylight/native/#{Platform.tuple}", __FILE__)
 
 SKYLIGHT_SOURCE_URL = ENV["SKYLIGHT_SOURCE_URL"]
 SKYLIGHT_VERSION = ENV["SKYLIGHT_VERSION"]
@@ -41,7 +42,7 @@ SKYLIGHT_CHECKSUM = ENV["SKYLIGHT_CHECKSUM"]
 SKYLIGHT_EXT_STRICT = ENV.key?("SKYLIGHT_EXT_STRICT") && ENV["SKYLIGHT_EXT_STRICT"] =~ /^true$/i
 
 # Setup logger
-LOG = Logger.new(MultiIO.new(STDOUT, File.open(SKYLIGHT_INSTALL_LOG, "a")))
+LOG = Logger.new(MultiIO.new($stdout, File.open(SKYLIGHT_INSTALL_LOG, "a")))
 
 # Handles terminating in the case of a failure. If we have a bug, we do not
 # want to break our customer's deploy, but extconf.rb requires a Makefile to be
@@ -68,7 +69,7 @@ end
 if Platform::OS == "darwin"
   # If the user installs Xcode-only, they have to approve the
   # license or no "xc*" tool will work.
-  if `/usr/bin/xcrun clang 2>&1` =~ /license/ && !$CHILD_STATUS.success?
+  if `/usr/bin/xcrun clang 2>&1` =~ /license/ && !$CHILD_STATUS.success? # rubocop:disable Style/SoleNestedConditional
     fail <<~MESSAGE
       You have not agreed to the Xcode license and so we are unable to build the native agent.
       To resolve this, you can agree to the license by opening Xcode.app or running:
@@ -206,8 +207,8 @@ find_header "skylight_dlopen.h", hdrpath
 fail "could not create Makefile; dlfcn.h missing" unless have_header "dlfcn.h"
 
 # For escaping the GVL
-unless have_func("rb_thread_call_without_gvl", "ruby/thread.h") || have_func("rb_thread_blocking_region")
-  abort "Ruby is unexpectedly missing rb_thread_blocking_region. This should not happen."
+unless have_func("rb_thread_call_without_gvl", "ruby/thread.h")
+  abort "Ruby is unexpectedly missing rb_thread_call_without_gvl. This should not happen."
 end
 
 # Previous comment stated:

data/ext/libskylight.yml

@@ -1,7 +1,8 @@
-version: "4.1.0-46806f5"
+# commit: ea59cc7bdbbee0f69d1cf7e69827974a9ea67645
+---
+version: "5.0.0-488d432"
 checksums:
-  x86-linux:         "4054632757516aa44e812860601fcb7f21c8c5d412b2d585868b46c7a8e3cae6"
-  x86_64-linux:      "4675694fba4448ba14787551237bd4e388443895d5c9e5d8a71c550eccd203db"
-  x86_64-linux-musl: "7a81880830d0a409a6f0f39ff42f90a1422542b05384e2480ae05515f6b5f831"
-  x86_64-darwin:     "84eef6c330818c2bad1628ff189b9cb88275c9d72a28afac88a0d054d8002230"
-  x86_64-freebsd:    "a86a52203fab4f9a56ad91d5700e14823e6e997b4df1bcb3ae2d9e4296c50b1f"
+  x86-linux:         "3c16b6db1508f35720258551783fbcd30fd231638bad316ea76748d659838399"
+  x86_64-linux:      "94383aa3359c3f2e9c0e3c16ee263d46c5673dd255f8842e6acadf5ec3131c06"
+  x86_64-linux-musl: "d2e2e2e61c321315f9bcaa157426f33aef8ffc2330ba46b2cdcbd9442e65f728"
+  x86_64-darwin:     "bcc925d0bcbae83a484f35dbc9729dcf262e1f3a8c29b8d387d0f58ad8f3afa3"

data/ext/skylight_native.c

@@ -86,14 +86,6 @@ typedef void* (*blocking_fn_t)(void*);
 #define WITHOUT_GVL(fn, a) \
   rb_thread_call_without_gvl((blocking_fn_t)(fn), (a), 0, 0)
 
-// Ruby 1.9
-#elif defined(HAVE_RB_THREAD_BLOCKING_REGION)
-
-typedef VALUE (*blocking_fn_t)(void*);
-#define WITHOUT_GVL(fn, a) \
-  rb_thread_blocking_region((blocking_fn_t)(fn), (a), 0, 0)
-
-
 #endif
 
 
@@ -103,7 +95,6 @@ typedef VALUE (*blocking_fn_t)(void*);
 
 VALUE rb_mSkylight;
 VALUE rb_eNativeError;
-VALUE rb_mCore;
 VALUE rb_mUtil;
 VALUE rb_cClock;
 VALUE rb_cTrace;
@@ -140,7 +131,7 @@ load_libskylight(VALUE klass, VALUE path) {
 
 /*
  *
- * class Skylight::Core::Util::Clock
+ * class Skylight::Util::Clock
  *
  */
 
@@ -152,7 +143,7 @@ clock_high_res_time(VALUE self) {
 
 /*
  *
- * class Skylight::Core::Instrumenter
+ * class Skylight::Instrumenter
  *
  */
 
@@ -243,33 +234,9 @@ instrumenter_submit_trace(VALUE self, VALUE rb_trace) {
   return Qnil;
 }
 
-static VALUE
-instrumenter_track_desc(VALUE self, VALUE rb_endpoint, VALUE rb_desc) {
-  int tracked;
-  sky_instrumenter_t* instrumenter;
-
-  CHECK_TYPE(rb_endpoint, T_STRING);
-  CHECK_TYPE(rb_desc, T_STRING);
-
-  tracked = 0;
-
-  My_Struct(instrumenter, sky_instrumenter_t, no_instrumenter_msg);
-
-  CHECK_FFI(
-      sky_instrumenter_track_desc(instrumenter, STR2BUF(rb_endpoint), STR2BUF(rb_desc), &tracked),
-      "Instrumenter#native_track_desc");
-
-  if (tracked) {
-    return Qtrue;
-  }
-  else {
-    return Qfalse;
-  }
-}
-
 /*
  *
- * class Skylight::Core::Trace
+ * class Skylight::Trace
  *
  */
 
@@ -503,9 +470,24 @@ trace_span_set_description(VALUE self, VALUE span, VALUE desc) {
 
 static VALUE
 trace_span_set_meta(VALUE self, VALUE span, VALUE meta) {
-  UNUSED(self);
-  UNUSED(span);
-  UNUSED(meta);
+  sky_trace_t* trace;
+  VALUE rb_source_location;
+
+  My_Struct(trace, sky_trace_t, consumed_trace_msg);
+
+  CHECK_TYPE(span, T_FIXNUM);
+  CHECK_TYPE(meta, T_HASH);
+
+  rb_source_location = rb_hash_lookup(meta, ID2SYM(rb_intern("source_location")));
+  if (rb_source_location != Qnil) {
+    sky_buf_t source_location;
+
+    CHECK_TYPE(rb_source_location, T_STRING);
+    source_location = STR2BUF(rb_source_location);
+
+    sky_trace_span_add_string_annotation(trace, FIX2UINT(span), 3, source_location);
+  }
+
   return Qnil;
 }
 
@@ -525,71 +507,14 @@ trace_span_set_exception(VALUE self, VALUE span, VALUE exception, VALUE exceptio
   return Qnil;
 }
 
-static VALUE
-trace_span_get_correlation_header(VALUE self, VALUE span_id) {
-  UNUSED(self);
-  UNUSED(span_id);
-  return Qnil;
-}
-
-static VALUE
-lex_sql(VALUE klass, VALUE rb_sql) {
-  sky_buf_t sql;
-  sky_buf_t title;
-  sky_buf_t statement;
-  uint8_t title_store[128];
-  VALUE rb_statement;
-  VALUE ret;
-
-  UNUSED(klass);
-  CHECK_TYPE(rb_sql, T_STRING);
-
-  sql = STR2BUF(rb_sql);
-  title = (sky_buf_t) {
-    .data = title_store,
-    .len = sizeof(title_store),
-  };
-
-  // The statement cannot be longer than the original sql string
-  rb_statement = rb_str_buf_new(RSTRING_LEN(rb_sql));
-  statement = (sky_buf_t) {
-    .data = RSTRING_PTR(rb_statement),
-    .len = RSTRING_LEN(rb_sql),
-  };
-
-  CHECK_FFI(sky_lex_sql(sql, &title, &statement),
-    "Skylight#lex_sql");
-
-  // Set the statement return
-  rb_str_set_len(rb_statement, statement.len);
-  rb_enc_associate(rb_statement, rb_utf8_encoding());
-
-  ret = rb_ary_new2(2);
-
-  if (title.len > 0) {
-    rb_ary_store(ret, 0, BUF2STR(title));
-  }
-  else {
-    rb_ary_store(ret, 0, Qnil);
-  }
-
-  rb_ary_store(ret, 1, rb_statement);
-
-  return ret;
-}
-
 void Init_skylight_native() {
   rb_mSkylight = rb_define_module("Skylight");
 
   rb_eNativeError = rb_const_get(rb_mSkylight, rb_intern("NativeError"));
 
   rb_define_singleton_method(rb_mSkylight, "load_libskylight", load_libskylight, 1);
-  rb_define_singleton_method(rb_mSkylight, "lex_sql", lex_sql, 1);
-
-  rb_mCore = rb_define_module_under(rb_mSkylight, "Core");
 
-  // FIXME: Don't put these under Core
-  rb_mUtil  = rb_define_module_under(rb_mCore, "Util");
+  rb_mUtil  = rb_define_module_under(rb_mSkylight, "Util");
   rb_cClock = rb_define_class_under(rb_mUtil, "Clock", rb_cObject);
   rb_define_method(rb_cClock, "native_hrtime", clock_high_res_time, 0);
 
@@ -612,12 +537,10 @@ void Init_skylight_native() {
   rb_define_method(rb_cTrace, "native_span_set_meta", trace_span_set_meta, 2);
   rb_define_method(rb_cTrace, "native_span_started", trace_span_started, 1);
   rb_define_method(rb_cTrace, "native_span_set_exception", trace_span_set_exception, 3);
-  rb_define_method(rb_cTrace, "native_span_get_correlation_header", trace_span_get_correlation_header, 1);
 
   rb_cInstrumenter = rb_const_get(rb_mSkylight, rb_intern("Instrumenter"));
   rb_define_singleton_method(rb_cInstrumenter, "native_new", instrumenter_new, 2);
   rb_define_method(rb_cInstrumenter, "native_start", instrumenter_start, 0);
   rb_define_method(rb_cInstrumenter, "native_stop", instrumenter_stop, 0);
   rb_define_method(rb_cInstrumenter, "native_submit_trace", instrumenter_submit_trace, 1);
-  rb_define_method(rb_cInstrumenter, "native_track_desc", instrumenter_track_desc, 2);
 }

data/lib/skylight.rb

@@ -1,36 +1,233 @@
 require "skylight/version"
-require "skylight/core"
 require "skylight/trace"
 require "skylight/instrumenter"
 require "skylight/middleware"
 require "skylight/api"
 require "skylight/helpers"
 require "skylight/config"
+require "skylight/user_config"
 require "skylight/errors"
 require "skylight/native"
+require "skylight/gc"
+require "skylight/vm/gc"
+require "skylight/util"
+require "skylight/deprecation"
+require "skylight/subscriber"
+require "skylight/sidekiq"
+require "skylight/probes"
 
 # For prettier global names
 require "English"
 
+require "active_support/notifications"
+
+# Specifically check for Railtie since we've had at least one case of a
+#   customer having Rails defined without having all of Rails loaded.
+if defined?(Rails::Railtie)
+  require "skylight/railtie"
+end
+
 module Skylight
   # Used from the CLI
   autoload :CLI, "skylight/cli"
+  # Is this autoload even useful?
+  autoload :Normalizers, "skylight/normalizers"
 
-  # Specifically check for Railtie since we've had at least one case of a
-  #   customer having Rails defined without having all of Rails loaded.
-  if defined?(Rails::Railtie)
-    require "skylight/railtie"
-  end
+  extend Util::Logging
 
-  include Core::Instrumentable
+  LOCK = Mutex.new
 
-  def self.instrumenter_class
-    Instrumenter
-  end
+  # @api private
+  TIERS = %w[
+    rack
+    api
+    app
+    view
+    db
+    noise
+    other
+  ].freeze
 
-  def self.config_class
-    Config
-  end
+  # @api private
+  TIER_REGEX = /^(?:#{TIERS.join('|')})(?:\.|$)/u.freeze
+
+  # @api private
+  CATEGORY_REGEX = /^[a-z0-9_-]+(?:\.[a-z0-9_-]+)*$/iu.freeze
+
+  # @api private
+  DEFAULT_CATEGORY = "app.block".freeze
+
+  # @api private
+  DEFAULT_OPTIONS = { category: DEFAULT_CATEGORY }.freeze
+
+  at_exit { stop! }
+
+  class << self
+    extend Util::InstrumenterMethod
+
+    def instrumenter
+      defined?(@instrumenter) && @instrumenter
+    end
+
+    def probe(*args)
+      Probes.probe(*args)
+    end
+
+    def enable_normalizer(*names)
+      Normalizers.enable(*names)
+    end
+
+    # Start instrumenting
+    def start!(config = nil)
+      return instrumenter if instrumenter
+
+      const_get(:LOCK).synchronize do
+        return instrumenter if instrumenter
+
+        config ||= {}
+        config = Config.load(config) unless config.is_a?(Config)
+
+        Probes.install!
+
+        @instrumenter = Instrumenter.new(config).start!
+      end
+    rescue => e
+      level, message =
+        if e.is_a?(ConfigError)
+          [:warn, format("Unable to start Instrumenter due to a configuration error: %<message>s",
+                         message: e.message)]
+        else
+          [:error, format("Unable to start Instrumenter; msg=%<message>s; class=%<klass>s",
+                          message: e.message, klass: e.class)]
+        end
+
+      if config.respond_to?("log_#{level}") && config.respond_to?(:log_trace)
+        config.send("log_#{level}", message)
+        config.log_trace e.backtrace.join("\n")
+      else
+        warn "[#{name.upcase}] #{message}"
+      end
+      false
+    end
+
+    def started?
+      !!instrumenter
+    end
+
+    # Stop instrumenting
+    def stop!
+      t { "stop!" }
 
-  Core::Probes.add_path(File.expand_path("skylight/probes", __dir__))
+      const_get(:LOCK).synchronize do
+        t { "stop! synchronized" }
+        return unless instrumenter
+
+        # This is only really helpful for getting specs to pass.
+        @instrumenter.current_trace = nil
+
+        @instrumenter.shutdown
+        @instrumenter = nil
+      end
+    end
+
+    # Check tracing
+    def tracing?
+      t { "checking tracing?; thread=#{Thread.current.object_id}" }
+      instrumenter&.current_trace
+    end
+
+    # Start a trace
+    def trace(endpoint = nil, cat = nil, title = nil, meta: nil, segment: nil, component: nil)
+      unless instrumenter
+        return yield if block_given?
+
+        return
+      end
+
+      if instrumenter.poisoned?
+        spawn_shutdown_thread!
+        return yield if block_given?
+
+        return
+      end
+
+      cat ||= DEFAULT_CATEGORY
+
+      if block_given?
+        instrumenter.trace(endpoint, cat, title, nil, meta: meta, segment: segment, component: component) do |tr|
+          yield tr
+        end
+      else
+        instrumenter.trace(endpoint, cat, title, nil, meta: meta, segment: segment, component: component)
+      end
+    end
+
+    # @overload instrument(opts)
+    #   @param [Hash] opts the options for instrumentation.
+    #   @option opts [String] :category (`DEFAULT_CATEGORY`) The category
+    #   @option opts [String] :title The title
+    #   @option opts [String] :description The description
+    #   @option opts [Hash] :meta The meta
+    #   @option opts [String] :source_location The source location
+    #   @option opts [String] :source_file The source file. (Will be sanitized.)
+    #   @option opts [String] :source_line The source line.
+    # @overload instrument(title)
+    #   Instrument with the specified title and the default category
+    #   @param [String] title The title
+    def instrument(opts = DEFAULT_OPTIONS, &block)
+      unless instrumenter
+        return yield if block_given?
+
+        return
+      end
+
+      if opts.is_a?(Hash)
+        category    = opts[:category] || DEFAULT_CATEGORY
+        title       = opts[:title]
+        desc        = opts[:description]
+        meta        = opts[:meta]
+      else
+        category    = DEFAULT_CATEGORY
+        title       = opts.to_s
+        desc        = nil
+        meta        = nil
+        opts        = {}
+      end
+
+      # NOTE: unless we have `:internal` (indicating a built-in Skylight instrument block),
+      # or we already have a `source_file` or `source_line` (probably set by `instrument_method`),
+      # we set the caller location to the second item on the stack
+      # (immediate caller of the `instrument` method).
+      unless opts[:source_file] || opts[:source_line] || opts[:internal]
+        opts = opts.merge(sk_instrument_location: caller_locations(1..1).first)
+      end
+
+      meta ||= {}
+
+      instrumenter.extensions.process_instrument_options(opts, meta)
+      instrumenter.instrument(category, title, desc, meta, &block)
+    end
+
+    instrumenter_method :config
+
+    instrumenter_method :mute, block: true
+    instrumenter_method :unmute, block: true
+    instrumenter_method :muted?
+
+    # End a span
+    instrumenter_method :done
+
+    instrumenter_method :broken!
+
+    # Temporarily disable
+    instrumenter_method :disable, block: true
+
+    # Runs the shutdown procedure in the background.
+    # This should do little more than unsubscribe from all ActiveSupport::Notifications
+    def spawn_shutdown_thread!
+      @shutdown_thread || const_get(:LOCK).synchronize do
+        @shutdown_thread ||= Thread.new { @instrumenter&.shutdown }
+      end
+    end
+  end
 end