lightstep 0.9.3 → 0.9.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e5efa372f2f58b87d7bf26d3e5719179999f9c2
-  data.tar.gz: 224eaaf3e5b7862e1ec8c135ccf7c3025f61e7c9
+  metadata.gz: 7d70ce2d0c73411b078a2855f102fb267db8cbc1
+  data.tar.gz: 6db2ee7c8186a11a4e7ace242c43eea628f4221d
 SHA512:
-  metadata.gz: 0ad636f2e47c99581df36d3773b0ecc7d7f4c395ed66743d782d0ddde4ccce1a5f1b081b1199213de6d99932181c4d88cd92f9387b5faacf8a9351c317ee6c3f
-  data.tar.gz: fb223d0e73cb22cf64287399a3c8cf82351bae8df229f9846f90e94645058fc8d4c628d63d7fa62f41a77d681b7ece462682e4ef1ea119b96fdbef4244efd3c1
+  metadata.gz: 101ec7f4c34bcca170f4152cc57289eefb3d7963d23b1ab3ff1f7615e825542a0aa120a9ade836706444accbdadff95da7c0d2edb0e49067afafd09cca4f31bc
+  data.tar.gz: 7031358b6681e84a40e26d75cf4403dabdd0c74bc272bbae2aa1303c95164f1322ae669d3eb5f468800ffafe0552c87894413c4e2549a3517a7bf655d76c5aec

data/README.md

@@ -45,9 +45,9 @@ The LightStep Tracer is threadsafe. For increased performance, you can add the
 `concurrent-ruby-ext` gem to your Gemfile. This will enable C extensions for
 concurrent operations.
 
-**The LightStep Tracer is not inherently Fork-safe**. When forking, you should
-`disable` the Tracer before forking, then `enable` it in the child Process
-and parent Process after forking. See the `fork_children` example for more.
+The LightStep Tracer is also Fork-safe. When forking, the child process will
+not inherit the unflushed spans of the parent, so they will only be flushed
+once.
 
 ## Development
 

data/examples/fork_children/main.rb

@@ -1,6 +1,5 @@
 # A simple, manual test ensuring that tracer instances still report after a
-# Process.fork. Currently this requires the tracer instance to be explicitly
-# disabled before the fork and reenabled afterward.
+# Process.fork.
 
 require 'bundler/setup'
 require 'lightstep'
@@ -14,10 +13,7 @@ puts 'Starting...'
 (1..20).each do |k|
   puts "Explicit reset iteration #{k}..."
 
-  # NOTE: the tracer is disabled and reenabled on either side of the fork
-  LightStep.disable
   pid = Process.fork do
-    LightStep.enable
     10.times do
       span = LightStep.start_span("my_forked_span-#{Process.pid}")
       sleep(0.0025 * rand(k))
@@ -25,9 +21,6 @@ puts 'Starting...'
     end
   end
 
-  # Also renable the parent process' tracer
-  LightStep.enable
-
   10.times do
     span = LightStep.start_span("my_process_span-#{Process.pid}")
     sleep(0.0025 * rand(k))
@@ -35,6 +28,8 @@ puts 'Starting...'
   end
 
   # Make sure redundant enable calls don't cause problems
+  # NOTE: disabling discards the buffer by default, so all spans
+  # get cleared here except the final toggle span
   10.times do
     LightStep.disable
     LightStep.enable
@@ -48,7 +43,7 @@ puts 'Starting...'
   end
 
   puts "Parent, pid #{Process.pid}, waiting on child pid #{pid}"
-  Process.wait
+  Process.wait(pid)
 end
 
 puts 'Done!'

data/lib/lightstep/global_tracer.rb

@@ -14,7 +14,7 @@ module LightStep
 
     # Configure the GlobalTracer
     # See {LightStep::Tracer#initialize}
-    def configure(opts = nil)
+    def configure(**options)
       raise ConfigurationError, 'Already configured' if configured
       self.configured = true
       super

data/lib/lightstep/reporter.rb

@@ -0,0 +1,166 @@
+require 'concurrent/channel'
+
+module LightStep
+  # Reporter builds up reports of spans and flushes them to a transport
+  class Reporter
+    attr_accessor :max_span_records
+
+    def initialize(max_span_records:, transport:, guid:, component_name:)
+      @max_span_records = max_span_records
+      @span_records = Concurrent::Array.new
+      @dropped_spans = Concurrent::AtomicFixnum.new
+      @dropped_span_logs = Concurrent::AtomicFixnum.new
+      @transport = transport
+
+      start_time = LightStep.micros(Time.now)
+      @guid = LightStep.guid
+      @report_start_time = start_time
+
+      @runtime = {
+        guid: guid,
+        start_micros: start_time,
+        group_name: component_name,
+        attrs: [
+          {Key: "lightstep.tracer_platform",         Value: "ruby"},
+          {Key: "lightstep.tracer_version",          Value: LightStep::VERSION},
+          {Key: "lightstep.tracer_platform_version", Value: RUBY_VERSION}
+        ]
+      }.freeze
+
+      reset_on_fork
+
+      at_exit do
+        @quit_signal << true
+        @thread.join
+      end
+    end
+
+    def add_span(span)
+      reset_on_fork
+
+      @span_records.push(span.to_h)
+      if @span_records.size > max_span_records
+        dropped = @span_records.shift
+        @dropped_spans.increment
+        @dropped_span_logs.increment(dropped[:log_records].size + dropped[:dropped_logs])
+      end
+
+      @span_signal << true
+    end
+
+    def clear
+      span_records = @span_records.slice!(0, @span_records.length)
+      @dropped_spans.increment(span_records.size)
+      @dropped_span_logs.increment(
+        span_records.reduce(0) {|memo, span|
+          memo + span[:log_records].size + span[:dropped_logs]
+        }
+      )
+    end
+
+    def flush
+      @flush_signal << true
+      ~@flush_response_signal
+    end
+
+    private
+    MIN_PERIOD_SECS = 1.5
+    MAX_PERIOD_SECS = 30.0
+
+    # When the process forks, reset the child. All data that was copied will be handled
+    # by the parent. Also, restart the thread since forking killed it
+    def reset_on_fork
+      if @pid != $$
+        @pid = $$
+        @span_signal = Concurrent::Channel.new(buffer: :dropping, capacity: 1)
+        @quit_signal = Concurrent::Channel.new(buffer: :dropping, capacity: 1)
+        @flush_signal = Concurrent::Channel.new
+        @flush_response_signal = Concurrent::Channel.new
+        @span_records.clear
+        @dropped_spans.value = 0
+        @dropped_span_logs.value = 0
+        report_spans
+      end
+    end
+
+    def perform_flush
+      reset_on_fork
+      return if @span_records.empty?
+
+      now = LightStep.micros(Time.now)
+
+      span_records = @span_records.slice!(0, @span_records.length)
+      dropped_spans = 0
+      @dropped_spans.update{|old| dropped_spans = old; 0 }
+
+      old_dropped_span_logs = 0
+      @dropped_span_logs.update{|old| old_dropped_span_logs = old; 0 }
+      dropped_logs = old_dropped_span_logs
+      dropped_logs = span_records.reduce(dropped_logs) do |memo, span|
+        memo += span.delete :dropped_logs
+      end
+
+      report_request = {
+        runtime: @runtime,
+        oldest_micros: @report_start_time,
+        youngest_micros: now,
+        span_records: span_records,
+        counters: [
+          {Name: "dropped_logs",  Value: dropped_logs},
+          {Name: "dropped_spans", Value: dropped_spans},
+        ]
+      }
+
+      @report_start_time = now
+
+      begin
+        @transport.report(report_request)
+      rescue
+        # an error occurs, add the previous dropped logs to the logs
+        # that were going to get reported, as well as the previous dropped
+        # spans and spans that would have been recorded
+        @dropped_spans.increment(dropped_spans + span_records.length)
+        @dropped_span_logs.increment(old_dropped_span_logs)
+      end
+    end
+
+    def report_spans
+      @thread = Thread.new do
+        begin
+          loop do
+            min_reached = false
+            max_reached = false
+            min_timer = Concurrent::Channel.timer(MIN_PERIOD_SECS)
+            max_timer = Concurrent::Channel.timer(MAX_PERIOD_SECS)
+            loop do
+              Concurrent::Channel.select do |s|
+                s.take(@span_signal) do
+                  # we'll check span count below
+                end
+                s.take(min_timer) do
+                  min_reached = true
+                end
+                s.take(max_timer) do
+                  max_reached = true
+                end
+                s.take(@quit_signal) do
+                  perform_flush
+                  Thread.exit
+                end
+                s.take(@flush_signal) do
+                  perform_flush
+                  @flush_response_signal << true
+                end
+              end
+              if max_reached || (min_reached && @span_records.size >= max_span_records / 2)
+                perform_flush
+              end
+            end
+          end
+        rescue => ex
+          # TODO: internally log the exception
+        end
+      end
+    end
+  end
+end

data/lib/lightstep/tracer.rb

@@ -2,6 +2,7 @@ require 'json'
 require 'concurrent'
 
 require 'lightstep/span'
+require 'lightstep/reporter'
 require 'lightstep/transport/http_json'
 require 'lightstep/transport/nil'
 require 'lightstep/transport/callback'
@@ -11,16 +12,6 @@ module LightStep
     FORMAT_TEXT_MAP = 1
     FORMAT_BINARY = 2
 
-    CARRIER_TRACER_STATE_PREFIX = 'ot-tracer-'.freeze
-    CARRIER_BAGGAGE_PREFIX = 'ot-baggage-'.freeze
-
-    DEFAULT_MAX_LOG_RECORDS = 1000
-    MIN_MAX_LOG_RECORDS = 1
-    DEFAULT_MAX_SPAN_RECORDS = 1000
-    MIN_MAX_SPAN_RECORDS = 1
-    DEFAULT_MIN_REPORTING_PERIOD_SECS = 1.5
-    DEFAULT_MAX_REPORTING_PERIOD_SECS = 30.0
-
     class Error < LightStep::Error; end
     class ConfigurationError < LightStep::Tracer::Error; end
 
@@ -51,22 +42,7 @@ module LightStep
 
     def max_span_records=(max)
       @max_span_records = [MIN_MAX_SPAN_RECORDS, max].max
-    end
-
-    def min_flush_period_micros
-      @min_flush_period_micros ||= DEFAULT_MIN_REPORTING_PERIOD_SECS * 1E6
-    end
-
-    def min_reporting_period_secs=(secs)
-      @min_flush_period_micros = [DEFAULT_MIN_REPORTING_PERIOD_SECS, secs].max * 1E6
-    end
-
-    def max_flush_period_micros
-      @max_flush_period_micros ||= DEFAULT_MAX_REPORTING_PERIOD_SECS * 1E6
-    end
-
-    def max_reporting_period_secs=(secs)
-      @max_flush_period_micros = [DEFAULT_MAX_REPORTING_PERIOD_SECS, secs].min * 1E6
+      @reporter.max_span_records = @max_span_records
     end
 
     # TODO(ngauthier@gmail.com) inherit SpanContext from references
@@ -146,93 +122,50 @@ module LightStep
     # @param discard [Boolean] whether to discard queued data
     def disable(discard: true)
       @enabled = false
-      @transport.clear if discard
-      @transport.flush
+      @reporter.clear if discard
+      @reporter.flush
     end
 
     # Flush to the Transport
     def flush
-      _flush_worker
+      return unless enabled?
+      @reporter.flush
     end
 
     # Internal use only.
     # @private
     def finish_span(span)
       return unless enabled?
-      @span_records.push(span.to_h)
-      if @span_records.size > max_span_records
-        @span_records.shift
-        @dropped_spans.increment
-        @dropped_span_logs.increment(span.logs_count + span.dropped_logs_count)
-      end
-      flush_if_needed
+      @reporter.add_span(span)
     end
 
     protected
 
-    def access_token=(token)
-      if !access_token.nil?
-        raise ConfigurationError, "access token cannot be changed"
-      end
-      @access_token = token
-    end
-
     def configure(component_name:, access_token: nil, transport: nil)
       raise ConfigurationError, "component_name must be a string" unless String === component_name
       raise ConfigurationError, "component_name cannot be blank"  if component_name.empty?
 
-      @span_records = Concurrent::Array.new
-      @dropped_spans = Concurrent::AtomicFixnum.new
-      @dropped_span_logs = Concurrent::AtomicFixnum.new
-
-      start_time = LightStep.micros(Time.now)
-      @guid = LightStep.guid
-      @report_start_time = start_time
-      @last_flush_micros = start_time
+      transport = Transport::HTTPJSON.new(access_token: access_token) if !access_token.nil?
+      raise ConfigurationError, "you must provide an access token or a transport" if transport.nil?
+      raise ConfigurationError, "#{transport} is not a LightStep transport class" if !(LightStep::Transport::Base === transport)
 
-      @runtime = {
+      @reporter = LightStep::Reporter.new(
+        max_span_records: max_span_records,
+        transport: transport,
         guid: guid,
-        start_micros: start_time,
-        group_name: component_name,
-        attrs: [
-          {Key: "lightstep.tracer_platform",         Value: "ruby"},
-          {Key: "lightstep.tracer_version",          Value: LightStep::VERSION},
-          {Key: "lightstep.tracer_platform_version", Value: RUBY_VERSION}
-        ]
-      }.freeze
-
-      if !transport.nil?
-        if !(LightStep::Transport::Base === transport)
-          raise ConfigurationError, "transport is not a LightStep transport class: #{transport}"
-        end
-        @transport = transport
-      else
-        if access_token.nil?
-          raise ConfigurationError, "you must provide an access token or a transport"
-        end
-        @transport = Transport::HTTPJSON.new(access_token: access_token)
-      end
-
-      # At exit, flush this objects data to the transport and close the transport
-      # (which in turn will send the flushed data over the network).
-      at_exit do
-        flush
-        @transport.close
-      end
+        component_name: component_name
+      )
     end
 
-    def flush_if_needed
-      return unless enabled?
-
-      delta = LightStep.micros(Time.now) - @last_flush_micros
-      return if delta < min_flush_period_micros
+    private
 
-      if delta > max_flush_period_micros || @span_records.size >= max_span_records / 2
-        flush
-      end
-    end
+    CARRIER_TRACER_STATE_PREFIX = 'ot-tracer-'.freeze
+    CARRIER_BAGGAGE_PREFIX = 'ot-baggage-'.freeze
 
-    private
+    DEFAULT_MAX_LOG_RECORDS = 1000
+    MIN_MAX_LOG_RECORDS = 1
+    DEFAULT_MAX_SPAN_RECORDS = 1000
+    MIN_MAX_SPAN_RECORDS = 1
 
     def inject_to_text_map(span, carrier)
       carrier[CARRIER_TRACER_STATE_PREFIX + 'spanid'] = span.guid
@@ -261,50 +194,5 @@ module LightStep
       end
       span
     end
-
-    def _flush_worker
-      return unless enabled?
-      # The thrift configuration has not yet been set: allow logs and spans
-      # to be buffered in this case, but flushes won't yet be possible.
-      return if @runtime.nil?
-      return if @span_records.empty?
-
-      now = LightStep.micros(Time.now)
-
-      span_records = @span_records.slice!(0, @span_records.length)
-      dropped_spans = 0
-      @dropped_spans.update{|old| dropped_spans = old; 0 }
-
-      old_dropped_span_logs = 0
-      @dropped_span_logs.update{|old| old_dropped_span_logs = old; 0 }
-      dropped_logs = old_dropped_span_logs
-      dropped_logs = span_records.reduce(dropped_logs) do |memo, span|
-        memo += span.delete :dropped_logs
-      end
-
-      report_request = {
-        runtime: @runtime,
-        oldest_micros: @report_start_time,
-        youngest_micros: now,
-        span_records: span_records,
-        counters: [
-            {Name: "dropped_logs",  Value: dropped_logs},
-            {Name: "dropped_spans", Value: dropped_spans},
-        ]
-      }
-
-      @last_flush_micros = now
-      @report_start_time = now
-
-      begin
-        @transport.report(report_request)
-      rescue LightStep::Transport::HTTPJSON::QueueFullError
-        # If the queue is full, add the previous dropped logs to the logs
-        # that were going to get reported, as well as the previous dropped
-        # spans and spans that would have been recorded
-        @dropped_spans.increment(dropped_spans + span_records.length)
-        @dropped_span_logs.increment(old_dropped_span_logs)
-      end
-    end
   end
 end

data/lib/lightstep/transport/base.rb

@@ -8,15 +8,6 @@ module LightStep
       def report(_report)
         nil
       end
-
-      def close
-      end
-
-      def clear
-      end
-
-      def flush
-      end
     end
   end
 end

data/lib/lightstep/transport/callback.rb

@@ -11,15 +11,6 @@ module LightStep
         @callback.call(report)
         nil
       end
-
-      def close
-      end
-
-      def clear
-      end
-
-      def flush
-      end
     end
   end
 end

data/lib/lightstep/transport/http_json.rb

@@ -13,18 +13,15 @@ module LightStep
     class HTTPJSON < Base
       LIGHTSTEP_HOST = "collector.lightstep.com"
       LIGHTSTEP_PORT = 443
-      QUEUE_SIZE = 16
 
       ENCRYPTION_TLS = 'tls'
       ENCRYPTION_NONE = 'none'
 
-      class QueueFullError < LightStep::Error; end
-
       # Initialize the transport
       # @param host [String] host of the domain to the endpoind to push data
       # @param port [Numeric] port on which to connect
       # @param verbose [Numeric] verbosity level. Right now 0-3 are supported
-      # @param secure [Boolean]
+      # @param encryption [ENCRYPTION_TLS, ENCRYPTION_NONE] kind of encryption to use
       # @param access_token [String] access token for LightStep server
       # @return [HTTPJSON]
       def initialize(host: LIGHTSTEP_HOST, port: LIGHTSTEP_PORT, verbose: 0, encryption: ENCRYPTION_TLS, access_token:)
@@ -36,75 +33,24 @@ module LightStep
         raise ConfigurationError, "access_token must be a string" unless String === access_token
         raise ConfigurationError, "access_token cannot be blank"  if access_token.empty?
         @access_token = access_token
-
-        start_queue
       end
 
       # Queue a report for sending
       def report(report)
         p report if @verbose >= 3
-        # TODO(ngauthier@gmail.com): the queue could be full here if we're
-        # lagging, which would cause this to block!
-        @queue.push({
-          host: @host,
-          port: @port,
-          encryption: @encryption,
-          access_token: @access_token,
-          content: report,
-          verbose: @verbose
-        }, true)
-        nil
-      rescue ThreadError
-        raise QueueFullError
-      end
-
-      # Flush the current queue
-      def flush
-        close
-        start_queue
-      end
-
-      # Clear the current queue, deleting pending items
-      def clear
-        @queue.clear
-      end
 
-      # Close the transport. No further data can be sent!
-      def close
-        @queue.close
-        @thread.join
-      end
-
-      private
-
-      def start_queue
-        @queue = SizedQueue.new(QUEUE_SIZE)
-        @thread = start_thread(@queue)
-      end
-
-      # TODO(ngauthier@gmail.com) abort on exception?
-      def start_thread(queue)
-        Thread.new do
-          while item = queue.pop
-            post_report(item)
-          end
-        end
-      end
-
-      def post_report(params)
-        https = Net::HTTP.new(params[:host], params[:port])
-        https.use_ssl = params[:encryption] == ENCRYPTION_TLS
+        https = Net::HTTP.new(@host, @port)
+        https.use_ssl = @encryption == ENCRYPTION_TLS
         req = Net::HTTP::Post.new('/api/v0/reports')
-        req['LightStep-Access-Token'] = params[:access_token]
+        req['LightStep-Access-Token'] = @access_token
         req['Content-Type'] = 'application/json'
         req['Connection'] = 'keep-alive'
-        req.body = params[:content].to_json
+        req.body = report.to_json
         res = https.request(req)
 
-        puts res.to_s if params[:verbose] >= 3
+        puts res.to_s if @verbose >= 3
 
-        # TODO(ngauthier@gmail.com): log unknown commands
-        # TODO(ngauthier@gmail.com): log errors from server
+        nil
       end
     end
   end

data/lib/lightstep/version.rb

@@ -1,3 +1,3 @@
 module LightStep
-  VERSION = '0.9.3'.freeze
+  VERSION = '0.9.6'.freeze
 end

data/lightstep.gemspec

@@ -16,9 +16,10 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'concurrent-ruby', '~> 1.0.0'
-  spec.add_development_dependency 'rake', '~> 11.3.0'
-  spec.add_development_dependency 'rack', '~> 2.0.0'
+  spec.add_dependency 'concurrent-ruby', '~> 1.0'
+  spec.add_dependency 'concurrent-ruby-edge', '= 0.2.2'
+  spec.add_development_dependency 'rake', '~> 11.3'
+  spec.add_development_dependency 'rack', '~> 2.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'bump', '~> 0.5'
   spec.add_development_dependency 'simplecov', '~> 0.12.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lightstep
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.6
 platform: ruby
 authors:
 - bcronin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-02 00:00:00.000000000 Z
+date: 2016-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -16,42 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 11.3.0
+        version: '11.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 11.3.0
+        version: '11.3'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +146,7 @@ files:
 - examples/rack/hello.rb
 - lib/lightstep.rb
 - lib/lightstep/global_tracer.rb
+- lib/lightstep/reporter.rb
 - lib/lightstep/span.rb
 - lib/lightstep/tracer.rb
 - lib/lightstep/transport/base.rb