d13n 0.5.2

data/lib/d13n/metric/stream.rb

@@ -0,0 +1,322 @@
+require 'd13n/metric/stream_state'
+require 'd13n/metric/instrumentation/controller_instrumentation'
+require 'd13n/metric/stream/stream_tracer_helpers'
+require 'd13n/metric/stream/span_tracer_helpers'
+require 'securerandom'
+
+module D13n::Metric
+  class Stream
+
+    SINATRA_PREFIX = 'controller.sinatra'.freeze
+    MIDDLEWARE_PREFIX = 'controller.middleware'.freeze
+
+    WEB_TRANSACTION_CATEGORIES   = [:controller, :uri, :rack, :sinatra, :websocket].freeze
+
+    APDEX_S = 'S'.freeze
+    APDEX_T = 'T'.freeze
+    APDEX_F = 'F'.freeze
+
+    attr_accessor :state, :started_at, :uuid
+    attr_accessor :http_response_code,
+                  :request_content_length,
+                  :response_content_type,
+                  :response_content_length, :frame_stack
+
+    def self.st_current
+      StreamState.st_get.current_stream
+    end
+
+    def self.set_default_stream_name(name, category = nil, node_name = nil)
+      stream = st_current
+      name = stream.make_stream_name(name, category)
+      stream.name_last_frame(node_name || name)
+      stream.set_default_stream_name(name, category)
+    end
+
+    def self.start(state, category, options)
+      category ||= :controller
+      stream = state.current_stream
+
+      if stream 
+        stream.create_nested_stream(state, category, options)
+      else
+        stream = start_new_stream(state, category, options)
+      end
+      stream
+    rescue => e
+      D13n.logger.error("Exception during Stream.start", e)
+      nil
+    end
+
+    def self.start_new_stream(state, category, options)
+      stream = new(category, options)
+      state.reset(stream)
+      stream.state = state
+      stream.start(state)
+      stream
+    end
+
+    def self.stop(state, ended_time=Time.now.to_f)
+      stream = state.current_stream
+      if stream.nil?
+        D13n.logger.error("Failed during Stream.stop because there is no current stream")
+        return
+      end
+
+      nested_frame = stream.frame_stack.pop
+      if stream.frame_stack.empty?
+        stream.stop(state, ended_time, nested_frame)
+        state.reset
+      else
+        nested_name = nested_stream_name(nested_frame.name)
+        begin
+          D13n::Metric::Stream::SpanTracerHelpers.trace_footer(state, nested_frame.start_time.to_f, nested_name, nested_frame, {})
+        rescue => e
+          D13n.logger.debug "Error in trace_footer #{e}"
+        end
+      end
+
+      #:stream_stopped
+    end
+
+    def self.notice_error(exception, options={})
+      state = D13n::Metric::StreamState.st_get
+      stream = state.current_stream
+      if stream
+        stream.notice_error(exception, options)
+      end
+    end
+
+    def self.apdex_bucket(duration, failed, apdex_t)
+    
+      case
+      when failed
+        :apdex_f
+      when duration <= apdex_t
+        :apdex_s
+      when duration <= (4 * apdex_t)
+        :apdex_t
+      else
+        :apdex_f
+      end
+    end
+
+    def self.nested_stream_name(name)
+      name
+    end
+
+    def initialize(category, options)
+      @frame_stack = []
+
+      @category = category
+      @started_at = Time.now.to_f
+
+      @default_name = options[:stream_name]
+      
+      @apdex_started_at = options[:apdex_started_at] || @started_at
+
+      @ignore_apdex = false
+      @ignore_this_stream = false
+
+      @uuid = nil
+      @exceptions = {}
+      @request_content_length = 0
+      @response_content_type = nil
+      @response_content_length = 0
+    end
+
+    def start(state)
+      @frame_stack.push D13n::Metric::Stream::SpanTracerHelpers.trace_header(state, @started_at)
+      name_last_frame @default_name
+    end
+
+    def stop(state, ended_time, outermost_frame)
+      trace_options = {}
+      if @has_children
+        name = self.class.nested_stream_name(outermost_frame.name)
+      else
+        name = @frozen_name
+      end
+
+      D13n::Metric::Stream::SpanTracerHelpers.trace_footer(state, @started_at.to_f, name, outermost_frame, trace_options, ended_time.to_f)
+      duration = ended_time - @started_at
+      exclusive = duration - outermost_frame.children_time
+      commit!(state, exclusive, ended_time) unless @ignore_this_stream
+    end
+
+    def commit!(state, exclusive, ended_time)
+      @metric_data = {:exclusive => exclusive}
+      collect_metric_data(state, @metric_data, ended_time)
+      collect_metrics(state, @metric_data)
+    end
+
+    def apdex_t
+      stream_specific_apdex_t || D13n.config[:'metric.app.http.in.apdex_t']
+    end
+
+    def stream_specific_apdex_t
+      key = "metric.app.http.in.apdex_t.#{@frozen_name}".to_sym
+      D13n.config[key]
+    end
+
+    def collect_metrics(state, metric_data)
+      StreamTracerHelpers.collect_metrics(state, metric_data)
+    end
+
+    def collect_metric_data(state, metric_data, ended_time = Time.now.to_f)
+      total_duration = ended_time - @apdex_started_at
+      action_duration = ended_time - @started_at.to_f
+
+      # collect_apdex_metric(total_duration, action_duration, apdex_t)
+      generate_metric_data(state, metric_data, @started_at, ended_time)
+    end
+
+    # def collect_apdex_metric(total_duration, action_duration, current_apdex_t)
+    #   apdex_bucket_global = apdex_bucket(total_duration, current_apdex_t)
+    #   apdex_bucket_stream = apdex_bucket(action_duration, current_apdex_t)
+    # end
+
+    def default_metric_data
+      metric_data = {
+        :name => @frozen_name || @default_name,
+        :uuid => uuid,
+        :error => false
+      }
+
+      generate_error_data(metric_data)
+      metric_data[:referring_stream_id] = @state.referring_stream_id if @state.referring_stream_id
+      metric_data
+    end
+
+    def generate_error_data(metric_data)
+      if had_error?
+        metric_data.merge!({
+         :error => true,
+         :errors => @exceptions 
+        })
+      end
+    end
+
+    def generate_default_metric_data(state, started_at, ended_time, metric_data)
+      duration = ended_time - started_at
+      
+      metric_data.merge!(default_metric_data)
+      metric_data.merge!({
+        :type => :request,
+        :started_at => started_at,
+        :duration => duration,
+      })
+      metric_data
+    end
+
+    def generate_metric_data(state, metric_data, started_at, ended_time)
+      duration = ended_time - started_at
+      metric_data ||= {}
+      generate_default_metric_data(state, started_at, ended_time, metric_data)
+      append_apdex_perf_zone(duration, metric_data)
+      append_web_response(@http_response_code, @response_content_type, @response_content_length, @request_content_length, metric_data) if recording_web_transaction?
+      metric_data
+    end
+
+    def had_error?
+      !@exceptions.empty?
+    end
+
+    def had_exception_affecting_apdex?
+      # all exceptions are affecting
+      had_error?  
+    end
+
+    def apdex_bucket(duration, current_apdex_t)
+      self.class.apdex_bucket(duration, had_exception_affecting_apdex?, current_apdex_t)
+    end
+
+    def append_apdex_perf_zone(duration, metric_data)
+      bucket = apdex_bucket(duration, apdex_t)
+
+      return unless bucket
+ 
+      bucket_str = case bucket
+      when :apdex_s then APDEX_S
+      when :apdex_t then APDEX_T
+      when :apdex_f then APDEX_F
+      else nil
+      end
+
+      metric_data[:apdex_perf_zone] = bucket_str if bucket_str
+    end
+
+    def append_web_response(http_response_code,response_content_type,response_content_length,request_content_length,metric_data)
+      return if http_response_code.nil?
+
+      metric_data[:http_response_code] = http_response_code
+      metric_data[:http_response_content_type] = response_content_type
+      metric_data[:http_response_content_length] = response_content_length
+      metric_data[:http_request_content_length] = request_content_length
+    end
+
+    def make_stream_name(name, category=nil)
+      namer = Instrumentation::ControllerInstrumentation::StreamNamer
+      "#{namer.prefix_for_category(self, category)}.#{name}"
+    end
+
+    def set_default_stream_name(name, category)
+      @default_name = name
+      @category = category if category
+    end
+    
+    def name_last_frame(name)
+      @frame_stack.last.name = name
+    end
+
+    def notice_error(exception, options={})
+      if @exceptions[exception]
+        @exceptions[exception].merge! options
+      else
+        @exceptions[exception] = options
+      end
+    end
+
+    def create_nested_stream(state, category, options)
+      @has_children = true
+
+      @frame_stack.push D13n::Metric::Stream::SpanTracerHelpers.trace_header(state, Time.now.to_f)
+      name_last_frame(options[:stream_name])
+
+      set_default_stream_name(options[:stream_name], category)
+    end
+
+    def set_default_stream_name(name, category)
+      return if name_frozen?
+      @default_name = name
+      @category = category if category
+    end
+
+    def name_frozen?
+      @frozen_name ? true : false
+    end
+
+    def name_last_frame(name)
+      @frame_stack.last.name = name
+    end
+
+    def recording_web_transaction?
+      web_category?(@category)
+    end
+
+    def web_category?(category)
+      WEB_TRANSACTION_CATEGORIES.include?(category)
+    end
+
+    def get_id
+      uuid
+    end
+
+    def uuid
+      return @uuid if @uuid
+      request_info = StreamState.request_info
+      @uuid = (request_info.nil? || request_info["request_id"].nil?) ? SecureRandom.hex(16) : request_info["request_id"]
+      @uuid
+    end
+  end
+end

data/lib/d13n/metric/stream_state.rb

@@ -0,0 +1,68 @@
+require 'd13n/metric/stream/traced_span_stack'
+module D13n::Metric
+  class StateError < D13n::Error;end
+  class StreamState
+
+    D13N_STREAM_HEADER = 'X-D13n-Stream-ID'
+    D13N_APP_HEADER = 'X-D13n-App'
+    REQUEST_ID_HEADER = 'HTTP_X_REQUEST_ID'
+
+    def self.st_get
+      st_state_for(Thread.current)
+    end
+
+    def self.st_state_for(thread)
+      thread[:d13n_stream_state] ||= new
+    end
+
+    def self.request_info
+      st_get.request_info
+    end
+
+    def self.default_metric_data
+      state = st_get
+      stream = state.current_stream
+      if stream
+        stream.default_metric_data
+      else
+        {}
+      end
+    end
+
+    attr_reader :traced_span_stack
+    attr_accessor :request_info
+    attr_accessor :tag_hash
+    attr_accessor :current_stream
+    attr_accessor :request
+    attr_accessor :referring_stream_id, :is_cross_app_caller
+
+    def initialize
+      @traced_span_stack = D13n::Metric::Stream::TracedSpanStack.new
+      @current_stream = nil
+      @referring_stream_id = nil
+      @is_cross_app_caller = false
+    end
+
+    def reset(stream=nil)
+      @traced_span_stack.clear
+      @current_stream = stream
+      @is_cross_app_caller = false
+    end
+
+    def notify_rack_call(request)
+      notify_call(request)
+    end
+
+    def notify_call(request)
+      save_referring_stream_id(request)
+    end
+
+    def save_referring_stream_id(request)
+      @referring_stream_id = request[D13N_STREAM_HEADER] if request[D13N_STREAM_HEADER]
+    end
+
+    def clear_referring_stream_id()
+      @referring_stream_id = nil
+    end
+  end
+end

data/lib/d13n/metric.rb

@@ -0,0 +1,11 @@
+module D13n::Metric
+  class MetricError < D13n::Error; end
+end
+require 'd13n/metric/stream'
+require 'd13n/metric/stream_state'
+require 'd13n/metric/http_clients'
+require 'd13n/metric/manager'
+
+module D13n::Metric
+  class MetricError < D13n::Error;end
+end

data/lib/d13n/rack/d13n_middleware.rb

@@ -0,0 +1,21 @@
+require 'd13n/metric/instrumentation/controller_instrumentation'
+module D13n
+  module Rack
+    class D13nMiddleware
+      attr_reader :stream_options, :category
+      def initialize(app, options={})
+        @app = app
+        @category = :middleware
+        @target  = self
+        @stream_options = {
+          :stream_name => build_stream_name
+        }
+      end
+
+      def build_stream_name
+        prefix = ::D13n::Metric::Instrumentation::ControllerInstrumentation::StreamNamer.prefix_for_category(nil, @category)
+        "#{prefix}.#{self.class.name}.call"
+      end
+    end
+  end
+end

data/lib/d13n/rack/metric_middleware.rb

@@ -0,0 +1,18 @@
+require 'd13n/metric/stream_state'
+require 'd13n/metric/instrumentation/middleware_tracing'
+require 'd13n/rack/d13n_middleware'
+module D13n
+  module Rack
+    class MetricMiddleware < D13nMiddleware
+      include D13n::Metric::Instrumentation::MiddlewareTracing
+      
+      def self.enabled?
+        D13n.config[:'metric.app.http.in.tracable']
+      end
+  
+      def traced_call(env)
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/d13n/service/background_job/sinatra.rb

@@ -0,0 +1,24 @@
+module Sinatra
+  class Base
+    class << self
+      alias_method :run_without_callback, :run!
+
+      def background_job(opts = {}, &block)
+        @background_jobs ||= []
+        @background_jobs << {opts: opts, blk: block}
+      end
+
+      def background_job_run!
+        return if @background_jobs.nil?
+        @background_jobs.each do |j|
+          j[:blk].call(j[:opts])
+        end
+      end
+
+      def run!(*args, &block)
+        background_job_run!
+        run_without_callback(*args, &block)
+      end
+    end
+  end
+end

data/lib/d13n/service/background_job.rb

@@ -0,0 +1 @@
+require 'd13n/service/background_job/sinatra' if defined?(::Sinatra) && defined?(::Sinatra::Base)

data/lib/d13n/service/start.rb

@@ -0,0 +1,75 @@
+require 'config_kit'
+
+module D13n
+  class Service
+    module Start
+      def start_metric_manager
+        begin
+          channel = determine_metric_channel
+          Metric::Manager.start(channel)
+        rescue Exception => e
+          D13n.logger.error "Unknown Instrument Manager error. #{e.message}"
+        end
+      end
+
+      def start_api_service
+        @started = true
+        Object.const_get("#{D13n.application.name}::Api::Service").run!(@service_conf)
+      end
+
+      def log_startup
+        log_environment
+        log_service_name
+      end
+
+      def log_environment
+        D13n.logger.info "Environment: #{D13n.service.instance.env}"
+      end
+
+      def log_service_name
+        D13n.logger.info "Service: #{D13n.config.app_name}"
+      end
+
+      def log_version_and_pid
+        D13n.logger.debug "#{D13n.config.app_name} Service #{D13n.service.app_class::VERSION}(D13n:#{D13n::VERSION::STRING}) Initialized: pid = #{$$}"
+      end
+
+      def query_server_for_configuration
+        begin
+          #
+          # TODO: Config Kit may enhance configurator
+          #
+          ConfigKit.config.url = D13n.config[:ck_url] || D13n.config[:'service.config_kit.url']
+          config_data = ConfigKit::Manager.get(D13n.config.app_name)
+        rescue ConfigKit::Client::ConfigKitReadError => e
+          D13n.logger.error "Config Kit reads server error. #{e.message}"
+          raise ServiceStartError.new(e.message) if config_server_raise_on_failure?
+        rescue Exception => e
+          D13n.logger.error "Unknown Config Kit error. #{e.message}"
+          D13n.logger.debug e
+          raise ServiceStartError.new(e.message) if config_server_raise_on_failure?
+        end
+        #
+        # TODO: Empty to raise exception
+        #
+        return if config_data.nil? || config_data.empty?
+
+        D13n.logger.debug "Server provided config: #{config_data.inspect}"
+
+        server_config = D13n::Configuration::ServerSource.build(config_data)
+        D13n.config.replace_or_add_config(server_config)
+      end
+
+      def config_server_raise_on_failure?
+        D13n.config[:'config_server.raise_on_failure'] == true || D13n.config[:'config_server.raise_on_failure'] == 'true'
+      end
+
+      private
+
+      def determine_metric_channel
+        D13n.config[:metric_channel_type] || D13n.config[:'metric.channel.type']
+      end
+
+    end
+  end
+end

data/lib/d13n/service.rb

@@ -0,0 +1,91 @@
+require 'd13n/ext/string'
+require 'd13n/service/start'
+module D13n
+  class Service
+    class ServiceError < Error; end
+    class ServiceStartError < ServiceError; end
+
+    include Start
+    class << self
+      def instance
+        @instance ||= new
+      end
+
+      def run!(opts)
+        D13n.service.instance.init(opts)
+        D13n.service.instance.start
+      end
+
+      def app_class
+        @app_kls ||= Object.const_get(name.split("::").first)
+      end
+
+      def inherited(descendant)
+        raise ServiceError, "You cannot have more than one D13n::Service" if D13n.service && descendant == D13n::Service
+        descendant.app_class.extend Application::ClassMethods
+        D13n.service = descendant
+      end
+    end
+
+    attr_reader :service_conf, :env, :service_prefix, :started
+    def initialize()
+      @started=false
+      @service_prefix = D13n.service.app_class.name.underscore.upcase
+    end
+
+    def init(opts)
+      determine_service_conf(opts)
+      determine_env(opts)
+      config_service(opts)
+      determine_logger(opts)
+    end
+
+    def determine_logger(opts)
+      if D13n.logger.is_startup_logger?
+        D13n.logger = D13n::Logger.new(root,opts.delete(:logger))
+      end
+    end
+
+    def start
+      return if started?
+      log_startup
+      query_server_for_configuration
+      start_metric_manager
+      log_version_and_pid
+      start_api_service
+    end
+
+    def determine_service_conf(opts={})
+      @service_conf ||= {}
+      @service_conf[:port] = opts.fetch(:port, 3000)
+      @service_conf[:bind] = opts.fetch(:host, 'localhost')
+    end
+
+    def determine_env(opts={})
+      @env = opts.fetch(:env, default_env).to_s
+    end
+
+    def config_service(opts)
+      config_file_path = D13n.config[:config_path]
+      D13n.config.replace_or_add_config(D13n::Configuration::YamlSource.new(config_file_path, env))
+    end
+
+    def started?
+      @started
+    end
+
+    def root
+      @root = ENV['SERVICE_ROOT'] || '.'
+    end
+
+    def settings
+      D13n.config.to_collecter_hash
+    end
+
+  private
+  
+    def default_env
+      ENV["#{D13n.app_prefix}_ENV"] || ENV['RACK_ENV'] || 'development'
+    end
+  end
+end

data/lib/d13n/support/request_id.rb

@@ -0,0 +1,29 @@
+require 'securerandom'
+
+module Rack
+  class RequestId
+    X_REQUEST_ID = "X-Request-Id".freeze
+
+    def initialize(app, opts = {})
+      @app = app
+    end
+
+    def call(env)
+      env['HTTP_X_REQUEST_ID'] = make_request_id(env[X_REQUEST_ID]||env['HTTP_X_REQUEST_ID'])
+      @app.call(env).tap {|_status, headers, _body| headers[X_REQUEST_ID] = env['HTTP_X_REQUEST_ID'] }
+    end
+
+    private
+    def make_request_id(request_id)
+      if request_id.nil?
+        internal_request_id
+      else
+        request_id
+      end
+    end
+
+    def internal_request_id
+      SecureRandom.hex(16)
+    end
+  end
+end

data/lib/d13n/version.rb

@@ -0,0 +1,14 @@
+module D13n
+  module VERSION
+    MAJOR     = 0
+    MINOR     = 5
+    TINY      = 2
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+
+    CODENAME = 'd13n'
+  end
+
+  NAME = 'd13n'.freeze
+  RELEASE  = "#{NAME} #{VERSION::STRING} codename #{VERSION::CODENAME}".freeze
+end