mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/obfuscator.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+#require 'base64'
+
+module Mmtrix
+  module Agent
+    class Obfuscator
+
+      attr_reader :key_bytes
+
+      EMPTY_KEY_BYTES = [0]
+      PACK_FORMAT = 'm'
+
+      # RUM uses a shortened key, so just trim it up front
+      def initialize(key, length=nil)
+        if key.nil? || key.empty?
+          @key_bytes = EMPTY_KEY_BYTES
+        else
+          @key_bytes = key.bytes.to_a
+          @key_bytes = @key_bytes.first(length) if length
+        end
+      end
+
+      def obfuscate(text)
+        [ encode(text) ].pack(PACK_FORMAT).gsub(/\n/, '')
+      end
+
+      def deobfuscate(text)
+        encode(text.unpack(PACK_FORMAT).first )
+      end
+
+      def encode(text)
+        return text unless key_bytes
+
+        encoded = ""
+        encoded.force_encoding('binary') if encoded.respond_to?( :force_encoding )
+        index = 0
+        text.each_byte do |byte|
+          encoded.concat((byte ^ key_bytes[index % key_bytes.length]))
+          index+=1
+        end
+        encoded
+      end
+
+    end
+  end
+end

data/lib/mmtrix/agent/parameter_filtering.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix"s license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module ParameterFiltering
+      extend self
+
+      def apply_filters(env, params)
+        params = filter_using_rails(env, params)
+        params = filter_rack_file_data(env, params)
+        params
+      end
+
+      def filter_using_rails(env, params)
+        return params unless defined?(ActionDispatch::Http::ParameterFilter) && env.key?("action_dispatch.parameter_filter")
+        filters = env["action_dispatch.parameter_filter"]
+        ActionDispatch::Http::ParameterFilter.new(filters).filter(params)
+      end
+
+      def filter_rack_file_data(env, params)
+        content_type = env["CONTENT_TYPE"]
+        multipart = content_type && content_type.start_with?("multipart")
+
+        params.inject({}) do |memo, (k,v)|
+          if multipart && v.is_a?(Hash) && v[:tempfile]
+            memo[k] = "[FILE]"
+          else
+            memo[k] = v
+          end
+          memo
+        end
+      end
+
+      def filter_rails_request_parameters(params)
+        result = params.dup
+        result.delete("controller")
+        result.delete("action")
+        result
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/pipe_channel_manager.rb

@@ -0,0 +1,277 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'base64'
+
+module Mmtrix
+  module Agent
+
+    #--
+    # Manages the registering and servicing of pipes used by child
+    # processes to report data to their parent, rather than directly
+    # to the collector.
+    module PipeChannelManager
+      extend self
+
+      def register_report_channel(id)
+        listener.register_pipe(id)
+      end
+
+      def channels
+        listener.pipes
+      end
+
+      def listener
+        @listener ||= Listener.new
+      end
+
+      # Expected initial sequence of events for Pipe usage:
+      #
+      # 1. Pipe is created in parent process (read and write ends open)
+      # 2. Parent process forks
+      # 3. An after_fork hook is invoked in the child
+      # 4. From after_fork hook, child closes read end of pipe, and
+      #    writes a ready marker on the pipe (after_fork_in_child).
+      # 5. The parent receives the ready marker, and closes the write end of the
+      #    pipe in response (after_fork_in_parent).
+      #
+      # After this sequence of steps, an exit (whether clean or not) of the
+      # child will result in the pipe being marked readable again, and giving an
+      # EOF marker (nil) when read. Note that closing of the unused ends of the
+      # pipe in the parent and child processes is essential in order for the EOF
+      # to be correctly triggered. The ready marker mechanism is used because
+      # there's no easy hook for after_fork in the parent process.
+      #
+      # This class provides message framing (separation of individual messages),
+      # but not serialization. Serialization / deserialization is the
+      # responsibility of clients.
+      #
+      # Message framing works like this:
+      #
+      # Each message sent across the pipe is preceded by a length tag that
+      # specifies the length of the message that immediately follows, in bytes.
+      # The length tags are serialized as unsigned big-endian long values, (4
+      # bytes each). This means that the maximum theoretical message size is
+      # 4 GB - much larger than we'd ever need or want for this application.
+      #
+      class Pipe
+        READY_MARKER = "READY"
+        NUM_LENGTH_BYTES = 4
+
+        attr_accessor :in, :out
+        attr_reader :last_read, :parent_pid
+
+        def initialize
+          @out, @in = IO.pipe
+          if defined?(::Encoding::ASCII_8BIT)
+            @in.set_encoding(::Encoding::ASCII_8BIT)
+          end
+          @last_read = Time.now
+          @parent_pid = $$
+        end
+
+        def close
+          @out.close unless @out.closed?
+          @in.close unless @in.closed?
+        end
+
+        def serialize_message_length(data)
+          [data.bytesize].pack("L>")
+        end
+
+        def deserialize_message_length(data)
+          data.unpack("L>").first
+        end
+
+        def write(data)
+          @out.close unless @out.closed?
+          @in << serialize_message_length(data)
+          @in << data
+        end
+
+        def read
+          @in.close unless @in.closed?
+          @last_read = Time.now
+          length_bytes = @out.read(NUM_LENGTH_BYTES)
+          if length_bytes
+            message_length = deserialize_message_length(length_bytes)
+            if message_length
+              @out.read(message_length)
+            else
+              length_hex = length_bytes.bytes.map { |b| b.to_s(16) }.join(' ')
+              Mmtrix::Agent.logger.error("Failed to deserialize message length from pipe. Bytes: [#{length_hex}]")
+              nil
+            end
+          else
+            Mmtrix::Agent.logger.error("Failed to read bytes for length from pipe.")
+            nil
+          end
+        end
+
+        def eof?
+          !@out.closed? && @out.eof?
+        end
+
+        def after_fork_in_child
+          @out.close unless @out.closed?
+          write(READY_MARKER)
+        end
+
+        def after_fork_in_parent
+          @in.close unless @in.closed?
+        end
+
+        def closed?
+          @out.closed? && @in.closed?
+        end
+      end
+
+      class Listener
+        attr_reader   :thread
+
+        # This attr_accessor intentionally provides unsynchronized access to the
+        # @pipes hash. It is used to look up the write end of the pipe from
+        # within the Resque child process, and must be unsynchronized in order
+        # to avoid a potential deadlock in which the PipeChannelManager::Listener
+        # thread in the parent process is holding the @pipes_lock at the time of
+        # the fork.
+        attr_accessor :pipes, :timeout, :select_timeout
+
+        def initialize
+          @pipes = {}
+          @pipes_lock = Mutex.new
+
+          @timeout = 360
+          @select_timeout = 60
+        end
+
+        def wakeup
+          wake.in << '.'
+        end
+
+        def register_pipe(id)
+          @pipes_lock.synchronize do
+            @pipes[id] = Pipe.new
+          end
+
+          wakeup
+        end
+
+        def start
+          return if @started == true
+          @started = true
+          @thread = Mmtrix::Agent::Threading::AgentThread.create('Pipe Channel Manager') do
+            now = nil
+            loop do
+              clean_up_pipes
+
+              pipes_to_listen_to = @pipes_lock.synchronize do
+                @pipes.values.map{|pipe| pipe.out} + [wake.out]
+              end
+
+              Mmtrix::Agent.record_metric('Supportability/Listeners',
+                (Time.now - now).to_f) if now
+
+              if ready = IO.select(pipes_to_listen_to, [], [], @select_timeout)
+                now = Time.now
+
+                ready_pipes = ready[0]
+                ready_pipes.each do |pipe|
+                  merge_data_from_pipe(pipe) unless pipe == wake.out
+                end
+
+                wake.out.read(1) if ready_pipes.include?(wake.out)
+              end
+
+              break unless should_keep_listening?
+            end
+          end
+          sleep 0.001 # give time for the thread to spawn
+        end
+
+        def stop_listener_thread
+          @started = false
+          wakeup
+          @thread.join
+        end
+
+        def stop
+          return unless @started == true
+          stop_listener_thread
+          close_all_pipes
+          @wake.close
+          @wake = nil
+        end
+
+        def close_all_pipes
+          @pipes_lock.synchronize do
+            @pipes.each do |id, pipe|
+              pipe.close if pipe
+            end
+            @pipes = {}
+          end
+        end
+
+        def wake
+          @wake ||= Pipe.new
+        end
+
+        def started?
+          @started
+        end
+
+        protected
+
+        def merge_data_from_pipe(pipe_handle)
+          pipe = find_pipe_for_handle(pipe_handle)
+          raw_payload = pipe.read
+          if raw_payload && !raw_payload.empty?
+            if raw_payload == Pipe::READY_MARKER
+              pipe.after_fork_in_parent
+            else
+              payload = unmarshal(raw_payload)
+              if payload
+                endpoint, items = payload
+                Mmtrix::Agent.agent.merge_data_for_endpoint(endpoint, items)
+              end
+            end
+          end
+
+          pipe.close if pipe.eof?
+        end
+
+        def unmarshal(data)
+          Mmtrix::LanguageSupport.with_cautious_gc do
+            Marshal.load(data)
+          end
+        rescue StandardError => e
+          ::Mmtrix::Agent.logger.error "Failure unmarshalling message from Resque child process", e
+          ::Mmtrix::Agent.logger.debug Base64.encode64(data)
+          nil
+        end
+
+        def should_keep_listening?
+          @started || @pipes_lock.synchronize { @pipes.values.find{|pipe| !pipe.in.closed?} }
+        end
+
+        def clean_up_pipes
+          @pipes_lock.synchronize do
+            @pipes.values.each do |pipe|
+              if pipe.last_read.to_f + @timeout < Time.now.to_f
+                pipe.close unless pipe.closed?
+              end
+            end
+            @pipes.reject! {|id, pipe| pipe.out.closed? }
+          end
+        end
+
+        def find_pipe_for_handle(out_handle)
+          @pipes_lock.synchronize do
+            @pipes.values.find{|pipe| pipe.out == out_handle }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/pipe_service.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class PipeService
+      attr_reader :channel_id, :buffer, :pipe
+      attr_accessor :request_timeout, :agent_id, :collector
+
+      def initialize(channel_id)
+        @channel_id = channel_id
+        @collector = Mmtrix::Control::Server.new(:name => 'parent',
+                                                   :port => 0)
+        @pipe = Mmtrix::Agent::PipeChannelManager.channels[@channel_id]
+        if @pipe && @pipe.parent_pid != $$
+          @pipe.after_fork_in_child
+        else
+          Mmtrix::Agent.logger.error("No communication channel to parent process, please see https://mmtrix.com/docs/ruby/resque-instrumentation for more information.")
+        end
+      end
+
+      def connect(config)
+        nil
+      end
+
+      def get_agent_commands
+        []
+      end
+
+      def analytic_event_data(events)
+        write_to_pipe(:analytic_event_data, events) if events
+      end
+
+      def custom_event_data(events)
+        write_to_pipe(:custom_event_data, events) if events
+      end
+
+      def metric_data(unsent_timeslice_data)
+        write_to_pipe(:metric_data, unsent_timeslice_data)
+        {}
+      end
+
+      def transaction_sample_data(transactions)
+        write_to_pipe(:transaction_sample_data, transactions) if transactions
+      end
+
+      def error_data(errors)
+        write_to_pipe(:error_data, errors) if errors
+      end
+
+      def sql_trace_data(sql)
+        write_to_pipe(:sql_trace_data, sql) if sql
+      end
+
+      def shutdown(time)
+        @pipe.close if @pipe
+      end
+
+      # Invokes the block it is passed.  This is used to implement HTTP
+      # keep-alive in the MmtrixService, and is a required interface for any
+      # Service class.
+      def session
+        yield
+      end
+
+      def reset_metric_id_cache
+        # we don't cache metric IDs, so nothing to do
+      end
+
+      private
+
+      def marshal_payload(data)
+        Mmtrix::LanguageSupport.with_cautious_gc do
+          Marshal.dump(data)
+        end
+      end
+
+      def write_to_pipe(endpoint, data)
+        @pipe.write(marshal_payload([endpoint, data])) if @pipe
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/rules_engine/replacement_rule.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class RulesEngine
+      class ReplacementRule
+        attr_reader(:terminate_chain, :each_segment, :ignore, :replace_all, :eval_order,
+                    :match_expression, :replacement)
+
+        def initialize(options)
+          if !options['match_expression']
+            raise ArgumentError.new('missing required match_expression')
+          end
+          if !options['replacement'] && !options['ignore']
+            raise ArgumentError.new('must specify replacement when ignore is false')
+          end
+
+          @match_expression = Regexp.new(options['match_expression'], Regexp::IGNORECASE)
+          @replacement      = options['replacement']
+          @ignore           = options['ignore'] || false
+          @eval_order       = options['eval_order'] || 0
+          @replace_all      = options['replace_all'] || false
+          @each_segment     = options['each_segment'] || false
+          @terminate_chain  = options['terminate_chain'] || false
+        end
+
+        def terminal?
+          @terminate_chain || @ignore
+        end
+
+        def matches?(string)
+          if @each_segment
+            string.split(SEGMENT_SEPARATOR).any? do |segment|
+              segment.match(@match_expression)
+            end
+          else
+            string.match @match_expression
+          end
+        end
+
+        def apply(string)
+          if @ignore
+            nil
+          elsif @each_segment
+            apply_to_each_segment(string)
+          else
+            apply_replacement(string)
+          end
+        end
+
+        def apply_replacement(string)
+          method = @replace_all ? :gsub : :sub
+          string.send(method, @match_expression, @replacement)
+        end
+
+        def apply_to_each_segment(string)
+          string        = string.dup
+          leading_slash = string.slice!(LEADING_SLASH_REGEX)
+          segments      = string.split(SEGMENT_SEPARATOR)
+
+          segments.map! do |segment|
+            apply_replacement(segment)
+          end
+
+          "#{leading_slash}#{segments.join(SEGMENT_SEPARATOR)}"
+        end
+
+        def <=>(other)
+          eval_order <=> other.eval_order
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/rules_engine/segment_terms_rule.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class RulesEngine
+      class SegmentTermsRule
+        SEGMENT_PLACEHOLDER               = '*'.freeze
+        ADJACENT_PLACEHOLDERS_REGEX       = %r{((?:^|/)\*)(?:/\*)*}.freeze
+        ADJACENT_PLACEHOLDERS_REPLACEMENT = '\1'.freeze
+
+        attr_reader :prefix, :terms
+
+        def initialize(options)
+          @prefix          = options['prefix']
+          @terms           = options['terms']
+          @trim_range      = (@prefix.size..-1)
+        end
+
+        def terminal?
+          true
+        end
+
+        def matches?(string)
+          string.start_with?(@prefix)
+        end
+
+        def apply(string)
+          rest          = string[@trim_range]
+          leading_slash = rest.slice!(LEADING_SLASH_REGEX)
+
+          segments = rest.split(SEGMENT_SEPARATOR)
+          segments.map! { |s| @terms.include?(s) ? s : SEGMENT_PLACEHOLDER }
+          transformed_suffix = collapse_adjacent_placeholder_segments(segments)
+
+          "#{@prefix}#{leading_slash}#{transformed_suffix}"
+        end
+
+        def collapse_adjacent_placeholder_segments(segments)
+          joined = segments.join(SEGMENT_SEPARATOR)
+          joined.gsub!(ADJACENT_PLACEHOLDERS_REGEX, ADJACENT_PLACEHOLDERS_REPLACEMENT)
+          joined
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/rules_engine.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/rules_engine/replacement_rule'
+require 'mmtrix/agent/rules_engine/segment_terms_rule'
+
+module Mmtrix
+  module Agent
+    class RulesEngine
+      SEGMENT_SEPARATOR = '/'.freeze
+      LEADING_SLASH_REGEX = %r{^/}.freeze
+
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@rules, :size, :inspect, :each, :clear
+
+      def self.create_metric_rules(connect_response)
+        specs = connect_response['metric_name_rules'] || []
+        rules = specs.map { |spec| ReplacementRule.new(spec) }
+        self.new(rules)
+      end
+
+      def self.create_transaction_rules(connect_response)
+        txn_name_specs     = connect_response['transaction_name_rules']    || []
+        segment_rule_specs = connect_response['transaction_segment_terms'] || []
+
+        txn_name_rules = txn_name_specs.map     { |s| ReplacementRule.new(s) }
+        segment_rules  = segment_rule_specs.map { |s| SegmentTermsRule.new(s) }
+
+        self.new(txn_name_rules, segment_rules)
+      end
+
+      def initialize(rules=[], segment_term_rules=[])
+        @rules = rules.sort
+        @segment_term_rules = segment_term_rules
+      end
+
+      def apply_rules(rules, string)
+        rules.each do |rule|
+          if rule.matches?(string)
+            string = rule.apply(string)
+            break if rule.terminal?
+          end
+        end
+        string
+      end
+
+      def rename(original_string)
+        renamed = apply_rules(@rules, original_string)
+        return nil unless renamed
+        renamed = apply_rules(@segment_term_rules, renamed)
+        renamed
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/sampled_buffer.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This class acts like an Array with a fixed capacity that randomly samples
+# from a stream of items such that the probability of each item being included
+# in the Array is equal. It uses reservoir sampling in order to achieve this:
+# http://xlinux.nist.gov/dads/HTML/reservoirSampling.html
+
+require 'mmtrix/agent/event_buffer'
+
+module Mmtrix
+  module Agent
+    class SampledBuffer < EventBuffer
+      attr_reader :seen_lifetime, :captured_lifetime
+
+      def initialize(capacity)
+        super
+        @captured_lifetime = 0
+        @seen_lifetime     = 0
+      end
+
+      def reset!
+        @captured_lifetime += @items.size
+        @seen_lifetime     += @seen
+        super
+      end
+
+      def append_event(x)
+        if @items.size < @capacity
+          @items << x
+          return x
+        else
+          m = rand(@seen) # [0, @seen)
+          if m < @capacity
+            @items[m] = x
+            return x
+          else
+            # discard current sample
+            return nil
+          end
+        end
+      end
+
+      def sample_rate_lifetime
+        @captured_lifetime > 0 ? (@captured_lifetime.to_f / @seen_lifetime) : 0.0
+      end
+    end
+  end
+end
+