catpm 0.1.0

data/lib/catpm/collector.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+module Catpm
+  module Collector
+    class << self
+      def process_action_controller(event)
+        return unless Catpm.enabled?
+
+        payload = event.payload
+        target = "#{payload[:controller]}##{payload[:action]}"
+        return if target.start_with?('Catpm::')
+        return if Catpm.config.ignored?(target)
+
+        duration = event.duration # milliseconds
+        status = payload[:status] || (payload[:exception] ? 500 : nil)
+        context = build_http_context(payload)
+        metadata = build_http_metadata(payload)
+
+        req_segments = Thread.current[:catpm_request_segments]
+        if req_segments
+          segment_data = req_segments.to_h
+          segments = segment_data[:segments]
+
+          # Compute full request duration from middleware start to now
+          # (event.duration only covers the controller action, not middleware)
+          total_request_duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - req_segments.request_start) * 1000.0
+
+          # Inject root request segment with full duration
+          root_segment = {
+            type: 'request',
+            detail: "#{payload[:method]} #{payload[:path]}",
+            duration: total_request_duration.round(2),
+            offset: 0.0
+          }
+          segments.each do |seg|
+            if seg.key?(:parent_index)
+              seg[:parent_index] += 1
+            else
+              seg[:parent_index] = 0
+            end
+          end
+          segments.unshift(root_segment)
+
+          # Inject synthetic middleware segment if there's a time gap before the controller action
+          # (only when real per-middleware segments are not present)
+          ctrl_idx = segments.index { |s| s[:type] == 'controller' }
+          if ctrl_idx
+            has_real_middleware = segments.any? { |s| s[:type] == 'middleware' }
+            ctrl_offset = (segments[ctrl_idx][:offset] || 0.0).to_f
+            if ctrl_offset > 0.5 && !has_real_middleware
+              middleware_seg = {
+                type: 'middleware',
+                detail: 'Middleware Stack',
+                duration: ctrl_offset.round(2),
+                offset: 0.0,
+                parent_index: 0
+              }
+              segments.insert(1, middleware_seg)
+              # Shift parent_index for segments that moved down
+              segments.each_with_index do |seg, i|
+                next if i <= 1
+                next unless seg.key?(:parent_index)
+                seg[:parent_index] += 1 if seg[:parent_index] >= 1
+              end
+              # Add to summary so Time Breakdown shows middleware
+              segment_data[:segment_summary][:middleware_count] = 1
+              segment_data[:segment_summary][:middleware_duration] = ctrl_offset.round(2)
+            end
+          end
+
+          # Fill untracked controller time with sampler data or synthetic segment
+          ctrl_idx = segments.index { |s| s[:type] == 'controller' }
+          if ctrl_idx
+            ctrl_seg = segments[ctrl_idx]
+            ctrl_dur = (ctrl_seg[:duration] || 0).to_f
+            child_dur = segments.each_with_index.sum do |pair|
+              seg, i = pair
+              next 0.0 if i == ctrl_idx
+              (seg[:parent_index] == ctrl_idx) ? (seg[:duration] || 0).to_f : 0.0
+            end
+            gap = ctrl_dur - child_dur
+
+            if gap > 1.0
+              inject_gap_segments(segments, req_segments, gap, ctrl_idx, ctrl_seg)
+            end
+          end
+
+          context[:segments] = segments
+          context[:segment_summary] = segment_data[:segment_summary]
+          context[:segments_capped] = segment_data[:segments_capped]
+
+          segment_data[:segment_summary].each do |k, v|
+            metadata[k] = v
+          end
+
+          # Use full request duration (including middleware) for the event
+          duration = total_request_duration
+        end
+
+        ev = Event.new(
+          kind: :http,
+          target: target,
+          operation: payload[:method] || 'GET',
+          duration: duration,
+          started_at: Time.current,
+          status: status,
+          context: scrub(context),
+          metadata: metadata,
+          error_class: payload[:exception]&.first,
+          error_message: payload[:exception]&.last,
+          backtrace: payload[:exception_object]&.backtrace
+        )
+
+        Catpm.buffer&.push(ev)
+      end
+
+      def process_active_job(event)
+        return unless Catpm.enabled?
+
+        payload = event.payload
+        job = payload[:job]
+        target = job.class.name
+        return if Catpm.config.ignored?(target)
+
+        duration = event.duration
+        exception = payload[:exception_object]
+
+        queue_wait = if job.respond_to?(:enqueued_at) && job.enqueued_at
+          ((Time.current - job.enqueued_at.to_time) * 1000.0) rescue nil
+        end
+
+        context = {
+          job_class: target,
+          job_id: job.job_id,
+          queue: job.queue_name,
+          attempts: job.executions
+        }
+
+        metadata = { queue_wait: queue_wait }.compact
+
+        ev = Event.new(
+          kind: :job,
+          target: target,
+          operation: job.queue_name,
+          duration: duration,
+          started_at: Time.current,
+          context: context,
+          metadata: metadata,
+          error_class: exception&.class&.name,
+          error_message: exception&.message,
+          backtrace: exception&.backtrace
+        )
+
+        Catpm.buffer&.push(ev)
+      end
+
+      def process_tracked(kind:, target:, operation:, duration:, context:, metadata:, error:, req_segments:)
+        return unless Catpm.enabled?
+        return if Catpm.config.ignored?(target)
+
+        context = (context || {}).dup
+        metadata = (metadata || {}).dup
+
+        if req_segments
+          segment_data = req_segments.to_h
+          segments = segment_data[:segments]
+
+          # Inject root request segment
+          root_segment = {
+            type: 'request',
+            detail: "#{operation.presence || kind} #{target}",
+            duration: duration.round(2),
+            offset: 0.0
+          }
+          segments.each do |seg|
+            if seg.key?(:parent_index)
+              seg[:parent_index] += 1
+            else
+              seg[:parent_index] = 0
+            end
+          end
+          segments.unshift(root_segment)
+
+          # Fill untracked controller time with sampler data or synthetic segment
+          ctrl_idx = segments.index { |s| s[:type] == 'controller' }
+          if ctrl_idx
+            ctrl_seg = segments[ctrl_idx]
+            ctrl_dur = (ctrl_seg[:duration] || 0).to_f
+            child_dur = segments.each_with_index.sum do |pair|
+              seg, i = pair
+              next 0.0 if i == ctrl_idx
+              (seg[:parent_index] == ctrl_idx) ? (seg[:duration] || 0).to_f : 0.0
+            end
+            gap = ctrl_dur - child_dur
+
+            if gap > 1.0
+              inject_gap_segments(segments, req_segments, gap, ctrl_idx, ctrl_seg)
+            end
+          end
+
+          context[:segments] = segments
+          context[:segment_summary] = segment_data[:segment_summary]
+          context[:segments_capped] = segment_data[:segments_capped]
+
+          segment_data[:segment_summary]&.each do |k, v|
+            metadata[k] = v
+          end
+        end
+
+        ev = Event.new(
+          kind: kind,
+          target: target,
+          operation: operation.to_s,
+          duration: duration,
+          started_at: Time.current,
+          status: error ? 500 : 200,
+          context: scrub(context),
+          metadata: metadata,
+          error_class: error&.class&.name,
+          error_message: error&.message,
+          backtrace: error&.backtrace
+        )
+
+        Catpm.buffer&.push(ev)
+      end
+
+      def process_custom(name:, duration:, metadata: {}, error: nil, context: {})
+        return unless Catpm.enabled?
+        return if Catpm.config.ignored?(name)
+
+        ev = Event.new(
+          kind: :custom,
+          target: name,
+          operation: '',
+          duration: duration,
+          started_at: Time.current,
+          context: context,
+          metadata: metadata || {},
+          error_class: error&.class&.name,
+          error_message: error&.message,
+          backtrace: error&.backtrace
+        )
+
+        Catpm.buffer&.push(ev)
+      end
+
+      private
+
+      def inject_gap_segments(segments, req_segments, gap, ctrl_idx, ctrl_seg)
+        sampler_groups = req_segments&.sampler_segments || []
+
+        if sampler_groups.any?
+          sampler_dur = 0.0
+
+          sampler_groups.each do |group|
+            parent = group[:parent]
+            children = group[:children] || []
+
+            parent_idx = segments.size
+            parent[:parent_index] = ctrl_idx
+            segments << parent
+            sampler_dur += (parent[:duration] || 0).to_f
+
+            children.each do |child|
+              child[:parent_index] = parent_idx
+              child[:collapsed] = true
+              segments << child
+            end
+          end
+
+          remaining = gap - sampler_dur
+          if remaining > 1.0
+            segments << {
+              type: 'other',
+              detail: 'Untracked',
+              duration: remaining.round(2),
+              offset: (ctrl_seg[:offset] || 0.0),
+              parent_index: ctrl_idx
+            }
+          end
+        else
+          segments << {
+            type: 'other',
+            detail: 'Untracked',
+            duration: gap.round(2),
+            offset: (ctrl_seg[:offset] || 0.0),
+            parent_index: ctrl_idx
+          }
+        end
+      end
+
+      def build_http_context(payload)
+        {
+          method: payload[:method],
+          path: payload[:path],
+          params: (payload[:params] || {}).except('controller', 'action'),
+          status: payload[:status]
+        }
+      end
+
+      def build_http_metadata(payload)
+        h = {}
+        h[:db_runtime] = payload[:db_runtime] if payload[:db_runtime]
+        h[:view_runtime] = payload[:view_runtime] if payload[:view_runtime]
+        h
+      end
+
+      def scrub(hash)
+        parameter_filter.filter(hash)
+      end
+
+      def parameter_filter
+        @parameter_filter ||= begin
+          filters = Rails.application.config.filter_parameters + Catpm.config.additional_filter_parameters
+          ActiveSupport::ParameterFilter.new(filters)
+        end
+      end
+    end
+  end
+end

data/lib/catpm/configuration.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Catpm
+  class Configuration
+    attr_accessor :enabled,
+                  :instrument_http,
+                  :instrument_jobs,
+                  :instrument_segments,
+                  :instrument_net_http,
+                  :instrument_stack_sampler,
+                  :max_segments_per_request,
+                  :segment_source_threshold,
+                  :max_sql_length,
+                  :slow_threshold,
+                  :slow_threshold_per_kind,
+                  :ignored_targets,
+                  :retention_period,
+                  :max_buffer_memory,
+                  :flush_interval,
+                  :flush_jitter,
+                  :max_error_contexts,
+                  :bucket_sizes,
+                  :error_handler,
+                  :http_basic_auth_user,
+                  :http_basic_auth_password,
+                  :access_policy,
+                  :additional_filter_parameters,
+                  :instrument_middleware_stack,
+                  :auto_instrument_methods,
+                  :service_base_classes,
+                  :random_sample_rate,
+                  :max_random_samples_per_endpoint,
+                  :max_slow_samples_per_endpoint,
+                  :cleanup_interval,
+                  :circuit_breaker_failure_threshold,
+                  :circuit_breaker_recovery_timeout,
+                  :sqlite_busy_timeout,
+                  :persistence_batch_size,
+                  :backtrace_lines,
+                  :shutdown_timeout,
+                  :events_enabled,
+                  :events_max_samples_per_name
+
+    def initialize
+      @enabled = true
+      @instrument_http = true
+      @instrument_jobs = false
+      @instrument_segments = true
+      @instrument_net_http = false
+      @instrument_stack_sampler = false
+      @instrument_middleware_stack = false
+      @max_segments_per_request = 50
+      @segment_source_threshold = 0.0 # ms — capture caller_locations for all segments (set higher to reduce overhead)
+      @max_sql_length = 200
+      @slow_threshold = 500 # milliseconds
+      @slow_threshold_per_kind = {}
+      @ignored_targets = []
+      @retention_period = nil # nil = keep forever (data is downsampled, not deleted)
+      @max_buffer_memory = 32.megabytes
+      @flush_interval = 30 # seconds
+      @flush_jitter = 5 # ±seconds
+      @max_error_contexts = 5
+      @bucket_sizes = { recent: 1.minute, medium: 5.minutes, hourly: 1.hour, daily: 1.day, weekly: 1.week }
+      @error_handler = ->(e) { Rails.logger.error("[catpm] #{e.message}") }
+      @http_basic_auth_user = nil
+      @http_basic_auth_password = nil
+      @access_policy = nil
+      @additional_filter_parameters = []
+      @auto_instrument_methods = []
+      @service_base_classes = nil # nil = auto-detect (ApplicationService, BaseService)
+      @random_sample_rate = 20
+      @max_random_samples_per_endpoint = 5
+      @max_slow_samples_per_endpoint = 5
+      @cleanup_interval = 1.hour
+      @circuit_breaker_failure_threshold = 5
+      @circuit_breaker_recovery_timeout = 60 # seconds
+      @sqlite_busy_timeout = 5_000 # milliseconds
+      @persistence_batch_size = 100
+      @backtrace_lines = 10
+      @shutdown_timeout = 5 # seconds
+      @events_enabled = false
+      @events_max_samples_per_name = 20
+    end
+
+    def slow_threshold_for(kind)
+      slow_threshold_per_kind.fetch(kind.to_sym, slow_threshold)
+    end
+
+    def ignored?(target)
+      ignored_targets.any? do |pattern|
+        case pattern
+        when Regexp then pattern.match?(target)
+        when String
+          if pattern.include?('*')
+            File.fnmatch(pattern, target)
+          else
+            pattern == target
+          end
+        end
+      end
+    end
+  end
+end

data/lib/catpm/custom_event.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Catpm
+  class CustomEvent
+    OBJECT_OVERHEAD = 40
+    REF_SIZE = 8
+
+    attr_accessor :name, :payload, :recorded_at
+
+    def initialize(name:, payload: {}, recorded_at: nil)
+      @name = name.to_s
+      @payload = payload || {}
+      @recorded_at = recorded_at || Time.current
+    end
+
+    def bucket_start
+      recorded_at.change(sec: 0)
+    end
+
+    def estimated_byte_size
+      OBJECT_OVERHEAD +
+        name.bytesize + REF_SIZE +
+        payload_bytes
+    end
+
+    # Alias to match Event interface used by Buffer
+    alias_method :estimated_bytes, :estimated_byte_size
+
+    private
+
+    def payload_bytes
+      payload.sum { |k, v| k.to_s.bytesize + v.to_s.bytesize + REF_SIZE }
+    rescue
+      0
+    end
+  end
+end

data/lib/catpm/engine.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Catpm
+  class Engine < ::Rails::Engine
+    isolate_namespace Catpm
+
+    initializer 'catpm.migrations' do |app|
+      config.paths['db/migrate'].expanded.each do |path|
+        app.config.paths['db/migrate'] << path unless app.config.paths['db/migrate'].include?(path)
+      end
+    end
+
+    initializer 'catpm.middleware' do |app|
+      app.middleware.insert_before 0, Catpm::Middleware
+    end
+
+    config.after_initialize do
+      if Catpm.enabled?
+        Catpm::Subscribers.subscribe!
+        Catpm::Lifecycle.register_hooks
+        Catpm::AutoInstrument.apply!
+
+        if Catpm.config.instrument_middleware_stack
+          app = Rails.application
+          names = app.middleware.filter_map { |m| m.name }.reject { |n| n.start_with?('Catpm::') }
+          names.reverse_each do |name|
+            app.middleware.insert_before(name, Catpm::MiddlewareProbe, name)
+          rescue ArgumentError, RuntimeError
+            # Middleware not found in stack — skip
+          end
+        end
+      end
+    end
+
+    config.to_prepare do
+      Catpm::AutoInstrument.apply! if Catpm.enabled?
+    end
+  end
+end

data/lib/catpm/errors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Catpm
+  class CircuitOpenError < StandardError; end
+  class UnsupportedAdapter < StandardError; end
+end

data/lib/catpm/event.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Catpm
+  class Event
+    OBJECT_OVERHEAD = 40 # bytes, Ruby object header
+    REF_SIZE = 8         # bytes, pointer on 64-bit
+    NUMERIC_FIELDS_SIZE = 64 # fixed numeric fields (duration, timestamps, etc.)
+
+    attr_accessor :kind, :target, :operation, :duration, :started_at,
+                  :metadata, :error_class, :error_message, :backtrace,
+                  :sample_type, :context, :status
+
+    def initialize(kind:, target:, operation: '', duration: 0.0, started_at: nil,
+                   metadata: {}, error_class: nil, error_message: nil, backtrace: nil,
+                   sample_type: nil, context: {}, status: nil)
+      @kind = kind.to_s
+      @target = target.to_s
+      @operation = (operation || '').to_s
+      @duration = duration.to_f
+      @started_at = started_at || Time.current
+      @metadata = metadata || {}
+      @error_class = error_class
+      @error_message = error_message
+      @backtrace = backtrace
+      @sample_type = sample_type
+      @context = context || {}
+      @status = status
+    end
+
+    def estimated_bytes
+      OBJECT_OVERHEAD +
+        target.bytesize + REF_SIZE +
+        operation.bytesize +
+        kind.bytesize +
+        (error_class&.bytesize || 0) +
+        (error_message&.bytesize || 0) +
+        backtrace_bytes +
+        context_bytes +
+        metadata_bytes +
+        NUMERIC_FIELDS_SIZE
+    end
+
+    def error?
+      !error_class.nil?
+    end
+
+    def success?
+      !error? && (status.nil? || (200..399).cover?(status.to_i))
+    end
+
+    def bucket_start
+      started_at.change(sec: 0) # Round to minute
+    end
+
+    private
+
+    def backtrace_bytes
+      return 0 unless backtrace
+
+      backtrace.sum { |line| line.bytesize + REF_SIZE } + REF_SIZE
+    end
+
+    def context_bytes
+      return 0 if context.empty?
+
+      context.to_json.bytesize + REF_SIZE
+    end
+
+    def metadata_bytes
+      return 0 if metadata.empty?
+
+      metadata.to_json.bytesize + REF_SIZE
+    end
+  end
+end

data/lib/catpm/fingerprint.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Catpm
+  module Fingerprint
+    # Generates a stable fingerprint for error grouping.
+    # Includes kind so the same exception in HTTP vs job = different groups.
+    def self.generate(kind:, error_class:, backtrace:)
+      normalized = normalize_backtrace(backtrace || [])
+      raw = "#{kind}:#{error_class}\n#{normalized}"
+      Digest::SHA256.hexdigest(raw)
+    end
+
+    def self.normalize_backtrace(backtrace)
+      app_frames = backtrace
+        .select { |line| app_frame?(line) }
+        .first(5)
+        .map { |line| strip_line_number(line) }
+
+      # If there are app frames, group by app code (like Sentry)
+      return app_frames.join("\n") if app_frames.any?
+
+      # No app frames = error in a gem/library. Group by crash location
+      # so the same bug is always one issue regardless of the caller.
+      backtrace
+        .reject { |line| line.include?('<internal:') }
+        .first(3)
+        .map { |line| strip_line_number(line) }
+        .join("\n")
+    end
+
+    # Checks if a backtrace line belongs to the host application (not a gem or stdlib)
+    def self.app_frame?(line)
+      return false if line.include?('/gems/')
+      return false if line.include?('/ruby/')
+      return false if line.include?('<internal:')
+      return false if line.include?('/catpm/')
+
+      if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+        return line.start_with?(Rails.root.to_s) if line.start_with?('/')
+      end
+
+      line.start_with?('app/') || line.include?('/app/')
+    end
+
+    # Strips line numbers: "app/models/user.rb:42:in `validate'" → "app/models/user.rb:in `validate'"
+    def self.strip_line_number(line)
+      line.sub(/:\d+:in /, ':in ')
+    end
+  end
+end