catpm 0.4.0 → 0.6.0

data/lib/catpm/flusher.rb

@@ -2,6 +2,8 @@
 
 module Catpm
   class Flusher
+    ERROR_LOG_BACKTRACE_LINES = 5
+
     attr_reader :running
 
     def initialize(buffer:, interval: nil, jitter: nil)
@@ -104,7 +106,7 @@ module Catpm
       events&.each { |ev| @buffer.push(ev) }
       @circuit.record_failure
       Catpm.config.error_handler.call(e)
-      Rails.logger.error("[catpm] flush error: #{e.class}: #{e.message}\n#{e.backtrace&.first(5)&.join("\n")}")
+      Rails.logger.error("[catpm] flush error: #{e.class}: #{e.message}\n#{e.backtrace&.first(ERROR_LOG_BACKTRACE_LINES)&.join("\n")}")
     end
 
     def reset!
@@ -190,7 +192,8 @@ module Catpm
           error[:last_occurred_at] = [ error[:last_occurred_at], event.started_at ].max
           error[:occurrence_times] << event.started_at
 
-          if error[:new_contexts].size < Catpm.config.max_error_contexts
+          max_ctx = Catpm.config.max_error_contexts
+          if max_ctx.nil? || error[:new_contexts].size < max_ctx
             error[:new_contexts] << build_error_context(event)
           end
         end
@@ -254,31 +257,40 @@ module Catpm
 
         case sample[:sample_type]
         when 'random'
-          cache_key = [kind, target, operation, 'random']
-          if (counts_cache[cache_key] || 0) >= Catpm.config.max_random_samples_per_endpoint
-            oldest = Catpm::Sample.joins(:bucket)
-              .where(catpm_buckets: { kind: kind, target: target, operation: operation })
-              .where(sample_type: 'random').order(recorded_at: :asc).first
-            oldest&.destroy
+          max_random = Catpm.config.max_random_samples_per_endpoint
+          if max_random
+            cache_key = [kind, target, operation, 'random']
+            if (counts_cache[cache_key] || 0) >= max_random
+              oldest = Catpm::Sample.joins(:bucket)
+                .where(catpm_buckets: { kind: kind, target: target, operation: operation })
+                .where(sample_type: 'random').order(recorded_at: :asc).first
+              oldest&.destroy
+            end
           end
         when 'slow'
-          cache_key = [kind, target, operation, 'slow']
-          if (counts_cache[cache_key] || 0) >= Catpm.config.max_slow_samples_per_endpoint
-            weakest = Catpm::Sample.joins(:bucket)
-              .where(catpm_buckets: { kind: kind, target: target, operation: operation })
-              .where(sample_type: 'slow').order(duration: :asc).first
-            if weakest && sample[:duration] > weakest.duration
-              weakest.destroy
-            else
-              sample[:_skip] = true
+          max_slow = Catpm.config.max_slow_samples_per_endpoint
+          if max_slow
+            cache_key = [kind, target, operation, 'slow']
+            if (counts_cache[cache_key] || 0) >= max_slow
+              weakest = Catpm::Sample.joins(:bucket)
+                .where(catpm_buckets: { kind: kind, target: target, operation: operation })
+                .where(sample_type: 'slow').order(duration: :asc).first
+              if weakest && sample[:duration] > weakest.duration
+                weakest.destroy
+              else
+                sample[:_skip] = true
+              end
             end
           end
         when 'error'
-          fp = sample[:error_fingerprint]
-          if fp && (error_counts[fp] || 0) >= Catpm.config.max_error_samples_per_fingerprint
-            oldest = Catpm::Sample.where(sample_type: 'error', error_fingerprint: fp)
-              .order(recorded_at: :asc).first
-            oldest&.destroy
+          max_err = Catpm.config.max_error_samples_per_fingerprint
+          if max_err
+            fp = sample[:error_fingerprint]
+            if fp && (error_counts[fp] || 0) >= max_err
+              oldest = Catpm::Sample.where(sample_type: 'error', error_fingerprint: fp)
+                .order(recorded_at: :asc).first
+              oldest&.destroy
+            end
           end
         end
       end
@@ -335,7 +347,7 @@ module Catpm
 
         max = Catpm.config.events_max_samples_per_name
         if event.payload.any?
-          if sample_counts[event.name] < max
+          if max.nil? || sample_counts[event.name] < max
             samples << { name: event.name, payload: event.payload, recorded_at: event.recorded_at }
             sample_counts[event.name] += 1
           elsif rand(Catpm.config.random_sample_rate) == 0
@@ -357,41 +369,42 @@ module Catpm
 
     def downsample_buckets
       bucket_sizes = Catpm.config.bucket_sizes
+      thresholds = Catpm.config.downsampling_thresholds
       adapter = Catpm::Adapter.current
 
       # Phase 1: Merge 1-minute buckets older than 1 hour into 5-minute buckets
       downsample_tier(
         target_interval: bucket_sizes[:medium],
-        age_threshold: 1.hour,
+        age_threshold: thresholds[:medium],
         adapter: adapter
       )
 
       # Phase 2: Merge 5-minute buckets older than 24 hours into 1-hour buckets
       downsample_tier(
         target_interval: bucket_sizes[:hourly],
-        age_threshold: 24.hours,
+        age_threshold: thresholds[:hourly],
         adapter: adapter
       )
 
       # Phase 3: Merge 1-hour buckets older than 1 week into 1-day buckets
       downsample_tier(
         target_interval: bucket_sizes[:daily],
-        age_threshold: 1.week,
+        age_threshold: thresholds[:daily],
         adapter: adapter
       )
 
       # Phase 4: Merge 1-day buckets older than 3 months into 1-week buckets
       downsample_tier(
         target_interval: bucket_sizes[:weekly],
-        age_threshold: 90.days,
+        age_threshold: thresholds[:weekly],
         adapter: adapter
       )
 
       # Event buckets: same downsampling tiers
-      downsample_event_tier(target_interval: bucket_sizes[:medium], age_threshold: 1.hour, adapter: adapter)
-      downsample_event_tier(target_interval: bucket_sizes[:hourly], age_threshold: 24.hours, adapter: adapter)
-      downsample_event_tier(target_interval: bucket_sizes[:daily], age_threshold: 1.week, adapter: adapter)
-      downsample_event_tier(target_interval: bucket_sizes[:weekly], age_threshold: 90.days, adapter: adapter)
+      downsample_event_tier(target_interval: bucket_sizes[:medium], age_threshold: thresholds[:medium], adapter: adapter)
+      downsample_event_tier(target_interval: bucket_sizes[:hourly], age_threshold: thresholds[:hourly], adapter: adapter)
+      downsample_event_tier(target_interval: bucket_sizes[:daily], age_threshold: thresholds[:daily], adapter: adapter)
+      downsample_event_tier(target_interval: bucket_sizes[:weekly], age_threshold: thresholds[:weekly], adapter: adapter)
     end
 
     def downsample_tier(target_interval:, age_threshold:, adapter:)
@@ -496,7 +509,7 @@ module Catpm
 
     def cleanup_expired_data
       cutoff = Catpm.config.retention_period.ago
-      batch_size = 1_000
+      batch_size = Catpm.config.cleanup_batch_size
 
       [ Catpm::Bucket, Catpm::Sample ].each do |model|
         time_column = model == Catpm::Sample ? :recorded_at : :bucket_start

data/lib/catpm/middleware.rb

@@ -21,6 +21,12 @@ module Catpm
         )
         env['catpm.segments'] = req_segments
         Thread.current[:catpm_request_segments] = req_segments
+
+        if Catpm.config.instrument_call_tree
+          call_tracer = CallTracer.new(request_segments: req_segments)
+          call_tracer.start
+          env['catpm.call_tracer'] = call_tracer
+        end
       end
 
       @app.call(env)
@@ -29,6 +35,7 @@ module Catpm
       raise
     ensure
       if Catpm.config.instrument_segments
+        env['catpm.call_tracer']&.stop
         req_segments&.stop_sampler
         Thread.current[:catpm_request_segments] = nil
       end

data/lib/catpm/request_segments.rb

@@ -36,7 +36,7 @@ module Catpm
         @tracked_ranges << [started_at, started_at + duration / 1000.0]
       end
 
-      if @segments.size < @max_segments
+      if @max_segments.nil? || @segments.size < @max_segments
         @segments << segment
       else
         @overflow = true
@@ -54,7 +54,7 @@ module Catpm
       segment[:offset] = offset if offset
       segment[:parent_index] = @span_stack.last if @span_stack.any?
 
-      return nil if @segments.size >= @max_segments
+      return nil if @max_segments && @segments.size >= @max_segments
 
       index = @segments.size
       @segments << segment

data/lib/catpm/segment_subscribers.rb

@@ -2,6 +2,8 @@
 
 module Catpm
   module SegmentSubscribers
+    MIN_INSTANTIATION_DURATION_MS = 0.1
+    CALLER_OFFSET = 4 # frames to skip to reach user code from this call site
     # Subscriber with start/finish callbacks so all segments (SQL, views, etc.)
     # fired during a controller action are automatically nested under the controller span.
     class ControllerSpanSubscriber
@@ -146,7 +148,7 @@ module Catpm
         return unless req_segments
 
         duration = event.duration
-        return if duration < 0.1 # skip trivial instantiations
+        return if duration < MIN_INSTANTIATION_DURATION_MS # skip trivial instantiations
 
         payload = event.payload
         record_count = payload[:record_count] || 0
@@ -230,7 +232,7 @@ module Catpm
       end
 
       def extract_source_location
-        locations = caller_locations(4, 50)
+        locations = caller_locations(CALLER_OFFSET, Catpm.config.caller_scan_depth)
         locations&.each do |loc|
           path = loc.path.to_s
           if Fingerprint.app_frame?(path)

data/lib/catpm/stack_sampler.rb

@@ -3,6 +3,8 @@
 module Catpm
   class StackSampler
     MS_PER_SECOND = 1000.0
+    MIN_SEGMENT_DURATION_MS = 1.0
+    SAMPLING_THREAD_PRIORITY = -1
 
     # Single global thread that samples all active requests.
     # Avoids creating a thread per request.
@@ -33,7 +35,7 @@ module Catpm
             sample_all
           end
         end
-        @thread.priority = -1
+        @thread.priority = SAMPLING_THREAD_PRIORITY
       end
 
       def sample_all
@@ -118,7 +120,7 @@ module Catpm
 
       groups.filter_map do |group|
         duration = estimate_duration(group)
-        next if duration < 1.0
+        next if duration < MIN_SEGMENT_DURATION_MS
 
         offset = ((group[:start_time] - @request_start) * MS_PER_SECOND).round(2)
         app_frame = group[:app_frame]
@@ -205,7 +207,7 @@ module Catpm
           (span[:end_time] - span[:start_time]) * MS_PER_SECOND,
           span[:count] * Catpm.config.stack_sample_interval * MS_PER_SECOND
         ].max
-        next if duration < 1.0
+        next if duration < MIN_SEGMENT_DURATION_MS
 
         frame = span[:frame]
         path = frame.path.to_s

data/lib/catpm/trace.rb

@@ -83,6 +83,11 @@ module Catpm
       )
       Thread.current[:catpm_request_segments] = req_segments
       owns_segments = true
+
+      if config.instrument_call_tree
+        call_tracer = CallTracer.new(request_segments: req_segments)
+        call_tracer.start
+      end
     end
 
     if req_segments
@@ -100,6 +105,7 @@ module Catpm
       raise
     ensure
       duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+      call_tracer&.stop
       req_segments&.pop_span(ctrl_idx) if ctrl_idx
       req_segments&.stop_sampler
 
@@ -152,8 +158,10 @@ module Catpm
 
   private
 
+  CALLER_OFFSET = 3 # frames to skip to reach user code from this call site
+
   def self.extract_trace_source
-    locations = caller_locations(3, 50)
+    locations = caller_locations(CALLER_OFFSET, Catpm.config.caller_scan_depth)
     locations&.each do |loc|
       path = loc.path.to_s
       if Fingerprint.app_frame?(path)

data/lib/catpm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Catpm
-  VERSION = '0.4.0'
+  VERSION = '0.6.0'
 end

data/lib/catpm.rb

@@ -12,6 +12,7 @@ require 'catpm/adapter'
 require 'catpm/fingerprint'
 require 'catpm/stack_sampler'
 require 'catpm/request_segments'
+require 'catpm/call_tracer'
 require 'catpm/flusher'
 require 'catpm/collector'
 require 'catpm/middleware'

data/lib/generators/catpm/templates/initializer.rb.tt

@@ -1,78 +1,90 @@
 # frozen_string_literal: true
 
 Catpm.configure do |config|
+  # === Enable/Disable ===
+  # config.enabled = Rails.env.production? || Rails.env.staging?
+
   # === Security ===
-  # Protect the dashboard with HTTP Basic Auth or a custom policy.
-  # If neither is configured, the dashboard is accessible to everyone.
-  #
   # config.http_basic_auth_user = ENV["CATPM_USER"]
   # config.http_basic_auth_password = ENV["CATPM_PASSWORD"]
   # config.access_policy = ->(request) { request.env["warden"].user&.admin? }
 
   # === PII Filtering ===
-  # Rails' filter_parameters are inherited automatically.
-  # Add extra patterns here:
-  #
   # config.additional_filter_parameters = [:card_number, :ssn]
 
   # === Instrumentation ===
-  #
-  config.instrument_http = true              # Track HTTP requests (default: true)
-  config.instrument_jobs = false             # Track ActiveJob (default: false)
-  config.instrument_segments = true          # Track SQL/view/cache segments (default: true)
-  # config.track_own_requests = false           # Track catpm dashboard requests (default: false)
-  # config.instrument_net_http = false         # Patch Net::HTTP for outbound tracking (default: false)
-  # config.instrument_middleware_stack = false # Decompose middleware into per-middleware segments (default: false)
-  # config.max_segments_per_request = 50       # Cap segments per request (keeps slowest)
-  # config.segment_source_threshold = 0.0      # ms — capture caller_locations above this (raise to reduce overhead)
-  # config.max_sql_length = 200                # Truncate SQL queries to this length
-  # config.slow_threshold = 500                # ms — threshold for slow samples (all kinds)
-  # config.slow_threshold_per_kind = {         # Override per kind
-  #   http: 500,
-  #   job: 5_000,
-  #   custom: 1_000
-  # }
-  # config.ignored_targets = [                 # Patterns to exclude from tracking
-  #   "HealthcheckController#index",
-  #   "/assets/*",
-  # ]
+  config.instrument_http = true
+  config.instrument_jobs = false
+  config.instrument_segments = true
+  # config.instrument_net_http = false
+  # config.instrument_middleware_stack = false
+  # config.instrument_stack_sampler = false
+  # config.instrument_call_tree = false
+  # config.show_untracked_segments = false
+  # config.track_own_requests = false
+
+  # === Auto-instrumentation ===
+  # config.service_base_classes = nil                  # nil = auto-detect (ApplicationService, BaseService)
+  # config.auto_instrument_methods = []                # e.g. ["Worker#process", "Gateway.charge"]
+
+  # === Segments ===
+  # config.max_segments_per_request = 50               # nil = unlimited
+  # config.segment_source_threshold = 0.0              # ms — capture caller_locations above this
+  # config.max_sql_length = 200                        # nil = no truncation
+  # config.slow_threshold = 500                        # ms
+  # config.slow_threshold_per_kind = {}                # { http: 500, job: 5_000, custom: 1_000 }
+  # config.ignored_targets = []
+
+  # === Stack Sampling ===
+  # config.stack_sample_interval = 0.005               # seconds (5ms)
+  # config.max_stack_samples_per_request = 200         # nil = unlimited
 
   # === Sampling ===
-  #
-  # config.random_sample_rate = 20                   # 1 in N requests sampled randomly
-  # config.max_random_samples_per_endpoint = 5       # Random samples kept per endpoint
-  # config.max_slow_samples_per_endpoint = 5         # Slow samples kept per endpoint
-
-  # === Tuning ===
-  #
-  # config.retention_period = nil                    # nil = keep forever (default); set to e.g. 1.year to delete older data
-  # config.max_buffer_memory = 32.megabytes          # Maximum in-memory buffer size
-  # config.flush_interval = 30                       # seconds
-  # config.flush_jitter = 5                          # +/- seconds random offset per cycle
-  # config.max_error_contexts = 5                    # Contexts kept per error fingerprint
-  # config.cleanup_interval = 1.hour                 # How often to run downsampling
-  # config.persistence_batch_size = 100              # Records per DB batch operation
+  # config.random_sample_rate = 20                     # 1 in N requests sampled randomly
+  # config.max_random_samples_per_endpoint = 5         # nil = unlimited
+  # config.max_slow_samples_per_endpoint = 5           # nil = unlimited
+  # config.max_error_samples_per_fingerprint = 20      # nil = unlimited
+
+  # === Errors ===
+  # config.max_error_contexts = 5                      # nil = unlimited
+  # config.backtrace_lines = nil
+
+  # === Events ===
+  # config.events_enabled = false
+  # config.events_max_samples_per_name = 20            # nil = unlimited
+
+  # === Buffering & Flushing ===
+  # config.max_buffer_memory = 8.megabytes
+  # config.flush_interval = 30                         # seconds
+  # config.flush_jitter = 5                            # ±seconds
+  # config.persistence_batch_size = 100
+
+  # === Data Retention ===
+  # config.retention_period = nil                      # nil = keep forever
+  # config.cleanup_interval = 1.hour
 
   # === Downsampling ===
-  # Data is kept forever with progressively coarser resolution:
-  # - Last hour: 1-minute buckets
-  # - 1h–24h: 5-minute buckets
-  # - 1d–1w: 1-hour buckets
-  # - 1w–3mo: 1-day buckets
-  # - Older than 3 months: 1-week buckets
-  #
   # config.bucket_sizes = { recent: 1.minute, medium: 5.minutes, hourly: 1.hour, daily: 1.day, weekly: 1.week }
+  # config.downsampling_thresholds = {                 # Age before tier is merged into the next coarser tier
+  #   medium: 1.hour,
+  #   hourly: 24.hours,
+  #   daily:  1.week,
+  #   weekly: 90.days
+  # }
+
+  # === Database ===
+  # config.sqlite_busy_timeout = 5_000                 # ms
+
+  # === Circuit Breaker ===
+  # config.circuit_breaker_failure_threshold = 5
+  # config.circuit_breaker_recovery_timeout = 60       # seconds
 
   # === Advanced ===
-  #
-  # config.circuit_breaker_failure_threshold = 5     # Failures before circuit opens
-  # config.circuit_breaker_recovery_timeout = 60     # seconds before retry
-  # config.sqlite_busy_timeout = 5_000               # ms — SQLite lock wait
-  # config.backtrace_lines = 10                      # Backtrace lines in error contexts
-  # config.shutdown_timeout = 5                      # seconds — graceful shutdown wait
+  # config.shutdown_timeout = 5                        # seconds
+  # config.max_error_detail_length = 200               # truncate error detail segments
+  # config.max_fingerprint_app_frames = 5              # app frames used for error fingerprint
+  # config.max_fingerprint_gem_frames = 3              # gem frames used when no app frames
+  # config.cleanup_batch_size = 1_000                  # rows per cleanup batch
+  # config.caller_scan_depth = 50                      # max frames to scan for app code
   # config.error_handler = ->(e) { Rails.logger.error("[catpm] #{e.message}") }
-
-  # === Enable/Disable ===
-  #
-  # config.enabled = Rails.env.production? || Rails.env.staging?
 end

data/lib/tasks/catpm_tasks.rake

@@ -19,5 +19,33 @@ namespace :catpm do
     else
       puts '[catpm] catpm_errors.occurrence_buckets already exists, skipping'
     end
+
+    unless connection.table_exists?(:catpm_endpoint_prefs)
+      connection.create_table :catpm_endpoint_prefs do |t|
+        t.string :kind, null: false
+        t.string :target, null: false
+        t.string :operation, null: false, default: ''
+        t.boolean :pinned, null: false, default: false
+        t.boolean :ignored, null: false, default: false
+      end
+      connection.add_index :catpm_endpoint_prefs, [:kind, :target, :operation],
+                           unique: true, name: 'idx_catpm_endpoint_prefs_unique'
+      puts '[catpm] Created catpm_endpoint_prefs table'
+    else
+      puts '[catpm] catpm_endpoint_prefs table already exists, skipping'
+    end
+
+    unless connection.table_exists?(:catpm_event_prefs)
+      connection.create_table :catpm_event_prefs do |t|
+        t.string :name, null: false
+        t.boolean :pinned, null: false, default: false
+        t.boolean :ignored, null: false, default: false
+      end
+      connection.add_index :catpm_event_prefs, :name,
+                           unique: true, name: 'idx_catpm_event_prefs_unique'
+      puts '[catpm] Created catpm_event_prefs table'
+    else
+      puts '[catpm] catpm_event_prefs table already exists, skipping'
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: catpm
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - ''
@@ -46,14 +46,18 @@ files:
 - app/mailers/catpm/application_mailer.rb
 - app/models/catpm/application_record.rb
 - app/models/catpm/bucket.rb
+- app/models/catpm/endpoint_pref.rb
 - app/models/catpm/error_record.rb
 - app/models/catpm/event_bucket.rb
+- app/models/catpm/event_pref.rb
 - app/models/catpm/event_sample.rb
 - app/models/catpm/sample.rb
 - app/views/catpm/endpoints/_sample_table.html.erb
+- app/views/catpm/endpoints/ignored.html.erb
 - app/views/catpm/endpoints/show.html.erb
 - app/views/catpm/errors/index.html.erb
 - app/views/catpm/errors/show.html.erb
+- app/views/catpm/events/ignored.html.erb
 - app/views/catpm/events/index.html.erb
 - app/views/catpm/events/show.html.erb
 - app/views/catpm/samples/show.html.erb
@@ -71,6 +75,7 @@ files:
 - lib/catpm/adapter/sqlite.rb
 - lib/catpm/auto_instrument.rb
 - lib/catpm/buffer.rb
+- lib/catpm/call_tracer.rb
 - lib/catpm/circuit_breaker.rb
 - lib/catpm/collector.rb
 - lib/catpm/configuration.rb