catpm 0.8.3 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68fffa206b7baa5919309c1d54110c58205fe7b48bf37111724ba00b5ed07ef7
-  data.tar.gz: eba9254577cdbc6afa64d1ec352d2bdc55827e1668890dd0dbba2ff16696ac3c
+  metadata.gz: f73d14442e21e40a399d3df0635a640cf6067dc10f34e17a74054ab855ac212a
+  data.tar.gz: db34666075f0f5e2c2506a6e7ddad705673f8328ae22edcf1264ae0b94fe999a
 SHA512:
-  metadata.gz: d7c6016b9f7f3087638e0b1ab47ecb140bd3eff31b2c0988ab953590c435f162336b29e380cc231a5eb66d95c5ad4c7b93a77399fd513a99f8bcf0e44aface84
-  data.tar.gz: 9a923abb3d7f882c76b9665509adea19173368d0395ec1b99b73d6288901bfdff6d0d89336d9081e5f7b5ecea96f6274e59e0cf62d7f17f6be7ddc221bdff18c
+  metadata.gz: 16edf4257962cc5b52ee5207604d6f4643fd93b3219ef15aaf4fbaac14779fa09f49b22aff387a97796d9435b741820f4e9380e2e5276df4939b47af9202f01d
+  data.tar.gz: c5e5b2de2aa7994d827760f77ff7e30fa313d77a51b58573a5129a6a09010815ba30deade3603c15482076b6a4e3cd888123cea7dd78ba026db6fbeab6bc7bc3

data/README.md

@@ -1,5 +1,5 @@
 gem build catpm.gemspec
-gem push catpm-0.8.0.gem
+gem push catpm-0.8.4.gem
 
 # Catpm
 

data/app/views/catpm/endpoints/show.html.erb

@@ -72,11 +72,13 @@
 <% end %>
 
 <%
+  instrumented_count = (@metadata["_instrumented"] || @metadata[:"_instrumented"] || 0).to_f
+  instrumented_count = @count.to_f if instrumented_count == 0  # backward compat with pre-sampling data
   type_data = segment_colors.map { |type, color|
     count = (@metadata["#{type}_count"] || @metadata[:"#{type}_count"] || 0).to_f
     dur = (@metadata["#{type}_duration"] || @metadata[:"#{type}_duration"] || 0).to_f
-    avg_dur = @count > 0 ? dur / @count : 0
-    avg_count = @count > 0 ? count / @count : 0
+    avg_dur = instrumented_count > 0 ? dur / instrumented_count : 0
+    avg_count = instrumented_count > 0 ? count / instrumented_count : 0
     text_color = segment_text_colors[type] || "#4b5563"
     { type: type, label: segment_labels[type] || type.capitalize, bg: color, text: text_color, count: count, dur: dur, avg_dur: avg_dur, avg_count: avg_count }
   }.select { |d| d[:count] > 0 }

data/app/views/catpm/errors/index.html.erb

@@ -15,7 +15,7 @@
     <% end %>
     <input type="text" class="search-input" id="error-search" placeholder="Search errors... (/)" oninput="filterByText('error-search','errors-table')">
     <% if @tab == "active" && @active_count > 0 %>
-      <span style="display:inline; margin-left:auto"><%= button_to "Resolve all", catpm.resolve_all_errors_path, method: :post, class: "btn", data: { confirm: "Resolve all #{@active_count} active errors?" } %></span>
+      <span style="display:inline; margin-left:auto"><%= button_to "Resolve all", catpm.resolve_all_errors_path, method: :post, class: "btn" %></span>
     <% end %>
   </div>
 <% end %>

data/app/views/catpm/samples/show.html.erb

@@ -17,8 +17,7 @@
     <span>Sample #<%= @sample.id %></span>
   </div>
   <%= button_to "Delete Sample", catpm.sample_path(@sample),
-      method: :delete, class: "btn btn-danger",
-      data: { confirm: "Delete this sample? This cannot be undone." } %>
+      method: :delete, class: "btn btn-danger" %>
 </div>
 
 <%# ─── Request Info Bar ─── %>

data/lib/catpm/collector.rb

@@ -19,6 +19,8 @@ module Catpm
         metadata = build_http_metadata(payload)
 
         req_segments = Thread.current[:catpm_request_segments]
+        instrumented = !req_segments.nil?
+
         if req_segments
           segment_data = req_segments.to_h
 
@@ -28,8 +30,17 @@ module Catpm
 
           # Segment summary is always needed for bucket metadata aggregation
           segment_data[:segment_summary].each { |k, v| metadata[k] = v }
+        else
+          # Non-instrumented request — compute duration from thread-local start time
+          request_start = Thread.current[:catpm_request_start]
+          if request_start
+            duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - request_start) * 1000.0
+          end
         end
 
+        # Track instrumented count for correct dashboard averaging
+        metadata[:_instrumented] = 1 if instrumented
+
         # Early sampling decision — only build heavy context for sampled events
         operation = payload[:method] || 'GET'
         sample_type = early_sample_type(
@@ -37,9 +48,15 @@ module Catpm
           duration: duration,
           kind: :http,
           target: target,
-          operation: operation
+          operation: operation,
+          instrumented: instrumented
         )
 
+        # Slow spike detection: force instrument next request for this endpoint
+        if !instrumented && (payload[:exception] || duration >= Catpm.config.slow_threshold_for(:http))
+          trigger_force_instrument(kind: :http, target: target, operation: operation)
+        end
+
         if sample_type
           context = build_http_context(payload)
 
@@ -145,7 +162,6 @@ module Catpm
               }
             end
 
-            req_segments.release! # free internal state for GC
           end
 
           context = scrub(context)
@@ -225,20 +241,30 @@ module Catpm
         return if Catpm.config.ignored?(target)
 
         metadata = (metadata || {}).dup
+        instrumented = !req_segments.nil?
 
         if req_segments
           segment_data = req_segments.to_h
           segment_data[:segment_summary]&.each { |k, v| metadata[k] = v }
         end
 
+        # Track instrumented count for correct dashboard averaging
+        metadata[:_instrumented] = 1 if instrumented
+
         sample_type = early_sample_type(
           error: error,
           duration: duration,
           kind: kind,
           target: target,
-          operation: operation
+          operation: operation,
+          instrumented: instrumented
         )
 
+        # Slow spike detection: force instrument next request for this endpoint
+        if !instrumented && (error || duration >= Catpm.config.slow_threshold_for(kind.to_sym))
+          trigger_force_instrument(kind: kind, target: target, operation: operation)
+        end
+
         if sample_type
           context = (context || {}).dup
 
@@ -317,7 +343,6 @@ module Catpm
               }
             end
 
-            req_segments.release! # free internal state for GC
           end
 
           context = scrub(context)
@@ -343,6 +368,53 @@ module Catpm
         Catpm.buffer&.push(ev)
       end
 
+      def process_checkpoint(kind:, target:, operation:, context:, metadata:, checkpoint_data:, request_start:)
+        return unless Catpm.enabled?
+
+        segments = checkpoint_data[:segments].dup
+        collapse_code_wrappers(segments)
+
+        duration_so_far = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - request_start) * 1000.0
+
+        # Inject root request segment
+        root_segment = {
+          type: 'request',
+          detail: "#{operation.presence || kind} #{target}",
+          duration: duration_so_far.round(2),
+          offset: 0.0
+        }
+        segments.each do |seg|
+          if seg.key?(:parent_index)
+            seg[:parent_index] += 1
+          else
+            seg[:parent_index] = 0
+          end
+        end
+        segments.unshift(root_segment)
+
+        checkpoint_context = (context || {}).dup
+        checkpoint_context[:segments] = segments
+        checkpoint_context[:segment_summary] = checkpoint_data[:summary]
+        checkpoint_context[:segments_capped] = checkpoint_data[:overflow]
+        checkpoint_context[:partial] = true
+        checkpoint_context[:checkpoint_number] = checkpoint_data[:checkpoint_number]
+        checkpoint_context = scrub(checkpoint_context)
+
+        ev = Event.new(
+          kind: kind,
+          target: target,
+          operation: operation.to_s,
+          duration: duration_so_far,
+          started_at: Time.current,
+          status: 200,
+          context: checkpoint_context,
+          sample_type: 'random',
+          metadata: (metadata || {}).dup.merge(checkpoint_data[:summary] || {})
+        )
+
+        Catpm.buffer&.push(ev)
+      end
+
       def process_custom(name:, duration:, metadata: {}, error: nil, context: {})
         return unless Catpm.enabled?
         return if Catpm.config.ignored?(name)
@@ -363,7 +435,64 @@ module Catpm
         Catpm.buffer&.push(ev)
       end
 
-      private
+      # --- Pre-sampling: decide BEFORE request whether to instrument ---
+
+      # For HTTP middleware where endpoint is unknown at start.
+      # Returns true if this request should get full instrumentation.
+      def should_instrument_request?
+        # Force after slow spike detection
+        if (@force_instrument_count || 0) > 0
+          @force_instrument_count -= 1
+          return true
+        end
+
+        rand(Catpm.config.random_sample_rate) == 0
+      end
+
+      # For track_request where endpoint is known at start.
+      # Filling phase ensures new endpoints get instrumented samples quickly.
+      def should_instrument?(kind, target, operation)
+        endpoint_key = [kind.to_s, target.to_s, (operation || '').to_s]
+
+        # Force after slow spike
+        if force_instrument_endpoints.delete(endpoint_key)
+          return true
+        end
+
+        # Filling phase — endpoint hasn't collected enough instrumented samples yet
+        max = Catpm.config.max_random_samples_per_endpoint
+        if max.nil? || instrumented_sample_counts[endpoint_key] < max
+          return true
+        end
+
+        rand(Catpm.config.random_sample_rate) == 0
+      end
+
+      # Called when a slow/error request had no instrumentation —
+      # forces the NEXT request(s) to be fully instrumented.
+      def trigger_force_instrument(kind: nil, target: nil, operation: nil)
+        if kind && target
+          endpoint_key = [kind.to_s, target.to_s, (operation || '').to_s]
+          force_instrument_endpoints[endpoint_key] = true
+        end
+        @force_instrument_count = (@force_instrument_count || 0) + 1
+      end
+
+      def reset_sample_counts!
+        @instrumented_sample_counts = nil
+        @force_instrument_endpoints = nil
+        @force_instrument_count = nil
+      end
+
+    private
+
+      def force_instrument_endpoints
+        @force_instrument_endpoints ||= {}
+      end
+
+      def instrumented_sample_counts
+        @instrumented_sample_counts ||= Hash.new(0)
+      end
 
       # Remove near-zero-duration "code" spans that merely wrap a "controller" span.
       # This happens when CallTracer (TracePoint) captures a thin dispatch method
@@ -419,31 +548,30 @@ module Catpm
       end
 
       # Determine sample type at event creation time so only sampled events
-      # carry full context in the buffer. Includes filling phase via
-      # process-level counter (resets on restart — acceptable approximation).
-      def early_sample_type(error:, duration:, kind:, target:, operation:)
+      # carry full context in the buffer.
+      #
+      # When instrumented: false, only error/slow get a sample_type —
+      # non-instrumented normal requests just contribute duration/count.
+      # Filling counter only increments for instrumented requests so
+      # non-instrumented requests don't waste filling slots.
+      def early_sample_type(error:, duration:, kind:, target:, operation:, instrumented: true)
         return 'error' if error
         return 'slow' if duration >= Catpm.config.slow_threshold_for(kind.to_sym)
 
-        # Filling phase: always sample until endpoint has enough random samples
+        # Non-instrumented requests have no segments — skip sample creation
+        return nil unless instrumented
+
+        # Filling phase: always sample until endpoint has enough instrumented samples
         endpoint_key = [kind.to_s, target, operation.to_s]
-        count = random_sample_counts[endpoint_key]
+        count = instrumented_sample_counts[endpoint_key]
         max_random = Catpm.config.max_random_samples_per_endpoint
         if max_random.nil? || count < max_random
-          random_sample_counts[endpoint_key] = count + 1
+          instrumented_sample_counts[endpoint_key] = count + 1
           return 'random'
         end
 
-        return 'random' if rand(Catpm.config.random_sample_rate) == 0
-        nil
-      end
-
-      def random_sample_counts
-        @random_sample_counts ||= Hash.new(0)
-      end
-
-      def reset_sample_counts!
-        @random_sample_counts = nil
+        # Instrumented request was already chosen by dice roll at start — always sample
+        'random'
       end
 
       def inject_gap_segments(segments, req_segments, gap, ctrl_idx, ctrl_seg)

data/lib/catpm/configuration.rb

@@ -43,6 +43,7 @@ module Catpm
       events_max_samples_per_name max_stack_samples_per_request
       max_error_detail_length max_fingerprint_app_frames
       max_fingerprint_gem_frames cleanup_batch_size caller_scan_depth
+      max_request_memory
     ].freeze
 
     (REQUIRED_NUMERIC + OPTIONAL_NUMERIC).each do |attr|
@@ -116,6 +117,7 @@ module Catpm
       @max_fingerprint_gem_frames = 3
       @cleanup_batch_size = 1_000
       @caller_scan_depth = 50
+      @max_request_memory = 2.megabytes
       @instrument_call_tree = false
       @show_untracked_segments = false
     end

data/lib/catpm/middleware.rb

@@ -12,14 +12,16 @@ module Catpm
       Catpm.flusher&.ensure_running!
 
       env['catpm.request_start'] = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      Thread.current[:catpm_request_start] = env['catpm.request_start']
 
-      if Catpm.config.instrument_segments
+      if Catpm.config.instrument_segments && Collector.should_instrument_request?
         use_sampler = Catpm.config.instrument_stack_sampler || Catpm.config.instrument_call_tree
         req_segments = RequestSegments.new(
           max_segments: Catpm.config.max_segments_per_request,
           request_start: env['catpm.request_start'],
           stack_sample: use_sampler,
-          call_tree: Catpm.config.instrument_call_tree
+          call_tree: Catpm.config.instrument_call_tree,
+          memory_limit: Catpm.config.max_request_memory
         )
         env['catpm.segments'] = req_segments
         Thread.current[:catpm_request_segments] = req_segments
@@ -30,11 +32,10 @@ module Catpm
       record_exception(env, e)
       raise
     ensure
-      if Catpm.config.instrument_segments
-        req_segments&.stop_sampler
-        req_segments&.release!
-        Thread.current[:catpm_request_segments] = nil
-      end
+      req_segments&.stop_sampler
+      req_segments&.release!
+      Thread.current[:catpm_request_segments] = nil
+      Thread.current[:catpm_request_start] = nil
     end
 
     private

data/lib/catpm/request_segments.rb

@@ -5,9 +5,13 @@ module Catpm
     # Pre-computed symbol pairs — each type computed once per process lifetime.
     SUMMARY_KEYS = Hash.new { |h, k| h[k] = [:"#{k}_count", :"#{k}_duration"] }
 
-    attr_reader :segments, :summary, :request_start
+    # Per-segment byte estimate: Hash overhead + typical keys (type, duration, detail, offset, source, parent_index)
+    SEGMENT_BASE_BYTES = Event::OBJECT_OVERHEAD + (6 * Event::HASH_ENTRY_SIZE)
+    SEGMENT_STRING_OVERHEAD = Event::OBJECT_OVERHEAD # per-string overhead in segment values
 
-    def initialize(max_segments:, request_start: nil, stack_sample: false, call_tree: false)
+    attr_reader :segments, :summary, :request_start, :estimated_bytes, :checkpoint_count
+
+    def initialize(max_segments:, request_start: nil, stack_sample: false, call_tree: false, memory_limit: nil)
       @max_segments = max_segments
       @request_start = request_start || Process.clock_gettime(Process::CLOCK_MONOTONIC)
       @segments = []
@@ -16,6 +20,10 @@ module Catpm
       @span_stack = []
       @tracked_ranges = []
       @call_tree = call_tree
+      @memory_limit = memory_limit
+      @estimated_bytes = 0
+      @checkpoint_callback = nil
+      @checkpoint_count = 0
 
       if stack_sample
         @sampler = StackSampler.new(target_thread: Thread.current, request_start: @request_start, call_tree: call_tree)
@@ -23,6 +31,10 @@ module Catpm
       end
     end
 
+    def on_checkpoint(&block)
+      @checkpoint_callback = block
+    end
+
     def add(type:, duration:, detail:, source: nil, started_at: nil)
       type_key = type.to_sym
       count_key, dur_key = SUMMARY_KEYS[type_key]
@@ -50,6 +62,9 @@ module Catpm
           @segments[min_idx] = segment
         end
       end
+
+      @estimated_bytes += estimate_segment_bytes(segment)
+      maybe_checkpoint
     end
 
     def push_span(type:, detail:, started_at: nil)
@@ -64,6 +79,7 @@ module Catpm
       index = @segments.size
       @segments << segment
       @span_stack.push(index)
+      @estimated_bytes += estimate_segment_bytes(segment)
       index
     end
 
@@ -114,6 +130,7 @@ module Catpm
       @summary = {}
       @tracked_ranges = []
       @sampler = nil
+      @estimated_bytes = 0
     end
 
     def overflowed?
@@ -127,5 +144,78 @@ module Catpm
         segments_capped: @overflow
       }
     end
+
+    private
+
+    def estimate_segment_bytes(segment)
+      bytes = SEGMENT_BASE_BYTES
+      bytes += segment[:detail].bytesize + SEGMENT_STRING_OVERHEAD if segment[:detail]
+      bytes += segment[:type].bytesize + SEGMENT_STRING_OVERHEAD if segment[:type]
+      bytes += segment[:source].bytesize + SEGMENT_STRING_OVERHEAD if segment[:source]
+      bytes
+    end
+
+    def maybe_checkpoint
+      return unless @memory_limit && @estimated_bytes > @memory_limit && @checkpoint_callback
+
+      checkpoint_data = {
+        segments: @segments,
+        summary: @summary,
+        overflow: @overflow,
+        sampler_segments: @sampler ? sampler_segments_for_checkpoint : [],
+        checkpoint_number: @checkpoint_count
+      }
+
+      @checkpoint_count += 1
+      rebuild_after_checkpoint
+      @checkpoint_callback.call(checkpoint_data)
+    end
+
+    def sampler_segments_for_checkpoint
+      if @call_tree
+        result = @sampler&.to_call_tree(tracked_ranges: @tracked_ranges) || []
+      else
+        result = @sampler&.to_segments(tracked_ranges: @tracked_ranges) || []
+      end
+      @sampler&.clear_samples!
+      result
+    end
+
+    # After checkpoint: keep only active spans from @span_stack, reset everything else.
+    def rebuild_after_checkpoint
+      if @span_stack.any?
+        # Clone active spans with corrected indices
+        new_segments = []
+        old_to_new = {}
+
+        @span_stack.each do |old_idx|
+          seg = @segments[old_idx]
+          next unless seg
+
+          new_idx = new_segments.size
+          old_to_new[old_idx] = new_idx
+          new_segments << seg.dup
+        end
+
+        # Fix parent_index references in cloned spans
+        new_segments.each do |seg|
+          if seg.key?(:parent_index) && old_to_new.key?(seg[:parent_index])
+            seg[:parent_index] = old_to_new[seg[:parent_index]]
+          else
+            seg.delete(:parent_index)
+          end
+        end
+
+        @span_stack = @span_stack.filter_map { |old_idx| old_to_new[old_idx] }
+        @segments = new_segments
+      else
+        @segments = []
+      end
+
+      @summary = Hash.new(0)
+      @tracked_ranges = []
+      @overflow = false
+      @estimated_bytes = 0
+    end
   end
 end

data/lib/catpm/trace.rb

@@ -76,15 +76,30 @@ module Catpm
     owns_segments = false
 
     if req_segments.nil? && config.instrument_segments
-      use_sampler = config.instrument_stack_sampler || config.instrument_call_tree
-      req_segments = RequestSegments.new(
-        max_segments: config.max_segments_per_request,
-        request_start: Process.clock_gettime(Process::CLOCK_MONOTONIC),
-        stack_sample: use_sampler,
-        call_tree: config.instrument_call_tree
-      )
-      Thread.current[:catpm_request_segments] = req_segments
-      owns_segments = true
+      if Collector.should_instrument?(kind, target, operation)
+        use_sampler = config.instrument_stack_sampler || config.instrument_call_tree
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        req_segments = RequestSegments.new(
+          max_segments: config.max_segments_per_request,
+          request_start: start_time,
+          stack_sample: use_sampler,
+          call_tree: config.instrument_call_tree,
+          memory_limit: config.max_request_memory
+        )
+        Thread.current[:catpm_request_segments] = req_segments
+        owns_segments = true
+
+        if config.max_request_memory
+          req_segments.on_checkpoint do |checkpoint_data|
+            Collector.process_checkpoint(
+              kind: kind, target: target, operation: operation,
+              context: context, metadata: metadata,
+              checkpoint_data: checkpoint_data,
+              request_start: start_time
+            )
+          end
+        end
+      end
     end
 
     if req_segments

data/lib/catpm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Catpm
-  VERSION = '0.8.3'
+  VERSION = '0.9.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: catpm
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.9.0
 platform: ruby
 authors:
 - ''