catpm 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7550489165d4ba6b3c476bd1f2637fc6e68eed8e3def5bd32420dfc0632ff2d7
-  data.tar.gz: ec9ed224bcd90cd67835842385305146736ce634310b9b62eaf33997bf7e2ff8
+  metadata.gz: 10d81d0f093dc28f0f22a3bd9eda141ca89dfa44daacabf4fda1190c549681e2
+  data.tar.gz: 80c8cd25d825e3dc3eae16bf1fc58adddee4c5cfe5bf420ef1f663d0ca295d2e
 SHA512:
-  metadata.gz: ad84a45f6d0b8b508c94a5a9d0d06b9671a7820fedd93eda19eeee070b41669e625a814a6a2fab6e69fa80ae83edbaa81edd9a8f20e915d921973ba8a6e70822
-  data.tar.gz: 75cd050d65178c666e0369eecbc13c1fe1cb18f8d159f75c2078ecea7422e1844e4836c87320681d6b6d68718c9624f0b8abd67276888d28b09410265b43f8ad
+  metadata.gz: aafdfdedafaa9558c06da29ce3fc6c7af9e4f3120b7c1556b8e5162cbd781e72b1d8613720078e2ae2eb33e2d24d0304922c3de04cc622aa6df17602299e8ad5
+  data.tar.gz: 9f9eb35a16e182155b215323beafa593af8617de530dc36a993187ba8d2783b1b324855a783b27c7d998d97dd6e59d40b4449df852a4f1b70563ffa8009190b7

data/README.md

@@ -1,5 +1,5 @@
 gem build catpm.gemspec
-gem push catpm-0.9.0.gem
+gem push catpm-0.9.2.gem
 
 # Catpm
 

data/app/models/catpm/sample.rb

@@ -14,6 +14,10 @@ module Catpm
     scope :recent, ->(period = 1.hour) { where(recorded_at: period.ago..) }
     scope :for_error, ->(fingerprint) { where(error_fingerprint: fingerprint) }
 
+    def self.request_id_supported?
+      column_names.include?('request_id')
+    end
+
     def parsed_context
       case context
       when Hash then context

data/db/migrate/20250601000001_create_catpm_tables.rb

@@ -33,11 +33,13 @@ class CreateCatpmTables < ActiveRecord::Migration[8.0]
       t.float :duration, null: false
       t.json :context
       t.string :error_fingerprint, limit: 64
+      t.string :request_id, limit: 16
     end
 
     add_index :catpm_samples, :recorded_at, name: 'idx_catpm_samples_time'
     add_index :catpm_samples, [:kind, :recorded_at], name: 'idx_catpm_samples_kind_time'
     add_index :catpm_samples, :error_fingerprint, name: 'idx_catpm_samples_error_fp'
+    add_index :catpm_samples, :request_id, name: 'idx_catpm_samples_request_id'
 
     create_table :catpm_errors do |t|
       t.string :fingerprint, null: false, limit: 64

data/lib/catpm/adapter/base.rb

@@ -29,7 +29,7 @@ module Catpm
               bucket = bucket_map[sample_data[:bucket_key]]
               next unless bucket
 
-              {
+              record = {
                 bucket_id: bucket.id,
                 kind: sample_data[:kind],
                 sample_type: sample_data[:sample_type],
@@ -38,6 +38,8 @@ module Catpm
                 context: sample_data[:context],
                 error_fingerprint: sample_data[:error_fingerprint]
               }
+              record[:request_id] = sample_data[:request_id] if Catpm::Sample.request_id_supported?
+              record
             end
 
             Catpm::Sample.insert_all(records) if records.any?

data/lib/catpm/collector.rb

@@ -60,11 +60,19 @@ module Catpm
           instrumented: instrumented
         )
 
-        # Slow spike detection: force the NEXT HTTP request through middleware
-        # to be fully instrumented (uses global counter for should_instrument_request?).
-        # Skip if already handled by track_request (tracked_instrumented).
-        if !instrumented && (payload[:exception] || duration >= Catpm.config.slow_threshold_for(:http))
-          trigger_force_instrument
+        # Force the NEXT HTTP request to be fully instrumented when this one
+        # wasn't. Covers slow/error spikes and filling phase (new endpoints that
+        # haven't collected enough instrumented samples yet).
+        if !instrumented
+          if payload[:exception] || duration >= Catpm.config.slow_threshold_for(:http)
+            trigger_force_instrument
+          else
+            max = Catpm.config.max_random_samples_per_endpoint
+            if max
+              endpoint_key = ['http', target, operation]
+              trigger_force_instrument if instrumented_sample_counts[endpoint_key] < max
+            end
+          end
         end
 
         if sample_type
@@ -360,6 +368,8 @@ module Catpm
           context = nil
         end
 
+        request_id = req_segments&.request_id
+
         ev = Event.new(
           kind: kind,
           target: target,
@@ -372,13 +382,14 @@ module Catpm
           metadata: metadata,
           error_class: error&.class&.name,
           error_message: error&.message,
-          backtrace: error&.backtrace
+          backtrace: error&.backtrace,
+          request_id: request_id
         )
 
         Catpm.buffer&.push(ev)
       end
 
-      def process_checkpoint(kind:, target:, operation:, context:, metadata:, checkpoint_data:, request_start:)
+      def process_checkpoint(kind:, target:, operation:, context:, metadata:, checkpoint_data:, request_start:, request_id: nil)
         return unless Catpm.enabled?
 
         segments = checkpoint_data[:segments].dup
@@ -419,7 +430,8 @@ module Catpm
           status: 200,
           context: checkpoint_context,
           sample_type: 'random',
-          metadata: (metadata || {}).dup.merge(checkpoint_data[:summary] || {})
+          metadata: (metadata || {}).dup.merge(checkpoint_data[:summary] || {}),
+          request_id: request_id
         )
 
         Catpm.buffer&.push(ev)
@@ -600,10 +612,12 @@ module Catpm
       # Determine sample type at event creation time so only sampled events
       # carry full context in the buffer.
       #
-      # When instrumented: false, only error/slow get a sample_type —
-      # non-instrumented normal requests just contribute duration/count.
-      # Filling counter only increments for instrumented requests so
-      # non-instrumented requests don't waste filling slots.
+      # Non-instrumented requests never get a sample (they have no segments).
+      # Filling phase is handled by the caller via trigger_force_instrument,
+      # so the NEXT request gets full instrumentation with segments.
+      #
+      # Post-filling: non-instrumented requests just contribute duration/count
+      # to the bucket, no sample created.
       def early_sample_type(error:, duration:, kind:, target:, operation:, instrumented: true)
         # Errors: only create sample for instrumented requests (with segments).
         # Non-instrumented errors are still tracked in error_groups via
@@ -617,7 +631,9 @@ module Catpm
         # don't count towards filling phase (they have no segments).
         return 'slow' if is_slow && !instrumented
 
-        # Non-instrumented requests have no segments — skip sample creation
+        # Non-instrumented requests have no segments — skip sample creation.
+        # Filling phase is handled by the caller via trigger_force_instrument
+        # so the NEXT request gets full instrumentation with segments.
         return nil unless instrumented
 
         # Count this instrumented request towards filling phase completion.

data/lib/catpm/event.rb

@@ -9,14 +9,14 @@ module Catpm
 
     attr_accessor :kind, :target, :operation, :duration, :started_at,
                   :metadata, :error_class, :error_message, :backtrace,
-                  :sample_type, :context, :status
+                  :sample_type, :context, :status, :request_id
 
     EMPTY_HASH = {}.freeze
     private_constant :EMPTY_HASH
 
     def initialize(kind:, target:, operation: '', duration: 0.0, started_at: nil,
                    metadata: nil, error_class: nil, error_message: nil, backtrace: nil,
-                   sample_type: nil, context: nil, status: nil)
+                   sample_type: nil, context: nil, status: nil, request_id: nil)
       @kind = kind.to_s
       @target = target.to_s
       @operation = (operation || '').to_s
@@ -32,6 +32,7 @@ module Catpm
       @sample_type = sample_type
       @context = context
       @status = status
+      @request_id = request_id
     end
 
     def estimated_bytes

data/lib/catpm/flusher.rb

@@ -3,6 +3,7 @@
 module Catpm
   class Flusher
     ERROR_LOG_BACKTRACE_LINES = 5
+    PARTIAL_STALE_TIMEOUT = 600 # seconds — orphaned partial samples cleaned after 10 minutes
 
     attr_reader :running
 
@@ -181,7 +182,8 @@ module Catpm
             sample_type: sample_type,
             recorded_at: event.started_at,
             duration: event.duration,
-            context: event.context || {}
+            context: event.context || {},
+            request_id: event.request_id
           }
           sample_hash[:error_fingerprint] = error_fp if error_fp
           samples << sample_hash
@@ -219,6 +221,8 @@ module Catpm
         b
       end
 
+      samples = merge_request_samples(samples)
+
       [ buckets, samples, error_groups.values ]
     end
 
@@ -346,6 +350,7 @@ module Catpm
       @last_cleanup_at = Time.now
       downsample_buckets
       cleanup_expired_data if Catpm.config.retention_period
+      cleanup_orphaned_partials
       Collector.reset_sample_counts!
     end
 
@@ -489,6 +494,136 @@ module Catpm
       combined.empty? ? nil : combined.serialize
     end
 
+    def merge_request_samples(samples)
+      return samples unless Catpm::Sample.request_id_supported?
+
+      by_request = {}     # request_id => { partials: [], final: nil }
+      regular = []
+
+      samples.each do |s|
+        rid = s[:request_id]
+        if rid
+          entry = (by_request[rid] ||= { partials: [], final: nil })
+          if s[:context].is_a?(Hash) && s[:context][:partial]
+            entry[:partials] << s
+          else
+            entry[:final] = s
+          end
+        else
+          regular << s
+        end
+      end
+
+      merged = []
+      by_request.each do |rid, entry|
+        if entry[:final]
+          # Merge in-batch partials
+          if entry[:partials].any?
+            merge_checkpoint_contexts(
+              entry[:final][:context],
+              entry[:partials].map { |p| p[:context] }
+            )
+          end
+
+          # Merge cross-batch partials from DB
+          db_partials = Catpm::Sample.where(request_id: rid)
+          if db_partials.exists?
+            merge_checkpoint_contexts(
+              entry[:final][:context],
+              db_partials.map(&:parsed_context)
+            )
+            db_partials.delete_all
+          end
+
+          # Clear request_id so persisted final sample won't be treated as orphan
+          entry[:final][:request_id] = nil
+          merged << entry[:final]
+        else
+          # Only partials, no final yet — persist as-is
+          merged.concat(entry[:partials])
+        end
+      end
+
+      regular + merged
+    end
+
+    def merge_checkpoint_contexts(final_ctx, checkpoint_ctxs)
+      final_segments = final_ctx[:segments] || final_ctx['segments']
+      return unless final_segments
+
+      final_ctrl_idx = final_segments.index { |s|
+        (s[:type] || s['type']) == 'controller'
+      }
+
+      sorted = checkpoint_ctxs.sort_by { |c|
+        c[:checkpoint_number] || c['checkpoint_number'] || 0
+      }
+
+      sorted.each do |cp_ctx|
+        cp_segments = cp_ctx[:segments] || cp_ctx['segments'] || []
+
+        old_to_new = {}
+        kept = []
+
+        cp_segments.each_with_index do |seg, i|
+          seg_type = seg[:type] || seg['type']
+          next if seg_type == 'request'
+          next if seg_type == 'controller'
+          old_to_new[i] = final_segments.size + kept.size
+          kept << seg.dup
+        end
+
+        kept.each do |seg|
+          pi_key = seg.key?(:parent_index) ? :parent_index : 'parent_index'
+          pi = seg[pi_key]
+          next unless pi
+
+          if old_to_new.key?(pi)
+            seg[pi_key] = old_to_new[pi]
+          else
+            seg[pi_key] = final_ctrl_idx || 0
+          end
+        end
+
+        final_segments.concat(kept)
+
+        # Merge summary
+        cp_summary = cp_ctx[:segment_summary] || cp_ctx['segment_summary']
+        if cp_summary
+          use_symbols = final_ctx.key?(:segment_summary)
+          summary_key = use_symbols ? :segment_summary : 'segment_summary'
+          final_summary = final_ctx[summary_key] ||= {}
+          cp_summary.each do |k, v|
+            nk = use_symbols ? k.to_sym : k.to_s
+            final_summary[nk] = (final_summary[nk] || 0) + v.to_f
+          end
+        end
+
+        # Merge capped flag
+        capped_key = final_ctx.key?(:segments_capped) ? :segments_capped : 'segments_capped'
+        cp_capped = cp_ctx[:segments_capped] || cp_ctx['segments_capped']
+        final_ctx[capped_key] = true if cp_capped
+      end
+
+      # Clean up checkpoint markers
+      final_ctx.delete(:partial)
+      final_ctx.delete('partial')
+      final_ctx.delete(:request_id)
+      final_ctx.delete('request_id')
+      final_ctx.delete(:checkpoint_number)
+      final_ctx.delete('checkpoint_number')
+    end
+
+    def cleanup_orphaned_partials
+      return unless Catpm::Sample.request_id_supported?
+
+      Catpm::Sample.where.not(request_id: nil)
+        .where(recorded_at: ..PARTIAL_STALE_TIMEOUT.seconds.ago)
+        .delete_all
+    rescue => e
+      Catpm.config.error_handler&.call(e)
+    end
+
     def cleanup_expired_data
       cutoff = Catpm.config.retention_period.ago
       batch_size = Catpm.config.cleanup_batch_size

data/lib/catpm/request_segments.rb

@@ -9,7 +9,7 @@ module Catpm
     SEGMENT_BASE_BYTES = Event::OBJECT_OVERHEAD + (6 * Event::HASH_ENTRY_SIZE)
     SEGMENT_STRING_OVERHEAD = Event::OBJECT_OVERHEAD # per-string overhead in segment values
 
-    attr_reader :segments, :summary, :request_start, :estimated_bytes, :checkpoint_count
+    attr_reader :segments, :summary, :request_start, :estimated_bytes, :checkpoint_count, :request_id
 
     def initialize(max_segments:, request_start: nil, stack_sample: false, call_tree: false, memory_limit: nil)
       @max_segments = max_segments
@@ -24,6 +24,7 @@ module Catpm
       @estimated_bytes = 0
       @checkpoint_callback = nil
       @checkpoint_count = 0
+      @request_id = memory_limit ? SecureRandom.hex(8) : nil
 
       if stack_sample
         @sampler = StackSampler.new(target_thread: Thread.current, request_start: @request_start, call_tree: call_tree)

data/lib/catpm/trace.rb

@@ -101,7 +101,8 @@ module Catpm
               kind: kind, target: target, operation: operation,
               context: context, metadata: metadata,
               checkpoint_data: checkpoint_data,
-              request_start: start_time
+              request_start: start_time,
+              request_id: req_segments.request_id
             )
           end
         end

data/lib/catpm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Catpm
-  VERSION = '0.9.2'
+  VERSION = '0.9.3'
 end

data/lib/tasks/catpm_tasks.rake

@@ -42,6 +42,14 @@ namespace :catpm do
       puts '[catpm] catpm_errors.pinned already exists, skipping'
     end
 
+    unless connection.column_exists?(:catpm_samples, :request_id)
+      connection.add_column :catpm_samples, :request_id, :string, limit: 16
+      connection.add_index :catpm_samples, :request_id, name: 'idx_catpm_samples_request_id'
+      puts '[catpm] Added request_id column to catpm_samples'
+    else
+      puts '[catpm] catpm_samples.request_id already exists, skipping'
+    end
+
     unless connection.table_exists?(:catpm_event_prefs)
       connection.create_table :catpm_event_prefs do |t|
         t.string :name, null: false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: catpm
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - ''