catpm 0.9.4 → 0.9.6

data/lib/catpm/flusher.rb

@@ -3,7 +3,6 @@
 module Catpm
   class Flusher
     ERROR_LOG_BACKTRACE_LINES = 5
-    PARTIAL_STALE_TIMEOUT = 600 # seconds — orphaned partial samples cleaned after 10 minutes
 
     attr_reader :running
 
@@ -182,8 +181,7 @@ module Catpm
             sample_type: sample_type,
             recorded_at: event.started_at,
             duration: event.duration,
-            context: event.context || {},
-            request_id: event.request_id
+            context: event.context || {}
           }
           sample_hash[:error_fingerprint] = error_fp if error_fp
           samples << sample_hash
@@ -221,8 +219,6 @@ module Catpm
         b
       end
 
-      samples = merge_request_samples(samples)
-
       [ buckets, samples, error_groups.values ]
     end
 
@@ -350,7 +346,6 @@ module Catpm
       @last_cleanup_at = Time.now
       downsample_buckets
       cleanup_expired_data if Catpm.config.retention_period
-      cleanup_orphaned_partials
       Collector.reset_sample_counts!
     end
 
@@ -398,83 +393,93 @@ module Catpm
       cutoff = age_threshold.ago
       target_seconds = target_interval.to_i
 
-      # Process in batches to avoid loading all old buckets into memory
-      Catpm::Bucket.where(bucket_start: ...cutoff)
-        .select(:id, :kind, :target, :operation, :bucket_start)
-        .group_by { |b| [b.kind, b.target, b.operation] }
-        .each do |(_kind, _target, _operation), endpoint_buckets|
-          groups = endpoint_buckets.group_by do |bucket|
-            epoch = bucket.bucket_start.to_i
-            aligned_epoch = epoch - (epoch % target_seconds)
-            Time.at(aligned_epoch).utc
-          end
+      # Get unique endpoint keys first (small set), then process per-endpoint
+      # to avoid loading all old buckets into memory at once
+      endpoint_keys = Catpm::Bucket.where(bucket_start: ...cutoff)
+        .distinct.pluck(:kind, :target, :operation)
 
-          groups.each do |aligned_start, stub_buckets|
-            next if stub_buckets.size == 1 && stub_buckets.first.bucket_start.to_i % target_seconds == 0
-
-            # Load full records only for groups that need merging
-            bucket_ids = stub_buckets.map(&:id)
-            buckets = Catpm::Bucket.where(id: bucket_ids).to_a
-
-            merged = {
-              kind: buckets.first.kind,
-              target: buckets.first.target,
-              operation: buckets.first.operation,
-              bucket_start: aligned_start,
-              count: buckets.sum(&:count),
-              success_count: buckets.sum(&:success_count),
-              failure_count: buckets.sum(&:failure_count),
-              duration_sum: buckets.sum(&:duration_sum),
-              duration_max: buckets.map(&:duration_max).max,
-              duration_min: buckets.map(&:duration_min).min,
-              metadata_sum: merge_bucket_metadata(buckets, adapter),
-              p95_digest: merge_bucket_digests(buckets)
-            }
-
-            survivor = buckets.first
-
-            # Reassign all samples to the survivor bucket
-            Catpm::Sample.where(bucket_id: bucket_ids).update_all(bucket_id: survivor.id)
-
-            # Delete non-survivor source buckets (now sample-free)
-            Catpm::Bucket.where(id: bucket_ids - [survivor.id]).delete_all
-
-            # Overwrite survivor with merged data
-            survivor.update!(
-              bucket_start: aligned_start,
-              count: merged[:count],
-              success_count: merged[:success_count],
-              failure_count: merged[:failure_count],
-              duration_sum: merged[:duration_sum],
-              duration_max: merged[:duration_max],
-              duration_min: merged[:duration_min],
-              metadata_sum: merged[:metadata_sum],
-              p95_digest: merged[:p95_digest]
-            )
-          end
+      endpoint_keys.each do |kind, target, operation|
+        endpoint_buckets = Catpm::Bucket
+          .where(kind: kind, target: target, operation: operation, bucket_start: ...cutoff)
+          .select(:id, :bucket_start).to_a
+
+        groups = endpoint_buckets.group_by do |bucket|
+          epoch = bucket.bucket_start.to_i
+          aligned_epoch = epoch - (epoch % target_seconds)
+          Time.at(aligned_epoch).utc
+        end
+
+        groups.each do |aligned_start, stub_buckets|
+          next if stub_buckets.size == 1 && stub_buckets.first.bucket_start.to_i % target_seconds == 0
+
+          # Load full records only for groups that need merging
+          bucket_ids = stub_buckets.map(&:id)
+          buckets = Catpm::Bucket.where(id: bucket_ids).to_a
+
+          merged = {
+            kind: buckets.first.kind,
+            target: buckets.first.target,
+            operation: buckets.first.operation,
+            bucket_start: aligned_start,
+            count: buckets.sum(&:count),
+            success_count: buckets.sum(&:success_count),
+            failure_count: buckets.sum(&:failure_count),
+            duration_sum: buckets.sum(&:duration_sum),
+            duration_max: buckets.map(&:duration_max).max,
+            duration_min: buckets.map(&:duration_min).min,
+            metadata_sum: merge_bucket_metadata(buckets, adapter),
+            p95_digest: merge_bucket_digests(buckets)
+          }
+
+          survivor = buckets.first
+
+          # Reassign all samples to the survivor bucket
+          Catpm::Sample.where(bucket_id: bucket_ids).update_all(bucket_id: survivor.id)
+
+          # Delete non-survivor source buckets (now sample-free)
+          Catpm::Bucket.where(id: bucket_ids - [survivor.id]).delete_all
+
+          # Overwrite survivor with merged data
+          survivor.update!(
+            bucket_start: aligned_start,
+            count: merged[:count],
+            success_count: merged[:success_count],
+            failure_count: merged[:failure_count],
+            duration_sum: merged[:duration_sum],
+            duration_max: merged[:duration_max],
+            duration_min: merged[:duration_min],
+            metadata_sum: merged[:metadata_sum],
+            p95_digest: merged[:p95_digest]
+          )
         end
+      end
     end
 
     def downsample_event_tier(target_interval:, age_threshold:, adapter:)
       cutoff = age_threshold.ago
       target_seconds = target_interval.to_i
 
-      source_buckets = Catpm::EventBucket.where(bucket_start: ...cutoff).to_a
-      return if source_buckets.empty?
+      # Process per-name to avoid loading all event buckets into memory
+      names = Catpm::EventBucket.where(bucket_start: ...cutoff).distinct.pluck(:name)
+      return if names.empty?
 
-      groups = source_buckets.group_by do |bucket|
-        epoch = bucket.bucket_start.to_i
-        aligned_epoch = epoch - (epoch % target_seconds)
-        aligned_start = Time.at(aligned_epoch).utc
-        [bucket.name, aligned_start]
-      end
+      names.each do |name|
+        buckets = Catpm::EventBucket.where(name: name, bucket_start: ...cutoff).to_a
+        next if buckets.empty?
 
-      groups.each do |(name, aligned_start), buckets|
-        next if buckets.size == 1 && buckets.first.bucket_start.to_i % target_seconds == 0
+        groups = buckets.group_by do |bucket|
+          epoch = bucket.bucket_start.to_i
+          aligned_epoch = epoch - (epoch % target_seconds)
+          Time.at(aligned_epoch).utc
+        end
+
+        groups.each do |aligned_start, group_buckets|
+          next if group_buckets.size == 1 && group_buckets.first.bucket_start.to_i % target_seconds == 0
 
-        merged = { name: name, bucket_start: aligned_start, count: buckets.sum(&:count) }
-        Catpm::EventBucket.where(id: buckets.map(&:id)).delete_all
-        adapter.persist_event_buckets([merged])
+          merged = { name: name, bucket_start: aligned_start, count: group_buckets.sum(&:count) }
+          Catpm::EventBucket.where(id: group_buckets.map(&:id)).delete_all
+          adapter.persist_event_buckets([merged])
+        end
       end
     end
 
@@ -494,136 +499,6 @@ module Catpm
       combined.empty? ? nil : combined.serialize
     end
 
-    def merge_request_samples(samples)
-      return samples unless Catpm::Sample.request_id_supported?
-
-      by_request = {}     # request_id => { partials: [], final: nil }
-      regular = []
-
-      samples.each do |s|
-        rid = s[:request_id]
-        if rid
-          entry = (by_request[rid] ||= { partials: [], final: nil })
-          if s[:context].is_a?(Hash) && s[:context][:partial]
-            entry[:partials] << s
-          else
-            entry[:final] = s
-          end
-        else
-          regular << s
-        end
-      end
-
-      merged = []
-      by_request.each do |rid, entry|
-        if entry[:final]
-          # Merge in-batch partials
-          if entry[:partials].any?
-            merge_checkpoint_contexts(
-              entry[:final][:context],
-              entry[:partials].map { |p| p[:context] }
-            )
-          end
-
-          # Merge cross-batch partials from DB
-          db_partials = Catpm::Sample.where(request_id: rid)
-          if db_partials.exists?
-            merge_checkpoint_contexts(
-              entry[:final][:context],
-              db_partials.map(&:parsed_context)
-            )
-            db_partials.delete_all
-          end
-
-          # Clear request_id so persisted final sample won't be treated as orphan
-          entry[:final][:request_id] = nil
-          merged << entry[:final]
-        else
-          # Only partials, no final yet — persist as-is
-          merged.concat(entry[:partials])
-        end
-      end
-
-      regular + merged
-    end
-
-    def merge_checkpoint_contexts(final_ctx, checkpoint_ctxs)
-      final_segments = final_ctx[:segments] || final_ctx['segments']
-      return unless final_segments
-
-      final_ctrl_idx = final_segments.index { |s|
-        (s[:type] || s['type']) == 'controller'
-      }
-
-      sorted = checkpoint_ctxs.sort_by { |c|
-        c[:checkpoint_number] || c['checkpoint_number'] || 0
-      }
-
-      sorted.each do |cp_ctx|
-        cp_segments = cp_ctx[:segments] || cp_ctx['segments'] || []
-
-        old_to_new = {}
-        kept = []
-
-        cp_segments.each_with_index do |seg, i|
-          seg_type = seg[:type] || seg['type']
-          next if seg_type == 'request'
-          next if seg_type == 'controller'
-          old_to_new[i] = final_segments.size + kept.size
-          kept << seg.dup
-        end
-
-        kept.each do |seg|
-          pi_key = seg.key?(:parent_index) ? :parent_index : 'parent_index'
-          pi = seg[pi_key]
-          next unless pi
-
-          if old_to_new.key?(pi)
-            seg[pi_key] = old_to_new[pi]
-          else
-            seg[pi_key] = final_ctrl_idx || 0
-          end
-        end
-
-        final_segments.concat(kept)
-
-        # Merge summary
-        cp_summary = cp_ctx[:segment_summary] || cp_ctx['segment_summary']
-        if cp_summary
-          use_symbols = final_ctx.key?(:segment_summary)
-          summary_key = use_symbols ? :segment_summary : 'segment_summary'
-          final_summary = final_ctx[summary_key] ||= {}
-          cp_summary.each do |k, v|
-            nk = use_symbols ? k.to_sym : k.to_s
-            final_summary[nk] = (final_summary[nk] || 0) + v.to_f
-          end
-        end
-
-        # Merge capped flag
-        capped_key = final_ctx.key?(:segments_capped) ? :segments_capped : 'segments_capped'
-        cp_capped = cp_ctx[:segments_capped] || cp_ctx['segments_capped']
-        final_ctx[capped_key] = true if cp_capped
-      end
-
-      # Clean up checkpoint markers
-      final_ctx.delete(:partial)
-      final_ctx.delete('partial')
-      final_ctx.delete(:request_id)
-      final_ctx.delete('request_id')
-      final_ctx.delete(:checkpoint_number)
-      final_ctx.delete('checkpoint_number')
-    end
-
-    def cleanup_orphaned_partials
-      return unless Catpm::Sample.request_id_supported?
-
-      Catpm::Sample.where.not(request_id: nil)
-        .where(recorded_at: ..PARTIAL_STALE_TIMEOUT.seconds.ago)
-        .delete_all
-    rescue => e
-      Catpm.config.error_handler&.call(e)
-    end
-
     def cleanup_expired_data
       cutoff = Catpm.config.retention_period.ago
       batch_size = Catpm.config.cleanup_batch_size

data/lib/catpm/lifecycle.rb

@@ -8,6 +8,7 @@ module Catpm
 
         initialize_buffer
         initialize_flusher
+        load_sample_counts
         apply_patches
 
         # Start the flusher in the current process.
@@ -24,6 +25,12 @@ module Catpm
 
       private
 
+      def load_sample_counts
+        Collector.load_sample_counts_eagerly!
+      rescue => e
+        Catpm.config.error_handler&.call(e)
+      end
+
       def apply_patches
         if Catpm.config.instrument_net_http
           if defined?(::Net::HTTP)
@@ -39,7 +46,7 @@ module Catpm
       end
 
       def initialize_buffer
-        Catpm.buffer ||= Buffer.new(max_bytes: Catpm.config.max_buffer_memory)
+        Catpm.buffer ||= Buffer.new(max_bytes: Catpm.config.effective_max_buffer_memory)
       end
 
       def initialize_flusher

data/lib/catpm/middleware.rb

@@ -20,8 +20,7 @@ module Catpm
           max_segments: Catpm.config.max_segments_per_request,
           request_start: env['catpm.request_start'],
           stack_sample: use_sampler,
-          call_tree: Catpm.config.instrument_call_tree,
-          memory_limit: Catpm.config.max_request_memory
+          call_tree: Catpm.config.instrument_call_tree
         )
         env['catpm.segments'] = req_segments
         Thread.current[:catpm_request_segments] = req_segments
@@ -47,10 +46,10 @@ module Catpm
       ev = Event.new(
         kind: :http,
         target: target_from_env(env),
-        operation: env['REQUEST_METHOD'] || 'GET',
+        operation: env['REQUEST_METHOD'] || '',
         duration: elapsed_ms(env),
         started_at: Time.current,
-        status: 500,
+        status: Collector::DEFAULT_ERROR_STATUS,
         sample_type: 'error',
         error_class: exception.class.name,
         error_message: exception.message,

data/lib/catpm/patches/httpclient.rb

@@ -17,7 +17,7 @@ module Catpm
         status = response.status rescue nil
         detail = "#{http_method} #{uri.host}#{uri.path}"
         detail += " (#{status})" if status
-        source = duration >= Catpm.config.segment_source_threshold ? extract_catpm_source : nil
+        source = Catpm.segment_storable?(duration) ? extract_catpm_source : nil
 
         segments.add(
           type: :http, duration: duration, detail: detail,

data/lib/catpm/patches/net_http.rb

@@ -12,7 +12,7 @@ module Catpm
         duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000.0
 
         detail = "#{req.method} #{@address}#{req.path} (#{response.code})"
-        source = duration >= Catpm.config.segment_source_threshold ? extract_catpm_source : nil
+        source = Catpm.segment_storable?(duration) ? extract_catpm_source : nil
 
         segments.add(
           type: :http, duration: duration, detail: detail,

data/lib/catpm/request_segments.rb

@@ -9,9 +9,9 @@ module Catpm
     SEGMENT_BASE_BYTES = Event::OBJECT_OVERHEAD + (6 * Event::HASH_ENTRY_SIZE)
     SEGMENT_STRING_OVERHEAD = Event::OBJECT_OVERHEAD # per-string overhead in segment values
 
-    attr_reader :segments, :summary, :request_start, :estimated_bytes, :checkpoint_count, :request_id
+    attr_reader :segments, :summary, :request_start, :estimated_bytes, :segments_filtered
 
-    def initialize(max_segments:, request_start: nil, stack_sample: false, call_tree: false, memory_limit: nil)
+    def initialize(max_segments:, request_start: nil, stack_sample: false, call_tree: false)
       @max_segments = max_segments
       @request_start = request_start || Process.clock_gettime(Process::CLOCK_MONOTONIC)
       @segments = []
@@ -20,11 +20,8 @@ module Catpm
       @span_stack = []
       @tracked_ranges = []
       @call_tree = call_tree
-      @memory_limit = memory_limit
       @estimated_bytes = 0
-      @checkpoint_callback = nil
-      @checkpoint_count = 0
-      @request_id = memory_limit ? SecureRandom.hex(8) : nil
+      @segments_filtered = 0
 
       if stack_sample
         @sampler = StackSampler.new(target_thread: Thread.current, request_start: @request_start, call_tree: call_tree)
@@ -32,16 +29,24 @@ module Catpm
       end
     end
 
-    def on_checkpoint(&block)
-      @checkpoint_callback = block
-    end
-
     def add(type:, duration:, detail:, source: nil, started_at: nil)
       type_key = type.to_sym
       count_key, dur_key = SUMMARY_KEYS[type_key]
       @summary[count_key] += 1
       @summary[dur_key] += duration
 
+      # Record time range so sampler can skip already-tracked periods
+      if started_at && duration > 0
+        @tracked_ranges << [started_at, started_at + duration / 1000.0]
+      end
+
+      # Skip storing segment below minimum duration threshold —
+      # summary counters above are still updated for accurate Time Breakdown.
+      unless Catpm.segment_storable?(duration)
+        @segments_filtered += 1
+        return
+      end
+
       offset = started_at ? ((started_at - @request_start) * 1000.0).round(2) : nil
 
       segment = { type: type.to_s, duration: duration.round(2), detail: detail }
@@ -49,11 +54,6 @@ module Catpm
       segment[:source] = source if source
       segment[:parent_index] = @span_stack.last if @span_stack.any?
 
-      # Record time range so sampler can skip already-tracked periods
-      if started_at && duration > 0
-        @tracked_ranges << [started_at, started_at + duration / 1000.0]
-      end
-
       if @max_segments.nil? || @segments.size < @max_segments
         @segments << segment
       else
@@ -65,7 +65,6 @@ module Catpm
       end
 
       @estimated_bytes += estimate_segment_bytes(segment)
-      maybe_checkpoint
     end
 
     def push_span(type:, detail:, started_at: nil)
@@ -132,6 +131,7 @@ module Catpm
       @tracked_ranges = []
       @sampler = nil
       @estimated_bytes = 0
+      @segments_filtered = 0
     end
 
     def overflowed?
@@ -142,7 +142,8 @@ module Catpm
       {
         segments: @segments,
         segment_summary: @summary,
-        segments_capped: @overflow
+        segments_capped: @overflow,
+        segments_filtered: @segments_filtered
       }
     end
 
@@ -156,67 +157,5 @@ module Catpm
       bytes
     end
 
-    def maybe_checkpoint
-      return unless @memory_limit && @estimated_bytes > @memory_limit && @checkpoint_callback
-
-      checkpoint_data = {
-        segments: @segments,
-        summary: @summary,
-        overflow: @overflow,
-        sampler_segments: @sampler ? sampler_segments_for_checkpoint : [],
-        checkpoint_number: @checkpoint_count
-      }
-
-      @checkpoint_count += 1
-      rebuild_after_checkpoint
-      @checkpoint_callback.call(checkpoint_data)
-    end
-
-    def sampler_segments_for_checkpoint
-      if @call_tree
-        result = @sampler&.to_call_tree(tracked_ranges: @tracked_ranges) || []
-      else
-        result = @sampler&.to_segments(tracked_ranges: @tracked_ranges) || []
-      end
-      @sampler&.clear_samples!
-      result
-    end
-
-    # After checkpoint: keep only active spans from @span_stack, reset everything else.
-    def rebuild_after_checkpoint
-      if @span_stack.any?
-        # Clone active spans with corrected indices
-        new_segments = []
-        old_to_new = {}
-
-        @span_stack.each do |old_idx|
-          seg = @segments[old_idx]
-          next unless seg
-
-          new_idx = new_segments.size
-          old_to_new[old_idx] = new_idx
-          new_segments << seg.dup
-        end
-
-        # Fix parent_index references in cloned spans
-        new_segments.each do |seg|
-          if seg.key?(:parent_index) && old_to_new.key?(seg[:parent_index])
-            seg[:parent_index] = old_to_new[seg[:parent_index]]
-          else
-            seg.delete(:parent_index)
-          end
-        end
-
-        @span_stack = @span_stack.filter_map { |old_idx| old_to_new[old_idx] }
-        @segments = new_segments
-      else
-        @segments = []
-      end
-
-      @summary = Hash.new(0)
-      @tracked_ranges = []
-      @overflow = false
-      @estimated_bytes = 0
-    end
   end
 end

data/lib/catpm/segment_subscribers.rb

@@ -155,7 +155,7 @@ module Catpm
         record_count = payload[:record_count] || 0
         class_name = payload[:class_name] || 'ActiveRecord'
         detail = "#{class_name} x#{record_count}"
-        source = duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
+        source = Catpm.segment_storable?(duration) ? extract_source_location : nil
 
         # Fold into sql summary for cleaner breakdown
         req_segments.add(
@@ -176,7 +176,7 @@ module Catpm
         sql = payload[:sql].to_s
         max_len = Catpm.config.max_sql_length
         sql = sql.truncate(max_len) if max_len && sql.length > max_len
-        source = duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
+        source = Catpm.segment_storable?(duration) ? extract_source_location : nil
 
         req_segments.add(
           type: :sql, duration: duration, detail: sql,
@@ -193,7 +193,7 @@ module Catpm
         hit = event.payload[:hit]
         detail = "cache.#{operation} #{key}"
         detail += hit ? ' (hit)' : ' (miss)' if operation == 'read' && !hit.nil?
-        source = duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
+        source = Catpm.segment_storable?(duration) ? extract_source_location : nil
 
         req_segments.add(
           type: :cache, duration: duration, detail: detail,
@@ -209,7 +209,7 @@ module Catpm
         mailer = payload[:mailer].to_s
         to = Array(payload[:to]).first.to_s
         detail = to.empty? ? mailer : "#{mailer} to #{to}"
-        source = event.duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
+        source = Catpm.segment_storable?(event.duration) ? extract_source_location : nil
 
         req_segments.add(
           type: :mailer, duration: event.duration, detail: detail,
@@ -224,7 +224,7 @@ module Catpm
         payload = event.payload
         key = payload[:key].to_s
         detail = "#{operation} #{key}"
-        source = event.duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
+        source = Catpm.segment_storable?(event.duration) ? extract_source_location : nil
 
         req_segments.add(
           type: :storage, duration: event.duration, detail: detail,

data/lib/catpm/trace.rb

@@ -47,7 +47,7 @@ module Catpm
 
       req_segments = Thread.current[:catpm_request_segments]
       if req_segments
-        source = duration_ms >= config.segment_source_threshold ? extract_trace_source : nil
+        source = segment_storable?(duration_ms) ? extract_trace_source : nil
         req_segments.add(
           type: :custom, duration: duration_ms, detail: name,
           source: source, started_at: start_time
@@ -89,23 +89,10 @@ module Catpm
           max_segments: config.max_segments_per_request,
           request_start: start_time,
           stack_sample: use_sampler,
-          call_tree: config.instrument_call_tree,
-          memory_limit: config.max_request_memory
+          call_tree: config.instrument_call_tree
         )
         Thread.current[:catpm_request_segments] = req_segments
         owns_segments = true
-
-        if config.max_request_memory
-          req_segments.on_checkpoint do |checkpoint_data|
-            Collector.process_checkpoint(
-              kind: kind, target: target, operation: operation,
-              context: context, metadata: metadata,
-              checkpoint_data: checkpoint_data,
-              request_start: start_time,
-              request_id: req_segments.request_id
-            )
-          end
-        end
       end
     end
 
@@ -162,7 +149,7 @@ module Catpm
 
       req_segments = Thread.current[:catpm_request_segments]
       if req_segments
-        source = duration_ms >= Catpm.config.segment_source_threshold ? Catpm.send(:extract_trace_source) : nil
+        source = Catpm.segment_storable?(duration_ms) ? Catpm.send(:extract_trace_source) : nil
         req_segments.add(
           type: :custom, duration: duration_ms, detail: @name,
           source: source, started_at: @start_time

data/lib/catpm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Catpm
-  VERSION = '0.9.4'
+  VERSION = '0.9.6'
 end