catpm 0.2.0 → 0.4.0

data/lib/catpm/flusher.rb

@@ -123,14 +123,6 @@ module Catpm
       samples = []
       error_groups = {}
 
-      # Pre-load existing random sample counts per endpoint for filling phase
-      @random_sample_counts = {}
-      Catpm::Sample.where(sample_type: 'random')
-        .joins(:bucket)
-        .group('catpm_buckets.kind', 'catpm_buckets.target', 'catpm_buckets.operation')
-        .count
-        .each { |(kind, target, op), cnt| @random_sample_counts[[ kind, target, op ]] = cnt }
-
       events.each do |event|
         # Bucket aggregation
         key = [ event.kind, event.target, event.operation, event.bucket_start ]
@@ -165,8 +157,8 @@ module Catpm
           )
         end
 
-        # Collect samples
-        sample_type = determine_sample_type(event)
+        # Collect samples (pre-determined by collector — only these events carry full context)
+        sample_type = event.sample_type
         if sample_type
           sample_hash = {
             bucket_key: key,
@@ -174,7 +166,7 @@ module Catpm
             sample_type: sample_type,
             recorded_at: event.started_at,
             duration: event.duration,
-            context: event.context
+            context: event.context || {}
           }
           sample_hash[:error_fingerprint] = error_fp if error_fp
           samples << sample_hash
@@ -231,43 +223,51 @@ module Catpm
       }
     end
 
-    def determine_sample_type(event)
-      return 'error' if event.error?
-
-      threshold = Catpm.config.slow_threshold_for(event.kind.to_sym)
-      return 'slow' if event.duration >= threshold
 
-      # Always sample if endpoint has few random samples (filling phase)
-      endpoint_key = [ event.kind, event.target, event.operation ]
-      existing_random = @random_sample_counts[endpoint_key] || 0
-      if existing_random < Catpm.config.max_random_samples_per_endpoint
-        @random_sample_counts[endpoint_key] = existing_random + 1
-        return 'random'
+    def rotate_samples(samples)
+      return samples if samples.empty?
+
+      # Pre-fetch counts for all endpoints and types in bulk
+      endpoint_keys = samples.map { |s| s[:bucket_key][0..2] }.uniq
+      error_fps = samples.filter_map { |s| s[:error_fingerprint] }.uniq
+
+      # Build counts cache: { [kind, target, op, type] => count }
+      counts_cache = {}
+      if endpoint_keys.any?
+        Catpm::Sample.joins(:bucket)
+          .where(catpm_buckets: { kind: endpoint_keys.map(&:first), target: endpoint_keys.map { |k| k[1] }, operation: endpoint_keys.map { |k| k[2] } })
+          .where(sample_type: %w[random slow])
+          .group('catpm_buckets.kind', 'catpm_buckets.target', 'catpm_buckets.operation', 'catpm_samples.sample_type')
+          .count
+          .each { |(kind, target, op, type), cnt| counts_cache[[kind, target, op, type]] = cnt }
       end
 
-      return 'random' if rand(Catpm.config.random_sample_rate) == 0
-
-      nil
-    end
+      error_counts = {}
+      if error_fps.any?
+        Catpm::Sample.where(sample_type: 'error', error_fingerprint: error_fps)
+          .group(:error_fingerprint).count
+          .each { |fp, cnt| error_counts[fp] = cnt }
+      end
 
-    def rotate_samples(samples)
       samples.each do |sample|
-        kind, target, operation = sample[:bucket_key][0], sample[:bucket_key][1], sample[:bucket_key][2]
-        endpoint_samples = Catpm::Sample
-          .joins(:bucket)
-          .where(catpm_buckets: { kind: kind, target: target, operation: operation })
+        kind, target, operation = sample[:bucket_key][0..2]
 
         case sample[:sample_type]
         when 'random'
-          existing = endpoint_samples.where(sample_type: 'random')
-          if existing.count >= Catpm.config.max_random_samples_per_endpoint
-            existing.order(recorded_at: :asc).first.destroy
+          cache_key = [kind, target, operation, 'random']
+          if (counts_cache[cache_key] || 0) >= Catpm.config.max_random_samples_per_endpoint
+            oldest = Catpm::Sample.joins(:bucket)
+              .where(catpm_buckets: { kind: kind, target: target, operation: operation })
+              .where(sample_type: 'random').order(recorded_at: :asc).first
+            oldest&.destroy
           end
         when 'slow'
-          existing = endpoint_samples.where(sample_type: 'slow')
-          if existing.count >= Catpm.config.max_slow_samples_per_endpoint
-            weakest = existing.order(duration: :asc).first
-            if sample[:duration] > weakest.duration
+          cache_key = [kind, target, operation, 'slow']
+          if (counts_cache[cache_key] || 0) >= Catpm.config.max_slow_samples_per_endpoint
+            weakest = Catpm::Sample.joins(:bucket)
+              .where(catpm_buckets: { kind: kind, target: target, operation: operation })
+              .where(sample_type: 'slow').order(duration: :asc).first
+            if weakest && sample[:duration] > weakest.duration
               weakest.destroy
             else
               sample[:_skip] = true
@@ -275,11 +275,10 @@ module Catpm
           end
         when 'error'
           fp = sample[:error_fingerprint]
-          if fp
-            existing = Catpm::Sample.where(sample_type: 'error', error_fingerprint: fp)
-            if existing.count >= Catpm.config.max_error_samples_per_fingerprint
-              existing.order(recorded_at: :asc).first.destroy
-            end
+          if fp && (error_counts[fp] || 0) >= Catpm.config.max_error_samples_per_fingerprint
+            oldest = Catpm::Sample.where(sample_type: 'error', error_fingerprint: fp)
+              .order(recorded_at: :asc).first
+            oldest&.destroy
           end
         end
       end
@@ -288,28 +287,25 @@ module Catpm
     end
 
     def build_error_context(event)
+      event_context = event.context || {}
       ctx = {
         occurred_at: event.started_at.iso8601,
         kind: event.kind,
-        operation: event.context.slice(:method, :path, :params, :job_class, :job_id, :queue, :target, :metadata),
-        backtrace: begin
-          bt = event.backtrace || []
-          limit = Catpm.config.backtrace_lines
-          limit ? bt.first(limit) : bt
-        end,
+        operation: event_context.slice(:method, :path, :params, :job_class, :job_id, :queue, :target, :metadata),
+        backtrace: event.backtrace || [],
         duration: event.duration,
         status: event.status
       }
 
       ctx[:target] = event.target if event.target.present?
 
-      if event.context[:segments]
-        ctx[:segments] = event.context[:segments]
-        ctx[:segments_capped] = event.context[:segments_capped]
+      if event_context[:segments]
+        ctx[:segments] = event_context[:segments]
+        ctx[:segments_capped] = event_context[:segments_capped]
       end
 
-      if event.context[:segment_summary]
-        ctx[:segment_summary] = event.context[:segment_summary]
+      if event_context[:segment_summary]
+        ctx[:segment_summary] = event_context[:segment_summary]
       end
 
       ctx
@@ -402,48 +398,61 @@ module Catpm
       cutoff = age_threshold.ago
       target_seconds = target_interval.to_i
 
-      # Find all buckets older than cutoff
-      source_buckets = Catpm::Bucket.where(bucket_start: ...cutoff).to_a
-      return if source_buckets.empty?
-
-      # Group by (kind, target, operation) + target-aligned bucket_start
-      groups = source_buckets.group_by do |bucket|
-        epoch = bucket.bucket_start.to_i
-        aligned_epoch = epoch - (epoch % target_seconds)
-        aligned_start = Time.at(aligned_epoch).utc
-
-        [bucket.kind, bucket.target, bucket.operation, aligned_start]
-      end
-
-      groups.each do |(kind, target, operation, aligned_start), buckets|
-        # Skip if only one bucket already at the target alignment
-        next if buckets.size == 1 && buckets.first.bucket_start.to_i % target_seconds == 0
+      # Process in batches to avoid loading all old buckets into memory
+      Catpm::Bucket.where(bucket_start: ...cutoff)
+        .select(:id, :kind, :target, :operation, :bucket_start)
+        .group_by { |b| [b.kind, b.target, b.operation] }
+        .each do |(_kind, _target, _operation), endpoint_buckets|
+          groups = endpoint_buckets.group_by do |bucket|
+            epoch = bucket.bucket_start.to_i
+            aligned_epoch = epoch - (epoch % target_seconds)
+            Time.at(aligned_epoch).utc
+          end
 
-        merged = {
-          kind: kind,
-          target: target,
-          operation: operation,
-          bucket_start: aligned_start,
-          count: buckets.sum(&:count),
-          success_count: buckets.sum(&:success_count),
-          failure_count: buckets.sum(&:failure_count),
-          duration_sum: buckets.sum(&:duration_sum),
-          duration_max: buckets.map(&:duration_max).max,
-          duration_min: buckets.map(&:duration_min).min,
-          metadata_sum: merge_bucket_metadata(buckets, adapter),
-          p95_digest: merge_bucket_digests(buckets)
-        }
-
-        source_ids = buckets.map(&:id)
-
-        # Delete source buckets first (to avoid unique constraint conflict
-        # if one source bucket has the same bucket_start as the target)
-        Catpm::Sample.where(bucket_id: source_ids).delete_all
-        Catpm::Bucket.where(id: source_ids).delete_all
-
-        # Create the merged bucket
-        adapter.persist_buckets([merged])
-      end
+          groups.each do |aligned_start, stub_buckets|
+            next if stub_buckets.size == 1 && stub_buckets.first.bucket_start.to_i % target_seconds == 0
+
+            # Load full records only for groups that need merging
+            bucket_ids = stub_buckets.map(&:id)
+            buckets = Catpm::Bucket.where(id: bucket_ids).to_a
+
+            merged = {
+              kind: buckets.first.kind,
+              target: buckets.first.target,
+              operation: buckets.first.operation,
+              bucket_start: aligned_start,
+              count: buckets.sum(&:count),
+              success_count: buckets.sum(&:success_count),
+              failure_count: buckets.sum(&:failure_count),
+              duration_sum: buckets.sum(&:duration_sum),
+              duration_max: buckets.map(&:duration_max).max,
+              duration_min: buckets.map(&:duration_min).min,
+              metadata_sum: merge_bucket_metadata(buckets, adapter),
+              p95_digest: merge_bucket_digests(buckets)
+            }
+
+            survivor = buckets.first
+
+            # Reassign all samples to the survivor bucket
+            Catpm::Sample.where(bucket_id: bucket_ids).update_all(bucket_id: survivor.id)
+
+            # Delete non-survivor source buckets (now sample-free)
+            Catpm::Bucket.where(id: bucket_ids - [survivor.id]).delete_all
+
+            # Overwrite survivor with merged data
+            survivor.update!(
+              bucket_start: aligned_start,
+              count: merged[:count],
+              success_count: merged[:success_count],
+              failure_count: merged[:failure_count],
+              duration_sum: merged[:duration_sum],
+              duration_max: merged[:duration_max],
+              duration_min: merged[:duration_min],
+              metadata_sum: merged[:metadata_sum],
+              p95_digest: merged[:p95_digest]
+            )
+          end
+        end
     end
 
     def downsample_event_tier(target_interval:, age_threshold:, adapter:)

data/lib/catpm/segment_subscribers.rb

@@ -171,6 +171,8 @@ module Catpm
 
         duration = event.duration
         sql = payload[:sql].to_s
+        max_len = Catpm.config.max_sql_length
+        sql = sql.truncate(max_len) if max_len && sql.length > max_len
         source = duration >= Catpm.config.segment_source_threshold ? extract_source_location : nil
 
         req_segments.add(

data/lib/catpm/stack_sampler.rb

@@ -2,30 +2,74 @@
 
 module Catpm
   class StackSampler
-    SAMPLE_INTERVAL = 0.005 # 5ms
+    MS_PER_SECOND = 1000.0
+
+    # Single global thread that samples all active requests.
+    # Avoids creating a thread per request.
+    class SamplingLoop
+      def initialize
+        @mutex = Mutex.new
+        @samplers = []
+        @thread = nil
+      end
+
+      def register(sampler)
+        @mutex.synchronize do
+          @samplers << sampler
+          start_thread unless @thread&.alive?
+        end
+      end
+
+      def unregister(sampler)
+        @mutex.synchronize { @samplers.delete(sampler) }
+      end
+
+      private
+
+      def start_thread
+        @thread = Thread.new do
+          loop do
+            sleep(Catpm.config.stack_sample_interval)
+            sample_all
+          end
+        end
+        @thread.priority = -1
+      end
+
+      def sample_all
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        targets = @mutex.synchronize { @samplers.dup }
+        targets.each { |s| s.capture(now) }
+      end
+    end
+
+    @loop = SamplingLoop.new
+
+    class << self
+      attr_reader :loop
+    end
 
     def initialize(target_thread:, request_start:)
       @target = target_thread
       @request_start = request_start
       @samples = []
-      @running = false
     end
 
     def start
-      @running = true
-      @thread = Thread.new do
-        while @running
-          locs = @target.backtrace_locations
-          @samples << [Process.clock_gettime(Process::CLOCK_MONOTONIC), locs] if locs
-          sleep(SAMPLE_INTERVAL)
-        end
-      end
-      @thread.priority = -1
+      self.class.loop.register(self)
     end
 
     def stop
-      @running = false
-      @thread&.join(0.1)
+      self.class.loop.unregister(self)
+    end
+
+    # Called by SamplingLoop from the global thread
+    def capture(now)
+      max = Catpm.config.max_stack_samples_per_request
+      return if max && @samples.size >= max
+
+      locs = @target.backtrace_locations
+      @samples << [now, locs] if locs
     end
 
     # Returns array of { parent: {segment}, children: [{segment}, ...] }
@@ -76,7 +120,7 @@ module Catpm
         duration = estimate_duration(group)
         next if duration < 1.0
 
-        offset = ((group[:start_time] - @request_start) * 1000.0).round(2)
+        offset = ((group[:start_time] - @request_start) * MS_PER_SECOND).round(2)
         app_frame = group[:app_frame]
         leaf = group[:leaves].first&.last
 
@@ -158,8 +202,8 @@ module Catpm
 
       spans.filter_map do |span|
         duration = [
-          (span[:end_time] - span[:start_time]) * 1000.0,
-          span[:count] * SAMPLE_INTERVAL * 1000.0
+          (span[:end_time] - span[:start_time]) * MS_PER_SECOND,
+          span[:count] * Catpm.config.stack_sample_interval * MS_PER_SECOND
         ].max
         next if duration < 1.0
 
@@ -170,7 +214,7 @@ module Catpm
           type: classify_path(path),
           detail: build_gem_detail(frame),
           duration: duration.round(2),
-          offset: ((span[:start_time] - @request_start) * 1000.0).round(2),
+          offset: ((span[:start_time] - @request_start) * MS_PER_SECOND).round(2),
           started_at: span[:start_time]
         }
       end
@@ -178,8 +222,8 @@ module Catpm
 
     def estimate_duration(group)
       [
-        (group[:end_time] - group[:start_time]) * 1000.0,
-        group[:count] * SAMPLE_INTERVAL * 1000.0
+        (group[:end_time] - group[:start_time]) * MS_PER_SECOND,
+        group[:count] * Catpm.config.stack_sample_interval * MS_PER_SECOND
       ].max
     end
 

data/lib/catpm/tdigest.rb

@@ -12,6 +12,7 @@ module Catpm
     Centroid = Struct.new(:mean, :weight)
 
     COMPRESSION = 100 # Controls accuracy vs. memory trade-off
+    BUFFER_FLUSH_FACTOR = 2 # Lower = more frequent flushes (better accuracy), higher = fewer flushes (better performance)
 
     attr_reader :count
 
@@ -22,7 +23,7 @@ module Catpm
       @min = Float::INFINITY
       @max = -Float::INFINITY
       @buffer = []
-      @buffer_limit = @compression * 5
+      @buffer_limit = @compression * BUFFER_FLUSH_FACTOR
     end
 
     def add(value, weight = 1)

data/lib/catpm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Catpm
-  VERSION = '0.2.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: catpm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - ''