catpm 0.1.0

data/lib/catpm/tdigest.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+module Catpm
+  # Minimal TDigest implementation for percentile estimation.
+  # Based on the merging digest variant of the t-digest algorithm
+  # by Ted Dunning (https://github.com/tdunning/t-digest).
+  #
+  # Provides accurate percentile estimates (p50, p95, p99) using
+  # fixed memory (~1-3 KB serialized). Supports lossless merging
+  # of two digests.
+  class TDigest
+    Centroid = Struct.new(:mean, :weight)
+
+    COMPRESSION = 100 # Controls accuracy vs. memory trade-off
+
+    attr_reader :count
+
+    def initialize(compression: COMPRESSION)
+      @compression = compression
+      @centroids = []
+      @count = 0
+      @min = Float::INFINITY
+      @max = -Float::INFINITY
+      @buffer = []
+      @buffer_limit = @compression * 5
+    end
+
+    def add(value, weight = 1)
+      value = value.to_f
+      @buffer << Centroid.new(value, weight)
+      @count += weight
+      @min = value if value < @min
+      @max = value if value > @max
+      flush_buffer if @buffer.size >= @buffer_limit
+      self
+    end
+
+    def percentile(p)
+      raise ArgumentError, 'percentile must be between 0 and 1' unless (0..1).cover?(p)
+      return nil if @count == 0
+
+      flush_buffer unless @buffer.empty?
+      return @centroids.first.mean if @centroids.size == 1
+
+      target = p * @count
+      cumulative = 0.0
+
+      @centroids.each_with_index do |centroid, i|
+        lower = cumulative
+        upper = cumulative + centroid.weight
+
+        if target < lower + centroid.weight / 2.0
+          if i == 0
+            return interpolate(@min, centroid.mean, target / (centroid.weight / 2.0))
+          else
+            prev = @centroids[i - 1]
+            prev_upper = lower
+            prev_mid = prev_upper - prev.weight / 2.0
+            curr_mid = lower + centroid.weight / 2.0
+            return interpolate(prev.mean, centroid.mean,
+              (target - prev_mid) / (curr_mid - prev_mid))
+          end
+        end
+
+        cumulative = upper
+      end
+
+      @centroids.last.mean
+    end
+
+    def merge(other)
+      return self if other.nil? || other.count == 0
+
+      other.send(:flush_buffer) unless other.send(:buffer).empty?
+      other.send(:centroids).each do |c|
+        add(c.mean, c.weight)
+      end
+      self
+    end
+
+    def empty?
+      @count == 0
+    end
+
+    # Binary serialization: [compression(f64), count(u64), n_centroids(u32), min(f64), max(f64), (mean(f64), weight(u32))...]
+    def serialize
+      flush_buffer unless @buffer.empty?
+
+      parts = []
+      parts << [@compression].pack('E')      # f64 little-endian
+      parts << [@count].pack('Q<')            # u64 little-endian
+      parts << [@centroids.size].pack('V')    # u32 little-endian
+      parts << [@min].pack('E')               # f64
+      parts << [@max].pack('E')               # f64
+
+      @centroids.each do |c|
+        parts << [c.mean].pack('E')           # f64
+        parts << [c.weight.to_i].pack('V')    # u32
+      end
+
+      parts.join.b
+    end
+
+    def self.deserialize(blob)
+      return new if blob.nil? || blob.empty?
+
+      blob = blob.b
+      offset = 0
+
+      compression = blob[offset, 8].unpack1('E'); offset += 8
+      count = blob[offset, 8].unpack1('Q<');       offset += 8
+      n = blob[offset, 4].unpack1('V');            offset += 4
+      min = blob[offset, 8].unpack1('E');          offset += 8
+      max = blob[offset, 8].unpack1('E');          offset += 8
+
+      digest = new(compression: compression.to_i)
+      digest.instance_variable_set(:@count, count)
+      digest.instance_variable_set(:@min, min)
+      digest.instance_variable_set(:@max, max)
+
+      centroids = []
+      n.times do
+        mean = blob[offset, 8].unpack1('E');   offset += 8
+        weight = blob[offset, 4].unpack1('V');  offset += 4
+        centroids << Centroid.new(mean, weight)
+      end
+      digest.instance_variable_set(:@centroids, centroids)
+
+      digest
+    end
+
+    private
+
+    attr_reader :centroids, :buffer
+
+    def flush_buffer
+      return if @buffer.empty?
+
+      all = @centroids + @buffer
+      @buffer = []
+      all.sort_by!(&:mean)
+
+      merged = []
+      weight_so_far = 0
+
+      all.each do |centroid|
+        if merged.empty?
+          merged << Centroid.new(centroid.mean, centroid.weight)
+        else
+          last = merged.last
+          q = (weight_so_far + last.weight / 2.0) / @count
+          limit = 4.0 * @count * q * (1 - q) / @compression
+
+          if last.weight + centroid.weight <= limit
+            # Merge into existing centroid
+            new_weight = last.weight + centroid.weight
+            last.mean = (last.mean * last.weight + centroid.mean * centroid.weight) / new_weight
+            last.weight = new_weight
+          else
+            weight_so_far += last.weight
+            merged << Centroid.new(centroid.mean, centroid.weight)
+          end
+        end
+      end
+
+      @centroids = merged
+    end
+
+    def interpolate(a, b, fraction)
+      fraction = fraction.clamp(0.0, 1.0)
+      a + (b - a) * fraction
+    end
+  end
+end

data/lib/catpm/trace.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Catpm
+  def self.span(name, type: :custom, &block)
+    unless enabled?
+      return block.call if block
+      return nil
+    end
+
+    req_segments = Thread.current[:catpm_request_segments]
+    unless req_segments
+      return trace(name, &block)
+    end
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    index = req_segments.push_span(type: type, detail: name, started_at: start_time)
+
+    begin
+      block.call
+    ensure
+      req_segments.pop_span(index)
+    end
+  end
+
+  def self.trace(name, metadata: {}, context: {}, &block)
+    unless enabled?
+      return block.call if block
+      return nil
+    end
+
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    error = nil
+
+    begin
+      result = block.call
+    rescue => e
+      error = e
+      raise
+    ensure
+      duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+
+      req_segments = Thread.current[:catpm_request_segments]
+      if req_segments
+        source = duration_ms >= config.segment_source_threshold ? extract_trace_source : nil
+        req_segments.add(
+          type: :custom, duration: duration_ms, detail: name,
+          source: source, started_at: start_time
+        )
+      elsif buffer
+        Collector.process_custom(
+          name: name, duration: duration_ms,
+          metadata: metadata, error: error, context: context
+        )
+      end
+    end
+
+    result
+  end
+
+  def self.start_trace(name, metadata: {}, context: {})
+    Span.new(name: name, metadata: metadata, context: context)
+  end
+
+  # Instrument a block as a full request — creates a controller span,
+  # collects all segments (SQL, cache, etc.), and pushes a complete Event.
+  # Use this for non-ActionController contexts (webhooks, custom endpoints).
+  #
+  #   Catpm.track_request(kind: :custom, target: "WebhookController#message") do
+  #     process_update(...)
+  #   end
+  #
+  def self.track_request(kind: :http, target:, operation: '', context: {}, metadata: {})
+    return yield unless enabled?
+
+    req_segments = Thread.current[:catpm_request_segments]
+    owns_segments = false
+
+    if req_segments.nil? && config.instrument_segments
+      req_segments = RequestSegments.new(
+        max_segments: config.max_segments_per_request,
+        request_start: Process.clock_gettime(Process::CLOCK_MONOTONIC),
+        stack_sample: config.instrument_stack_sampler
+      )
+      Thread.current[:catpm_request_segments] = req_segments
+      owns_segments = true
+    end
+
+    if req_segments
+      started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      ctrl_idx = req_segments.push_span(type: :controller, detail: target, started_at: started_at)
+    end
+
+    error = nil
+    start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    begin
+      yield
+    rescue => e
+      error = e
+      raise
+    ensure
+      duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0
+      req_segments&.pop_span(ctrl_idx) if ctrl_idx
+      req_segments&.stop_sampler
+
+      Collector.process_tracked(
+        kind: kind, target: target, operation: operation,
+        duration: duration, context: context, metadata: metadata,
+        error: error, req_segments: req_segments
+      )
+
+      if owns_segments
+        Thread.current[:catpm_request_segments] = nil
+      end
+    end
+  end
+
+  class Span
+    def initialize(name:, metadata: {}, context: {})
+      @name = name
+      @metadata = metadata
+      @context = context
+      @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @finished = false
+    end
+
+    def finish(error: nil)
+      return if @finished
+
+      @finished = true
+      duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time) * 1000.0
+
+      req_segments = Thread.current[:catpm_request_segments]
+      if req_segments
+        source = duration_ms >= Catpm.config.segment_source_threshold ? Catpm.send(:extract_trace_source) : nil
+        req_segments.add(
+          type: :custom, duration: duration_ms, detail: @name,
+          source: source, started_at: @start_time
+        )
+      elsif Catpm.enabled? && Catpm.buffer
+        Collector.process_custom(
+          name: @name, duration: duration_ms,
+          metadata: @metadata, error: error, context: @context
+        )
+      end
+    end
+
+    def finished?
+      @finished
+    end
+  end
+
+  private
+
+  def self.extract_trace_source
+    locations = caller_locations(3, 50)
+    locations&.each do |loc|
+      path = loc.path.to_s
+      if Fingerprint.app_frame?(path)
+        return "#{path}:#{loc.lineno}"
+      end
+    end
+    nil
+  end
+end

data/lib/catpm/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Catpm
+  VERSION = '0.1.0'
+end

data/lib/catpm.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'catpm/version'
+require 'catpm/configuration'
+require 'catpm/event'
+require 'catpm/custom_event'
+require 'catpm/tdigest'
+require 'catpm/errors'
+require 'catpm/buffer'
+require 'catpm/circuit_breaker'
+require 'catpm/adapter'
+require 'catpm/fingerprint'
+require 'catpm/stack_sampler'
+require 'catpm/request_segments'
+require 'catpm/flusher'
+require 'catpm/collector'
+require 'catpm/middleware'
+require 'catpm/middleware_probe'
+require 'catpm/subscribers'
+require 'catpm/segment_subscribers'
+require 'catpm/lifecycle'
+require 'catpm/trace'
+require 'catpm/span_helpers'
+require 'catpm/auto_instrument'
+require 'catpm/engine'
+
+module Catpm
+  class << self
+    def configure
+      yield(config)
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def reset_config!
+      @config = Configuration.new
+      @buffer = nil
+      @flusher = nil
+    end
+
+    def enabled?
+      config.enabled
+    end
+
+    attr_writer :buffer, :flusher
+
+    attr_reader :buffer
+
+    attr_reader :flusher
+
+    def stats
+      @stats ||= { dropped_events: 0, circuit_opens: 0, flushes: 0 }
+    end
+
+    def reset_stats!
+      @stats = { dropped_events: 0, circuit_opens: 0, flushes: 0 }
+    end
+
+    def event(name, **payload)
+      return unless enabled? && config.events_enabled
+      buffer&.push(CustomEvent.new(name: name, payload: payload))
+    end
+  end
+end

data/lib/generators/catpm/install_generator.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Catpm
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Install catpm: copy migrations, create initializer, mount engine'
+
+      def copy_migrations
+        rake 'catpm:install:migrations'
+      end
+
+      def create_initializer
+        template 'initializer.rb.tt', 'config/initializers/catpm.rb'
+      end
+
+      def mount_engine
+        route 'mount Catpm::Engine => "/catpm"'
+      end
+
+      def show_post_install
+        say ''
+        say 'catpm installed successfully!', :green
+        say ''
+        say 'Next steps:'
+        say '  1. Run: rails db:migrate'
+        say '  2. Review config/initializers/catpm.rb'
+        say '  3. Visit /catpm in your browser'
+        say ''
+      end
+    end
+  end
+end

data/lib/generators/catpm/templates/initializer.rb.tt

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+Catpm.configure do |config|
+  # === Security ===
+  # Protect the dashboard with HTTP Basic Auth or a custom policy.
+  # If neither is configured, the dashboard is accessible to everyone.
+  #
+  # config.http_basic_auth_user = ENV["CATPM_USER"]
+  # config.http_basic_auth_password = ENV["CATPM_PASSWORD"]
+  # config.access_policy = ->(request) { request.env["warden"].user&.admin? }
+
+  # === PII Filtering ===
+  # Rails' filter_parameters are inherited automatically.
+  # Add extra patterns here:
+  #
+  # config.additional_filter_parameters = [:card_number, :ssn]
+
+  # === Instrumentation ===
+  #
+  # config.instrument_http = true              # Track HTTP requests (default: true)
+  # config.instrument_jobs = false             # Track ActiveJob (default: false)
+  # config.instrument_segments = true          # Track SQL/view/cache segments (default: true)
+  # config.instrument_net_http = false         # Patch Net::HTTP for outbound tracking (default: false)
+  # config.instrument_middleware_stack = false # Decompose middleware into per-middleware segments (default: false)
+  # config.max_segments_per_request = 50       # Cap segments per request (keeps slowest)
+  # config.segment_source_threshold = 0.0      # ms — capture caller_locations above this (raise to reduce overhead)
+  # config.max_sql_length = 200                # Truncate SQL queries to this length
+  # config.slow_threshold = 500                # ms — threshold for slow samples (all kinds)
+  # config.slow_threshold_per_kind = {         # Override per kind
+  #   http: 500,
+  #   job: 5_000,
+  #   custom: 1_000
+  # }
+  # config.ignored_targets = [                 # Patterns to exclude from tracking
+  #   "HealthcheckController#index",
+  #   "/assets/*",
+  # ]
+
+  # === Sampling ===
+  #
+  # config.random_sample_rate = 20                   # 1 in N requests sampled randomly
+  # config.max_random_samples_per_endpoint = 5       # Random samples kept per endpoint
+  # config.max_slow_samples_per_endpoint = 5         # Slow samples kept per endpoint
+
+  # === Tuning ===
+  #
+  # config.retention_period = nil                    # nil = keep forever (default); set to e.g. 1.year to delete older data
+  # config.max_buffer_memory = 32.megabytes          # Maximum in-memory buffer size
+  # config.flush_interval = 30                       # seconds
+  # config.flush_jitter = 5                          # +/- seconds random offset per cycle
+  # config.max_error_contexts = 5                    # Contexts kept per error fingerprint
+  # config.cleanup_interval = 1.hour                 # How often to run downsampling
+  # config.persistence_batch_size = 100              # Records per DB batch operation
+
+  # === Downsampling ===
+  # Data is kept forever with progressively coarser resolution:
+  # - Last hour: 1-minute buckets
+  # - 1h–24h: 5-minute buckets
+  # - 1d–1w: 1-hour buckets
+  # - 1w–3mo: 1-day buckets
+  # - Older than 3 months: 1-week buckets
+  #
+  # config.bucket_sizes = { recent: 1.minute, medium: 5.minutes, hourly: 1.hour, daily: 1.day, weekly: 1.week }
+
+  # === Advanced ===
+  #
+  # config.circuit_breaker_failure_threshold = 5     # Failures before circuit opens
+  # config.circuit_breaker_recovery_timeout = 60     # seconds before retry
+  # config.sqlite_busy_timeout = 5_000               # ms — SQLite lock wait
+  # config.backtrace_lines = 10                      # Backtrace lines in error contexts
+  # config.shutdown_timeout = 5                      # seconds — graceful shutdown wait
+  # config.error_handler = ->(e) { Rails.logger.error("[catpm] #{e.message}") }
+
+  # === Enable/Disable ===
+  #
+  # config.enabled = Rails.env.production? || Rails.env.staging?
+end

data/lib/tasks/catpm_seed.rake

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+namespace :catpm do
+  desc 'Seed realistic custom event data (7 days, 9 event types)'
+  task seed_events: :environment do
+    EVENT_TYPES = {
+      'allowed'    => { weight: 35, payload: -> { { user_id: rand(1..5000), chat_id: rand(1..2000) } } },
+      'gpt'        => { weight: 15, payload: -> { { model: %w[gpt-4o gpt-4o-mini].sample, tokens: rand(50..2000), user_id: rand(1..5000) } } },
+      'cache'      => { weight: 15, payload: -> { { hit: [true, false].sample, key: "msg:#{rand(1..10000)}" } } },
+      'spam'       => { weight: 10, payload: -> { { user_id: rand(1..5000), score: rand(0.7..1.0).round(2), action: 'blocked' } } },
+      'moderation' => { weight: 10, payload: -> { { user_id: rand(1..5000), category: %w[hate violence self-harm].sample, flagged: [true, false].sample } } },
+      'ratelimit'  => { weight: 5,  payload: -> { { user_id: rand(1..5000), limit: %w[messages commands media].sample } } },
+      'command'    => { weight: 4,  payload: -> { { command: %w[/start /help /settings /stats /premium].sample, user_id: rand(1..5000) } } },
+      'payment'    => { weight: 3,  payload: -> { { amount: [299, 499, 999].sample, currency: 'USD', user_id: rand(1..5000) } } },
+      'error'      => { weight: 3,  payload: -> { { error_class: %w[Timeout::Error Net::ReadTimeout Redis::CannotConnectError].sample, context: 'background' } } }
+    }.freeze
+
+    # Build weighted pool
+    pool = EVENT_TYPES.flat_map { |name, cfg| [name] * cfg[:weight] }
+
+    days = 7
+    start_time = days.days.ago.beginning_of_hour
+    end_time = Time.current
+
+    puts "Seeding event buckets from #{start_time} to #{end_time}..."
+
+    # Generate 1-minute buckets
+    bucket_records = []
+    current = start_time
+
+    while current < end_time
+      hour = current.hour
+      # Lower activity at night (2am-7am)
+      base_rate = (hour >= 2 && hour < 7) ? rand(30..80) : rand(200..500)
+
+      # Distribute events across types for this minute
+      counts = Hash.new(0)
+      base_rate.times { counts[pool.sample] += 1 }
+
+      counts.each do |name, count|
+        bucket_records << {
+          name: name,
+          bucket_start: current,
+          count: count
+        }
+      end
+
+      current += 1.minute
+
+      # Batch insert every hour of data
+      if bucket_records.size >= 500
+        Catpm::EventBucket.insert_all(bucket_records)
+        bucket_records.clear
+        print '.'
+      end
+    end
+
+    Catpm::EventBucket.insert_all(bucket_records) if bucket_records.any?
+    puts "\nInserted #{Catpm::EventBucket.count} event buckets."
+
+    # Generate samples (20 per event type)
+    puts 'Seeding event samples...'
+    sample_records = []
+    EVENT_TYPES.each do |name, cfg|
+      20.times do
+        recorded_at = start_time + rand((end_time - start_time).to_i).seconds
+        sample_records << {
+          name: name,
+          payload: cfg[:payload].call.to_json,
+          recorded_at: recorded_at
+        }
+      end
+    end
+
+    Catpm::EventSample.insert_all(sample_records) if sample_records.any?
+    puts "Inserted #{Catpm::EventSample.count} event samples."
+    puts 'Done!'
+  end
+end

data/lib/tasks/catpm_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# desc "Explaining what the task does"
+# task :catpm do
+#   # Task goes here
+# end