heapscope 0.6.0

data/lib/heapscope/collector.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module HeapScope
+  # Collects heap snapshots at configurable fidelity with pause/object budgets.
+  class Collector
+    RUNTIME_NOISE = %w[
+      Class Module Object BasicObject String Array Hash Integer Float Symbol
+      Proc Method UnboundMethod Binding Encoding Range Regexp MatchData
+      Thread Fiber Mutex Rational Complex NilClass TrueClass FalseClass
+      RubyVM RubyVM::InstructionSequence IO File Dir Time Random
+    ].freeze
+
+    def initialize(runtime: Runtime.current, config: HeapScope.config)
+      @runtime = runtime
+      @config = config
+    end
+
+    def capture(mode: nil, metadata: {}, max_objects: nil, max_pause: nil, sample_rate: nil)
+      mode = @config.effective_snapshot_mode(mode)
+      max_objects ||= @config.max_objects
+      sample_rate ||= @config.object_sample_rate
+      max_pause_ms = normalize_pause(max_pause || @config.max_pause_ms)
+
+      started = monotonic_ms
+      limitations = []
+
+      gc_stat = @runtime.gc_stat
+      rss = safe_rss
+      limitations << "rss_tracking_unavailable" if rss.nil? && !@runtime.rss_tracking?
+
+      class_stats, allocation_sites, object_sample, truncated, more_limits =
+        collect_objects(mode, max_objects, sample_rate, max_pause_ms, started)
+      limitations.concat(more_limits)
+      limitations << "analysis_truncated_pause_budget" if truncated && max_pause_ms
+
+      duration = monotonic_ms - started
+
+      Snapshot.new(
+        id: SecureRandom.hex(8),
+        timestamp: Time.now.utc,
+        mode: mode,
+        gc_stat: gc_stat,
+        rss_bytes: rss,
+        class_stats: class_stats,
+        allocation_sites: allocation_sites,
+        metadata: metadata,
+        limitations: limitations.uniq,
+        sampled: sample_rate < 1.0 || mode != :lightweight,
+        process: @runtime.process_metadata,
+        duration_ms: duration.round(2),
+        object_sample: object_sample
+      )
+    rescue CapabilityError
+      raise
+    rescue StandardError => e
+      raise SnapshotError, "Failed to capture snapshot: #{e.class}: #{e.message}"
+    end
+
+    private
+
+    def collect_objects(mode, max_objects, sample_rate, max_pause_ms, started)
+      return [{}, {}, [], false, ["each_object_unavailable"]] unless @runtime.each_object?
+
+      counts = Hash.new { |h, k| h[k] = { name: k, count: 0, shallow_bytes: 0 } }
+      allocation_sites = Hash.new { |h, k| h[k] = { site: k, count: 0, shallow_bytes: 0, classes: Hash.new(0) } }
+      object_sample = []
+      seen = 0
+      truncated = false
+      limitations = []
+
+      estimate_memsize = mode != :lightweight && @runtime.memsize?
+      track_alloc = mode != :lightweight && (@config.track_allocations || mode == :deep) && @runtime.allocation_tracing?
+      sample_refs = mode == :deep
+
+      limitations << "memsize_unavailable" unless @runtime.memsize?
+      limitations << "allocation_tracing_not_enabled" if mode != :lightweight && !track_alloc
+
+      @runtime.each_object do |object|
+        break truncated = true if seen >= max_objects
+        break truncated = true if max_pause_ms && (monotonic_ms - started) > max_pause_ms
+
+        seen += 1
+        next if sample_rate < 1.0 && rand > sample_rate
+
+        name = safe_class_name(object)
+        next if name.nil?
+
+        counts[name][:count] += 1
+        bytes = estimate_memsize ? (@runtime.memsize_of(object) || 0) : 0
+        counts[name][:shallow_bytes] += bytes
+
+        if track_alloc
+          info = @runtime.allocation_info(object)
+          if info && info[:file]
+            site = "#{info[:file]}:#{info[:line]}"
+            allocation_sites[site][:count] += 1
+            allocation_sites[site][:shallow_bytes] += bytes
+            allocation_sites[site][:classes][name] += 1
+            allocation_sites[site][:method_id] ||= info[:method_id]&.to_s
+          end
+        end
+
+        next unless sample_refs && object_sample.size < 50 && !noise?(name)
+
+        object_sample << {
+          local_id: object_sample.size,
+          class: name,
+          shallow_bytes: bytes,
+          allocation: track_alloc ? @runtime.allocation_info(object) : nil
+        }.compact
+      end
+
+      limitations << "object_budget_reached" if truncated && seen >= max_objects
+
+      class_stats = counts.transform_values do |stats|
+        avg = stats[:count].positive? ? (stats[:shallow_bytes].to_f / stats[:count]) : 0.0
+        stats.merge(average_bytes: avg.round(2))
+      end
+
+      # Scale sampled counts when sample_rate < 1
+      if sample_rate < 1.0 && sample_rate.positive?
+        factor = 1.0 / sample_rate
+        class_stats = class_stats.transform_values do |stats|
+          stats.merge(
+            count: (stats[:count] * factor).round,
+            shallow_bytes: (stats[:shallow_bytes] * factor).round,
+            sampled: true,
+            sample_rate: sample_rate
+          )
+        end
+        limitations << "object_counts_extrapolated_from_sample"
+      end
+
+      [class_stats, allocation_sites, object_sample, truncated, limitations]
+    end
+
+    def safe_class_name(object)
+      object.class.name || "#<Anonymous:#{object.class.object_id}>"
+    rescue StandardError
+      nil
+    end
+
+    def noise?(name)
+      RUNTIME_NOISE.include?(name) || name.start_with?("RubyVM")
+    end
+
+    def safe_rss
+      @runtime.rss_bytes
+    rescue StandardError
+      nil
+    end
+
+    def normalize_pause(value)
+      return nil if value.nil?
+      return value if value.is_a?(Numeric)
+      return value.to_f if value.respond_to?(:to_f)
+
+      nil
+    end
+
+    def monotonic_ms
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+    end
+  end
+end

data/lib/heapscope/config.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module HeapScope
+  class Config
+    MODES = %i[lightweight standard deep production_safe development].freeze
+    SNAPSHOT_MODES = %i[lightweight standard deep].freeze
+
+    attr_accessor :mode,
+                  :track_allocations,
+                  :max_graph_depth,
+                  :max_objects,
+                  :max_edges,
+                  :max_pause_ms,
+                  :object_sample_rate,
+                  :force_gc_default,
+                  :include_runtime_objects,
+                  :ignore_patterns,
+                  :ignored_classes,
+                  :inspect_values,
+                  :verbose,
+                  :debug,
+                  :rails,
+                  :redactor,
+                  :logger,
+                  :warmup_done,
+                  :min_retention_cycles,
+                  :detect_globals,
+                  :detect_closures,
+                  :detect_fibers,
+                  :detect_thread_locals,
+                  :apply_noise_defaults,
+                  :quiet
+
+    def initialize
+      @mode = :standard
+      @track_allocations = false
+      @max_graph_depth = 6
+      @max_objects = 100_000
+      @max_edges = 500_000
+      @max_pause_ms = nil
+      @object_sample_rate = 1.0
+      @force_gc_default = false
+      @include_runtime_objects = false
+      @ignore_patterns = [/^HeapScope::/]
+      @ignored_classes = []
+      @inspect_values = false
+      @verbose = false
+      @debug = false
+      @rails = false
+      @redactor = nil
+      @logger = nil
+      @warmup_done = false
+      @min_retention_cycles = 3
+      @detect_globals = true
+      @detect_closures = false
+      @detect_fibers = true
+      @detect_thread_locals = true
+      @apply_noise_defaults = true
+      @quiet = false
+    end
+
+    def production_safe?
+      mode == :production_safe
+    end
+
+    def development?
+      mode == :development
+    end
+
+    def effective_snapshot_mode(requested = nil)
+      requested ||= snapshot_mode_for_config
+      return :lightweight if production_safe? && requested == :deep
+
+      requested
+    end
+
+    def snapshot_mode_for_config
+      case mode
+      when :lightweight, :production_safe then :lightweight
+      when :deep, :development then :deep
+      else :standard
+      end
+    end
+
+    def ignore_class?(name)
+      return true if ignored_classes.map(&:to_s).include?(name.to_s)
+
+      ignore_patterns.any? { |pattern| name.to_s.match?(pattern) }
+    end
+
+    def redact(object)
+      return nil unless inspect_values
+      return redactor.call(object) if redactor
+
+      :"[redacted]"
+    end
+
+    def log(level, message)
+      return unless logger || verbose || debug
+      return if level == :debug && !debug
+
+      (logger || $stderr).puts("[HeapScope] #{message}")
+    end
+  end
+
+  # Load Ruby or YAML configuration files.
+  module ConfigLoader
+    STARTER_YAML = <<~YAML
+      # HeapScope local config — no telemetry, no uploads
+      heapscope:
+        mode: standard
+        track_allocations: false
+        apply_noise_defaults: true
+        detect_globals: true
+        detect_fibers: true
+        detect_closures: false
+        detect_thread_locals: true
+        retention:
+          min_cycles: 3
+        ignore_patterns:
+          - "^Zeitwerk"
+          - "^Bootsnap"
+          - "^HeapScope::"
+    YAML
+
+    module_function
+
+    def load!(path)
+      raise ConfigurationError, "Config not found: #{path}" unless File.exist?(path)
+
+      case File.extname(path)
+      when ".yml", ".yaml"
+        load_yaml!(path)
+      when ".rb"
+        load(path)
+      else
+        raise ConfigurationError, "Unsupported config type: #{path}"
+      end
+      HeapScope.config
+    end
+
+    def load_yaml!(path)
+      require "yaml"
+      data = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: true) || {}
+      data = data["heapscope"] || data[:heapscope] || data
+      HeapScope.configure do |c|
+        c.mode = data["mode"]&.to_sym if data["mode"]
+        c.track_allocations = data["track_allocations"] unless data["track_allocations"].nil?
+        c.max_graph_depth = data["max_graph_depth"] if data["max_graph_depth"]
+        c.max_objects = data["max_objects"] if data["max_objects"]
+        c.object_sample_rate = data["object_sample_rate"] if data["object_sample_rate"]
+        Array(data["ignore_patterns"]).each { |p| c.ignore_patterns << Regexp.new(p) }
+        c.min_retention_cycles = data.dig("retention", "min_cycles") if data.dig("retention", "min_cycles")
+        c.detect_globals = data["detect_globals"] unless data["detect_globals"].nil?
+        c.detect_closures = data["detect_closures"] unless data["detect_closures"].nil?
+        c.detect_fibers = data["detect_fibers"] unless data["detect_fibers"].nil?
+        c.apply_noise_defaults = data["apply_noise_defaults"] unless data["apply_noise_defaults"].nil?
+      end
+    end
+
+    def write_starter!(path = "heapscope.yml", force: false)
+      raise ConfigurationError, "Refusing to overwrite existing #{path}" if File.exist?(path) && !force
+
+      File.write(path, STARTER_YAML)
+      path
+    end
+  end
+
+  class << self
+    def config
+      @config ||= Config.new
+    end
+
+    def configure
+      yield config
+      self
+    end
+
+    def reset_config!
+      @config = Config.new
+    end
+
+    def ignore_class(klass)
+      config.ignored_classes << klass
+    end
+
+    def after_warmup
+      yield if block_given?
+      config.warmup_done = true
+    end
+
+    def load_config!(path)
+      ConfigLoader.load!(path)
+    end
+  end
+end

data/lib/heapscope/detectors.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module HeapScope
+  # Pattern detectors for cache-vs-leak, unbounded collections, and thread-locals.
+  # Findings remain evidence-based: facts, derived behavior, hypothesis, suspected cause.
+  module Detectors
+    module_function
+
+    def analyze_thread_locals(runtime: Runtime.current)
+      findings = []
+      inventory = Graph.new(runtime: runtime).thread_local_inventory
+      inventory.each do |thread|
+        thread[:keys].each do |key, meta|
+          bytes = meta[:shallow_bytes].to_i
+          next if bytes < 64_000 && !suspicious_key?(key)
+
+          findings << Finding.new(
+            code: "HS003",
+            severity: bytes >= 1_000_000 ? :high : :medium,
+            subject: "#{thread[:name]}#{key}",
+            facts: [
+              "Observed fact: thread #{thread[:name]} holds key #{key} of class #{meta[:class]}."
+            ],
+            derived: [
+              "Derived: shallow size estimate #{format_bytes(bytes)}."
+            ],
+            hypothesis: "Thread-local state may retain request/job context across reuse of this thread.",
+            suspected_cause: "Thread-local #{key}",
+            suggestions: [
+              "Clear the key in an ensure block after each request/job",
+              "Avoid storing full request graphs on long-lived Puma/Sidekiq threads",
+              "Confirm whether this cache is intentional and bounded"
+            ],
+            evidence: [{ thread: thread[:name], key: key, meta: meta }]
+          )
+        end
+      end
+      { inventory: inventory, findings: findings }
+    end
+
+    def classify_collection_growth(name:, sizes:, owner: nil)
+      return { kind: :insufficient_data } if sizes.size < 3
+
+      trend = Growth.analyze(sizes)
+      max_size = sizes.max
+      min_size = sizes.min
+      last = sizes.last(3)
+      plateau = last.max - last.min <= [last.max * 0.05, 2].max
+      shrinking = sizes.each_cons(2).any? { |a, b| b < a }
+
+      kind =
+        if plateau && max_size > 0 && sizes.first < max_size
+          :likely_cache
+        elsif %i[monotonic_growth linear_growth exponential_like].include?(trend[:pattern]) && !shrinking
+          :unbounded_candidate
+        elsif trend[:pattern] == :sawtooth
+          :recovering_collection
+        else
+          :stable_or_noisy
+        end
+
+      {
+        name: name,
+        owner: owner,
+        kind: kind,
+        trend: trend,
+        sizes: sizes,
+        max: max_size,
+        min: min_size
+      }
+    end
+
+    def collection_findings(class_series)
+      findings = []
+      class_series.each_value do |entry|
+        next unless %w[Array Hash Set].include?(entry[:name]) || entry[:name].end_with?("Registry", "Cache", "Store")
+
+        classification = classify_collection_growth(name: entry[:name], sizes: entry[:series])
+        case classification[:kind]
+        when :unbounded_candidate
+          findings << Finding.new(
+            code: "HS004",
+            severity: entry[:delta] > 200 ? :high : :medium,
+            subject: entry[:name],
+            facts: ["Observed fact: #{entry[:name]} sizes #{entry[:series].inspect}."],
+            derived: [
+              "Derived: pattern=#{classification[:trend][:pattern]}, no shrinkage observed across samples."
+            ],
+            hypothesis: "This structure is an unbounded collection candidate.",
+            suggestions: [
+              "Identify the owner/root retaining the collection",
+              "Cap growth or add eviction",
+              "Verify appends are not accidental per-request registrations"
+            ],
+            evidence: [classification]
+          )
+        when :likely_cache
+          findings << Finding.new(
+            code: "HS001",
+            severity: :low,
+            subject: entry[:name],
+            title: "Likely cache plateau",
+            facts: ["Observed fact: #{entry[:name]} reached a plateau near #{classification[:max]}."],
+            derived: ["Derived: growth appears bounded (cache-like)."],
+            hypothesis: "Intentional cache behavior is more likely than a leak.",
+            suggestions: ["Confirm max size / LRU / TTL expectations"],
+            evidence: [classification]
+          )
+        end
+      end
+      findings
+    end
+
+    def suspicious_key?(key)
+      key.to_s.match?(/context|request|session|payload|current|job|controller|env/i)
+    end
+
+    def format_bytes(bytes)
+      return "n/a" if bytes.nil?
+
+      abs = bytes.abs.to_f
+      if abs >= 1024 * 1024
+        format("%.2f MB", abs / (1024 * 1024))
+      elsif abs >= 1024
+        format("%.1f KB", abs / 1024)
+      else
+        "#{bytes} B"
+      end
+    end
+  end
+end

data/lib/heapscope/diff.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module HeapScope
+  # Rich comparison between two snapshots.
+  class Diff
+    attr_reader :before, :after, :class_deltas, :rss_delta, :heap_live_delta,
+                :allocated_delta, :freed_delta, :warnings, :allocation_deltas
+
+    def initialize(before, after)
+      @before = before
+      @after = after
+      @warnings = []
+      @warnings << "snapshots_from_different_processes" unless before.same_process?(after)
+      @class_deltas = compute_class_deltas
+      @rss_delta = delta(after.rss_bytes, before.rss_bytes)
+      @heap_live_delta = delta(after.heap_live_slots, before.heap_live_slots)
+      @allocated_delta = delta(after.total_allocated_objects, before.total_allocated_objects)
+      @freed_delta = delta(after.total_freed_objects, before.total_freed_objects)
+      @allocation_deltas = compute_allocation_deltas
+    end
+
+    def surviving_estimate
+      return nil if allocated_delta.nil? || freed_delta.nil?
+
+      allocated_delta - freed_delta
+    end
+
+    def retention_ratio
+      return nil if allocated_delta.nil? || allocated_delta <= 0
+
+      surviving = surviving_estimate
+      return nil if surviving.nil?
+
+      surviving.to_f / allocated_delta
+    end
+
+    def top_growth(limit = 20)
+      class_deltas.sort_by { |d| -d[:delta_count].abs }.first(limit)
+    end
+
+    def growing_classes(limit = 20)
+      class_deltas.select { |d| d[:delta_count].positive? }
+                  .sort_by { |d| -d[:delta_count] }
+                  .first(limit)
+    end
+
+    def freed_classes(limit = 20)
+      class_deltas.select { |d| d[:delta_count].negative? }
+                  .sort_by { |d| d[:delta_count] }
+                  .first(limit)
+    end
+
+    def new_classes
+      class_deltas.select { |d| d[:before_count].zero? && d[:after_count].positive? }
+    end
+
+    def native_memory_mismatch?
+      return false if rss_delta.nil? || heap_live_delta.nil?
+
+      rss_delta > 10 * 1024 * 1024 && heap_live_delta < 50_000 &&
+        (rss_delta > heap_bytes_estimate_delta * 3)
+    end
+
+    def heap_bytes_estimate_delta
+      class_deltas.sum { |d| d[:delta_bytes] }
+    end
+
+    def to_h
+      {
+        before_id: before.id,
+        after_id: after.id,
+        rss_delta: rss_delta,
+        heap_live_delta: heap_live_delta,
+        allocated_delta: allocated_delta,
+        freed_delta: freed_delta,
+        surviving_estimate: surviving_estimate,
+        retention_ratio: retention_ratio,
+        class_deltas: class_deltas,
+        allocation_deltas: allocation_deltas,
+        warnings: warnings,
+        native_memory_mismatch: native_memory_mismatch?
+      }
+    end
+
+    def to_table(format: :ascii, limit: 15)
+      Tables.class_growth(self, limit: limit, format: format)
+    end
+
+    private
+
+    def compute_class_deltas
+      names = (before.class_stats.keys + after.class_stats.keys).uniq
+      names.map do |name|
+        b = before.class_stats[name] || { count: 0, shallow_bytes: 0 }
+        a = after.class_stats[name] || { count: 0, shallow_bytes: 0 }
+        {
+          name: name,
+          before_count: b[:count].to_i,
+          after_count: a[:count].to_i,
+          delta_count: a[:count].to_i - b[:count].to_i,
+          before_bytes: b[:shallow_bytes].to_i,
+          after_bytes: a[:shallow_bytes].to_i,
+          delta_bytes: a[:shallow_bytes].to_i - b[:shallow_bytes].to_i
+        }
+      end
+    end
+
+    def compute_allocation_deltas
+      sites = (before.allocation_sites.keys + after.allocation_sites.keys).uniq
+      sites.map do |site|
+        b = before.allocation_sites[site] || { count: 0, shallow_bytes: 0 }
+        a = after.allocation_sites[site] || { count: 0, shallow_bytes: 0 }
+        {
+          site: site,
+          before_count: b[:count].to_i,
+          after_count: a[:count].to_i,
+          delta_count: a[:count].to_i - b[:count].to_i,
+          delta_bytes: a[:shallow_bytes].to_i - b[:shallow_bytes].to_i,
+          classes: a[:classes] || b[:classes] || {}
+        }
+      end.sort_by { |d| -d[:delta_count] }
+    end
+
+    def delta(after_v, before_v)
+      return nil if after_v.nil? || before_v.nil?
+
+      after_v - before_v
+    end
+  end
+end