txray 0.1.0

data/lib/txray/reporters/live.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "io/console"
+
+module Txray
+  module Reporters
+    class Live
+      SPARKS = %w[▁ ▂ ▃ ▄ ▅ ▆ ▇ █].freeze
+      HIDE_CURSOR = "\e[?25l"
+      SHOW_CURSOR = "\e[?25h"
+
+      def initialize(path:, io: $stdout, threshold_ms: 250)
+        @io = io
+        @path = path
+        @threshold_ms = threshold_ms
+      end
+
+      def open = @io.print(HIDE_CURSOR)
+
+      def close(monitor)
+        @width, @height = viewport
+        @io.print(SHOW_CURSOR)
+        @io.puts
+        summary(monitor)
+      end
+
+      def draw(monitor)
+        @width, @height = viewport
+        @io.print("\e[H\e[2J")
+        header(monitor)
+        stats(monitor)
+        feed(monitor)
+        hotspots(monitor)
+        @io.print(dim("  ctrl-c to stop"))
+        @io.flush
+      end
+
+      private
+
+      def viewport
+        rows, columns = IO.console&.winsize
+        [ columns.to_i.positive? ? columns : 100, rows.to_i.positive? ? rows : 40 ]
+      rescue StandardError
+        [ 100, 40 ]
+      end
+
+      def feed_limit = (@height - 19).clamp(3, 9)
+
+      def clip(text, budget)
+        budget = [ budget, 12 ].max
+        text.length <= budget ? text : "..#{text[-(budget - 2)..]}"
+      end
+
+      def header(monitor)
+        @io.puts
+        @io.puts "  #{bold(magenta("txray"))} #{dim("watching #{@path}")}#{" " * 2}#{dim(clock(monitor.uptime))}"
+        @io.puts
+      end
+
+      def stats(monitor)
+        if monitor.empty?
+          @io.puts "  #{dim("waiting for the application to open a transaction")}"
+          @io.puts
+          return
+        end
+
+        @io.puts "  #{label("transactions")}#{count(monitor)}"
+        @io.puts "  #{label("duration")}#{spread(monitor)}"
+        @io.puts "  #{label("")}#{sparkline(monitor)}"
+        @io.puts
+      end
+
+      def count(monitor)
+        [ "#{monitor.transactions.size} seen",
+          tint("#{monitor.slow} slow", monitor.slow.zero? ? :dim : :yellow),
+          tint("#{monitor.flagged} with findings", monitor.flagged.zero? ? :dim : :red) ].join(dim("   "))
+      end
+
+      def spread(monitor)
+        [ "p50 #{duration(monitor.percentile(0.5))}",
+          "p95 #{duration(monitor.percentile(0.95))}",
+          "max #{duration(monitor.durations.max.to_f)}" ].join(dim("   "))
+      end
+
+      def sparkline(monitor)
+        values = monitor.durations.last(48)
+        return dim("-") if values.empty?
+
+        peak = [ values.max, 1.0 ].max
+        values.map { |value| SPARKS[((value / peak) * (SPARKS.size - 1)).round] }.join
+      end
+
+      def feed(monitor)
+        rows = monitor.recent(feed_limit)
+        return if rows.empty?
+
+        @io.puts "  #{dim("LIVE")}"
+        rows.each { |event| @io.puts(event[:type] == "transaction" ? transaction_row(event) : violation_row(event)) }
+        @io.puts
+      end
+
+      def transaction_row(event)
+        marker = if event[:violations].to_a.any?
+                   red("x")
+                 else
+                   (event[:duration_ms].to_f >= @threshold_ms ? yellow("!") : green("."))
+                 end
+        line = "  #{dim(time(event))} #{duration(event[:duration_ms]).rjust(18)}  #{marker}  " \
+               "#{clip(source(event), @width - 36)}"
+        [ line, *event[:violations].to_a.map { |violation| finding_row(violation) } ].join("\n")
+      end
+
+      def violation_row(event)
+        "  #{dim(time(event))} #{elapsed(event[:duration_ms]).rjust(18)}  #{red("x")}  " \
+          "#{red(event[:rule].to_s)} #{dim(clip(source(event), @width - 36 - event[:rule].to_s.length))}"
+      end
+
+      def elapsed(milliseconds) = milliseconds ? duration(milliseconds) : dim("-")
+
+      def finding_row(violation)
+        detail = [ violation[:message], duration_suffix(violation) ].compact.join(" ")
+        rule = violation[:rule].to_s
+        "  #{" " * 24}#{dim("|")} #{yellow(rule)} #{dim(clip(detail, @width - 28 - rule.length))}"
+      end
+
+      def duration_suffix(violation)
+        "(#{duration(violation[:duration_ms])})" if violation[:duration_ms]
+      end
+
+      def source(event) = event[:source].to_s.sub(/:in\s+[`'"](.+)[`'"]\z/, " \\1")
+
+      def hotspots(monitor)
+        rows = monitor.hotspots(5)
+        return if rows.empty?
+
+        @io.puts "  #{dim("HOTSPOTS")}"
+        rows.each do |entry|
+          @io.puts "  #{"#{entry[:count]}x".rjust(4)}  #{yellow(entry[:rule].to_s.ljust(32))} " \
+                   "#{dim(clip(source(entry), @width - 42))}"
+        end
+        @io.puts
+      end
+
+      def summary(monitor)
+        return @io.puts("  no transactions observed") if monitor.empty?
+
+        @io.puts "  #{monitor.transactions.size} transactions, #{monitor.slow} slow, " \
+                 "#{monitor.findings.size} findings over #{clock(monitor.uptime)}"
+        monitor.hotspots(10).each do |entry|
+          @io.puts "  #{"#{entry[:count]}x".rjust(4)}  #{entry[:rule].to_s.ljust(32)} #{source(entry)}"
+        end
+      end
+
+      def duration(milliseconds)
+        value = milliseconds.to_f
+        return "#{value.round}ms" if value < 1000
+
+        "#{(value / 1000).round(2)}s"
+      end
+
+      def time(event) = Time.at(event[:at].to_f).strftime("%H:%M:%S")
+
+      def clock(seconds)
+        format("%02d:%02d:%02d", seconds / 3600, (seconds % 3600) / 60, seconds % 60)
+      end
+
+      def label(text) = text.ljust(16)
+
+      def tint(text, color) = color == :dim ? dim(text) : send(color, text)
+
+      def bold(text) = "\e[1m#{text}\e[0m"
+      def dim(text) = "\e[2m#{text}\e[0m"
+      def red(text) = "\e[31m#{text}\e[0m"
+      def green(text) = "\e[32m#{text}\e[0m"
+      def yellow(text) = "\e[33m#{text}\e[0m"
+      def magenta(text) = "\e[35m#{text}\e[0m"
+    end
+  end
+end

data/lib/txray/reporters/sarif.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Txray
+  module Reporters
+    class Sarif
+      LEVELS = { high: "error", medium: "warning", low: "note" }.freeze
+
+      def initialize(io: $stdout)
+        @io = io
+      end
+
+      def report(result)
+        @io.puts JSON.pretty_generate(
+          "$schema" => "https://json.schemastore.org/sarif-2.1.0.json",
+          version: "2.1.0",
+          runs: [ { tool: tool, results: result.offenses.map { |offense| sarif_result(offense) } } ]
+        )
+      end
+
+      private
+
+      def tool
+        {
+          driver: {
+            name: "txray",
+            version: Txray::VERSION,
+            informationUri: "https://github.com/theowecker/txray",
+            rules: Rules.all.values.map do |rule|
+              { id: rule.id, shortDescription: { text: rule.id.tr("-", " ") }, fullDescription: { text: rule.remedy } }
+            end
+          }
+        }
+      end
+
+      def sarif_result(offense)
+        {
+          ruleId: offense.id,
+          level: LEVELS.fetch(offense.severity, "warning"),
+          message: { text: "#{offense.message}. #{offense.rule.remedy}" },
+          locations: [ {
+            physicalLocation: {
+              artifactLocation: { uri: offense.path },
+              region: { startLine: offense.line, startColumn: offense.column }
+            }
+          } ]
+        }
+      end
+    end
+  end
+end

data/lib/txray/reporters/text.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Txray
+  module Reporters
+    class Text
+      COLORS = { high: 31, medium: 33, low: 36 }.freeze
+
+      def initialize(io: $stdout, color: io.tty?)
+        @io = io
+        @color = color
+      end
+
+      def report(result)
+        result.offenses.group_by(&:path).each do |path, offenses|
+          @io.puts bold(path)
+          offenses.each { |offense| print_offense(offense) }
+          @io.puts
+        end
+        print_summary(result)
+      end
+
+      private
+
+      def print_offense(offense)
+        @io.puts "  #{offense.line}:#{offense.column}  #{tint(offense.severity.to_s.ljust(6), offense.severity)} #{offense.id}"
+        @io.puts "    #{offense.message}"
+        offense.trace.each { |frame| @io.puts dim("      via #{frame}") }
+        @io.puts dim("    #{offense.rule.remedy}")
+      end
+
+      def print_summary(result)
+        counts = result.counts
+        summary = Offense::SEVERITIES.reverse.filter_map { |s| "#{counts[s]} #{s}" if counts[s] }
+        @io.puts "#{result.files.size} files scanned, #{result.offenses.size} offenses" \
+                 "#{" (#{summary.join(", ")})" unless summary.empty?}"
+        return if result.skipped.to_a.empty?
+
+        @io.puts dim("#{result.skipped.size} files could not be parsed and were skipped:")
+        result.skipped.each { |path| @io.puts dim("  #{path}") }
+      end
+
+      def tint(text, severity) = @color ? "\e[#{COLORS.fetch(severity, 0)}m#{text}\e[0m" : text
+      def bold(text) = @color ? "\e[1m#{text}\e[0m" : text
+      def dim(text) = @color ? "\e[2m#{text}\e[0m" : text
+    end
+  end
+end

data/lib/txray/reporters.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "reporters/text"
+require_relative "reporters/json"
+require_relative "reporters/sarif"
+require_relative "reporters/github"
+require_relative "reporters/live"
+
+module Txray
+  module Reporters
+    FORMATS = { "text" => Text, "json" => Json, "sarif" => Sarif, "github" => Github }.freeze
+
+    def self.build(format, io: $stdout)
+      klass = FORMATS[format.to_s] or raise Error, "unknown format #{format}, expected one of #{FORMATS.keys.join(", ")}"
+      klass.new(io: io)
+    end
+  end
+end

data/lib/txray/rule.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Txray
+  Rule = Struct.new(:id, :category, :severity, :message, :remedy, keyword_init: true)
+
+  module Rules
+    def self.all = @all ||= {}
+
+    def self.register(id:, category:, severity:, message:, remedy:)
+      all[id] = Rule.new(id: id, category: category, severity: severity, message: message, remedy: remedy)
+    end
+
+    def self.[](id) = all.fetch(id)
+    def self.ids = all.keys
+
+    register(
+      id: "http-in-transaction",
+      category: :transaction,
+      severity: :high,
+      message: "HTTP request `%{snippet}` runs inside %{scope}",
+      remedy: "Move the request outside the transaction, or enqueue it from an after_commit callback."
+    )
+
+    register(
+      id: "external-service-in-transaction",
+      category: :transaction,
+      severity: :high,
+      message: "External service call `%{snippet}` runs inside %{scope}",
+      remedy: "Third party clients hold the connection and the row locks for their full round trip. Call them after commit."
+    )
+
+    register(
+      id: "mail-in-transaction",
+      category: :transaction,
+      severity: :high,
+      message: "Synchronous mail delivery `%{snippet}` runs inside %{scope}",
+      remedy: "Use deliver_later from an after_commit callback so SMTP latency stays out of the transaction."
+    )
+
+    register(
+      id: "shell-in-transaction",
+      category: :transaction,
+      severity: :high,
+      message: "Subprocess `%{snippet}` runs inside %{scope}",
+      remedy: "Shelling out blocks the connection for an unbounded time. Run it after the transaction commits."
+    )
+
+    register(
+      id: "sleep-in-transaction",
+      category: :transaction,
+      severity: :high,
+      message: "`%{snippet}` deliberately blocks inside %{scope}",
+      remedy: "Sleeping while holding row locks stalls every writer behind you. Sleep outside the transaction."
+    )
+
+    register(
+      id: "job-enqueue-in-transaction",
+      category: :transaction,
+      severity: :medium,
+      message: "Background job `%{snippet}` is enqueued inside %{scope}",
+      remedy: "The worker can pick the job up before the transaction commits and read stale or missing rows. Enqueue from after_commit."
+    )
+
+    register(
+      id: "upload-in-transaction",
+      category: :transaction,
+      severity: :medium,
+      message: "Attachment operation `%{snippet}` runs inside %{scope}",
+      remedy: "Active Storage uploads to object storage over the network. Attach after commit or upload before opening the transaction."
+    )
+
+    register(
+      id: "iteration-in-transaction",
+      category: :transaction,
+      severity: :medium,
+      message: "Loop `%{snippet}` performs database work per iteration inside %{scope}",
+      remedy: "Transaction duration grows with collection size. Batch the writes or move the loop outside the transaction."
+    )
+
+    register(
+      id: "broadcast-in-transaction",
+      category: :transaction,
+      severity: :medium,
+      message: "Broadcast `%{snippet}` is pushed inside %{scope}",
+      remedy: "Rendering and pushing before the commit lets subscribers see rows that are not committed yet, " \
+              "or that roll back. Broadcast from after_commit."
+    )
+
+    register(
+      id: "dynamic-dispatch-in-transaction",
+      category: :transaction,
+      severity: :low,
+      message: "`%{snippet}` dispatches to a name txray cannot resolve inside %{scope}",
+      remedy: "Static analysis stops here. Check by hand that the target does no network or filesystem work, or enable the runtime guard."
+    )
+
+    register(
+      id: "blocking-io-in-transaction",
+      category: :transaction,
+      severity: :medium,
+      message: "File or media work `%{snippet}` runs inside %{scope}",
+      remedy: "Parsing, rendering and image processing scale with input size. Do the work before opening the transaction."
+    )
+
+    register(
+      id: "cache-in-transaction",
+      category: :transaction,
+      severity: :low,
+      message: "Cache or key value call `%{snippet}` runs inside %{scope}",
+      remedy: "A cache round trip is still network latency held against an open transaction. Read it before the transaction."
+    )
+  end
+end

data/lib/txray/runtime/sink.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Txray
+  module Runtime
+    class Sink
+      def self.build(path)
+        path ? new(path) : Null.new
+      end
+
+      def initialize(path)
+        @path = path
+      end
+
+      def write(event)
+        prepare
+        File.open(@path, "a") do |file|
+          file.flock(File::LOCK_EX)
+          file.puts(JSON.generate(event))
+        end
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def prepare
+        return if @prepared
+
+        FileUtils.mkdir_p(File.dirname(@path))
+        @prepared = true
+      end
+
+      class Null
+        def write(_event) = nil
+      end
+    end
+  end
+end

data/lib/txray/runtime/transaction.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Txray
+  module Runtime
+    class Transaction
+      attr_reader :source, :violations
+
+      def initialize(source)
+        @source = source
+        @violations = []
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def record(violation) = @violations << violation
+
+      def duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at) * 1000).round(1)
+
+      def to_event(outcome)
+        {
+          type: "transaction",
+          at: Time.now.to_f,
+          pid: Process.pid,
+          duration_ms: duration_ms,
+          outcome: outcome.to_s,
+          source: source,
+          violations: violations
+        }
+      end
+    end
+
+    class Stack
+      KEY = :txray_transactions
+
+      class << self
+        def push(source)
+          frames.push(Transaction.new(source))
+        end
+
+        def pop = frames.pop
+        def current = frames.last
+        def open? = !frames.empty?
+
+        private
+
+        def frames = Thread.current[KEY] ||= []
+      end
+    end
+  end
+end

data/lib/txray/runtime.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require "logger"
+require "fileutils"
+
+require_relative "runtime/sink"
+require_relative "runtime/transaction"
+
+module Txray
+  module Runtime
+    class Violation < Error; end
+
+    IGNORE_KEY = :txray_ignoring
+    GEM_ROOT = File.expand_path("..", __dir__)
+
+    DEFAULTS = {
+      threshold_ms: 250,
+      on_violation: :log,
+      guard_http: true,
+      guard_jobs: true,
+      guard_mail: true,
+      log_path: nil,
+      ignore: [],
+      logger: nil
+    }.freeze
+
+    class << self
+      attr_reader :options
+
+      def install(**overrides)
+        return if @installed
+
+        @installed = true
+        @options = DEFAULTS.merge(overrides.compact)
+        @ignore = Array(@options[:ignore]).map { |pattern| Regexp.new(pattern.to_s) }
+        @sink = Sink.build(@options[:log_path])
+        @subscribers = []
+
+        watch_transactions
+        watch_jobs if @options[:guard_jobs]
+        watch_mail if @options[:guard_mail]
+        patch_net_http if @options[:guard_http]
+      end
+
+      def uninstall
+        @subscribers.to_a.each { |subscriber| ActiveSupport::Notifications.unsubscribe(subscriber) }
+        @subscribers = []
+        @installed = nil
+      end
+
+      def installed? = @installed == true
+
+      def ignore
+        previous = Thread.current[IGNORE_KEY]
+        Thread.current[IGNORE_KEY] = true
+        yield
+      ensure
+        Thread.current[IGNORE_KEY] = previous
+      end
+
+      def ignoring? = Thread.current[IGNORE_KEY] == true
+
+      def transaction_open?
+        return false unless defined?(ActiveRecord::Base)
+
+        pool = ActiveRecord::Base.connection_pool
+        return false unless pool.active_connection?
+
+        connection = pool.respond_to?(:lease_connection) ? pool.lease_connection : pool.connection
+        connection.transaction_open?
+      rescue StandardError
+        false
+      end
+
+      def violation(rule, message, duration_ms: nil)
+        return if ignoring?
+
+        source = app_backtrace.first
+        return if ignored?("#{message} #{source}")
+
+        record = { rule: rule, message: message, duration_ms: duration_ms, source: source }
+        open = Stack.current
+        open&.record(record)
+        report(record.merge(type: "violation", at: Time.now.to_f, pid: Process.pid)) if open.nil?
+        announce(rule, message)
+      end
+
+      def report(event)
+        @sink.write(event)
+      end
+
+      private
+
+      def announce(rule, message)
+        text = "[txray] #{rule}: #{message}\n  #{app_backtrace.join("\n  ")}"
+        raise Violation, text if @options[:on_violation] == :raise
+
+        logger.warn(text)
+      end
+
+      def ignored?(text) = @ignore.any? { |pattern| pattern.match?(text) }
+
+      def logger = @options[:logger] || (defined?(Rails) && Rails.logger) || Logger.new($stderr)
+
+      def app_backtrace
+        root = defined?(Rails) && Rails.root ? Rails.root.to_s : Dir.pwd
+        frames = caller.reject { |line| vendored?(line) }
+        own = frames.select { |line| line.start_with?(root) }
+        own = frames if own.empty?
+        own.map { |line| line.delete_prefix("#{root}/") }.first(5)
+      end
+
+      def vendored?(line)
+        line.start_with?(GEM_ROOT, RbConfig::CONFIG["libdir"]) || line.include?("/gems/")
+      end
+
+      def subscribe(event, &block)
+        @subscribers << ActiveSupport::Notifications.subscribe(event) do |*args|
+          block.call(ActiveSupport::Notifications::Event.new(*args))
+        end
+      end
+
+      def watch_transactions
+        subscribe("start_transaction.active_record") { open_transaction }
+        subscribe("transaction.active_record") { |event| close_transaction(event) }
+      end
+
+      def open_transaction
+        Stack.push(app_backtrace.first)
+      end
+
+      def close_transaction(event)
+        transaction = Stack.pop
+        return if ignoring?
+
+        duration = transaction&.duration_ms || event.duration.round(1)
+        slow = duration >= @options[:threshold_ms]
+        violations = transaction&.violations.to_a
+        return unless slow || violations.any?
+
+        outcome = event.payload[:outcome] || :commit
+        report((transaction&.to_event(outcome) || fallback_event(duration, outcome)).merge(duration_ms: duration))
+        announce_slow(duration) if slow
+      end
+
+      def fallback_event(duration, outcome)
+        { type: "transaction", at: Time.now.to_f, pid: Process.pid, duration_ms: duration,
+          outcome: outcome.to_s, source: app_backtrace.first, violations: [] }
+      end
+
+      def announce_slow(duration)
+        announce("slow-transaction",
+                 "transaction held for #{duration}ms, over the #{@options[:threshold_ms]}ms threshold")
+      end
+
+      def watch_jobs
+        subscribe("enqueue.active_job") do |event|
+          next unless transaction_open?
+
+          violation("job-enqueue-in-transaction", "#{event.payload[:job].class} enqueued inside an open transaction")
+        end
+      end
+
+      def watch_mail
+        subscribe("deliver.action_mailer") do |event|
+          next unless transaction_open?
+
+          violation("mail-in-transaction", "#{event.payload[:mailer]} delivered mail inside an open transaction",
+                    duration_ms: event.duration.round(1))
+        end
+      end
+
+      def patch_net_http
+        return if @http_patched
+
+        require "net/http"
+        Net::HTTP.prepend(NetHttpGuard)
+        @http_patched = true
+      end
+    end
+
+    module NetHttpGuard
+      def request(req, body = nil, &)
+        return super unless Txray::Runtime.transaction_open?
+
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        begin
+          super
+        ensure
+          elapsed = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round(1)
+          Txray::Runtime.violation("http-in-transaction", "#{req.method} #{address}#{req.path} inside an open " \
+                                                          "transaction", duration_ms: elapsed)
+        end
+      end
+    end
+  end
+end