greenroom 0.1.0

data/lib/greenroom/cli/judge.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+require "date"
+require "fileutils"
+require "json"
+require "open3"
+require "optparse"
+require "tmpdir"
+require "time"
+require "yaml"
+require_relative "check"
+
+module Greenroom
+  module CLI
+    module Judge
+      DEFAULTS = {
+        "minimum_comparisons" => 1000,
+        "consecutive_clean_nights" => 2,
+        "expiry_days" => 30,
+        "minimum_comparison_ratio" => 0.8
+      }.freeze
+      Verdict = Data.define(:experiment, :verdict, :reason, :patch, :patch_error)
+
+      class << self
+        def run(arguments, stdout: $stdout, stderr: $stderr)
+          options = {repo: ".", json: false}
+          parser = option_parser(options)
+          parser.parse!(arguments)
+          return usage(stderr, parser, "The --events option is required.") unless options[:events]
+          return usage(stderr, parser, "The --config option is required.") unless options[:config]
+          return usage(stderr, parser, "Unexpected arguments are present.") unless arguments.empty?
+
+          # The host application CI owns the service credentials and pull
+          # request delivery. JSON input keeps these service boundaries outside
+          # the gem.
+          events = JSON.parse(File.read(options[:events]))
+          config = DEFAULTS.merge(YAML.safe_load_file(options[:config]) || {})
+          require "rubocop-ast"
+          verdicts, errors = Evaluation.new(events, config, options[:repo]).call
+          print_results(stdout, verdicts, errors, options[:json])
+          # The exit status reports when a person must act. A successful promotion
+          # stays green because the host CI uses the JSON result to open its
+          # pull request.
+          (verdicts.any? { |item| %w[nogo attention expired].include?(item.verdict) }) ? 1 : 0
+        rescue OptionParser::ParseError, JSON::ParserError, Errno::ENOENT, Psych::Exception => error
+          stderr.puts(error.message)
+          2
+        end
+
+        private
+
+        def option_parser(options)
+          OptionParser.new do |parser|
+            parser.banner = "Usage: greenroom judge --events PATH --config PATH [--repo PATH] [--json]"
+            parser.on("--events PATH") { |value| options[:events] = value }
+            parser.on("--config PATH") { |value| options[:config] = value }
+            parser.on("--repo PATH") { |value| options[:repo] = value }
+            parser.on("--json") { options[:json] = true }
+          end
+        end
+
+        def usage(stderr, parser, message)
+          stderr.puts(message)
+          stderr.puts(parser)
+          2
+        end
+
+        def print_results(stdout, verdicts, errors, json)
+          if json
+            stdout.puts(JSON.generate({verdicts: verdicts.map(&:to_h), errors:}))
+            return
+          end
+
+          verdicts.each do |item|
+            line = "#{item.experiment}: #{item.verdict} (#{item.reason})"
+            if item.patch_error
+              detail = item.patch_error.sub(/\A[A-Z]/) { |character| character.downcase }.delete_suffix(".")
+              line = "#{line} -- no patch: #{detail}"
+            end
+            stdout.puts(line)
+            stdout.write(item.patch) if item.patch
+          end
+          errors.each { |error| stdout.puts("error: #{error}") }
+        end
+      end
+
+      class Evaluation
+        def initialize(events, config, repository)
+          @events = events
+          @config = config
+          @repository = File.expand_path(repository)
+        end
+
+        def call
+          valid, errors = readable_events
+          verdicts = valid.group_by { |event| event.fetch("experiment") }.map do |name, entries|
+            evaluate(name, deduplicate(entries))
+          end
+          [verdicts, errors]
+        end
+
+        private
+
+        def readable_events
+          valid = []
+          errors = []
+          @events.each_with_index do |event, index|
+            if event["timestamp"]
+              valid << event
+            else
+              errors << "Event #{index + 1} has no timestamp."
+            end
+          end
+          [valid, errors]
+        end
+
+        def deduplicate(events)
+          # A retry can report the same run again with corrected data. Assignment
+          # keeps the later report as the run's final value.
+          events.each_with_object({}) { |event, runs| runs[event.fetch("run_id")] = event }.values
+            .sort_by { |event| Time.iso8601(event.fetch("timestamp")) }
+        end
+
+        def evaluate(name, runs)
+          latest = runs.last
+          previous = runs[-2]
+          verdict, reason = decision(runs, latest, previous)
+          patch = %w[go nogo expired].include?(verdict) ? Patch.new(@repository, name, verdict).call : nil
+          patch_error = nil
+          if patch.is_a?(Array)
+            # Event data determines the verdict reason, but repository state
+            # determines the patch result. One result must not replace the other.
+            patch_error = patch.last
+            patch = nil
+          end
+          Verdict.new(name, verdict, reason, patch, patch_error)
+        end
+
+        def decision(runs, latest, previous)
+          # Candidate failures, row failures, and a reduced sample question the
+          # census itself. A person must validate the census before any
+          # patch.
+          attention = attention_reason(latest, previous)
+          return ["attention", attention] if attention
+          return ["nogo", "#{latest.fetch("mismatches")} mismatches"] if latest.fetch("mismatches").positive?
+          if expired?(runs, latest)
+            return ["expired", "#{latest.fetch("comparisons")} comparisons after #{@config.fetch("expiry_days")} days"]
+          end
+
+          # Scheduled runs do not replace a missed night. Counting reported runs
+          # prevents a deployment pause from becoming a failed observation.
+          clean = runs.reverse.take_while { |run| clean?(run) }.length
+          needed = @config.fetch("consecutive_clean_nights")
+          return ["go", "#{needed} clean nights, #{latest.fetch("comparisons")} comparisons"] if clean >= needed
+
+          # Most nightly runs cannot support a final decision. Waiting avoids an
+          # early promotion and avoids repeated alerts for normal evidence
+          # collection.
+          ["waiting", "#{clean} of #{needed} clean nights, #{latest.fetch("comparisons")} comparisons"]
+        end
+
+        def attention_reason(latest, previous)
+          # Control errors describe the established production path. Ignored
+          # comparisons remain operational data, but version 1 does not use them
+          # for decisions. If either metric caused attention, normal nightly runs
+          # would alert a person. Frequent alerts would make each alert less useful.
+          return "#{latest.fetch("candidate_errors")} candidate errors" if latest.fetch("candidate_errors").positive?
+          return "#{latest.fetch("row_errors")} row errors" if latest.fetch("row_errors").positive?
+          # A smaller denominator can make zero mismatches look reliable after
+          # the census examines too little data.
+          # The comparison needs a prior positive value because a ratio against
+          # a missing run or zero is undefined.
+          return unless previous&.fetch("comparisons")&.positive?
+          return unless latest.fetch("comparisons") < previous.fetch("comparisons") * @config.fetch("minimum_comparison_ratio")
+
+          "comparisons fell from #{previous.fetch("comparisons")} to #{latest.fetch("comparisons")}"
+        end
+
+        def expired?(runs, latest)
+          # Expiry requires a reported run below the threshold. A missing event
+          # means that the census did not run, not that it examined zero rows.
+          return false unless latest.fetch("comparisons") < @config.fetch("minimum_comparisons")
+
+          first = Time.iso8601(runs.first.fetch("timestamp"))
+          last = Time.iso8601(latest.fetch("timestamp"))
+          last - first >= @config.fetch("expiry_days") * 86_400
+        end
+
+        def clean?(run)
+          run.fetch("mismatches").zero? && run.fetch("comparisons") >= @config.fetch("minimum_comparisons")
+        end
+      end
+
+      class Patch
+        def initialize(repository, experiment, verdict)
+          @repository = repository
+          @experiment = experiment
+          @verdict = verdict
+        end
+
+        def call
+          matches = find_matches
+          # Multiple locations make the patch target ambiguous. The judge stops
+          # instead of changing a location that it cannot identify safely.
+          return [nil, "The experiment has #{matches.length} locations."] unless matches.one?
+
+          path, source, match = matches.first
+          changed = (@verdict == "go") ? promote(source, match) : revert(source, match)
+          build(path, source, changed)
+        end
+
+        private
+
+        def find_matches
+          Dir.glob(File.join(@repository, "**", "*.rb")).filter_map do |path|
+            source = File.read(path)
+            next unless source.include?(@experiment)
+
+            # The merge gate defines the accepted experiment structure. Reusing
+            # its analyzer prevents the judge from patching a shape that the
+            # gate rejects.
+            analyzer = Check::Analyzer.new(source, source)
+            analyzer.experiments.filter { |item| item.name == @experiment }.map { |item| [path, source, item] }
+          rescue Check::Analyzer::SourceError
+            []
+          end.flatten(1)
+        end
+
+        def promote(source, match)
+          string = match.method.each_descendant(:str).find { |node| node.value == @experiment }
+          offset = string.source_range.end_pos
+          source.dup.insert(offset, ', run: "candidate"')
+        end
+
+        def revert(source, match)
+          body = match.method.children[2]
+          use_body = match.use_block.children[2]
+          changed = source.dup
+          candidate_range = full_line_range(changed, match.candidate.source_range)
+          # Revert reviews compare the result byte for byte with the source from
+          # before the wrap. The separator before the candidate belongs to it.
+          preceding_line = changed.rindex("\n", candidate_range.begin - 2)&.+(1) || 0
+          candidate_range = preceding_line..candidate_range.end if changed[preceding_line...candidate_range.begin].strip.empty?
+          changed.slice!(candidate_range)
+          replace(changed, body.source_range, use_body.source)
+        end
+
+        def replace(source, range, replacement)
+          source.dup.tap { |value| value[range.begin_pos...range.end_pos] = replacement }
+        end
+
+        def full_line_range(source, range)
+          start = source.rindex("\n", range.begin_pos - 1)&.+(1) || 0
+          finish = source.index("\n", range.end_pos) || source.length - 1
+          start..finish
+        end
+
+        def build(path, before, after)
+          # The host application owns its working tree and pull requests. A
+          # unified diff lets that host inspect and apply the change itself.
+          relative = path.delete_prefix("#{@repository}/")
+          Dir.mktmpdir("greenroom-judge-patch") do |directory|
+            %w[a b].each { |side| FileUtils.mkdir_p(File.join(directory, side, File.dirname(relative))) }
+            File.write(File.join(directory, "a", relative), before)
+            File.write(File.join(directory, "b", relative), after)
+            stdout, _stderr, _status = Open3.capture3("git", "diff", "--no-index", "--", "a/#{relative}", "b/#{relative}", chdir: directory)
+            stdout.gsub("a/a/#{relative}", "a/#{relative}").gsub("b/b/#{relative}", "b/#{relative}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/greenroom/cli.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Greenroom
+  module CLI
+    class << self
+      def run(arguments, stdout: $stdout, stderr: $stderr)
+        command = arguments.first
+        return usage(stderr, "A command is required.") unless command
+        if command == "judge"
+          require_relative "cli/judge"
+          return Judge.run(arguments.drop(1), stdout:, stderr:)
+        end
+        return usage(stderr, "Unknown command: #{command}.") unless command == "check"
+
+        require_relative "cli/check"
+        Check.run(arguments.drop(1), stdout:, stderr:)
+      end
+
+      private
+
+      def usage(stderr, message)
+        stderr.puts(message)
+        stderr.puts("Usage: greenroom <check|judge> [options]")
+        2
+      end
+    end
+  end
+end

data/lib/greenroom/digest.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Greenroom
+  # Turns an observed value into a short, fixed-length digest for a sample
+  # event. Never carries the value itself, only evidence that two values did
+  # or did not match.
+  module Digest
+    module_function
+
+    # Value stability is the host's job, via Scientist's own `clean` block --
+    # this module only shortens whatever that block already produced. The
+    # default #inspect can embed an object's memory address, so a host that
+    # needs a digest to stay stable across runs must write a `clean` block
+    # rather than rely on this method to normalize the value.
+    #
+    # `::Digest::SHA256`, not `SHA256`: this module is itself named `Digest`,
+    # so an unqualified `Digest::SHA256` would resolve to
+    # `Greenroom::Digest::SHA256` (itself) under Ruby's lexical scoping,
+    # rather than the standard library's `::Digest` module.
+    def of(value)
+      return nil if value.nil?
+
+      ::Digest::SHA256.hexdigest(value.inspect)
+    end
+  end
+end

data/lib/greenroom/experiment_behavior.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Greenroom
+  # The two behaviours `Scientist::Experiment` requires from its includer,
+  # plus a non-raising override of `raised`.
+  #
+  # A module, not a concrete class: `Scientist::Experiment` has exactly one
+  # registration slot per process (`Scientist::Experiment.set_default`), and
+  # a later `include` silently steals it from an earlier one. If this gem
+  # shipped a class for a host to include, requiring this gem at all would
+  # risk taking over that slot from the host's own experiment class. A host
+  # includes both `Scientist::Experiment` and this module into its own
+  # class instead, keeping the registration decision with the host.
+  #
+  # A host that wants an experiment to run outside a census (for example,
+  # sampling live traffic) overrides `enabled?` in its own class; that is
+  # why `publish` still works when no recorder is installed.
+  module ExperimentBehavior
+    # True exactly while a census job has a recorder installed on this
+    # thread. The recorder's presence is what turns an experiment on, so
+    # there is only one mechanism to reason about, not two that could drift
+    # out of sync.
+    def enabled?
+      !Recorder.current.nil?
+    end
+
+    # A recorder installed on this thread means a census is in progress:
+    # hand it the result to tally instead of sending an event per
+    # comparison. With no recorder installed, an experiment that a host
+    # enabled outside a census (see the module documentation) still gets a
+    # single event per comparison, sent directly.
+    def publish(result)
+      recorder = Recorder.current
+      if recorder
+        recorder.record(result)
+      else
+        publish_comparison(result)
+      end
+    end
+
+    # Scientist's default `raised` re-raises, which would let a failure in
+    # this gem's own instrumentation (a bad `clean` block, a reporter
+    # outage) break the host's actual call. Recording the failure and
+    # returning nil keeps that failure from ever reaching the caller.
+    def raised(operation, error)
+      begin
+        Greenroom.reporter.record("GreenroomInternalError", {
+          experiment: name,
+          operation: operation,
+          error: error.class.name
+        })
+      rescue
+        # Recording the failure must never itself fail the call that
+        # triggered it -- a reporter outage would otherwise take down the
+        # very requests this gem exists to observe, in a way that a broken
+        # bit of measurement code has no business doing.
+      end
+      nil
+    end
+
+    private
+
+    def publish_comparison(result)
+      candidate = result.candidates.first
+      Greenroom.reporter.record("GreenroomComparison", {
+        experiment: result.experiment_name,
+        matched: result.matched?,
+        control_duration_ms: duration_ms(result.control),
+        candidate_duration_ms: duration_ms(candidate),
+        candidate_error: error_class_name(candidate)
+      })
+    end
+
+    def duration_ms(observation)
+      observation && observation.duration * 1000
+    end
+
+    def error_class_name(observation)
+      observation.exception.class.name if observation&.raised?
+    end
+  end
+end

data/lib/greenroom/recorder.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require_relative "digest"
+
+module Greenroom
+  # The tallying half of a census run. A census job installs one of these on
+  # the thread it is running on; while it is installed, `enabled?` is true
+  # and comparisons flow into it instead of going nowhere. Uninstalling it
+  # turns comparisons back off, so the presence of a recorder is the one
+  # mechanism behind both "is the experiment running" and "where do the
+  # results go".
+  class Recorder
+    # A mutable set of counters for one experiment name. Kept separate from
+    # `Recorder` itself because a single run can, in principle, touch more
+    # than one experiment name (a misconfigured target, or a host that reuses
+    # one job for several experiments), and each needs its own counts.
+    class Tally
+      attr_accessor :comparisons, :mismatches, :ignored, :candidate_errors, :control_errors
+
+      def initialize
+        @comparisons = 0
+        @mismatches = 0
+        @ignored = 0
+        @candidate_errors = 0
+        @control_errors = 0
+      end
+
+      # String keys, not symbols: a Tally's hash only ever leaves this class
+      # inside `Recorder#snapshot`, and that Hash must already be shaped like
+      # JSON, because it travels through `Sidekiq.dump_json` / `load_json`
+      # (which always returns string keys) on the way to and from a cursor.
+      def to_h
+        {
+          "comparisons" => comparisons,
+          "mismatches" => mismatches,
+          "ignored" => ignored,
+          "candidate_errors" => candidate_errors,
+          "control_errors" => control_errors
+        }
+      end
+    end
+
+    # census jobs process one row per thread, so the thread is the unit a
+    # recorder must be scoped to: `Thread.current` (fiber-local by default)
+    # gives each thread its own slot without one thread's recorder leaking
+    # into a concurrently running, unrelated job on another thread.
+    THREAD_KEY = :greenroom_recorder
+    private_constant :THREAD_KEY
+
+    class << self
+      def current
+        Thread.current[THREAD_KEY]
+      end
+
+      def install(recorder)
+        Thread.current[THREAD_KEY] = recorder
+      end
+
+      # Returns the recorder that was installed, so a caller that wants to
+      # flush it (`on_stop`) does not need to have held onto a reference
+      # itself.
+      def uninstall
+        recorder = Thread.current[THREAD_KEY]
+        Thread.current[THREAD_KEY] = nil
+        recorder
+      end
+
+      # For tests and any plain call site that wants a recorder installed
+      # for the duration of a block. `ensure` uninstalls even if the block
+      # raises, so a failing test cannot leave a recorder behind for the
+      # next one.
+      def with(recorder)
+        install(recorder)
+        yield
+      ensure
+        uninstall
+      end
+    end
+
+    attr_reader :run_id, :sample_limit, :rows_scanned, :tallies, :samples
+    attr_accessor :subject
+
+    def initialize(run_id:, experiment: nil, sample_limit: 100)
+      @run_id = run_id
+      @sample_limit = sample_limit
+      @rows_scanned = 0
+      @row_errors = 0
+      @tallies = {}
+      @samples = []
+      @subject = nil
+
+      # A run that never triggers a single comparison (a module-only
+      # include, or a target whose `call` never reaches `Scientist.run`)
+      # would otherwise leave `tallies` empty and `flush_total` silent. A
+      # silent run and a run that never started look identical from the
+      # outside, so the zero must be sent, not inferred from an absence.
+      tally_for(experiment) if experiment
+    end
+
+    def count_row
+      @rows_scanned += 1
+    end
+
+    def record_row_error(_error)
+      @row_errors += 1
+    end
+
+    # Receives a `Scientist::Result` from `publish`. Counts are split into
+    # "mismatched" and "ignored" so that a host's `ignore` block, which
+    # exists to suppress noisy known-differences, does not also hide how
+    # much it is suppressing.
+    def record(result)
+      tally = tally_for(result.experiment_name)
+      tally.comparisons += 1
+      tally.mismatches += 1 if result.mismatched?
+      tally.ignored += 1 if result.ignored?
+      tally.candidate_errors += result.candidates.count(&:raised?)
+      tally.control_errors += 1 if result.control&.raised?
+
+      sample(result) if result.mismatched?
+    end
+
+    def snapshot
+      {
+        "rows_scanned" => @rows_scanned,
+        "row_errors" => @row_errors,
+        "tallies" => @tallies.transform_values(&:to_h)
+      }
+    end
+
+    def restore(snapshot)
+      @rows_scanned = snapshot.fetch("rows_scanned", 0).to_i
+      @row_errors = snapshot.fetch("row_errors", 0).to_i
+      # Merges into whatever `@tallies` already holds instead of replacing
+      # it outright. A resumed segment's `Recorder.new(experiment:)` seeds a
+      # zero tally under that name (see the comment in `initialize`) before
+      # `restore` ever runs; a snapshot taken before the first comparison
+      # for that name does not carry it, and replacing `@tallies` wholesale
+      # would erase the seeded zero and lose the name -- the run's later
+      # `flush_total` would then report `experiment: nil` instead of the
+      # name it was actually comparing.
+      snapshot.fetch("tallies", {}).each do |name, counts|
+        tally = tally_for(name)
+        tally.comparisons = counts.fetch("comparisons", 0).to_i
+        tally.mismatches = counts.fetch("mismatches", 0).to_i
+        tally.ignored = counts.fetch("ignored", 0).to_i
+        tally.candidate_errors = counts.fetch("candidate_errors", 0).to_i
+        tally.control_errors = counts.fetch("control_errors", 0).to_i
+      end
+    end
+
+    # Sends one progress event (this run is alive, with the counts so far)
+    # and one mismatch event per sample gathered since the last flush. The
+    # samples that were just sent are discarded afterward: a sample is
+    # evidence, not part of the denominator, so re-sending it on resume
+    # (see `Census::Cursor`) is harmless, and keeping it around forever
+    # would only grow memory for no benefit.
+    def flush_segment(reporter)
+      emit(reporter, "GreenroomCensusProgress")
+      @samples.each { |sample| reporter.record("GreenroomMismatch", sample) }
+      @samples.clear
+    end
+
+    # Sends the run's aggregate event: the one number a judge reads as the
+    # denominator. Splitting counts across `flush_segment` calls and asking
+    # a judge to sum them would make that sum itself something to verify for
+    # double-counting or gaps; a single event at the end avoids that.
+    def flush_total(reporter)
+      emit(reporter, "GreenroomCensus")
+    end
+
+    private
+
+    def tally_for(name)
+      @tallies[name] ||= Tally.new
+    end
+
+    def sample(result)
+      return if @samples.size >= @sample_limit
+
+      candidate = result.mismatched.first
+      @samples << {
+        run_id: run_id,
+        experiment: result.experiment_name,
+        subject: subject,
+        control_digest: Digest.of(result.control&.cleaned_value),
+        candidate_digest: Digest.of(candidate&.cleaned_value),
+        control_error: error_class_name(result.control),
+        candidate_error: error_class_name(candidate)
+      }
+    end
+
+    def error_class_name(observation)
+      observation.exception.class.name if observation&.raised?
+    end
+
+    # Shared by `flush_segment` and `flush_total`: both send one event per
+    # experiment tallied, plus the run-wide `rows_scanned` and `row_errors`.
+    # When no comparison has run yet, `tallies` is empty; a run that is
+    # merely quiet still needs one event (see the comment in `initialize`),
+    # so this falls back to a single zero tally keyed by `nil`.
+    def emit(reporter, event_type)
+      entries = @tallies.empty? ? {nil => Tally.new} : @tallies
+      entries.each do |name, tally|
+        reporter.record(event_type, {
+          run_id: run_id,
+          experiment: name,
+          rows_scanned: rows_scanned,
+          comparisons: tally.comparisons,
+          mismatches: tally.mismatches,
+          ignored: tally.ignored,
+          candidate_errors: tally.candidate_errors,
+          control_errors: tally.control_errors,
+          row_errors: @row_errors
+        })
+      end
+    end
+  end
+end

data/lib/greenroom/reporter/new_relic.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "greenroom/reporter"
+
+module Greenroom
+  # Sends recorded events to New Relic as custom events. Not required by
+  # `greenroom.rb`: a host that wants this delivery requires this file on
+  # purpose, after loading its own New Relic agent, so the gem's own runtime
+  # dependencies never include the agent gem.
+  class Reporter::NewRelic < Reporter
+    def record(event_type, attributes)
+      ::NewRelic::Agent.record_custom_event(event_type, truncate(attributes))
+    end
+  end
+end