rspec-hopper 0.1.0

data/lib/rspec/hopper/queue/redis_streams.rb

@@ -0,0 +1,307 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "redis"
+
+module RSpec
+  module Hopper
+    module Queue
+      # The one queue adapter: Redis Streams with a consumer group per unit
+      # stream, two Lua scripts (initialization and state transition), and a
+      # four-part reservation handle that fences every mutation of an in-flight
+      # entry. See docs/DESIGN.md for the contract this implements.
+      #
+      # The adapter never opens a connection; the caller passes a `Redis`.
+      class RedisStreams
+        LUA_DIR = File.expand_path("redis_streams/lua", __dir__)
+        SCRIPTS = {
+          init: File.read(File.join(LUA_DIR, "init.lua")),
+          transition: File.read(File.join(LUA_DIR, "transition.lua"))
+        }.freeze
+        LEASE_SECONDS = 60
+        INIT_ERRORS = {
+          "LEASE_LOST" => LeaseLost,
+          "ALREADY_INITIALIZED" => PreviouslyInitialized,
+          "ALREADY_READY" => AlreadyInitialized
+        }.freeze
+        BUDGET_FIELDS = %i[max_requeues requeue_tolerance max_reclaims timeout ttl].freeze
+
+        attr_reader :redis, :build_id, :keyset, :ttl, :tombstone_ttl, :timeout,
+                    :max_requeues, :requeue_tolerance, :max_reclaims
+
+        # Times are seconds; the adapter converts to milliseconds for Redis.
+        def initialize(redis:, build_id:, ttl: Config::WORK_DEFAULTS[:ttl],
+                       tombstone_ttl: Config::WORK_DEFAULTS[:tombstone_ttl],
+                       timeout: Config::WORK_DEFAULTS[:timeout],
+                       max_requeues: Config::WORK_DEFAULTS[:max_requeues],
+                       requeue_tolerance: Config::WORK_DEFAULTS[:requeue_tolerance],
+                       max_reclaims: Config::WORK_DEFAULTS[:max_reclaims])
+          @redis = redis
+          @build_id = build_id
+          @keyset = Keys.new(build_id)
+          @ttl = ttl
+          @tombstone_ttl = tombstone_ttl
+          @timeout = timeout
+          @max_requeues = max_requeues
+          @requeue_tolerance = requeue_tolerance
+          @max_reclaims = max_reclaims
+          @shas = {}
+        end
+
+        # -- build lifecycle ---------------------------------------------------
+
+        # One MULTI: meta and the tombstone read together so a caller can tell
+        # "never initialized" from "initialized, state gone".
+        def status
+          meta, tombstone = redis.multi do |m|
+            m.hgetall(keyset.meta)
+            m.exists(keyset.exists)
+          end
+          meta = nil if meta.empty?
+          Status.new(state: meta && meta["state"], tombstone: tombstone.positive?, meta: meta)
+        end
+
+        def acquire_leader(worker_id)
+          token = "#{worker_id}:#{SecureRandom.hex(8)}"
+          redis.set(keyset.leader, token, nx: true, ex: LEASE_SECONDS) ? token : nil
+        end
+
+        def initialize_build(token:, manifest:, unit_ids:)
+          ids = unit_ids.map { |u| u.respond_to?(:id) ? u.id : u }
+          init_script("success", token, manifest, JSON.generate(ids))
+          :ready
+        end
+
+        def fail_initialization(token:, manifest:)
+          init_script("failure", token, manifest, "[]")
+          :init_failed
+        end
+
+        def manifest
+          meta = status.meta
+          meta && Manifest.from_meta(meta)
+        end
+
+        # -- reservation -------------------------------------------------------
+
+        # Atomically claims one entry idle past `timeout` (priority stream
+        # first) and records the reclaim. Returns nil when nothing was claimed
+        # or when the script finalized the unit as reclaim_budget_exhausted.
+        def reclaim_lost(worker_id)
+          reply = check!(transition("reclaim", worker_id, ms(timeout), max_reclaims), :reclaim, nil)
+          return nil unless reply.first == "OK"
+
+          _, stream, entry_id, unit_id, unit_type, delivery_count, retry_index, reclaim_count = reply
+          Reservation.new(unit_id: unit_id, unit_type: unit_type, stream: stream, entry_id: entry_id,
+                          consumer: worker_id, delivery_count: delivery_count,
+                          retry_index: retry_index, reclaim_count: reclaim_count)
+        end
+
+        # Priority stream without blocking, then units with BLOCK, then delivery
+        # accounting. Returns nil on an empty queue or when the entry was
+        # reclaimed between the read and the accounting.
+        def reserve(worker_id, block_ms: 1000)
+          stream, entry_id, fields = read_group(worker_id, "units:priority", nil) ||
+                                     read_group(worker_id, "units", block_ms)
+          return nil unless entry_id
+
+          reply = transition("delivery_accounting", worker_id, stream, entry_id, fields["id"])
+          return nil if reply.first == "STALE"
+
+          check!(reply, :delivery_accounting, fields["id"])
+          Reservation.new(unit_id: fields["id"], unit_type: fields["type"], stream: stream, entry_id: entry_id,
+                          consumer: worker_id, delivery_count: reply[1],
+                          retry_index: reply[2], reclaim_count: reply[3])
+        end
+
+        # -- fenced mutations --------------------------------------------------
+        # The protocol says these return `true` on success and raise otherwise.
+        # rubocop:disable Naming/PredicateMethod
+
+        def heartbeat(reservation)
+          fenced(:heartbeat, reservation, reservation.unit_id)
+          true
+        end
+
+        def finalize(reservation, outcome:, duration_ms:, reason: nil, errors: nil)
+          outcome = outcome.to_sym
+          raise ArgumentError, "outcome must be one of #{OUTCOMES.join(", ")}" unless OUTCOMES.include?(outcome)
+          raise ArgumentError, "reason is required when outcome is failed" if outcome == :failed && reason.nil?
+          raise ArgumentError, "unknown reason #{reason}" if reason && !REASONS.include?(reason.to_s)
+
+          fenced(:finalize, reservation, reservation.unit_id, outcome.to_s, duration_ms.to_i,
+                 reason.to_s, errors_json(errors))
+          true
+        end
+
+        def requeue(reservation, duration_ms:, failure_summary:, errors:)
+          reply = fenced(:requeue, reservation, reservation.unit_id, reservation.unit_type,
+                         max_requeues, requeue_tolerance, duration_ms.to_i, failure_summary.to_s,
+                         errors_json(errors))
+          status = reply.first == "REQUEUED" ? :requeued : :finalized
+          RequeueResult.new(status: status, retry_index: reply[1])
+        end
+
+        def record_abandoned(reservation, elapsed_ms:)
+          fenced(:abandoned, reservation, reservation.unit_id, elapsed_ms.to_i)
+          true
+        end
+
+        # -- unfenced writes ---------------------------------------------------
+
+        # Works before initialization: a boot error may precede `ready`.
+        def record_worker_error(worker_id:, phase:, error:, unit_id: nil)
+          phase = phase.to_s
+          raise ArgumentError, "phase must be one of #{PHASES.join(", ")}" unless PHASES.include?(phase)
+
+          event = {
+            "type" => "worker_error", "worker_id" => worker_id, "phase" => phase, "unit_id" => unit_id,
+            "class" => error.class.name,
+            "message" => ErrorPayload.truncate(error.message.to_s, ErrorPayload::MESSAGE_BYTES),
+            "backtrace" => (error.backtrace || []).first(ErrorPayload::BACKTRACE_LINES)
+          }
+          transition("worker_error", worker_id, JSON.generate(event))
+          true
+        end
+
+        def record_stale_rejected(reservation, operation:)
+          event = {
+            "type" => "stale_rejected", "unit_id" => reservation.unit_id, "worker_id" => reservation.consumer,
+            "retry_index" => reservation.retry_index, "reclaim_count" => reservation.reclaim_count,
+            "ownership_generation" => reservation.ownership_generation, "operation" => operation.to_s,
+            "stream" => reservation.stream, "entry_id" => reservation.entry_id,
+            "delivery_count" => reservation.delivery_count
+          }
+          transition("stale_rejected", reservation.consumer, JSON.generate(event))
+          true
+        end
+
+        def touch_liveness(worker_id, current_unit: nil)
+          check!(transition("liveness", worker_id, current_unit.to_s), :liveness, current_unit)
+          true
+        end
+        # rubocop:enable Naming/PredicateMethod
+
+        # -- completion --------------------------------------------------------
+
+        def complete?
+          finalized, total = meta_fields("finalized_count", "total_units")
+          finalized.to_i == total.to_i
+        end
+
+        def finalized_count = meta_fields("finalized_count").first.to_i
+        def total_units = meta_fields("total_units").first.to_i
+
+        # -- reads -------------------------------------------------------------
+
+        def attempt_events
+          redis.xrange(keyset.attempts).map { |_id, fields| JSON.parse(fields["json"]) }
+        end
+
+        def workers = json_hash(keyset.workers)
+        def unit_states = json_hash(keyset.unit_state)
+
+        # Existing key names of this build, for TTL scans.
+        def keys
+          found = []
+          redis.scan_each(match: keyset.pattern) { |k| found << k }
+          found.sort
+        end
+
+        private
+
+        def ms(seconds) = (seconds * 1000).to_i
+
+        def json_hash(key)
+          redis.hgetall(key).transform_values { |v| JSON.parse(v) }
+        end
+
+        def meta_fields(*names)
+          values = redis.hmget(keyset.meta, *names)
+          raise BuildStateMissing, "build #{build_id}: meta is missing" if values.all?(&:nil?)
+
+          values
+        end
+
+        def errors_json(errors)
+          return "" if errors.nil?
+
+          JSON.generate(ErrorPayload.cap(errors))
+        end
+
+        def budget_meta
+          BUDGET_FIELDS.to_h { |f| [f.to_s, public_send(f).to_s] }
+        end
+
+        def init_script(mode, token, manifest, unit_ids_json)
+          fields = manifest.to_meta.merge(budget_meta)
+          argv = [mode, token, ms(ttl), ms(tombstone_ttl), JSON.generate(fields), unit_ids_json, "file"]
+          run_script(:init, init_keys, argv)
+        rescue Redis::CommandError => e
+          code = e.message[/\A(?:ERR\s+)?([A-Z_]+)/, 1]
+          raise INIT_ERRORS.fetch(code), "build #{build_id}: #{code}" if INIT_ERRORS.key?(code)
+
+          raise
+        end
+
+        def init_keys
+          [keyset.units, keyset.units_priority, keyset.attempts, keyset.meta, keyset.unit_state,
+           keyset.leader, keyset.exists]
+        end
+
+        def transition_keys
+          [keyset.units, keyset.units_priority, keyset.attempts, keyset.meta, keyset.unit_state, keyset.workers]
+        end
+
+        def transition(mode, worker_id, *argv)
+          run_script(:transition, transition_keys, [mode, worker_id, ms(ttl), *argv])
+        rescue Redis::CommandError => e
+          raise CorruptBuild, "build #{build_id}: #{e.message}" if e.message.start_with?("NOGROUP")
+
+          raise
+        end
+
+        # Runs a mode carrying the reservation handle and maps STALE/CORRUPT.
+        def fenced(mode, reservation, *argv)
+          reply = transition(mode.to_s, reservation.consumer, reservation.stream, reservation.entry_id,
+                             reservation.delivery_count, *argv)
+          check!(reply, mode, reservation.unit_id)
+        end
+
+        def check!(reply, operation, unit_id)
+          case reply.first
+          when "STALE"
+            raise StaleReservation, "#{operation} rejected for #{unit_id}: ownership has moved"
+          when "CORRUPT"
+            raise CorruptBuild, "build #{build_id}: state missing during #{operation}#{" of #{unit_id}" if unit_id}"
+          end
+          reply
+        end
+
+        # SCRIPT LOAD once per process, EVALSHA thereafter, EVAL if the server
+        # lost the script (restart, SCRIPT FLUSH).
+        def run_script(name, keys, argv)
+          sha = (@shas[name] ||= redis.script(:load, SCRIPTS.fetch(name)))
+          redis.evalsha(sha, keys: keys, argv: argv)
+        rescue Redis::NoScriptError
+          @shas.delete(name)
+          redis.eval(SCRIPTS.fetch(name), keys: keys, argv: argv)
+        end
+
+        def read_group(worker_id, stream, block_ms)
+          key = keyset.stream(stream)
+          reply = redis.xreadgroup(Keys::CONSUMER_GROUP, worker_id, key, ">", count: 1, block: block_ms)
+          entry = reply[key]&.first
+          return nil unless entry
+
+          [stream, entry[0], entry[1]]
+        rescue Redis::CommandError => e
+          raise CorruptBuild, "build #{build_id}: #{e.message}" if e.message.start_with?("NOGROUP")
+
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/rspec/hopper/queue.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # Storage-agnostic queue protocol. See docs/DESIGN.md for the full contract.
+    # The only adapter is Queue::RedisStreams. No RSpec knowledge lives here.
+    module Queue
+      Status = Data.define(:state, :tombstone, :meta) do
+        def ready? = state == "ready"
+        def init_failed? = state == "init_failed"
+        def present? = !meta.nil?
+      end
+
+      RequeueResult = Data.define(:status, :retry_index) do
+        def requeued? = status == :requeued
+        def finalized? = status == :finalized
+      end
+
+      PHASES = %w[boot init reserve execution redis formatter].freeze
+      REASONS = %w[test_failure retry_budget_exhausted reclaim_budget_exhausted].freeze
+      OUTCOMES = %i[passed failed].freeze
+    end
+  end
+end

data/lib/rspec/hopper/report.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "redis"
+
+module RSpec
+  module Hopper
+    # Produces the one authoritative verdict for a build. Never loads spec files,
+    # never writes to Redis, and knows nothing about RSpec.
+    class Report
+      VERDICTS = %w[passed failed incomplete init_failed missing expired unreachable].freeze
+
+      Outcome = Data.define(:verdict, :exit_code, :headline, :details) do
+        def initialize(verdict:, exit_code:, headline:, details: [])
+          super
+        end
+      end
+
+      MONOTONIC = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      EPOCH_MS = -> { Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) }
+
+      attr_reader :config, :queue, :summary
+
+      # `clock` (monotonic seconds) budgets --timeout and --init-timeout.
+      # `wall_clock` (epoch ms) is compared with Redis-stamped `ready_at` and
+      # `workers.last_seen` for --inactive-timeout.
+      def initialize(config:, queue:, clock: MONOTONIC, sleeper: ->(s) { sleep(s) }, poll_interval: 1.0,
+                     wall_clock: EPOCH_MS)
+        @config = config
+        @queue = queue
+        @clock = clock
+        @sleeper = sleeper
+        @poll_interval = poll_interval
+        @wall_clock = wall_clock
+        @summary = nil
+      end
+
+      # Runs the wait phases, prints a human summary, writes the configured
+      # output files and returns the exit code.
+      def run(out: $stdout)
+        reset
+        outcome = evaluate
+        @summary = build_summary(outcome)
+        write_outputs
+        print_outcome(out, outcome)
+        outcome.exit_code
+      end
+
+      def attempt_log = @attempt_log ||= AttemptLog.new(@events)
+
+      private
+
+      def reset
+        @started = @clock.call
+        @state = nil
+        @manifest = nil
+        @finalized_count = nil
+        @workers = {}
+        @events = []
+        @attempt_log = nil
+      end
+
+      def evaluate
+        status = wait_for_status
+        return missing unless status
+        return expired unless status.present?
+
+        @state = status.state
+        @manifest = Manifest.from_meta(status.meta)
+        return init_failed if status.init_failed?
+
+        incomplete_reason = wait_for_completion
+        collect_events
+        incomplete_reason ? incomplete(*incomplete_reason) : verdict
+      rescue Redis::BaseConnectionError, RedisUnreachable => e
+        Outcome.new(verdict: "unreachable", exit_code: ExitCode::INFRASTRUCTURE,
+                    headline: "redis unreachable: #{e.message}")
+      rescue BuildStateMissing
+        expired
+      end
+
+      # --- wait phases -------------------------------------------------------
+
+      def wait_for_status
+        loop do
+          status = @queue.status
+          return status if status.present? || status.tombstone
+          return nil if elapsed >= config.init_timeout
+
+          @sleeper.call(@poll_interval)
+        end
+      end
+
+      # Returns nil once every unit is finalized, otherwise why it gave up:
+      # `[:timeout]` or `[:inactive, idle_seconds]`.
+      def wait_for_completion
+        loop do
+          @finalized_count = @queue.finalized_count
+          @workers = @queue.workers
+          return nil if @finalized_count >= @manifest.total_units
+          return [:timeout] if elapsed >= config.timeout
+
+          idle = idle_seconds
+          return [:inactive, idle] if idle > config.inactive_timeout
+
+          @sleeper.call(@poll_interval)
+        end
+      end
+
+      def collect_events
+        @events = @queue.attempt_events
+        @attempt_log = nil
+      end
+
+      def elapsed = @clock.call - @started
+
+      # Seconds since the later of ready_at and the most recent worker liveness.
+      def idle_seconds
+        stamps = [@manifest.ready_at, *@workers.values.map { |w| w["last_seen"] }].compact
+        return 0.0 if stamps.empty?
+
+        (@wall_clock.call - stamps.max) / 1000.0
+      end
+
+      # --- outcomes ------------------------------------------------------------
+
+      def missing
+        Outcome.new(verdict: "missing", exit_code: ExitCode::INFRASTRUCTURE,
+                    headline: "build #{config.build_id} never initialized " \
+                              "(no manifest or tombstone after #{config.init_timeout}s)")
+      end
+
+      def expired
+        Outcome.new(verdict: "expired", exit_code: ExitCode::INCOMPLETE,
+                    headline: "build #{config.build_id} was previously initialized; its state is gone " \
+                              "(expired or evicted)")
+      end
+
+      def init_failed
+        collect_events
+        Outcome.new(verdict: "init_failed", exit_code: ExitCode::INCOMPLETE,
+                    headline: "build #{config.build_id} failed to initialize: " \
+                              "#{@manifest.load_errors.size} spec file load error(s)",
+                    details: @manifest.load_errors)
+      end
+
+      # Built only after collect_events so the never-finalized list is current.
+      def incomplete(cause, idle = nil)
+        headline = if cause == :timeout
+                     "incomplete: #{gap} after #{config.timeout}s (--timeout)"
+                   else
+                     "incomplete: workers inactive for #{idle.round}s " \
+                       "(--inactive-timeout #{config.inactive_timeout}s); #{gap}"
+                   end
+        Outcome.new(verdict: "incomplete", exit_code: ExitCode::INCOMPLETE, headline: headline,
+                    details: never_finalized_lines)
+      end
+
+      def verdict
+        if @manifest.empty? && !config.allow_empty
+          Outcome.new(verdict: "failed", exit_code: ExitCode::TEST_FAILURE,
+                      headline: "#{count(@manifest.file_args.size, "file")} given, 0 examples selected",
+                      details: @manifest.file_args)
+        elsif @manifest.total_examples < config.min_examples
+          Outcome.new(verdict: "failed", exit_code: ExitCode::TEST_FAILURE,
+                      headline: "#{@manifest.total_examples} examples selected, " \
+                                "fewer than --min-examples #{config.min_examples}")
+        elsif attempt_log.failed.any?
+          Outcome.new(verdict: "failed", exit_code: ExitCode::TEST_FAILURE,
+                      headline: "#{attempt_log.failed.size} of #{@manifest.total_units} units failed",
+                      details: failed_lines)
+        else
+          Outcome.new(verdict: "passed", exit_code: ExitCode::OK, headline: "passed")
+        end
+      end
+
+      def gap = "#{@finalized_count} of #{@manifest.total_units} units finalized"
+
+      def count(number, noun) = "#{number} #{number == 1 ? noun : "#{noun}s"}"
+
+      def never_finalized
+        return [] unless @manifest
+
+        attempt_log.never_finalized(@manifest.unit_ids)
+      end
+
+      def never_finalized_lines
+        never_finalized.map do |entry|
+          holder = entry[:last_worker_id] ? "last worker #{entry[:last_worker_id]}" : "never delivered"
+          "never finalized: #{entry[:unit_id]} (#{holder})"
+        end
+      end
+
+      def failed_lines
+        attempt_log.failed.map do |entry|
+          line = "failed: #{entry[:unit_id]} (#{entry[:reason]}, worker #{entry[:worker_id]})"
+          first = entry[:errors].find { |e| e.is_a?(Hash) && e["message"] }
+          line += " #{first["class"]}: #{first["message"].lines.first.to_s.strip}" if first
+          line
+        end
+      end
+
+      # --- output --------------------------------------------------------------
+
+      def print_outcome(out, outcome)
+        out.puts "rspec-hopper report: build #{config.build_id}: #{outcome.verdict}"
+        out.puts "  #{outcome.headline}"
+        out.puts "  units: #{totals_line}"
+        outcome.details.each { |line| out.puts "  #{line}" }
+        print_log_notes(out)
+      end
+
+      def totals_line
+        return "unknown (no manifest)" unless @manifest
+
+        finalized = @finalized_count.nil? ? "" : ", #{@finalized_count} finalized"
+        "#{@manifest.total_units} total#{finalized}; examples: #{@manifest.total_examples} selected"
+      end
+
+      def print_log_notes(out)
+        return unless @manifest
+
+        flaky = attempt_log.flaky
+        out.puts "  flaky (passed after requeue): #{flaky.join(", ")}" if flaky.any?
+        abandoned = attempt_log.abandoned
+        out.puts "  abandoned (exceeded --max-unit-duration): #{abandoned.join(", ")}" if abandoned.any?
+        attempt_log.worker_errors.each do |e|
+          out.puts "  worker error: #{e["worker_id"]} #{e["phase"]} #{e["class"]}: #{e["message"]}"
+        end
+      end
+
+      def build_summary(outcome)
+        {
+          "build_id" => config.build_id,
+          "state" => @state,
+          "verdict" => outcome.verdict,
+          "exit_code" => outcome.exit_code,
+          "message" => outcome.headline,
+          "total_units" => @manifest&.total_units,
+          "total_examples" => @manifest&.total_examples,
+          "finalized_count" => @finalized_count,
+          **log_summary,
+          "workers" => @workers,
+          **manifest_summary
+        }
+      end
+
+      def log_summary
+        {
+          "failed" => attempt_log.failed.map { |h| h.transform_keys(&:to_s) },
+          "flaky" => attempt_log.flaky,
+          "never_finalized" => never_finalized.map { |h| h.transform_keys(&:to_s) },
+          "abandoned" => attempt_log.abandoned,
+          "retry_counts" => attempt_log.retry_counts,
+          "reclaim_counts" => attempt_log.reclaim_counts,
+          "worker_errors" => attempt_log.worker_errors,
+          "stale_rejections" => attempt_log.stale_rejections.size
+        }
+      end
+
+      def manifest_summary
+        {
+          "load_errors" => @manifest&.load_errors || [],
+          "file_args" => @manifest&.file_args || [],
+          "seed" => @manifest&.seed,
+          "fingerprint" => @manifest&.fingerprint,
+          "revision" => @manifest&.revision
+        }
+      end
+
+      def write_outputs
+        write_file(config.summary_out, "#{JSON.pretty_generate(@summary)}\n") if config.summary_out
+        return unless config.failed_out
+
+        ids = @summary["failed"].map { |f| f["unit_id"] } + @summary["never_finalized"].map { |n| n["unit_id"] }
+        write_file(config.failed_out, ids.map { |id| "#{id}\n" }.join)
+      end
+
+      def write_file(path, content)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, content)
+      end
+    end
+  end
+end

data/lib/rspec/hopper/reservation.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Hopper
+    # The opaque handle for an in-flight stream entry. Every mutation of the
+    # entry is fenced on (stream, entry_id, consumer, delivery_count).
+    Reservation = Data.define(
+      :unit_id, :unit_type, :stream, :entry_id, :consumer, :delivery_count,
+      :retry_index, :reclaim_count
+    ) do
+      def unit = Unit.new(id: unit_id, type: unit_type)
+
+      # Logical ownership generation across stream-entry replacement on retry.
+      # Recorded in events, never compared to a limit.
+      def ownership_generation = 1 + retry_index + reclaim_count
+    end
+  end
+end