wurk 1.9.0 → 1.10.0

data/lib/wurk/configuration.rb

@@ -9,6 +9,8 @@ require_relative 'topology'
 require_relative 'cpu_count'
 require_relative 'redis_options'
 require_relative 'keys'
+require_relative 'error_coalescer'
+require_relative 'metrics/process_counters'
 
 module Wurk
   # Owns runtime knobs (concurrency, queues, timeouts, lifecycle events,
@@ -98,7 +100,11 @@ module Wurk
       # the fetch path reads this once at boot, so an app that caps nothing
       # resolves to "no capped queues" before the first fetch and never asks
       # again.
-      global_concurrency: {}
+      global_concurrency: {},
+      # See #swarm_crash_exit_after=. Long enough to outlast a slow rollout or a
+      # dependency coming back, short enough that a bad release does not sit
+      # looking Running for an afternoon.
+      swarm_crash_exit_after: 600
     }.freeze
 
     # :fork fires only inside swarm children, after fork + internal AR/Redis
@@ -151,7 +157,7 @@ module Wurk
       Wurk::Metrics::Statsd.reset!
     end
 
-    def initialize(options = {})
+    def initialize(options = {}) # rubocop:disable Metrics/AbcSize
       @options = deep_dup_defaults.merge(options)
       # Through the same door `global_concurrency=` uses, so a cap passed to
       # `Configuration.new` is validated rather than trusted, and the default
@@ -162,14 +168,16 @@ module Wurk
       @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
       @capsules = {}
       @directory = {}
-      @client_chain = Middleware::Chain.new
-      @server_chain = Middleware::Chain.new(on_registration: method(:warn_on_late_server_registration))
+      @client_chain = Middleware::Chain.new(self)
+      @server_chain = Middleware::Chain.new(self, on_registration: method(:warn_on_late_server_registration))
       @redis_config = { url: ENV.fetch('REDIS_URL', 'redis://localhost:6379/0') }
       @web_redis_pool = nil
       @logger = nil
       @thread_priority = DEFAULT_THREAD_PRIORITY
       @telemetry_installed = false
       @frozen = false
+      @deferred_server_blocks = []
+      @fetch_error_coalescer = ErrorCoalescer.new
     end
 
     # --- Hash-like options access -----------------------------------------
@@ -192,6 +200,7 @@ module Wurk
     end
 
     def dig(*keys) = @options.dig(*keys)
+    def to_json(*) = Wurk.dump_json(@options)
 
     # --- Default capsule shortcuts ---------------------------------------
 
@@ -250,6 +259,7 @@ module Wurk
     def redis=(hash)
       guard_frozen!
       RedisOptions.validate!(hash)
+      warn_on_late_redis_assignment
       @redis_config = @redis_config.merge(hash.transform_keys(&:to_sym))
     end
 
@@ -263,6 +273,11 @@ module Wurk
       default_capsule.redis_pool
     end
 
+    # Parsed INFO, without the pool's own counters RedisPool#info merges in.
+    def redis_info
+      redis_pool.info.except('size', 'available')
+    end
+
     # Disconnect and drop every capsule's cached pools (main + fetch) plus the
     # web pool. Used by Wurk::Swarm so the parent never leaks sockets into forks
     # and each child can build fresh ones.
@@ -272,6 +287,12 @@ module Wurk
       @web_redis_pool = nil
     end
 
+    # Non-terminal counterpart, for a fork in a parent whose threads are live.
+    def reload_redis_pools!
+      @capsules.each_value(&:reload_redis_pools!)
+      @web_redis_pool&.reload!
+    end
+
     def new_redis_pool(size, name = 'custom')
       build_redis_pool(size: size, name: name)
     end
@@ -403,6 +424,20 @@ module Wurk
       @options[:swarm_heartbeat_timeout]
     end
 
+    # Seconds every swarm slot may spend crash-looping before the standalone
+    # runner reports it and exits non-zero. nil or false keeps respawning forever.
+    def swarm_crash_exit_after=(seconds)
+      unless seconds.nil? || seconds == false || (seconds.is_a?(Numeric) && seconds.positive?)
+        raise ArgumentError, "swarm_crash_exit_after must be nil, false, or a positive number (got #{seconds.inspect})"
+      end
+
+      @options[:swarm_crash_exit_after] = seconds || nil
+    end
+
+    def swarm_crash_exit_after
+      @options[:swarm_crash_exit_after]
+    end
+
     # --- Reliability (Sidekiq Pro drop-in toggles) -----------------------
 
     # Sidekiq Pro's opt-in toggle for reliable fetch. Already the default in
@@ -411,9 +446,8 @@ module Wurk
     # callback. It exists so a Pro initializer drops in unchanged instead of
     # raising NoMethodError.
     #
-    # NOTE: `reliable_scheduler!` below is NOT a no-op. The default
-    # `scheduled_enq` pops then pushes and has a job-loss window
-    # (Wurk::Scheduled::Enq); only the toggle swaps in the atomic promoter.
+    # NOTE: `reliable_scheduler!` below is NOT a no-op: it swaps in the atomic
+    # promoter, much faster but skipping client middleware on promotion.
     #
     # The optional block is Pro's recovery callback: `|jobstr, pill|`, fired
     # once per orphan recovery (`pill` nil) and once on a poison kill (`pill`
@@ -426,8 +460,9 @@ module Wurk
 
     # Pro reliable scheduler (§4): promote due jobs from retry/schedule onto
     # their target queue in a single atomic Lua (ZRANGEBYSCORE+ZREM+LPUSH),
-    # closing the pop→push job-loss window of the default poller. Swaps the
-    # pluggable `scheduled_enq` for the atomic promoter; idempotent.
+    # much faster than the default poller but skipping client middleware on
+    # promotion. Swaps the pluggable `scheduled_enq` for the atomic promoter;
+    # idempotent.
     def reliable_scheduler!(*)
       self[:scheduled_enq] = Wurk::Scheduled::ReliableEnq
       nil
@@ -765,6 +800,7 @@ module Wurk
     attr_writer :logger
 
     def handle_exception(ex, ctx = {})
+      ctx = redact_jobstr(ctx)
       if error_handlers.empty?
         logger.error("#{redacted_context(ctx)} #{ex.class}: #{ex.message}")
       else
@@ -776,11 +812,26 @@ module Wurk
       end
     end
 
+    # What the fetch loop reports through. It sleeps 1s and retries on every
+    # thread of every worker, so an outage would otherwise call each handler once
+    # per thread per second. The count lands in the metric, the handlers see the
+    # first of each exception class and one per window after that.
+    def report_fetch_error(exception)
+      Metrics::ProcessCounters.increment(:fetch_errors)
+      suppressed = @fetch_error_coalescer.admit(exception.class)
+      return unless suppressed
+
+      context = { context: 'Error fetching job' }
+      context[:suppressed] = suppressed if suppressed.positive?
+      handle_exception(exception, context)
+    end
+
     # The error context with the job's args passed through the `redact_args`
     # hook, for the two reporters that print or tag the whole context: the
     # fallback log line and the default ERROR_HANDLER (whose Wurk::Context a
     # JSON log formatter emits). Host-registered error handlers still receive
-    # the real ctx — they are the host's own code.
+    # the real `job` — they are the host's own code (`jobstr` is another matter:
+    # see #redact_jobstr).
     def redacted_context(ctx)
       hook = @options[:redact_args]
       job = ctx.is_a?(Hash) ? ctx[:job] : nil
@@ -790,16 +841,36 @@ module Wurk
       ctx.merge(job: job.merge('args' => Wurk::Redact.args(job, hook)))
     end
 
+    # The raw payload a job that failed to parse or dispatch is reported with. Unlike
+    # `job`, this reaches host handlers scrubbed too: it is a string nobody expects
+    # to carry the args, so handlers forward it to log aggregators and trackers as
+    # is. What it cannot be parsed from cannot be scrubbed, so that is withheld.
+    def redact_jobstr(ctx)
+      hook = @options[:redact_args]
+      raw = ctx.is_a?(Hash) ? ctx[:jobstr] : nil
+      return ctx unless hook && raw
+
+      require_relative 'redact'
+      ctx.merge(jobstr: Wurk::Redact.jobstr(raw, hook))
+    end
+    private :redact_jobstr
+
     # --- Configure blocks (Sidekiq.configure_server / _client) -----------
 
-    # The gate is evaluated per call: a skipped block is never yielded and
-    # never replayed, and an allowed one yields the configuration and returns
-    # the block's value. A skip reports itself once per gate per instance —
+    # The gate is evaluated per call: an allowed block yields the configuration
+    # and returns the block's value. A skip reports itself once per gate per instance —
     # the flags below decide only whether the skip is reported again, and
     # they live outside @options so a frozen configuration can still warn.
-    def configure_server
-      return unless block_given?
+    #
+    # A skipped block is kept for #enter_server_mode. Gems call this at load
+    # time, and under the railtie that is Bundler.require, before the
+    # `wurk.server_mode` initializer: dropping it left sidekiq-cron, sentry-sidekiq
+    # and the like silently absent from a process that does run workers.
+    def configure_server(&block)
+      return unless block
       return yield self if server?
+
+      @deferred_server_blocks << block
       return if @configure_server_skip_warned
 
       @configure_server_skip_warned = true
@@ -807,6 +878,11 @@ module Wurk
       nil
     end
 
+    def enter_server_mode
+      self[:server] = true
+      @deferred_server_blocks.shift.call(self) until @deferred_server_blocks.empty?
+    end
+
     # A web process that also boots the swarm (Wurk::RailsBoot) is flagged
     # server, yet every enqueue it makes from a request is client work, so
     # `client_in_server` lets its configure_client blocks run too. Same
@@ -896,6 +972,16 @@ module Wurk
       self
     end
 
+    # A swarm child inherits the parent's prepare_for_fork! freeze, but its copy
+    # is its own, and Launcher#run freezes it again. In between it is what a
+    # non-swarm process is: open to `:fork` / `:startup` hooks and to gems that
+    # prepend Launcher#initialize and write an option, as sidekiq-cron does.
+    def reopen_in_forked_child
+      @options = @options.dup
+      @frozen = false
+      self
+    end
+
     def frozen?
       @frozen
     end
@@ -906,6 +992,18 @@ module Wurk
 
     private
 
+    # Pools read the hash once, when built, so the later assignment silently does
+    # nothing to this process's connections until something rebuilds them.
+    def warn_on_late_redis_assignment
+      return unless @capsules.each_value.any?(&:redis_pools_built?)
+
+      logger.warn do
+        'config.redis= called after a Redis pool was already built; existing connections keep the old ' \
+          'settings until the pools are rebuilt (a swarm child rebuilds them, a single process does not). ' \
+          'Set config.redis before anything uses Redis.'
+      end
+    end
+
     def warn_on_late_server_registration(klass)
       affected = @capsules.each_value.filter_map do |capsule|
         capsule.name if capsule.prepared? && capsule.server_middleware_missing?(klass)
@@ -1023,7 +1121,7 @@ module Wurk
       threads = total_concurrency
       return if size >= threads
 
-      logger.warn do
+      logger.error do
         "ActiveRecord connection pool size (#{size}) is smaller than this process's job concurrency " \
           "(#{threads}); jobs that use the database will wait for connections and may raise " \
           'ActiveRecord::ConnectionTimeoutError. Raise `pool:` in config/database.yml to at least ' \

data/lib/wurk/cron.rb

@@ -461,8 +461,53 @@ module Wurk
         loop_obj
       end
 
+      # Registers a sidekiq-cron shaped schedule: `{ name => { cron:, class:,
+      # queue:, args:, retry:, tz:, status: } }`, or a list of those carrying
+      # `name:`. Other keys (`description`, ...) are ignored. Raises ArgumentError
+      # on an entry with no cron or class, and on anything but a 5-field crontab.
+      def load_hash(schedule)
+        entries = schedule
+        entries = schedule.map { |name, entry| { 'name' => name }.merge(entry.to_h) } if schedule.is_a?(::Hash)
+        entries.map { |entry| register_entry(entry.to_h.transform_keys(&:to_s)) }
+      end
+
+      # The same shape from a YAML file (ERB allowed), like a `config/schedule.yml`.
+      def load_file(path)
+        require 'erb'
+        require 'yaml'
+        load_hash(::YAML.safe_load(::ERB.new(::File.read(path)).result, aliases: true) || {})
+      end
+
+      # Call after the last registration to make this block the whole truth:
+      # every other loop in the cluster is deleted. Off unless asked because a
+      # process registering only part of the schedule would delete the rest, and
+      # because during a rolling deploy old and new code then disagree.
+      def authoritative!
+        return self unless @prune
+
+        keep = @loops.map(&:lid)
+        LoopSet.new(@config).each { |existing| Cron.unregister(existing.lid) unless keep.include?(existing.lid) }
+        self
+      end
+
       private
 
+      def register_entry(entry)
+        cron = entry['cron'].to_s.strip
+        klass = (entry['class'] || entry['klass']).to_s
+        if cron.empty? || klass.empty?
+          raise ArgumentError, "schedule entry #{entry['name'].inspect} needs `cron` and `class`"
+        end
+
+        register(cron, klass, **entry_options(entry))
+      end
+
+      def entry_options(entry)
+        opts = { label: entry['name']&.to_s, queue: entry['queue'], args: entry['args'], retry: entry['retry'],
+                 tz: entry['tz'], paused: entry['status'] == 'disabled' ? true : nil }
+        opts.compact
+      end
+
       # Once per class per boot: everything else registered for that class is
       # a leftover from a previous deploy (the lid hashes schedule+options, so
       # an edited `register` line yields a new loop and orphans the old one).

data/lib/wurk/debounce.rb

@@ -81,8 +81,8 @@ module Wurk
         reject_shorter_cap!(seconds, cap)
 
         argv = [seconds, cap, JobUtil.scheduled_member(job), GRACE]
-        # Replay-safe: a re-run finds the member it just wrote, ZREMs it and
-        # ZADDs it back, so a pool retry after a lost reply converges on one
+        # Replay-safe: a re-run finds the member it just wrote and ZADDs it
+        # over itself, so a pool retry after a lost reply converges on one
         # entry rather than a duplicate — the whole point of doing this in a
         # script rather than a pipeline.
         #

data/lib/wurk/error_coalescer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Wurk
+  # Lets the first report of a key through, then one per window, and says how
+  # many it swallowed in between. Per process: every forked child speaks for
+  # itself, so a fleet still reports at worst children x keys per window.
+  class ErrorCoalescer
+    DEFAULT_WINDOW = 60
+
+    def initialize(window: DEFAULT_WINDOW, clock: -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) })
+      @window = window
+      @clock = clock
+      @lock = ::Mutex.new
+      @entries = {}
+    end
+
+    # The number of reports suppressed since the last one when this one should
+    # go out (0 for a first report); nil when this one should be swallowed.
+    def admit(key)
+      @lock.synchronize do
+        now = @clock.call
+        entry = (@entries[key] ||= [nil, 0])
+        if entry[0] && now - entry[0] < @window
+          entry[1] += 1
+          next nil
+        end
+
+        suppressed = entry[1]
+        entry[0] = now
+        entry[1] = 0
+        suppressed
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/at_maxmemory.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative 'oom_drain'
+
+module Wurk
+  class Fetcher
+    # {Fetcher::Reliable}'s fetch once Redis has refused one with OOM: a full
+    # Redis refuses LMOVE, which would stop every worker draining the queues that
+    # filled it. See OomDrain for what makes this possible. Only reached after an
+    # OutOfMemoryError, so it costs a healthy fetch nothing.
+    module AtMaxmemory
+      # Seconds between error reports while draining a full Redis. Every thread
+      # lands here on every fetch for as long as it lasts.
+      REPORT_INTERVAL = 60
+
+      private
+
+      # Costs the wall clock a BLMOVE would have on an empty pass, for the reason
+      # #retrieve_work's short-circuits do. An ACK is an LREM, which a full Redis
+      # still takes.
+      def claim_at_maxmemory(queues, error)
+        flush_pending_acks
+        report_maxmemory(error)
+        fetchable = fetchable_at_maxmemory(queues)
+        raise error if fetchable.empty?
+
+        index, job = claim_past_maxmemory(fetchable, error)
+        return unit_at_maxmemory(fetchable[index - 1], job) if job
+
+        sleep poll_interval
+        nil
+      end
+
+      # A pipeline applies everything it can before raising; the guard makes what
+      # already went back a no-op here.
+      def requeue_past_maxmemory(units, error)
+        config.redis(idempotent: true) do |conn|
+          units.each { |uow| OomDrain.requeue(conn, error, queue_keys(uow.queue).first, uow.queue, uow.job) }
+        end
+      end
+
+      def claim_past_maxmemory(fetchable, error)
+        keys = fetchable.flat_map { |public_q| [public_q, queue_keys(public_q).first] }
+        config.redis(idempotent: true) do |conn|
+          OomDrain.claim(conn, error, keys: keys, info: maxmemory_info, beat: ::Time.now.to_f.to_s)
+        end
+      end
+
+      def unit_at_maxmemory(public_q, job)
+        priv, name = queue_keys(public_q)
+        unit_of_work(public_q, priv, name, job)
+      end
+
+      # Capped overrides this: a cap cannot be honoured by a script that skips
+      # the ledger's write.
+      def fetchable_at_maxmemory(queues)
+        queues
+      end
+
+      # What the claim script writes if this process has no heartbeat hash (it
+      # booted into the full Redis). The first ordinary beat replaces it.
+      def maxmemory_info
+        Wurk.dump_json(
+          'hostname' => hostname, 'pid' => ::Process.pid, 'identity' => Component.identity,
+          'started_at' => ::Time.now.to_f, 'tag' => '', 'labels' => [], 'concurrency' => config.concurrency,
+          'queues' => config.queues.uniq, 'version' => Wurk::VERSION, 'embedded' => false
+        )
+      end
+
+      def report_maxmemory(error)
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        return if @oom_reported_at && now - @oom_reported_at < REPORT_INTERVAL
+
+        @oom_reported_at = now
+        handle_exception(error, { context: 'Redis is at maxmemory: draining with allow-oom scripts' })
+      end
+    end
+  end
+end

data/lib/wurk/fetcher/capped.rb

@@ -210,6 +210,10 @@ module Wurk
         blocked ? [poll_interval, CAPPED_BACKOFF].min : poll_interval
       end
 
+      def fetchable_at_maxmemory(queues)
+        queues.select { |public_q| gate_for(public_q).nil? }
+      end
+
       def gate_for(public_q)
         @gates[queue_keys(public_q)[1]]
       end

data/lib/wurk/fetcher/many_queues.rb

@@ -23,6 +23,16 @@ module Wurk
     module ManyQueues
       NO_ARGV = [].freeze
 
+      # How long a script pass that found every queue empty answers for the whole
+      # fetcher. The script sends two keys per queue, per thread, per poll: 120 KB
+      # a pass at 1000 queues, which at concurrency 500 cost a process 70% of a
+      # core and Redis 26% of one while idle. A thread skipping it parks in BLMOVE
+      # on the first queue for the whole poll, unaware of anything that lands on a
+      # later queue meanwhile, so a job can wait out a poll interval (about 2s)
+      # with a thread idle. That is the same bound an empty script pass gives it.
+      # Not a config knob.
+      REST_MISS_BACKOFF = 0.25
+
       private
 
       # Queues 2..n. Two queues is one more LMOVE — a one-key script costs more
@@ -34,11 +44,17 @@ module Wurk
       def claim_rest(queues)
         return lmove(queues[1]) if queues.size == 2
 
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        return nil if now < @rest_empty_until
+
         keys = claim_keys(queues)
         index, job = config.redis(idempotent: true) do |conn|
           Wurk::Lua::Loader.eval_cached(conn, :fetch_first, keys: keys, argv: NO_ARGV)
         end
-        return nil unless job
+        unless job
+          @rest_empty_until = now + REST_MISS_BACKOFF
+          return nil
+        end
 
         public_q = queues[index]
         priv, name = queue_keys(public_q)

data/lib/wurk/fetcher/oom_drain.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative '../keys'
+require_relative '../lua'
+
+module Wurk
+  class Fetcher
+    # The way out of a full Redis.
+    #
+    # LMOVE and ZADD are `denyoom` commands, so at `maxmemory` under the
+    # `noeviction` policy Wurk requires, every worker would be refused its next
+    # job and could therefore never free the memory that is refusing it. A
+    # Redis 7 script flagged `allow-oom` runs those commands anyway, and moving a
+    # job between two lists grows nothing but a small key.
+    #
+    # Deliberately not under lib/wurk/lua/: everything there is uploaded at boot,
+    # and Dragonfly refuses to compile a `#!lua` shebang, so one script in that
+    # directory would crash every child at boot. The same refusal is why these
+    # go by EVAL and not EVALSHA: a path taken only while Redis is full has no
+    # use for a NOSCRIPT recovery arm. A server that refuses the script raises
+    # the original OOM, which is exactly the behaviour without this class.
+    module OomDrain
+      HEADER = "#!lua flags=allow-oom\n"
+
+      # Heartbeat::TTL_SECONDS. Spelled out because the fetcher loads without the
+      # heartbeat, which pulls in the Processor.
+      HEARTBEAT_TTL = 60
+
+      # Claiming proves this process alive, and it has to say so itself: the
+      # heartbeat's own HSET is refused too. The hash then lapses (a booting
+      # process never gets one), other hosts' reapers drain the jobs it is
+      # running, and the swarm's liveness check sees its beat stand still.
+      #
+      # KEYS = [processes, identity, public, private, public, private, ...]
+      # ARGV = [info_json, beat, ttl_seconds]
+      # Returns {n, payload} for the n-th queue, nil when all were empty.
+      CLAIM = <<~LUA.freeze
+        #{HEADER}if redis.call('EXISTS', KEYS[2]) == 0 then
+          redis.call('HSET', KEYS[2], 'info', ARGV[1])
+          redis.call('SADD', KEYS[1], KEYS[2])
+        end
+        redis.call('HSET', KEYS[2], 'beat', ARGV[2])
+        redis.call('EXPIRE', KEYS[2], ARGV[3])
+        for i = 3, #KEYS, 2 do
+          local job = redis.call('LMOVE', KEYS[i], KEYS[i + 1], 'RIGHT', 'LEFT')
+          if job then
+            return { (i - 1) / 2, job }
+          end
+        end
+        return nil
+      LUA
+
+      RECLAIM = (HEADER + Lua::RELIABLE_RECLAIM).freeze
+      REQUEUE = (HEADER + Lua::RELIABLE_REQUEUE).freeze
+
+      # KEYS = [lock], ARGV = [ttl_ms]. The reaper's sweep lock: refused, no sweep
+      # would ever run past maxmemory. Same reply as the SET it stands in for.
+      LOCK = <<~LUA.freeze
+        #{HEADER}return redis.call('SET', KEYS[1], '1', 'NX', 'PX', ARGV[1])
+      LUA
+
+      # KEYS = [sorted_set], ARGV = [score, member]
+      ZADD = <<~LUA.freeze
+        #{HEADER}return redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2])
+      LUA
+
+      module_function
+
+      # `error` is the OOM that sent the caller here, raised again if the server
+      # cannot run the script. Returns [queue_index, payload] or nil.
+      def claim(conn, error, keys:, info:, beat:)
+        run(conn, error, CLAIM, [Keys::PROCESSES, Component.identity, *keys], [info, beat, HEARTBEAT_TTL])
+      end
+
+      def reclaim(conn, error, source, destination)
+        run(conn, error, RECLAIM, [source, destination], [])
+      end
+
+      # Returns 1 when moved, 0 when the job has left the private list since.
+      def requeue(conn, error, private_list, public_q, job)
+        run(conn, error, REQUEUE, [private_list, public_q], [job])
+      end
+
+      def lock(conn, error, key, ttl_ms)
+        run(conn, error, LOCK, [key], [ttl_ms])
+      end
+
+      def zadd(conn, error, key, score, member)
+        run(conn, error, ZADD, [key], [score, member])
+      end
+
+      def run(conn, error, script, keys, argv)
+        conn.call('EVAL', script, keys.size, *keys, *argv)
+      rescue RedisClient::ConnectionError
+        raise
+      rescue RedisClient::CommandError
+        raise error
+      end
+      private_class_method :run
+    end
+  end
+end