kicks_liveness 0.1.0

data/lib/kicks_liveness/heartbeat.rb

@@ -0,0 +1,178 @@
+module KicksLiveness
+  # The liveness mark on the filesystem: written by the worker, read by the
+  # probe.
+  #
+  # The directory must live on tmpfs — in Kubernetes, an emptyDir with
+  # <tt>medium: Memory</tt>. Put it on a real disk and the probe starts
+  # depending on the disk again, and a stalling disk is one of the most common
+  # causes of false restarts.
+  #
+  # This file deliberately runs no +require+ on the probe's path and refers to
+  # nothing else in the gem: the probe can load it alone, with no application
+  # code and no Rails behind it. That keeps the optional direct form at ~43 ms
+  # and lets it run on a bare interpreter with +--disable-gems+ when the image
+  # allows it -- whether Bundler is in the picture depends on where the image
+  # put its gems, not on this file. The one +require+ it does contain is lazy,
+  # in a branch only the worker reaches.
+  #
+  # @see file:docs/DESIGN.md#why-the-heartbeat-file-has-no-require-of-its-own
+  class Heartbeat
+    # @return [String] marks directory used when the environment says nothing
+    DEFAULT_DIR = '/opt/app/tmp/health'.freeze
+    # @return [Integer] seconds after which a mark is stale, by default
+    DEFAULT_MAX_AGE = 45
+
+    # @return [Hash{Symbol => String}] setting name to environment variable
+    ENV_NAMES = {
+      dir: 'KICKS_LIVENESS_DIR',
+      max_age: 'KICKS_LIVENESS_MAX_AGE',
+      tick: 'KICKS_LIVENESS_TICK'
+    }.freeze
+
+    # An empty string counts as unset: in a ConfigMap that is what you get by
+    # declaring a key and leaving it blank.
+    #
+    # @param key [Symbol] one of the keys of {ENV_NAMES}
+    # @return [String, nil]
+    def self.env_raw(key)
+      value = ENV.fetch(ENV_NAMES.fetch(key), nil)
+      value unless value.nil? || value.empty?
+    end
+
+    # Values arrive from a ConfigMap, and a typo there is no reason to bring a
+    # worker down — garbage falls back to the default.
+    #
+    # Both settings are durations, so a value has to be parseable *and*
+    # positive. Zero and negative numbers parse perfectly well and are the more
+    # dangerous half: a tick of zero turns the monitor into a hot loop, a
+    # negative one used to kill the monitor thread on its first sleep, and a
+    # negative max_age makes every mark stale on arrival, so the probe can never
+    # pass again.
+    #
+    # @param key [Symbol] one of the keys of {ENV_NAMES}
+    # @param default [Integer]
+    # @return [Integer] the value from the environment when it is a positive
+    #   integer, the default otherwise
+    def self.env_int(key, default)
+      value = Integer(env_raw(key) || default)
+      value.positive? ? value : default
+    rescue ArgumentError, TypeError
+      default
+    end
+
+    # @return [String] marks directory
+    def self.env_dir
+      env_raw(:dir) || DEFAULT_DIR
+    end
+
+    # @return [Integer] seconds after which a mark is considered stale
+    def self.env_max_age
+      env_int(:max_age, DEFAULT_MAX_AGE)
+    end
+
+    # @param dir [String] marks directory
+    # @param max_age [Integer] seconds after which a mark is considered stale
+    def initialize(dir: Heartbeat.env_dir, max_age: Heartbeat.env_max_age)
+      @dir = dir
+      @max_age = max_age
+    end
+
+    attr_reader :dir, :max_age
+
+    # Records how many forks the probe must wait for. Every fork writes the same
+    # value. Without it the probe would pass as soon as any single mark was
+    # fresh, while half the workers had not subscribed yet.
+    #
+    # Written through a rename, because a plain write truncates first: a probe
+    # reading in that window finds the file empty and reports that the worker
+    # has not started. The window is real and recurring — every fork rewrites
+    # this file on every tick, while the liveness probe reads it on a schedule
+    # of its own. Rename is atomic on tmpfs.
+    #
+    # @param processes [Integer]
+    # @return [void]
+    def declare!(processes)
+      make_dir
+      # The pid keeps concurrent forks from sharing the temporary file.
+      tmp = "#{expected_path}.#{Process.pid}"
+      File.write(tmp, processes)
+      File.rename(tmp, expected_path)
+    end
+
+    # Refreshes this fork's mark.
+    #
+    # The file is named by supervisor slot, not by PID: a fork killed with
+    # SIGKILL is respawned into the same slot and overwrites its own file. With a
+    # PID in the name that file would stay stale forever and the probe would fail
+    # permanently.
+    #
+    # The contents exist only for a human running <tt>kubectl exec ... cat</tt>;
+    # the probe decides on mtime alone.
+    #
+    # @param slot [Integer] supervisor slot of this fork
+    # @return [Integer] bytes written
+    def touch!(slot)
+      make_dir
+      File.write(slot_path(slot), "#{Time.now.utc.strftime('%FT%TZ')} pid=#{Process.pid} slot=#{slot}\n")
+    end
+
+    # The probe side: is every declared fork's mark present and fresh?
+    #
+    # The returned message names slots the way the files are named, so that a
+    # human reading the Unhealthy event knows which file to look at.
+    #
+    # @param now [Time] injected in specs
+    # @return [Array(Boolean, String)] health, and the reason to print on stdout
+    def check(now: Time.now.utc)
+      processes = expected
+      return [false, "no #{expected_path}: worker has not started yet"] unless processes&.positive?
+
+      problems = (0...processes).filter_map do |slot|
+        age = age_of(slot_path(slot), now)
+        next "worker-#{slot} missing" if age.nil?
+
+        "worker-#{slot} stale #{age.round}s > #{@max_age}s" if age > @max_age
+      end
+
+      problems.empty? ? [true, "#{processes} process(es) healthy"] : [false, problems.join('; ')]
+    end
+
+    private
+
+    # One syscall on the happy path, and whichever fork gets there first wins.
+    #
+    # +Dir.mkdir+ creates a single level and raises ENOENT when the parent is
+    # missing, which any nested KICKS_LIVENESS_DIR hits. FileUtils is required
+    # here rather than at the top of the file: only the worker ever creates the
+    # directory, and the probe's path through {#check} must stay free of
+    # requires.
+    def make_dir
+      Dir.mkdir(@dir)
+    rescue Errno::EEXIST
+      nil
+    rescue Errno::ENOENT
+      require 'fileutils'
+      FileUtils.mkdir_p(@dir)
+    end
+
+    def expected_path
+      File.join(@dir, 'expected')
+    end
+
+    def slot_path(slot)
+      File.join(@dir, "worker-#{slot}")
+    end
+
+    def expected
+      Integer(File.read(expected_path))
+    rescue StandardError
+      nil
+    end
+
+    def age_of(path, now)
+      now - File.mtime(path)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/kicks_liveness/hooks.rb

@@ -0,0 +1,93 @@
+module KicksLiveness
+  # The two modules prepended into the worker gem. Installed together by
+  # {KicksLiveness.install!}.
+  #
+  # @see file:docs/DESIGN.md#where-the-hooks-attach
+  # @api private
+  module Hooks
+    # Prepended to +Sneakers::Worker+. The worker is registered only after a
+    # successful subscribe: if +subscribe+ raised, it never reaches the registry
+    # and the probe honestly fails.
+    # @api private
+    module Worker
+      # @return [void]
+      def run
+        super
+        Registry.add(self)
+      end
+
+      # @return [void]
+      def stop
+        Registry.remove(self)
+        super
+      end
+    end
+
+    # Prepended to +Sneakers::WorkerGroup+.
+    #
+    # +ServerEngine::Server#create_worker+ calls +w.extend(worker_module)+, so
+    # +WorkerGroup+ lands in the singleton class of the instance rather than
+    # being included in a class. A prepend to the module still sits ahead of it
+    # in that lookup chain; +worker_id+ and +config+ come from
+    # +ServerEngine::Worker+.
+    #
+    # @see file:docs/DESIGN.md#where-the-hooks-attach
+    # @api private
+    module WorkerGroup
+      # Starts the monitor from +after_fork+ rather than from +run+.
+      #
+      # +Sneakers::WorkerGroup#run+ calls +after_fork+ as its very first
+      # statement and only afterwards resolves +worker_classes+, so this is the
+      # earliest point in the fork at which the application's own +after_fork+
+      # hook has already run. That ordering matters because resolving the
+      # expected consumer count may run application code: under
+      # sneakers:active_job the set is a callable registry, and the standard use
+      # of +after_fork+ is to re-establish the connections a fork inherited.
+      # Resolving it first would run that code against a fork that is not ready.
+      #
+      # Wrapped in a rescue: instrumentation has no right to prevent a worker
+      # from starting, so the worst outcome is a missing mark and a pod restart
+      # rather than a pod that stays up with silent queues.
+      #
+      # @return [void]
+      def after_fork
+        super
+
+        begin
+          KicksLiveness.start!(
+            slot: worker_id,
+            processes: config[:workers] || 1,
+            consumers: kicks_liveness_expected_consumers
+          )
+        rescue StandardError => e
+          KicksLiveness.config.resolved_logger&.error("[liveness] failed to start: #{e.class}: #{e.message}")
+        end
+      end
+
+      # Marks the shutdown as deliberate before the workers unsubscribe.
+      #
+      # +Sneakers::WorkerGroup#stop+ calls +stop+ on each worker, and
+      # +Sneakers::Worker#stop+ waits for the thread pool to drain, which takes
+      # as long as the longest in-flight job. The registry empties at the start
+      # of that, so without this flag the monitor would report a fault on every
+      # ordinary deploy.
+      #
+      # @return [void]
+      def stop
+        Registry.stopping!
+        super
+      end
+
+      private
+
+      # The same set the worker gem itself builds its workers from, so the
+      # queue list is never duplicated and cannot drift. An array of classes
+      # under sneakers:run, a callable registry under sneakers:active_job.
+      def kicks_liveness_expected_consumers
+        classes = config[:worker_classes]
+        classes = classes.call if classes.respond_to?(:call)
+        classes.size
+      end
+    end
+  end
+end

data/lib/kicks_liveness/monitor.rb

@@ -0,0 +1,178 @@
+module KicksLiveness
+  # The thread that publishes the liveness mark. Created inside the fork,
+  # because threads do not survive +fork+.
+  #
+  # @see file:docs/DESIGN.md#logging-events-not-the-pulse
+  # @api private
+  class Monitor
+    # @param slot [Integer] supervisor slot of this fork; names the mark file
+    # @param processes [Integer] how many forks the probe must wait for
+    # @param consumers [Integer] how many workers must subscribe in this process
+    # @param config [Configuration]
+    # @param heartbeat [Heartbeat, nil] injected in specs; {Attempts} follows its
+    #   directory, so injecting it separately is not needed
+    def initialize(slot:, processes:, consumers:, config:, heartbeat: nil)
+      @slot = slot
+      @processes = processes
+      @consumers = consumers
+      @config = config
+      @heartbeat = heartbeat || Heartbeat.new
+      @attempts = Attempts.new(@heartbeat.dir)
+    end
+
+    # Declares the expected fork count and starts the tick thread.
+    # @return [Thread]
+    def start!
+      attempt, elapsed = @attempts.record!(@slot)
+      @heartbeat.declare!(@processes)
+      attempt > 1 ? report_respawn(attempt, elapsed) : report_first_start
+
+      Thread.new do
+        Thread.current.name = 'kicks-liveness'
+        run_loop
+      end
+    end
+
+    # A single step: check, mark if healthy, log any transition. Public so that
+    # specs do not have to drive the thread.
+    #
+    # @return [Boolean] whether the process is healthy at this moment
+    def tick!
+      # Re-declared on every tick, not only at startup. This file is the probe's
+      # only source for how many forks to expect, and nothing else restores it:
+      # if the directory is wiped, `touch!` brings back the slot marks while
+      # `expected` stays missing, and the probe reports "worker has not started"
+      # for the rest of the pod's life. It also lets a respawned set of forks
+      # correct the count after the supervisor was told to run fewer of them.
+      @heartbeat.declare!(@processes)
+
+      if Registry.stopping?
+        shutdown_tick!
+        return true
+      end
+
+      healthy = Registry.healthy?(@consumers)
+      if healthy
+        @heartbeat.touch!(@slot)
+        clear_attempt_once!
+      end
+      @unhealthy_ticks = healthy ? 0 : unhealthy_ticks + 1
+      report(healthy)
+      healthy
+    end
+
+    private
+
+    def report_first_start
+      log(:info, "started: dir=#{@heartbeat.dir} max_age=#{@heartbeat.max_age}s tick=#{@config.tick}s " \
+                 "processes=#{@processes} consumers=#{@consumers}")
+    end
+
+    # A slot that starts again without ever having become healthy is a respawn
+    # loop: the supervisor brings the fork back after `start_worker_delay`,
+    # which is a fraction of a second, so the process dies long before any
+    # single monitor's grace period expires. Two things follow.
+    #
+    # The start line and the first transition are suppressed. Seeding
+    # `@previous` is what does the second part: without it the first tick reads
+    # a nil previous state and logs `waiting for N consumers`, which in a storm
+    # means hundreds of identical INFO lines a minute and no diagnosis in any of
+    # them.
+    #
+    # The escalation is driven off the marks directory instead of off
+    # `unhealthy_ticks`, because that counter dies with the fork and can never
+    # reach the threshold. Reporting once per grace window rather than once per
+    # respawn is the difference between one line a minute and five a second.
+    def report_respawn(attempt, elapsed)
+      @previous = false
+
+      return if elapsed < grace_seconds
+
+      log(:error, "not healthy #{elapsed.round}s over #{attempt} starts, expected #{@consumers} consumers")
+      @attempts.restart_window!(@slot)
+    end
+
+    def grace_seconds
+      @config.startup_grace_ticks * interval
+    end
+
+    # Once the slot is healthy the respawn counter has done its job, and the
+    # next start of this slot deserves to be treated as a fresh one.
+    def clear_attempt_once!
+      return if @attempt_cleared
+
+      @attempts.clear!(@slot)
+      @attempt_cleared = true
+    end
+
+    def unhealthy_ticks
+      @unhealthy_ticks ||= 0
+    end
+
+    # From the moment shutdown begins the consumer count says nothing about
+    # health: the workers unsubscribe one by one while the pool drains, and the
+    # process is doing exactly what it was told. So the predicate is dropped and
+    # the mark is kept fresh, which leaves a slow drain the whole of
+    # terminationGracePeriodSeconds instead of having the probe cut it short at
+    # max_age.
+    def shutdown_tick!
+      @heartbeat.touch!(@slot)
+
+      return if @shutdown_reported
+
+      log(:info, 'shutting down: consumers are no longer checked')
+      @shutdown_reported = true
+    end
+
+    # Nothing inside this loop may be allowed to kill the thread. It is the only
+    # thing writing the mark, and its death is close to invisible: the exception
+    # surfaces as a bare backtrace on stderr, never through the gem's logger,
+    # while the pod restarts on every probe budget with nothing in the
+    # application log to explain it. `sleep` used to sit outside the guarded
+    # block, which is exactly how a non-positive tick killed the monitor.
+    def run_loop
+      loop do
+        tick!
+        sleep interval
+      rescue StandardError => e
+        log(:error, "monitor loop failed: #{e.class}: #{e.message}")
+        sleep Configuration::DEFAULT_TICK
+      end
+    end
+
+    # Values coming from the environment are already clamped by
+    # {Heartbeat.env_int}; this covers a tick set programmatically.
+    def interval
+      tick = @config.tick
+      tick.is_a?(Numeric) && tick.positive? ? tick : Configuration::DEFAULT_TICK
+    end
+
+    # Events are logged, not the pulse — the pulse lives in the mark's mtime.
+    # ERROR is reserved for genuine failure, because a logger usually comes up
+    # with LOG_LEVEL defaulting to error.
+    def report(healthy)
+      if healthy != @previous
+        report_transition(healthy)
+        @previous = healthy
+      elsif unhealthy_ticks == @config.startup_grace_ticks
+        # Startup took longer than normal, which is a failure by now. Staying
+        # silent would leave a pod that never comes up saying nothing about it.
+        log(:error, "not healthy #{unhealthy_ticks * @config.tick}s, expected #{@consumers} consumers")
+      end
+    end
+
+    def report_transition(healthy)
+      if healthy
+        log(:info, 'healthy')
+      elsif @previous.nil?
+        log(:info, "waiting for #{@consumers} consumers")
+      else
+        log(:error, "became unhealthy (expected #{@consumers} consumers)")
+      end
+    end
+
+    def log(level, message)
+      @config.resolved_logger&.public_send(level, "[liveness] slot #{@slot}: #{message}")
+    end
+  end
+end

data/lib/kicks_liveness/probe.rb

@@ -0,0 +1,16 @@
+# Probe entry point. Requiring this file performs the check and terminates the
+# process, so do not require it from an application.
+#
+# In a container manifest:
+#   command: ["bundle", "exec", "kicks-liveness"]
+#
+# The gem namespace is deliberately not loaded: this pulls in one
+# dependency-free file. Images that install gems in +GEM_HOME+ can invoke this
+# file directly as a faster, optional form; see
+# docs/KUBERNETES.md#where-your-image-puts-its-gems.
+require_relative 'heartbeat'
+
+ok, message = KicksLiveness::Heartbeat.new.check
+
+puts message
+exit(ok ? 0 : 1)

data/lib/kicks_liveness/railtie.rb

@@ -0,0 +1,18 @@
+require 'rails/railtie'
+
+module KicksLiveness
+  # Installs the hooks by itself, so a Rails application only needs the
+  # configuration block.
+  #
+  # +to_prepare+ rather than an initializer: an application's lib is normally
+  # managed by Zeitwerk and reloadable, and such constants must not be
+  # referenced while the application is initialising.
+  #
+  # @see file:docs/SETUP.md#rails
+  # @api private
+  class Railtie < ::Rails::Railtie
+    initializer 'kicks_liveness.install' do |app|
+      app.config.to_prepare { KicksLiveness.install! }
+    end
+  end
+end

data/lib/kicks_liveness/registry.rb

@@ -0,0 +1,102 @@
+module KicksLiveness
+  # Registry of the workers that have subscribed in the current process, and the
+  # health predicate computed over them.
+  #
+  # The predicate reads only in-process state: no network calls, no disk access.
+  # @see file:docs/DESIGN.md#the-health-predicate
+  # @api private
+  module Registry
+    # Initialised at file-load time: single-threaded, and before any fork.
+    @workers = []
+    @mutex = Mutex.new
+    @stopping = false
+
+    class << self
+      # Records a worker as subscribed.
+      #
+      # Registering the same object twice would break the size comparison in
+      # {healthy?} and leave the process permanently unhealthy, so identity is
+      # checked first. +equal?+ rather than +==+ on purpose: this is a registry
+      # of objects, and a worker class is free to define equality however it
+      # likes.
+      #
+      # @param worker [Sneakers::Worker]
+      # @return [Array] the registry
+      def add(worker)
+        @mutex.synchronize do
+          @workers << worker unless @workers.any? { |registered| registered.equal?(worker) }
+          @workers
+        end
+      end
+
+      # Drops a worker from the registry, on shutdown.
+      # @param worker [Sneakers::Worker]
+      # @return [Sneakers::Worker, nil]
+      def remove(worker)
+        @mutex.synchronize { @workers.delete(worker) }
+      end
+
+      # @return [Integer] how many workers are currently registered
+      def size
+        @mutex.synchronize { @workers.size }
+      end
+
+      # Records that this process is shutting down on purpose.
+      #
+      # Without it a graceful shutdown is indistinguishable from a failure: the
+      # registry empties as each worker unsubscribes, so the monitor would see
+      # an incomplete set and report a genuine fault on every deploy.
+      #
+      # @return [true]
+      def stopping!
+        @stopping = true
+      end
+
+      # @return [Boolean] whether shutdown has begun
+      def stopping?
+        @stopping
+      end
+
+      # Whether this process is consuming everything it is supposed to consume.
+      #
+      # The registry size is compared against +expected+ rather than only
+      # checking the workers that did register: four healthy workers out of five
+      # look fine one object at a time.
+      #
+      # @param expected [Integer] how many workers must have subscribed in this process
+      # @return [Boolean]
+      def healthy?(expected)
+        workers = @mutex.synchronize { @workers.dup }
+        return false unless expected.positive? && workers.size == expected
+
+        workers.all? { |worker| alive?(worker) }
+      end
+
+      private
+
+      def alive?(worker)
+        channel = worker.queue.channel
+        # No subscription yet; the startup probe is what holds the pod.
+        return false if channel.nil?
+
+        connection = channel.connection
+        return true if recovering?(connection)
+
+        # Connection open but no consumers: the worker silently stopped
+        # consuming, and a restart is the only cure.
+        channel.open? && connection.open? && channel.any_consumers?
+      rescue StandardError
+        false
+      end
+
+      # Bunny marks this method @private, so an upgrade may remove it. The
+      # guard keeps that from turning every tick into NoMethodError -> rescued
+      # -> unhealthy; the cost is that the recovery exemption would then vanish
+      # silently.
+      def recovering?(connection)
+        connection.respond_to?(:recovering_from_network_failure?) &&
+          connection.recovering_from_network_failure?
+      end
+    end
+  end
+end

data/lib/kicks_liveness/version.rb

@@ -0,0 +1,4 @@
+module KicksLiveness
+  # @return [String] gem version
+  VERSION = '0.1.0'.freeze
+end

data/lib/kicks_liveness.rb

@@ -0,0 +1,86 @@
+require_relative 'kicks_liveness/version'
+require_relative 'kicks_liveness/heartbeat'
+require_relative 'kicks_liveness/attempts'
+require_relative 'kicks_liveness/configuration'
+require_relative 'kicks_liveness/registry'
+require_relative 'kicks_liveness/monitor'
+require_relative 'kicks_liveness/hooks'
+
+# Liveness probe for Kicks and Sneakers workers, backed by a tmpfs heartbeat.
+#
+# The worker publishes a mark from inside its own process, checking its Bunny
+# consumers in memory; the probe reads only the mark's mtime. No Rails, no call
+# to the broker.
+#
+# @see file:docs/SETUP.md
+# @see file:docs/DESIGN.md
+module KicksLiveness
+  class << self
+    # @return [Configuration] the process-wide configuration
+    def config
+      @config ||= Configuration.new
+    end
+
+    # Yields the configuration for the application to adjust.
+    #
+    # @example
+    #   KicksLiveness.configure do |config|
+    #     config.enabled = !Rails.env.local?
+    #   end
+    #
+    # @yieldparam config [Configuration]
+    # @return [Configuration]
+    def configure
+      yield(config)
+      config
+    end
+
+    # Installs the two hooks the gem works through. Must run before the runner
+    # starts; in Rails a Railtie does it. Idempotent — prepending the same
+    # module twice is a no-op.
+    #
+    # Two requires, not one: +Sneakers::Worker+ is defined by lib/sneakers.rb,
+    # while +Sneakers::WorkerGroup+ comes only with sneakers/workergroup, which
+    # ships with the runner and is absent in web processes. Relying on the
+    # application having required sneakers itself is not safe: with
+    # <tt>gem 'kicks', require: false</tt> the to_prepare hook runs earlier and
+    # would fail on NameError.
+    #
+    # @raise [LoadError] if neither +kicks+ nor +sneakers+ is available
+    # @return [Module]
+    # @see file:docs/SETUP.md#installing-the-hooks
+    def install!
+      begin
+        require 'sneakers'
+        require 'sneakers/workergroup'
+      rescue LoadError => e
+        # The original message is kept: this rescue fires just as readily when
+        # the worker gem is installed but something inside it fails to load — a
+        # bunny or serverengine version that cannot be required, an extension
+        # not built for the image. Reporting that as "the gem is missing" sends
+        # the reader to check a Gemfile.lock that is perfectly fine.
+        raise LoadError, 'kicks_liveness requires the kicks (>= 3.0) or sneakers (>= 2.11) gem ' \
+                         "(#{e.message})"
+      end
+
+      ::Sneakers::Worker.prepend(Hooks::Worker)
+      ::Sneakers::WorkerGroup.prepend(Hooks::WorkerGroup)
+    end
+
+    # Starts the monitor thread. Called from the WorkerGroup hook, already
+    # inside the fork.
+    #
+    # @param slot [Integer] supervisor slot of this fork
+    # @param processes [Integer] how many forks the probe must wait for
+    # @param consumers [Integer] how many workers must subscribe here
+    # @return [Thread, nil] nil when disabled by configuration
+    # @api private
+    def start!(slot:, processes:, consumers:)
+      return unless config.enabled?
+
+      Monitor.new(slot: slot, processes: processes, consumers: consumers, config: config).start!
+    end
+  end
+end
+
+require_relative 'kicks_liveness/railtie' if defined?(Rails::Railtie)