cronwatch 0.3.1

data/lib/cronwatch/evaluate.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  # Pure decisions about a job's health. Each function takes the current state
+  # and returns the new state plus the alerts that should go out. Nothing here
+  # touches a store or a network, which is what makes it testable.
+  module Evaluate
+    DEFAULT_GRACE_MS = 10 * 60_000
+    DEFAULT_TIMEOUT_MS = 60 * 60_000
+    # Runs faster than this are never called slow, whatever the baseline says.
+    SLOW_FLOOR_MS = 10_000
+    # How many earlier runs a baseline needs before it is trusted.
+    BASELINE_MIN_RUNS = 5
+    # How many successful runs a baseline looks at, and how many runs a summary covers.
+    BASELINE_WINDOW = 20
+
+    Evaluation = Struct.new(:state, :alerts, keyword_init: true)
+    CheckEvaluation = Struct.new(:state, :alerts, :next_expected_at, :due_at, keyword_init: true)
+    SlowThreshold = Struct.new(:threshold_ms, :basis, keyword_init: true)
+
+    module_function
+
+    def empty_state(job)
+      JobState.new(job: job, open: {}, consecutive_failures: 0, silenced_until: nil, last_alert_at: nil,
+                   pending_recovery: [], undelivered: [])
+    end
+
+    # A stored state with every field present, or a fresh one. State written by an older version lacks the newer fields.
+    def normalize_state(state, job)
+      return empty_state(job) if state.nil?
+
+      state = JobState.from_h(state)
+      JobState.new(
+        job: state.job.nil? ? job : state.job,
+        open: (state.open || {}).dup,
+        consecutive_failures: state.consecutive_failures.nil? ? 0 : state.consecutive_failures,
+        silenced_until: state.silenced_until,
+        last_alert_at: state.last_alert_at,
+        pending_recovery: (state.pending_recovery || []).dup,
+        undelivered: (state.undelivered || []).dup,
+        version: state.version,
+      )
+    end
+
+    def clone_state(state)
+      normalize_state(state, state.job)
+    end
+
+    def open_condition(state, condition, now)
+      return false if state.open.key?(condition)
+
+      state.open[condition] = now
+      true
+    end
+
+    # Every open condition has alerted, so closing one owes a recovered message.
+    # It is remembered until a successful run leaves nothing open and sends it.
+    def close_condition(state, condition)
+      return false unless state.open.key?(condition)
+
+      state.open.delete(condition)
+      state.pending_recovery ||= []
+      state.pending_recovery << condition unless state.pending_recovery.include?(condition)
+      true
+    end
+
+    def open_conditions(state)
+      state.open.keys
+    end
+
+    def grace_ms(definition)
+      definition.grace.nil? ? DEFAULT_GRACE_MS : Duration.parse(definition.grace, "grace")
+    end
+
+    def timeout_ms(definition)
+      definition.timeout.nil? ? DEFAULT_TIMEOUT_MS : Duration.parse(definition.timeout, "timeout")
+    end
+
+    # Slow threshold for a successful run, or nil when there is nothing to compare against yet.
+    def slow_threshold(definition, history)
+      unless definition.max_duration.nil?
+        return SlowThreshold.new(threshold_ms: Duration.parse(definition.max_duration, "maxDuration"), basis: "maxDuration")
+      end
+
+      durations = history.select { |r| r.status == :ok && !r.duration_ms.nil? }.first(BASELINE_WINDOW).map(&:duration_ms)
+      return nil if durations.length < BASELINE_MIN_RUNS
+
+      p95 = Stats.percentile(durations, 95)
+      SlowThreshold.new(
+        threshold_ms: [2 * p95, SLOW_FLOOR_MS].max,
+        basis: "twice the p95 of the last #{durations.length} runs (#{Duration.format(p95)})",
+      )
+    end
+
+    def budget_breaches(definition, run, history)
+      breaches = []
+      budget = definition.budget
+      metrics = run.metrics || {}
+      JS.object_keys(metrics).each do |metric|
+        value = metrics[metric]
+        ceiling = budget&.[](metric.to_s)
+        unless ceiling.nil?
+          breaches << { metric: metric.to_s, value: value, limit: ceiling, basis: "budget" } if value > ceiling
+          next
+        end
+        past = history.select { |r| r.status == :ok && (r.metrics || {})[metric].is_a?(Numeric) }
+                      .first(BASELINE_WINDOW).map { |r| r.metrics[metric] }
+        next if past.length < BASELINE_MIN_RUNS
+
+        usual = Stats.median(past)
+        if usual.positive? && value > 3 * usual
+          breaches << { metric: metric.to_s, value: value, limit: 3 * usual, basis: "three times the usual #{format_number(usual)}" }
+        end
+      end
+      breaches
+    end
+
+    # Whether `history` (newest first) holds a full baseline window of successful runs.
+    def full_baseline?(history)
+      history.count { |r| r.status == :ok } >= BASELINE_WINDOW
+    end
+
+    # toLocaleString("en-US"): digit groups, and a fraction rounded half up to
+    # at most four places. Worked on the shortest decimal digits, as ICU does.
+    def format_number(n)
+      return (n.negative? ? "-" : "") + (n.nan? ? "NaN" : "\u221e") if n.is_a?(Float) && !n.finite?
+
+      negative = n.negative? || (n.is_a?(Float) && n.zero? && (1.0 / n).negative?)
+      if n.is_a?(Integer)
+        whole = n.abs.to_s
+        fraction = ""
+      else
+        digits, point = JS.decimal(n.abs.to_f)
+        if point >= digits.length
+          whole = digits + ("0" * (point - digits.length))
+          fraction = ""
+        elsif point.positive?
+          whole = digits[0, point]
+          fraction = digits[point..]
+        else
+          whole = "0"
+          fraction = ("0" * -point) + digits
+        end
+        if fraction.length > 4
+          up = fraction[4].to_i >= 5
+          fraction = fraction[0, 4]
+          if up
+            rounded = ((whole + fraction).to_i + 1).to_s.rjust(whole.length + 4, "0")
+            whole = rounded[0...-4]
+            fraction = rounded[-4..]
+          end
+        end
+        fraction = fraction.sub(/0+\z/, "")
+      end
+      whole = whole.reverse.scan(/\d{1,3}/).join(",").reverse
+      text = fraction.empty? ? whole : "#{whole}.#{fraction}"
+      negative ? "-#{text}" : text
+    end
+
+    # Called when a run starts. Missed and stuck are about the absence of a run,
+    # so a run starting closes them without an alert; the recovered message
+    # waits for a successful finish.
+    def on_run_start(state)
+      next_state = clone_state(state)
+      close_condition(next_state, :missed)
+      close_condition(next_state, :stuck)
+      next_state
+    end
+
+    # Called when a run finishes with status ok, failed or timeout. `history` is
+    # the job's earlier runs, newest first, not including this one.
+    def on_run_finish(definition, run, state, history, now)
+      next_state = clone_state(state)
+      alerts = []
+
+      if run.status == :ok
+        next_state.consecutive_failures = 0
+        close_condition(next_state, :missed)
+        close_condition(next_state, :stuck)
+        close_condition(next_state, :failed)
+
+        slow = slow_threshold(definition, history)
+        if slow && !run.duration_ms.nil? && run.duration_ms > slow.threshold_ms
+          if open_condition(next_state, :slow, now)
+            alerts << AlertDraft.new(type: :slow, run: run,
+                                     details: { duration_ms: run.duration_ms, threshold_ms: slow.threshold_ms, basis: slow.basis })
+          end
+        else
+          close_condition(next_state, :slow)
+        end
+
+        breaches = budget_breaches(definition, run, history)
+        if breaches.any?
+          alerts << AlertDraft.new(type: :over_budget, run: run, details: { breaches: breaches }) if open_condition(next_state, :over_budget, now)
+        else
+          close_condition(next_state, :over_budget)
+        end
+
+        pending = next_state.pending_recovery || []
+        if pending.any? && open_conditions(next_state).empty?
+          alerts << AlertDraft.new(type: :recovered, run: run, details: { after: pending.dup })
+          next_state.pending_recovery = []
+        end
+        return Evaluation.new(state: next_state, alerts: alerts)
+      end
+
+      # failed or timeout
+      next_state.consecutive_failures += 1
+      close_condition(next_state, :missed)
+      threshold = [1, definition.failures_before_alert || 1].max
+      condition = run.status == :timeout ? :stuck : :failed
+      if next_state.consecutive_failures >= threshold && open_condition(next_state, condition, now)
+        alerts << AlertDraft.new(type: condition, run: run,
+                                 details: { consecutive_failures: next_state.consecutive_failures, threshold: threshold })
+      end
+      Evaluation.new(state: next_state, alerts: alerts)
+    end
+
+    # Called by check. Decides whether the schedule has been missed: the run
+    # the schedule wants next (see Schedule.expectation) has not started and its
+    # grace has run out. `last_run` is the most recent run of any status.
+    def on_check(definition, stored, last_run, state, now)
+      next_state = clone_state(state)
+      alerts = []
+      schedule = definition.schedule
+      if schedule.nil? || schedule == ""
+        return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: nil, due_at: nil)
+      end
+
+      parsed = Schedule.parse(schedule, definition.timezone)
+      grace = grace_ms(definition)
+      last_run_at = last_run&.started_at
+      exp = Schedule.expectation(parsed, last_run_at, stored.created_at, grace)
+      next_expected_at =
+        if parsed.interval? then Schedule.next_fire(parsed, stored.created_at, last_run_at)
+        else Schedule.next_fire(parsed, now, nil)
+        end
+      return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: nil) unless exp
+
+      # An interval's next run is due a period after the last one started. If that
+      # run is still going, the job is busy, not late; stuck covers one that never ends.
+      if parsed.interval? && last_run&.status == :running
+        return CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: exp.due_at)
+      end
+
+      if now > exp.deadline
+        if open_condition(next_state, :missed, now)
+          alerts << AlertDraft.new(type: :missed, run: last_run,
+                                   details: { due_at: exp.due_at, deadline: exp.deadline, grace_ms: grace, last_run_at: last_run_at })
+        end
+      else
+        # A run has started since it opened, or the grace was widened.
+        close_condition(next_state, :missed)
+      end
+      CheckEvaluation.new(state: next_state, alerts: alerts, next_expected_at: next_expected_at, due_at: exp.due_at)
+    end
+
+    # Whether a running run has gone on longer than the job's timeout.
+    def stuck?(definition, run, now)
+      run.status == :running && now - run.started_at > timeout_ms(definition)
+    end
+
+    # While a job is silenced nothing new is recorded as an incident: conditions
+    # may close (so a job that recovered during the silence shows as healthy) but
+    # none may open, so the first problem after the silence ends alerts normally.
+    def mute_opens(previous, next_state)
+      muted = clone_state(next_state)
+      open_conditions(muted).each { |condition| muted.open.delete(condition) unless previous.open.key?(condition) }
+      muted
+    end
+
+    def silenced?(state, now)
+      !state.silenced_until.nil? && state.silenced_until > now
+    end
+
+    # An evaluation as it is saved and sent: while the job was silenced when it
+    # began, nothing opens and nothing is sent.
+    def apply_silence(previous, evaluation, now)
+      return evaluation unless silenced?(previous, now)
+
+      Evaluation.new(state: mute_opens(previous, evaluation.state), alerts: [])
+    end
+
+    # Whether an alert waiting to be retried no longer describes the job, so it
+    # is dropped rather than sent late. An alert for a condition is stale once
+    # that condition has closed, or has closed and opened again (it opened at a
+    # time other than the alert's). A recovery is stale when any condition it
+    # names is open again; while they all stay closed it is kept.
+    def stale_alert?(alert, state)
+      return Array(alert.details[:after]).any? { |condition| state.open.key?(condition.to_sym) } if alert.type == :recovered
+
+      state.open[alert.type] != alert.at
+    end
+
+    # How a job looks at a glance. Silence wins, then stuck, failing and late.
+    def job_health(definition, last_run, state, now)
+      open = open_conditions(state)
+      return :silenced if silenced?(state, now)
+      return :stuck if open.include?(:stuck) || (last_run && stuck?(definition, last_run, now))
+      return :failing if open.include?(:failed) || %i[failed timeout].include?(last_run&.status)
+      return :late if open.include?(:missed)
+      return :never_ran if last_run.nil?
+
+      :healthy
+    end
+
+    # A job's summary from its most recent runs (newest first; the first
+    # BASELINE_WINDOW are used) and its state. Stats cover runs of any status;
+    # the percentiles are over the successful ones among them.
+    def summarize(stored, recent, state, next_expected_at, now)
+      summary(stored, recent, state, next_expected_at) { |last_run| job_health(stored.definition, last_run, state, now) }
+    end
+
+    # The summary of a job that could not be evaluated, say because its stored
+    # schedule no longer parses. It reads nothing from the definition. The job
+    # shows as failing (or silenced, while it is), since it needs a look, and
+    # nothing is known about when it is next due.
+    def unevaluable_summary(stored, recent, state, now)
+      summary(stored, recent, state, nil) { silenced?(state, now) ? :silenced : :failing }
+    end
+
+    def summary(stored, recent, state, next_expected_at)
+      window = recent.first(BASELINE_WINDOW)
+      last_run = window.first
+      finished = window.reject { |r| r.status == :running }
+      ok_durations = window.select { |r| r.status == :ok && !r.duration_ms.nil? }.map(&:duration_ms)
+      JobSummary.new(
+        name: stored.name,
+        definition: stored.definition,
+        health: yield(last_run),
+        open: open_conditions(state),
+        last_run: last_run,
+        next_expected_at: next_expected_at,
+        consecutive_failures: state.consecutive_failures,
+        silenced_until: state.silenced_until,
+        stats: JobStats.new(
+          runs: finished.length,
+          ok_rate: finished.empty? ? 1 : finished.count { |r| r.status == :ok }.fdiv(finished.length),
+          p50_ms: Stats.percentile(ok_durations, 50),
+          p95_ms: Stats.percentile(ok_durations, 95),
+        ),
+      )
+    end
+  end
+end

data/lib/cronwatch/flight.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  class Client
+    # One shared check: the first caller runs it, the others wait for its result.
+    class Flight
+      def initialize
+        @lock = Mutex.new
+        @done = ConditionVariable.new
+        @finished = false
+      end
+
+      def resolve(value)
+        settle(value, nil)
+      end
+
+      def reject(error)
+        settle(nil, error)
+      end
+
+      def value
+        @lock.synchronize do
+          @done.wait(@lock) until @finished
+          raise @error if @error
+
+          @value
+        end
+      end
+
+      private
+
+      def settle(value, error)
+        @lock.synchronize do
+          @value = value
+          @error = error
+          @finished = true
+          @done.broadcast
+        end
+      end
+    end
+  end
+end

data/lib/cronwatch/format.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  module Format
+    NAMED = /\A[A-Za-z_$][A-Za-z0-9_$]*: /
+
+    module_function
+
+    def at_time(at, now)
+      return "never" if at.nil?
+
+      "#{JS.iso(at).tr("T", " ")[0, 19]} UTC (#{Duration.relative(at, now)})"
+    end
+
+    def first_lines(text, n)
+      return "" if text.nil? || text.empty?
+
+      text.split("\n", -1).first(n).join("\n")
+    end
+
+    def tail(text, n)
+      return "" if text.nil? || text.empty?
+
+      lines = JS.trim_end(text).split("\n", -1)
+      lines.last(n).join("\n")
+    end
+
+    # "Error: x" for a bare message, but not "Error: TypeError: x" for one that already names itself.
+    def error_line(error)
+      text = first_lines(error, 4)
+      NAMED.match?(text) ? text : "Error: #{text}"
+    end
+
+    # Turns a draft into the title and message every channel shows.
+    def compose_alert(draft, definition, now)
+      name = definition.name
+      run = draft.run
+      details = draft.details
+      lines = []
+
+      title =
+        case draft.type
+        when :missed
+          lines << "Due #{at_time(details[:due_at], now)}, and no run had started by #{at_time(details[:deadline], now)} (grace #{Duration.format(details[:grace_ms])})."
+          lines << "Schedule: #{definition.schedule.nil? ? "undefined" : definition.schedule}#{definition.timezone.to_s.empty? ? "" : " (#{definition.timezone})"}."
+          lines << "Last run: #{run ? "#{run.status} #{at_time(run.started_at, now)}" : "never"}."
+          "#{name} missed its scheduled run"
+        when :failed
+          n = details[:consecutive_failures]
+          lines << "#{n} consecutive failures." if n > 1
+          if run
+            lines << "Started #{at_time(run.started_at, now)}#{run.duration_ms.nil? ? "" : ", ran #{Duration.format(run.duration_ms)}"}."
+            lines << error_line(run.error) if run.error && !run.error.empty?
+            out = tail(run.output, 8)
+            lines << "Output (tail):\n#{out}" unless out.empty?
+          end
+          "#{name} failed"
+        when :stuck
+          if run
+            lines << "Started #{at_time(run.started_at, now)} and never reported finishing. Marked as timed out after #{Duration.format(run.duration_ms.nil? ? now - run.started_at : run.duration_ms)}."
+            out = tail(run.output, 8)
+            lines << "Output so far (tail):\n#{out}" unless out.empty?
+          end
+          lines << "If the process was killed mid-run (a serverless timeout, a deploy), this is what that looks like."
+          "#{name} is stuck"
+        when :slow
+          lines << "Took #{Duration.format(details[:duration_ms])}; the limit is #{Duration.format(details[:threshold_ms])} (#{details[:basis]})."
+          lines << "Started #{at_time(run.started_at, now)}." if run
+          "#{name} was slow"
+        when :over_budget
+          details[:breaches].each do |b|
+            lines << "#{b[:metric]}: #{Evaluate.format_number(b[:value])}, limit #{Evaluate.format_number(b[:limit])} (#{b[:basis]})."
+          end
+          lines << "Started #{at_time(run.started_at, now)}." if run
+          "#{name} went over budget"
+        when :recovered
+          after = details[:after].map { |c| c.to_s.sub("_", " ") }.join(", ")
+          lines << "A run #{run ? at_time(run.started_at, now) : "just now"} succeeded#{after.empty? ? "" : " after: #{after}"}."
+          lines << "Ran #{Duration.format(run.duration_ms)}." if run && !run.duration_ms.nil?
+          "#{name} recovered"
+        else
+          raise ArgumentError, "unknown alert type #{draft.type.inspect}"
+        end
+
+      Alert.new(type: draft.type, run: run, details: details, job: name, definition: definition, title: title,
+                message: lines.join("\n"), at: now)
+    end
+  end
+end

data/lib/cronwatch/http.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Cronwatch
+  # POSTs for the alert channels. Anything with `post(url, body, headers)`
+  # returning a Response can stand in for it, which is how the tests run.
+  module HTTP
+    TIMEOUT = 10
+
+    Response = Struct.new(:status, :body, keyword_init: true) do
+      def ok? = status >= 200 && status < 300
+    end
+
+    class NetHTTP
+      def post(url, body, headers)
+        uri = URI(url)
+        request = Net::HTTP::Post.new(uri.request_uri)
+        headers.each { |k, v| request[k] = v }
+        request.body = body
+        response = connection(uri).request(request)
+        Response.new(status: response.code.to_i, body: response.body.to_s)
+      end
+
+      # A connection that gives up after ten seconds, as the SDK's requests do.
+      def connection(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = TIMEOUT
+        http.read_timeout = TIMEOUT
+        http.write_timeout = TIMEOUT
+        http
+      end
+    end
+
+    module_function
+
+    def default
+      @default ||= NetHTTP.new
+    end
+
+    # Compares two secrets without stopping at the first differing character.
+    def constant_time_equal?(a, b)
+      return false unless a.bytesize == b.bytesize
+
+      diff = 0
+      a.bytes.zip(b.bytes) { |x, y| diff |= x ^ y }
+      diff.zero?
+    end
+  end
+end

data/lib/cronwatch/job.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative "abort_signal"
+
+module Cronwatch
+  # What a job's block receives.
+  class JobContext
+    attr_reader :name, :run_id, :started_at, :signal
+
+    def initialize(run, signal, recorder)
+      @name = run.job
+      @run_id = run.id
+      @started_at = run.started_at
+      @signal = signal
+      @recorder = recorder
+    end
+
+    # Append a line of output. Kept with the run, capped at 16 KB, shown in alerts and the dashboard.
+    def log(*parts)
+      @recorder.log(parts.map { |part| JobContext.stringify(part) }.join(" "))
+      nil
+    end
+
+    # Report a number for this run: tokens, cost, rows, anything. Watched against budgets and baselines.
+    def metric(name, value)
+      unless value.is_a?(Numeric) && value.real? && JS.finite?(value)
+        raise ArgumentError, "metric \"#{name}\" must be a finite number"
+      end
+
+      @recorder.metric(Output.utf8(name), value)
+      nil
+    end
+
+    def metrics(values = nil, **more)
+      (values || {}).merge(more).each { |k, v| metric(k, v) }
+      nil
+    end
+
+    # True once the job's timeout has passed. Honour it if the work can stop.
+    def aborted?
+      @signal.aborted?
+    end
+
+    # A logged value as text, the way the SDK writes it, always valid UTF-8.
+    def self.stringify(part)
+      text =
+        case part
+        when String then part
+        when Symbol then part.to_s
+        when Exception then "#{part.class.name || part.class}: #{Output.utf8(part.message)}"
+        when Hash, Array, Numeric, true, false, nil, Struct then JS.json(part)
+        else part.to_s
+        end
+      Output.utf8(text)
+    end
+  end
+
+  # A declared job. Keep it, and call #run with a block.
+  class JobHandle
+    attr_reader :name, :definition
+
+    def initialize(client, definition)
+      @client = client
+      @definition = definition
+      @name = definition.name
+    end
+
+    # Run the block now, recording the run. Returns what the block returns and
+    # re-raises what it raises, after the run is recorded.
+    def run(trigger: "run", &block)
+      raise ArgumentError, "run needs a block" unless block
+
+      outcome = @client.execute(@definition, trigger, &block)
+      raise outcome.error if outcome.threw
+
+      outcome.result
+    end
+  end
+
+  # Collects a run's output and metrics while its block runs.
+  class RunRecorder
+    # Lines are dropped from the front once the output is well past the cap;
+    # Output.cap trims it exactly at the end.
+    KEEP = 64 * 1024
+
+    attr_reader :context, :signal
+
+    def initialize(run, timeout_ms)
+      @lines = []
+      @size = 0
+      # The first lines logged, up to the cap, and whether any line has been
+      # dropped from @lines: what expect_text needs once the output runs long.
+      @head = []
+      @head_size = 0
+      @dropped = false
+      @metrics = {}
+      @lock = Mutex.new
+      @signal = AbortSignal.new(timeout_ms)
+      @context = JobContext.new(run, @signal, self)
+    end
+
+    def log(line)
+      length = JS.length16(line)
+      @lock.synchronize do
+        if @head_size < Output::CAP
+          @head << line
+          @head_size += length + 1
+        end
+        @lines << line
+        @size += length + 1
+        while @size > KEEP && @lines.length > 1
+          @size -= JS.length16(@lines.shift) + 1
+          @dropped = true
+        end
+      end
+    end
+
+    def metric(name, value)
+      @lock.synchronize { @metrics[name] = value }
+    end
+
+    def output
+      @lock.synchronize { @lines.empty? ? nil : Output.cap(@lines.join("\n")) }
+    end
+
+    # What an expect rule is checked against: everything logged, or when that
+    # ran long, the first 16 KB and the last 16 KB. The stored output keeps
+    # only the tail, so a "done" line printed early would otherwise be lost.
+    def expect_text
+      @lock.synchronize do
+        next nil if @lines.empty?
+
+        all = @lines.join("\n")
+        next all if !@dropped && JS.length16(all) <= 2 * Output::CAP
+
+        "#{JS.head16(@head.join("\n"), Output::CAP)}\n#{JS.tail16(all, Output::CAP)}"
+      end
+    end
+
+    def metrics
+      @lock.synchronize { @metrics.dup }
+    end
+  end
+end