cronwatch 0.3.1 → 0.5.0

data/lib/cronwatch/run_handle.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module Cronwatch
+  # A run recorded by JobHandle#start or found by JobHandle#resume, to finish
+  # later, perhaps in another process. Lines and metrics wait in the handle
+  # until #flush or #finish. Store failures go to on_error; none of these
+  # methods raises for them.
+  #
+  #   run = SYNC.start(id: event_id)
+  #   # later, perhaps elsewhere
+  #   run = SYNC.resume(event_id)
+  #   run.log("sent 40 emails")
+  #   run.finish                  # or run.fail(error)
+  class RunHandle
+    # Raised by the client's side of finish when the store failed part way
+    # and nothing was recorded (already reported): the handle stays active.
+    #
+    # @api private
+    class Retry < StandardError; end
+
+    # `started_at` is nil when a resumed run could not be read.
+    attr_reader :id, :job, :started_at
+
+    # Made by the client (Client#run_handle), not by apps. `finish` and
+    # `flush` are the client's side of each; `inactive` says why finish has
+    # nothing to do, or is nil.
+    #
+    # @api private
+    def initialize(id:, job:, started_at:, inactive:, finish:, flush:, ignored:)
+      @id = id
+      @job = job
+      @started_at = started_at
+      @inactive = inactive
+      @finish = finish
+      @flush = flush
+      @ignored = ignored
+      # @state guards the flags and which recorder is current; @turn keeps
+      # flush and finish in order, one at a time.
+      @state = Mutex.new
+      @turn = Mutex.new
+      @recorder = fresh_recorder
+      @finished = !inactive.nil?
+      @finish_called = false
+      # The first Output::CAP characters of every line flushed from this
+      # handle, unredacted, or nil before the first flush. The stored output
+      # keeps only the tail, so without it an expect rule at finish would
+      # miss a line logged early, which run would have seen.
+      @head = nil
+    end
+
+    # False once finished, and from the start for a resumed run that already
+    # finished or does not exist.
+    def active?
+      @state.synchronize { !@finished }
+    end
+
+    # Add a line of output, as JobContext#log does. Kept in the handle until flush or finish.
+    def log(*parts)
+      @state.synchronize { @recorder.context.log(*parts) }
+    end
+
+    # Report a number for this run. A later value for the same name replaces an earlier one.
+    def metric(name, value)
+      @state.synchronize { @recorder.context.metric(name, value) }
+    end
+
+    def metrics(values = nil, **more)
+      @state.synchronize { @recorder.context.metrics(values, **more) }
+    end
+
+    # Append the lines and metrics added so far to the stored run, which must
+    # still be running and belong to this job. Output is redacted as it is
+    # written. A read, change and write of the run's row, written only while
+    # it is still running: two processes appending to one run at the same
+    # moment can lose one's lines, but a flush never undoes a finish. When
+    # the write fails the lines stay here for finish. The first 16 KB of
+    # everything flushed stay in the handle, so an expect rule at finish
+    # sees an early line as run would.
+    def flush
+      @turn.synchronize do
+        next if @flush.nil? || @state.synchronize { @finished }
+
+        # Lines logged while this waits on the store go to a new recorder.
+        taken = @state.synchronize do
+          current = @recorder
+          @recorder = fresh_recorder
+          current
+        end
+        lines = taken.output
+        values = taken.metrics
+        next if lines.nil? && values.empty?
+
+        if @flush.call(lines, values)
+          keep_head(taken.expect_text)
+        else
+          put_back(taken)
+        end
+      end
+      nil
+    end
+
+    # Finish the run, judge it like any other and send what that produces.
+    #
+    #   finish                        # ok
+    #   finish(status: "ok")          # ok
+    #   finish(error: e)              # failed, recorded like an error run caught
+    #   finish("text")                # like run's return value: the output when
+    #   finish(result: "text")        # nothing was logged, checked by expect
+    #
+    # Returns the run as recorded, or nil when nothing was: the run was
+    # already finished (here or elsewhere), was not found, or belongs to
+    # another job, which is reported to on_error. When several processes
+    # finish one run, only the one whose write lands judges it. A store that
+    # fails is reported, nothing is recorded, and the handle stays active so
+    # finish can be called again.
+    def finish(outcome = nil)
+      was_inactive = nil
+      again = @state.synchronize do
+        next true if @finish_called
+
+        @finish_called = true
+        was_inactive = @finished
+        @finished = true
+        false
+      end
+      if again
+        @ignored.call("was already finished by this handle")
+        return nil
+      end
+
+      @turn.synchronize do
+        if was_inactive
+          @ignored.call(@inactive)
+          next nil
+        end
+
+        begin
+          @finish.call(@state.synchronize { @recorder }, outcome, @head)
+        rescue Retry
+          reopen
+          nil
+        rescue Exception # rubocop:disable Lint/RescueException
+          # An Interrupt or Timeout mid-finish: the run may still be running,
+          # so the handle stays open (lines kept) for finish to be called again.
+          reopen
+          raise
+        end
+      end
+    end
+
+    # finish(error: error).
+    def fail(error)
+      finish({ error: error })
+    end
+
+    # [failed, result, error] from what finish was given. A Hash with an
+    # :error key is a failure; a String, or a Hash's :result, is the result.
+    #
+    # @api private
+    def self.read_outcome(outcome)
+      case outcome
+      when String then [false, outcome, nil]
+      when Hash
+        key = [:error, "error"].find { |k| outcome.key?(k) }
+        return [true, nil, outcome[key]] if key
+
+        [false, outcome.key?(:result) ? outcome[:result] : outcome["result"], nil]
+      else [false, nil, nil]
+      end
+    end
+
+    private
+
+    def fresh_recorder
+      RunRecorder.new(Run.new(id: @id, job: @job, started_at: @started_at || 0), nil)
+    end
+
+    # A flush that could not write: its lines go back ahead of any logged since.
+    def put_back(taken)
+      @state.synchronize do
+        later = @recorder
+        @recorder = fresh_recorder
+        [taken.expect_text, later.expect_text].each { |text| @recorder.log(text) unless text.nil? }
+        taken.metrics.merge(later.metrics).each { |name, value| @recorder.metric(name, value) }
+      end
+    end
+
+    # A finish that recorded nothing leaves the handle active, to be finished again.
+    def reopen
+      @state.synchronize do
+        @finish_called = false
+        @finished = false
+      end
+    end
+
+    # Keeps the start of what a flush wrote, up to the cap, for expect at finish.
+    def keep_head(text)
+      return if text.nil? || (!@head.nil? && JS.length16(@head) >= Output::CAP)
+
+      @head = JS.head16(@head.nil? || @head.empty? ? text : "#{@head}\n#{text}", Output::CAP)
+    end
+  end
+end

data/lib/cronwatch/schedule.rb

@@ -93,6 +93,34 @@ module Cronwatch
       nil
     end
 
+    # Every fire of a cron strictly after `from` and at or before `to`,
+    # ascending, or nil when there are more than `limit`. Walks the fires in
+    # batches, as firesBetween in schedule.ts asks croner for them, and drops
+    # any that do not move forward (see fire_after).
+    def fires_between(parsed, from, to, limit)
+      raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
+
+      out = []
+      probe = from
+      last = from
+      1000.times do
+        batch = next_runs(parsed, [limit + 1 - out.length, 24].min, probe)
+        return out if batch.empty?
+
+        batch.each do |t|
+          next if t <= last
+          return out if t > to
+
+          out << t
+          last = t
+          return nil if out.length > limit
+        end
+        finish = batch.last
+        probe = finish > probe ? finish : probe + 3_600_000
+      end
+      out
+    end
+
     # Up to `count` fires, each found from the one before, as croner's nextRuns.
     def next_runs(parsed, count, from)
       runs = []

data/lib/cronwatch/stores/active_record.rb

@@ -216,6 +216,18 @@ module Cronwatch
         nil
       end
 
+      # update_run, only while the stored status is one of `from_statuses`,
+      # in one statement (sql.ts updateRunIfSql). Returns whether it wrote.
+      def update_run_if(run, from_statuses)
+        statuses = Array(from_statuses).map(&:to_s)
+        return false if statuses.empty?
+
+        binds = [run.status.to_s, run.finished_at, run.duration_ms, run.error, run.output,
+                 JS.json(run.metrics || {}), run.id, *statuses]
+        changed = with_connection { |conn| conn.exec_update(update_run_if_sql(conn, statuses.length), NAME, binds) }
+        changed.to_i.positive?
+      end
+
       def get_run(id)
         row = read(:get_run, [id]).first
         row && row_to_run(row)
@@ -313,6 +325,17 @@ module Cronwatch
         (@statements[dialect] ||= statements(dialect)).fetch(key)
       end
 
+      # sql.ts `updateRunIfSql`: update_run's statement, only while the
+      # stored status is one of `count` statuses, bound value by value.
+      def update_run_if_sql(conn, count)
+        text = "UPDATE #{@prefix}runs SET status = ?, finished_at = ?, duration_ms = ?, error = ?, output = ?, metrics = ? " \
+               "WHERE id = ? AND status IN (#{Array.new(count, "?").join(", ")})"
+        return text unless self.class.dialect(conn) == :postgres
+
+        n = 0
+        text.gsub("?") { "$#{n += 1}" }
+      end
+
       # sql.ts `statements`: the same text, with `?` numbered for Postgres.
       def statements(dialect)
         p = @prefix

data/lib/cronwatch/stores/memory.rb

@@ -7,7 +7,8 @@ module Cronwatch
     # so a missed run cannot be noticed across one.
     #
     # Every store answers the same methods: init (optional), upsert_job,
-    # get_job, list_jobs, delete_job, insert_run, update_run, get_run,
+    # get_job, list_jobs, delete_job, insert_run, update_run, update_run_if
+    # (optional: without it the client reads the run, then writes it), get_run,
     # list_runs, last_run, running_runs, get_state, set_state, compare_and_set_state
     # (optional: without it the client falls back to set_state), prune and close
     # (optional). They take and return the types in types.rb.
@@ -56,8 +57,11 @@ module Cronwatch
         nil
       end
 
+      # Like SQL's primary key: an id already recorded is refused, never overwritten.
       def insert_run(run)
         sync do
+          raise "run #{run.id} already exists" if @runs.key?(run.id)
+
           @runs[run.id] = clone(run, Run)
           @order[run.id] = (@seq += 1)
         end
@@ -70,19 +74,25 @@ module Cronwatch
           existing = @runs[run.id]
           next unless existing
 
-          copy = clone(run, Run)
-          @runs[run.id] = existing.dup.tap do |r|
-            r.status = copy.status
-            r.finished_at = copy.finished_at
-            r.duration_ms = copy.duration_ms
-            r.error = copy.error
-            r.output = copy.output
-            r.metrics = copy.metrics
-          end
+          @runs[run.id] = finished_fields(existing, run)
         end
         nil
       end
 
+      # update_run, only while the stored run's status is one of
+      # `from_statuses`, in one step. Returns whether it wrote. What lets
+      # exactly one of several processes finishing a run evaluate it.
+      def update_run_if(run, from_statuses)
+        statuses = Array(from_statuses).map(&:to_sym)
+        sync do
+          existing = @runs[run.id]
+          next false unless existing && statuses.include?(existing.status)
+
+          @runs[run.id] = finished_fields(existing, run)
+          true
+        end
+      end
+
       def get_run(id)
         sync { (run = @runs[id]) && clone(run, Run) }
       end
@@ -149,6 +159,19 @@ module Cronwatch
 
       private
 
+      # `existing` with the fields an update writes taken from `run`.
+      def finished_fields(existing, run)
+        copy = clone(run, Run)
+        existing.dup.tap do |r|
+          r.status = copy.status
+          r.finished_at = copy.finished_at
+          r.duration_ms = copy.duration_ms
+          r.error = copy.error
+          r.output = copy.output
+          r.metrics = copy.metrics
+        end
+      end
+
       def sync(&block)
         @lock.synchronize(&block)
       end

data/lib/cronwatch/types.rb

@@ -226,6 +226,7 @@ module Cronwatch
       run = Naming.fetch(hash, "run")
       details = Naming.from_json_value(Naming.fetch(hash, "details") || {})
       details[:after] = details[:after].map(&:to_sym) if details[:after].is_a?(Array)
+      details[:reason] = details[:reason].to_sym if details[:reason].is_a?(String)
       alert = new(
         type: Naming.fetch(hash, "type")&.to_sym,
         run: run && Run.from_h(run),

data/lib/cronwatch/version.rb

@@ -2,5 +2,5 @@
 
 module Cronwatch
   # Kept in step with @cronwatch/sdk.
-  VERSION = "0.3.1"
+  VERSION = "0.5.0"
 end

data/lib/cronwatch/web/app.rb

@@ -3,6 +3,7 @@
 require "base64"
 require "digest"
 require "securerandom"
+require "uri"
 
 module Cronwatch
   # The dashboard and the small JSON API, as a Rack app: the SDK's routes
@@ -26,16 +27,37 @@ module Cronwatch
   #            client's cron_secret as a bearer, for a platform cron.
   # base_path: where the app is mounted, so links resolve. Defaults to the
   #            mount point (SCRIPT_NAME), which is right under Rails' `mount`.
+  # origin:    the public origin the dashboard is served from, such as
+  #            "https://app.example.com". By default the request's own origin
+  #            is used, as Rack reads it (Rack::Request#base_url, which follows
+  #            X-Forwarded-Proto and X-Forwarded-Host the way Rails does). Set
+  #            this to pin it, for an app behind a proxy that does not set
+  #            those headers or one that should not trust them. Used in place
+  #            of the request's origin for the cross-site check on writes, the
+  #            sign-in cookie's Secure flag, the Referer the redirect back after
+  #            a form follows, and the development sign-in line. Read as
+  #            `new URL(value).origin` reads it (Web::Origin) and normalised to
+  #            scheme://host[:port]; anything that is not an absolute http or
+  #            https URL, or has a port outside 1 to 65535, raises
+  #            ArgumentError here. "" counts as unset. Behind more than one
+  #            proxy set it: Rack takes the last of several forwarded values.
   class Web
     # Tells "token not given" (read CRONWATCH_TOKEN) from "token: nil" (open on purpose).
     UNSET = Object.new.freeze
     COOKIE = "cronwatch_token"
     DEFAULT_RUNS = 20
     MAX_RUNS = 500
+    # Runs per job the board reads in one go: the table's sparkline, and most jobs' lanes.
+    BOARD_PAGE_RUNS = 20
     COOKIE_MAX_AGE = 60 * 60 * 24 * 30
 
-    CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src 'self' data:; form-action 'self'; " \
-          "frame-ancestors 'none'; base-uri 'none'"
+    # 'self' only for what the app shell needs: app.js (which registers the
+    # service worker and nothing else), the manifest, the worker and the icons.
+    # No inline script, and the pages work without any.
+    CSP = "default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; img-src 'self' data:; " \
+          "manifest-src 'self'; worker-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'"
+    # For the SVG icons, should one be opened on its own.
+    ASSET_CSP = "default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'"
     # same-origin rather than no-referrer: under no-referrer browsers send
     # `Origin: null` on form posts, which the CSRF check would refuse, and the
     # forms redirect back to the page named by the same-origin Referer.
@@ -46,8 +68,9 @@ module Cronwatch
     DECIMAL = /\A[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?\z/
     RADIX = { "x" => 16, "o" => 8, "b" => 2 }.freeze
 
-    def initialize(client = nil, token: UNSET, base_path: nil)
+    def initialize(client = nil, token: UNSET, base_path: nil, origin: nil)
       @client = client
+      @origin = Web.configured_origin(origin)
       @opted_out = token.nil?
       given = token.equal?(UNSET) ? nil : token
       @token = @opted_out ? nil : [given, ENV.fetch("CRONWATCH_TOKEN", nil)].map(&:to_s).find { |t| !t.empty? }
@@ -77,6 +100,14 @@ module Cronwatch
         "Sign in: #{origin}#{base}/?token=#{token}"
     end
 
+    # The `origin` option, normalised to scheme://host[:port], or nil when
+    # unset. Raises on a value that is not an http or https origin, so a typo
+    # fails at startup.
+    # Read as the SDK's `new URL(value).origin` reads it (Web::Origin).
+    def self.configured_origin(value)
+      Origin.parse(value)
+    end
+
     # The client given, or Cronwatch.client when none was.
     def client
       @client || Cronwatch.client
@@ -112,6 +143,18 @@ module Cronwatch
 
       announce(request, base) if @generated && !@announced
 
+      # The app shell: the manifest, icons, service worker, app.js and the
+      # offline page. Served to anyone, since a browser fetches some of it
+      # without cookies and none of it says anything about the jobs.
+      if %w[GET HEAD].include?(method)
+        if path == "/offline"
+          return html(HTML.message_page("You are offline", "CronWatch shows live data from your app, so it needs a connection.", base), 200, "no-cache")
+        end
+
+        asset = PWA.asset(path, base)
+        return shell(asset, base) if asset
+      end
+
       # No token outside development: fail closed.
       if !@token && !@opted_out
         return wants_html ? html(HTML.message_page("CronWatch routes are locked", "Set CRONWATCH_TOKEN (or pass token: to Cronwatch::Web.new), or pass token: nil to serve them open behind your own auth.", base), 503) : api({ ok: false, error: "CRONWATCH_TOKEN is not set" }, 503)
@@ -136,13 +179,13 @@ module Cronwatch
           end
         unless cron_secret_ok || token_ok
           if @generated
-            return wants_html ? html(HTML.message_page("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401)
+            return wants_html ? html(HTML.message_page("Sign in", "CRONWATCH_TOKEN is not set, so this development server made a token. The sign-in link is in the server log: open it once and this browser stays signed in.", base, sign_in: true), 401) : api({ ok: false, error: "Unauthorized: CRONWATCH_TOKEN is not set, so this development server made a token; it is in the server log" }, 401)
           end
-          return wants_html ? html(HTML.message_page("Sign in", "Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.", base), 401) : api({ ok: false, error: "Unauthorized" }, 401)
+          return wants_html ? html(HTML.message_page("Sign in", "Open this page with ?token=<your CRONWATCH_TOKEN> once and it will stay signed in.", base, sign_in: true), 401) : api({ ok: false, error: "Unauthorized" }, 401)
         end
         unless query.nil?
           # Move the token from the URL into a cookie so it is not in history or logs.
-          secure = request.https? ? "; Secure" : ""
+          secure = public_origin(request).start_with?("https:") ? "; Secure" : ""
           return redirect(request.pathname + request.search_without("token"),
                           "set-cookie" => "#{COOKIE}=#{cookie_value(@token)}; Path=#{base.empty? ? "/" : base}; HttpOnly; SameSite=Lax; Max-Age=#{COOKIE_MAX_AGE}#{secure}")
         end
@@ -155,15 +198,20 @@ module Cronwatch
 
       # HTML
       if method == "GET" && path == "/"
-        entries = cw.jobs_with_runs(20)
+        entries = cw.jobs_with_runs(BOARD_PAGE_RUNS)
+        now = cw.now
         runs_by_job = entries.to_h { |entry| [entry.job.name, entry.runs] }
-        return html(HTML.dashboard_page(entries.map(&:job), runs_by_job, cw.now, base, nil))
+        return html(HTML.dashboard_page(entries.map(&:job), runs_by_job, now, base, nil, board_lanes(cw, entries, now)))
       end
       if method == "GET" && parts[0] == "jobs" && parts.length == 2
         job = cw.job_summary(parts[1])
         return html(HTML.message_page("No such job", "#{parts[1]} is not in the store.", base), 404) unless job
 
-        return html(HTML.job_page(job, cw.runs(job.name, 50), cw.now, base))
+        now = cw.now
+        # Enough runs to draw the job's week; the page lists the newest fifty.
+        limit = Timeline.week_runs_limit(job, now)
+        runs = cw.runs(job.name, limit)
+        return html(HTML.job_page(job, runs, now, base, runs.length < limit))
       end
       if method == "POST" && path == "/check"
         cw.check
@@ -246,6 +294,22 @@ module Cronwatch
       api({ ok: false, error: "Not found" }, 404)
     end
 
+    # The board's timeline lanes, the first Timeline::BOARD_LANES jobs. The
+    # runs already read for the table usually cover the last day; only a job
+    # whose twenty newest runs all fall inside it (one that runs more often
+    # than every hour or so) is read again, deeper.
+    def board_lanes(cw, entries, now)
+      from = now - Timeline::BOARD_BEHIND_MS
+      entries.first(Timeline::BOARD_LANES).map do |entry|
+        runs = entry.runs
+        short = runs.length >= BOARD_PAGE_RUNS && runs.last.started_at > from
+        next Timeline::LaneInput.new(job: entry.job, runs: runs, complete: true) unless short
+
+        deeper = cw.runs(entry.job.name, Timeline::BOARD_RUNS)
+        Timeline::LaneInput.new(job: entry.job, runs: deeper, complete: deeper.length < Timeline::BOARD_RUNS)
+      end
+    end
+
     # Prints the development sign-in link, once per app.
     def announce(request, base)
       first = @announce_lock.synchronize do
@@ -255,7 +319,7 @@ module Cronwatch
       end
       return unless first
 
-      $stdout.puts(Web.development_sign_in_line(request.origin, base, @token))
+      $stdout.puts(Web.development_sign_in_line(public_origin(request), base, @token))
       $stdout.flush
     end
 
@@ -295,7 +359,7 @@ module Cronwatch
     # a page cannot forge either. Non-browser clients send neither.
     def cross_site?(request)
       origin = request.header("origin")
-      return true if !origin.nil? && origin != request.origin
+      return true if !origin.nil? && origin != public_origin(request)
 
       site = request.header("sec-fetch-site")
       !site.nil? && site != "same-origin" && site != "none"
@@ -378,7 +442,12 @@ module Cronwatch
 
     def redirect_back(request, base)
       referer = request.header("referer") || ""
-      redirect(referer.start_with?("#{request.origin}/") ? referer : "#{base}/")
+      redirect(referer.start_with?("#{public_origin(request)}/") ? referer : "#{base}/")
+    end
+
+    # The origin a browser sees: the `origin` option when set, otherwise the request's own.
+    def public_origin(request)
+      @origin || request.origin
     end
 
     def api(body, status = 200, headers = {})
@@ -393,9 +462,19 @@ module Cronwatch
       [303, { "location" => location, "cache-control" => "no-store", **SECURITY_HEADERS, **headers }, []]
     end
 
-    def html(body, status = 200)
+    # An app shell file. The worker may be scoped to the base (it is served
+    # from there anyway); the SVGs get a CSP of their own.
+    def shell(asset, base)
+      headers = { "content-type" => asset.type, "cache-control" => asset.cache, **SECURITY_HEADERS }
+      headers["content-security-policy"] = ASSET_CSP if asset.type == "image/svg+xml"
+      headers["service-worker-allowed"] = "#{base}/" if asset.worker
+      headers["content-length"] = asset.body.bytesize.to_s
+      [200, headers, [asset.body]]
+    end
+
+    def html(body, status = 200, cache = "no-store")
       [status, {
-        "content-type" => "text/html; charset=utf-8", "cache-control" => "no-store",
+        "content-type" => "text/html; charset=utf-8", "cache-control" => cache,
         "content-security-policy" => CSP, "x-frame-options" => "DENY", "content-length" => body.bytesize.to_s,
         **SECURITY_HEADERS,
       }, [body]]
@@ -426,13 +505,11 @@ module Cronwatch
         @env[key]
       end
 
-      # scheme://host[:port], the page's origin.
+      # scheme://host[:port], the page's origin, lowercased as a browser
+      # writes it in Origin and Referer (Rack's base_url keeps the case of
+      # the Host or X-Forwarded-Host it read).
       def origin
-        @rack.base_url
-      end
-
-      def https?
-        @rack.scheme == "https"
+        @rack.base_url.downcase
       end
 
       # URLSearchParams#get: the first value, or nil.