ductwork 1.3.0 → 1.5.0

data/app/controllers/ductwork/application_controller.rb

@@ -4,13 +4,20 @@ module Ductwork
   class ApplicationController < ActionController::Base
     DEFAULT_PER_PAGE = 50
 
+    helper_method :current_page, :per_page
+
     def query_pipeline_runs
-      Ductwork::Run
-        .includes(steps: { job: { executions: :result } })
-        .then(&method(:filter_by_klass))
-        .then(&method(:filter_by_status))
+      filtered_runs
         .then(&method(:paginate))
-        .order(started_at: :desc)
+        .order(started_at: :desc, id: :desc)
+    end
+
+    def filtered_runs
+      runs_of_klass.then(&method(:filter_by_status))
+    end
+
+    def runs_of_klass
+      Ductwork::Run.all.then(&method(:filter_by_klass))
     end
 
     def filter_by_klass(relation)
@@ -30,10 +37,34 @@ module Ductwork
     end
 
     def paginate(relation)
-      per_page = @per_page || DEFAULT_PER_PAGE
-      offset = params[:page].to_i * per_page
+      relation.limit(per_page).offset(current_page * per_page)
+    end
+
+    def per_page
+      @per_page || DEFAULT_PER_PAGE
+    end
+
+    def current_page
+      [params[:page].to_i, 0].max
+    end
+
+    def error_counts(run_ids)
+      Ductwork::Result
+        .failure
+        .joins(execution: { job: :step })
+        .where(ductwork_steps: { run_id: run_ids })
+        .group("ductwork_steps.run_id")
+        .count
+    end
 
-      relation.limit(per_page).offset(offset)
+    def halt_reasons(run_ids)
+      Ductwork::Branch
+        .where(run_id: run_ids, status: "halted")
+        .group(:run_id, :halt_reason)
+        .count
+        .each_with_object({}) do |((run_id, reason), count), reasons|
+          (reasons[run_id] ||= {})[reason] = count
+        end
     end
   end
 end

data/app/controllers/ductwork/dashboards_controller.rb

@@ -2,18 +2,51 @@
 
 module Ductwork
   class DashboardsController < Ductwork::ApplicationController
+    RECENT_RUNS = 10
+    HALTED_RUNS = 5
+
     def show
-      @metrics = Ductwork::Pipeline
-                 .group(:status)
-                 .count
-                 .slice("completed", "in_progress", "halted")
-                 .with_indifferent_access
-                 .reverse_merge(completed: 0, in_progress: 0, halted: 0)
-      @metrics[:step_errors] = Ductwork::Result.failure.count
-      @klasses = Ductwork::Pipeline.group(:klass).pluck(:klass).sort
-      @statuses = Ductwork::Pipeline.statuses.keys
-      @per_page = 25
-      @runs = query_pipeline_runs
+      @metrics = metrics
+      @halted_runs = Ductwork::Run
+                     .halted
+                     .where(pipeline_id: Ductwork::Pipeline.halted.select(:id))
+                     .order(halted_at: :desc, id: :desc)
+                     .limit(HALTED_RUNS)
+      @halts = halt_reasons(@halted_runs.map(&:id))
+      @class_summaries = class_summaries
+      @runs = Ductwork::Run.order(started_at: :desc, id: :desc).limit(RECENT_RUNS)
+      @error_counts = error_counts(@runs.map(&:id))
+    end
+
+    private
+
+    def metrics
+      Ductwork::Pipeline
+        .group(:status)
+        .count
+        .slice("completed", "in_progress", "halted")
+        .with_indifferent_access
+        .reverse_merge(completed: 0, in_progress: 0, halted: 0)
+        .merge(step_errors: Ductwork::Result.failure.count)
+    end
+
+    def class_summaries
+      last_started_at = Ductwork::Run.group(:pipeline_klass).maximum(:started_at)
+      counts = Ductwork::Pipeline.group(:klass, :status).count
+
+      counts.keys.map(&:first).uniq.sort.map do |klass|
+        total = counts.sum { |(key, _), count| key == klass ? count : 0 }
+        halted = counts.fetch([klass, "halted"], 0)
+        completed = counts.fetch([klass, "completed"], 0)
+
+        {
+          klass: klass,
+          active: total - halted - completed,
+          halted: halted,
+          completed: completed,
+          last_started_at: last_started_at[klass],
+        }
+      end
     end
   end
 end

data/app/controllers/ductwork/pipelines_controller.rb

@@ -3,29 +3,69 @@
 module Ductwork
   class PipelinesController < Ductwork::ApplicationController
     def index
+      @run_count = filtered_runs.count
+      @status_counts = runs_of_klass.group(:status).count
       @runs = query_pipeline_runs
+      @error_counts = error_counts(@runs.map(&:id))
       @klasses = Ductwork::Pipeline.group(:klass).pluck(:klass).sort
       @statuses = Ductwork::Pipeline.statuses.keys
     end
 
     def show
+      @per_page = 25
       @pipeline = Ductwork::Pipeline.find(params[:id])
       @last_run = @pipeline.runs.order(started_at: :desc).first
-      @per_page = 10
+      @run_count = @pipeline.runs.count
+      @graph = Ductwork::PipelineGraph.new(@last_run)
+      @error_count = error_counts([@last_run.id]).fetch(@last_run.id, 0)
+      @halts = halt_reasons([@last_run.id]).fetch(@last_run.id, {})
+      load_steps
+    end
+
+    private
+
+    def load_steps
+      @status_counts = steps_in_scope.group(:status).count
+      @step_total = @last_run.steps.count
+      @step_count = filtered_steps.count
       @steps = query_steps
       @klasses = @last_run.steps.group(:klass).pluck(:klass).sort
+      @nodes = @last_run.steps.group(:node).pluck(:node).sort
       @statuses = Ductwork::Step.statuses.keys
     end
 
-    private
-
     def query_steps
+      filtered_steps
+        .includes(job: { executions: %i[availability attempt result] })
+        .then(&method(:paginate))
+        .order(:started_at, :id)
+    end
+
+    def filtered_steps
+      steps_in_scope.then(&method(:filter_by_status))
+    end
+
+    def steps_in_scope
       @last_run
         .steps
-        .then(&method(:filter_by_klass))
-        .then(&method(:filter_by_status))
-        .then(&method(:paginate))
-        .order(:started_at)
+        .then(&method(:filter_by_step_klass))
+        .then(&method(:filter_by_node))
+    end
+
+    def filter_by_step_klass(relation)
+      if params[:klass].present?
+        relation.where(klass: params[:klass])
+      else
+        relation
+      end
+    end
+
+    def filter_by_node(relation)
+      if params[:node].present?
+        relation.where(node: params[:node])
+      else
+        relation
+      end
     end
   end
 end

data/app/controllers/ductwork/step_errors_controller.rb

@@ -3,23 +3,27 @@
 module Ductwork
   class StepErrorsController < Ductwork::ApplicationController
     def index
+      @error_count = filtered_errors.count
       @step_errors = query_step_errors
       @klasses = Ductwork::Result
                  .failure
                  .joins(execution: { job: :step })
                  .group("ductwork_steps.klass")
                  .pluck("ductwork_steps.klass")
+                 .sort
     end
 
     private
 
     def query_step_errors
-      Ductwork::Result
-        .failure
-        .then(&method(:filter_by_step_klass))
+      filtered_errors
         .then(&method(:paginate))
-        .includes(execution: { job: :step })
-        .order(created_at: :desc)
+        .includes(execution: { job: { step: :run } })
+        .order(created_at: :desc, id: :desc)
+    end
+
+    def filtered_errors
+      Ductwork::Result.failure.then(&method(:filter_by_step_klass))
     end
 
     def filter_by_step_klass(relation)

data/app/helpers/ductwork/application_helper.rb

@@ -2,38 +2,127 @@
 
 module Ductwork
   module ApplicationHelper
+    NO_VALUE = "—"
+    SHORT_ID_LENGTH = 8
+
     def formatted_time_distance(started_at, completed_at)
       elapsed = completed_at - started_at
+
+      return "#{elapsed.round(2)}s" if elapsed < 60
+
       hours = (elapsed / 3600).floor
       minutes = ((elapsed % 3600) / 60).floor
-      seconds = (elapsed % 60).round(2)
-      hours_string = hours.zero? ? "" : "#{hours}h "
-      minutes_string = minutes.zero? ? "" : "#{minutes}m "
-      seconds_string = seconds.zero? ? "" : "#{seconds}s"
+      seconds = (elapsed % 60).floor
+      parts = { "h" => hours, "m" => minutes, "s" => seconds }.reject { |_, value| value.zero? }
+
+      parts.map { |unit, value| "#{value}#{unit}" }.join(" ")
+    end
+
+    def runtime(record)
+      finished_at = record.completed_at || record.try(:halted_at)
+
+      if finished_at.present?
+        formatted_time_distance(record.started_at, finished_at)
+      elsif record.started_at.nil? || record.try(:failed?)
+        no_value
+      else
+        tag.span(
+          formatted_time_distance(record.started_at, Time.current),
+          data: { started_at: record.started_at.iso8601(3) }
+        )
+      end
+    end
+
+    def timestamp(time)
+      return no_value if time.nil?
+
+      iso8601 = time.iso8601(3)
+
+      tag.time(time.strftime("%b %-d, %H:%M:%S"), datetime: iso8601, title: iso8601)
+    end
+
+    # NOTE: Primary keys are UUID v7, so the leading characters are a timestamp
+    # and match across records created together. The tail is the random part.
+    def short_id(id)
+      tag.code(id.to_s.last(SHORT_ID_LENGTH), title: id)
+    end
+
+    def status_badge(status)
+      tag.span(status_text(status), class: "status", data: { status: })
+    end
+
+    def status_text(status)
+      status.tr("_", "-")
+    end
+
+    def transition_text(transition)
+      transition == "default" ? "chain" : transition
+    end
 
-      "#{hours_string}#{minutes_string}#{seconds_string}"
+    def count_badge(count, tone: "alert")
+      count.zero? ? tag.span(0, class: "faint") : tag.span(number_with_delimiter(count), class: "count-#{tone}")
+    end
+
+    def halt_summary(reasons)
+      reasons
+        .map do |reason, count|
+          text = reason&.humanize&.downcase || "unknown reason"
+
+          "#{pluralize(count, "branch")}: #{text}"
+        end
+        .join(", ")
+    end
+
+    def pretty_json(value)
+      value.nil? ? "no value" : JSON.pretty_generate(value)
+    end
+
+    def step_input(job)
+      JSON.parse(job.input_args)["args"] if job&.input_args.present?
+    end
+
+    def step_output(job)
+      job.return_value if job&.output_payload.present?
+    end
+
+    def filtered_path(anchor: nil, **changes)
+      url_for(**link_params.except(:page).merge(changes).compact, anchor:)
     end
 
     def first_page?
-      params[:page].to_i.zero?
+      current_page.zero?
+    end
+
+    def last_page?(total)
+      (current_page + 1) * per_page >= total
     end
 
-    def next_page_path
-      next_page = params[:page].to_i + 1
-      next_params = params
-                    .permit(:controller, :action, :id, :klass, :status, :page)
-                    .merge(page: next_page)
+    def page_summary(total)
+      first = (current_page * per_page) + 1
+      last = [first + per_page - 1, total].min
+
+      first > total ? "Showing 0 of #{total}" : "Showing #{first}-#{last} of #{total}"
+    end
 
-      url_for(**next_params)
+    def next_page_path(anchor: nil)
+      url_for(**link_params, page: current_page + 1, anchor: anchor)
     end
 
-    def previous_page_path
-      previous_page = params[:page].to_i - 1
-      previous_params = params
-                        .permit(:controller, :action, :id, :klass, :status, :page)
-                        .merge(page: previous_page)
+    def previous_page_path(anchor: nil)
+      url_for(**link_params, page: [current_page - 1, 0].max, anchor: anchor)
+    end
+
+    private
+
+    def link_params
+      params
+        .permit(:controller, :action, :id, :klass, :node, :status, :page)
+        .to_h
+        .symbolize_keys
+    end
 
-      url_for(**previous_params)
+    def no_value
+      tag.span(NO_VALUE, class: "faint")
     end
   end
 end

data/app/views/ductwork/dashboards/show.html.erb

@@ -1,136 +1,85 @@
-<div>
-  <div class="grid metrics">
-    <%= link_to(pipelines_path(status: "completed"), class: "stealthy-link") do %>
-      <article>
-        <div class="metric-heading">Completed Pipelines</div>
-        <h1 class="metric-value"><%= number_with_delimiter(@metrics[:completed]) %></h1>
-      </article>
-    <% end %>
+<% content_for(:title, "Overview") %>
 
-    <%= link_to(pipelines_path(status: "in_progress"), class: "stealthy-link") do %>
-      <article>
-        <div class="metric-heading">In-Progress Pipelines</div>
-        <h1 class="metric-value"><%= number_with_delimiter(@metrics[:in_progress]) %></h1>
-      </article>
+<section class="section">
+  <div class="panel stats">
+    <%= link_to(pipelines_path(status: "in_progress"), class: "stat") do %>
+      <span class="stat-value"><%= number_with_delimiter(@metrics[:in_progress]) %></span>
+      In progress
     <% end %>
-
-    <%= link_to(pipelines_path(status: "halted"), class: "stealthy-link") do %>
-      <article>
-        <div class="metric-heading">Halted Pipelines</div>
-        <h1 class="metric-value"><%= number_with_delimiter(@metrics[:halted]) %></h1>
-      </article>
+    <%= link_to(pipelines_path(status: "halted"), class: "stat", data: { alert: @metrics[:halted].positive? }) do %>
+      <span class="stat-value"><%= number_with_delimiter(@metrics[:halted]) %></span>
+      Halted
     <% end %>
-
-    <%= link_to(step_errors_path, class: "stealthy-link") do %>
-      <article>
-        <div class="metric-heading">Step Errors</div>
-        <h1 class="metric-value"><%= @metrics[:step_errors] %></h1>
-      </article>
+    <%= link_to(pipelines_path(status: "completed"), class: "stat") do %>
+      <span class="stat-value"><%= number_with_delimiter(@metrics[:completed]) %></span>
+      Completed
+    <% end %>
+    <%= link_to(step_errors_path, class: "stat") do %>
+      <span class="stat-value"><%= number_with_delimiter(@metrics[:step_errors]) %></span>
+      Step errors
     <% end %>
   </div>
+</section>
 
-  <div>
-    <h1>All Pipelines</h1>
-    <div class="separate-content">
-      <div class="filter-group">
-        <span class="filter-group-item">Filters:</span>
-
-        <details class="dropdown filter-group-item">
-          <summary>Class</summary>
-          <ul>
-            <% @klasses.each do |klass| %>
-              <li>
-                <%= link_to(klass, pipelines_path(klass:)) %>
-              </li>
-            <% end %>
-          </ul>
-        </details>
-
-        <details class="dropdown filter-group-item">
-          <summary>Status</summary>
-          <ul>
-            <% @statuses.each do |status| %>
-              <li>
-                <%= link_to(status.gsub("_", "-"), pipelines_path(status:)) %>
-              </li>
-            <% end %>
-          </ul>
-        </details>
-      </div>
-
-      <div role="group" style="width: auto;">
-        <button <%= first_page? && "disabled" %> data-href="<%= previous_page_path %>">
-          &larr;
-        </button>
-        <button data-href="<%= next_page_path %>">
-          &rarr;
-        </button>
-      </div>
+<% if @halted_runs.any? %>
+  <section class="section">
+    <div class="section-head">
+      <h2 class="section-title">Needs attention</h2>
+      <%= link_to("All halted", pipelines_path(status: "halted")) %>
+    </div>
+    <div class="panel">
+      <% @halted_runs.each do |run| %>
+        <div class="halt">
+          <div>
+            <%= link_to(run.pipeline_klass, pipeline_path(run.pipeline_id), class: "table-primary") %>
+            <%= short_id(run.pipeline_id) %>
+          </div>
+          <div class="halt-reason"><%= halt_summary(@halts.fetch(run.id, {})) %></div>
+          <div class="halt-time"><%= timestamp(run.halted_at) %></div>
+        </div>
+      <% end %>
     </div>
+  </section>
+<% end %>
 
-    <table class="striped">
-      <thead>
-        <tr>
-          <th>ID</th>
-          <th>Class</th>
-          <th>Status</th>
-          <th>Errors</th>
-          <th>Started At</th>
-          <th>Runtime</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @runs.each do |run| %>
-          <tr class="row-link" data-href="<%= pipeline_path(run.pipeline_id) %>">
-            <td>
-              <code>
-                <%= run.pipeline_id %>
-              </code>
-            </td>
-            <td>
-              <code>
-                <%= run.pipeline_klass %>
-              </code>
-            </td>
-            <td>
-              <div class="status-pill <%= run.status %>">
-                <%= run.status.gsub("_", "-") %>
-              </div>
-            </td>
-            <td>
-              <%
-                count = Ductwork::Result
-                        .joins(execution: { job: { step: { run: :pipeline }}})
-                        .failure
-                        .where(ductwork_pipelines: { id: run.pipeline_id })
-                        .count
-              %>
-              <% if count.zero? %>
-                0
-              <% else %>
-                <div class="errors">
-                  <%= image_tag("ductwork/octagon-x.svg", class: "icon") %>
-                  <%= count %>
-                </div>
-              <% end %>
-            </td>
-            <td>
-              <div class="tight-timestamp">
-                <%= run.started_at.iso8601(3) %>
-              </div>
-            </td>
-            <td>
-              <% if run.completed_at.nil? %>
-                <div data-started-at=<%= run.started_at.iso8601 %>>
-                  <span id="elapsed-timer">0h 0m 0s</span>
-                </div>
-              <% else %>
-                <%= formatted_time_distance(run.started_at, run.completed_at) %>
-              <% end %>
-            </td>
+<section class="section">
+  <div class="section-head">
+    <h2 class="section-title">Pipelines by class</h2>
+  </div>
+  <div class="panel table-scroller">
+    <% if @class_summaries.empty? %>
+      <p class="empty">No pipelines have been triggered yet.</p>
+    <% else %>
+      <table class="table">
+        <thead>
+          <tr>
+            <th>Class</th>
+            <th class="numeric">Active</th>
+            <th class="numeric">Halted</th>
+            <th class="numeric">Completed</th>
+            <th>Last started</th>
           </tr>
-        <% end %>
-      </tbody>
-    </table>
+        </thead>
+        <tbody>
+          <% @class_summaries.each do |summary| %>
+            <tr data-href="<%= pipelines_path(klass: summary[:klass]) %>">
+              <td><%= link_to(summary[:klass], pipelines_path(klass: summary[:klass]), class: "table-primary") %></td>
+              <td class="numeric"><%= number_with_delimiter(summary[:active]) %></td>
+              <td class="numeric"><%= count_badge(summary[:halted]) %></td>
+              <td class="numeric"><%= number_with_delimiter(summary[:completed]) %></td>
+              <td><%= timestamp(summary[:last_started_at]) %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    <% end %>
+  </div>
+</section>
+
+<section class="section">
+  <div class="section-head">
+    <h2 class="section-title">Recent runs</h2>
+    <%= link_to("All pipelines", pipelines_path) %>
   </div>
-</div>
+  <%= render("ductwork/shared/runs_table", runs: @runs, error_counts: @error_counts) %>
+</section>

data/app/views/ductwork/pipelines/_graph.html.erb

@@ -0,0 +1,37 @@
+<div class="graph-scroller">
+  <svg class="graph" width="<%= graph.width %>" height="<%= graph.height %>" viewBox="0 0 <%= graph.width %> <%= graph.height %>" role="figure" aria-label="Pipeline graph">
+    <% graph.edges.each do |edge| %>
+      <path class="graph-edge" data-state="<%= edge.state %>" d="<%= edge.path %>"></path>
+    <% end %>
+
+    <% graph.nodes.each do |node| %>
+      <g class="graph-node" data-state="<%= node.state %>" data-selected="<%= params[:node] == node.id %>">
+        <title><%= node.klass %> (<%= node.id %>)</title>
+        <rect class="graph-card" x="<%= node.x %>" y="<%= node.y %>" width="<%= graph.card_width %>" height="<%= graph.card_height %>" rx="8"></rect>
+        <rect class="graph-bar-track" x="<%= node.x + Ductwork::PipelineGraph::BAR_INSET %>" y="<%= node.y + Ductwork::PipelineGraph::BAR_TOP %>" width="<%= graph.bar_width %>" height="<%= Ductwork::PipelineGraph::BAR_HEIGHT %>" rx="3"></rect>
+        <% node.segments.each do |segment| %>
+          <rect class="graph-bar-segment" data-status="<%= segment.status %>" x="<%= node.x + segment.x %>" y="<%= node.y + Ductwork::PipelineGraph::BAR_TOP %>" width="<%= segment.width %>" height="<%= Ductwork::PipelineGraph::BAR_HEIGHT %>"></rect>
+        <% end %>
+
+        <foreignObject x="<%= node.x %>" y="<%= node.y %>" width="<%= graph.card_width %>" height="<%= Ductwork::PipelineGraph::HEADER_HEIGHT %>">
+          <div class="graph-header">
+            <div class="graph-transition"><%= node.transition %></div>
+            <%= link_to(node.name, params[:node] == node.id ? pipeline_path(@pipeline.id, anchor: "steps") : pipeline_path(@pipeline.id, node: node.id, anchor: "steps"), class: "graph-name") %>
+          </div>
+        </foreignObject>
+
+        <foreignObject x="<%= node.x %>" y="<%= node.y + Ductwork::PipelineGraph::LABELS_TOP %>" width="<%= graph.card_width %>" height="<%= graph.card_height - Ductwork::PipelineGraph::LABELS_TOP %>">
+          <div class="graph-labels">
+            <% node.labels.each do |label| %>
+              <% if label.status %>
+                <%= link_to(label.text, pipeline_path(@pipeline.id, node: node.id, status: label.status, anchor: "steps"), class: "graph-label", data: { tone: label.tone }) %>
+              <% else %>
+                <span class="graph-label <%= "graph-label-wide" if label.wide %>" data-tone="<%= label.tone %>" title="<%= label.text %>"><%= label.text %></span>
+              <% end %>
+            <% end %>
+          </div>
+        </foreignObject>
+      </g>
+    <% end %>
+  </svg>
+</div>