good_pipeline 0.1.0

data/app/models/good_pipeline/dependency_record.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class DependencyRecord < ActiveRecord::Base
+    self.table_name = "good_pipeline_dependencies"
+    self.record_timestamps = false
+
+    belongs_to :pipeline,
+               class_name: "GoodPipeline::PipelineRecord",
+               foreign_key: :pipeline_id,
+               inverse_of: :dependencies
+
+    belongs_to :step,
+               class_name: "GoodPipeline::StepRecord",
+               foreign_key: :step_id,
+               inverse_of: :upstream_dependencies
+
+    belongs_to :depends_on_step,
+               class_name: "GoodPipeline::StepRecord",
+               foreign_key: :depends_on_step_id,
+               inverse_of: :downstream_dependencies
+  end
+end

data/app/models/good_pipeline/pipeline_record.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class PipelineRecord < ActiveRecord::Base
+    self.table_name = "good_pipeline_pipelines"
+    self.inheritance_column = nil
+
+    TERMINAL_STATUSES = %w[succeeded failed halted skipped].freeze
+
+    VALID_TRANSITIONS = {
+      "pending" => %w[running skipped],
+      "running" => %w[succeeded failed halted]
+    }.freeze
+
+    enum :status, {
+      pending: "pending",
+      running: "running",
+      succeeded: "succeeded",
+      failed: "failed",
+      halted: "halted",
+      skipped: "skipped"
+    }
+
+    enum :on_failure_strategy, { halt: "halt", continue: "continue", ignore: "ignore" }
+
+    has_many :steps,
+             class_name: "GoodPipeline::StepRecord",
+             foreign_key: :pipeline_id,
+             inverse_of: :pipeline,
+             dependent: :delete_all
+
+    has_many :dependencies,
+             class_name: "GoodPipeline::DependencyRecord",
+             foreign_key: :pipeline_id,
+             inverse_of: :pipeline,
+             dependent: :delete_all
+
+    has_many :downstream_chains,
+             class_name: "GoodPipeline::ChainRecord",
+             foreign_key: :upstream_pipeline_id,
+             inverse_of: :upstream_pipeline,
+             dependent: :delete_all
+
+    has_many :downstream_pipelines,
+             through: :downstream_chains,
+             source: :downstream_pipeline
+
+    has_many :upstream_chains,
+             class_name: "GoodPipeline::ChainRecord",
+             foreign_key: :downstream_pipeline_id,
+             inverse_of: :downstream_pipeline,
+             dependent: :delete_all
+
+    has_many :upstream_pipelines,
+             through: :upstream_chains,
+             source: :upstream_pipeline
+
+    def terminal?
+      TERMINAL_STATUSES.include?(status)
+    end
+
+    def transition_to!(new_status)
+      new_status = new_status.to_s
+      allowed = VALID_TRANSITIONS.fetch(status, [])
+
+      unless allowed.include?(new_status)
+        raise InvalidTransition, "cannot transition pipeline from '#{status}' to '#{new_status}'"
+      end
+
+      update!(status: new_status)
+    end
+  end
+end

data/app/models/good_pipeline/step_record.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class StepRecord < ActiveRecord::Base
+    self.table_name = "good_pipeline_steps"
+
+    TERMINAL_COORDINATION_STATUSES = %w[succeeded failed skipped].freeze
+
+    VALID_COORDINATION_TRANSITIONS = {
+      "pending" => %w[enqueued skipped],
+      "enqueued" => %w[succeeded failed]
+    }.freeze
+
+    enum :coordination_status, {
+      pending: "pending",
+      enqueued: "enqueued",
+      succeeded: "succeeded",
+      failed: "failed",
+      skipped: "skipped"
+    }
+
+    enum :on_failure_strategy, { halt: "halt", continue: "continue", ignore: "ignore" }
+
+    belongs_to :pipeline,
+               class_name: "GoodPipeline::PipelineRecord",
+               foreign_key: :pipeline_id,
+               inverse_of: :steps
+
+    has_many :upstream_dependencies,
+             class_name: "GoodPipeline::DependencyRecord",
+             foreign_key: :step_id,
+             inverse_of: :step,
+             dependent: :delete_all
+
+    has_many :upstream_steps,
+             through: :upstream_dependencies,
+             source: :depends_on_step
+
+    has_many :downstream_dependencies,
+             class_name: "GoodPipeline::DependencyRecord",
+             foreign_key: :depends_on_step_id,
+             inverse_of: :depends_on_step,
+             dependent: :delete_all
+
+    has_many :downstream_steps,
+             through: :downstream_dependencies,
+             source: :step
+
+    def duration
+      return nil unless good_job_id
+
+      good_job = GoodJob::Job.find_by(id: good_job_id)
+      return nil unless good_job&.performed_at && good_job.finished_at
+
+      good_job.finished_at - good_job.performed_at
+    end
+
+    def terminal_coordination_status?
+      TERMINAL_COORDINATION_STATUSES.include?(coordination_status)
+    end
+
+    def transition_coordination_status_to!(new_status)
+      new_status = new_status.to_s
+      allowed = VALID_COORDINATION_TRANSITIONS.fetch(coordination_status, [])
+
+      unless allowed.include?(new_status)
+        raise InvalidTransition,
+              "cannot transition step '#{key}' coordination_status from '#{coordination_status}' to '#{new_status}'"
+      end
+
+      update!(coordination_status: new_status)
+    end
+  end
+end

data/app/views/good_pipeline/pipelines/_chain_links.html.erb

@@ -0,0 +1,30 @@
+<% if pipeline.upstream_pipelines.any? || pipeline.downstream_pipelines.any? %>
+  <div class="chain-cards">
+    <% if pipeline.upstream_pipelines.any? %>
+      <article>
+        <header><h4>Upstream</h4></header>
+        <div class="chain-chips">
+          <% pipeline.upstream_pipelines.each do |upstream| %>
+            <%= link_to pipeline_path(upstream), class: "chain-chip" do %>
+              <span class="chain-label"><%= humanized_type(upstream.type) %></span>
+              <%= status_badge(upstream.status) %>
+            <% end %>
+          <% end %>
+        </div>
+      </article>
+    <% end %>
+    <% if pipeline.downstream_pipelines.any? %>
+      <article>
+        <header><h4>Downstream</h4></header>
+        <div class="chain-chips">
+          <% pipeline.downstream_pipelines.each do |downstream| %>
+            <%= link_to pipeline_path(downstream), class: "chain-chip" do %>
+              <span class="chain-label"><%= humanized_type(downstream.type) %></span>
+              <%= status_badge(downstream.status) %>
+            <% end %>
+          <% end %>
+        </div>
+      </article>
+    <% end %>
+  </div>
+<% end %>

data/app/views/good_pipeline/pipelines/_pagination.html.erb

@@ -0,0 +1,24 @@
+<% if params[:after_created_at].present? || @has_next_page %>
+  <nav>
+    <ul>
+      <li>
+        <% if params[:after_created_at].present? %>
+          <%= link_to "\u2190 First page", pipelines_path(status: @status, pipeline_type: @pipeline_type) %>
+        <% end %>
+      </li>
+    </ul>
+    <ul>
+      <li>
+        <% if @has_next_page %>
+          <% last_pipeline = @pipelines.last %>
+          <%= link_to "Next \u2192", pipelines_path(
+            status: @status,
+            pipeline_type: @pipeline_type,
+            after_created_at: last_pipeline.created_at.iso8601(6),
+            after_id: last_pipeline.id
+          ) %>
+        <% end %>
+      </li>
+    </ul>
+  </nav>
+<% end %>

data/app/views/good_pipeline/pipelines/_pipeline_row.html.erb

@@ -0,0 +1,7 @@
+<tr data-href="<%= pipeline_path(pipeline) %>" class="<%= 'row-running' if pipeline.running? %>">
+  <td><span class="execution-id"><%= pipeline.id.to_s.first(8) %></span></td>
+  <td><span class="pipeline-name"><%= humanized_type(pipeline.type) %></span></td>
+  <td><%= status_badge(pipeline.status) %></td>
+  <td><%= pipeline.created_at.strftime("%Y-%m-%d %I:%M %p") %></td>
+  <td><%= pipeline_duration(pipeline) || "\u2014" %></td>
+</tr>

data/app/views/good_pipeline/pipelines/_steps_table.html.erb

@@ -0,0 +1,33 @@
+<header><h3>Steps</h3></header>
+<table>
+  <thead>
+    <tr>
+      <th>Key</th>
+      <th>Job Class</th>
+      <th>Status</th>
+      <th>Duration</th>
+      <th>Error</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% pipeline.steps.each do |step| %>
+      <tr>
+        <td>
+          <% if (url = good_job_step_url(step)) %>
+            <%= link_to step.key, url, target: "_blank", rel: "noopener" %>
+          <% else %>
+            <%= step.key %>
+          <% end %>
+        </td>
+        <td><%= step.job_class %></td>
+        <td><%= status_badge(step.coordination_status) %></td>
+        <td><%= step.duration ? "#{step.duration.round(2)}s" : "\u2014" %></td>
+        <td>
+          <% if step.error_class %>
+            <small><%= step.error_class %>: <%= step.error_message %></small>
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/app/views/good_pipeline/pipelines/definitions.html.erb

@@ -0,0 +1,49 @@
+<h1 class="page-title">Pipeline Definitions</h1>
+
+<% if @pipelines.any? %>
+  <div class="definitions-layout">
+    <div class="definitions-sidebar">
+      <% @pipelines.each_with_index do |pipeline, index| %>
+        <button type="button" class="definition-item <%= 'active' if index.zero? %>" data-pipeline="<%= pipeline.type %>">
+          <span class="definition-item-name"><%= humanized_type(pipeline.type) %></span>
+          <span class="definition-item-meta"><%= pipeline.steps.size %> steps</span>
+        </button>
+      <% end %>
+    </div>
+
+    <div class="definitions-detail">
+      <% @pipelines.each_with_index do |pipeline, index| %>
+        <div class="definition-panel <%= 'active' if index.zero? %>" data-pipeline="<%= pipeline.type %>">
+          <article>
+            <header class="definition-header">
+              <div>
+                <h2><%= humanized_type(pipeline.type) %></h2>
+                <span class="definition-subtitle">on failure: <%= pipeline.on_failure_strategy %></span>
+              </div>
+              <%= link_to "View Executions \u2192", pipelines_path(pipeline_type: pipeline.type), class: "definition-executions-link" %>
+            </header>
+            <% if pipeline.steps.any? %>
+              <pre class="mermaid"><%= raw mermaid_definition_diagram(pipeline) %></pre>
+            <% else %>
+              <p class="empty-state">No steps defined.</p>
+            <% end %>
+          </article>
+        </div>
+      <% end %>
+    </div>
+  </div>
+
+  <script>
+    document.querySelectorAll('.definition-item').forEach(function(button) {
+      button.addEventListener('click', function() {
+        var type = this.dataset.pipeline;
+        document.querySelectorAll('.definition-item').forEach(function(b) { b.classList.remove('active'); });
+        document.querySelectorAll('.definition-panel').forEach(function(p) { p.classList.remove('active'); });
+        this.classList.add('active');
+        document.querySelector('.definition-panel[data-pipeline="' + type + '"]').classList.add('active');
+      });
+    });
+  </script>
+<% else %>
+  <p class="empty-state">No pipeline executions found yet. Run a pipeline to see its definition here.</p>
+<% end %>

data/app/views/good_pipeline/pipelines/index.html.erb

@@ -0,0 +1,43 @@
+<h1 class="page-title">Pipeline Executions</h1>
+
+<div class="filters-row">
+  <ul class="status-tabs">
+  <li><%= link_to "All (#{@total_count})", pipelines_path(pipeline_type: @pipeline_type), class: ("active" if @status.nil?) %></li>
+  <% %w[running succeeded failed halted skipped].each do |status| %>
+    <li><%= link_to "#{status.titleize} (#{@status_counts[status] || 0})", pipelines_path(status: status, pipeline_type: @pipeline_type), class: ("active" if @status == status), data: { status: status } %></li>
+  <% end %>
+  </ul>
+  <select class="pipeline-type-select" onchange="window.location = this.value;">
+    <option value="<%= pipelines_path(status: @status) %>" <%= "selected" if @pipeline_type.nil? %>>All Pipelines</option>
+    <% @pipeline_types.each do |pipeline_type| %>
+      <option value="<%= pipelines_path(status: @status, pipeline_type: pipeline_type) %>" <%= "selected" if @pipeline_type == pipeline_type %>>
+        <%= humanized_type(pipeline_type) %>
+      </option>
+    <% end %>
+  </select>
+</div>
+
+<% if @pipelines.any? %>
+  <div class="table-card">
+    <table>
+      <thead>
+        <tr>
+          <th>Execution ID</th>
+          <th>Pipeline Name</th>
+          <th>Status</th>
+          <th>Start Time</th>
+          <th>Duration</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @pipelines.each do |pipeline| %>
+          <%= render "pipeline_row", pipeline: pipeline %>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+
+  <%= render "pagination" %>
+<% else %>
+  <p class="empty-state">No pipelines found.</p>
+<% end %>

data/app/views/good_pipeline/pipelines/show.html.erb

@@ -0,0 +1,40 @@
+<p><%= link_to "\u2190 All Pipelines", pipelines_path %></p>
+
+<div class="pipeline-detail-grid">
+  <div class="pipeline-detail-left">
+    <article class="pipeline-details">
+      <header><h2><%= humanized_type(@pipeline.type) %></h2></header>
+      <dl>
+        <dt>Status</dt>
+        <dd><%= status_badge(@pipeline.status) %></dd>
+        <dt>Failure Strategy</dt>
+        <dd><span class="strategy-label"><%= @pipeline.on_failure_strategy %></span></dd>
+        <dt>Params</dt>
+        <dd><code><%= @pipeline.params&.to_json %></code></dd>
+        <dt>Created</dt>
+        <dd><%= relative_time_tag(@pipeline.created_at) %></dd>
+        <% if pipeline_duration(@pipeline) %>
+          <dt>Duration</dt>
+          <dd><%= pipeline_duration(@pipeline) %></dd>
+        <% end %>
+      </dl>
+    </article>
+
+    <%= render "chain_links", pipeline: @pipeline %>
+
+    <article class="table-card">
+      <%= render "steps_table", pipeline: @pipeline %>
+    </article>
+  </div>
+
+  <div class="pipeline-detail-right">
+    <article>
+      <header><h3>Pipeline Graph</h3></header>
+      <% if @pipeline.steps.any? %>
+        <pre class="mermaid"><%= raw mermaid_diagram(@pipeline) %></pre>
+      <% else %>
+        <p class="empty-state">No steps defined.</p>
+      <% end %>
+    </article>
+  </div>
+</div>

data/app/views/layouts/good_pipeline/application.html.erb

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en" data-theme="light">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>GoodPipeline</title>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@picocss/pico@2/css/pico.min.css">
+  <link rel="stylesheet" href="<%= frontend_static_path(id: 'style', format: :css) %>">
+</head>
+<body>
+  <header class="dashboard-header">
+    <span class="header-left">
+      <%= link_to "GoodPipeline", root_path %>
+      <%= link_to "Definitions", definitions_pipelines_path, class: "header-nav-link" %>
+    </span>
+    <span class="header-right">
+      <span class="header-version">v<%= GoodPipeline::VERSION %></span>
+      <a href="https://github.com/milkstrawai/good_pipeline" target="_blank" rel="noopener" class="header-github" title="GitHub">
+        <svg height="18" width="18" viewBox="0 0 16 16" fill="currentColor"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"/></svg>
+      </a>
+    </span>
+  </header>
+  <main class="dashboard-content">
+    <%= yield %>
+  </main>
+  <script src="https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js"></script>
+  <script>
+    mermaid.initialize({
+      startOnLoad: true,
+      theme: 'neutral',
+      flowchart: { curve: 'basis', nodeSpacing: 30, rankSpacing: 50 }
+    });
+    document.querySelectorAll('tr[data-href]').forEach(function(row) {
+      row.addEventListener('click', function() {
+        window.location = this.dataset.href;
+      });
+    });
+  </script>
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+GoodPipeline::Engine.routes.draw do
+  root "pipelines#index"
+
+  resources :pipelines, only: %i[index show] do
+    get :definitions, on: :collection
+  end
+
+  scope :frontend, controller: :frontends, defaults: { version: GoodPipeline::VERSION.tr(".", "-") } do
+    get "static/:version/:id", action: :static, as: :frontend_static
+  end
+end

data/demo/Rakefile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/demo/app/jobs/always_failing_job.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class AlwaysFailingJob < ApplicationJob
+  class AlwaysFailingError < StandardError
+  end
+
+  retry_on AlwaysFailingError, wait: 0, attempts: 3
+
+  def perform(**_kwargs)
+    raise AlwaysFailingError, "always fails"
+  end
+end

data/demo/app/jobs/application_job.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+end

data/demo/app/jobs/cleanup_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class CleanupJob < ApplicationJob
+  def perform(**); end
+end

data/demo/app/jobs/download_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class DownloadJob < ApplicationJob
+  def perform(**); end
+end

data/demo/app/jobs/failing_job.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class FailingJob < ApplicationJob
+  class FailingError < StandardError
+  end
+
+  discard_on FailingError
+
+  def perform(**)
+    raise FailingError, "intentional failure"
+  end
+end

data/demo/app/jobs/publish_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class PublishJob < ApplicationJob
+  def perform(**); end
+end

data/demo/app/jobs/retryable_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class RetryableJob < ApplicationJob
+  class RetryableError < StandardError
+  end
+
+  retry_on RetryableError, wait: 0, attempts: 5
+
+  def perform(tracker_key:)
+    result = ActiveRecord::Base.connection.execute(
+      ActiveRecord::Base.sanitize_sql(
+        ["UPDATE attempt_trackers SET count = count + 1 WHERE key = ? RETURNING count", tracker_key]
+      )
+    )
+    count = result.first["count"]
+
+    raise RetryableError, "attempt #{count}" if count < 3
+  end
+end

data/demo/app/jobs/thumbnail_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ThumbnailJob < ApplicationJob
+  def perform(**); end
+end

data/demo/app/jobs/transcode_job.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TranscodeJob < ApplicationJob
+  def perform(**); end
+end

data/demo/app/pipelines/analytics_pipeline.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AnalyticsPipeline < GoodPipeline::Pipeline
+  def configure(**_kwargs)
+    run :analyze, DownloadJob
+  end
+end

data/demo/app/pipelines/archive_pipeline.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class ArchivePipeline < GoodPipeline::Pipeline
+  def configure(**_kwargs)
+    run :archive, DownloadJob
+  end
+end

data/demo/app/pipelines/continue_test_pipeline.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class ContinueTestPipeline < GoodPipeline::Pipeline
+  failure_strategy :continue
+
+  def configure(**_kwargs)
+    run :step_a, FailingJob
+    run :step_b, DownloadJob
+    run :step_c, DownloadJob, after: %i[step_a step_b]
+  end
+end

data/demo/app/pipelines/halt_test_pipeline.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class HaltTestPipeline < GoodPipeline::Pipeline
+  failure_strategy :halt
+
+  def configure(**_kwargs)
+    run :step_a, FailingJob
+    run :step_b, DownloadJob, after: :step_a
+  end
+end

data/demo/app/pipelines/notification_pipeline.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class NotificationPipeline < GoodPipeline::Pipeline
+  def configure(**_kwargs)
+    run :notify, DownloadJob
+  end
+end

data/demo/app/pipelines/test_pipeline.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class TestPipeline < GoodPipeline::Pipeline
+  def configure(**) = run(:default, DownloadJob)
+end

data/demo/app/pipelines/video_processing_pipeline.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class VideoProcessingPipeline < GoodPipeline::Pipeline
+  description "Downloads, transcodes and publishes a video"
+  failure_strategy :halt
+
+  def configure(video_id:, **)
+    run :download, DownloadJob, with: { video_id: video_id }
+    run :transcode, TranscodeJob, after: :download
+    run :thumbnail, ThumbnailJob, after: :download
+    run :publish, PublishJob, after: %i[transcode thumbnail]
+    run :cleanup, CleanupJob, after: :publish
+  end
+end

data/demo/bin/rails

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/demo/config/application.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "boot"
+require "rails/all"
+require "good_job"
+require "good_job/engine"
+require "good_pipeline"
+require "good_pipeline/engine"
+
+Bundler.require(*Rails.groups)
+
+module TestApp
+  class Application < Rails::Application
+    config.load_defaults Rails::VERSION::STRING.to_f
+
+    config.active_job.queue_adapter = :good_job
+  end
+end