geneva_drive 0.4.0 → 0.6.0

data/README.md

@@ -164,12 +164,14 @@ class PremiumWorkflow < BaseWorkflow
   cancel_if { hero.subscription_expired? }
   set_step_job_options queue: :premium, priority: 10
 
-  step :premium_feature do
+  step :premium_feature, job_options: {priority: -1} do
     # Inherits cancel_if from parent
   end
 end
 ```
 
+Step job options override workflow defaults when the step is initially enqueued, reattempted, or resumed. Options are passed directly to Active Job's `set` method.
+
 ### Hero Deletion Handling
 
 By default, workflows cancel if their hero is deleted:

data/Rakefile

@@ -1,6 +1,17 @@
 require "bundler/setup"
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+require "standard/rake"
+require "yard"
+
+YARD::Rake::YardocTask.new("yard")
+
+task doc: [:yard] do
+  outfile = "doc/MANUAL.html"
+  `bin/md2html MANUAL.md #{outfile}`
+  warn "#{outfile} written"
+end
 
 APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
 load "rails/tasks/engine.rake"
-
-require "bundler/gem_tasks"

data/bin/md2html

@@ -7,9 +7,11 @@ require "bundler/inline"
 gemfile do
   source "https://rubygems.org"
   gem "redcarpet"
+  gem "nokogiri"
 end
 
 require "redcarpet"
+require "nokogiri"
 require "cgi"
 require "erb"
 
@@ -40,7 +42,24 @@ class SyntaxHighlightingHTML < Redcarpet::Render::HTML
   end
 end
 
-def html_template(title, content)
+def build_toc(html_body)
+  doc = Nokogiri::HTML::DocumentFragment.parse(html_body)
+  headings = doc.css("h1, h2")
+  return "" if headings.empty?
+
+  toc = +%(<nav class="toc-pane"><ul>\n)
+  headings.each do |h|
+    id = h["id"]
+    next unless id
+    level = h.name # "h1" or "h2"
+    css_class = (level == "h2") ? ' class="toc-h2"' : ""
+    toc << %(<li#{css_class}><a href="##{CGI.escapeHTML(id)}">#{CGI.escapeHTML(h.text)}</a></li>\n)
+  end
+  toc << %(</ul></nav>\n)
+  toc
+end
+
+def html_template(title, content, toc: "")
   <<~HTML
 <!DOCTYPE html>
 <html lang="en">
@@ -63,21 +82,73 @@ def html_template(title, content)
       --table-border: #dfe2e5;
       --blockquote-color: #6a737d;
       --blockquote-border: #dfe2e5;
+      --toc-width: 280px;
     }
 
     * {
       box-sizing: border-box;
     }
 
-    body {
+    html, body {
+      margin: 0;
+      padding: 0;
       font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
       font-size: 16px;
       line-height: 1.6;
+      color: var(--text-color);
+      background-color: var(--bg-color);
+    }
+
+    /* TOC sidebar — fixed, scrolls independently */
+    .toc-pane {
+      position: fixed;
+      top: 0;
+      left: 0;
+      width: var(--toc-width);
+      height: 100vh;
+      overflow-y: auto;
+      border-right: 1px solid var(--border-color);
+      padding: 2rem 1.25rem;
+      font-size: 13px;
+      line-height: 1.5;
+      background-color: var(--bg-color);
+    }
+
+    .toc-pane ul {
+      list-style: none;
+      padding: 0;
+      margin: 0;
+    }
+
+    .toc-pane li {
+      margin: 0.25em 0;
+    }
+
+    .toc-pane li.toc-h2 {
+      padding-left: 1em;
+    }
+
+    .toc-pane a {
+      color: var(--link-color);
+      text-decoration: underline;
+    }
+
+    .toc-pane a:hover {
+      color: #0550ae;
+    }
+
+    /* Body scrolls normally — browser scrollbar at window edge */
+    body.has-toc {
+      margin-left: var(--toc-width);
+      padding: 2rem 3rem;
+      max-width: calc(960px + var(--toc-width));
+    }
+
+    /* Without TOC, body is a simple centered column */
+    body:not(.has-toc) {
       max-width: 900px;
       margin: 0 auto;
       padding: 2rem;
-      color: var(--text-color);
-      background-color: var(--bg-color);
     }
 
     /* Headings */
@@ -241,12 +312,7 @@ def html_template(title, content)
       height: auto;
     }
 
-    /* Print footer element - not needed with @page margin boxes */
-    .print-footer {
-      display: none;
-    }
-
-    /* Print styles - using Chrome 131+ @page margin boxes */
+    /* Print styles */
     @media print {
       @page {
         margin: 1.5cm;
@@ -265,7 +331,12 @@ def html_template(title, content)
         }
       }
 
-      body {
+      .toc-pane {
+        display: none;
+      }
+
+      body, body.has-toc {
+        margin-left: 0;
         max-width: none;
         padding: 0;
         font-size: 11pt;
@@ -296,8 +367,8 @@ def html_template(title, content)
     }
   </style>
 </head>
-<body>
-#{content}
+<body#{' class="has-toc"' unless toc.empty?}>
+#{toc}#{content}
 
 <!-- Prism.js for syntax highlighting -->
 <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-core.min.js"></script>
@@ -338,7 +409,7 @@ def html_template(title, content)
   HTML
 end
 
-def convert_markdown_to_html(input_path, output_path = nil)
+def convert_markdown_to_html(input_path, output_path = nil, toc: true)
   unless File.exist?(input_path)
     abort "Error: File not found: #{input_path}"
   end
@@ -373,7 +444,8 @@ def convert_markdown_to_html(input_path, output_path = nil)
   })
 
   html_body = markdown.render(markdown_content)
-  html_document = html_template(title, html_body)
+  toc_html = toc ? build_toc(html_body) : ""
+  html_document = html_template(title, html_body, toc: toc_html)
 
   File.write(output_path, html_document)
   puts "Created: #{output_path}"
@@ -382,7 +454,7 @@ def convert_markdown_to_html(input_path, output_path = nil)
 end
 
 if __FILE__ == $0
-  options = {open: true}
+  options = {open: false, toc: true}
 
   parser = OptionParser.new do |opts|
     opts.banner = "Usage: #{$0} [options] INPUT.md [OUTPUT.html]"
@@ -390,14 +462,19 @@ if __FILE__ == $0
     opts.separator "Converts a GitHub-Flavored Markdown file to styled HTML with:"
     opts.separator "  - Prism.js syntax highlighting"
     opts.separator "  - Mermaid.js diagram rendering"
+    opts.separator "  - Table of contents sidebar"
     opts.separator "  - Print-optimized CSS with page numbers"
     opts.separator ""
     opts.separator "If OUTPUT is not specified, uses INPUT filename with .html extension."
     opts.separator ""
     opts.separator "Options:"
 
-    opts.on("--no-open", "Don't open the result in the browser") do
-      options[:open] = false
+    opts.on("--no-toc", "Don't generate table of contents sidebar") do
+      options[:toc] = false
+    end
+
+    opts.on("--open", "Open the result in the browser") do
+      options[:open] = true
     end
 
     opts.on("-h", "--help", "Show this help message") do
@@ -416,7 +493,7 @@ if __FILE__ == $0
   input_path = ARGV[0]
   output_path = ARGV[1]
 
-  result_path = convert_markdown_to_html(input_path, output_path)
+  result_path = convert_markdown_to_html(input_path, output_path, toc: options[:toc])
 
   # Open in browser if on macOS and not disabled
   if options[:open] && RUBY_PLATFORM =~ /darwin/

data/lib/generators/geneva_drive/install/install_generator.rb

@@ -40,6 +40,31 @@ module GenevaDrive
           "add_error_class_name_to_step_executions.rb",
           "db/migrate/add_error_class_name_to_geneva_drive_step_executions.rb"
         )
+
+        migration_template(
+          "add_metadata_to_step_executions.rb",
+          "db/migrate/add_metadata_to_geneva_drive_step_executions.rb"
+        )
+
+        migration_template(
+          "add_metadata_to_workflows.rb",
+          "db/migrate/add_metadata_to_geneva_drive_workflows.rb"
+        )
+
+        migration_template(
+          "allow_null_hero_on_workflows.rb",
+          "db/migrate/allow_null_hero_on_geneva_drive_workflows.rb"
+        )
+
+        migration_template(
+          "add_started_at_index_to_step_executions.rb",
+          "db/migrate/add_started_at_index_to_geneva_drive_step_executions.rb"
+        )
+
+        migration_template(
+          "add_resumable_step_support.rb",
+          "db/migrate/add_resumable_step_support_to_geneva_drive_step_executions.rb"
+        )
       end
 
       # Creates the initializer file.

data/lib/generators/geneva_drive/install/templates/add_metadata_to_step_executions.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddMetadataToGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
+  def change
+    return if column_exists?(:geneva_drive_step_executions, :metadata)
+
+    adapter = connection.adapter_name.downcase
+
+    if adapter.include?("postgresql")
+      add_column :geneva_drive_step_executions, :metadata, :jsonb
+    elsif adapter.include?("mysql")
+      add_column :geneva_drive_step_executions, :metadata, :text, limit: 4_294_967_295
+    else
+      add_column :geneva_drive_step_executions, :metadata, :text
+    end
+  end
+end

data/lib/generators/geneva_drive/install/templates/add_metadata_to_workflows.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class AddMetadataToGenevaDriveWorkflows < ActiveRecord::Migration[7.2]
+  def change
+    return if column_exists?(:geneva_drive_workflows, :metadata)
+
+    adapter = connection.adapter_name.downcase
+
+    if adapter.include?("postgresql")
+      add_column :geneva_drive_workflows, :metadata, :jsonb
+    elsif adapter.include?("mysql")
+      add_column :geneva_drive_workflows, :metadata, :text, limit: 4_294_967_295
+    else
+      add_column :geneva_drive_workflows, :metadata, :text
+    end
+  end
+end

data/lib/generators/geneva_drive/install/templates/add_resumable_step_support.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class AddResumableStepSupportToGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
+  include GenevaDrive::MigrationHelpers
+
+  def change
+    unless column_exists?(:geneva_drive_step_executions, :cursor)
+      # Cursor for resumable steps. Use database-native JSON type:
+      # - PostgreSQL: jsonb (indexed, efficient, supports containment queries)
+      # - MySQL 5.7+: json (native validation and storage)
+      # - SQLite: json (Rails handles as TEXT with serialization)
+      if connection.adapter_name.downcase.include?("postgresql")
+        add_column :geneva_drive_step_executions, :cursor, :jsonb
+      else
+        add_column :geneva_drive_step_executions, :cursor, :json
+      end
+    end
+
+    unless column_exists?(:geneva_drive_step_executions, :continues_from_id)
+      # Link successor executions to their predecessor, chaining the execution
+      # records of a resumable step. Match the primary key type (bigint or uuid)
+      # of the step_executions table.
+      # No foreign key constraint - SQLite rewrites the table on add_foreign_key,
+      # which can destroy data.
+      add_column :geneva_drive_step_executions, :continues_from_id, geneva_drive_key_type
+      add_index :geneva_drive_step_executions, :continues_from_id
+    end
+  end
+end

data/lib/generators/geneva_drive/install/templates/add_started_at_index_to_step_executions.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class AddStartedAtIndexToGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
+  # Partial index scoped to state = 'in_progress'. HousekeepingJob's
+  # recover_stuck_in_progress! filters `state = 'in_progress' AND
+  # started_at < cutoff LIMIT N`; without this the planner walks the whole
+  # in-progress set looking for old rows and trips the caller's
+  # statement_timeout on large / mostly-fresh workloads.
+  INDEX_NAME = :index_geneva_drive_step_executions_in_progress_started_at
+
+  disable_ddl_transaction!
+
+  def up
+    return if index_name_exists?(:geneva_drive_step_executions, INDEX_NAME)
+
+    adapter = connection.adapter_name.downcase
+
+    if adapter.include?("postgresql")
+      add_index :geneva_drive_step_executions, :started_at,
+        name: INDEX_NAME,
+        where: "state = 'in_progress'",
+        algorithm: :concurrently
+    elsif adapter.include?("sqlite")
+      # SQLite supports partial indexes but has no CONCURRENTLY.
+      add_index :geneva_drive_step_executions, :started_at,
+        name: INDEX_NAME,
+        where: "state = 'in_progress'"
+    else
+      # MySQL does not support partial indexes; fall back to a plain index.
+      # Composite (state, started_at) so the planner can seek by state first.
+      add_index :geneva_drive_step_executions, [:state, :started_at],
+        name: INDEX_NAME
+    end
+  end
+
+  def down
+    return unless index_name_exists?(:geneva_drive_step_executions, INDEX_NAME)
+
+    adapter = connection.adapter_name.downcase
+    if adapter.include?("postgresql")
+      remove_index :geneva_drive_step_executions,
+        name: INDEX_NAME,
+        algorithm: :concurrently
+    else
+      remove_index :geneva_drive_step_executions, name: INDEX_NAME
+    end
+  end
+end

data/lib/generators/geneva_drive/install/templates/allow_null_hero_on_workflows.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+class AllowNullHeroOnGenevaDriveWorkflows < ActiveRecord::Migration[7.2]
+  def change
+    return unless column_exists?(:geneva_drive_workflows, :hero_type)
+
+    adapter = connection.adapter_name.downcase
+    if adapter.include?("sqlite")
+      allow_null_hero_sqlite
+    else
+      change_column_null :geneva_drive_workflows, :hero_type, true
+      change_column_null :geneva_drive_workflows, :hero_id, true
+    end
+  end
+
+  private
+
+  def allow_null_hero_sqlite
+    reversible do |direction|
+      direction.up do
+        # SQLite cannot ALTER COLUMN to remove NOT NULL without a full table
+        # recreation. We perform an atomic table swap to avoid foreign-key
+        # corruption issues. See CLAUDE.md / AGENTS.md for context.
+
+        # 1. Introspect the existing table via ActiveRecord
+        old_columns = connection.columns(:geneva_drive_workflows)
+        old_pk = connection.primary_key(:geneva_drive_workflows)
+        pk_col = old_columns.find { |c| c.name == old_pk }
+        data_cols = old_columns.reject { |c| c.name == old_pk }
+
+        # 2. Create the replacement table using the schema DSL
+        create_table :geneva_drive_workflows_new, id: pk_col.type do |t|
+          data_cols.each do |col|
+            nullable = %w[hero_type hero_id].include?(col.name) || col.null
+            opts = {null: nullable}
+            opts[:default] = col.default_function || col.default unless col.default.nil? && col.default_function.nil?
+            t.column col.name, col.type, **opts
+          end
+        end
+
+        # 3. Copy all data
+        col_names = old_columns.map(&:name).join(", ")
+        execute "INSERT INTO geneva_drive_workflows_new (#{col_names}) SELECT #{col_names} FROM geneva_drive_workflows"
+
+        # 4. Atomic swap
+        drop_table :geneva_drive_workflows
+        rename_table :geneva_drive_workflows_new, :geneva_drive_workflows
+
+        # 5. Recreate indexes (they are lost during the swap)
+        add_index :geneva_drive_workflows, :type
+        add_index :geneva_drive_workflows, :state
+        add_index :geneva_drive_workflows, [:hero_type, :hero_id]
+        execute <<-SQL
+          CREATE UNIQUE INDEX index_geneva_drive_workflows_unique_ongoing
+          ON geneva_drive_workflows (type, hero_type, hero_id)
+          WHERE state NOT IN ('finished', 'canceled') AND allow_multiple = 0;
+        SQL
+      end
+
+      direction.down do
+        raise ActiveRecord::IrreversibleMigration,
+          "Cannot re-add NOT NULL on hero columns in SQLite without risking data loss"
+      end
+    end
+  end
+end

data/lib/generators/geneva_drive/install/templates/create_step_executions_migration.rb

@@ -5,14 +5,18 @@ class CreateGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
 
   def change
     key_type = geneva_drive_key_type
+    adapter = connection.adapter_name.downcase
+
+    # Build reference options - we add foreign key separately to avoid MySQL type mismatch
+    reference_options = {
+      null: false,
+      index: true
+    }
+    reference_options[:type] = key_type if key_type == :uuid
 
     create_table :geneva_drive_step_executions, **geneva_drive_table_options do |t|
       # Link to workflow (cascade delete when workflow is deleted)
-      t.references :workflow,
-        null: false,
-        foreign_key: {to_table: :geneva_drive_workflows, on_delete: :cascade},
-        index: true,
-        type: key_type
+      t.references :workflow, **reference_options
 
       # Which step this execution represents
       t.string :step_name, null: false
@@ -41,6 +45,15 @@ class CreateGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
       # Job tracking (for debugging)
       t.string :job_id
 
+      # Freeform JSON metadata for executor bookkeeping
+      if adapter.include?("postgresql")
+        t.jsonb :metadata
+      elsif adapter.include?("mysql")
+        t.text :metadata, limit: 4_294_967_295
+      else
+        t.text :metadata
+      end
+
       t.timestamps
     end
 
@@ -55,6 +68,13 @@ class CreateGenevaDriveStepExecutions < ActiveRecord::Migration[7.2]
     # Index for common query patterns
     add_index :geneva_drive_step_executions, [:workflow_id, :state]
 
+    # Add foreign key separately to avoid MySQL type mismatch (UNSIGNED vs SIGNED bigint)
+    # MySQL creates primary keys as UNSIGNED but references as SIGNED, causing FK constraint failure
+    unless adapter.include?("mysql")
+      add_foreign_key :geneva_drive_step_executions, :geneva_drive_workflows,
+        column: :workflow_id, on_delete: :cascade
+    end
+
     reversible do |direction|
       direction.up { create_db_specific_indices }
     end

data/lib/generators/geneva_drive/install/templates/create_workflows_migration.rb

@@ -4,13 +4,15 @@ class CreateGenevaDriveWorkflows < ActiveRecord::Migration[7.2]
   include GenevaDrive::MigrationHelpers
 
   def change
+    adapter = connection.adapter_name.downcase
+
     create_table :geneva_drive_workflows, **geneva_drive_table_options do |t|
       # Core identification (STI)
       t.string :type, null: false, index: true
 
-      # Polymorphic association to the hero of the workflow
-      t.string :hero_type, null: false
-      t.column :hero_id, geneva_drive_key_type, null: false
+      # Polymorphic association to the hero of the workflow (optional)
+      t.string :hero_type
+      t.column :hero_id, geneva_drive_key_type
 
       # State machine
       t.string :state, null: false, default: "ready", index: true
@@ -22,6 +24,15 @@ class CreateGenevaDriveWorkflows < ActiveRecord::Migration[7.2]
       # Multiple workflows of same type for same hero
       t.boolean :allow_multiple, default: false, null: false
 
+      # Freeform JSON metadata
+      if adapter.include?("postgresql")
+        t.jsonb :metadata
+      elsif adapter.include?("mysql")
+        t.text :metadata, limit: 4_294_967_295
+      else
+        t.text :metadata
+      end
+
       # Timestamps
       t.datetime :started_at
       t.datetime :transitioned_at

data/lib/generators/geneva_drive/install/templates/initializer.rb.tt

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # How long to keep completed/canceled workflows before automatic deletion.
-# Set to nil to disable automatic cleanup (default).
+# Set to nil to disable automatic cleanup.
 # When enabled, finished and canceled workflows older than this threshold
 # will be deleted along with their step execution records.
 #
@@ -18,7 +18,7 @@
 # still in "scheduled" state before being considered stuck. This can
 # happen if jobs fail to enqueue or are lost by the queue backend.
 #
-# GenevaDrive.stuck_scheduled_threshold = 1.hour
+# GenevaDrive.stuck_scheduled_threshold = 15.minutes
 
 # Action to take when recovering stuck step executions.
 # - :reattempt - Mark as recovered and schedule a retry (default)
@@ -30,3 +30,23 @@
 # housekeeping run. Prevents runaway processing if there's a large backlog.
 #
 # GenevaDrive.housekeeping_batch_size = 1000
+
+# Maximum size in bytes of a resumable step cursor once serialized to JSON.
+# The cursor is a position marker (an id, page number, or pagination token),
+# not a place to store the data being processed - it is rewritten on every
+# checkpoint and copied to every successor execution. Exceeding the limit
+# raises GenevaDrive::CursorTooLargeError. Set to nil to disable the check.
+#
+# GenevaDrive.max_cursor_size = 128.kilobytes
+
+# Whether to defer job enqueueing until after the database transaction commits.
+# In production this ensures step execution records are visible to job workers
+# before the job runs. In test environments (the default when Rails.env.test?)
+# this is set to false so that jobs are enqueued immediately — transactional
+# tests (especially with SQLite) never commit the outermost transaction, which
+# can cause after_commit callbacks to misbehave or not fire at all.
+#
+# Set to true in tests if you need strict transaction semantics for a specific
+# test suite, or if you are not using transactional tests.
+#
+# GenevaDrive.enqueue_after_commit = !Rails.env.test?