job-workflow 0.5.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.agents/instructions/coding-style.md +38 -0
  3. data/.agents/instructions/domain.md +37 -0
  4. data/.agents/instructions/environment.md +44 -0
  5. data/.agents/instructions/general.md +29 -0
  6. data/.agents/instructions/security.md +20 -0
  7. data/.agents/instructions/structure.md +43 -0
  8. data/.agents/instructions/tech-stack.md +40 -0
  9. data/.agents/instructions/testing.md +46 -0
  10. data/.agents/instructions/workflow.md +39 -0
  11. data/.rubocop.yml +1 -2
  12. data/AGENTS.md +23 -0
  13. data/CHANGELOG.md +26 -0
  14. data/README.md +1 -1
  15. data/app/controllers/job_workflow/monitoring/application_controller.rb +11 -0
  16. data/app/controllers/job_workflow/monitoring/executions_controller.rb +28 -0
  17. data/app/controllers/job_workflow/monitoring/workflows_controller.rb +11 -0
  18. data/app/views/job_workflow/monitoring/executions/index.html.erb +57 -0
  19. data/app/views/job_workflow/monitoring/executions/show.html.erb +200 -0
  20. data/app/views/job_workflow/monitoring/workflows/index.html.erb +39 -0
  21. data/app/views/layouts/job_workflow/monitoring/application.html.erb +117 -0
  22. data/config/routes.rb +8 -0
  23. data/guides/API_REFERENCE.md +9 -6
  24. data/guides/DEPENDENCY_WAIT.md +9 -5
  25. data/guides/MONITORING_UI.md +74 -0
  26. data/guides/PARALLEL_PROCESSING.md +33 -21
  27. data/guides/PRODUCTION_DEPLOYMENT.md +1 -1
  28. data/guides/README.md +6 -1
  29. data/guides/THROTTLING.md +24 -0
  30. data/guides/WORKFLOW_STATUS_QUERY.md +7 -1
  31. data/lib/job_workflow/context.rb +7 -5
  32. data/lib/job_workflow/dsl.rb +0 -4
  33. data/lib/job_workflow/instrumentation/opentelemetry_subscriber.rb +1 -1
  34. data/lib/job_workflow/instrumentation.rb +14 -14
  35. data/lib/job_workflow/job_status.rb +16 -1
  36. data/lib/job_workflow/monitoring/dag_layout.rb +186 -0
  37. data/lib/job_workflow/monitoring/engine.rb +15 -0
  38. data/lib/job_workflow/monitoring/execution_page.rb +16 -0
  39. data/lib/job_workflow/monitoring/execution_registry.rb +50 -0
  40. data/lib/job_workflow/monitoring/execution_view_model.rb +258 -0
  41. data/lib/job_workflow/monitoring/parameter_filter.rb +37 -0
  42. data/lib/job_workflow/monitoring/workflow_definition.rb +24 -0
  43. data/lib/job_workflow/monitoring/workflow_registry.rb +24 -0
  44. data/lib/job_workflow/monitoring.rb +120 -0
  45. data/lib/job_workflow/queue_adapters/abstract.rb +7 -2
  46. data/lib/job_workflow/queue_adapters/null_adapter.rb +12 -1
  47. data/lib/job_workflow/queue_adapters/solid_queue_adapter.rb +42 -12
  48. data/lib/job_workflow/railtie.rb +2 -0
  49. data/lib/job_workflow/runner.rb +5 -3
  50. data/lib/job_workflow/sub_task_job.rb +93 -0
  51. data/lib/job_workflow/task_enqueue.rb +19 -12
  52. data/lib/job_workflow/version.rb +1 -1
  53. data/lib/job_workflow/workflow_status.rb +39 -3
  54. data/lib/job_workflow.rb +2 -0
  55. data/rbs_collection.lock.yaml +11 -11
  56. data/sig/generated/job_workflow/context.rbs +7 -7
  57. data/sig/generated/job_workflow/instrumentation/opentelemetry_subscriber.rbs +0 -1
  58. data/sig/generated/job_workflow/instrumentation.rbs +28 -28
  59. data/sig/generated/job_workflow/job_status.rbs +5 -2
  60. data/sig/generated/job_workflow/monitoring/dag_layout.rbs +80 -0
  61. data/sig/generated/job_workflow/monitoring/engine.rbs +8 -0
  62. data/sig/generated/job_workflow/monitoring/execution_page.rbs +14 -0
  63. data/sig/generated/job_workflow/monitoring/execution_registry.rbs +21 -0
  64. data/sig/generated/job_workflow/monitoring/execution_view_model.rbs +111 -0
  65. data/sig/generated/job_workflow/monitoring/parameter_filter.rbs +16 -0
  66. data/sig/generated/job_workflow/monitoring/workflow_definition.rbs +18 -0
  67. data/sig/generated/job_workflow/monitoring/workflow_registry.rbs +13 -0
  68. data/sig/generated/job_workflow/monitoring.rbs +38 -0
  69. data/sig/generated/job_workflow/queue_adapters/abstract.rbs +7 -4
  70. data/sig/generated/job_workflow/queue_adapters/null_adapter.rbs +5 -2
  71. data/sig/generated/job_workflow/queue_adapters/solid_queue_adapter.rbs +18 -6
  72. data/sig/generated/job_workflow/runner.rbs +1 -1
  73. data/sig/generated/job_workflow/sub_task_job.rbs +40 -0
  74. data/sig/generated/job_workflow/task_enqueue.rbs +5 -8
  75. data/sig/generated/job_workflow/workflow_status.rbs +18 -2
  76. data/sig-private/job-workflow.rbs +11 -0
  77. data/sig-private/rails.rbs +5 -0
  78. metadata +42 -1
@@ -0,0 +1,200 @@
1
+ <% status_class = case @execution.workflow_status
2
+ when :succeeded then "text-bg-success"
3
+ when :failed then "text-bg-danger"
4
+ when :running then "text-bg-primary"
5
+ else "text-bg-secondary"
6
+ end %>
7
+ <% dag_layout = @execution.dag_layout %>
8
+
9
+ <section class="d-flex flex-column flex-lg-row justify-content-between align-items-lg-start gap-3 mb-4">
10
+ <div>
11
+ <div class="text-body-secondary small mb-1">Execution</div>
12
+ <h1 class="display-6 fw-semibold mb-3"><%= @execution.job_class_name %></h1>
13
+ <div class="d-flex flex-wrap gap-2">
14
+ <span class="badge <%= status_class %>"><%= @execution.workflow_status %></span>
15
+ <span class="badge text-bg-light border text-dark">task <%= @execution.current_task_name || "-" %></span>
16
+ <span class="badge text-bg-light border text-dark">failed <%= @execution.failed_task_name || "-" %></span>
17
+ </div>
18
+ </div>
19
+ <div class="d-flex flex-wrap gap-2">
20
+ <%= link_to "Back to executions", workflow_executions_path(@workflow.name), class: "btn btn-outline-secondary" %>
21
+ <% if @execution.mission_control_job_path %>
22
+ <%= link_to "Mission Control Jobs", @execution.mission_control_job_path, class: "btn btn-primary" %>
23
+ <% end %>
24
+ </div>
25
+ </section>
26
+
27
+ <div class="row g-4 mb-4">
28
+ <div class="col-12 col-xl-4">
29
+ <div class="card jw-card h-100">
30
+ <div class="card-header bg-white fw-semibold">Execution Summary</div>
31
+ <div class="card-body">
32
+ <dl class="row mb-0">
33
+ <dt class="col-sm-4">Job ID</dt>
34
+ <dd class="col-sm-8 jw-mono"><%= @execution.job_id %></dd>
35
+ <dt class="col-sm-4">Queue</dt>
36
+ <dd class="col-sm-8"><%= @execution.queue_name || "-" %></dd>
37
+ <dt class="col-sm-4">Current</dt>
38
+ <dd class="col-sm-8"><%= @execution.current_task_name || "-" %></dd>
39
+ <dt class="col-sm-4">Failed</dt>
40
+ <dd class="col-sm-8"><%= @execution.failed_task_name || "-" %></dd>
41
+ </dl>
42
+ </div>
43
+ </div>
44
+ </div>
45
+
46
+ <div class="col-12 col-xl-8">
47
+ <div class="card jw-card h-100">
48
+ <div class="card-header bg-white fw-semibold">Arguments</div>
49
+ <div class="card-body">
50
+ <pre class="jw-pre"><%= JSON.pretty_generate(@execution.filtered_arguments.as_json) %></pre>
51
+ </div>
52
+ </div>
53
+ </div>
54
+ </div>
55
+
56
+ <div class="card jw-card">
57
+ <div class="card-header bg-white fw-semibold">DAG</div>
58
+ <% if dag_layout[:nodes].any? %>
59
+ <div class="card-body border-bottom" aria-hidden="true">
60
+ <div class="small text-body-secondary mb-3">graph overview</div>
61
+ <div class="jw-dag-viewport">
62
+ <div
63
+ class="jw-dag-canvas"
64
+ style="width: <%= dag_layout[:width] %>px; height: <%= dag_layout[:height] %>px;"
65
+ >
66
+ <svg
67
+ class="jw-dag-svg"
68
+ viewBox="0 0 <%= dag_layout[:width] %> <%= dag_layout[:height] %>"
69
+ preserveAspectRatio="xMinYMin meet"
70
+ >
71
+ <% dag_layout[:edges].each do |edge| %>
72
+ <path class="jw-dag-edge" d="<%= edge[:path] %>"></path>
73
+ <% end %>
74
+ </svg>
75
+
76
+ <% dag_layout[:nodes].each do |task| %>
77
+ <% graph_status_class = case task[:status]
78
+ when :succeeded then "text-bg-success"
79
+ when :failed then "text-bg-danger"
80
+ when :running then "text-bg-primary"
81
+ else "text-bg-secondary"
82
+ end %>
83
+ <div
84
+ class="jw-dag-node"
85
+ style="left: <%= task[:x] %>px; top: <%= task[:y] %>px; width: <%= task[:width] %>px; height: <%= task[:height] %>px;"
86
+ >
87
+ <div class="d-flex flex-column align-items-start gap-1">
88
+ <div class="fw-semibold jw-dag-node-title" title="<%= task[:label] %>"><%= task[:truncated_label] %></div>
89
+ <span class="badge <%= graph_status_class %>"><%= task[:status] %></span>
90
+ </div>
91
+
92
+ <% if task[:meta_label] %>
93
+ <div class="jw-dag-node-meta"><%= task[:meta_label] %></div>
94
+ <% end %>
95
+ </div>
96
+ <% end %>
97
+ </div>
98
+ </div>
99
+ </div>
100
+ <% end %>
101
+ <div class="list-group list-group-flush">
102
+ <% @execution.tasks.each do |task| %>
103
+ <% task_status_class = case task[:status]
104
+ when :succeeded then "text-bg-success"
105
+ when :failed then "text-bg-danger"
106
+ when :running then "text-bg-primary"
107
+ else "text-bg-secondary"
108
+ end %>
109
+ <div class="list-group-item p-4">
110
+ <div class="d-flex flex-column flex-lg-row justify-content-between gap-3">
111
+ <div>
112
+ <div class="d-flex flex-wrap align-items-center gap-2 mb-2">
113
+ <span class="fw-semibold"><%= task[:name] %></span>
114
+ <span class="badge <%= task_status_class %>"><%= task[:status] %></span>
115
+ <% if task[:each] %>
116
+ <span class="badge text-bg-light border text-dark">
117
+ each <%= task[:each_progress][:succeeded] %>/<%= task[:each_progress][:total] %>
118
+ </span>
119
+ <% end %>
120
+ </div>
121
+ <% if task[:depends_on].any? %>
122
+ <div class="text-body-secondary small">depends on <%= task[:depends_on].join(", ") %></div>
123
+ <% end %>
124
+ </div>
125
+
126
+ <% if task[:each] %>
127
+ <div class="text-lg-end">
128
+ <div class="small text-body-secondary mb-1">fan-out progress</div>
129
+ <div class="progress" role="progressbar" aria-valuemin="0" aria-valuemax="<%= task[:each_progress][:total] %>" aria-valuenow="<%= task[:each_progress][:succeeded] %>">
130
+ <div
131
+ class="progress-bar"
132
+ style="width: <%= task[:each_progress][:total].positive? ? (task[:each_progress][:succeeded] * 100 / task[:each_progress][:total]) : 0 %>%"
133
+ ></div>
134
+ </div>
135
+ </div>
136
+ <% end %>
137
+ </div>
138
+
139
+ <div class="row g-3 mt-1">
140
+ <div class="col-12 col-xl-6">
141
+ <div class="small text-body-secondary mb-2">configuration</div>
142
+ <pre class="jw-pre"><%= JSON.pretty_generate(task[:configuration].as_json) %></pre>
143
+ </div>
144
+
145
+ <div class="col-12 col-xl-6">
146
+ <div class="small text-body-secondary mb-2">progress</div>
147
+ <pre class="jw-pre"><%= JSON.pretty_generate(task[:each_progress].as_json) %></pre>
148
+ </div>
149
+ </div>
150
+
151
+ <% if task[:outputs].any? %>
152
+ <div class="mt-3">
153
+ <div class="small text-body-secondary mb-2">outputs</div>
154
+ <pre class="jw-pre"><%= JSON.pretty_generate(task[:outputs].as_json) %></pre>
155
+ </div>
156
+ <% end %>
157
+
158
+ <% if Array(task[:sub_task_jobs]).any? %>
159
+ <div class="mt-3">
160
+ <div class="small text-body-secondary mb-2">sub task jobs</div>
161
+ <div class="table-responsive">
162
+ <table class="table table-sm align-middle mb-0">
163
+ <thead>
164
+ <tr>
165
+ <th>Each</th>
166
+ <th>Status</th>
167
+ <th>Job ID</th>
168
+ <th class="text-end">Mission Control</th>
169
+ </tr>
170
+ </thead>
171
+ <tbody>
172
+ <% Array(task[:sub_task_jobs]).each do |sub_task_job| %>
173
+ <% sub_task_status_class = case sub_task_job[:status]
174
+ when :succeeded then "text-bg-success"
175
+ when :failed then "text-bg-danger"
176
+ when :running then "text-bg-primary"
177
+ else "text-bg-secondary"
178
+ end %>
179
+ <tr>
180
+ <td><%= sub_task_job[:each_index] %></td>
181
+ <td><span class="badge <%= sub_task_status_class %>"><%= sub_task_job[:status] %></span></td>
182
+ <td class="jw-mono"><%= sub_task_job[:job_id] %></td>
183
+ <td class="text-end">
184
+ <% if sub_task_job[:mission_control_job_path] %>
185
+ <%= link_to "Open", sub_task_job[:mission_control_job_path], class: "btn btn-sm btn-outline-primary" %>
186
+ <% else %>
187
+ <span class="text-body-secondary">-</span>
188
+ <% end %>
189
+ </td>
190
+ </tr>
191
+ <% end %>
192
+ </tbody>
193
+ </table>
194
+ </div>
195
+ </div>
196
+ <% end %>
197
+ </div>
198
+ <% end %>
199
+ </div>
200
+ </div>
@@ -0,0 +1,39 @@
1
+ <section class="mb-4">
2
+ <h1 class="display-6 fw-semibold mb-2">Workflow Definitions</h1>
3
+ <p class="text-body-secondary mb-0">Root execution only. Select one workflow, then drill into its executions.</p>
4
+ </section>
5
+
6
+ <div class="card jw-card">
7
+ <div class="card-body p-0">
8
+ <% if @workflows.empty? %>
9
+ <div class="p-4 text-body-secondary">No workflows are registered.</div>
10
+ <% else %>
11
+ <div class="table-responsive">
12
+ <table class="table table-hover align-middle mb-0">
13
+ <thead class="table-light">
14
+ <tr>
15
+ <th class="ps-4">Workflow</th>
16
+ <th>Tasks</th>
17
+ <th class="text-end pe-4">Executions</th>
18
+ </tr>
19
+ </thead>
20
+ <tbody>
21
+ <% @workflows.each do |workflow| %>
22
+ <tr>
23
+ <td class="ps-4">
24
+ <div class="fw-semibold"><%= workflow.job_class_name %></div>
25
+ </td>
26
+ <td>
27
+ <span class="badge text-bg-secondary"><%= workflow.task_count %> tasks</span>
28
+ </td>
29
+ <td class="text-end pe-4">
30
+ <%= link_to "View executions", workflow_executions_path(workflow.job_class_name), class: "btn btn-sm btn-primary" %>
31
+ </td>
32
+ </tr>
33
+ <% end %>
34
+ </tbody>
35
+ </table>
36
+ </div>
37
+ <% end %>
38
+ </div>
39
+ </div>
@@ -0,0 +1,117 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>JobWorkflow Monitoring</title>
7
+ <link
8
+ href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.8/dist/css/bootstrap.min.css"
9
+ rel="stylesheet"
10
+ integrity="sha384-sRIl4kxILFvY47J16cr9ZwB07vP4J8+LH7qKQnuqkuIAvNWLzeN8tE5YBujZqJLB"
11
+ crossorigin="anonymous"
12
+ >
13
+ <style>
14
+ :root {
15
+ --jw-surface: #f8f9fa;
16
+ --jw-border: #dee2e6;
17
+ }
18
+
19
+ body {
20
+ background:
21
+ radial-gradient(circle at top right, rgba(13, 110, 253, 0.08), transparent 24rem),
22
+ linear-gradient(180deg, #f8fbff 0%, #f8f9fa 100%);
23
+ }
24
+
25
+ .jw-shell {
26
+ max-width: 1200px;
27
+ }
28
+
29
+ .jw-mono {
30
+ font-family: ui-monospace, SFMono-Regular, SFMono-Regular, Menlo, Consolas, monospace;
31
+ word-break: break-all;
32
+ }
33
+
34
+ .jw-pre {
35
+ background: #212529;
36
+ color: #f8f9fa;
37
+ border-radius: 0.75rem;
38
+ padding: 1rem;
39
+ margin: 0;
40
+ white-space: pre-wrap;
41
+ }
42
+
43
+ .jw-card {
44
+ border-color: var(--jw-border);
45
+ box-shadow: 0 0.5rem 1.5rem rgba(33, 37, 41, 0.06);
46
+ }
47
+
48
+ .jw-dag-viewport {
49
+ overflow-x: auto;
50
+ padding-bottom: 0.25rem;
51
+ }
52
+
53
+ .jw-dag-canvas {
54
+ position: relative;
55
+ min-height: 12rem;
56
+ }
57
+
58
+ .jw-dag-svg {
59
+ position: absolute;
60
+ inset: 0;
61
+ overflow: visible;
62
+ }
63
+
64
+ .jw-dag-edge {
65
+ fill: none;
66
+ stroke: #adb5bd;
67
+ stroke-width: 2;
68
+ stroke-linecap: round;
69
+ stroke-linejoin: round;
70
+ }
71
+
72
+ .jw-dag-node {
73
+ position: absolute;
74
+ display: flex;
75
+ flex-direction: column;
76
+ justify-content: flex-start;
77
+ gap: 0.4rem;
78
+ background: #fff;
79
+ border: 1px solid var(--jw-border);
80
+ border-radius: 0.85rem;
81
+ box-shadow: 0 0.4rem 1rem rgba(33, 37, 41, 0.08);
82
+ padding: 0.75rem 0.875rem;
83
+ }
84
+
85
+ .jw-dag-node-title {
86
+ font-size: 0.875rem;
87
+ line-height: 1.15;
88
+ max-width: 10rem;
89
+ overflow: hidden;
90
+ text-overflow: ellipsis;
91
+ white-space: nowrap;
92
+ }
93
+
94
+ .jw-dag-node-meta {
95
+ color: #6c757d;
96
+ font-size: 0.72rem;
97
+ }
98
+ </style>
99
+ </head>
100
+ <body>
101
+ <nav class="navbar navbar-expand-lg bg-white border-bottom sticky-top">
102
+ <div class="container-fluid jw-shell px-3 px-lg-4">
103
+ <a class="navbar-brand fw-semibold" href="<%= root_path %>">JobWorkflow Monitoring</a>
104
+ </div>
105
+ </nav>
106
+
107
+ <main class="container-fluid jw-shell py-4 py-lg-5 px-3 px-lg-4">
108
+ <%= yield %>
109
+ </main>
110
+
111
+ <script
112
+ src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.8/dist/js/bootstrap.bundle.min.js"
113
+ integrity="sha384-FKyoEForCGlyvwx9Hj09JcYn3nv7wiPVlz7YYwJrWVcXK/BmnVDxM+D2scQbITxI"
114
+ crossorigin="anonymous"
115
+ ></script>
116
+ </body>
117
+ </html>
data/config/routes.rb ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ JobWorkflow::Monitoring::Engine.routes.draw do
4
+ resources :workflows, only: [:index], param: :job_class_name do
5
+ resources :executions, only: %i[index show]
6
+ end
7
+ root "workflows#index"
8
+ end
@@ -18,10 +18,9 @@ task(name, **options, &block)
18
18
  - `depends_on` (Symbol | Array[Symbol]): Dependent tasks
19
19
  - `each` (Proc): Proc that returns an enumerable for map task execution
20
20
  - `enqueue` (Hash | Proc | bool): Controls whether task iterations are enqueued as sub-jobs
21
- - Hash format (recommended): `{ condition: Proc, queue: String, concurrency: Integer }`
21
+ - Hash format (recommended): `{ condition: Proc, queue: String }`
22
22
  - `condition` (Proc | bool): Determines if task should be enqueued (default: true if Hash is not empty)
23
23
  - `queue` (String): Custom queue name for the task (optional)
24
- - `concurrency` (Integer): Concurrency limit for parallel processing (default: unlimited)
25
24
  - Proc format (legacy): Proc that returns boolean
26
25
  - bool format: true/false for simple cases
27
26
  - Default: nil (synchronous execution)
@@ -75,7 +74,8 @@ end
75
74
  # Parallel processing with collection
76
75
  task :process_items,
77
76
  each: ->(ctx) { ctx.arguments.items },
78
- enqueue: { concurrency: 5 },
77
+ enqueue: true,
78
+ throttle: 5,
79
79
  output: { result: "String" } do |ctx|
80
80
  item = ctx.each_value
81
81
  { result: ProcessService.handle(item) }
@@ -187,7 +187,8 @@ class ImportJob < ApplicationJob
187
187
 
188
188
  task :process,
189
189
  each: ->(ctx) { ctx.arguments.items },
190
- enqueue: { concurrency: 5 },
190
+ enqueue: true,
191
+ throttle: 5,
191
192
  output: { result: "String" } do |ctx|
192
193
  { result: handle(ctx.each_value) }
193
194
  end
@@ -211,7 +212,8 @@ class BatchImportJob < ApplicationJob
211
212
 
212
213
  task :process,
213
214
  each: ->(ctx) { ctx.arguments.items },
214
- enqueue: { concurrency: 5 },
215
+ enqueue: true,
216
+ throttle: 5,
215
217
  output: { result: "String" } do |ctx|
216
218
  { result: handle(ctx.each_value) }
217
219
  end
@@ -228,7 +230,8 @@ argument :items, "Array[String]"
228
230
 
229
231
  task :process_items,
230
232
  each: ->(ctx) { ctx.arguments.items },
231
- enqueue: { concurrency: 5 },
233
+ enqueue: true,
234
+ throttle: 5,
232
235
  output: { result: "String", status: "Symbol" } do |ctx|
233
236
  item = ctx.each_value
234
237
  {
@@ -14,7 +14,8 @@ class ExampleJob < ApplicationJob
14
14
 
15
15
  task :process_items,
16
16
  each: ->(ctx) { ctx.arguments.items },
17
- enqueue: { concurrency: 5 },
17
+ enqueue: true,
18
+ throttle: 5,
18
19
  output: { result: "Integer" } do |ctx|
19
20
  # This creates many sub-jobs
20
21
  { result: ctx.each_value * 2 }
@@ -172,7 +173,8 @@ class DataPipelineJob < ApplicationJob
172
173
  # Extract data from multiple sources in parallel
173
174
  task :extract_data,
174
175
  each: ->(ctx) { %w[users orders products inventory] },
175
- enqueue: { concurrency: 4 },
176
+ enqueue: true,
177
+ throttle: 4,
176
178
  output: { source: "String", count: "Integer" } do |ctx|
177
179
  source = ctx.each_value
178
180
  data = DataSource.fetch(source, date: ctx.arguments.date)
@@ -210,10 +212,11 @@ class APIAggregatorJob < ApplicationJob
210
212
 
211
213
  argument :user_ids, "Array[Integer]"
212
214
 
213
- # Fetch user data with rate limiting
215
+ # Fetch user data with rate limiting.
216
+ # Async fan-out is unbounded here; the official execution cap is throttle: 5.
214
217
  task :fetch_users,
215
218
  each: ->(ctx) { ctx.arguments.user_ids },
216
- enqueue: { concurrency: 10 },
219
+ enqueue: true,
217
220
  throttle: { key: "external_api", limit: 5 },
218
221
  output: { user_id: "Integer", data: "Hash" } do |ctx|
219
222
  user_id = ctx.each_value
@@ -270,7 +273,8 @@ dependency_wait: { poll_timeout: 60, reschedule_delay: 10 }
270
273
  # ✅ Good: dependency_wait with parallel sub-jobs
271
274
  task :process,
272
275
  each: ->(ctx) { ctx.arguments.items },
273
- enqueue: { concurrency: 10 } do |ctx|
276
+ enqueue: true,
277
+ throttle: 10 do |ctx|
274
278
  heavy_process(ctx.each_value)
275
279
  end
276
280
 
@@ -0,0 +1,74 @@
1
+ # Monitoring UI
2
+
3
+ ## Overview
4
+
5
+ JobWorkflow ships with a workflow-oriented monitoring UI. Instead of listing mixed job rows first, the UI starts
6
+ from workflow definitions, then lets you drill into one workflow's root executions and finally into one execution's
7
+ DAG state.
8
+
9
+ This view is intended to answer workflow-level questions such as:
10
+
11
+ - which workflow is currently stuck
12
+ - which task is running or failed
13
+ - how `each` fan-out is progressing
14
+ - which arguments and outputs shaped the current execution
15
+
16
+ ## What the UI shows
17
+
18
+ The current scope includes:
19
+
20
+ - workflow definition list
21
+ - paginated root execution list per workflow
22
+ - execution detail with a DAG overview, task state, arguments, outputs, and failed task
23
+ - fan-out progress and sub-task job links into Mission Control Jobs
24
+
25
+ History analytics, retries, and dry-run launch flows are out of scope for now.
26
+
27
+ ## Navigation
28
+
29
+ The UI is organized around workflows rather than a cross-workflow execution feed:
30
+
31
+ ```text
32
+ workflow definitions
33
+ └─ one workflow's root executions
34
+ └─ one root execution with sub-task-job detail
35
+ ```
36
+
37
+ The UI is intentionally scoped to one workflow at a time, so the first screen stays focused on definitions and the
38
+ execution list stays easy to scan.
39
+
40
+ ## Mounting the engine
41
+
42
+ Add the engine to your application's routes:
43
+
44
+ ```ruby
45
+ # config/routes.rb
46
+ mount JobWorkflow::Monitoring::Engine => "/job_workflow"
47
+ ```
48
+
49
+ After mounting, open `/job_workflow` to browse workflow definitions and executions.
50
+
51
+ ## Authentication and controller inheritance
52
+
53
+ By default, monitoring controllers inherit from `ApplicationController`. If you already use a dedicated authenticated
54
+ controller for admin tooling, configure monitoring to inherit from it:
55
+
56
+ ```ruby
57
+ config.job_workflow.monitoring.base_controller_class = "AdminController"
58
+ ```
59
+
60
+ If `config.job_workflow.monitoring.base_controller_class` is not set and `MissionControl::Jobs` is installed,
61
+ monitoring falls back to `MissionControl::Jobs.base_controller_class`.
62
+
63
+ ## Root executions and sub-task jobs
64
+
65
+ Execution lists show **root jobs only**. `SubTaskJob` rows do not appear in the workflow execution list. Instead, the
66
+ detail page shows sub-task job state only after you open one root execution.
67
+
68
+ This keeps the list view focused on workflow-level monitoring while still preserving the full fan-out story on the
69
+ detail page.
70
+
71
+ ## Query behavior
72
+
73
+ Root executions are paginated with a cursor and scoped by workflow class. As a user, this means the execution list is
74
+ ordered newest-first within the workflow you selected, without mixing in unrelated job rows.
@@ -54,9 +54,9 @@ task :process_items,
54
54
  end
55
55
  ```
56
56
 
57
- ### Asynchronous Execution with Concurrency
57
+ ### Asynchronous Execution with Throttled Concurrency
58
58
 
59
- To execute map task iterations in separate sub-jobs with concurrency control, use the `enqueue:` option with a Hash containing `condition:` and `concurrency:`:
59
+ To execute map task iterations in separate sub-jobs, enable `enqueue:` and use `throttle:` when you need to cap concurrent execution:
60
60
 
61
61
  ```ruby
62
62
  # Simplest form: enable parallel execution with default settings
@@ -69,21 +69,23 @@ end
69
69
  # Process up to 10 items concurrently in sub-jobs
70
70
  task :process_items,
71
71
  each: ->(ctx) { ctx.arguments.items },
72
- enqueue: { condition: ->(_ctx) { true }, concurrency: 10 } do |ctx|
72
+ enqueue: true,
73
+ throttle: 10 do |ctx|
73
74
  process_item(ctx.each_value)
74
75
  end
75
76
 
76
- # Simplified syntax when condition is implicitly true
77
+ # Conditional enqueue with a concurrency cap
77
78
  task :process_items,
78
79
  each: ->(ctx) { ctx.arguments.items },
79
- enqueue: { concurrency: 10 } do |ctx|
80
+ enqueue: { condition: ->(ctx) { ctx.arguments.use_async? } },
81
+ throttle: 10 do |ctx|
80
82
  process_item(ctx.each_value)
81
83
  end
82
84
 
83
85
  # When enqueue is enabled:
84
86
  # - Each iteration is executed in a separate sub-job
85
87
  # - Sub-jobs are created via perform_all_later
86
- # - Concurrency limit controls how many sub-jobs run in parallel
88
+ # - `throttle` controls how many iterations can execute concurrently
87
89
  # - Parent job waits for all sub-jobs to complete before continuing
88
90
  # - Outputs from sub-jobs are automatically collected
89
91
  ```
@@ -99,14 +101,18 @@ The `enqueue:` option determines how map task iterations are executed:
99
101
 
100
102
  - **`enqueue: true`**: Each iteration is enqueued as a separate sub-job with default settings
101
103
  - Simplest way to enable parallel execution
102
- - No concurrency limit (executes as fast as workers allow)
104
+ - No throttle limit (executes as fast as workers allow)
103
105
  - Good for I/O-bound operations with many workers
104
106
 
105
- - **`enqueue: { condition: ->(_ctx) { true }, concurrency: 10 }`**: Each iteration is enqueued as a separate sub-job
106
- - Enables true parallel execution across multiple workers
107
+ - **`enqueue: { condition: ... }`**: Each iteration is enqueued as a separate sub-job when the condition passes
107
108
  - Better for I/O-bound operations (API calls, database queries)
108
- - Can accept dynamic condition: `enqueue: { condition: ->(ctx) { ctx.arguments.use_concurrency? } }`
109
- - Supports `queue:` option for custom queue: `enqueue: { queue: "critical", concurrency: 5 }`
109
+ - Can accept dynamic condition: `enqueue: { condition: ->(ctx) { ctx.arguments.use_async? } }`
110
+ - Supports `queue:` option for custom queue: `enqueue: { queue: "critical" }`
111
+
112
+ - **`throttle: 10`**: Limits how many task executions can run concurrently
113
+ - This is the official way to cap async map task parallelism
114
+ - It is enforced at perform time via JobWorkflow semaphores
115
+ - It does not use SolidQueue `ready` / `blocked` dispatch-state controls
110
116
 
111
117
  **Note**: `enqueue:` works with both regular tasks and map tasks. For map tasks, it enables asynchronous sub-job execution. For regular tasks, it allows conditional enqueueing as a separate job. Legacy syntax (`enqueue: ->(_ctx) { true }` as a Proc) is still supported for backward compatibility.
112
118
 
@@ -129,7 +135,8 @@ class ImportJob < ApplicationJob
129
135
 
130
136
  task :process_items,
131
137
  each: ->(ctx) { ctx.arguments.items },
132
- enqueue: { concurrency: 5 },
138
+ enqueue: true,
139
+ throttle: 5,
133
140
  output: { result: "String" } do |ctx|
134
141
  { result: process(ctx.each_value) }
135
142
  end
@@ -238,7 +245,8 @@ class DataProcessingJob < ApplicationJob
238
245
  task :process_by_region,
239
246
  each: ->(ctx) { ctx.arguments.regions },
240
247
  output: { region: "String", results: "Array[Hash]" },
241
- enqueue: { concurrency: 5 } do |ctx|
248
+ enqueue: true,
249
+ throttle: 5 do |ctx|
242
250
  region = ctx.each_value
243
251
  # This will create sub-tasks for each region
244
252
  { region: region, results: [] }
@@ -261,7 +269,8 @@ class DataProcessingJob < ApplicationJob
261
269
  },
262
270
  depends_on: [:process_by_region],
263
271
  output: { region: "String", data_type: "String", result: "Hash" },
264
- enqueue: { concurrency: 10 } do |ctx|
272
+ enqueue: true,
273
+ throttle: 10 do |ctx|
265
274
  item = ctx.each_value
266
275
  region = item[:region]
267
276
  data_type = item[:data_type]
@@ -290,7 +299,7 @@ DataProcessingJob.perform_later(
290
299
  regions: ["us-east-1", "us-west-1", "eu-west-1"],
291
300
  data_types: ["user", "order", "product"]
292
301
  )
293
- # => 3 regions × 3 data types = 9 parallel iterations (with concurrency limits)
302
+ # => 3 regions × 3 data types = 9 parallel iterations (with throttled concurrency)
294
303
  ```
295
304
 
296
305
  ### Advanced Matrix with Filtering
@@ -318,7 +327,8 @@ task :process_filtered_matrix,
318
327
  end
319
328
  },
320
329
  output: { region: "String", data_type: "String", status: "Symbol" },
321
- enqueue: { concurrency: 10 } do |ctx|
330
+ enqueue: true,
331
+ throttle: 10 do |ctx|
322
332
  combo = ctx.each_value
323
333
  region = combo[:region]
324
334
  data_type = combo[:data_type]
@@ -335,9 +345,9 @@ end
335
345
 
336
346
  When implementing matrix processing:
337
347
 
338
- 1. **Concurrency Control**: Set appropriate `concurrency:` limits to avoid overwhelming workers
339
- - High concurrency (20+): Suitable for I/O-bound operations (API calls, database queries)
340
- - Low concurrency (2-5): Better for CPU-bound operations or rate-limited APIs
348
+ 1. **Concurrency Control**: Set appropriate `throttle:` limits to avoid overwhelming workers
349
+ - High throttle (20+): Suitable for I/O-bound operations (API calls, database queries)
350
+ - Low throttle (2-5): Better for CPU-bound operations or rate-limited APIs
341
351
 
342
352
  2. **Output Size**: Watch out for large output collections
343
353
  - With N×M combinations, the output array will have N×M elements
@@ -348,7 +358,8 @@ When implementing matrix processing:
348
358
  task :process_matrix,
349
359
  each: ->(_ctx) { combinations },
350
360
  timeout: 300.seconds, # 5 minutes per iteration
351
- enqueue: { concurrency: 5 } do |ctx|
361
+ enqueue: true,
362
+ throttle: 5 do |ctx|
352
363
  # ...
353
364
  end
354
365
  ```
@@ -358,7 +369,8 @@ When implementing matrix processing:
358
369
  task :process_matrix,
359
370
  each: ->(_ctx) { combinations },
360
371
  retry: { count: 3, strategy: :exponential },
361
- enqueue: { concurrency: 5 } do |ctx|
372
+ enqueue: true,
373
+ throttle: 5 do |ctx|
362
374
  # ...
363
375
  end
364
376
  ```