dispatch_policy 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/MIT-LICENSE +16 -17
- data/README.md +433 -388
- data/app/assets/stylesheets/dispatch_policy/application.css +157 -0
- data/app/controllers/dispatch_policy/application_controller.rb +45 -1
- data/app/controllers/dispatch_policy/dashboard_controller.rb +91 -0
- data/app/controllers/dispatch_policy/partitions_controller.rb +122 -0
- data/app/controllers/dispatch_policy/policies_controller.rb +94 -267
- data/app/controllers/dispatch_policy/staged_jobs_controller.rb +9 -0
- data/app/models/dispatch_policy/adaptive_concurrency_stats.rb +11 -81
- data/app/models/dispatch_policy/inflight_job.rb +12 -0
- data/app/models/dispatch_policy/partition.rb +21 -0
- data/app/models/dispatch_policy/staged_job.rb +4 -97
- data/app/models/dispatch_policy/tick_sample.rb +11 -0
- data/app/views/dispatch_policy/dashboard/index.html.erb +109 -0
- data/app/views/dispatch_policy/partitions/index.html.erb +63 -0
- data/app/views/dispatch_policy/partitions/show.html.erb +106 -0
- data/app/views/dispatch_policy/policies/index.html.erb +15 -37
- data/app/views/dispatch_policy/policies/show.html.erb +139 -223
- data/app/views/dispatch_policy/shared/_capacity.html.erb +67 -0
- data/app/views/dispatch_policy/shared/_hints.html.erb +13 -0
- data/app/views/dispatch_policy/shared/_partition_row.html.erb +12 -0
- data/app/views/dispatch_policy/staged_jobs/show.html.erb +31 -0
- data/app/views/layouts/dispatch_policy/application.html.erb +95 -238
- data/config/routes.rb +18 -2
- data/db/migrate/20260501000001_create_dispatch_policy_tables.rb +103 -0
- data/lib/dispatch_policy/bypass.rb +23 -0
- data/lib/dispatch_policy/config.rb +85 -0
- data/lib/dispatch_policy/context.rb +50 -0
- data/lib/dispatch_policy/cursor_pagination.rb +121 -0
- data/lib/dispatch_policy/decision.rb +22 -0
- data/lib/dispatch_policy/engine.rb +4 -27
- data/lib/dispatch_policy/forwarder.rb +63 -0
- data/lib/dispatch_policy/gate.rb +10 -38
- data/lib/dispatch_policy/gates/adaptive_concurrency.rb +99 -97
- data/lib/dispatch_policy/gates/concurrency.rb +45 -26
- data/lib/dispatch_policy/gates/throttle.rb +65 -41
- data/lib/dispatch_policy/inflight_tracker.rb +174 -0
- data/lib/dispatch_policy/job_extension.rb +155 -0
- data/lib/dispatch_policy/operator_hints.rb +126 -0
- data/lib/dispatch_policy/pipeline.rb +48 -0
- data/lib/dispatch_policy/policy.rb +61 -59
- data/lib/dispatch_policy/policy_dsl.rb +120 -0
- data/lib/dispatch_policy/railtie.rb +35 -0
- data/lib/dispatch_policy/registry.rb +46 -0
- data/lib/dispatch_policy/repository.rb +723 -0
- data/lib/dispatch_policy/serializer.rb +36 -0
- data/lib/dispatch_policy/tick.rb +260 -256
- data/lib/dispatch_policy/tick_loop.rb +59 -26
- data/lib/dispatch_policy/version.rb +1 -1
- data/lib/dispatch_policy.rb +71 -52
- data/lib/generators/dispatch_policy/install/install_generator.rb +70 -0
- data/lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt +95 -0
- data/lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt +53 -0
- data/lib/generators/dispatch_policy/install/templates/initializer.rb.tt +11 -0
- metadata +101 -43
- data/CHANGELOG.md +0 -43
- data/app/models/dispatch_policy/partition_inflight_count.rb +0 -42
- data/app/models/dispatch_policy/partition_observation.rb +0 -76
- data/app/models/dispatch_policy/throttle_bucket.rb +0 -41
- data/db/migrate/20260424000001_create_dispatch_policy_tables.rb +0 -80
- data/db/migrate/20260424000002_create_adaptive_concurrency_stats.rb +0 -22
- data/db/migrate/20260424000003_create_adaptive_concurrency_samples.rb +0 -25
- data/db/migrate/20260424000004_rename_samples_to_partition_observations.rb +0 -32
- data/db/migrate/20260425000001_add_duration_to_partition_observations.rb +0 -8
- data/lib/dispatch_policy/active_job_perform_all_later_patch.rb +0 -32
- data/lib/dispatch_policy/dispatch_context.rb +0 -53
- data/lib/dispatch_policy/dispatchable.rb +0 -123
- data/lib/dispatch_policy/gates/fair_interleave.rb +0 -32
- data/lib/dispatch_policy/gates/global_cap.rb +0 -26
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dispatch_policy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- José Galisteo
|
|
@@ -10,7 +10,7 @@ cert_chain: []
|
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
|
-
name:
|
|
13
|
+
name: rails
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
@@ -24,77 +24,119 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '7.1'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: pg
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - ">="
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
32
|
+
version: '1.4'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - ">="
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
39
|
+
version: '1.4'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
|
-
name:
|
|
41
|
+
name: minitest
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - "~>"
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '5.20'
|
|
47
|
+
type: :development
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '5.20'
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: rake
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - "~>"
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '13.0'
|
|
61
|
+
type: :development
|
|
62
|
+
prerelease: false
|
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - "~>"
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '13.0'
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: good_job
|
|
42
70
|
requirement: !ruby/object:Gem::Requirement
|
|
43
71
|
requirements:
|
|
44
72
|
- - ">="
|
|
45
73
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
47
|
-
type: :
|
|
74
|
+
version: '4.0'
|
|
75
|
+
type: :development
|
|
48
76
|
prerelease: false
|
|
49
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
50
78
|
requirements:
|
|
51
79
|
- - ">="
|
|
52
80
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '
|
|
81
|
+
version: '4.0'
|
|
54
82
|
- !ruby/object:Gem::Dependency
|
|
55
|
-
name:
|
|
83
|
+
name: solid_queue
|
|
56
84
|
requirement: !ruby/object:Gem::Requirement
|
|
57
85
|
requirements:
|
|
58
86
|
- - ">="
|
|
59
87
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '0'
|
|
88
|
+
version: '1.0'
|
|
61
89
|
type: :development
|
|
62
90
|
prerelease: false
|
|
63
91
|
version_requirements: !ruby/object:Gem::Requirement
|
|
64
92
|
requirements:
|
|
65
93
|
- - ">="
|
|
66
94
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '0'
|
|
95
|
+
version: '1.0'
|
|
68
96
|
- !ruby/object:Gem::Dependency
|
|
69
|
-
name:
|
|
97
|
+
name: turbo-rails
|
|
70
98
|
requirement: !ruby/object:Gem::Requirement
|
|
71
99
|
requirements:
|
|
72
100
|
- - ">="
|
|
73
101
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '
|
|
102
|
+
version: '1.5'
|
|
75
103
|
type: :development
|
|
76
104
|
prerelease: false
|
|
77
105
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
106
|
requirements:
|
|
79
107
|
- - ">="
|
|
80
108
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: '
|
|
109
|
+
version: '1.5'
|
|
82
110
|
- !ruby/object:Gem::Dependency
|
|
83
|
-
name:
|
|
111
|
+
name: puma
|
|
84
112
|
requirement: !ruby/object:Gem::Requirement
|
|
85
113
|
requirements:
|
|
86
114
|
- - ">="
|
|
87
115
|
- !ruby/object:Gem::Version
|
|
88
|
-
version: '0'
|
|
116
|
+
version: '6.0'
|
|
89
117
|
type: :development
|
|
90
118
|
prerelease: false
|
|
91
119
|
version_requirements: !ruby/object:Gem::Requirement
|
|
92
120
|
requirements:
|
|
93
121
|
- - ">="
|
|
94
122
|
- !ruby/object:Gem::Version
|
|
95
|
-
version: '0'
|
|
123
|
+
version: '6.0'
|
|
124
|
+
- !ruby/object:Gem::Dependency
|
|
125
|
+
name: foreman
|
|
126
|
+
requirement: !ruby/object:Gem::Requirement
|
|
127
|
+
requirements:
|
|
128
|
+
- - ">="
|
|
129
|
+
- !ruby/object:Gem::Version
|
|
130
|
+
version: '0.87'
|
|
131
|
+
type: :development
|
|
132
|
+
prerelease: false
|
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - ">="
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '0.87'
|
|
96
138
|
- !ruby/object:Gem::Dependency
|
|
97
|
-
name:
|
|
139
|
+
name: debug
|
|
98
140
|
requirement: !ruby/object:Gem::Requirement
|
|
99
141
|
requirements:
|
|
100
142
|
- - ">="
|
|
@@ -107,60 +149,77 @@ dependencies:
|
|
|
107
149
|
- - ">="
|
|
108
150
|
- !ruby/object:Gem::Version
|
|
109
151
|
version: '0'
|
|
110
|
-
description:
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
152
|
+
description: Stages perform_later into a dedicated table, runs a tick loop that admits
|
|
153
|
+
jobs through declared gates (throttle, concurrency), then forwards survivors to
|
|
154
|
+
the real ActiveJob adapter. Embedded as a periodic job. Compatible with good_job
|
|
155
|
+
and solid_queue.
|
|
114
156
|
email:
|
|
115
157
|
- ceritium@gmail.com
|
|
116
158
|
executables: []
|
|
117
159
|
extensions: []
|
|
118
160
|
extra_rdoc_files: []
|
|
119
161
|
files:
|
|
120
|
-
- CHANGELOG.md
|
|
121
162
|
- MIT-LICENSE
|
|
122
163
|
- README.md
|
|
164
|
+
- app/assets/stylesheets/dispatch_policy/application.css
|
|
123
165
|
- app/controllers/dispatch_policy/application_controller.rb
|
|
166
|
+
- app/controllers/dispatch_policy/dashboard_controller.rb
|
|
167
|
+
- app/controllers/dispatch_policy/partitions_controller.rb
|
|
124
168
|
- app/controllers/dispatch_policy/policies_controller.rb
|
|
169
|
+
- app/controllers/dispatch_policy/staged_jobs_controller.rb
|
|
125
170
|
- app/models/dispatch_policy/adaptive_concurrency_stats.rb
|
|
126
171
|
- app/models/dispatch_policy/application_record.rb
|
|
127
|
-
- app/models/dispatch_policy/
|
|
128
|
-
- app/models/dispatch_policy/
|
|
172
|
+
- app/models/dispatch_policy/inflight_job.rb
|
|
173
|
+
- app/models/dispatch_policy/partition.rb
|
|
129
174
|
- app/models/dispatch_policy/staged_job.rb
|
|
130
|
-
- app/models/dispatch_policy/
|
|
175
|
+
- app/models/dispatch_policy/tick_sample.rb
|
|
176
|
+
- app/views/dispatch_policy/dashboard/index.html.erb
|
|
177
|
+
- app/views/dispatch_policy/partitions/index.html.erb
|
|
178
|
+
- app/views/dispatch_policy/partitions/show.html.erb
|
|
131
179
|
- app/views/dispatch_policy/policies/index.html.erb
|
|
132
180
|
- app/views/dispatch_policy/policies/show.html.erb
|
|
181
|
+
- app/views/dispatch_policy/shared/_capacity.html.erb
|
|
182
|
+
- app/views/dispatch_policy/shared/_hints.html.erb
|
|
183
|
+
- app/views/dispatch_policy/shared/_partition_row.html.erb
|
|
184
|
+
- app/views/dispatch_policy/staged_jobs/show.html.erb
|
|
133
185
|
- app/views/layouts/dispatch_policy/application.html.erb
|
|
134
186
|
- config/routes.rb
|
|
135
|
-
- db/migrate/
|
|
136
|
-
- db/migrate/20260424000002_create_adaptive_concurrency_stats.rb
|
|
137
|
-
- db/migrate/20260424000003_create_adaptive_concurrency_samples.rb
|
|
138
|
-
- db/migrate/20260424000004_rename_samples_to_partition_observations.rb
|
|
139
|
-
- db/migrate/20260425000001_add_duration_to_partition_observations.rb
|
|
187
|
+
- db/migrate/20260501000001_create_dispatch_policy_tables.rb
|
|
140
188
|
- lib/dispatch_policy.rb
|
|
141
|
-
- lib/dispatch_policy/
|
|
142
|
-
- lib/dispatch_policy/
|
|
143
|
-
- lib/dispatch_policy/
|
|
189
|
+
- lib/dispatch_policy/bypass.rb
|
|
190
|
+
- lib/dispatch_policy/config.rb
|
|
191
|
+
- lib/dispatch_policy/context.rb
|
|
192
|
+
- lib/dispatch_policy/cursor_pagination.rb
|
|
193
|
+
- lib/dispatch_policy/decision.rb
|
|
144
194
|
- lib/dispatch_policy/engine.rb
|
|
195
|
+
- lib/dispatch_policy/forwarder.rb
|
|
145
196
|
- lib/dispatch_policy/gate.rb
|
|
146
197
|
- lib/dispatch_policy/gates/adaptive_concurrency.rb
|
|
147
198
|
- lib/dispatch_policy/gates/concurrency.rb
|
|
148
|
-
- lib/dispatch_policy/gates/fair_interleave.rb
|
|
149
|
-
- lib/dispatch_policy/gates/global_cap.rb
|
|
150
199
|
- lib/dispatch_policy/gates/throttle.rb
|
|
200
|
+
- lib/dispatch_policy/inflight_tracker.rb
|
|
201
|
+
- lib/dispatch_policy/job_extension.rb
|
|
202
|
+
- lib/dispatch_policy/operator_hints.rb
|
|
203
|
+
- lib/dispatch_policy/pipeline.rb
|
|
151
204
|
- lib/dispatch_policy/policy.rb
|
|
205
|
+
- lib/dispatch_policy/policy_dsl.rb
|
|
206
|
+
- lib/dispatch_policy/railtie.rb
|
|
207
|
+
- lib/dispatch_policy/registry.rb
|
|
208
|
+
- lib/dispatch_policy/repository.rb
|
|
209
|
+
- lib/dispatch_policy/serializer.rb
|
|
152
210
|
- lib/dispatch_policy/tick.rb
|
|
153
211
|
- lib/dispatch_policy/tick_loop.rb
|
|
154
212
|
- lib/dispatch_policy/version.rb
|
|
213
|
+
- lib/generators/dispatch_policy/install/install_generator.rb
|
|
214
|
+
- lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt
|
|
215
|
+
- lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt
|
|
216
|
+
- lib/generators/dispatch_policy/install/templates/initializer.rb.tt
|
|
155
217
|
homepage: https://github.com/ceritium/dispatch_policy
|
|
156
218
|
licenses:
|
|
157
219
|
- MIT
|
|
158
220
|
metadata:
|
|
159
221
|
homepage_uri: https://github.com/ceritium/dispatch_policy
|
|
160
222
|
source_code_uri: https://github.com/ceritium/dispatch_policy
|
|
161
|
-
bug_tracker_uri: https://github.com/ceritium/dispatch_policy/issues
|
|
162
|
-
changelog_uri: https://github.com/ceritium/dispatch_policy/blob/master/CHANGELOG.md
|
|
163
|
-
rubygems_mfa_required: 'true'
|
|
164
223
|
rdoc_options: []
|
|
165
224
|
require_paths:
|
|
166
225
|
- lib
|
|
@@ -168,7 +227,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
168
227
|
requirements:
|
|
169
228
|
- - ">="
|
|
170
229
|
- !ruby/object:Gem::Version
|
|
171
|
-
version:
|
|
230
|
+
version: 3.1.0
|
|
172
231
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
173
232
|
requirements:
|
|
174
233
|
- - ">="
|
|
@@ -177,6 +236,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
177
236
|
requirements: []
|
|
178
237
|
rubygems_version: 4.0.3
|
|
179
238
|
specification_version: 4
|
|
180
|
-
summary: Per-partition admission control
|
|
181
|
-
for ActiveJob.
|
|
239
|
+
summary: Per-partition admission control for ActiveJob (Postgres).
|
|
182
240
|
test_files: []
|
data/CHANGELOG.md
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
# Changelog
|
|
2
|
-
|
|
3
|
-
## 0.2.0
|
|
4
|
-
|
|
5
|
-
### Added
|
|
6
|
-
- `round_robin_by` supports `weight: :time` to balance per-tick quanta
|
|
7
|
-
by recent perform compute time instead of by request count (#3).
|
|
8
|
-
- GitHub Actions CI matrix covering Ruby 3.4 and Rails 7.2 / 8.1 (#5).
|
|
9
|
-
- Integration tests for gate combinations and throttle bucket
|
|
10
|
-
boundaries (#12).
|
|
11
|
-
- Resilience tests covering failure paths and dedupe state transitions
|
|
12
|
-
(#13).
|
|
13
|
-
- `bin/release` wrapper around `rake release` (#2).
|
|
14
|
-
|
|
15
|
-
### Changed
|
|
16
|
-
- Admin partition breakdown caps its aggregations to keep the page
|
|
17
|
-
responsive on policies with many partitions (#9).
|
|
18
|
-
- Admin pending list no longer loads the `arguments` jsonb column
|
|
19
|
-
(#6).
|
|
20
|
-
|
|
21
|
-
### Fixed
|
|
22
|
-
- Admission is reverted when the underlying adapter silently declines
|
|
23
|
-
to enqueue, so the staged row doesn't stay marked as admitted (#14).
|
|
24
|
-
- `consumed_ms_by_partition` window is padded to survive
|
|
25
|
-
minute-boundary races in the time-weighted round-robin fetch (#11).
|
|
26
|
-
- ThrottleBucket row locks are taken in a deterministic key order to
|
|
27
|
-
remove a deadlock window when multiple ticks contend on the same
|
|
28
|
-
set of partitions (#8).
|
|
29
|
-
|
|
30
|
-
### Removed
|
|
31
|
-
- Stale custom `InstallGenerator` — the engine's migration generator
|
|
32
|
-
is the supported install path (#7).
|
|
33
|
-
|
|
34
|
-
## 0.1.0
|
|
35
|
-
|
|
36
|
-
Initial release.
|
|
37
|
-
|
|
38
|
-
- Rails engine + ActiveJob integration (`DispatchPolicy::Dispatchable`).
|
|
39
|
-
- Gates: `:throttle`, `:concurrency`, `:global_cap`, `:fair_interleave`, `:adaptive_concurrency`.
|
|
40
|
-
- Staged jobs with dedupe, round-robin fairness, per-partition counters, and throttle buckets.
|
|
41
|
-
- Admin UI (Chart.js + Turbo) with watched partitions, sparklines, and EWMA queue-lag charts.
|
|
42
|
-
- PostgreSQL required (uses `FOR UPDATE SKIP LOCKED`, `ON CONFLICT`, and `jsonb`).
|
|
43
|
-
- Experimental — being trialed on pulso.run.
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
class PartitionInflightCount < ApplicationRecord
|
|
5
|
-
self.table_name = "dispatch_policy_partition_counts"
|
|
6
|
-
|
|
7
|
-
def self.fetch_many(policy_name:, gate_name:, partition_keys:)
|
|
8
|
-
return {} if partition_keys.empty?
|
|
9
|
-
|
|
10
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_keys)
|
|
11
|
-
.pluck(:partition_key, :in_flight).to_h
|
|
12
|
-
.tap { |h| partition_keys.each { |k| h[k] ||= 0 } }
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def self.total_for(policy_name:, gate_name:)
|
|
16
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s).sum(:in_flight)
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def self.increment(policy_name:, gate_name:, partition_key:, by: 1)
|
|
20
|
-
now = Time.current
|
|
21
|
-
sql = <<~SQL.squish
|
|
22
|
-
INSERT INTO #{quoted_table_name}
|
|
23
|
-
(policy_name, gate_name, partition_key, in_flight, created_at, updated_at)
|
|
24
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
25
|
-
ON CONFLICT (policy_name, gate_name, partition_key)
|
|
26
|
-
DO UPDATE SET
|
|
27
|
-
in_flight = #{quoted_table_name}.in_flight + EXCLUDED.in_flight,
|
|
28
|
-
updated_at = EXCLUDED.updated_at
|
|
29
|
-
SQL
|
|
30
|
-
connection.exec_update(
|
|
31
|
-
sanitize_sql_array([ sql, policy_name, gate_name.to_s, partition_key.to_s, by, now, now ])
|
|
32
|
-
)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def self.decrement(policy_name:, gate_name:, partition_key:, by: 1)
|
|
36
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
|
|
37
|
-
.update_all([
|
|
38
|
-
"in_flight = GREATEST(in_flight - ?, 0), updated_at = ?", by, Time.current
|
|
39
|
-
])
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
end
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
# Minute-bucketed observability per (policy, partition). Any gate with
|
|
5
|
-
# partition_by gets an observation row here — adaptive, throttle,
|
|
6
|
-
# concurrency, whatever — so the admin chart shows queue lag / throughput
|
|
7
|
-
# for all partitioned policies, not just the adaptive ones.
|
|
8
|
-
#
|
|
9
|
-
# One row per (policy, partition, minute): total_lag_ms accumulates the
|
|
10
|
-
# sum of queue_lag_ms observations in that minute, total_duration_ms
|
|
11
|
-
# accumulates perform durations (used by :time_budget and :fair_time_share),
|
|
12
|
-
# observation_count increments, max_lag_ms / max_duration_ms track worst
|
|
13
|
-
# spikes. Averages are derived on read as total / count.
|
|
14
|
-
class PartitionObservation < ApplicationRecord
|
|
15
|
-
self.table_name = "dispatch_policy_partition_observations"
|
|
16
|
-
|
|
17
|
-
OBSERVATION_TTL = 2 * 60 * 60 # 2 hours
|
|
18
|
-
|
|
19
|
-
def self.observe!(policy_name:, partition_key:, queue_lag_ms:, duration_ms: 0, current_max: nil)
|
|
20
|
-
return if partition_key.nil? || partition_key.to_s.empty?
|
|
21
|
-
|
|
22
|
-
now = Time.current
|
|
23
|
-
lag = queue_lag_ms.to_i
|
|
24
|
-
dur = duration_ms.to_i
|
|
25
|
-
sql = <<~SQL.squish
|
|
26
|
-
INSERT INTO #{quoted_table_name}
|
|
27
|
-
(policy_name, partition_key, minute_bucket,
|
|
28
|
-
total_lag_ms, total_duration_ms, observation_count,
|
|
29
|
-
max_lag_ms, max_duration_ms, current_max,
|
|
30
|
-
created_at, updated_at)
|
|
31
|
-
VALUES (?, ?, date_trunc('minute', ?::timestamp), ?, ?, 1, ?, ?, ?, ?, ?)
|
|
32
|
-
ON CONFLICT (policy_name, partition_key, minute_bucket)
|
|
33
|
-
DO UPDATE SET
|
|
34
|
-
total_lag_ms = #{quoted_table_name}.total_lag_ms + EXCLUDED.total_lag_ms,
|
|
35
|
-
total_duration_ms = #{quoted_table_name}.total_duration_ms + EXCLUDED.total_duration_ms,
|
|
36
|
-
observation_count = #{quoted_table_name}.observation_count + 1,
|
|
37
|
-
max_lag_ms = GREATEST(#{quoted_table_name}.max_lag_ms, EXCLUDED.max_lag_ms),
|
|
38
|
-
max_duration_ms = GREATEST(#{quoted_table_name}.max_duration_ms, EXCLUDED.max_duration_ms),
|
|
39
|
-
current_max = COALESCE(EXCLUDED.current_max, #{quoted_table_name}.current_max),
|
|
40
|
-
updated_at = EXCLUDED.updated_at
|
|
41
|
-
SQL
|
|
42
|
-
connection.exec_update(
|
|
43
|
-
sanitize_sql_array([
|
|
44
|
-
sql, policy_name, partition_key.to_s, now,
|
|
45
|
-
lag, dur, lag, dur, current_max, now, now
|
|
46
|
-
])
|
|
47
|
-
)
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
# Sum of perform durations per partition over the last `window` seconds.
|
|
51
|
-
# Used by :fair_time_share to bias admission ordering toward partitions
|
|
52
|
-
# that have consumed less compute time recently.
|
|
53
|
-
def self.consumed_ms_by_partition(policy_name:, partition_keys:, window:)
|
|
54
|
-
return {} if partition_keys.empty?
|
|
55
|
-
|
|
56
|
-
# minute_bucket is floored on insert (date_trunc('minute', now)).
|
|
57
|
-
# An observation written T seconds ago lives in a bucket up to 60s
|
|
58
|
-
# earlier than T. Add a one-bucket pad to the lower bound so the
|
|
59
|
-
# most recent bucket is always inside the window — without it, the
|
|
60
|
-
# previous-minute bucket is silently excluded as soon as the wall
|
|
61
|
-
# clock crosses a minute boundary.
|
|
62
|
-
since = Time.current - window - 60
|
|
63
|
-
rows = where(policy_name: policy_name, partition_key: partition_keys.map(&:to_s))
|
|
64
|
-
.where("minute_bucket >= ?", since)
|
|
65
|
-
.group(:partition_key)
|
|
66
|
-
.pluck(Arel.sql("partition_key, SUM(total_duration_ms), SUM(observation_count)"))
|
|
67
|
-
rows.each_with_object({}) do |(key, total, count), acc|
|
|
68
|
-
acc[key] = { consumed_ms: total.to_i, count: count.to_i }
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def self.prune!
|
|
73
|
-
where("minute_bucket < ?", Time.current - OBSERVATION_TTL).delete_all
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
class ThrottleBucket < ApplicationRecord
|
|
5
|
-
self.table_name = "dispatch_policy_throttle_buckets"
|
|
6
|
-
|
|
7
|
-
def self.lock(policy_name:, gate_name:, partition_key:, burst:)
|
|
8
|
-
now = Time.current
|
|
9
|
-
seed_sql = <<~SQL.squish
|
|
10
|
-
INSERT INTO #{quoted_table_name}
|
|
11
|
-
(policy_name, gate_name, partition_key, tokens, refilled_at, created_at, updated_at)
|
|
12
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
13
|
-
ON CONFLICT (policy_name, gate_name, partition_key) DO NOTHING
|
|
14
|
-
SQL
|
|
15
|
-
connection.exec_update(
|
|
16
|
-
sanitize_sql_array([
|
|
17
|
-
seed_sql, policy_name, gate_name.to_s, partition_key.to_s,
|
|
18
|
-
burst.to_f, now, now, now
|
|
19
|
-
])
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
|
|
23
|
-
.lock("FOR UPDATE")
|
|
24
|
-
.first!
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def refill!(rate:, per:, burst:)
|
|
28
|
-
now = Time.current
|
|
29
|
-
elapsed = (now - refilled_at).to_f
|
|
30
|
-
new_tokens = tokens + (rate * elapsed / per)
|
|
31
|
-
self.tokens = [ new_tokens, burst.to_f ].min
|
|
32
|
-
self.refilled_at = now
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def consume(n = 1)
|
|
36
|
-
return false if tokens < n
|
|
37
|
-
self.tokens -= n
|
|
38
|
-
true
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
end
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateDispatchPolicyTables < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_staged_jobs do |t|
|
|
6
|
-
t.string :job_class, null: false
|
|
7
|
-
t.string :policy_name, null: false
|
|
8
|
-
t.jsonb :arguments, null: false
|
|
9
|
-
t.jsonb :snapshot, null: false, default: {}
|
|
10
|
-
t.jsonb :context, null: false, default: {}
|
|
11
|
-
t.integer :priority, null: false, default: 100
|
|
12
|
-
t.datetime :not_before_at
|
|
13
|
-
t.datetime :staged_at, null: false
|
|
14
|
-
t.datetime :admitted_at
|
|
15
|
-
t.datetime :completed_at
|
|
16
|
-
t.datetime :lease_expires_at
|
|
17
|
-
t.string :active_job_id
|
|
18
|
-
t.string :dedupe_key
|
|
19
|
-
t.string :round_robin_key
|
|
20
|
-
t.jsonb :partitions, null: false, default: {}
|
|
21
|
-
|
|
22
|
-
t.timestamps
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
add_index :dispatch_policy_staged_jobs,
|
|
26
|
-
%i[policy_name priority staged_at],
|
|
27
|
-
where: "admitted_at IS NULL",
|
|
28
|
-
name: "idx_dp_staged_dispatch_order"
|
|
29
|
-
|
|
30
|
-
add_index :dispatch_policy_staged_jobs,
|
|
31
|
-
%i[policy_name dedupe_key],
|
|
32
|
-
unique: true,
|
|
33
|
-
where: "dedupe_key IS NOT NULL AND completed_at IS NULL",
|
|
34
|
-
name: "idx_dp_staged_dedupe_active"
|
|
35
|
-
|
|
36
|
-
add_index :dispatch_policy_staged_jobs,
|
|
37
|
-
%i[lease_expires_at],
|
|
38
|
-
where: "admitted_at IS NOT NULL",
|
|
39
|
-
name: "idx_dp_staged_lease_expires"
|
|
40
|
-
|
|
41
|
-
add_index :dispatch_policy_staged_jobs,
|
|
42
|
-
%i[completed_at],
|
|
43
|
-
where: "completed_at IS NOT NULL",
|
|
44
|
-
name: "idx_dp_staged_completed_at"
|
|
45
|
-
|
|
46
|
-
add_index :dispatch_policy_staged_jobs,
|
|
47
|
-
%i[policy_name round_robin_key priority staged_at],
|
|
48
|
-
where: "admitted_at IS NULL AND round_robin_key IS NOT NULL",
|
|
49
|
-
name: "idx_dp_staged_round_robin"
|
|
50
|
-
|
|
51
|
-
create_table :dispatch_policy_partition_counts do |t|
|
|
52
|
-
t.string :policy_name, null: false
|
|
53
|
-
t.string :gate_name, null: false
|
|
54
|
-
t.string :partition_key, null: false, default: "default"
|
|
55
|
-
t.integer :in_flight, null: false, default: 0
|
|
56
|
-
|
|
57
|
-
t.timestamps
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
add_index :dispatch_policy_partition_counts,
|
|
61
|
-
%i[policy_name gate_name partition_key],
|
|
62
|
-
unique: true,
|
|
63
|
-
name: "idx_dp_partition_counts_unique"
|
|
64
|
-
|
|
65
|
-
create_table :dispatch_policy_throttle_buckets do |t|
|
|
66
|
-
t.string :policy_name, null: false
|
|
67
|
-
t.string :gate_name, null: false
|
|
68
|
-
t.string :partition_key, null: false, default: "default"
|
|
69
|
-
t.float :tokens, null: false
|
|
70
|
-
t.datetime :refilled_at, null: false
|
|
71
|
-
|
|
72
|
-
t.timestamps
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
add_index :dispatch_policy_throttle_buckets,
|
|
76
|
-
%i[policy_name gate_name partition_key],
|
|
77
|
-
unique: true,
|
|
78
|
-
name: "idx_dp_throttle_buckets_unique"
|
|
79
|
-
end
|
|
80
|
-
end
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateAdaptiveConcurrencyStats < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_adaptive_concurrency_stats do |t|
|
|
6
|
-
t.string :policy_name, null: false
|
|
7
|
-
t.string :gate_name, null: false
|
|
8
|
-
t.string :partition_key, null: false, default: "default"
|
|
9
|
-
t.integer :current_max, null: false
|
|
10
|
-
t.float :ewma_latency_ms, null: false, default: 0
|
|
11
|
-
t.integer :sample_count, null: false, default: 0
|
|
12
|
-
t.datetime :last_observed_at
|
|
13
|
-
|
|
14
|
-
t.timestamps
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
add_index :dispatch_policy_adaptive_concurrency_stats,
|
|
18
|
-
%i[policy_name gate_name partition_key],
|
|
19
|
-
unique: true,
|
|
20
|
-
name: "idx_dp_adaptive_concurrency_stats_unique"
|
|
21
|
-
end
|
|
22
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateAdaptiveConcurrencySamples < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_adaptive_concurrency_samples do |t|
|
|
6
|
-
t.string :policy_name, null: false
|
|
7
|
-
t.string :gate_name, null: false
|
|
8
|
-
t.string :partition_key, null: false
|
|
9
|
-
t.datetime :minute_bucket, null: false
|
|
10
|
-
t.float :ewma_latency_ms, null: false, default: 0
|
|
11
|
-
t.integer :current_max, null: false
|
|
12
|
-
|
|
13
|
-
t.timestamps
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
add_index :dispatch_policy_adaptive_concurrency_samples,
|
|
17
|
-
%i[policy_name gate_name partition_key minute_bucket],
|
|
18
|
-
unique: true,
|
|
19
|
-
name: "idx_dp_adaptive_concurrency_samples_unique"
|
|
20
|
-
|
|
21
|
-
add_index :dispatch_policy_adaptive_concurrency_samples,
|
|
22
|
-
:minute_bucket,
|
|
23
|
-
name: "idx_dp_adaptive_concurrency_samples_time"
|
|
24
|
-
end
|
|
25
|
-
end
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class RenameSamplesToPartitionObservations < ActiveRecord::Migration[7.1]
|
|
4
|
-
def up
|
|
5
|
-
drop_table :dispatch_policy_adaptive_concurrency_samples, if_exists: true
|
|
6
|
-
|
|
7
|
-
create_table :dispatch_policy_partition_observations do |t|
|
|
8
|
-
t.string :policy_name, null: false
|
|
9
|
-
t.string :partition_key, null: false
|
|
10
|
-
t.datetime :minute_bucket, null: false
|
|
11
|
-
t.bigint :total_lag_ms, null: false, default: 0
|
|
12
|
-
t.integer :observation_count, null: false, default: 0
|
|
13
|
-
t.integer :max_lag_ms, null: false, default: 0
|
|
14
|
-
t.integer :current_max
|
|
15
|
-
|
|
16
|
-
t.timestamps
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
add_index :dispatch_policy_partition_observations,
|
|
20
|
-
%i[policy_name partition_key minute_bucket],
|
|
21
|
-
unique: true,
|
|
22
|
-
name: "idx_dp_partition_observations_unique"
|
|
23
|
-
|
|
24
|
-
add_index :dispatch_policy_partition_observations,
|
|
25
|
-
:minute_bucket,
|
|
26
|
-
name: "idx_dp_partition_observations_time"
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def down
|
|
30
|
-
drop_table :dispatch_policy_partition_observations
|
|
31
|
-
end
|
|
32
|
-
end
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class AddDurationToPartitionObservations < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
add_column :dispatch_policy_partition_observations, :total_duration_ms, :bigint, null: false, default: 0
|
|
6
|
-
add_column :dispatch_policy_partition_observations, :max_duration_ms, :integer, null: false, default: 0
|
|
7
|
-
end
|
|
8
|
-
end
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
# Rails 7.1's ActiveJob.perform_all_later(*jobs) bypasses ActiveJob::Base#enqueue
|
|
5
|
-
# and calls queue_adapter.enqueue_all directly. Dispatchable hooks on #enqueue,
|
|
6
|
-
# so without this patch the batch path would skip staging.
|
|
7
|
-
module ActiveJobPerformAllLaterPatch
|
|
8
|
-
def perform_all_later(*jobs)
|
|
9
|
-
jobs.flatten!
|
|
10
|
-
|
|
11
|
-
staged, remaining = jobs.partition do |job|
|
|
12
|
-
klass = job.class
|
|
13
|
-
klass.respond_to?(:dispatch_policy?) &&
|
|
14
|
-
klass.dispatch_policy? &&
|
|
15
|
-
DispatchPolicy.enabled?
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
staged_count = 0
|
|
19
|
-
staged.group_by(&:class).each do |klass, group|
|
|
20
|
-
staged_count += DispatchPolicy::StagedJob.stage_many!(
|
|
21
|
-
policy: klass.resolved_dispatch_policy,
|
|
22
|
-
jobs: group
|
|
23
|
-
)
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
remaining_count = remaining.empty? ? 0 : super(*remaining)
|
|
27
|
-
staged_count + remaining_count.to_i
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
ActiveJob.singleton_class.prepend(DispatchPolicy::ActiveJobPerformAllLaterPatch)
|