dispatch_policy 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -28
- data/MIT-LICENSE +16 -17
- data/README.md +452 -388
- data/app/assets/images/dispatch_policy/logo-large.svg +9 -0
- data/app/assets/images/dispatch_policy/logo-small.svg +7 -0
- data/app/assets/javascripts/dispatch_policy/turbo.es2017-umd.min.js +35 -0
- data/app/assets/stylesheets/dispatch_policy/application.css +294 -0
- data/app/controllers/dispatch_policy/application_controller.rb +45 -1
- data/app/controllers/dispatch_policy/assets_controller.rb +31 -0
- data/app/controllers/dispatch_policy/dashboard_controller.rb +91 -0
- data/app/controllers/dispatch_policy/partitions_controller.rb +122 -0
- data/app/controllers/dispatch_policy/policies_controller.rb +94 -267
- data/app/controllers/dispatch_policy/staged_jobs_controller.rb +9 -0
- data/app/models/dispatch_policy/adaptive_concurrency_stats.rb +11 -81
- data/app/models/dispatch_policy/inflight_job.rb +12 -0
- data/app/models/dispatch_policy/partition.rb +21 -0
- data/app/models/dispatch_policy/staged_job.rb +4 -97
- data/app/models/dispatch_policy/tick_sample.rb +11 -0
- data/app/views/dispatch_policy/dashboard/index.html.erb +109 -0
- data/app/views/dispatch_policy/partitions/index.html.erb +63 -0
- data/app/views/dispatch_policy/partitions/show.html.erb +106 -0
- data/app/views/dispatch_policy/policies/index.html.erb +15 -37
- data/app/views/dispatch_policy/policies/show.html.erb +139 -223
- data/app/views/dispatch_policy/shared/_capacity.html.erb +67 -0
- data/app/views/dispatch_policy/shared/_hints.html.erb +13 -0
- data/app/views/dispatch_policy/shared/_partition_row.html.erb +12 -0
- data/app/views/dispatch_policy/staged_jobs/show.html.erb +31 -0
- data/app/views/layouts/dispatch_policy/application.html.erb +164 -231
- data/config/routes.rb +21 -2
- data/db/migrate/20260501000001_create_dispatch_policy_tables.rb +103 -0
- data/lib/dispatch_policy/assets.rb +38 -0
- data/lib/dispatch_policy/bypass.rb +23 -0
- data/lib/dispatch_policy/config.rb +85 -0
- data/lib/dispatch_policy/context.rb +50 -0
- data/lib/dispatch_policy/cursor_pagination.rb +121 -0
- data/lib/dispatch_policy/decision.rb +22 -0
- data/lib/dispatch_policy/engine.rb +5 -27
- data/lib/dispatch_policy/forwarder.rb +63 -0
- data/lib/dispatch_policy/gate.rb +10 -38
- data/lib/dispatch_policy/gates/adaptive_concurrency.rb +99 -97
- data/lib/dispatch_policy/gates/concurrency.rb +45 -26
- data/lib/dispatch_policy/gates/throttle.rb +65 -41
- data/lib/dispatch_policy/inflight_tracker.rb +174 -0
- data/lib/dispatch_policy/job_extension.rb +155 -0
- data/lib/dispatch_policy/operator_hints.rb +126 -0
- data/lib/dispatch_policy/pipeline.rb +48 -0
- data/lib/dispatch_policy/policy.rb +61 -59
- data/lib/dispatch_policy/policy_dsl.rb +120 -0
- data/lib/dispatch_policy/railtie.rb +35 -0
- data/lib/dispatch_policy/registry.rb +46 -0
- data/lib/dispatch_policy/repository.rb +723 -0
- data/lib/dispatch_policy/serializer.rb +36 -0
- data/lib/dispatch_policy/tick.rb +260 -256
- data/lib/dispatch_policy/tick_loop.rb +59 -26
- data/lib/dispatch_policy/version.rb +1 -1
- data/lib/dispatch_policy.rb +72 -52
- data/lib/generators/dispatch_policy/install/install_generator.rb +70 -0
- data/lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt +95 -0
- data/lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt +53 -0
- data/lib/generators/dispatch_policy/install/templates/initializer.rb.tt +11 -0
- metadata +134 -42
- data/app/models/dispatch_policy/partition_inflight_count.rb +0 -42
- data/app/models/dispatch_policy/partition_observation.rb +0 -76
- data/app/models/dispatch_policy/throttle_bucket.rb +0 -41
- data/db/migrate/20260424000001_create_dispatch_policy_tables.rb +0 -80
- data/db/migrate/20260424000002_create_adaptive_concurrency_stats.rb +0 -22
- data/db/migrate/20260424000003_create_adaptive_concurrency_samples.rb +0 -25
- data/db/migrate/20260424000004_rename_samples_to_partition_observations.rb +0 -32
- data/db/migrate/20260425000001_add_duration_to_partition_observations.rb +0 -8
- data/lib/dispatch_policy/active_job_perform_all_later_patch.rb +0 -32
- data/lib/dispatch_policy/dispatch_context.rb +0 -53
- data/lib/dispatch_policy/dispatchable.rb +0 -123
- data/lib/dispatch_policy/gates/fair_interleave.rb +0 -32
- data/lib/dispatch_policy/gates/global_cap.rb +0 -26
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: dispatch_policy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- José Galisteo
|
|
@@ -10,7 +10,7 @@ cert_chain: []
|
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
|
-
name:
|
|
13
|
+
name: rails
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
@@ -24,77 +24,119 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '7.1'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: pg
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - ">="
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
|
-
version: '
|
|
32
|
+
version: '1.4'
|
|
33
33
|
type: :runtime
|
|
34
34
|
prerelease: false
|
|
35
35
|
version_requirements: !ruby/object:Gem::Requirement
|
|
36
36
|
requirements:
|
|
37
37
|
- - ">="
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
|
-
version: '
|
|
39
|
+
version: '1.4'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: minitest
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - "~>"
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '5.20'
|
|
47
|
+
type: :development
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - "~>"
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '5.20'
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: rake
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - "~>"
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '13.0'
|
|
61
|
+
type: :development
|
|
62
|
+
prerelease: false
|
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - "~>"
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: '13.0'
|
|
40
68
|
- !ruby/object:Gem::Dependency
|
|
41
|
-
name:
|
|
69
|
+
name: good_job
|
|
42
70
|
requirement: !ruby/object:Gem::Requirement
|
|
43
71
|
requirements:
|
|
44
72
|
- - ">="
|
|
45
73
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
47
|
-
type: :
|
|
74
|
+
version: '4.0'
|
|
75
|
+
type: :development
|
|
48
76
|
prerelease: false
|
|
49
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
50
78
|
requirements:
|
|
51
79
|
- - ">="
|
|
52
80
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '
|
|
81
|
+
version: '4.0'
|
|
54
82
|
- !ruby/object:Gem::Dependency
|
|
55
|
-
name:
|
|
83
|
+
name: solid_queue
|
|
56
84
|
requirement: !ruby/object:Gem::Requirement
|
|
57
85
|
requirements:
|
|
58
86
|
- - ">="
|
|
59
87
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '0'
|
|
88
|
+
version: '1.0'
|
|
61
89
|
type: :development
|
|
62
90
|
prerelease: false
|
|
63
91
|
version_requirements: !ruby/object:Gem::Requirement
|
|
64
92
|
requirements:
|
|
65
93
|
- - ">="
|
|
66
94
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '0'
|
|
95
|
+
version: '1.0'
|
|
68
96
|
- !ruby/object:Gem::Dependency
|
|
69
|
-
name:
|
|
97
|
+
name: turbo-rails
|
|
70
98
|
requirement: !ruby/object:Gem::Requirement
|
|
71
99
|
requirements:
|
|
72
100
|
- - ">="
|
|
73
101
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '
|
|
102
|
+
version: '1.5'
|
|
75
103
|
type: :development
|
|
76
104
|
prerelease: false
|
|
77
105
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
106
|
requirements:
|
|
79
107
|
- - ">="
|
|
80
108
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: '
|
|
109
|
+
version: '1.5'
|
|
82
110
|
- !ruby/object:Gem::Dependency
|
|
83
|
-
name:
|
|
111
|
+
name: puma
|
|
84
112
|
requirement: !ruby/object:Gem::Requirement
|
|
85
113
|
requirements:
|
|
86
114
|
- - ">="
|
|
87
115
|
- !ruby/object:Gem::Version
|
|
88
|
-
version: '0'
|
|
116
|
+
version: '6.0'
|
|
89
117
|
type: :development
|
|
90
118
|
prerelease: false
|
|
91
119
|
version_requirements: !ruby/object:Gem::Requirement
|
|
92
120
|
requirements:
|
|
93
121
|
- - ">="
|
|
94
122
|
- !ruby/object:Gem::Version
|
|
95
|
-
version: '0'
|
|
123
|
+
version: '6.0'
|
|
96
124
|
- !ruby/object:Gem::Dependency
|
|
97
|
-
name:
|
|
125
|
+
name: foreman
|
|
126
|
+
requirement: !ruby/object:Gem::Requirement
|
|
127
|
+
requirements:
|
|
128
|
+
- - ">="
|
|
129
|
+
- !ruby/object:Gem::Version
|
|
130
|
+
version: '0.87'
|
|
131
|
+
type: :development
|
|
132
|
+
prerelease: false
|
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - ">="
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '0.87'
|
|
138
|
+
- !ruby/object:Gem::Dependency
|
|
139
|
+
name: debug
|
|
98
140
|
requirement: !ruby/object:Gem::Requirement
|
|
99
141
|
requirements:
|
|
100
142
|
- - ">="
|
|
@@ -107,10 +149,38 @@ dependencies:
|
|
|
107
149
|
- - ">="
|
|
108
150
|
- !ruby/object:Gem::Version
|
|
109
151
|
version: '0'
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
152
|
+
- !ruby/object:Gem::Dependency
|
|
153
|
+
name: capybara
|
|
154
|
+
requirement: !ruby/object:Gem::Requirement
|
|
155
|
+
requirements:
|
|
156
|
+
- - ">="
|
|
157
|
+
- !ruby/object:Gem::Version
|
|
158
|
+
version: '3.40'
|
|
159
|
+
type: :development
|
|
160
|
+
prerelease: false
|
|
161
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
162
|
+
requirements:
|
|
163
|
+
- - ">="
|
|
164
|
+
- !ruby/object:Gem::Version
|
|
165
|
+
version: '3.40'
|
|
166
|
+
- !ruby/object:Gem::Dependency
|
|
167
|
+
name: selenium-webdriver
|
|
168
|
+
requirement: !ruby/object:Gem::Requirement
|
|
169
|
+
requirements:
|
|
170
|
+
- - ">="
|
|
171
|
+
- !ruby/object:Gem::Version
|
|
172
|
+
version: '4.20'
|
|
173
|
+
type: :development
|
|
174
|
+
prerelease: false
|
|
175
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
176
|
+
requirements:
|
|
177
|
+
- - ">="
|
|
178
|
+
- !ruby/object:Gem::Version
|
|
179
|
+
version: '4.20'
|
|
180
|
+
description: Stages perform_later into a dedicated table, runs a tick loop that admits
|
|
181
|
+
jobs through declared gates (throttle, concurrency), then forwards survivors to
|
|
182
|
+
the real ActiveJob adapter. Embedded as a periodic job. Compatible with good_job
|
|
183
|
+
and solid_queue.
|
|
114
184
|
email:
|
|
115
185
|
- ceritium@gmail.com
|
|
116
186
|
executables: []
|
|
@@ -120,47 +190,70 @@ files:
|
|
|
120
190
|
- CHANGELOG.md
|
|
121
191
|
- MIT-LICENSE
|
|
122
192
|
- README.md
|
|
193
|
+
- app/assets/images/dispatch_policy/logo-large.svg
|
|
194
|
+
- app/assets/images/dispatch_policy/logo-small.svg
|
|
195
|
+
- app/assets/javascripts/dispatch_policy/turbo.es2017-umd.min.js
|
|
196
|
+
- app/assets/stylesheets/dispatch_policy/application.css
|
|
123
197
|
- app/controllers/dispatch_policy/application_controller.rb
|
|
198
|
+
- app/controllers/dispatch_policy/assets_controller.rb
|
|
199
|
+
- app/controllers/dispatch_policy/dashboard_controller.rb
|
|
200
|
+
- app/controllers/dispatch_policy/partitions_controller.rb
|
|
124
201
|
- app/controllers/dispatch_policy/policies_controller.rb
|
|
202
|
+
- app/controllers/dispatch_policy/staged_jobs_controller.rb
|
|
125
203
|
- app/models/dispatch_policy/adaptive_concurrency_stats.rb
|
|
126
204
|
- app/models/dispatch_policy/application_record.rb
|
|
127
|
-
- app/models/dispatch_policy/
|
|
128
|
-
- app/models/dispatch_policy/
|
|
205
|
+
- app/models/dispatch_policy/inflight_job.rb
|
|
206
|
+
- app/models/dispatch_policy/partition.rb
|
|
129
207
|
- app/models/dispatch_policy/staged_job.rb
|
|
130
|
-
- app/models/dispatch_policy/
|
|
208
|
+
- app/models/dispatch_policy/tick_sample.rb
|
|
209
|
+
- app/views/dispatch_policy/dashboard/index.html.erb
|
|
210
|
+
- app/views/dispatch_policy/partitions/index.html.erb
|
|
211
|
+
- app/views/dispatch_policy/partitions/show.html.erb
|
|
131
212
|
- app/views/dispatch_policy/policies/index.html.erb
|
|
132
213
|
- app/views/dispatch_policy/policies/show.html.erb
|
|
214
|
+
- app/views/dispatch_policy/shared/_capacity.html.erb
|
|
215
|
+
- app/views/dispatch_policy/shared/_hints.html.erb
|
|
216
|
+
- app/views/dispatch_policy/shared/_partition_row.html.erb
|
|
217
|
+
- app/views/dispatch_policy/staged_jobs/show.html.erb
|
|
133
218
|
- app/views/layouts/dispatch_policy/application.html.erb
|
|
134
219
|
- config/routes.rb
|
|
135
|
-
- db/migrate/
|
|
136
|
-
- db/migrate/20260424000002_create_adaptive_concurrency_stats.rb
|
|
137
|
-
- db/migrate/20260424000003_create_adaptive_concurrency_samples.rb
|
|
138
|
-
- db/migrate/20260424000004_rename_samples_to_partition_observations.rb
|
|
139
|
-
- db/migrate/20260425000001_add_duration_to_partition_observations.rb
|
|
220
|
+
- db/migrate/20260501000001_create_dispatch_policy_tables.rb
|
|
140
221
|
- lib/dispatch_policy.rb
|
|
141
|
-
- lib/dispatch_policy/
|
|
142
|
-
- lib/dispatch_policy/
|
|
143
|
-
- lib/dispatch_policy/
|
|
222
|
+
- lib/dispatch_policy/assets.rb
|
|
223
|
+
- lib/dispatch_policy/bypass.rb
|
|
224
|
+
- lib/dispatch_policy/config.rb
|
|
225
|
+
- lib/dispatch_policy/context.rb
|
|
226
|
+
- lib/dispatch_policy/cursor_pagination.rb
|
|
227
|
+
- lib/dispatch_policy/decision.rb
|
|
144
228
|
- lib/dispatch_policy/engine.rb
|
|
229
|
+
- lib/dispatch_policy/forwarder.rb
|
|
145
230
|
- lib/dispatch_policy/gate.rb
|
|
146
231
|
- lib/dispatch_policy/gates/adaptive_concurrency.rb
|
|
147
232
|
- lib/dispatch_policy/gates/concurrency.rb
|
|
148
|
-
- lib/dispatch_policy/gates/fair_interleave.rb
|
|
149
|
-
- lib/dispatch_policy/gates/global_cap.rb
|
|
150
233
|
- lib/dispatch_policy/gates/throttle.rb
|
|
234
|
+
- lib/dispatch_policy/inflight_tracker.rb
|
|
235
|
+
- lib/dispatch_policy/job_extension.rb
|
|
236
|
+
- lib/dispatch_policy/operator_hints.rb
|
|
237
|
+
- lib/dispatch_policy/pipeline.rb
|
|
151
238
|
- lib/dispatch_policy/policy.rb
|
|
239
|
+
- lib/dispatch_policy/policy_dsl.rb
|
|
240
|
+
- lib/dispatch_policy/railtie.rb
|
|
241
|
+
- lib/dispatch_policy/registry.rb
|
|
242
|
+
- lib/dispatch_policy/repository.rb
|
|
243
|
+
- lib/dispatch_policy/serializer.rb
|
|
152
244
|
- lib/dispatch_policy/tick.rb
|
|
153
245
|
- lib/dispatch_policy/tick_loop.rb
|
|
154
246
|
- lib/dispatch_policy/version.rb
|
|
247
|
+
- lib/generators/dispatch_policy/install/install_generator.rb
|
|
248
|
+
- lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt
|
|
249
|
+
- lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt
|
|
250
|
+
- lib/generators/dispatch_policy/install/templates/initializer.rb.tt
|
|
155
251
|
homepage: https://github.com/ceritium/dispatch_policy
|
|
156
252
|
licenses:
|
|
157
253
|
- MIT
|
|
158
254
|
metadata:
|
|
159
255
|
homepage_uri: https://github.com/ceritium/dispatch_policy
|
|
160
256
|
source_code_uri: https://github.com/ceritium/dispatch_policy
|
|
161
|
-
bug_tracker_uri: https://github.com/ceritium/dispatch_policy/issues
|
|
162
|
-
changelog_uri: https://github.com/ceritium/dispatch_policy/blob/master/CHANGELOG.md
|
|
163
|
-
rubygems_mfa_required: 'true'
|
|
164
257
|
rdoc_options: []
|
|
165
258
|
require_paths:
|
|
166
259
|
- lib
|
|
@@ -168,7 +261,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
168
261
|
requirements:
|
|
169
262
|
- - ">="
|
|
170
263
|
- !ruby/object:Gem::Version
|
|
171
|
-
version:
|
|
264
|
+
version: 3.1.0
|
|
172
265
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
173
266
|
requirements:
|
|
174
267
|
- - ">="
|
|
@@ -177,6 +270,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
177
270
|
requirements: []
|
|
178
271
|
rubygems_version: 4.0.3
|
|
179
272
|
specification_version: 4
|
|
180
|
-
summary: Per-partition admission control
|
|
181
|
-
for ActiveJob.
|
|
273
|
+
summary: Per-partition admission control for ActiveJob (Postgres).
|
|
182
274
|
test_files: []
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
class PartitionInflightCount < ApplicationRecord
|
|
5
|
-
self.table_name = "dispatch_policy_partition_counts"
|
|
6
|
-
|
|
7
|
-
def self.fetch_many(policy_name:, gate_name:, partition_keys:)
|
|
8
|
-
return {} if partition_keys.empty?
|
|
9
|
-
|
|
10
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_keys)
|
|
11
|
-
.pluck(:partition_key, :in_flight).to_h
|
|
12
|
-
.tap { |h| partition_keys.each { |k| h[k] ||= 0 } }
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def self.total_for(policy_name:, gate_name:)
|
|
16
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s).sum(:in_flight)
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def self.increment(policy_name:, gate_name:, partition_key:, by: 1)
|
|
20
|
-
now = Time.current
|
|
21
|
-
sql = <<~SQL.squish
|
|
22
|
-
INSERT INTO #{quoted_table_name}
|
|
23
|
-
(policy_name, gate_name, partition_key, in_flight, created_at, updated_at)
|
|
24
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
25
|
-
ON CONFLICT (policy_name, gate_name, partition_key)
|
|
26
|
-
DO UPDATE SET
|
|
27
|
-
in_flight = #{quoted_table_name}.in_flight + EXCLUDED.in_flight,
|
|
28
|
-
updated_at = EXCLUDED.updated_at
|
|
29
|
-
SQL
|
|
30
|
-
connection.exec_update(
|
|
31
|
-
sanitize_sql_array([ sql, policy_name, gate_name.to_s, partition_key.to_s, by, now, now ])
|
|
32
|
-
)
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def self.decrement(policy_name:, gate_name:, partition_key:, by: 1)
|
|
36
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
|
|
37
|
-
.update_all([
|
|
38
|
-
"in_flight = GREATEST(in_flight - ?, 0), updated_at = ?", by, Time.current
|
|
39
|
-
])
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
|
-
end
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
# Minute-bucketed observability per (policy, partition). Any gate with
|
|
5
|
-
# partition_by gets an observation row here — adaptive, throttle,
|
|
6
|
-
# concurrency, whatever — so the admin chart shows queue lag / throughput
|
|
7
|
-
# for all partitioned policies, not just the adaptive ones.
|
|
8
|
-
#
|
|
9
|
-
# One row per (policy, partition, minute): total_lag_ms accumulates the
|
|
10
|
-
# sum of queue_lag_ms observations in that minute, total_duration_ms
|
|
11
|
-
# accumulates perform durations (used by :time_budget and :fair_time_share),
|
|
12
|
-
# observation_count increments, max_lag_ms / max_duration_ms track worst
|
|
13
|
-
# spikes. Averages are derived on read as total / count.
|
|
14
|
-
class PartitionObservation < ApplicationRecord
|
|
15
|
-
self.table_name = "dispatch_policy_partition_observations"
|
|
16
|
-
|
|
17
|
-
OBSERVATION_TTL = 2 * 60 * 60 # 2 hours
|
|
18
|
-
|
|
19
|
-
def self.observe!(policy_name:, partition_key:, queue_lag_ms:, duration_ms: 0, current_max: nil)
|
|
20
|
-
return if partition_key.nil? || partition_key.to_s.empty?
|
|
21
|
-
|
|
22
|
-
now = Time.current
|
|
23
|
-
lag = queue_lag_ms.to_i
|
|
24
|
-
dur = duration_ms.to_i
|
|
25
|
-
sql = <<~SQL.squish
|
|
26
|
-
INSERT INTO #{quoted_table_name}
|
|
27
|
-
(policy_name, partition_key, minute_bucket,
|
|
28
|
-
total_lag_ms, total_duration_ms, observation_count,
|
|
29
|
-
max_lag_ms, max_duration_ms, current_max,
|
|
30
|
-
created_at, updated_at)
|
|
31
|
-
VALUES (?, ?, date_trunc('minute', ?::timestamp), ?, ?, 1, ?, ?, ?, ?, ?)
|
|
32
|
-
ON CONFLICT (policy_name, partition_key, minute_bucket)
|
|
33
|
-
DO UPDATE SET
|
|
34
|
-
total_lag_ms = #{quoted_table_name}.total_lag_ms + EXCLUDED.total_lag_ms,
|
|
35
|
-
total_duration_ms = #{quoted_table_name}.total_duration_ms + EXCLUDED.total_duration_ms,
|
|
36
|
-
observation_count = #{quoted_table_name}.observation_count + 1,
|
|
37
|
-
max_lag_ms = GREATEST(#{quoted_table_name}.max_lag_ms, EXCLUDED.max_lag_ms),
|
|
38
|
-
max_duration_ms = GREATEST(#{quoted_table_name}.max_duration_ms, EXCLUDED.max_duration_ms),
|
|
39
|
-
current_max = COALESCE(EXCLUDED.current_max, #{quoted_table_name}.current_max),
|
|
40
|
-
updated_at = EXCLUDED.updated_at
|
|
41
|
-
SQL
|
|
42
|
-
connection.exec_update(
|
|
43
|
-
sanitize_sql_array([
|
|
44
|
-
sql, policy_name, partition_key.to_s, now,
|
|
45
|
-
lag, dur, lag, dur, current_max, now, now
|
|
46
|
-
])
|
|
47
|
-
)
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
# Sum of perform durations per partition over the last `window` seconds.
|
|
51
|
-
# Used by :fair_time_share to bias admission ordering toward partitions
|
|
52
|
-
# that have consumed less compute time recently.
|
|
53
|
-
def self.consumed_ms_by_partition(policy_name:, partition_keys:, window:)
|
|
54
|
-
return {} if partition_keys.empty?
|
|
55
|
-
|
|
56
|
-
# minute_bucket is floored on insert (date_trunc('minute', now)).
|
|
57
|
-
# An observation written T seconds ago lives in a bucket up to 60s
|
|
58
|
-
# earlier than T. Add a one-bucket pad to the lower bound so the
|
|
59
|
-
# most recent bucket is always inside the window — without it, the
|
|
60
|
-
# previous-minute bucket is silently excluded as soon as the wall
|
|
61
|
-
# clock crosses a minute boundary.
|
|
62
|
-
since = Time.current - window - 60
|
|
63
|
-
rows = where(policy_name: policy_name, partition_key: partition_keys.map(&:to_s))
|
|
64
|
-
.where("minute_bucket >= ?", since)
|
|
65
|
-
.group(:partition_key)
|
|
66
|
-
.pluck(Arel.sql("partition_key, SUM(total_duration_ms), SUM(observation_count)"))
|
|
67
|
-
rows.each_with_object({}) do |(key, total, count), acc|
|
|
68
|
-
acc[key] = { consumed_ms: total.to_i, count: count.to_i }
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def self.prune!
|
|
73
|
-
where("minute_bucket < ?", Time.current - OBSERVATION_TTL).delete_all
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
class ThrottleBucket < ApplicationRecord
|
|
5
|
-
self.table_name = "dispatch_policy_throttle_buckets"
|
|
6
|
-
|
|
7
|
-
def self.lock(policy_name:, gate_name:, partition_key:, burst:)
|
|
8
|
-
now = Time.current
|
|
9
|
-
seed_sql = <<~SQL.squish
|
|
10
|
-
INSERT INTO #{quoted_table_name}
|
|
11
|
-
(policy_name, gate_name, partition_key, tokens, refilled_at, created_at, updated_at)
|
|
12
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
13
|
-
ON CONFLICT (policy_name, gate_name, partition_key) DO NOTHING
|
|
14
|
-
SQL
|
|
15
|
-
connection.exec_update(
|
|
16
|
-
sanitize_sql_array([
|
|
17
|
-
seed_sql, policy_name, gate_name.to_s, partition_key.to_s,
|
|
18
|
-
burst.to_f, now, now, now
|
|
19
|
-
])
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
|
|
23
|
-
.lock("FOR UPDATE")
|
|
24
|
-
.first!
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def refill!(rate:, per:, burst:)
|
|
28
|
-
now = Time.current
|
|
29
|
-
elapsed = (now - refilled_at).to_f
|
|
30
|
-
new_tokens = tokens + (rate * elapsed / per)
|
|
31
|
-
self.tokens = [ new_tokens, burst.to_f ].min
|
|
32
|
-
self.refilled_at = now
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def consume(n = 1)
|
|
36
|
-
return false if tokens < n
|
|
37
|
-
self.tokens -= n
|
|
38
|
-
true
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
end
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateDispatchPolicyTables < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_staged_jobs do |t|
|
|
6
|
-
t.string :job_class, null: false
|
|
7
|
-
t.string :policy_name, null: false
|
|
8
|
-
t.jsonb :arguments, null: false
|
|
9
|
-
t.jsonb :snapshot, null: false, default: {}
|
|
10
|
-
t.jsonb :context, null: false, default: {}
|
|
11
|
-
t.integer :priority, null: false, default: 100
|
|
12
|
-
t.datetime :not_before_at
|
|
13
|
-
t.datetime :staged_at, null: false
|
|
14
|
-
t.datetime :admitted_at
|
|
15
|
-
t.datetime :completed_at
|
|
16
|
-
t.datetime :lease_expires_at
|
|
17
|
-
t.string :active_job_id
|
|
18
|
-
t.string :dedupe_key
|
|
19
|
-
t.string :round_robin_key
|
|
20
|
-
t.jsonb :partitions, null: false, default: {}
|
|
21
|
-
|
|
22
|
-
t.timestamps
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
add_index :dispatch_policy_staged_jobs,
|
|
26
|
-
%i[policy_name priority staged_at],
|
|
27
|
-
where: "admitted_at IS NULL",
|
|
28
|
-
name: "idx_dp_staged_dispatch_order"
|
|
29
|
-
|
|
30
|
-
add_index :dispatch_policy_staged_jobs,
|
|
31
|
-
%i[policy_name dedupe_key],
|
|
32
|
-
unique: true,
|
|
33
|
-
where: "dedupe_key IS NOT NULL AND completed_at IS NULL",
|
|
34
|
-
name: "idx_dp_staged_dedupe_active"
|
|
35
|
-
|
|
36
|
-
add_index :dispatch_policy_staged_jobs,
|
|
37
|
-
%i[lease_expires_at],
|
|
38
|
-
where: "admitted_at IS NOT NULL",
|
|
39
|
-
name: "idx_dp_staged_lease_expires"
|
|
40
|
-
|
|
41
|
-
add_index :dispatch_policy_staged_jobs,
|
|
42
|
-
%i[completed_at],
|
|
43
|
-
where: "completed_at IS NOT NULL",
|
|
44
|
-
name: "idx_dp_staged_completed_at"
|
|
45
|
-
|
|
46
|
-
add_index :dispatch_policy_staged_jobs,
|
|
47
|
-
%i[policy_name round_robin_key priority staged_at],
|
|
48
|
-
where: "admitted_at IS NULL AND round_robin_key IS NOT NULL",
|
|
49
|
-
name: "idx_dp_staged_round_robin"
|
|
50
|
-
|
|
51
|
-
create_table :dispatch_policy_partition_counts do |t|
|
|
52
|
-
t.string :policy_name, null: false
|
|
53
|
-
t.string :gate_name, null: false
|
|
54
|
-
t.string :partition_key, null: false, default: "default"
|
|
55
|
-
t.integer :in_flight, null: false, default: 0
|
|
56
|
-
|
|
57
|
-
t.timestamps
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
add_index :dispatch_policy_partition_counts,
|
|
61
|
-
%i[policy_name gate_name partition_key],
|
|
62
|
-
unique: true,
|
|
63
|
-
name: "idx_dp_partition_counts_unique"
|
|
64
|
-
|
|
65
|
-
create_table :dispatch_policy_throttle_buckets do |t|
|
|
66
|
-
t.string :policy_name, null: false
|
|
67
|
-
t.string :gate_name, null: false
|
|
68
|
-
t.string :partition_key, null: false, default: "default"
|
|
69
|
-
t.float :tokens, null: false
|
|
70
|
-
t.datetime :refilled_at, null: false
|
|
71
|
-
|
|
72
|
-
t.timestamps
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
add_index :dispatch_policy_throttle_buckets,
|
|
76
|
-
%i[policy_name gate_name partition_key],
|
|
77
|
-
unique: true,
|
|
78
|
-
name: "idx_dp_throttle_buckets_unique"
|
|
79
|
-
end
|
|
80
|
-
end
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateAdaptiveConcurrencyStats < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_adaptive_concurrency_stats do |t|
|
|
6
|
-
t.string :policy_name, null: false
|
|
7
|
-
t.string :gate_name, null: false
|
|
8
|
-
t.string :partition_key, null: false, default: "default"
|
|
9
|
-
t.integer :current_max, null: false
|
|
10
|
-
t.float :ewma_latency_ms, null: false, default: 0
|
|
11
|
-
t.integer :sample_count, null: false, default: 0
|
|
12
|
-
t.datetime :last_observed_at
|
|
13
|
-
|
|
14
|
-
t.timestamps
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
add_index :dispatch_policy_adaptive_concurrency_stats,
|
|
18
|
-
%i[policy_name gate_name partition_key],
|
|
19
|
-
unique: true,
|
|
20
|
-
name: "idx_dp_adaptive_concurrency_stats_unique"
|
|
21
|
-
end
|
|
22
|
-
end
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class CreateAdaptiveConcurrencySamples < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
create_table :dispatch_policy_adaptive_concurrency_samples do |t|
|
|
6
|
-
t.string :policy_name, null: false
|
|
7
|
-
t.string :gate_name, null: false
|
|
8
|
-
t.string :partition_key, null: false
|
|
9
|
-
t.datetime :minute_bucket, null: false
|
|
10
|
-
t.float :ewma_latency_ms, null: false, default: 0
|
|
11
|
-
t.integer :current_max, null: false
|
|
12
|
-
|
|
13
|
-
t.timestamps
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
add_index :dispatch_policy_adaptive_concurrency_samples,
|
|
17
|
-
%i[policy_name gate_name partition_key minute_bucket],
|
|
18
|
-
unique: true,
|
|
19
|
-
name: "idx_dp_adaptive_concurrency_samples_unique"
|
|
20
|
-
|
|
21
|
-
add_index :dispatch_policy_adaptive_concurrency_samples,
|
|
22
|
-
:minute_bucket,
|
|
23
|
-
name: "idx_dp_adaptive_concurrency_samples_time"
|
|
24
|
-
end
|
|
25
|
-
end
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class RenameSamplesToPartitionObservations < ActiveRecord::Migration[7.1]
|
|
4
|
-
def up
|
|
5
|
-
drop_table :dispatch_policy_adaptive_concurrency_samples, if_exists: true
|
|
6
|
-
|
|
7
|
-
create_table :dispatch_policy_partition_observations do |t|
|
|
8
|
-
t.string :policy_name, null: false
|
|
9
|
-
t.string :partition_key, null: false
|
|
10
|
-
t.datetime :minute_bucket, null: false
|
|
11
|
-
t.bigint :total_lag_ms, null: false, default: 0
|
|
12
|
-
t.integer :observation_count, null: false, default: 0
|
|
13
|
-
t.integer :max_lag_ms, null: false, default: 0
|
|
14
|
-
t.integer :current_max
|
|
15
|
-
|
|
16
|
-
t.timestamps
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
add_index :dispatch_policy_partition_observations,
|
|
20
|
-
%i[policy_name partition_key minute_bucket],
|
|
21
|
-
unique: true,
|
|
22
|
-
name: "idx_dp_partition_observations_unique"
|
|
23
|
-
|
|
24
|
-
add_index :dispatch_policy_partition_observations,
|
|
25
|
-
:minute_bucket,
|
|
26
|
-
name: "idx_dp_partition_observations_time"
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def down
|
|
30
|
-
drop_table :dispatch_policy_partition_observations
|
|
31
|
-
end
|
|
32
|
-
end
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
class AddDurationToPartitionObservations < ActiveRecord::Migration[7.1]
|
|
4
|
-
def change
|
|
5
|
-
add_column :dispatch_policy_partition_observations, :total_duration_ms, :bigint, null: false, default: 0
|
|
6
|
-
add_column :dispatch_policy_partition_observations, :max_duration_ms, :integer, null: false, default: 0
|
|
7
|
-
end
|
|
8
|
-
end
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module DispatchPolicy
|
|
4
|
-
# Rails 7.1's ActiveJob.perform_all_later(*jobs) bypasses ActiveJob::Base#enqueue
|
|
5
|
-
# and calls queue_adapter.enqueue_all directly. Dispatchable hooks on #enqueue,
|
|
6
|
-
# so without this patch the batch path would skip staging.
|
|
7
|
-
module ActiveJobPerformAllLaterPatch
|
|
8
|
-
def perform_all_later(*jobs)
|
|
9
|
-
jobs.flatten!
|
|
10
|
-
|
|
11
|
-
staged, remaining = jobs.partition do |job|
|
|
12
|
-
klass = job.class
|
|
13
|
-
klass.respond_to?(:dispatch_policy?) &&
|
|
14
|
-
klass.dispatch_policy? &&
|
|
15
|
-
DispatchPolicy.enabled?
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
staged_count = 0
|
|
19
|
-
staged.group_by(&:class).each do |klass, group|
|
|
20
|
-
staged_count += DispatchPolicy::StagedJob.stage_many!(
|
|
21
|
-
policy: klass.resolved_dispatch_policy,
|
|
22
|
-
jobs: group
|
|
23
|
-
)
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
remaining_count = remaining.empty? ? 0 : super(*remaining)
|
|
27
|
-
staged_count + remaining_count.to_i
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
ActiveJob.singleton_class.prepend(DispatchPolicy::ActiveJobPerformAllLaterPatch)
|