sidekiq 6.5.1 → 7.3.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +376 -12
  3. data/README.md +43 -35
  4. data/bin/multi_queue_bench +271 -0
  5. data/bin/sidekiq +3 -8
  6. data/bin/sidekiqload +213 -118
  7. data/bin/sidekiqmon +3 -0
  8. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +88 -0
  9. data/lib/generators/sidekiq/job_generator.rb +2 -0
  10. data/lib/sidekiq/api.rb +378 -173
  11. data/lib/sidekiq/capsule.rb +132 -0
  12. data/lib/sidekiq/cli.rb +61 -63
  13. data/lib/sidekiq/client.rb +89 -40
  14. data/lib/sidekiq/component.rb +6 -2
  15. data/lib/sidekiq/config.rb +305 -0
  16. data/lib/sidekiq/deploy.rb +64 -0
  17. data/lib/sidekiq/embedded.rb +63 -0
  18. data/lib/sidekiq/fetch.rb +11 -14
  19. data/lib/sidekiq/iterable_job.rb +55 -0
  20. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  21. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  22. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  23. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  24. data/lib/sidekiq/job/iterable.rb +294 -0
  25. data/lib/sidekiq/job.rb +382 -10
  26. data/lib/sidekiq/job_logger.rb +8 -7
  27. data/lib/sidekiq/job_retry.rb +89 -46
  28. data/lib/sidekiq/job_util.rb +53 -15
  29. data/lib/sidekiq/launcher.rb +77 -69
  30. data/lib/sidekiq/logger.rb +2 -27
  31. data/lib/sidekiq/manager.rb +9 -11
  32. data/lib/sidekiq/metrics/query.rb +158 -0
  33. data/lib/sidekiq/metrics/shared.rb +106 -0
  34. data/lib/sidekiq/metrics/tracking.rb +148 -0
  35. data/lib/sidekiq/middleware/chain.rb +84 -48
  36. data/lib/sidekiq/middleware/current_attributes.rb +87 -20
  37. data/lib/sidekiq/middleware/modules.rb +2 -0
  38. data/lib/sidekiq/monitor.rb +19 -5
  39. data/lib/sidekiq/paginator.rb +11 -3
  40. data/lib/sidekiq/processor.rb +67 -56
  41. data/lib/sidekiq/rails.rb +22 -16
  42. data/lib/sidekiq/redis_client_adapter.rb +31 -71
  43. data/lib/sidekiq/redis_connection.rb +44 -117
  44. data/lib/sidekiq/ring_buffer.rb +2 -0
  45. data/lib/sidekiq/scheduled.rb +62 -35
  46. data/lib/sidekiq/systemd.rb +2 -0
  47. data/lib/sidekiq/testing.rb +37 -46
  48. data/lib/sidekiq/transaction_aware_client.rb +11 -5
  49. data/lib/sidekiq/version.rb +6 -1
  50. data/lib/sidekiq/web/action.rb +15 -5
  51. data/lib/sidekiq/web/application.rb +94 -24
  52. data/lib/sidekiq/web/csrf_protection.rb +10 -7
  53. data/lib/sidekiq/web/helpers.rb +118 -45
  54. data/lib/sidekiq/web/router.rb +5 -2
  55. data/lib/sidekiq/web.rb +67 -15
  56. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  57. data/lib/sidekiq.rb +78 -266
  58. data/sidekiq.gemspec +12 -10
  59. data/web/assets/javascripts/application.js +46 -1
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +192 -0
  64. data/web/assets/javascripts/dashboard.js +11 -250
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +4 -0
  67. data/web/assets/stylesheets/application-rtl.css +10 -89
  68. data/web/assets/stylesheets/application.css +98 -295
  69. data/web/locales/ar.yml +70 -70
  70. data/web/locales/cs.yml +62 -62
  71. data/web/locales/da.yml +60 -53
  72. data/web/locales/de.yml +65 -65
  73. data/web/locales/el.yml +43 -24
  74. data/web/locales/en.yml +83 -69
  75. data/web/locales/es.yml +68 -68
  76. data/web/locales/fa.yml +65 -65
  77. data/web/locales/fr.yml +80 -67
  78. data/web/locales/gd.yml +98 -0
  79. data/web/locales/he.yml +65 -64
  80. data/web/locales/hi.yml +59 -59
  81. data/web/locales/it.yml +85 -54
  82. data/web/locales/ja.yml +72 -68
  83. data/web/locales/ko.yml +52 -52
  84. data/web/locales/lt.yml +66 -66
  85. data/web/locales/nb.yml +61 -61
  86. data/web/locales/nl.yml +52 -52
  87. data/web/locales/pl.yml +45 -45
  88. data/web/locales/pt-br.yml +78 -69
  89. data/web/locales/pt.yml +51 -51
  90. data/web/locales/ru.yml +67 -66
  91. data/web/locales/sv.yml +53 -53
  92. data/web/locales/ta.yml +60 -60
  93. data/web/locales/tr.yml +100 -0
  94. data/web/locales/uk.yml +85 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +67 -67
  97. data/web/locales/zh-cn.yml +42 -16
  98. data/web/locales/zh-tw.yml +41 -8
  99. data/web/views/_footer.erb +17 -2
  100. data/web/views/_job_info.erb +18 -2
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +1 -1
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +1 -1
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +49 -33
  107. data/web/views/dashboard.erb +28 -6
  108. data/web/views/filtering.erb +6 -0
  109. data/web/views/layout.erb +6 -6
  110. data/web/views/metrics.erb +90 -0
  111. data/web/views/metrics_for_job.erb +59 -0
  112. data/web/views/morgue.erb +5 -9
  113. data/web/views/queue.erb +15 -15
  114. data/web/views/queues.erb +9 -3
  115. data/web/views/retries.erb +5 -9
  116. data/web/views/scheduled.erb +12 -13
  117. metadata +61 -26
  118. data/lib/sidekiq/.DS_Store +0 -0
  119. data/lib/sidekiq/delay.rb +0 -43
  120. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  121. data/lib/sidekiq/extensions/active_record.rb +0 -43
  122. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  123. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  124. data/lib/sidekiq/worker.rb +0 -367
  125. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -0,0 +1,298 @@
1
+ class JobMetricsOverviewChart extends BaseChart {
2
+ constructor(el, options) {
3
+ super(el, { ...options, chartType: "line" });
4
+ this.swatches = [];
5
+ this.visibleKls = options.visibleKls;
6
+
7
+ const countBuckets = this.options.labels.length / 60;
8
+ this.labelBuckets = this.options.labels.reduce((acc, label, index) => {
9
+ const bucket = Math.floor(index / countBuckets);
10
+ acc[bucket] = acc[bucket] || [];
11
+ acc[bucket].push(label);
12
+ return acc;
13
+ }, []);
14
+
15
+ this.init();
16
+ }
17
+
18
+ get datasets() {
19
+ return Object.entries(this.options.series)
20
+ .filter(([kls, _]) => this.visibleKls.includes(kls))
21
+ .map(([kls, _]) => this.buildDataset(kls));
22
+ }
23
+
24
+ get metric() {
25
+ return this._metric || this.options.initialMetric;
26
+ }
27
+
28
+ set metric(m) {
29
+ this._metric = m;
30
+ }
31
+
32
+ registerSwatch(id) {
33
+ const el = document.getElementById(id);
34
+ el.addEventListener("change", () => this.toggleKls(el.value, el.checked));
35
+ this.swatches[el.value] = el;
36
+ this.updateSwatch(el.value, el.checked);
37
+ }
38
+
39
+ updateSwatch(kls, checked) {
40
+ const el = this.swatches[kls];
41
+ el.checked = checked;
42
+ el.style.color = this.colors.assignments[kls] || "";
43
+ }
44
+
45
+ toggleKls(kls, visible) {
46
+ if (visible) {
47
+ this.chart.data.datasets.push(this.buildDataset(kls));
48
+ } else {
49
+ const i = this.chart.data.datasets.findIndex((ds) => ds.label == kls);
50
+ this.colors.checkIn(kls);
51
+ this.chart.data.datasets.splice(i, 1);
52
+ }
53
+
54
+ this.updateSwatch(kls, visible);
55
+ this.update();
56
+ }
57
+
58
+ buildDataset(kls) {
59
+ const color = this.colors.checkOut(kls);
60
+
61
+ return {
62
+ label: kls,
63
+ data: this.buildSeries(kls),
64
+ borderColor: color,
65
+ backgroundColor: color,
66
+ borderWidth: 2,
67
+ pointRadius: 2,
68
+ };
69
+ }
70
+
71
+ buildSeries(kls) {
72
+ // `series` is an object that maps labels to counts => { "20:15" => 2, "20:16" => 3, ... }
73
+ const series = this.options.series[kls];
74
+ return this.labelBuckets.reduce((acc, labels) => {
75
+ const bucketValues = labels.map(label => series[label]).filter(v => v);
76
+ if (bucketValues.length > 0) {
77
+ // Sum up the values for each bucket that has data.
78
+ // The new label is the bucket's first label, its start time.
79
+ acc[labels[0]] = bucketValues.reduce((a, b) => a + b, 0);
80
+ }
81
+ return acc;
82
+ }, {});
83
+ }
84
+
85
+ buildTooltipTitle(items) {
86
+ const [first, ...rest] = this.labelBuckets.find((labels) => labels[0] === items[0].label);
87
+ const title = [first, rest[rest.length - 1]].filter(v => v).join(" - ");
88
+ return `${title} UTC`
89
+ }
90
+
91
+ get chartOptions() {
92
+ return {
93
+ ...super.chartOptions,
94
+ aspectRatio: 4,
95
+ scales: {
96
+ ...super.chartOptions.scales,
97
+ y: {
98
+ ...super.chartOptions.scales.y,
99
+ beginAtZero: true,
100
+ title: {
101
+ text: this.options.yLabel,
102
+ display: true,
103
+ },
104
+ },
105
+ },
106
+ plugins: {
107
+ ...super.chartOptions.plugins,
108
+ tooltip: {
109
+ ...super.chartOptions.plugins.tooltip,
110
+ callbacks: {
111
+ title: (items) => this.buildTooltipTitle(items),
112
+ label: (item) =>
113
+ `${item.dataset.label}: ${item.parsed.y.toFixed(1)} ` +
114
+ `${this.options.units}`,
115
+ footer: (items) => {
116
+ const bucket = items[0].label;
117
+ const marks = this.options.marks.filter(([b, _]) => b == bucket);
118
+ return marks.map(
119
+ ([b, msg]) => `${this.options.markLabel}: ${msg}`
120
+ );
121
+ },
122
+ },
123
+ },
124
+ },
125
+ };
126
+ }
127
+ }
128
+
129
+ class HistTotalsChart extends BaseChart {
130
+ constructor(el, options) {
131
+ super(el, { ...options, chartType: "bar" });
132
+ this.init();
133
+ }
134
+
135
+ get datasets() {
136
+ return [
137
+ {
138
+ data: this.options.series,
139
+ backgroundColor: this.colors.primary,
140
+ borderWidth: 0,
141
+ },
142
+ ];
143
+ }
144
+
145
+ get chartOptions() {
146
+ return {
147
+ ...super.chartOptions,
148
+ aspectRatio: 6,
149
+ scales: {
150
+ ...super.chartOptions.scales,
151
+ y: {
152
+ ...super.chartOptions.scales.y,
153
+ beginAtZero: true,
154
+ title: {
155
+ text: this.options.yLabel,
156
+ display: true,
157
+ },
158
+ },
159
+ x: {
160
+ ...super.chartOptions.scales.x,
161
+ title: {
162
+ text: this.options.xLabel,
163
+ display: true,
164
+ },
165
+ },
166
+ },
167
+ plugins: {
168
+ ...super.chartOptions.plugins,
169
+ tooltip: {
170
+ ...super.chartOptions.plugins.tooltip,
171
+ callbacks: {
172
+ label: (item) => `${item.parsed.y} ${this.options.units}`,
173
+ },
174
+ },
175
+ },
176
+ };
177
+ }
178
+ }
179
+
180
+ class HistBubbleChart extends BaseChart {
181
+ constructor(el, options) {
182
+ super(el, { ...options, chartType: "bubble" });
183
+ this.init();
184
+ }
185
+
186
+ get datasets() {
187
+ const data = [];
188
+ let maxCount = 0;
189
+
190
+ Object.entries(this.options.hist).forEach(([bucket, hist]) => {
191
+ hist.forEach((count, histBucket) => {
192
+ if (count > 0) {
193
+ // histogram data is ordered fastest to slowest, but this.histIntervals is
194
+ // slowest to fastest (so it displays correctly on the chart).
195
+ const index = this.options.histIntervals.length - 1 - histBucket;
196
+
197
+ data.push({
198
+ x: bucket,
199
+ y: this.options.histIntervals[index] / 1000,
200
+ count: count,
201
+ });
202
+
203
+ if (count > maxCount) maxCount = count;
204
+ }
205
+ });
206
+ });
207
+
208
+ // Chart.js will not calculate the bubble size. We have to do that.
209
+ const maxRadius = this.el.offsetWidth / this.options.labels.length;
210
+ const minRadius = 1;
211
+ const multiplier = (maxRadius / maxCount) * 1.5;
212
+ data.forEach((entry) => {
213
+ entry.r = entry.count * multiplier + minRadius;
214
+ });
215
+
216
+ return [
217
+ {
218
+ data: data,
219
+ backgroundColor: this.colors.primary,
220
+ borderColor: this.colors.primary,
221
+ },
222
+ ];
223
+ }
224
+
225
+ get chartOptions() {
226
+ return {
227
+ ...super.chartOptions,
228
+ aspectRatio: 3,
229
+ scales: {
230
+ ...super.chartOptions.scales,
231
+ x: {
232
+ ...super.chartOptions.scales.x,
233
+ type: "category",
234
+ labels: this.options.labels,
235
+ },
236
+ y: {
237
+ ...super.chartOptions.scales.y,
238
+ title: {
239
+ text: this.options.yLabel,
240
+ display: true,
241
+ },
242
+ },
243
+ },
244
+ plugins: {
245
+ ...super.chartOptions.plugins,
246
+ tooltip: {
247
+ ...super.chartOptions.plugins.tooltip,
248
+ callbacks: {
249
+ title: (items) => `${items[0].raw.x} UTC`,
250
+ label: (item) =>
251
+ `${item.parsed.y} ${this.options.yUnits}: ${item.raw.count} ${this.options.zUnits}`,
252
+ footer: (items) => {
253
+ const bucket = items[0].raw.x;
254
+ const marks = this.options.marks.filter(([b, _]) => b == bucket);
255
+ return marks.map(
256
+ ([b, msg]) => `${this.options.markLabel}: ${msg}`
257
+ );
258
+ },
259
+ },
260
+ },
261
+ },
262
+ };
263
+ }
264
+ }
265
+
266
+ var ch = document.getElementById("job-metrics-overview-chart");
267
+ if (ch != null) {
268
+ var jm = new JobMetricsOverviewChart(ch, JSON.parse(ch.textContent));
269
+ document.querySelectorAll(".metrics-swatch-wrapper > input[type=checkbox]").forEach((imp) => {
270
+ jm.registerSwatch(imp.id)
271
+ });
272
+ window.jobMetricsChart = jm;
273
+ }
274
+
275
+ var htc = document.getElementById("hist-totals-chart");
276
+ if (htc != null) {
277
+ var tc = new HistTotalsChart(htc, JSON.parse(htc.textContent));
278
+ window.histTotalsChart = tc
279
+ }
280
+
281
+ var hbc = document.getElementById("hist-bubble-chart");
282
+ if (hbc != null) {
283
+ var bc = new HistBubbleChart(hbc, JSON.parse(hbc.textContent));
284
+ window.histBubbleChart = bc
285
+ }
286
+
287
+ var form = document.getElementById("metrics-form")
288
+ document.querySelectorAll("#period-selector").forEach(node => {
289
+ node.addEventListener("input", debounce(() => form.submit()))
290
+ })
291
+
292
+ function debounce(func, timeout = 300) {
293
+ let timer;
294
+ return (...args) => {
295
+ clearTimeout(timer);
296
+ timer = setTimeout(() => { func.apply(this, args); }, timeout);
297
+ };
298
+ }
@@ -72,6 +72,10 @@ table {
72
72
  background-color: #31708f;
73
73
  }
74
74
 
75
+ .alert-warning {
76
+ background-color: #c47612;
77
+ }
78
+
75
79
  a:link, a:active, a:hover, a:visited {
76
80
  color: #ddd;
77
81
  }
@@ -102,22 +102,8 @@ div.interval-slider {
102
102
  float: left;
103
103
  }
104
104
 
105
- #realtime-legend,
106
- #history-legend {
107
- text-align: right;
108
- float: left;
109
- }
110
- #realtime-legend .timestamp,
111
- #history-legend .timestamp {
112
- text-align: left;
113
- }
114
- #realtime-legend .line,
115
- #history-legend .line {
116
- margin: 0 20px 0 0;
117
- }
118
- #realtime-legend .swatch,
119
- #history-legend .swatch {
120
- margin: 0 0 0 8px;
105
+ #realtime-legend {
106
+ justify-content: start;
121
107
  }
122
108
 
123
109
  /* Beacon
@@ -166,77 +152,12 @@ div.interval-slider {
166
152
  }
167
153
  }
168
154
 
169
- /* Rickshaw */
170
-
171
- .rickshaw_graph .detail .x_label.left {
172
- right: 0
173
- }
174
- .rickshaw_graph .detail .x_label.right {
175
- left: 0
176
- }
177
- .rickshaw_graph .detail .item.left {
178
- right: 0
179
- }
180
- .rickshaw_graph .detail .item.right {
181
- left: 0
182
- }
183
- .rickshaw_graph .detail .item.left:after {
184
- left: 0;
185
- right: -5px;
186
- border-right-color: unset;
187
- border-left-color: rgba(0, 0, 0, .8);
188
- border-right-width: 0;
189
- border-left-width: unset;
190
- }
191
- .rickshaw_graph .detail .item.right:after {
192
- right: 0;
193
- left: -5px;
194
- border-left-color: unset;
195
- border-right-color: rgba(0, 0, 0, .8);
196
- border-left-width: 0;
197
- border-right-width: unset;
198
- }
199
- .rickshaw_graph .detail .dot {
200
- margin-right: -3px;
201
- margin-left: unset;
202
- }
203
- .rickshaw_graph .x_tick {
204
- border-left: unset;
205
- border-right: 1px dotted rgba(0, 0, 0, .2);
206
- }
207
- .rickshaw_graph .x_tick .title {
208
- margin-right: 3px;
209
- margin-left: unset;
210
- }
211
- .rickshaw_annotation_timeline .annotation {
212
- margin-right: -2px;
213
- margin-left: unset;
214
- }
215
- .rickshaw_graph .annotation_line {
216
- border-right: 2px solid rgba(0, 0, 0, .3);
217
- border-left: unset;
218
- }
219
- .rickshaw_annotation_timeline .annotation .content {
220
- left: unset;
221
- right: -11px;
222
- }
223
- .rickshaw_graph .x_tick.glow .title,
224
- .rickshaw_graph .y_ticks.glow text {
225
- text-shadow: 1px 1px 0 rgba(255, 255, 255, .1), -1px -1px 0 rgba(255, 255, 255, .1), -1px 1px 0 rgba(255, 255, 255, .1), 0 1px 0 rgba(255, 255, 255, .1), 0 -1px 0 rgba(255, 255, 255, .1), -1px 0 0 rgba(255, 255, 255, .1), 1px 0 0 rgba(255, 255, 255, .1), 1px -1px 0 rgba(255, 255, 255, .1)
226
- }
227
- .rickshaw_graph .x_tick.inverse .title,
228
- .rickshaw_graph .y_ticks.inverse text {
229
- text-shadow: 1px 1px 0 rgba(0, 0, 0, .8), -1px -1px 0 rgba(0, 0, 0, .8), -1px 1px 0 rgba(0, 0, 0, .8), 0 1px 0 rgba(0, 0, 0, .8), 0 -1px 0 rgba(0, 0, 0, .8), -1px 0 0 rgba(0, 0, 0, .8), 1px 0 0 rgba(0, 0, 0, .8), 1px -1px 0 rgba(0, 0, 0, .8)
230
- }
231
- .rickshaw_legend .line {
232
- padding-left: 15px;
233
- padding-right: unset;
234
- }
235
- .rickshaw_legend .line .swatch {
236
- margin-left: 3px;
237
- margin-right: unset;
238
- }
239
- .rickshaw_legend .action {
240
- margin-left: .2em;
241
- margin-right: unset;
155
+ #locale-select {
156
+ float: right;
242
157
  }
158
+
159
+ @media (max-width: 767px) {
160
+ #locale-select {
161
+ float: none;
162
+ }
163
+ }