sidekiq 6.5.12 → 7.2.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +224 -20
  3. data/README.md +43 -35
  4. data/bin/multi_queue_bench +271 -0
  5. data/bin/sidekiq +3 -8
  6. data/bin/sidekiqload +204 -118
  7. data/bin/sidekiqmon +3 -0
  8. data/lib/sidekiq/api.rb +187 -135
  9. data/lib/sidekiq/capsule.rb +127 -0
  10. data/lib/sidekiq/cli.rb +59 -75
  11. data/lib/sidekiq/client.rb +66 -37
  12. data/lib/sidekiq/component.rb +4 -1
  13. data/lib/sidekiq/config.rb +287 -0
  14. data/lib/sidekiq/deploy.rb +62 -0
  15. data/lib/sidekiq/embedded.rb +61 -0
  16. data/lib/sidekiq/fetch.rb +11 -14
  17. data/lib/sidekiq/job.rb +371 -10
  18. data/lib/sidekiq/job_logger.rb +2 -2
  19. data/lib/sidekiq/job_retry.rb +36 -18
  20. data/lib/sidekiq/job_util.rb +51 -15
  21. data/lib/sidekiq/launcher.rb +71 -65
  22. data/lib/sidekiq/logger.rb +2 -27
  23. data/lib/sidekiq/manager.rb +9 -11
  24. data/lib/sidekiq/metrics/query.rb +7 -4
  25. data/lib/sidekiq/metrics/shared.rb +8 -7
  26. data/lib/sidekiq/metrics/tracking.rb +27 -21
  27. data/lib/sidekiq/middleware/chain.rb +19 -18
  28. data/lib/sidekiq/middleware/current_attributes.rb +52 -20
  29. data/lib/sidekiq/monitor.rb +16 -3
  30. data/lib/sidekiq/paginator.rb +2 -2
  31. data/lib/sidekiq/processor.rb +46 -51
  32. data/lib/sidekiq/rails.rb +15 -10
  33. data/lib/sidekiq/redis_client_adapter.rb +23 -66
  34. data/lib/sidekiq/redis_connection.rb +15 -117
  35. data/lib/sidekiq/scheduled.rb +22 -23
  36. data/lib/sidekiq/testing.rb +32 -41
  37. data/lib/sidekiq/transaction_aware_client.rb +11 -5
  38. data/lib/sidekiq/version.rb +2 -1
  39. data/lib/sidekiq/web/action.rb +8 -3
  40. data/lib/sidekiq/web/application.rb +108 -15
  41. data/lib/sidekiq/web/csrf_protection.rb +10 -7
  42. data/lib/sidekiq/web/helpers.rb +52 -38
  43. data/lib/sidekiq/web.rb +17 -16
  44. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  45. data/lib/sidekiq.rb +76 -274
  46. data/sidekiq.gemspec +12 -10
  47. data/web/assets/javascripts/application.js +39 -0
  48. data/web/assets/javascripts/base-charts.js +106 -0
  49. data/web/assets/javascripts/dashboard-charts.js +182 -0
  50. data/web/assets/javascripts/dashboard.js +10 -232
  51. data/web/assets/javascripts/metrics.js +151 -115
  52. data/web/assets/stylesheets/application-dark.css +4 -0
  53. data/web/assets/stylesheets/application-rtl.css +10 -89
  54. data/web/assets/stylesheets/application.css +45 -298
  55. data/web/locales/ar.yml +70 -70
  56. data/web/locales/cs.yml +62 -62
  57. data/web/locales/da.yml +60 -53
  58. data/web/locales/de.yml +65 -65
  59. data/web/locales/el.yml +2 -7
  60. data/web/locales/en.yml +78 -70
  61. data/web/locales/es.yml +68 -68
  62. data/web/locales/fa.yml +65 -65
  63. data/web/locales/fr.yml +81 -67
  64. data/web/locales/gd.yml +99 -0
  65. data/web/locales/he.yml +65 -64
  66. data/web/locales/hi.yml +59 -59
  67. data/web/locales/it.yml +53 -53
  68. data/web/locales/ja.yml +67 -69
  69. data/web/locales/ko.yml +52 -52
  70. data/web/locales/lt.yml +66 -66
  71. data/web/locales/nb.yml +61 -61
  72. data/web/locales/nl.yml +52 -52
  73. data/web/locales/pl.yml +45 -45
  74. data/web/locales/pt-br.yml +79 -69
  75. data/web/locales/pt.yml +51 -51
  76. data/web/locales/ru.yml +67 -66
  77. data/web/locales/sv.yml +53 -53
  78. data/web/locales/ta.yml +60 -60
  79. data/web/locales/uk.yml +62 -61
  80. data/web/locales/ur.yml +64 -64
  81. data/web/locales/vi.yml +67 -67
  82. data/web/locales/zh-cn.yml +20 -18
  83. data/web/locales/zh-tw.yml +10 -1
  84. data/web/views/_footer.erb +17 -2
  85. data/web/views/_job_info.erb +18 -2
  86. data/web/views/_metrics_period_select.erb +12 -0
  87. data/web/views/_paging.erb +2 -0
  88. data/web/views/_poll_link.erb +1 -1
  89. data/web/views/_summary.erb +7 -7
  90. data/web/views/busy.erb +46 -35
  91. data/web/views/dashboard.erb +26 -5
  92. data/web/views/filtering.erb +7 -0
  93. data/web/views/metrics.erb +46 -24
  94. data/web/views/metrics_for_job.erb +41 -69
  95. data/web/views/morgue.erb +5 -9
  96. data/web/views/queue.erb +10 -14
  97. data/web/views/queues.erb +9 -3
  98. data/web/views/retries.erb +5 -9
  99. data/web/views/scheduled.erb +12 -13
  100. metadata +44 -38
  101. data/lib/sidekiq/delay.rb +0 -43
  102. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  103. data/lib/sidekiq/extensions/active_record.rb +0 -43
  104. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  105. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  106. data/lib/sidekiq/metrics/deploy.rb +0 -47
  107. data/lib/sidekiq/worker.rb +0 -370
  108. data/web/assets/javascripts/graph.js +0 -16
  109. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -0,0 +1,182 @@
1
+ class DashboardChart extends BaseChart {
2
+ constructor(el, options) {
3
+ super(el, { ...options, chartType: "line" });
4
+ this.init();
5
+ }
6
+
7
+ get data() {
8
+ return [this.options.processed, this.options.failed];
9
+ }
10
+
11
+ get datasets() {
12
+ return [
13
+ {
14
+ label: this.options.processedLabel,
15
+ data: this.data[0],
16
+ borderColor: this.colors.success,
17
+ backgroundColor: this.colors.success,
18
+ borderWidth: 2,
19
+ pointRadius: 2,
20
+ },
21
+ {
22
+ label: this.options.failedLabel,
23
+ data: this.data[1],
24
+ borderColor: this.colors.failure,
25
+ backgroundColor: this.colors.failure,
26
+ borderWidth: 2,
27
+ pointRadius: 2,
28
+ },
29
+ ];
30
+ }
31
+
32
+ get chartOptions() {
33
+ return {
34
+ ...super.chartOptions,
35
+ aspectRatio: 4,
36
+ scales: {
37
+ ...super.chartOptions.scales,
38
+ x: {
39
+ ...super.chartOptions.scales.x,
40
+ ticks: {
41
+ ...super.chartOptions.scales.x.ticks,
42
+ callback: function (value, index, ticks) {
43
+ // Remove the year from the date string
44
+ return this.getLabelForValue(value).split("-").slice(1).join("-");
45
+ },
46
+ },
47
+ },
48
+ y: {
49
+ ...super.chartOptions.scales.y,
50
+ beginAtZero: true,
51
+ },
52
+ },
53
+ };
54
+ }
55
+ }
56
+
57
+ class RealtimeChart extends DashboardChart {
58
+ constructor(el, options) {
59
+ super(el, options);
60
+ let d = parseInt(localStorage.sidekiqTimeInterval) || 5000;
61
+ if (d < 2000) { d = 2000; }
62
+ this.delay = d
63
+ this.startPolling();
64
+ document.addEventListener("interval:update", this.handleUpdate.bind(this));
65
+ }
66
+
67
+ async startPolling() {
68
+ // Fetch initial values so we can show diffs moving forward
69
+ this.stats = await this.fetchStats();
70
+ this._interval = setInterval(this.poll.bind(this), this.delay);
71
+ }
72
+
73
+ async poll() {
74
+ const stats = await this.fetchStats();
75
+ const processed = stats.sidekiq.processed - this.stats.sidekiq.processed;
76
+ const failed = stats.sidekiq.failed - this.stats.sidekiq.failed;
77
+
78
+ this.chart.data.labels.shift();
79
+ this.chart.data.datasets[0].data.shift();
80
+ this.chart.data.datasets[1].data.shift();
81
+ this.chart.data.labels.push(new Date().toUTCString().split(" ")[4]);
82
+ this.chart.data.datasets[0].data.push(processed);
83
+ this.chart.data.datasets[1].data.push(failed);
84
+ this.chart.update();
85
+
86
+ updateStatsSummary(this.stats.sidekiq);
87
+ updateRedisStats(this.stats.redis);
88
+ updateFooterUTCTime(this.stats.server_utc_time);
89
+ updateNumbers();
90
+ pulseBeacon();
91
+
92
+ this.stats = stats;
93
+ }
94
+
95
+ async fetchStats() {
96
+ const response = await fetch(this.options.updateUrl);
97
+ return await response.json();
98
+ }
99
+
100
+ handleUpdate(e) {
101
+ this.delay = parseInt(e.detail);
102
+ clearInterval(this._interval);
103
+ this.startPolling();
104
+ }
105
+
106
+ registerLegend(el) {
107
+ this.legend = el;
108
+ }
109
+
110
+ renderLegend(dp) {
111
+ this.legend.innerHTML = `
112
+ <span>
113
+ <span class="swatch" style="background-color: ${dp[0].dataset.borderColor};"></span>
114
+ <span>${dp[0].dataset.label}: ${dp[0].formattedValue}</span>
115
+ </span>
116
+ <span>
117
+ <span class="swatch" style="background-color: ${dp[1].dataset.borderColor};"></span>
118
+ <span>${dp[1].dataset.label}: ${dp[1].formattedValue}</span>
119
+ </span>
120
+ <span class="time">${dp[0].label}</span>
121
+ `;
122
+ }
123
+
124
+ renderCursor(dp) {
125
+ if (this.cursorX != dp[0].label) {
126
+ this.cursorX = dp[0].label;
127
+ this.update()
128
+ }
129
+ }
130
+
131
+ get chartOptions() {
132
+ return {
133
+ ...super.chartOptions,
134
+ scales: {
135
+ ...super.chartOptions.scales,
136
+ x: {
137
+ ...super.chartOptions.scales.x,
138
+ display: false,
139
+ },
140
+ },
141
+ plugins: {
142
+ ...super.chartOptions.plugins,
143
+ tooltip: {
144
+ ...super.chartOptions.plugins.tooltip,
145
+ enabled: false,
146
+ external: (context) => {
147
+ const dp = context.tooltip.dataPoints;
148
+ if (dp && dp.length == 2 && this.legend) {
149
+ this.renderLegend(dp);
150
+ this.renderCursor(dp);
151
+ }
152
+ },
153
+ },
154
+ annotation: {
155
+ annotations: {
156
+ ...super.chartOptions.plugins.annotation.annotations,
157
+ cursor: this.cursorX && {
158
+ type: "line",
159
+ borderColor: "rgba(0, 0, 0, 0.3)",
160
+ xMin: this.cursorX,
161
+ xMax: this.cursorX,
162
+ borderWidth: 1,
163
+ },
164
+ },
165
+ },
166
+ },
167
+ };
168
+ }
169
+ }
170
+
171
+ var rc = document.getElementById("realtime-chart")
172
+ if (rc != null) {
173
+ var rtc = new RealtimeChart(rc, JSON.parse(rc.textContent))
174
+ rtc.registerLegend(document.getElementById("realtime-legend"))
175
+ window.realtimeChart = rtc
176
+ }
177
+
178
+ var hc = document.getElementById("history-chart")
179
+ if (hc != null) {
180
+ var htc = new DashboardChart(hc, JSON.parse(hc.textContent))
181
+ window.historyChart = htc
182
+ }
@@ -1,197 +1,13 @@
1
1
  Sidekiq = {};
2
2
 
3
- var nf = new Intl.NumberFormat();
4
- var poller;
5
- var realtimeGraph = function(updatePath) {
6
- var timeInterval = parseInt(localStorage.sidekiqTimeInterval) || 5000;
7
- var graphElement = document.getElementById("realtime");
8
-
9
- var graph = new Rickshaw.Graph( {
10
- element: graphElement,
11
- width: responsiveWidth(),
12
- height: 200,
13
- renderer: 'line',
14
- interpolation: 'linear',
15
-
16
- series: new Rickshaw.Series.FixedDuration([{ name: graphElement.dataset.failedLabel, color: '#af0014' }, { name: graphElement.dataset.processedLabel, color: '#006f68' }], undefined, {
17
- timeInterval: timeInterval,
18
- maxDataPoints: 100,
19
- })
20
- });
21
-
22
- var y_axis = new Rickshaw.Graph.Axis.Y( {
23
- graph: graph,
24
- tickFormat: Rickshaw.Fixtures.Number.formatKMBT,
25
- ticksTreatment: 'glow'
26
- });
27
-
28
- graph.render();
29
-
30
- var legend = document.getElementById('realtime-legend');
31
- var Hover = Rickshaw.Class.create(Rickshaw.Graph.HoverDetail, {
32
- render: function(args) {
33
- legend.innerHTML = "";
34
-
35
- var timestamp = document.createElement('div');
36
- timestamp.className = 'timestamp';
37
- timestamp.innerHTML = args.formattedXValue;
38
- legend.appendChild(timestamp);
39
-
40
- args.detail.sort(function(a, b) { return a.order - b.order }).forEach( function(d) {
41
- var line = document.createElement('div');
42
- line.className = 'line';
43
-
44
- var swatch = document.createElement('div');
45
- swatch.className = 'swatch';
46
- swatch.style.backgroundColor = d.series.color;
47
-
48
- var label = document.createElement('div');
49
- label.className = 'tag';
50
- label.innerHTML = d.name + ": " + nf.format(Math.floor(d.formattedYValue));
51
-
52
- line.appendChild(swatch);
53
- line.appendChild(label);
54
- legend.appendChild(line);
55
-
56
- var dot = document.createElement('div');
57
- dot.className = 'dot';
58
- dot.style.top = graph.y(d.value.y0 + d.value.y) + 'px';
59
- dot.style.borderColor = d.series.color;
60
-
61
- this.element.appendChild(dot);
62
- dot.className = 'dot active';
63
- this.show();
64
- }, this );
65
- }
66
- });
67
- var hover = new Hover( { graph: graph } );
68
-
69
- var i = 0;
70
- poller = setInterval(function() {
71
- var url = document.getElementById("history").getAttribute("data-update-url");
72
-
73
- fetch(url).then(response => response.json()).then(data => {
74
- if (i === 0) {
75
- var processed = data.sidekiq.processed;
76
- var failed = data.sidekiq.failed;
77
- } else {
78
- var processed = data.sidekiq.processed - Sidekiq.processed;
79
- var failed = data.sidekiq.failed - Sidekiq.failed;
80
- }
81
-
82
- dataPoint = {};
83
- dataPoint[graphElement.dataset.failedLabel] = failed;
84
- dataPoint[graphElement.dataset.processedLabel] = processed;
85
-
86
- graph.series.addData(dataPoint);
87
- graph.render();
88
-
89
- Sidekiq.processed = data.sidekiq.processed;
90
- Sidekiq.failed = data.sidekiq.failed;
91
-
92
- updateStatsSummary(data.sidekiq);
93
- updateRedisStats(data.redis);
94
- updateFooterUTCTime(data.server_utc_time)
95
-
96
- pulseBeacon();
97
- });
98
-
99
- i++;
100
- }, timeInterval);
101
- }
102
-
103
- var historyGraph = function() {
104
- var h = document.getElementById("history");
105
- processed = createSeries(h.getAttribute("data-processed"));
106
- failed = createSeries(h.getAttribute("data-failed"));
107
-
108
- var graphElement = h;
109
- var graph = new Rickshaw.Graph( {
110
- element: graphElement,
111
- width: responsiveWidth(),
112
- height: 200,
113
- renderer: 'line',
114
- interpolation: 'linear',
115
- series: [
116
- {
117
- color: "#af0014",
118
- data: failed,
119
- name: graphElement.dataset.failedLabel
120
- }, {
121
- color: "#006f68",
122
- data: processed,
123
- name: graphElement.dataset.processedLabel
124
- }
125
- ]
126
- } );
127
- var x_axis = new Rickshaw.Graph.Axis.Time( { graph: graph } );
128
- var y_axis = new Rickshaw.Graph.Axis.Y({
129
- graph: graph,
130
- tickFormat: Rickshaw.Fixtures.Number.formatKMBT,
131
- ticksTreatment: 'glow'
132
- });
133
-
134
- graph.render();
135
-
136
- var legend = document.getElementById('history-legend');
137
- var Hover = Rickshaw.Class.create(Rickshaw.Graph.HoverDetail, {
138
- render: function(args) {
139
- legend.innerHTML = "";
140
-
141
- var timestamp = document.createElement('div');
142
- timestamp.className = 'timestamp';
143
- timestamp.innerHTML = args.formattedXValue;
144
- legend.appendChild(timestamp);
145
-
146
- args.detail.sort(function(a, b) { return a.order - b.order }).forEach( function(d) {
147
- var line = document.createElement('div');
148
- line.className = 'line';
149
-
150
- var swatch = document.createElement('div');
151
- swatch.className = 'swatch';
152
- swatch.style.backgroundColor = d.series.color;
153
-
154
- var label = document.createElement('div');
155
- label.className = 'tag';
156
- label.innerHTML = d.name + ": " + nf.format(Math.floor(d.formattedYValue));
157
-
158
- line.appendChild(swatch);
159
- line.appendChild(label);
160
- legend.appendChild(line);
161
-
162
- var dot = document.createElement('div');
163
- dot.className = 'dot';
164
- dot.style.top = graph.y(d.value.y0 + d.value.y) + 'px';
165
- dot.style.borderColor = d.series.color;
166
-
167
- this.element.appendChild(dot);
168
- dot.className = 'dot active';
169
- this.show();
170
- }, this );
171
- }
172
- });
173
- var hover = new Hover( { graph: graph } );
174
- }
175
-
176
- var createSeries = function(data) {
177
- var obj = JSON.parse(data);
178
- var series = [];
179
- for (var date in obj) {
180
- var value = obj[date];
181
- var point = { x: Date.parse(date)/1000, y: value };
182
- series.unshift(point);
183
- }
184
- return series;
185
- };
186
-
187
3
  var updateStatsSummary = function(data) {
188
- document.getElementById("txtProcessed").innerText = nf.format(data.processed);
189
- document.getElementById("txtFailed").innerText = nf.format(data.failed);
190
- document.getElementById("txtBusy").innerText = nf.format(data.busy);
191
- document.getElementById("txtScheduled").innerText = nf.format(data.scheduled);
192
- document.getElementById("txtRetries").innerText = nf.format(data.retries);
193
- document.getElementById("txtEnqueued").innerText = nf.format(data.enqueued);
194
- document.getElementById("txtDead").innerText = nf.format(data.dead);
4
+ document.getElementById("txtProcessed").innerText = data.processed;
5
+ document.getElementById("txtFailed").innerText = data.failed;
6
+ document.getElementById("txtBusy").innerText = data.busy;
7
+ document.getElementById("txtScheduled").innerText = data.scheduled;
8
+ document.getElementById("txtRetries").innerText = data.retries;
9
+ document.getElementById("txtEnqueued").innerText = data.enqueued;
10
+ document.getElementById("txtDead").innerText = data.dead;
195
11
  }
196
12
 
197
13
  var updateRedisStats = function(data) {
@@ -211,12 +27,6 @@ var pulseBeacon = function() {
211
27
  window.setTimeout(() => { document.getElementById('beacon').classList.remove('pulse'); }, 1000);
212
28
  }
213
29
 
214
- // Render graphs
215
- var renderGraphs = function() {
216
- realtimeGraph();
217
- historyGraph();
218
- };
219
-
220
30
  var setSliderLabel = function(val) {
221
31
  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' sec';
222
32
  }
@@ -227,8 +37,6 @@ var ready = (callback) => {
227
37
  }
228
38
 
229
39
  ready(() => {
230
- renderGraphs();
231
-
232
40
  var sldr = document.getElementById('sldr');
233
41
  if (typeof localStorage.sidekiqTimeInterval !== 'undefined') {
234
42
  sldr.value = localStorage.sidekiqTimeInterval;
@@ -236,44 +44,14 @@ ready(() => {
236
44
  }
237
45
 
238
46
  sldr.addEventListener("change", event => {
239
- clearInterval(poller);
240
47
  localStorage.sidekiqTimeInterval = sldr.value;
241
48
  setSliderLabel(sldr.value);
242
- resetGraphs();
243
- renderGraphs();
49
+ sldr.dispatchEvent(
50
+ new CustomEvent("interval:update", { bubbles: true, detail: sldr.value })
51
+ );
244
52
  });
245
53
 
246
54
  sldr.addEventListener("mousemove", event => {
247
55
  setSliderLabel(sldr.value);
248
56
  });
249
57
  });
250
-
251
- // Reset graphs
252
- var resetGraphs = function() {
253
- document.getElementById('realtime').innerHTML = '';
254
- document.getElementById('history').innerHTML = '';
255
- };
256
-
257
- // Resize graphs after resizing window
258
- var debounce = function(fn, timeout) {
259
- var timeoutID = -1;
260
- return function() {
261
- if (timeoutID > -1) {
262
- window.clearTimeout(timeoutID);
263
- }
264
- timeoutID = window.setTimeout(fn, timeout);
265
- }
266
- };
267
-
268
- window.onresize = function() {
269
- var prevWidth = window.innerWidth;
270
- return debounce(function () {
271
- var currWidth = window.innerWidth;
272
- if (prevWidth !== currWidth) {
273
- prevWidth = currWidth;
274
- clearInterval(poller);
275
- resetGraphs();
276
- renderGraphs();
277
- }
278
- }, 125);
279
- }();