sidekiq 6.5.10 → 7.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +159 -18
  3. data/README.md +42 -34
  4. data/bin/sidekiq +3 -8
  5. data/bin/sidekiqload +204 -118
  6. data/bin/sidekiqmon +3 -0
  7. data/lib/sidekiq/api.rb +114 -128
  8. data/lib/sidekiq/capsule.rb +127 -0
  9. data/lib/sidekiq/cli.rb +56 -74
  10. data/lib/sidekiq/client.rb +66 -37
  11. data/lib/sidekiq/component.rb +4 -1
  12. data/lib/sidekiq/config.rb +287 -0
  13. data/lib/sidekiq/deploy.rb +62 -0
  14. data/lib/sidekiq/embedded.rb +61 -0
  15. data/lib/sidekiq/fetch.rb +11 -14
  16. data/lib/sidekiq/job.rb +371 -10
  17. data/lib/sidekiq/job_logger.rb +2 -2
  18. data/lib/sidekiq/job_retry.rb +33 -15
  19. data/lib/sidekiq/job_util.rb +51 -15
  20. data/lib/sidekiq/launcher.rb +66 -62
  21. data/lib/sidekiq/logger.rb +1 -26
  22. data/lib/sidekiq/manager.rb +9 -11
  23. data/lib/sidekiq/metrics/query.rb +3 -3
  24. data/lib/sidekiq/metrics/shared.rb +8 -7
  25. data/lib/sidekiq/metrics/tracking.rb +20 -18
  26. data/lib/sidekiq/middleware/chain.rb +19 -18
  27. data/lib/sidekiq/middleware/current_attributes.rb +52 -20
  28. data/lib/sidekiq/monitor.rb +16 -3
  29. data/lib/sidekiq/paginator.rb +1 -1
  30. data/lib/sidekiq/processor.rb +46 -51
  31. data/lib/sidekiq/rails.rb +7 -17
  32. data/lib/sidekiq/redis_client_adapter.rb +10 -69
  33. data/lib/sidekiq/redis_connection.rb +12 -111
  34. data/lib/sidekiq/scheduled.rb +21 -22
  35. data/lib/sidekiq/testing.rb +24 -39
  36. data/lib/sidekiq/transaction_aware_client.rb +4 -5
  37. data/lib/sidekiq/version.rb +2 -1
  38. data/lib/sidekiq/web/action.rb +3 -3
  39. data/lib/sidekiq/web/application.rb +76 -11
  40. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  41. data/lib/sidekiq/web/helpers.rb +39 -24
  42. data/lib/sidekiq/web.rb +17 -16
  43. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  44. data/lib/sidekiq.rb +76 -274
  45. data/sidekiq.gemspec +12 -10
  46. data/web/assets/javascripts/application.js +18 -0
  47. data/web/assets/javascripts/base-charts.js +106 -0
  48. data/web/assets/javascripts/dashboard-charts.js +168 -0
  49. data/web/assets/javascripts/dashboard.js +3 -223
  50. data/web/assets/javascripts/metrics.js +117 -115
  51. data/web/assets/stylesheets/application-dark.css +4 -0
  52. data/web/assets/stylesheets/application-rtl.css +2 -91
  53. data/web/assets/stylesheets/application.css +23 -298
  54. data/web/locales/ar.yml +70 -70
  55. data/web/locales/cs.yml +62 -62
  56. data/web/locales/da.yml +60 -53
  57. data/web/locales/de.yml +65 -65
  58. data/web/locales/el.yml +2 -7
  59. data/web/locales/en.yml +78 -70
  60. data/web/locales/es.yml +68 -68
  61. data/web/locales/fa.yml +65 -65
  62. data/web/locales/fr.yml +81 -67
  63. data/web/locales/gd.yml +99 -0
  64. data/web/locales/he.yml +65 -64
  65. data/web/locales/hi.yml +59 -59
  66. data/web/locales/it.yml +53 -53
  67. data/web/locales/ja.yml +67 -69
  68. data/web/locales/ko.yml +52 -52
  69. data/web/locales/lt.yml +66 -66
  70. data/web/locales/nb.yml +61 -61
  71. data/web/locales/nl.yml +52 -52
  72. data/web/locales/pl.yml +45 -45
  73. data/web/locales/pt-br.yml +79 -69
  74. data/web/locales/pt.yml +51 -51
  75. data/web/locales/ru.yml +67 -66
  76. data/web/locales/sv.yml +53 -53
  77. data/web/locales/ta.yml +60 -60
  78. data/web/locales/uk.yml +62 -61
  79. data/web/locales/ur.yml +64 -64
  80. data/web/locales/vi.yml +67 -67
  81. data/web/locales/zh-cn.yml +20 -18
  82. data/web/locales/zh-tw.yml +10 -1
  83. data/web/views/_footer.erb +5 -2
  84. data/web/views/_job_info.erb +18 -2
  85. data/web/views/_metrics_period_select.erb +12 -0
  86. data/web/views/_paging.erb +2 -0
  87. data/web/views/_poll_link.erb +1 -1
  88. data/web/views/busy.erb +39 -28
  89. data/web/views/dashboard.erb +36 -5
  90. data/web/views/filtering.erb +7 -0
  91. data/web/views/metrics.erb +33 -20
  92. data/web/views/metrics_for_job.erb +25 -44
  93. data/web/views/morgue.erb +5 -9
  94. data/web/views/queue.erb +10 -14
  95. data/web/views/queues.erb +3 -1
  96. data/web/views/retries.erb +5 -9
  97. data/web/views/scheduled.erb +12 -13
  98. metadata +44 -39
  99. data/lib/sidekiq/delay.rb +0 -43
  100. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  101. data/lib/sidekiq/extensions/active_record.rb +0 -43
  102. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  103. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  104. data/lib/sidekiq/metrics/deploy.rb +0 -47
  105. data/lib/sidekiq/worker.rb +0 -370
  106. data/web/assets/javascripts/graph.js +0 -16
  107. /data/{LICENSE → LICENSE.txt} +0 -0
@@ -1,89 +1,66 @@
1
- if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
2
- Chart.defaults.borderColor = "#333"
3
- Chart.defaults.color = "#aaa"
4
- }
1
+ class JobMetricsOverviewChart extends BaseChart {
2
+ constructor(el, options) {
3
+ super(el, { ...options, chartType: "line" });
4
+ this.swatches = [];
5
+ this.visibleKls = options.visibleKls;
5
6
 
6
- class BaseChart {
7
- constructor(id, options) {
8
- this.ctx = document.getElementById(id);
9
- this.options = options
10
- this.fallbackColor = "#999";
11
- this.colors = [
12
- // Colors taken from https://www.chartjs.org/docs/latest/samples/utils.html
13
- "#537bc4",
14
- "#4dc9f6",
15
- "#f67019",
16
- "#f53794",
17
- "#acc236",
18
- "#166a8f",
19
- "#00a950",
20
- "#58595b",
21
- "#8549ba",
22
- "#991b1b",
23
- ];
7
+ const countBuckets = this.options.labels.length / 60;
8
+ this.labelBuckets = this.options.labels.reduce((acc, label, index) => {
9
+ const bucket = Math.floor(index / countBuckets);
10
+ acc[bucket] = acc[bucket] || [];
11
+ acc[bucket].push(label);
12
+ return acc;
13
+ }, []);
24
14
 
25
- this.chart = new Chart(this.ctx, {
26
- type: this.options.chartType,
27
- data: { labels: this.options.labels, datasets: this.datasets },
28
- options: this.chartOptions,
29
- });
15
+ this.init();
30
16
  }
31
17
 
32
- addMarksToChart() {
33
- this.options.marks.forEach(([bucket, label], i) => {
34
- this.chart.options.plugins.annotation.annotations[`deploy-${i}`] = {
35
- type: "line",
36
- xMin: bucket,
37
- xMax: bucket,
38
- borderColor: "rgba(220, 38, 38, 0.4)",
39
- borderWidth: 2,
40
- };
41
- });
18
+ get datasets() {
19
+ return Object.entries(this.options.series)
20
+ .filter(([kls, _]) => this.visibleKls.includes(kls))
21
+ .map(([kls, _]) => this.buildDataset(kls));
42
22
  }
43
- }
44
23
 
45
- class JobMetricsOverviewChart extends BaseChart {
46
- constructor(id, options) {
47
- super(id, { ...options, chartType: "line" });
48
- this.swatches = [];
24
+ get metric() {
25
+ return this._metric || this.options.initialMetric;
26
+ }
49
27
 
50
- this.addMarksToChart();
51
- this.chart.update();
28
+ set metric(m) {
29
+ this._metric = m;
52
30
  }
53
31
 
54
32
  registerSwatch(id) {
55
33
  const el = document.getElementById(id);
56
- el.onchange = () => this.toggle(el.value, el.checked);
34
+ el.addEventListener("change", () => this.toggleKls(el.value, el.checked));
57
35
  this.swatches[el.value] = el;
58
- this.updateSwatch(el.value);
36
+ this.updateSwatch(el.value, el.checked);
59
37
  }
60
38
 
61
- updateSwatch(kls) {
39
+ updateSwatch(kls, checked) {
62
40
  const el = this.swatches[kls];
63
- const ds = this.chart.data.datasets.find((ds) => ds.label == kls);
64
- el.checked = !!ds;
65
- el.style.color = ds ? ds.borderColor : null;
41
+ el.checked = checked;
42
+ el.style.color = this.colors.assignments[kls] || "";
66
43
  }
67
44
 
68
- toggle(kls, visible) {
45
+ toggleKls(kls, visible) {
69
46
  if (visible) {
70
- this.chart.data.datasets.push(this.dataset(kls));
47
+ this.chart.data.datasets.push(this.buildDataset(kls));
71
48
  } else {
72
49
  const i = this.chart.data.datasets.findIndex((ds) => ds.label == kls);
73
- this.colors.unshift(this.chart.data.datasets[i].borderColor);
50
+ this.colors.checkIn(kls);
74
51
  this.chart.data.datasets.splice(i, 1);
75
52
  }
76
53
 
77
- this.updateSwatch(kls);
78
- this.chart.update();
54
+ this.updateSwatch(kls, visible);
55
+ this.update();
79
56
  }
80
57
 
81
- dataset(kls) {
82
- const color = this.colors.shift() || this.fallbackColor;
58
+ buildDataset(kls) {
59
+ const color = this.colors.checkOut(kls);
83
60
 
84
61
  return {
85
62
  label: kls,
86
- data: this.options.series[kls],
63
+ data: this.buildSeries(kls),
87
64
  borderColor: color,
88
65
  backgroundColor: color,
89
66
  borderWidth: 2,
@@ -91,40 +68,56 @@ class JobMetricsOverviewChart extends BaseChart {
91
68
  };
92
69
  }
93
70
 
94
- get datasets() {
95
- return Object.entries(this.options.series)
96
- .filter(([kls, _]) => this.options.visible.includes(kls))
97
- .map(([kls, _]) => this.dataset(kls));
71
+ buildSeries(kls) {
72
+ // `series` is an object that maps labels to counts => { "20:15" => 2, "20:16" => 3, ... }
73
+ const series = this.options.series[kls];
74
+ return this.labelBuckets.reduce((acc, labels) => {
75
+ const bucketValues = labels.map(label => series[label]).filter(v => v);
76
+ if (bucketValues.length > 0) {
77
+ // Sum up the values for each bucket that has data.
78
+ // The new label is the bucket's first label, its start time.
79
+ acc[labels[0]] = bucketValues.reduce((a, b) => a + b, 0);
80
+ }
81
+ return acc;
82
+ }, {});
83
+ }
84
+
85
+ buildTooltipTitle(items) {
86
+ const [first, ...rest] = this.labelBuckets.find((labels) => labels[0] === items[0].label);
87
+ const title = [first, rest[rest.length - 1]].filter(v => v).join(" - ");
88
+ return `${title} UTC`
98
89
  }
99
90
 
100
91
  get chartOptions() {
101
92
  return {
93
+ ...super.chartOptions,
102
94
  aspectRatio: 4,
103
95
  scales: {
96
+ ...super.chartOptions.scales,
104
97
  y: {
98
+ ...super.chartOptions.scales.y,
105
99
  beginAtZero: true,
106
100
  title: {
107
- text: "Total execution time (sec)",
101
+ text: this.options.yLabel,
108
102
  display: true,
109
103
  },
110
104
  },
111
105
  },
112
- interaction: {
113
- mode: "x",
114
- },
115
106
  plugins: {
116
- legend: {
117
- display: false,
118
- },
107
+ ...super.chartOptions.plugins,
119
108
  tooltip: {
109
+ ...super.chartOptions.plugins.tooltip,
120
110
  callbacks: {
121
- title: (items) => `${items[0].label} UTC`,
111
+ title: (items) => this.buildTooltipTitle(items),
122
112
  label: (item) =>
123
- `${item.dataset.label}: ${item.parsed.y.toFixed(1)} seconds`,
113
+ `${item.dataset.label}: ${item.parsed.y.toFixed(1)} ` +
114
+ `${this.options.units}`,
124
115
  footer: (items) => {
125
116
  const bucket = items[0].label;
126
117
  const marks = this.options.marks.filter(([b, _]) => b == bucket);
127
- return marks.map(([b, msg]) => `Deploy: ${msg}`);
118
+ return marks.map(
119
+ ([b, msg]) => `${this.options.markLabel}: ${msg}`
120
+ );
128
121
  },
129
122
  },
130
123
  },
@@ -134,40 +127,49 @@ class JobMetricsOverviewChart extends BaseChart {
134
127
  }
135
128
 
136
129
  class HistTotalsChart extends BaseChart {
137
- constructor(id, options) {
138
- super(id, { ...options, chartType: "bar" });
130
+ constructor(el, options) {
131
+ super(el, { ...options, chartType: "bar" });
132
+ this.init();
139
133
  }
140
134
 
141
135
  get datasets() {
142
- return [{
143
- data: this.options.series,
144
- backgroundColor: this.colors[0],
145
- borderWidth: 0,
146
- }];
136
+ return [
137
+ {
138
+ data: this.options.series,
139
+ backgroundColor: this.colors.primary,
140
+ borderWidth: 0,
141
+ },
142
+ ];
147
143
  }
148
144
 
149
145
  get chartOptions() {
150
146
  return {
147
+ ...super.chartOptions,
151
148
  aspectRatio: 6,
152
149
  scales: {
150
+ ...super.chartOptions.scales,
153
151
  y: {
152
+ ...super.chartOptions.scales.y,
154
153
  beginAtZero: true,
155
154
  title: {
156
- text: "Total jobs",
155
+ text: this.options.yLabel,
156
+ display: true,
157
+ },
158
+ },
159
+ x: {
160
+ ...super.chartOptions.scales.x,
161
+ title: {
162
+ text: this.options.xLabel,
157
163
  display: true,
158
164
  },
159
165
  },
160
- },
161
- interaction: {
162
- mode: "x",
163
166
  },
164
167
  plugins: {
165
- legend: {
166
- display: false,
167
- },
168
+ ...super.chartOptions.plugins,
168
169
  tooltip: {
170
+ ...super.chartOptions.plugins.tooltip,
169
171
  callbacks: {
170
- label: (item) => `${item.parsed.y} jobs`,
172
+ label: (item) => `${item.parsed.y} ${this.options.units}`,
171
173
  },
172
174
  },
173
175
  },
@@ -176,11 +178,9 @@ class HistTotalsChart extends BaseChart {
176
178
  }
177
179
 
178
180
  class HistBubbleChart extends BaseChart {
179
- constructor(id, options) {
180
- super(id, { ...options, chartType: "bubble" });
181
-
182
- this.addMarksToChart();
183
- this.chart.update();
181
+ constructor(el, options) {
182
+ super(el, { ...options, chartType: "bubble" });
183
+ this.init();
184
184
  }
185
185
 
186
186
  get datasets() {
@@ -190,13 +190,13 @@ class HistBubbleChart extends BaseChart {
190
190
  Object.entries(this.options.hist).forEach(([bucket, hist]) => {
191
191
  hist.forEach((count, histBucket) => {
192
192
  if (count > 0) {
193
+ // histogram data is ordered fastest to slowest, but this.histIntervals is
194
+ // slowest to fastest (so it displays correctly on the chart).
195
+ const index = this.options.histIntervals.length - 1 - histBucket;
196
+
193
197
  data.push({
194
198
  x: bucket,
195
- // histogram data is ordered fastest to slowest, but this.histIntervals is
196
- // slowest to fastest (so it displays correctly on the chart).
197
- y:
198
- this.options.histIntervals[this.options.histIntervals.length - 1 - histBucket] /
199
- 1000,
199
+ y: this.options.histIntervals[index] / 1000,
200
200
  count: count,
201
201
  });
202
202
 
@@ -206,53 +206,55 @@ class HistBubbleChart extends BaseChart {
206
206
  });
207
207
 
208
208
  // Chart.js will not calculate the bubble size. We have to do that.
209
- const maxRadius = this.ctx.offsetWidth / this.options.labels.length;
210
- const minRadius = 1
209
+ const maxRadius = this.el.offsetWidth / this.options.labels.length;
210
+ const minRadius = 1;
211
211
  const multiplier = (maxRadius / maxCount) * 1.5;
212
212
  data.forEach((entry) => {
213
213
  entry.r = entry.count * multiplier + minRadius;
214
214
  });
215
215
 
216
- return [{
217
- data: data,
218
- backgroundColor: "#537bc4",
219
- borderColor: "#537bc4",
220
- }];
216
+ return [
217
+ {
218
+ data: data,
219
+ backgroundColor: this.colors.primary,
220
+ borderColor: this.colors.primary,
221
+ },
222
+ ];
221
223
  }
222
224
 
223
225
  get chartOptions() {
224
226
  return {
227
+ ...super.chartOptions,
225
228
  aspectRatio: 3,
226
229
  scales: {
230
+ ...super.chartOptions.scales,
227
231
  x: {
232
+ ...super.chartOptions.scales.x,
228
233
  type: "category",
229
234
  labels: this.options.labels,
230
235
  },
231
236
  y: {
237
+ ...super.chartOptions.scales.y,
232
238
  title: {
233
- text: "Execution time (sec)",
239
+ text: this.options.yLabel,
234
240
  display: true,
235
241
  },
236
242
  },
237
243
  },
238
- interaction: {
239
- mode: "x",
240
- },
241
244
  plugins: {
242
- legend: {
243
- display: false,
244
- },
245
+ ...super.chartOptions.plugins,
245
246
  tooltip: {
247
+ ...super.chartOptions.plugins.tooltip,
246
248
  callbacks: {
247
249
  title: (items) => `${items[0].raw.x} UTC`,
248
250
  label: (item) =>
249
- `${item.parsed.y} seconds: ${item.raw.count} job${
250
- item.raw.count == 1 ? "" : "s"
251
- }`,
251
+ `${item.parsed.y} ${this.options.yUnits}: ${item.raw.count} ${this.options.zUnits}`,
252
252
  footer: (items) => {
253
253
  const bucket = items[0].raw.x;
254
254
  const marks = this.options.marks.filter(([b, _]) => b == bucket);
255
- return marks.map(([b, msg]) => `Deploy: ${msg}`);
255
+ return marks.map(
256
+ ([b, msg]) => `${this.options.markLabel}: ${msg}`
257
+ );
256
258
  },
257
259
  },
258
260
  },
@@ -72,6 +72,10 @@ table {
72
72
  background-color: #31708f;
73
73
  }
74
74
 
75
+ .alert-warning {
76
+ background-color: #c47612;
77
+ }
78
+
75
79
  a:link, a:active, a:hover, a:visited {
76
80
  color: #ddd;
77
81
  }
@@ -102,22 +102,8 @@ div.interval-slider {
102
102
  float: left;
103
103
  }
104
104
 
105
- #realtime-legend,
106
- #history-legend {
107
- text-align: right;
108
- float: left;
109
- }
110
- #realtime-legend .timestamp,
111
- #history-legend .timestamp {
112
- text-align: left;
113
- }
114
- #realtime-legend .line,
115
- #history-legend .line {
116
- margin: 0 20px 0 0;
117
- }
118
- #realtime-legend .swatch,
119
- #history-legend .swatch {
120
- margin: 0 0 0 8px;
105
+ #realtime-legend {
106
+ justify-content: start;
121
107
  }
122
108
 
123
109
  /* Beacon
@@ -165,78 +151,3 @@ div.interval-slider {
165
151
  padding-left: 5px;
166
152
  }
167
153
  }
168
-
169
- /* Rickshaw */
170
-
171
- .rickshaw_graph .detail .x_label.left {
172
- right: 0
173
- }
174
- .rickshaw_graph .detail .x_label.right {
175
- left: 0
176
- }
177
- .rickshaw_graph .detail .item.left {
178
- right: 0
179
- }
180
- .rickshaw_graph .detail .item.right {
181
- left: 0
182
- }
183
- .rickshaw_graph .detail .item.left:after {
184
- left: 0;
185
- right: -5px;
186
- border-right-color: unset;
187
- border-left-color: rgba(0, 0, 0, .8);
188
- border-right-width: 0;
189
- border-left-width: unset;
190
- }
191
- .rickshaw_graph .detail .item.right:after {
192
- right: 0;
193
- left: -5px;
194
- border-left-color: unset;
195
- border-right-color: rgba(0, 0, 0, .8);
196
- border-left-width: 0;
197
- border-right-width: unset;
198
- }
199
- .rickshaw_graph .detail .dot {
200
- margin-right: -3px;
201
- margin-left: unset;
202
- }
203
- .rickshaw_graph .x_tick {
204
- border-left: unset;
205
- border-right: 1px dotted rgba(0, 0, 0, .2);
206
- }
207
- .rickshaw_graph .x_tick .title {
208
- margin-right: 3px;
209
- margin-left: unset;
210
- }
211
- .rickshaw_annotation_timeline .annotation {
212
- margin-right: -2px;
213
- margin-left: unset;
214
- }
215
- .rickshaw_graph .annotation_line {
216
- border-right: 2px solid rgba(0, 0, 0, .3);
217
- border-left: unset;
218
- }
219
- .rickshaw_annotation_timeline .annotation .content {
220
- left: unset;
221
- right: -11px;
222
- }
223
- .rickshaw_graph .x_tick.glow .title,
224
- .rickshaw_graph .y_ticks.glow text {
225
- text-shadow: 1px 1px 0 rgba(255, 255, 255, .1), -1px -1px 0 rgba(255, 255, 255, .1), -1px 1px 0 rgba(255, 255, 255, .1), 0 1px 0 rgba(255, 255, 255, .1), 0 -1px 0 rgba(255, 255, 255, .1), -1px 0 0 rgba(255, 255, 255, .1), 1px 0 0 rgba(255, 255, 255, .1), 1px -1px 0 rgba(255, 255, 255, .1)
226
- }
227
- .rickshaw_graph .x_tick.inverse .title,
228
- .rickshaw_graph .y_ticks.inverse text {
229
- text-shadow: 1px 1px 0 rgba(0, 0, 0, .8), -1px -1px 0 rgba(0, 0, 0, .8), -1px 1px 0 rgba(0, 0, 0, .8), 0 1px 0 rgba(0, 0, 0, .8), 0 -1px 0 rgba(0, 0, 0, .8), -1px 0 0 rgba(0, 0, 0, .8), 1px 0 0 rgba(0, 0, 0, .8), 1px -1px 0 rgba(0, 0, 0, .8)
230
- }
231
- .rickshaw_legend .line {
232
- padding-left: 15px;
233
- padding-right: unset;
234
- }
235
- .rickshaw_legend .line .swatch {
236
- margin-left: 3px;
237
- margin-right: unset;
238
- }
239
- .rickshaw_legend .action {
240
- margin-left: .2em;
241
- margin-right: unset;
242
- }