wurk 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/wurk/metrics/history.rb +24 -38
- data/lib/wurk/profiler.rb +7 -5
- data/lib/wurk/version.rb +1 -1
- data/vendor/assets/dashboard/wurk-manifest.json +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4ed68ef2a8264d31364b725a97e468e55f2dcd823b00ed3a1104cc5ea39dddba
|
|
4
|
+
data.tar.gz: '03281f4525a7a9b8892f04acafa464a8b8eb28d51bd82c6cc6a3a0241e8b7dfe'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dbc0fcc7b8aa362b63792352d9e0099df9d45a5d6b03f5ef53997ba55d70b973ccbcd59aa8bbef4404a5a7c09dc6890dbac935cb413a5c3e463a3e7799e4c986
|
|
7
|
+
data.tar.gz: 2a60efb7fa8697421f01d0d426ca348da2d191be707f8806444fbaaad1494643443432dca62e255b3645421f9e646a3dfa7d5c64859b39143ee996f7474ec3c0
|
data/lib/wurk/metrics/history.rb
CHANGED
|
@@ -6,42 +6,42 @@ module Wurk
|
|
|
6
6
|
module Metrics
|
|
7
7
|
# Ent feature parity (§5): server middleware that records per-job-class
|
|
8
8
|
# execution metrics into Redis time-buckets. The on-the-wire schema is
|
|
9
|
-
# wire-compat with Sidekiq
|
|
10
|
-
# `j
|
|
9
|
+
# wire-compat with Sidekiq's history pane — Sidekiq keys the per-minute
|
|
10
|
+
# HASH as `j|<YYYYMMDD>|<H>:<M>`, so dashboards (and Sidekiq data migrated
|
|
11
|
+
# in place) keep resolving against the same key after a drop-in swap.
|
|
11
12
|
#
|
|
12
13
|
# Bucket layout (spec: docs/target/sidekiq-free.md §1.6):
|
|
13
14
|
#
|
|
14
|
-
# j|
|
|
15
|
-
# <klass>|p
|
|
16
|
-
# <klass>|f
|
|
17
|
-
# <klass>|ms
|
|
15
|
+
# j|YYYYMMDD|H:M HASH per-minute bucket, TTL = MID_TERM (3 days)
|
|
16
|
+
# <klass>|p INT processed count
|
|
17
|
+
# <klass>|f INT failed count
|
|
18
|
+
# <klass>|ms INT total ms spent
|
|
18
19
|
#
|
|
19
|
-
#
|
|
20
|
-
# TTL = SHORT_TERM (8 hours) — short window for
|
|
21
|
-
# quick aggregate queries without scanning 600 minute keys.
|
|
20
|
+
# <klass>-YYYYMMDD-H HASH per-class hourly histogram, TTL = MID_TERM
|
|
22
21
|
#
|
|
23
|
-
#
|
|
22
|
+
# We deliberately do NOT write a `H:m0` 10-minute rollup. Its key format
|
|
23
|
+
# collides with the real minute-0 bucket, so rolling x1..x9 into it turns
|
|
24
|
+
# that minute's value into a decade total — and the read side (Query) then
|
|
25
|
+
# sums the minute-0 bucket alongside x1..x9 and double-counts. Sidekiq
|
|
26
|
+
# itself doesn't keep that rollup (the daily/hourly rollups are commented
|
|
27
|
+
# out in its ExecutionTracker); Query reads the last N per-minute keys.
|
|
24
28
|
#
|
|
25
|
-
# Every bucket TTL is set on
|
|
26
|
-
#
|
|
27
|
-
# write would keep the bucket alive indefinitely while traffic continues,
|
|
28
|
-
# but that's the desired behavior here: as long as a class keeps running,
|
|
29
|
-
# we keep the minute bucket around for the retention window measured from
|
|
29
|
+
# Every bucket TTL is set on every write (not NX): as long as a class keeps
|
|
30
|
+
# running we keep its bucket around for the retention window measured from
|
|
30
31
|
# *last write*, not from first write. So we EXPIRE unconditionally.
|
|
31
32
|
#
|
|
32
|
-
# The middleware is hot-path — every successful job pays for it. Writes
|
|
33
|
-
#
|
|
34
|
-
#
|
|
33
|
+
# The middleware is hot-path — every successful job pays for it. Writes are
|
|
34
|
+
# pipelined in a single round-trip per job (1 HINCRBY × 2 + 1 EXPIRE per
|
|
35
|
+
# bucket × 2 buckets = 6 commands, batched).
|
|
35
36
|
class History
|
|
36
37
|
include Wurk::Middleware::ServerMiddleware
|
|
37
38
|
|
|
38
39
|
# Per spec §1.6 — naming mirrors the upstream constants so anyone
|
|
39
40
|
# grepping the Sidekiq source for `MID_TERM` lands here.
|
|
40
41
|
MID_TERM = 3 * 24 * 60 * 60 # 3 days, in seconds
|
|
41
|
-
SHORT_TERM = 8 * 60 * 60 # 8 hours, in seconds
|
|
42
42
|
|
|
43
43
|
MINUTE_KEY_PREFIX = 'j|'
|
|
44
|
-
DATE_FORMAT = '%
|
|
44
|
+
DATE_FORMAT = '%Y%m%d' # YYYYMMDD — matches Sidekiq's j| key
|
|
45
45
|
|
|
46
46
|
def call(_worker, job, _queue)
|
|
47
47
|
klass = job['class']
|
|
@@ -74,28 +74,20 @@ module Wurk
|
|
|
74
74
|
|
|
75
75
|
ms = duration_ms.to_i
|
|
76
76
|
ms = 0 if ms.negative?
|
|
77
|
-
buckets = { minute: minute_key(at),
|
|
77
|
+
buckets = { minute: minute_key(at), hour: hour_key(klass, at) }
|
|
78
78
|
with_pool(redis_pool) { |conn| pipeline_write(conn, klass, ms, success, buckets) }
|
|
79
79
|
nil
|
|
80
80
|
end
|
|
81
81
|
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
# `p|f|ms`. Pipeline all
|
|
82
|
+
# The minute bucket carries per-class `<klass>|p|f|ms` fields; the
|
|
83
|
+
# hourly bucket is already class-scoped so its fields are bare
|
|
84
|
+
# `p|f|ms`. Pipeline all 6 commands in one round-trip.
|
|
85
85
|
def pipeline_write(conn, klass, ms, success, buckets)
|
|
86
86
|
outcome = success ? 'p' : 'f'
|
|
87
87
|
class_outcome = "#{klass}|#{outcome}"
|
|
88
88
|
class_ms = "#{klass}|ms"
|
|
89
89
|
conn.pipelined do |pipe|
|
|
90
90
|
incr_bucket(pipe, buckets[:minute], [class_outcome, class_ms, ms, MID_TERM])
|
|
91
|
-
# The minute key and the 10-min rollup key coincide whenever the
|
|
92
|
-
# minute ends in 0 (rollup zeroes the last digit). Writing both
|
|
93
|
-
# would double-count the shared field — the minute write above
|
|
94
|
-
# already lands on it — so skip the rollup write then. Minutes
|
|
95
|
-
# x1..x9 still accumulate into the x0 rollup key as normal.
|
|
96
|
-
unless buckets[:rollup] == buckets[:minute]
|
|
97
|
-
incr_bucket(pipe, buckets[:rollup], [class_outcome, class_ms, ms, SHORT_TERM])
|
|
98
|
-
end
|
|
99
91
|
incr_bucket(pipe, buckets[:hour], [outcome, 'ms', ms, MID_TERM])
|
|
100
92
|
end
|
|
101
93
|
end
|
|
@@ -115,12 +107,6 @@ module Wurk
|
|
|
115
107
|
date: t.strftime(DATE_FORMAT), hr: t.hour, min: t.min)
|
|
116
108
|
end
|
|
117
109
|
|
|
118
|
-
def rollup_key(time)
|
|
119
|
-
t = time.utc
|
|
120
|
-
format("#{MINUTE_KEY_PREFIX}%<date>s|%<hr>d:%<min>d",
|
|
121
|
-
date: t.strftime(DATE_FORMAT), hr: t.hour, min: (t.min / 10) * 10)
|
|
122
|
-
end
|
|
123
|
-
|
|
124
110
|
def hour_key(klass, time)
|
|
125
111
|
t = time.utc
|
|
126
112
|
"#{klass}-#{t.strftime(DATE_FORMAT)}-#{t.hour}"
|
data/lib/wurk/profiler.rb
CHANGED
|
@@ -50,11 +50,13 @@ module Wurk
|
|
|
50
50
|
key = profile_key(token, jid)
|
|
51
51
|
gz = gzip(gecko_json)
|
|
52
52
|
with_pool(pool) do |conn|
|
|
53
|
-
conn.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
53
|
+
conn.pipelined do |pipe|
|
|
54
|
+
pipe.call('HSET', key, 'jid', jid, 'type', type, 'token', token,
|
|
55
|
+
'started_at', started_at.to_i, 'elapsed', elapsed_ms.to_i,
|
|
56
|
+
'size', gz.bytesize, 'sid', sid.to_s, 'data', gz)
|
|
57
|
+
pipe.call('EXPIRE', key, TTL)
|
|
58
|
+
pipe.call('ZADD', Keys::PROFILES, (now + TTL).to_i, key)
|
|
59
|
+
end
|
|
58
60
|
end
|
|
59
61
|
key
|
|
60
62
|
end
|
data/lib/wurk/version.rb
CHANGED