robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Pausing one job class across every queue (Robin-only).
|
|
5
|
+
#
|
|
6
|
+
# Robin::JobClass.new(ReportJob).pause!
|
|
7
|
+
# Robin::JobClass.new("ReportJob").unpause!
|
|
8
|
+
#
|
|
9
|
+
# A paused class's jobs wait where they are (queued, scheduled, retrying)
|
|
10
|
+
# until a worker fetches one: then, instead of running, it's parked. On
|
|
11
|
+
# unpausing, the parked jobs go back on the front of their queues, the
|
|
12
|
+
# oldest first in line. Jobs already running finish.
|
|
13
|
+
#
|
|
14
|
+
# Servers refresh their copy of the paused classes on every heartbeat, so
|
|
15
|
+
# a pause takes effect within one, and checking a fetched job costs no
|
|
16
|
+
# round trip. A fetched job is parked only if its class is still paused,
|
|
17
|
+
# checked atomically as it's moved, so an unpause never strands one that
|
|
18
|
+
# a server still thought paused.
|
|
19
|
+
#
|
|
20
|
+
# An Active Job job's class is the one it wraps.
|
|
21
|
+
class JobClass
|
|
22
|
+
# Parks a fetched job if its class is still paused: moves it from its
|
|
23
|
+
# in-flight key onto the class's parked list, the newest on the left.
|
|
24
|
+
#
|
|
25
|
+
# KEYS: paused classes set, in-flight key, parked list
|
|
26
|
+
# ARGV: class name
|
|
27
|
+
PARK = Redis::Script.new(<<~LUA)
|
|
28
|
+
if redis.call('SISMEMBER', KEYS[1], ARGV[1]) == 0 then return 0 end
|
|
29
|
+
if redis.call('LMOVE', KEYS[2], KEYS[3], 'RIGHT', 'LEFT') then return 1 end
|
|
30
|
+
return 0
|
|
31
|
+
LUA
|
|
32
|
+
|
|
33
|
+
# Moves up to ARGV[2] of a class's parked jobs back onto the front of
|
|
34
|
+
# their queues: the newest first, so the oldest end up first in line.
|
|
35
|
+
# Returns how many it moved and how many are still parked.
|
|
36
|
+
#
|
|
37
|
+
# KEYS: parked list, queues set
|
|
38
|
+
# ARGV: queue key prefix, how many
|
|
39
|
+
UNPARK = Redis::Script.new(<<~LUA)
|
|
40
|
+
local moved = 0
|
|
41
|
+
while moved < tonumber(ARGV[2]) do
|
|
42
|
+
local raw = redis.call('LPOP', KEYS[1])
|
|
43
|
+
if not raw then break end
|
|
44
|
+
local queue = cjson.decode(raw)['queue'] or 'default'
|
|
45
|
+
redis.call('SADD', KEYS[2], queue)
|
|
46
|
+
redis.call('RPUSH', ARGV[1] .. queue, raw)
|
|
47
|
+
moved = moved + 1
|
|
48
|
+
end
|
|
49
|
+
return {moved, redis.call('LLEN', KEYS[1])}
|
|
50
|
+
LUA
|
|
51
|
+
|
|
52
|
+
# The paused classes and how many of each one's jobs are parked: name,
|
|
53
|
+
# count, name, count...
|
|
54
|
+
#
|
|
55
|
+
# KEYS: paused classes set
|
|
56
|
+
# ARGV: parked list key prefix
|
|
57
|
+
LIST = Redis::Script.new(<<~LUA)
|
|
58
|
+
local out = {}
|
|
59
|
+
for _, name in ipairs(redis.call('SMEMBERS', KEYS[1])) do
|
|
60
|
+
table.insert(out, name)
|
|
61
|
+
table.insert(out, redis.call('LLEN', ARGV[1] .. name))
|
|
62
|
+
end
|
|
63
|
+
return out
|
|
64
|
+
LUA
|
|
65
|
+
|
|
66
|
+
@cache = {}
|
|
67
|
+
|
|
68
|
+
class << self
|
|
69
|
+
# The paused classes' names, sorted.
|
|
70
|
+
def paused(config: Robin.config) = config.redis { |c| c.call("SMEMBERS", config.key("classes", "paused")) }.sort
|
|
71
|
+
|
|
72
|
+
# Each paused class, by name: its name and how many of its jobs are
|
|
73
|
+
# parked. One round trip.
|
|
74
|
+
#
|
|
75
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
76
|
+
# prefix bare or wrapped in an Array is identical.
|
|
77
|
+
def list(config: Robin.config)
|
|
78
|
+
flat = config.redis { |c| c.evalsha(LIST, [config.key("classes", "paused")], ["#{config.key("classes", "parked")}:"]) }
|
|
79
|
+
flat.each_slice(2).map { |name, parked| {"name" => name, "parked" => parked} }.sort_by { |row| row.fetch("name") }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# The command that reads the paused classes, for a heartbeat to send
|
|
83
|
+
# along with others; pass its reply to #store.
|
|
84
|
+
def listing(config) = ["SMEMBERS", config.key("classes", "paused")]
|
|
85
|
+
|
|
86
|
+
# Replaces the local copy of the paused classes.
|
|
87
|
+
def store(config, names)
|
|
88
|
+
@cache[config.object_id] = Set.new(names)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Of +payload+'s classes (the one an Active Job job wraps, and the
|
|
92
|
+
# job's), the one this process last saw paused, or nil. Allocates
|
|
93
|
+
# nothing: every job is checked.
|
|
94
|
+
def paused_class(config, payload)
|
|
95
|
+
set = @cache[config.object_id] or return
|
|
96
|
+
|
|
97
|
+
wrapped = payload["wrapped"]
|
|
98
|
+
return wrapped if set.include?(wrapped)
|
|
99
|
+
|
|
100
|
+
klass = payload["class"]
|
|
101
|
+
klass if set.include?(klass)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Parks +unit+, a fetched job of the paused class +name+, unless
|
|
105
|
+
# +name+ was unpaused meanwhile; whether it did.
|
|
106
|
+
#
|
|
107
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
108
|
+
# name bare or wrapped in an Array is identical.
|
|
109
|
+
def park(config, unit, name)
|
|
110
|
+
keys = [config.key("classes", "paused"), unit.key, config.key("classes", "parked", name)]
|
|
111
|
+
config.redis { |c| c.evalsha(PARK, keys, [name]) } == 1
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# This process's own pauses and unpauses show in its copy at once,
|
|
115
|
+
# before the next heartbeat.
|
|
116
|
+
def noted(config, name, paused)
|
|
117
|
+
set = @cache[config.object_id] or return
|
|
118
|
+
|
|
119
|
+
paused ? set.add(name) : set.delete(name)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
attr_reader :name
|
|
124
|
+
|
|
125
|
+
# +klass+: a job class, or its name.
|
|
126
|
+
def initialize(klass, config: Robin.config)
|
|
127
|
+
@name = klass.to_s
|
|
128
|
+
@config = config
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def paused? = @config.redis { |c| c.call("SISMEMBER", @config.key("classes", "paused"), name) } == 1
|
|
132
|
+
|
|
133
|
+
# How many of its jobs are parked.
|
|
134
|
+
def parked = @config.redis { |c| c.call("LLEN", @config.key("classes", "parked", name)) }
|
|
135
|
+
|
|
136
|
+
def pause!
|
|
137
|
+
@config.redis { |c| c.call("SADD", @config.key("classes", "paused"), name) }
|
|
138
|
+
JobClass.noted(@config, name, true)
|
|
139
|
+
true
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Unpauses the class and puts its parked jobs back on their queues,
|
|
143
|
+
# +chunk+ of them a round trip; returns how many it put back. Once it's
|
|
144
|
+
# unpaused no more are parked (see PARK), so none are left behind.
|
|
145
|
+
def unpause!(chunk: 1000)
|
|
146
|
+
@config.redis { |c| c.call("SREM", @config.key("classes", "paused"), name) }
|
|
147
|
+
JobClass.noted(@config, name, false)
|
|
148
|
+
keys = [@config.key("classes", "parked", name), @config.key("queues")]
|
|
149
|
+
total = 0
|
|
150
|
+
loop do
|
|
151
|
+
moved, left = @config.redis { |c| c.evalsha(UNPARK, keys, ["#{@config.key("queue")}:", chunk]) }
|
|
152
|
+
total += moved
|
|
153
|
+
return total if left.zero?
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Per-thread logging context (class, jid, bid...), included in every log
|
|
7
|
+
# line written while a job runs.
|
|
8
|
+
module Context
|
|
9
|
+
KEY = :robin_context
|
|
10
|
+
|
|
11
|
+
def self.current
|
|
12
|
+
Thread.current[KEY] ||= {}
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.with(hash)
|
|
16
|
+
previous = current
|
|
17
|
+
Thread.current[KEY] = previous.merge(hash)
|
|
18
|
+
yield
|
|
19
|
+
ensure
|
|
20
|
+
Thread.current[KEY] = previous
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class JobLogger
|
|
25
|
+
class Formatter < ::Logger::Formatter
|
|
26
|
+
def call(severity, time, _progname, message)
|
|
27
|
+
extra = Context.current.map { |k, v| " #{k}=#{v}" }.join
|
|
28
|
+
"#{time.utc.iso8601(3)} pid=#{Process.pid} tid=#{JobLogger.tid}#{extra} #{severity}: #{message.is_a?(String) ? message : message.inspect}\n"
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# One-line JSON per log entry, for log aggregators. Strings that
|
|
33
|
+
# aren't valid UTF-8 (binary error messages, say) are scrubbed rather
|
|
34
|
+
# than making the log call raise.
|
|
35
|
+
class JSONFormatter < ::Logger::Formatter
|
|
36
|
+
def call(severity, time, _progname, message)
|
|
37
|
+
entry = {ts: time.utc.iso8601(3), pid: Process.pid, tid: JobLogger.tid, lvl: severity,
|
|
38
|
+
msg: Robin.utf8(message.is_a?(String) ? message : message.inspect)}
|
|
39
|
+
ctx = Context.current
|
|
40
|
+
entry[:ctx] = clean(ctx) unless ctx.empty?
|
|
41
|
+
Robin.dump_json(entry) << "\n"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def clean(value)
|
|
47
|
+
case value
|
|
48
|
+
when String then Robin.utf8(value)
|
|
49
|
+
when Array then value.map { |v| clean(v) }
|
|
50
|
+
when Hash then value.transform_values { |v| clean(v) }
|
|
51
|
+
else value
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The logging context of a job: its class, jid, and bid and tags when
|
|
57
|
+
# it has them.
|
|
58
|
+
def self.context_for(payload)
|
|
59
|
+
ctx = {class: Robin.display_class(payload), jid: payload["jid"]}
|
|
60
|
+
ctx[:bid] = payload.fetch("bid") if payload["bid"]
|
|
61
|
+
ctx[:tags] = payload.fetch("tags").join(",") if payload["tags"].is_a?(Array) && !payload.fetch("tags").empty?
|
|
62
|
+
ctx
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.tid
|
|
66
|
+
Thread.current[:robin_tid] ||= (Thread.current.object_id ^ Process.pid).to_s(36)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def initialize(config)
|
|
70
|
+
@config = config
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
LEVELS = %w[debug info warn error].freeze
|
|
74
|
+
ENABLED = LEVELS.to_h { |level| [level, :"#{level}?"] }.freeze
|
|
75
|
+
|
|
76
|
+
# Logs "start", then "done" or "fail", around a job, with its context
|
|
77
|
+
# on every line logged meanwhile. Jobs choose the level of those lines
|
|
78
|
+
# with +log_level+ (e.g. robin_options log_level: :debug for chatty
|
|
79
|
+
# jobs), or turn them off with +logging: false+.
|
|
80
|
+
def call(payload)
|
|
81
|
+
ctx = self.class.context_for(payload)
|
|
82
|
+
level = level_for(payload)
|
|
83
|
+
return Context.with(ctx) { yield } unless level
|
|
84
|
+
|
|
85
|
+
Context.with(ctx) do
|
|
86
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
87
|
+
log(level, "start")
|
|
88
|
+
begin
|
|
89
|
+
result = yield
|
|
90
|
+
Context.current[:elapsed] = elapsed(start)
|
|
91
|
+
log(level, "done")
|
|
92
|
+
result
|
|
93
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
94
|
+
Context.current[:elapsed] = elapsed(start)
|
|
95
|
+
log(level, "fail")
|
|
96
|
+
raise
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
private
|
|
102
|
+
|
|
103
|
+
# The level of the job's lines; nil for none at all, including when
|
|
104
|
+
# the logger would drop them (so their timing isn't even formatted).
|
|
105
|
+
def level_for(payload)
|
|
106
|
+
return if payload["logging"] == false
|
|
107
|
+
|
|
108
|
+
level = payload["log_level"].to_s
|
|
109
|
+
level = "info" unless LEVELS.include?(level)
|
|
110
|
+
level if @config.logger.public_send(ENABLED.fetch(level))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def log(level, message)
|
|
114
|
+
@config.logger.public_send(level, message)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def elapsed(start)
|
|
118
|
+
format("%.3f", Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|