robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
@@ -0,0 +1,60 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Things worth knowing about that aren't errors, as events: what Robin
5
+ # otherwise only logs. Handlers are callables taking the event's name and
6
+ # a Hash about it:
7
+ #
8
+ # Robin.configure do |config|
9
+ # config.event_handlers << ->(name, payload) { Metrics.increment("robin.#{name}") }
10
+ # end
11
+ #
12
+ # The Rails integration sends them on to ActiveSupport::Notifications as
13
+ # "<name>.robin". The events, and what their payloads hold:
14
+ #
15
+ # job.expired:: job (the payload): it expired before it ran
16
+ # job.skipped:: job, reason ("cancelled" or "superseded", by a later
17
+ # debounced push)
18
+ # job.discarded:: job, error: it raised Robin::Discard
19
+ # job.parked:: job, class (the paused one: see Robin::JobClass):
20
+ # set aside until its class is unpaused
21
+ # job.rate_limited:: job, limiter (its name), delay (seconds until it
22
+ # runs again)
23
+ # job.recovered:: job, poison (true when it went to the dead set
24
+ # instead of its queue): it was running when its
25
+ # process died
26
+ # jobs.orphaned:: count: in-flight jobs no worker held, put back
27
+ # jobs.interrupted:: count: jobs still running when a stopping process's
28
+ # timeout ran out
29
+ # iteration.slow:: job, seconds: an iterable job's iteration took
30
+ # longer than the shutdown timeout
31
+ # worker.started:: capsule (its name): a worker thread started; handlers
32
+ # run on that thread
33
+ # worker.died:: error, delay (seconds before its replacement starts;
34
+ # nil when stopping, with no replacement)
35
+ # workers.resized:: capsule, from, to: a capsule's worker count changed
36
+ # (Robin::Launcher#resize)
37
+ # redis.slow:: rtt_us: Redis round trips take over 50ms
38
+ # gvl.contended:: percent: GVL contention (see Robin::GVL) stayed at 50%
39
+ # or more for three heartbeats in a row
40
+ # heartbeat.lapsed:: (nothing): no heartbeat for longer than process_ttl
41
+ # memory.exceeded:: rss_mb, limit_mb: the process restarts gracefully
42
+ # swarm.child_exited:: index, pid, status (exit status or signal), delay
43
+ # push.buffered:: jobs, error: pushed while Redis was unreachable,
44
+ # held by reliable push
45
+ # push.dropped:: jobs: reliable push's buffer was full
46
+ # push.recovered:: count: buffered jobs pushed once Redis was back
47
+ # push.lost:: jobs: still buffered when the process exited
48
+ module Events
49
+ # Calls every handler; one that raises is reported and the others
50
+ # still run.
51
+ def self.emit(config, name, **payload)
52
+ config.event_handlers.each do |handler|
53
+ handler.call(name, payload)
54
+ rescue => e
55
+ config.handle_exception(e, {context: "event handler", event: name})
56
+ end
57
+ nil
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Expiring jobs: work that's worthless after a deadline.
5
+ #
6
+ # class RefreshDashboard
7
+ # include Robin::Job
8
+ # robin_options expires_in: 600 # seconds after enqueue
9
+ # end
10
+ #
11
+ # RefreshDashboard.set(expires_in: 30).perform_async(user_id)
12
+ #
13
+ # The client turns +expires_in+ into an absolute "expires_at" (epoch ms).
14
+ # Jobs fetched after that moment are skipped and count as done.
15
+ module Expiry
16
+ def self.expired?(payload, now = Robin.now_ms)
17
+ at = payload["expires_at"]
18
+ !at.nil? && at < now
19
+ end
20
+
21
+ class ServerMiddleware
22
+ include Middleware::ConfigAware
23
+ include Middleware::Stateless
24
+
25
+ def self.applies?(_job, payload, _queue) = payload.key?("expires_at")
26
+
27
+ def call(_job, payload, _queue)
28
+ return yield unless Expiry.expired?(payload)
29
+
30
+ logger.info { "skipping expired job #{payload.fetch("class")} jid=#{payload.fetch("jid")}" }
31
+ config.statsd&.increment("jobs.expired", tags: {class: Robin.display_class(payload)})
32
+ Events.emit(config, "job.expired", job: payload)
33
+ # It won't run, but it's finished as far as its batch (if any) is
34
+ # concerned.
35
+ bid = payload["bid"]
36
+ Batch.job_succeeded(config, bid, payload.fetch("jid")) if bid
37
+ nil
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,441 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Reliable fetch: jobs are atomically moved from their queue into an
5
+ # in-flight key of their process and only removed from it once they
6
+ # finish (successfully, or after being recorded for retry). If a process
7
+ # crashes or is killed, Robin::Recovery puts its in-flight jobs back.
8
+ #
9
+ # Each in-flight job has a key of its own, inflight:<identity>:<n>,
10
+ # registered in the process's in-flight set (inflight:<identity>) before
11
+ # a job is first moved into it. So a job is acknowledged (or put back)
12
+ # by its key: its payload only ever travels from Redis to the process,
13
+ # once. The key of an acknowledged job is used again once its
14
+ # acknowledgement has gone through, so keys are registered once, not
15
+ # for every job.
16
+ class Fetcher
17
+ # A fetched job.
18
+ class UnitOfWork
19
+ # +key+ is the job's in-flight key.
20
+ attr_reader :queue, :raw, :fetcher, :key
21
+
22
+ def initialize(queue, raw, fetcher, key)
23
+ @queue = queue
24
+ @raw = raw
25
+ @fetcher = fetcher
26
+ @key = key
27
+ end
28
+
29
+ # Removes the job from its in-flight key, right away or as part of
30
+ # +conn+ (a pipeline or transaction). The key is retired with it.
31
+ def acknowledge(conn = nil)
32
+ return fetcher.config.redis { |c| c.pipelined { |p| acknowledge(p) } } unless conn
33
+
34
+ fetcher.retire(conn, key)
35
+ end
36
+
37
+ # Puts the job back on the front of its queue.
38
+ def requeue = fetcher.config.redis { |c| fetcher.put_back(c, [self]) }
39
+ end
40
+
41
+ # Seconds a job fetched for another worker may wait for one before
42
+ # it goes back on its queue (checked every heartbeat), so jobs fetched
43
+ # just before every worker got busy with long jobs can run elsewhere.
44
+ STALE = 1
45
+
46
+ # Seconds a finished job's acknowledgement waits for a round trip to
47
+ # take it before a thread of the launcher sends it (see #acker). Under
48
+ # load, a round trip comes sooner; but a worker that takes a job
49
+ # fetched for it makes none, and long jobs can keep every worker busy.
50
+ # A job finished but not yet acknowledged when its process crashes
51
+ # runs again, so this bounds how long before a crash such a job may
52
+ # have finished.
53
+ ACK_WAIT = 0.002
54
+
55
+ # Jobs larger than this (in bytes, judged by the first job of the last
56
+ # round trip) make a batch from a single queue cheaper to take with
57
+ # plain commands than with the script (see #take_from).
58
+ LARGE = 2048
59
+
60
+ @minted = 0
61
+ @minting = Mutex.new
62
+
63
+ # A number for a new in-flight key. They're unique within the process,
64
+ # whichever capsule's fetcher uses them, as they share its in-flight
65
+ # set.
66
+ def self.mint = @minting.synchronize { @minted += 1 }
67
+
68
+ attr_reader :config, :capsule, :inflight_key
69
+ attr_accessor :paused
70
+
71
+ # A queue the fetcher puts itself on when acknowledgements are left
72
+ # waiting, once until #send_acks takes them: the launcher's thread pops
73
+ # it, waits ACK_WAIT and calls #send_acks.
74
+ attr_accessor :acker
75
+
76
+ def initialize(capsule, identity)
77
+ @capsule = capsule
78
+ @config = capsule.config
79
+ @inflight_key = config.key("inflight", identity)
80
+ @paused_key = config.key("paused")
81
+ @paused = []
82
+ @lock = Mutex.new
83
+ # [UnitOfWork, fetched at] for jobs fetched for other workers, oldest
84
+ # first.
85
+ @buffer = []
86
+ # In-flight keys of finished jobs waiting to be acknowledged with the
87
+ # next round trip, and (by thread) those being acknowledged in one
88
+ # now.
89
+ @acks = []
90
+ @sending = {}
91
+ @acker = Thread::Queue.new
92
+ # In-flight keys known to be empty, and (by thread) those a round
93
+ # trip is filling now.
94
+ @free = []
95
+ @filling = {}
96
+ # Keys the sweep took over (see #forget), as hash keys.
97
+ @lost = {}
98
+ # Were the jobs of the last round trip large (see LARGE)?
99
+ @large = false
100
+ # Workers inside #fetch without a job yet.
101
+ @wanting = 0
102
+ # Workers (threads, as hash keys) that last waited on their capsule's
103
+ # only queue and got nothing (see #fetch).
104
+ @idle = {}
105
+ end
106
+
107
+ # Returns the next UnitOfWork, or nil after waiting briefly with no
108
+ # work available. +ack+ is the in-flight key of the worker's last
109
+ # finished job: it's acknowledged with the next round trip, or by
110
+ # #send_acks (once it's passed in, it's the fetcher's to acknowledge).
111
+ #
112
+ # A round trip takes a job for every worker of the capsule fetching at
113
+ # the same time, up to config.fetch_batch; the extras go to the next
114
+ # workers to ask, without a round trip. They stay in flight meanwhile,
115
+ # so a crash loses nothing. With nothing to take, the worker blocks on
116
+ # a queue: +slot+ spreads idle workers across queues so every queue has
117
+ # one blocked on it, giving instant pickup.
118
+ def fetch(slot = 0, ack: nil)
119
+ reserved = @lock.synchronize do
120
+ @acks << ack if ack
121
+ unit, = @buffer.shift
122
+ if unit
123
+ wake_acker unless @acks.empty?
124
+ return unit
125
+ end
126
+ # A worker that just waited on its capsule's only queue for
127
+ # nothing waits again: a round trip would find nothing there
128
+ # either that blocking on it doesn't. (It finished no job, so it
129
+ # has nothing to acknowledge.)
130
+ unless @idle.delete(Thread.current)
131
+ @wanting += 1
132
+ [take_acks, reserve([@wanting, config.fetch_batch].min)]
133
+ end
134
+ end #: [Array[String], [Array[String], Array[String]]]?
135
+ return wait(slot, capsule.queue_order) unless reserved
136
+
137
+ acks, (keys, minted) = reserved
138
+
139
+ begin
140
+ order = capsule.queue_order
141
+ units, unused = take(order, acks, keys, minted)
142
+ ensure
143
+ @lock.synchronize do
144
+ @wanting -= 1
145
+ @sending.delete(Thread.current)
146
+ settle(unused)
147
+ if units
148
+ free(acks)
149
+ keep(units.drop(1))
150
+ else
151
+ @acks.concat(acks) # for the next round trip
152
+ wake_acker unless @acks.empty?
153
+ end
154
+ end
155
+ end
156
+ units.first || wait(slot, order)
157
+ end
158
+
159
+ # Called every heartbeat with the paused queues (the heartbeat reads
160
+ # them once for every capsule): takes the jobs fetched for other
161
+ # workers that are from a queue paused since, or that no worker took
162
+ # within STALE seconds. The heartbeat hands back every capsule's in one
163
+ # transaction (see #hand_back), then frees their keys (#freed).
164
+ def stale(paused)
165
+ @paused = paused
166
+ cutoff = monotonic - STALE
167
+ @lock.synchronize do
168
+ back, @buffer = @buffer.partition { |unit, at| at < cutoff || @paused.include?(unit.queue) }
169
+ back.map(&:first)
170
+ end
171
+ end
172
+
173
+ # Frees the keys of jobs handed back.
174
+ def freed(units) = free_keys(units.map(&:key))
175
+
176
+ # Acknowledges the finished jobs waiting for a round trip, if any, in
177
+ # one of its own (see ACK_WAIT). If that fails, they wait again, for a
178
+ # round trip or for a worker to leave more waiting.
179
+ def send_acks
180
+ acks = @lock.synchronize do
181
+ @ack_queued = false
182
+ take_acks unless @acks.empty?
183
+ end
184
+ return unless acks
185
+
186
+ begin
187
+ config.redis { |conn| conn.call("DEL", *acks) }
188
+ sent = true
189
+ ensure
190
+ @lock.synchronize do
191
+ @sending.delete(Thread.current)
192
+ sent ? free(acks) : @acks.concat(acks)
193
+ end
194
+ end
195
+ end
196
+
197
+ # In-flight keys this fetcher is responsible for: free ones, ones
198
+ # being filled, those of jobs fetched but not handed out yet, and those
199
+ # of finished jobs not acknowledged yet.
200
+ def held = @lock.synchronize { holding }
201
+
202
+ # Lets go of +key+, found sitting unheld with the same job (or none) in
203
+ # it on two sweeps, so its job can be put back: it won't be used again
204
+ # (were it acknowledged after all). False if it's held now.
205
+ def forget(key)
206
+ @lock.synchronize do
207
+ held = holding.include?(key)
208
+ @lost[key] = true unless held
209
+ !held
210
+ end
211
+ end
212
+
213
+ # Queues on +conn+ (a pipeline or transaction) the removal of the job
214
+ # in +key+: the key is deleted and unregistered, and never used again
215
+ # (so it can't be filled while its deletion may still be pending).
216
+ def retire(conn, key)
217
+ conn.call("DEL", key)
218
+ conn.call("SREM", @inflight_key, key)
219
+ end
220
+
221
+ # Puts jobs (units of work) back on the front of their queues, the
222
+ # first one first in line, and frees their keys; returns nil.
223
+ def put_back(conn, units)
224
+ send_back(conn, units)
225
+ freed(units)
226
+ nil
227
+ end
228
+
229
+ # Puts the jobs fetched but not handed out back on their queues, and
230
+ # acknowledges the finished ones, e.g. when the workers stop: in one
231
+ # transaction.
232
+ def release
233
+ units, acks = @lock.synchronize do
234
+ taken = [@buffer.map(&:first), @acks] #: [Array[UnitOfWork], Array[String]]
235
+ @buffer = []
236
+ @acks = []
237
+ taken
238
+ end
239
+ return if units.empty? && acks.empty?
240
+
241
+ config.redis do |conn|
242
+ conn.multi do |tx|
243
+ hand_back(tx, units)
244
+ tx.call("DEL", *acks) unless acks.empty?
245
+ end
246
+ end
247
+ free_keys(units.map(&:key) + acks)
248
+ end
249
+
250
+ # Queues on +tx+ (a transaction) the commands moving jobs (units of
251
+ # work) back from their in-flight keys onto the front of their queues,
252
+ # the first one first in line. The caller frees their keys once it's
253
+ # sent (see #freed).
254
+ def hand_back(tx, units)
255
+ units.reverse_each { |unit| tx.call("LMOVE", unit.key, config.key("queue", unit.queue), "RIGHT", "RIGHT") }
256
+ end
257
+
258
+ # Acknowledges the finished jobs in +keys+ in one round trip, then
259
+ # frees the keys.
260
+ def acknowledge(conn, keys)
261
+ return if keys.empty?
262
+
263
+ conn.call("DEL", *keys)
264
+ free_keys(keys)
265
+ end
266
+
267
+ private
268
+
269
+ # With the lock held: puts the fetcher on the acker queue, unless it's
270
+ # there already (@ack_queued, until #send_acks).
271
+ def wake_acker
272
+ return if @ack_queued
273
+
274
+ @ack_queued = true
275
+ @acker << self
276
+ end
277
+
278
+ # With the lock held: takes the waiting acknowledgements for this
279
+ # thread to send.
280
+ def take_acks
281
+ acks = @sending[Thread.current] = @acks
282
+ @acks = []
283
+ acks
284
+ end
285
+
286
+ # With the lock held: buffers +units+ for other workers.
287
+ def keep(units)
288
+ at = monotonic
289
+ units.each { |unit| @buffer << [unit, at] }
290
+ end
291
+
292
+ # With the lock held: +count+ in-flight keys for this thread's round
293
+ # trip to fill, free ones first, and the ones among them just minted
294
+ # (the round trip registers them before filling them).
295
+ def reserve(count)
296
+ keys = @free.pop(count)
297
+ minted = (keys.size...count).map { "#{@inflight_key}:#{Fetcher.mint}" }
298
+ @filling[Thread.current] = keys.concat(minted)
299
+ [keys, minted]
300
+ end
301
+
302
+ # With the lock held, once this thread's round trip is over: the keys
303
+ # it didn't fill (+unused+) are free (they're empty). After a failure
304
+ # (no +unused+) the round trip may have filled any of its keys without
305
+ # the reply getting here, so they're let go of instead: the sweep puts
306
+ # back whatever reached them (see Launcher#sweep_inflight).
307
+ def settle(unused)
308
+ @filling.delete(Thread.current)
309
+ free(unused) if unused
310
+ end
311
+
312
+ # With the lock held: +keys+ are empty, so they can be filled again,
313
+ # unless the sweep took them over.
314
+ def free(keys)
315
+ keys.each { |key| @free << key unless @lost.delete(key) }
316
+ end
317
+
318
+ # Frees +keys+, taking the lock.
319
+ def free_keys(keys) = @lock.synchronize { free(keys) }
320
+
321
+ # With the lock held: the keys this fetcher is responsible for right
322
+ # now (see #held).
323
+ def holding = @free + @filling.values.flatten + @buffer.map { |unit, _at| unit.key } + @acks + @sending.values.flatten
324
+
325
+ # Acknowledges +acks+ and fills +keys+ with up to as many jobs; returns
326
+ # the units of work and the keys left unused.
327
+ def take(order, acks, keys, minted)
328
+ units, unused = config.redis do |conn|
329
+ if order.size == 1 && !@paused.include?(order.first) && (keys.size == 1 || @large)
330
+ take_from(conn, order.first, acks, keys, minted)
331
+ else
332
+ script_keys = [@paused_key, *keys, *order.map { |q| config.key("queue", q) }, *acks]
333
+ argv = [keys.size, *order]
334
+ taken = if minted.empty?
335
+ conn.evalsha(Scripts::FETCH, script_keys, argv)
336
+ else
337
+ conn.pipelined do |p|
338
+ register(p, minted)
339
+ p.evalsha(Scripts::FETCH, script_keys, argv)
340
+ end.last
341
+ end
342
+ unused = keys.dup # filled in order
343
+ [taken.each_slice(2).map { |queue, raw| UnitOfWork.new(queue, raw, self, unused.shift) }, unused]
344
+ end
345
+ end
346
+ @large = units.first.raw.bytesize > LARGE unless units.empty?
347
+ [units, unused]
348
+ end
349
+
350
+ # Scripts::FETCH for a single queue, as plain commands in one round
351
+ # trip. For one job, Redis runs them faster than the script. For
352
+ # several, they cost a command each, but the script copies every job
353
+ # into Lua and back out, which costs more once jobs are large. Jobs
354
+ # taken from a queue paused since the last refresh go straight back.
355
+ def take_from(conn, queue, acks, keys, minted)
356
+ source = config.key("queue", queue)
357
+ replies = conn.pipelined do |p|
358
+ p.call("DEL", *acks) unless acks.empty?
359
+ register(p, minted)
360
+ p.call("SISMEMBER", @paused_key, queue)
361
+ keys.each { |key| p.call("LMOVE", source, key, "RIGHT", "LEFT") }
362
+ end
363
+ paused, *raws = replies.last(keys.size + 1)
364
+ received(conn, queue, paused, raws, keys)
365
+ end
366
+
367
+ # Queues the registration of newly minted in-flight keys, ahead of the
368
+ # commands that fill them.
369
+ def register(pipe, minted)
370
+ pipe.call("SADD", @inflight_key, *minted) unless minted.empty?
371
+ end
372
+
373
+ # The units for the jobs (+raws+, nil where there was nothing to take)
374
+ # a round trip moved from +queue+ into +keys+, and the keys left
375
+ # unused; all of them, the jobs handed straight back, if +paused+ (1)
376
+ # says the queue was paused meanwhile.
377
+ def received(conn, queue, paused, raws, keys)
378
+ units = [] #: Array[UnitOfWork]
379
+ unused = [] #: Array[String]
380
+ # (raws and keys are the same size, so key is never nil.)
381
+ raws.zip(keys) do |raw, key|
382
+ if raw
383
+ units << UnitOfWork.new(queue, raw, self, key) # steep:ignore ArgumentTypeMismatch
384
+ else
385
+ unused << key # steep:ignore ArgumentTypeMismatch
386
+ end
387
+ end
388
+ return [units, unused] unless paused == 1
389
+
390
+ send_back(conn, units)
391
+ [[], keys]
392
+ end
393
+
394
+ # Blocks for up to config.fetch_timeout for the next job, unless
395
+ # another worker fetched one for this one meanwhile.
396
+ def wait(slot, order)
397
+ active = order - @paused
398
+ reserved = @lock.synchronize do
399
+ unit, = @buffer.shift
400
+ return unit if unit
401
+
402
+ reserve(1) unless active.empty?
403
+ end #: [Array[String], Array[String]]?
404
+
405
+ wait = config.fetch_timeout
406
+ unless reserved
407
+ sleep wait
408
+ return
409
+ end
410
+
411
+ keys, minted = reserved
412
+ queue = active.fetch(slot % active.size)
413
+ begin
414
+ units, unused = config.redis do |conn|
415
+ # The queue may have been paused while we were blocked: Redis runs
416
+ # the SISMEMBER once the BLMOVE returns, so it answers for the
417
+ # moment the job came, without a round trip of its own.
418
+ raw, paused = conn.pipelined(timeout: wait) do |p|
419
+ register(p, minted)
420
+ p.call("BLMOVE", config.key("queue", queue), keys.first, "RIGHT", "LEFT", wait)
421
+ p.call("SISMEMBER", @paused_key, queue)
422
+ end.last(2)
423
+ received(conn, queue, paused, [raw], keys)
424
+ end
425
+ ensure
426
+ @lock.synchronize do
427
+ settle(unused)
428
+ @idle[Thread.current] = true if units&.empty? && order.size == 1
429
+ end
430
+ end
431
+ units.first
432
+ end
433
+
434
+ # Hands back +units+ (see #hand_back) in a transaction of their own.
435
+ def send_back(conn, units)
436
+ conn.multi { |tx| hand_back(tx, units) } unless units.empty?
437
+ end
438
+
439
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
440
+ end
441
+ end
data/lib/robin/gvl.rb ADDED
@@ -0,0 +1,62 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Measures GVL contention: how much of the time a process's threads could
5
+ # have spent waiting for the GVL they did spend waiting. Only one thread
6
+ # runs Ruby code at a time, so over t seconds n threads can wait at most
7
+ # (n - 1) * t seconds between them. Contention is the share of that they
8
+ # waited, as a percentage: 0 when no thread ever waited, near 100 when
9
+ # every thread wanted to run all the time, so more threads wouldn't run
10
+ # more Ruby code (more processes would).
11
+ #
12
+ # Threads that sleep or wait on I/O don't wait for the GVL, so a mostly
13
+ # idle process stays near 0 however many threads it has.
14
+ #
15
+ # Needs the C extension (Robin::NativeGVL);
16
+ # elsewhere (JRuby, which has no GVL) it measures nothing.
17
+ class GVL
18
+ def self.native = (NativeGVL if Robin.const_defined?(:NativeGVL))
19
+
20
+ def initialize(native = GVL.native, threads: -> { Thread.list.size })
21
+ @native = native
22
+ @threads = threads
23
+ end
24
+
25
+ def available? = !@native.nil?
26
+
27
+ # Starts measuring; #sample then reports from here.
28
+ def start
29
+ # @type ivar @native: Object & _Native
30
+ return unless @native
31
+
32
+ @native.start
33
+ @last = reading
34
+ end
35
+
36
+ def stop = @native&.stop
37
+
38
+ # The contention since the last sample (or #start), as a percentage
39
+ # rounded to 0.1, or nil when it isn't measuring.
40
+ def sample
41
+ return unless @last
42
+
43
+ now = reading
44
+ waited = now.first - @last.first
45
+ elapsed = now.last - @last.last
46
+ @last = now
47
+ possible = (@threads.call - 1) * elapsed
48
+ return 0.0 unless possible.positive?
49
+
50
+ [100.0 * waited / possible, 100.0].min.round(1) #: Float
51
+ end
52
+
53
+ private
54
+
55
+ # Nanoseconds waited so far, and now (monotonic nanoseconds).
56
+ def reading
57
+ # (Only called once #start found the extension.)
58
+ # @type ivar @native: Object & _Native
59
+ [@native.wait_ns, Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)]
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Robin
6
+ # A tiny HTTP endpoint a server opens for liveness and readiness probes
7
+ # (Kubernetes, load balancers, uptime monitors):
8
+ #
9
+ # Robin.configure { |c| c.health_port = 7433 } # or --health-port, ROBIN_HEALTH_PORT
10
+ #
11
+ # Any GET answers 200 while the process is healthy and 503 otherwise,
12
+ # with the details as JSON (see Launcher#health). Swarm children listen
13
+ # on the port plus their index. `robin health` asks the same endpoint, for
14
+ # exec probes.
15
+ class HealthServer
16
+ REASONS = {200 => "OK", 503 => "Service Unavailable"}.freeze
17
+ READ_TIMEOUT = 1
18
+ REQUEST_LIMIT = 16_384
19
+
20
+ # The port listened on (assigned by the OS when given 0).
21
+ attr_reader :port
22
+
23
+ def initialize(launcher, port:, bind: "0.0.0.0")
24
+ @launcher = launcher
25
+ @server = TCPServer.new(bind, port)
26
+ @port = @server.addr[1]
27
+ @thread = Thread.new do
28
+ Thread.current.name = "robin.health"
29
+ serve
30
+ end
31
+ end
32
+
33
+ def stop
34
+ begin
35
+ @server.close
36
+ rescue IOError
37
+ # JRuby 10.0 can raise, in the thread that closes the server, the
38
+ # "stream closed in another thread" meant for the one in accept.
39
+ # It's closed all the same, and the error mustn't replace the one a
40
+ # failed boot is stopping for.
41
+ end
42
+ @thread.join(1)
43
+ end
44
+
45
+ # The HTTP response for a Launcher#health report.
46
+ def self.response(health)
47
+ code = health.fetch("healthy") ? 200 : 503
48
+ body = Robin.dump_json(health)
49
+ "HTTP/1.1 #{code} #{REASONS.fetch(code)}\r\ncontent-type: application/json\r\n" \
50
+ "content-length: #{body.bytesize}\r\nconnection: close\r\n\r\n#{body}"
51
+ end
52
+
53
+ private
54
+
55
+ def serve
56
+ loop { answer(@server.accept) }
57
+ rescue IOError, SystemCallError
58
+ nil # closed by #stop
59
+ end
60
+
61
+ # Waits briefly for the request (whatever it says), answers and hangs
62
+ # up. A silent client is dropped without an answer. The request is read
63
+ # first: closing a socket with unread data resets the connection, and
64
+ # the client could lose the answer.
65
+ def answer(client)
66
+ return unless client.wait_readable(READ_TIMEOUT)
67
+
68
+ client.readpartial(REQUEST_LIMIT)
69
+ client.write(self.class.response(@launcher.health))
70
+ rescue IOError, SystemCallError
71
+ nil # the client went away
72
+ ensure
73
+ client.close
74
+ end
75
+ end
76
+ end