robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
data/lib/robin/client.rb CHANGED
@@ -1,37 +1,371 @@
1
- require_relative 'config'
2
- require_relative 'user'
3
- require 'twitter'
1
+ # frozen_string_literal: true
4
2
 
5
3
  module Robin
4
+ # Pushes jobs into Redis.
5
+ #
6
+ # Robin::Client.push("class" => "ReportJob", "args" => [1], "queue" => "low")
7
+ # Robin::Client.push_bulk("class" => ReportJob, "args" => [[1], [2], [3]])
6
8
  class Client
7
- Robin::User.load_credentials
8
- @client = Twitter::Client.new(
9
- :consumer_key => Robin::Config.key,
10
- :consumer_secret => Robin::Config.secret,
11
- :oauth_token => Robin::User.token,
12
- :oauth_token_secret => Robin::User.secret)
9
+ class ArgumentError < ::ArgumentError; end
13
10
 
14
- def self.home_timeline
15
- @client.home_timeline
11
+ class << self
12
+ # A callable that may take over delivery (Robin::Testing uses it to
13
+ # keep jobs in memory). It returns truthy when it handled the push.
14
+ attr_accessor :interceptor
15
+
16
+ # Callables that may rewrite a job hash (string keys) before #push
17
+ # handles it, each returning the hash to use. The Rails integration
18
+ # adds one that turns an ActiveJob class into its wrapper, so
19
+ # periodic jobs (and Client.push) can name ActiveJob classes.
20
+ def rewriters = (@rewriters ||= [])
21
+ end
22
+
23
+ # Sends every push made by this thread inside the block to +pool+ (a
24
+ # Robin::Redis::Pool) instead of the configured Redis, e.g. to spread
25
+ # jobs over several Redis shards:
26
+ #
27
+ # SHARDS = [Robin::Redis::Pool.new { Robin::Redis::Connection.new(url: A) }, ...]
28
+ # Robin::Client.via(SHARDS.sample) { ReportJob.perform_async(1) }
29
+ #
30
+ # Each shard needs its own Robin processes to work its jobs.
31
+ def self.via(pool)
32
+ previous = Thread.current[:robin_client_pool]
33
+ Thread.current[:robin_client_pool] = pool
34
+ yield
35
+ ensure
36
+ Thread.current[:robin_client_pool] = previous
37
+ end
38
+
39
+ # The pool set by an enclosing Client.via block, if any.
40
+ def self.current_pool = Thread.current[:robin_client_pool]
41
+
42
+ # Holds back the jobs this thread's pushes send inside the block, and
43
+ # sends them all when it finishes: in one round trip when they go
44
+ # straight to the same Redis. If the block raises, none are sent.
45
+ # Returns the block's value.
46
+ #
47
+ # Robin::Client.together { reports.each { |r| ReportJob.perform_async(r.id) } }
48
+ #
49
+ # mutant:disable -- on CRuby, setting a fiber-local to nil removes it,
50
+ # so key? would do; on JRuby the key stays, and every later block would
51
+ # take itself for a nested one.
52
+ def self.together
53
+ return yield if Thread.current[:robin_client_together]
54
+
55
+ held = Thread.current[:robin_client_together] = []
56
+ begin
57
+ result = yield
58
+ ensure
59
+ Thread.current[:robin_client_together] = nil
60
+ end
61
+ send_together(held)
62
+ result
63
+ end
64
+
65
+ # Sends what a Client.together block held: [client, payloads] each.
66
+ def self.send_together(held)
67
+ return if held.empty?
68
+
69
+ pool = held.first.first.redis_pool
70
+ if held.all? { |client, _payloads| client.redis_pool.equal?(pool) && client.plain? }
71
+ pool.with { |conn| conn.pipelined { |pipe| held.each { |client, payloads| client.queue_push(pipe, payloads) } } }
72
+ else
73
+ held.each { |client, payloads| client.__send__(:send_payloads, payloads) }
74
+ end
75
+ end
76
+ private_class_method :send_together
77
+
78
+ def self.push(item) = new.push(item)
79
+
80
+ def self.push_bulk(items) = new.push_bulk(items)
81
+
82
+ # Converts a delay (seconds), a Time, or an epoch timestamp (seconds or
83
+ # milliseconds) to epoch milliseconds.
84
+ def self.timestamp_ms(value, now = Robin.now_ms)
85
+ case value
86
+ when nil then raise ArgumentError, "a time is required, not nil"
87
+ when Numeric
88
+ if value > 100_000_000_000 then Integer(value) # already epoch milliseconds
89
+ elsif value > 1_000_000_000 then Integer(value * 1000) # epoch seconds
90
+ else now + Integer(value * 1000) # a delay in seconds
91
+ end
92
+ when Time then Integer(value.to_r * 1000) # Time#to_time needs the date library
93
+ else
94
+ if value.respond_to?(:to_time) && !value.respond_to?(:parts)
95
+ Integer(value.to_time.to_r * 1000) # Date, DateTime...
96
+ else
97
+ now + Integer(value.to_f * 1000) # e.g. ActiveSupport::Duration
98
+ end
99
+ end
100
+ end
101
+
102
+ # +count+ times spread evenly over +interval+ seconds, the first at
103
+ # +start+ (anything #timestamp_ms takes; default now), as epoch
104
+ # milliseconds.
105
+ def self.spread(count, interval, start = 0)
106
+ raise ArgumentError, "'spread_interval' must be a positive number of seconds" unless interval.is_a?(Numeric) && interval.positive?
107
+
108
+ first = timestamp_ms(start)
109
+ Array.new(count) { |i| first + Integer(interval * 1000 * i / count) }
110
+ end
111
+
112
+ attr_reader :config
113
+
114
+ # +pool+ pushes to another Redis (see Client.via); +chain+ replaces the
115
+ # configured client middleware for this client's pushes.
116
+ def initialize(config: Robin.config, pool: nil, chain: nil)
117
+ @config = config
118
+ @pool = pool || Client.current_pool
119
+ @chain = chain
120
+ end
121
+
122
+ def middleware = @chain || @config.client_middleware
123
+
124
+ def redis_pool
125
+ @pool || @config.redis_pool
126
+ end
127
+
128
+ # Pushes a single job. Returns its jid, or nil if a client middleware
129
+ # stopped the push (e.g. a duplicate unique job).
130
+ def push(item)
131
+ item = Client.rewriters.reduce(item.transform_keys(&:to_s)) { |rewritten, rewriter| rewriter.call(rewritten) }
132
+ normalized = normalize(item)
133
+ # Inside a batch's +jobs+ block, the job joins the batch as it's sent
134
+ # (see Batch.holding).
135
+ payload = Batch.holding do
136
+ pushed = invoke_middleware(normalized)
137
+ deliver([pushed]) if pushed
138
+ pushed
139
+ end
140
+ payload&.fetch("jid")
141
+ end
142
+
143
+ # Pushes many jobs of the same class at once. "args" is an array of
144
+ # argument arrays; "at" may be a single time or one per job, and
145
+ # "spread_interval" (seconds) spreads the jobs evenly over that long,
146
+ # from "at" or now. Returns the jids (nil for any job stopped by
147
+ # middleware).
148
+ def push_bulk(item)
149
+ base = item.transform_keys(&:to_s)
150
+ args_list = base["args"]
151
+ raise ArgumentError, "push_bulk needs 'args' to be an Array of Arrays" unless args_list.is_a?(Array) && args_list.all?(Array)
152
+
153
+ ats = base["at"]
154
+ if ats.is_a?(Array) && !ats.size.equal?(args_list.size)
155
+ raise ArgumentError, "push_bulk 'at' must have the same size as 'args'"
156
+ end
157
+ if (interval = base.delete("spread_interval"))
158
+ raise ArgumentError, "push_bulk takes 'spread_interval' or an 'at' per job, not both" if ats.is_a?(Array)
159
+
160
+ # (Steep can't tell that ats, not an Array, splats to one argument.)
161
+ ats = Client.spread(args_list.size, interval, *ats) # steep:ignore UnexpectedPositionalArgument
162
+ end
163
+
164
+ jids = base.delete("jids")
165
+ jid = base.delete("jid")
166
+ # What every job shares is worked out once; then each gets its own
167
+ # arguments, jid and (when given one per job) time.
168
+ template = normalize(base.merge({"args" => [], "at" => (ats unless ats.is_a?(Array)), "jid" => ""}))
169
+ built = args_list.each_with_index.map { |args, i| build(template, args, jids&.fetch(i) || jid, (ats.fetch(i) if ats.is_a?(Array))) }
170
+ # Inside a batch's +jobs+ block, the jobs join the batch together.
171
+ payloads = Batch.holding do
172
+ # Their unique locks are taken together too.
173
+ Unique.claiming(middleware, built) { built.map { |payload| run_middleware(payload) } }
174
+ .tap { |list| deliver(list.compact) }
175
+ end
176
+ payloads.map { |p| p&.fetch("jid") }
16
177
  end
17
178
 
18
- def self.user_timeline(user = nil)
19
- if user
20
- @client.user_timeline(user)
179
+ # Fills in defaults and validates a job hash. Mutates and returns it.
180
+ def normalize(item)
181
+ item = item.transform_keys(&:to_s)
182
+ klass = item["class"]
183
+ raise ArgumentError, "job must have a 'class'" unless klass
184
+ raise ArgumentError, "job 'args' must be an Array" unless item["args"].is_a?(Array)
185
+
186
+ if klass.instance_of?(Class)
187
+ # (A job class: Steep only knows it's a Class.)
188
+ item = klass.robin_options.merge(item) if klass.respond_to?(:robin_options) # steep:ignore NoMethod
189
+ item["class"] = klass.name
21
190
  else
22
- @client.user_timeline
191
+ item["class"] = klass.to_s
23
192
  end
193
+ queue = item["queue"].to_s
194
+ item["queue"] = queue.empty? ? "default" : queue
195
+ item["retry"] = true unless item.key?("retry")
196
+ item["jid"] ||= Robin.generate_jid
197
+ item["created_at"] ||= Robin.now_ms
198
+ at = item.delete("at")
199
+ at_ms = self.class.timestamp_ms(at) if at
200
+ item["at"] = Robin.epoch_seconds(at_ms) if at_ms
201
+ if (expires_in = item.delete("expires_in"))
202
+ # The clock starts when the job is due to run.
203
+ item["expires_at"] = [at_ms, item.fetch("created_at")].compact.max + Integer(expires_in.to_f * 1000)
204
+ end
205
+ if (window = item["retry_for"]) && !(window.is_a?(Numeric) && window.positive?)
206
+ raise ArgumentError, "job 'retry_for' must be a positive number of seconds"
207
+ end
208
+ item["retry_queue"] = item.fetch("retry_queue").to_s if item["retry_queue"]
209
+ %w[wait wait_until sync].each { |k| item.delete(k) }
210
+ check_args(item) if @config.strict_args
211
+ item
212
+ end
213
+
214
+ private
215
+
216
+ # The work push_bulk does for each job is in methods of their own:
217
+ # JRuby compiles a method after 50 calls, so these are compiled after
218
+ # 50 jobs, rather than with push_bulk after 50 bulk pushes.
219
+
220
+ # A bulk-pushed job: +template+ with its own arguments, jid (or a new
221
+ # one) and time, if it has one.
222
+ def build(template, args, jid, at)
223
+ payload = template.merge({"args" => args, "jid" => jid || Robin.generate_jid})
224
+ schedule(payload, self.class.timestamp_ms(at)) if at
225
+ payload
226
+ end
227
+
228
+ # The job as client middleware leaves it, or nil if it stopped it.
229
+ def run_middleware(payload)
230
+ check_args(payload) if @config.strict_args
231
+ invoke_middleware(payload)
232
+ end
233
+
234
+ def invoke_middleware(payload)
235
+ middleware.invoke(payload.fetch("class"), payload, payload.fetch("queue"), redis_pool) { payload }
236
+ .tap { |result| Debounce.carry(payload, result) }
24
237
  end
25
238
 
26
- def self.tweet(message)
27
- @client.update(message)
239
+ # Sets a bulk-pushed job's own time (epoch ms); its expiry, if any,
240
+ # counts from then (the template's counted from its creation).
241
+ def schedule(payload, at_ms)
242
+ payload["at"] = Robin.epoch_seconds(at_ms)
243
+ payload["expires_at"] += [at_ms - payload.fetch("created_at"), 0].max if payload["expires_at"]
244
+ end
245
+
246
+ # Config#push_deferrer may take over delivery (e.g. until a database
247
+ # transaction commits); otherwise the jobs go out now.
248
+ def deliver(payloads)
249
+ # An atomic batch block sends its jobs once it finishes.
250
+ hold = Batch.hold
251
+ return hold.pushes << [self, payloads] if hold
252
+
253
+ deliver_now(payloads)
254
+ end
255
+
256
+ public
257
+
258
+ # Would #deliver_now send these jobs to Redis right away (with no
259
+ # interceptor, push deferrer or reliable push buffer in the way)?
260
+ def direct?(payloads)
261
+ plain? && (!@config.push_deferrer || payloads.any? { |p| p.key?("bid") })
262
+ end
263
+
264
+ # Does #send_payloads go straight to Redis (with no interceptor or
265
+ # reliable push buffer in the way)?
266
+ def plain? = !Client.interceptor && !@config.reliable_push
267
+
268
+ # Delivers now, without waiting for an atomic batch block (see
269
+ # Batch#jobs).
270
+ def deliver_now(payloads)
271
+ # Batch jobs are never deferred: their batch has already counted them,
272
+ # and a deferred push that never happens (e.g. a rolled-back
273
+ # transaction) would leave the batch waiting forever.
274
+ deferrable = payloads.none? { |p| p.key?("bid") }
275
+ return if deferrable && @config.push_deferrer&.call(payloads) { send_payloads(payloads) }
276
+
277
+ send_payloads(payloads)
28
278
  end
29
279
 
30
- def self.followers(user = nil)
31
- if user
32
- @client.followers(user).users
280
+ private
281
+
282
+ def send_payloads(payloads)
283
+ held = Thread.current[:robin_client_together]
284
+ return held << [self, payloads] if held
285
+
286
+ if @config.reliable_push
287
+ ReliablePush.for(@config, @pool).push(payloads) { raw_push(payloads) }
33
288
  else
34
- @client.followers.users
289
+ raw_push(payloads)
290
+ end
291
+ end
292
+
293
+ def raw_push(payloads)
294
+ return if Client.interceptor&.call(payloads)
295
+
296
+ redis_pool.with { |conn| conn.pipelined { |pipe| queue_push(pipe, payloads) } }
297
+ end
298
+
299
+ public
300
+
301
+ # Queues the commands pushing +payloads+ into +pipe+ (a pipeline or
302
+ # transaction on #redis_pool).
303
+ def queue_push(pipe, payloads)
304
+ Debounce.queue_claims(pipe, payloads)
305
+ now = Robin.now_ms
306
+ due = Robin.epoch_seconds(now)
307
+ payloads.group_by { |p| (p.key?("at") && p.fetch("at") > due) ? :later : p.fetch("queue") }.each do |queue, group|
308
+ if queue == :later
309
+ # The score says when; enqueued_at is set when it's moved onto
310
+ # its queue.
311
+ args = group.flat_map { |p| [p.fetch("at"), Robin.dump_json(p.except("at", "enqueued_at"))] }
312
+ pipe.call("ZADD", @config.key("schedule"), *args)
313
+ else
314
+ entries = group.map do |p|
315
+ p.delete("at")
316
+ p["enqueued_at"] = now
317
+ Robin.dump_json(p)
318
+ end
319
+ pipe.call("SADD", @config.key("queues"), queue)
320
+ pipe.call("LPUSH", @config.key("queue", queue), *entries)
321
+ end
322
+ end
323
+ index_jids(pipe, payloads)
324
+ end
325
+
326
+ private
327
+
328
+ # Records each job's queue for Robin::JobLocator (see Config#jid_index).
329
+ def index_jids(pipe, payloads)
330
+ ttl = @config.jid_index
331
+ return unless ttl
332
+
333
+ payloads.each { |p| pipe.call("SET", @config.key("jid", p.fetch("jid")), p.fetch("queue"), "EX", ttl) }
334
+ end
335
+
336
+ def check_args(item)
337
+ bad = find_non_native(item.fetch("args"))
338
+ return unless bad
339
+
340
+ message = "Job arguments for #{item["class"]} must be JSON-native types (String, Integer, finite Float, " \
341
+ "true, false, nil, Array, or Hash with String keys); got #{bad.inspect} (#{bad.class}). " \
342
+ "Pass IDs or plain values instead of objects."
343
+ case @config.strict_args
344
+ when :warn then @config.logger.warn(message)
345
+ else raise ArgumentError, message
346
+ end
347
+ end
348
+
349
+ # Returns the first value (or hash key) that doesn't survive a JSON round
350
+ # trip unchanged, or nil. (Such a value is never nil or false: those are
351
+ # JSON-native, and come back from the last branch as "nothing found".)
352
+ def find_non_native(obj)
353
+ case obj
354
+ when String, Integer, true then nil
355
+ when Float then (obj unless obj.finite?) # (JSON has no NaN or Infinity)
356
+ when Array
357
+ obj.each do |o|
358
+ bad = find_non_native(o)
359
+ return bad if bad
360
+ end
361
+ nil
362
+ when Hash
363
+ obj.each do |k, v|
364
+ bad = k.is_a?(String) ? find_non_native(v) : k
365
+ return bad if bad
366
+ end
367
+ nil
368
+ else obj
35
369
  end
36
370
  end
37
371
  end