cogworker 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +7 -0
  2. data/exe/cogworker +6 -0
  3. data/exe/cogworkerswarm +6 -0
  4. data/lib/cogworker/basic_fetch.rb +27 -0
  5. data/lib/cogworker/cli.rb +64 -0
  6. data/lib/cogworker/client.rb +61 -0
  7. data/lib/cogworker/component.rb +19 -0
  8. data/lib/cogworker/config.rb +84 -0
  9. data/lib/cogworker/config_loader.rb +22 -0
  10. data/lib/cogworker/heartbeat.rb +150 -0
  11. data/lib/cogworker/history/middleware.rb +18 -0
  12. data/lib/cogworker/history/storage.rb +81 -0
  13. data/lib/cogworker/history.rb +33 -0
  14. data/lib/cogworker/job.rb +75 -0
  15. data/lib/cogworker/job_record.rb +24 -0
  16. data/lib/cogworker/job_util.rb +60 -0
  17. data/lib/cogworker/launcher.rb +93 -0
  18. data/lib/cogworker/logging.rb +18 -0
  19. data/lib/cogworker/manager.rb +71 -0
  20. data/lib/cogworker/middleware/chain.rb +64 -0
  21. data/lib/cogworker/periodic/claim.lua +27 -0
  22. data/lib/cogworker/periodic/entry.rb +24 -0
  23. data/lib/cogworker/periodic/manager.rb +28 -0
  24. data/lib/cogworker/periodic/release_middleware.rb +27 -0
  25. data/lib/cogworker/periodic/ticker.rb +123 -0
  26. data/lib/cogworker/process.rb +36 -0
  27. data/lib/cogworker/process_set.rb +29 -0
  28. data/lib/cogworker/processor.rb +129 -0
  29. data/lib/cogworker/prometheus/exporter.rb +62 -0
  30. data/lib/cogworker/queue.rb +60 -0
  31. data/lib/cogworker/redis_connection.rb +37 -0
  32. data/lib/cogworker/redis_keys.rb +32 -0
  33. data/lib/cogworker/scheduled.rb +67 -0
  34. data/lib/cogworker/signals.rb +15 -0
  35. data/lib/cogworker/stats.rb +47 -0
  36. data/lib/cogworker/status/client_middleware.rb +19 -0
  37. data/lib/cogworker/status/server_middleware.rb +31 -0
  38. data/lib/cogworker/status/storage.rb +30 -0
  39. data/lib/cogworker/status/worker.rb +27 -0
  40. data/lib/cogworker/status.rb +40 -0
  41. data/lib/cogworker/swarm.rb +169 -0
  42. data/lib/cogworker/testing.rb +109 -0
  43. data/lib/cogworker/unique_jobs/client_middleware.rb +31 -0
  44. data/lib/cogworker/unique_jobs/release_middleware.rb +30 -0
  45. data/lib/cogworker/unique_jobs.rb +32 -0
  46. data/lib/cogworker/version.rb +5 -0
  47. data/lib/cogworker/web/action.rb +63 -0
  48. data/lib/cogworker/web/application.rb +62 -0
  49. data/lib/cogworker/web/assets/ag-grid/ag-grid-community.min.js +1 -0
  50. data/lib/cogworker/web/assets/ag-grid/ag-grid.min.css +7 -0
  51. data/lib/cogworker/web/assets/ag-grid/ag-theme-alpine.min.css +2 -0
  52. data/lib/cogworker/web/assets/chart.umd.min.js +13 -0
  53. data/lib/cogworker/web/assets/htmx.min.js +1 -0
  54. data/lib/cogworker/web/assets/tailwind.css +1 -0
  55. data/lib/cogworker/web/layout.rb +352 -0
  56. data/lib/cogworker/web/router.rb +27 -0
  57. data/lib/cogworker/web/routes/busy.rb +99 -0
  58. data/lib/cogworker/web/routes/dead.rb +93 -0
  59. data/lib/cogworker/web/routes/history.rb +226 -0
  60. data/lib/cogworker/web/routes/periodic.rb +67 -0
  61. data/lib/cogworker/web/routes/queues.rb +101 -0
  62. data/lib/cogworker/web/routes/retries.rb +89 -0
  63. data/lib/cogworker/web/routes/save_session.rb +21 -0
  64. data/lib/cogworker/web/routes/scheduled.rb +49 -0
  65. data/lib/cogworker/web/routes/stats.rb +298 -0
  66. data/lib/cogworker/web/views.rb +25 -0
  67. data/lib/cogworker/web.rb +257 -0
  68. data/lib/cogworker/work.rb +19 -0
  69. data/lib/cogworker/work_set.rb +23 -0
  70. data/lib/cogworker/worker.rb +5 -0
  71. data/lib/cogworker/workers.rb +8 -0
  72. data/lib/cogworker.rb +114 -0
  73. metadata +300 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 5aad05a1deaf85505c91f0b6ebe11199fc918b6b3e3831efa5ddb753381cf824
4
+ data.tar.gz: 760b88ec2cc3075f4bbfe152062bc8f1e24b71ce2c71984f9702c0cbdb7da81f
5
+ SHA512:
6
+ metadata.gz: fc050c6eff33d2fcc5f61629d46e88452912d3f798c49b49fdc53d21ca525403a575e89e13c9e34307cba312eab7307baebc80a187661a032592fb31eb797854
7
+ data.tar.gz: a9ef2e674633abe4085b6e6e21eb74c3ec735b3ab7c4f6305f274172049e41058477b03e5a5e8105cc6b0afdeadcafc9babba07bbdd81855b54532585a9a946e
data/exe/cogworker ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'cogworker'
5
+
6
+ Cogworker::CLI.new.run(ARGV)
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'cogworker'
5
+
6
+ Cogworker::Swarm.new(ARGV).run
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ # Pops jobs off Redis lists. Queue names repeated in the process's queue
5
+ # list represent weight; since BRPOP itself scans its key list
6
+ # strictly left-to-right, weighting is implemented by shuffling the
7
+ # (already-expanded, so repeats survive) key list before every fetch cycle
8
+ # — the more often a name appears, the more likely it lands first.
9
+ class BasicFetch
10
+ TIMEOUT = 2 # seconds; also how often a stopped/quieted processor notices and exits its fetch loop.
11
+
12
+ def initialize(queues)
13
+ @queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
14
+ end
15
+
16
+ def retrieve_work
17
+ keys = @queue_keys.shuffle
18
+ result = Cogworker.config.redis { |c| c.brpop(*keys, timeout: TIMEOUT) }
19
+ return nil unless result
20
+
21
+ queue_key, raw_job = result
22
+ UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
23
+ end
24
+
25
+ UnitOfWork = Struct.new(:queue, :raw_job)
26
+ end
27
+ end
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'optparse'
4
+
5
+ module Cogworker
6
+ # Shared flag parsing + boot sequence for both the single-process and
7
+ # swarm-child entry points: `-e/--environment`, `-c/--concurrency`,
8
+ # `-r/--require <file>`, `-C/--config <file>`, `-L/--logfile <file>`,
9
+ # `-q/--queue <name>[,<weight>]` (repeatable).
10
+ class CLI
11
+ def self.parse(argv)
12
+ options = { queues: [], require_path: nil, environment: ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development' }
13
+
14
+ OptionParser.new do |o|
15
+ o.on('-e ENV', '--environment ENV') { |v| options[:environment] = v }
16
+ o.on('-c INT', '--concurrency INT', Integer) { |v| options[:concurrency] = v }
17
+ o.on('-r PATH', '--require PATH') { |v| options[:require_path] = v }
18
+ o.on('-C PATH', '--config PATH') { |v| options[:config_path] = v }
19
+ o.on('-L PATH', '--logfile PATH') { |v| options[:logfile] = v }
20
+ o.on('-q QUEUE', '--queue QUEUE') { |v| options[:queues].concat(parse_queue_weight(v)) }
21
+ end.parse!(argv.dup)
22
+
23
+ options
24
+ end
25
+
26
+ def self.parse_queue_weight(value)
27
+ name, weight = value.split(',')
28
+ Array.new([weight.to_i, 1].max, name)
29
+ end
30
+
31
+ # Boots one OS process fully: applies config, requires app code, and
32
+ # blocks running the Launcher loop until a stop signal is handled.
33
+ # `argv` is parsed independently per call so a swarm parent can boot each
34
+ # forked child with the exact same flags it was started with.
35
+ def run(argv)
36
+ options = self.class.parse(argv)
37
+ file_config = ConfigLoader.load(options[:config_path])
38
+
39
+ Cogworker.server_process!
40
+ Cogworker.config.concurrency = options[:concurrency] || file_config[:concurrency]&.to_i || Cogworker.config.concurrency
41
+ Cogworker.config.queues = options[:queues].any? ? options[:queues] : (file_config[:queues] || Cogworker.config.queues)
42
+
43
+ redirect_logfile(options[:logfile]) if options[:logfile]
44
+ require_app_code(options[:require_path]) if options[:require_path]
45
+
46
+ Launcher.new.run
47
+ end
48
+
49
+ private
50
+
51
+ def redirect_logfile(path)
52
+ Cogworker.logger = Logging.default_logger(File.open(path, 'a'))
53
+ end
54
+
55
+ def require_app_code(path)
56
+ expanded = File.expand_path(path)
57
+ if File.directory?(expanded)
58
+ require File.join(expanded, 'config/environment')
59
+ else
60
+ require expanded
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,61 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Cogworker
6
+ # Enqueues jobs: normalizes the job hash, runs the client middleware
7
+ # chain, then pushes onto the target queue or schedule ZSET.
8
+ class Client
9
+ class << self
10
+ # Cogworker::Client.push(hash) — the single funnel every enqueue path
11
+ # (perform_async/perform_in/perform_at, and direct calls) goes through.
12
+ def push(item)
13
+ job = JobUtil.normalize_item(item)
14
+ queue = job['queue']
15
+
16
+ Cogworker.config.client_chain.invoke(job['class'], job, queue, Cogworker.config.redis_pool) do
17
+ dispatch(job)
18
+ end
19
+
20
+ # UniqueJobs::ClientMiddleware sets this when the job was a
21
+ # duplicate of an already-enqueued/running `unique: :until_executed`
22
+ # job and never actually reached `dispatch` — nil signals "nothing
23
+ # was pushed", same as `job['jid']` would be meaningless otherwise.
24
+ job['unique_skipped'] ? nil : job['jid']
25
+ end
26
+
27
+ def push_bulk(items)
28
+ items.map { |item| push(item) }
29
+ end
30
+
31
+ private
32
+
33
+ # Testing.fake?/.inline? divert the enqueue entirely; both still ran
34
+ # the client middleware chain above them in #push, same as the real
35
+ # push would.
36
+ def dispatch(job)
37
+ if Testing.fake?
38
+ Testing.jobs_for(job['class']) << job
39
+ elsif Testing.inline?
40
+ Testing.perform_inline(job)
41
+ else
42
+ raw_push(job)
43
+ end
44
+ end
45
+
46
+ def raw_push(job)
47
+ Cogworker.config.redis do |conn|
48
+ if job['at']
49
+ conn.zadd(RedisKeys::SCHEDULE, job['at'].to_f, JSON.generate(job))
50
+ else
51
+ job['enqueued_at'] = Time.now.to_f
52
+ conn.multi do |pipeline|
53
+ pipeline.sadd(RedisKeys::QUEUES, job['queue'])
54
+ pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ # Mixin giving any class `#identity`/`#logger`, delegating to the
5
+ # process-global `Cogworker.identity`/`Cogworker.logger`. Deliberately global
6
+ # (not instance-scoped config injection): each OS process — including every
7
+ # cogworkerswarm child — has exactly one Cogworker::Config, so a singleton-backed
8
+ # mixin is sufficient and keeps `include Cogworker::Component` usable with zero
9
+ # wiring, as existing custom middleware (e.g. WorkerKiller) expects.
10
+ module Component
11
+ def identity
12
+ Cogworker.identity
13
+ end
14
+
15
+ def logger
16
+ Cogworker.logger
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ # Process-global configuration, built once and mutated in place by
5
+ # (possibly several, per the real init code) `configure_server`/
6
+ # `configure_client` calls, so every block sees and extends the same
7
+ # server/client middleware chains.
8
+ class Config
9
+ attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options
10
+ attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
11
+
12
+ def initialize
13
+ @server_chain = Middleware::Chain.new
14
+ @client_chain = Middleware::Chain.new
15
+ @periodic_manager = Periodic::Manager.new
16
+ @redis_options = {}
17
+ @concurrency = 10
18
+ @queues = ['default']
19
+ # Default true: an entry's most-recently-due slot fires immediately on
20
+ # a process's first tick, same as always. Set to false to skip that
21
+ # one-time catch-up fire instead — see Periodic::Ticker#priming_first_slot?
22
+ # for why a cold start against an empty/reset Redis otherwise fires
23
+ # every registered entry at once.
24
+ @periodic_catch_up = true
25
+ # Safety-net TTL (seconds) on a regular `unique: :until_executed`
26
+ # job's lock — `UniqueJobs::ReleaseMiddleware` is what actually clears
27
+ # it on success/terminal failure; this only bounds how long a lock
28
+ # can stay stuck if a process dies mid-job before that ever runs
29
+ # (same class of gap as a crashed process losing its in-flight job
30
+ # generally — see the Status section of CLAUDE.md). 24h by default;
31
+ # set higher/lower to match how long a unique job might legitimately
32
+ # run plus however long its retries can take.
33
+ @unique_lock_ttl = 24 * 60 * 60
34
+ register_default_middleware
35
+ end
36
+
37
+ def redis=(options)
38
+ @redis_options = options
39
+ end
40
+
41
+ def redis_pool
42
+ @redis_pool ||= RedisConnection.create(@redis_options.merge(size: concurrency + 5))
43
+ end
44
+
45
+ def redis(&block)
46
+ pool = redis_pool
47
+ if block_given?
48
+ pool.with(&block)
49
+ else
50
+ pool
51
+ end
52
+ end
53
+
54
+ def server_middleware
55
+ yield server_chain if block_given?
56
+ server_chain
57
+ end
58
+
59
+ def client_middleware
60
+ yield client_chain if block_given?
61
+ client_chain
62
+ end
63
+
64
+ # `config.periodic(&PERIODIC_JOBS)` — the block receives the manager and
65
+ # registers cron entries synchronously as it runs. No-op-safe: an app
66
+ # that never calls this simply has an empty periodic_manager, and the
67
+ # Ticker (started later, per process) has nothing to do.
68
+ def periodic(&block)
69
+ block&.call(periodic_manager)
70
+ periodic_manager
71
+ end
72
+
73
+ private
74
+
75
+ # Always present, regardless of whether config.periodic/a unique job is
76
+ # ever used — each is a no-op for a job that doesn't carry its own
77
+ # marker (`periodic_pjid`/`unique: :until_executed`).
78
+ def register_default_middleware
79
+ @server_chain.add(Periodic::ReleaseMiddleware)
80
+ @client_chain.add(UniqueJobs::ClientMiddleware)
81
+ @server_chain.add(UniqueJobs::ReleaseMiddleware)
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+ require 'erb'
5
+
6
+ module Cogworker
7
+ # Loads the process config file (`:concurrency:`/`:queues:` YAML, symbol
8
+ # keys, ERB-interpolated before parsing — e.g. `<%= ENV["CONCURRENCY"] %>`).
9
+ # A queue name repeated in `:queues:` is weight, not a duplicate: the
10
+ # repetition survives untouched here and is what BasicFetch's per-cycle
11
+ # shuffle uses to weight fetches.
12
+ module ConfigLoader
13
+ module_function
14
+
15
+ def load(path)
16
+ return {} unless path
17
+
18
+ erb_result = ERB.new(File.read(path)).result
19
+ YAML.safe_load(erb_result, permitted_classes: [Symbol], aliases: true) || {}
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+ require 'redis'
5
+
6
+ module Cogworker
7
+ # Publishes this process's presence (for ProcessSet) and listens for
8
+ # remote quiet!/stop! requests (for Process#quiet!/#stop! issued by another
9
+ # process, e.g. a self-targeting WorkerKiller or the Web UI). Both the
10
+ # heartbeat loop and the pub/sub subscriber must only start *after* a
11
+ # cogworkerswarm fork, never before, in the parent — otherwise the thread
12
+ # simply doesn't exist in the child, and a lock that thread held could
13
+ # leave the child permanently deadlocked.
14
+ class Heartbeat
15
+ INTERVAL = 5
16
+ TTL = 60
17
+
18
+ def initialize(manager)
19
+ @manager = manager
20
+ end
21
+
22
+ def start!
23
+ @stopping = false
24
+ @beat_thread = Thread.new { beat_loop }
25
+ @signal_thread = Thread.new { subscribe_loop }
26
+ end
27
+
28
+ # Tears down both background threads, not just the Redis keys — leaving
29
+ # them running would leak a thread (and a dedicated pub/sub connection)
30
+ # per Heartbeat instance for the remaining life of the process.
31
+ def stop!
32
+ @stopping = true
33
+ @beat_thread&.kill
34
+ @subscribe_client&.unsubscribe
35
+ @signal_thread&.kill
36
+ cleanup_presence!
37
+ rescue StandardError
38
+ nil
39
+ end
40
+
41
+ private
42
+
43
+ def cleanup_presence!
44
+ Cogworker.config.redis do |c|
45
+ c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
46
+ c.srem(RedisKeys::PROCESSES, Cogworker.identity)
47
+ end
48
+ end
49
+
50
+ def beat_loop
51
+ loop do
52
+ beat
53
+ sleep(INTERVAL)
54
+ end
55
+ rescue StandardError => e
56
+ Cogworker.logger.error { "Heartbeat died: #{e.class}: #{e.message}" }
57
+ end
58
+
59
+ def beat
60
+ identity = Cogworker.identity
61
+ info = {
62
+ 'hostname' => Cogworker.hostname,
63
+ 'pid' => ::Process.pid,
64
+ 'concurrency' => Cogworker.config.concurrency,
65
+ 'queues' => @manager.queues,
66
+ 'started_at' => (@started_at ||= Time.now.to_f),
67
+ 'rss_kb' => current_rss_kb
68
+ }
69
+
70
+ Cogworker.config.redis do |c|
71
+ c.sadd(RedisKeys::PROCESSES, identity)
72
+ c.hset(RedisKeys.process(identity),
73
+ 'info', JSON.generate(info),
74
+ 'busy', @manager.busy_count.to_s,
75
+ 'quiet', @manager.quiet?.to_s)
76
+ c.expire(RedisKeys.process(identity), TTL)
77
+ c.expire(RedisKeys.workers(identity), TTL)
78
+ end
79
+ end
80
+
81
+ # Current resident set size, in KB — read straight from the kernel
82
+ # (`/proc/self/status`, present on any real Linux deployment target)
83
+ # when available, since that's a plain file read with no extra process;
84
+ # `ps` (a real fork+exec every heartbeat) is only the fallback for
85
+ # platforms without `/proc` (macOS in local dev). `nil` — not `0` — on
86
+ # any failure, so the Web UI can render "n/a" rather than a misleading
87
+ # "0M" if this ever can't be measured.
88
+ def current_rss_kb
89
+ proc_status = '/proc/self/status'
90
+ if File.readable?(proc_status)
91
+ matched = File.read(proc_status)[/^VmRSS:\s+(\d+)\s+kB/, 1]
92
+ return matched.to_i if matched
93
+ end
94
+
95
+ out = `ps -o rss= -p #{::Process.pid} 2>/dev/null`.strip
96
+ out.empty? ? nil : out.to_i
97
+ rescue StandardError
98
+ nil
99
+ end
100
+
101
+ # Uses its own dedicated (non-pooled) Redis client: SUBSCRIBE blocks the
102
+ # connection for as long as the subscription is open, which would starve
103
+ # the shared job-execution pool if it borrowed a connection from there.
104
+ def subscribe_loop
105
+ @subscribe_client = ::Redis.new(RedisConnection.client_options(Cogworker.config.redis_options))
106
+ @subscribe_client.subscribe(RedisKeys.signal(Cogworker.identity)) do |on|
107
+ on.message do |_channel, message|
108
+ dispatch(message)
109
+ end
110
+ end
111
+ rescue StandardError => e
112
+ return if @stopping
113
+
114
+ Cogworker.logger.error { "Signal subscriber died: #{e.class}: #{e.message}" }
115
+ sleep(1)
116
+ retry
117
+ end
118
+
119
+ def dispatch(message)
120
+ case message
121
+ when 'quiet' then @manager.quiet!
122
+ when 'stop' then remote_stop!
123
+ else Cogworker.logger.warn { "Unknown signal message: #{message}" }
124
+ end
125
+ end
126
+
127
+ # A *remote* stop (published by another process — typically the Web
128
+ # UI's `Process#stop!` — as opposed to this process's own `TERM`/`INT`,
129
+ # handled by `Launcher#stop!`/`#watch_signals` on the main thread) has
130
+ # to actually end this OS process, not just quiesce the Manager
131
+ # forever: nothing here naturally returns from a running `Launcher#run`
132
+ # loop the way breaking out of `watch_signals` does — this dispatch
133
+ # runs on the pub/sub subscriber's own background thread, so without an
134
+ # explicit exit the process would otherwise sit there, still
135
+ # heartbeating, permanently "quiet", never picking up work again,
136
+ # until someone `kill`s it for real or the swarm is restarted.
137
+ # `::Process.exit!` (not `Kernel#exit`, which only raises `SystemExit`
138
+ # on the *calling* thread and wouldn't actually end the process from
139
+ # here) terminates immediately and unconditionally, from any thread —
140
+ # deliberately *after* `@manager.stop!` has already drained in-flight
141
+ # work and `cleanup_presence!` has already removed this process from
142
+ # Redis, so both happen before the process can vanish out from under
143
+ # them.
144
+ def remote_stop!
145
+ @manager.stop!
146
+ cleanup_presence!
147
+ ::Process.exit!(true)
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ module History
5
+ # Server middleware: times every run and records it via Storage,
6
+ # success or failure, regardless of what other middleware/worker code does.
7
+ class Middleware
8
+ def call(_worker, job, queue)
9
+ started_at = Time.now.to_f
10
+ yield
11
+ Storage.record(job, queue, started_at, Time.now.to_f, 'success')
12
+ rescue Exception => e # rubocop:disable Lint/RescueException
13
+ Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e)
14
+ raise e
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Cogworker
6
+ module History
7
+ # Three parallel ZSETs (score = finished_at), so any of the three
8
+ # filters (all/success/failed) is a plain `ZREVRANGE` — no per-request
9
+ # `ZUNIONSTORE`/scan needed. Each is trimmed to `History.max_entries`
10
+ # independently on every write, oldest first.
11
+ module Storage
12
+ LIST_KEYS = {
13
+ 'all' => 'cogworker:history:all',
14
+ 'success' => 'cogworker:history:success',
15
+ 'failed' => 'cogworker:history:failed'
16
+ }.freeze
17
+
18
+ module_function
19
+
20
+ def record(job, queue, started_at, finished_at, status, error: nil)
21
+ entry = {
22
+ 'jid' => job['jid'], 'class' => job['class'], 'queue' => queue, 'args' => job['args'],
23
+ 'status' => status, 'started_at' => started_at, 'finished_at' => finished_at
24
+ }
25
+ if error
26
+ entry['error_class'] = error.class.name
27
+ entry['error_message'] = error.message.to_s[0, 10_000]
28
+ entry['backtrace'] = (error.backtrace || []).first(200)
29
+ end
30
+
31
+ raw = JSON.generate(entry)
32
+ max = Cogworker::History.max_entries
33
+ Cogworker.config.redis do |c|
34
+ write_and_trim(c, LIST_KEYS.fetch('all'), raw, finished_at, max)
35
+ write_and_trim(c, LIST_KEYS.fetch(status), raw, finished_at, max)
36
+ end
37
+ end
38
+
39
+ # Newest-first page of `count`/`status` entries: `[entries, total]`.
40
+ def page(status, page_number, per_page)
41
+ key = LIST_KEYS.fetch(status, LIST_KEYS.fetch('all'))
42
+ start = [(page_number - 1), 0].max * per_page
43
+ stop = start + per_page - 1
44
+
45
+ Cogworker.config.redis do |c|
46
+ total = c.zcard(key)
47
+ raw_entries = c.zrevrange(key, start, stop)
48
+ [raw_entries.map { |raw| JSON.parse(raw) }, total]
49
+ end
50
+ end
51
+
52
+ def write_and_trim(conn, key, raw, score, max)
53
+ conn.zadd(key, score, raw)
54
+ conn.zremrangebyrank(key, 0, -(max + 1))
55
+ end
56
+
57
+ # Success/failed counts per UTC calendar day, for the last `days` days
58
+ # (today included) — `{ 'YYYY-MM-DD' => { 'success' => n, 'failed' => n } }`,
59
+ # a day with no entries simply absent from the Hash. Reads the shared
60
+ # "all" list once (rather than "success" and "failed" separately) since
61
+ # every entry already carries its own `status`. Bucketing uses UTC,
62
+ # not the viewer's browser timezone (unlike `Layout.time_tag`
63
+ # elsewhere) — this is a server-rendered daily aggregate, not a single
64
+ # instant, so there's no one "browser day" to convert into.
65
+ # Same caveat as any other read of this list: only entries still
66
+ # within `History.max_entries` are counted, so a very busy queue's
67
+ # oldest requested days may already have been trimmed away.
68
+ def daily_counts(days)
69
+ since = Time.now.to_f - (days * 86_400)
70
+ Cogworker.config.redis do |c|
71
+ c.zrangebyscore(LIST_KEYS.fetch('all'), since, '+inf').each_with_object({}) do |raw, counts|
72
+ entry = JSON.parse(raw)
73
+ day = Time.at(entry['finished_at']).utc.strftime('%Y-%m-%d')
74
+ bucket = (counts[day] ||= { 'success' => 0, 'failed' => 0 })
75
+ bucket[entry['status']] += 1 if bucket.key?(entry['status'])
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ # Execution history: every job run (success and failure alike) is recorded
5
+ # with its full args, timing, and — on failure — error class/message/
6
+ # backtrace. Distinct from the Status layer (`Cogworker::Status`), which
7
+ # tracks *current* per-jid state with a TTL; History is an append-only,
8
+ # length-capped log meant for browsing/auditing past runs.
9
+ module History
10
+ DEFAULT_MAX_ENTRIES = 1000
11
+
12
+ class << self
13
+ # How many entries are retained (oldest trimmed first), independently
14
+ # for the "all", "success", and "failed" lists — read fresh on every
15
+ # write, so changing it takes effect immediately, no need to rebuild
16
+ # the middleware chain. Configurable via
17
+ # `configure_server_middleware(config, max_entries: N)`, or directly:
18
+ # `Cogworker::History.max_entries = 5000`.
19
+ attr_writer :max_entries
20
+
21
+ def max_entries
22
+ @max_entries ||= DEFAULT_MAX_ENTRIES
23
+ end
24
+ end
25
+
26
+ module_function
27
+
28
+ def configure_server_middleware(config, max_entries: DEFAULT_MAX_ENTRIES)
29
+ self.max_entries = max_entries
30
+ config.server_middleware { |chain| chain.add(Middleware) }
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cogworker
4
+ # `include Cogworker::Worker` (alias `Cogworker::Job`) DSL.
5
+ module Job
6
+ def self.included(base)
7
+ base.include(Component)
8
+ base.extend(ClassMethods)
9
+ base.include(InstanceMethods)
10
+ end
11
+
12
+ # `cogworker_options`/`perform_async`/`perform_in`/`perform_at`, extended
13
+ # onto the including class.
14
+ module ClassMethods
15
+ # Accepts a plain Hash (both `key: value` and `:key => value` call-site
16
+ # syntax already collect into the same Hash literal in Ruby — nothing
17
+ # special to implement). Merges into the class's option set rather than
18
+ # replacing it, and never filters keys: arbitrary custom options (e.g.
19
+ # `lock_run: :while_executing`) ride along unchanged into the job hash.
20
+ def cogworker_options(opts = {})
21
+ cogworker_options_hash.merge!(opts.transform_keys(&:to_sym))
22
+ cogworker_options_hash
23
+ end
24
+
25
+ def cogworker_options_hash
26
+ @cogworker_options_hash ||= if superclass.respond_to?(:cogworker_options_hash)
27
+ superclass.cogworker_options_hash.dup
28
+ else
29
+ {}
30
+ end
31
+ end
32
+
33
+ def perform_async(*args)
34
+ Client.push(job_payload('args' => args))
35
+ end
36
+
37
+ def perform_in(interval, *args)
38
+ ts = interval_to_ts(interval)
39
+ Client.push(job_payload('args' => args, 'at' => ts))
40
+ end
41
+ alias perform_at perform_in
42
+
43
+ # `SomeJob.jobs`/`SomeJob.clear` — reads/clears this class's entries in
44
+ # `Cogworker::Testing`'s fake queue (empty outside `Testing.fake!`).
45
+ def jobs
46
+ Testing.jobs_for(name)
47
+ end
48
+
49
+ def clear
50
+ Testing.jobs_for(name).clear
51
+ end
52
+
53
+ private
54
+
55
+ def job_payload(extra)
56
+ cogworker_options_hash.transform_keys(&:to_s).merge('class' => name).merge(extra)
57
+ end
58
+
59
+ def interval_to_ts(interval)
60
+ numeric = interval.respond_to?(:to_f) ? interval.to_f : interval.to_time.to_f
61
+ # A value below a billion is treated as "seconds from now"
62
+ # (perform_in), anything
63
+ # at or above it as an absolute unix timestamp (perform_at, or a
64
+ # Time passed straight through — Time#to_f is already an epoch
65
+ # timestamp, always well above the threshold).
66
+ numeric < 1_000_000_000 ? Time.now.to_f + numeric : numeric
67
+ end
68
+ end
69
+
70
+ # `jid`, included onto the including class's instances.
71
+ module InstanceMethods
72
+ attr_accessor :jid
73
+ end
74
+ end
75
+ end