cogworker 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/exe/cogworker +6 -0
- data/exe/cogworkerswarm +6 -0
- data/lib/cogworker/basic_fetch.rb +27 -0
- data/lib/cogworker/cli.rb +64 -0
- data/lib/cogworker/client.rb +61 -0
- data/lib/cogworker/component.rb +19 -0
- data/lib/cogworker/config.rb +84 -0
- data/lib/cogworker/config_loader.rb +22 -0
- data/lib/cogworker/heartbeat.rb +150 -0
- data/lib/cogworker/history/middleware.rb +18 -0
- data/lib/cogworker/history/storage.rb +81 -0
- data/lib/cogworker/history.rb +33 -0
- data/lib/cogworker/job.rb +75 -0
- data/lib/cogworker/job_record.rb +24 -0
- data/lib/cogworker/job_util.rb +60 -0
- data/lib/cogworker/launcher.rb +93 -0
- data/lib/cogworker/logging.rb +18 -0
- data/lib/cogworker/manager.rb +71 -0
- data/lib/cogworker/middleware/chain.rb +64 -0
- data/lib/cogworker/periodic/claim.lua +27 -0
- data/lib/cogworker/periodic/entry.rb +24 -0
- data/lib/cogworker/periodic/manager.rb +28 -0
- data/lib/cogworker/periodic/release_middleware.rb +27 -0
- data/lib/cogworker/periodic/ticker.rb +123 -0
- data/lib/cogworker/process.rb +36 -0
- data/lib/cogworker/process_set.rb +29 -0
- data/lib/cogworker/processor.rb +129 -0
- data/lib/cogworker/prometheus/exporter.rb +62 -0
- data/lib/cogworker/queue.rb +60 -0
- data/lib/cogworker/redis_connection.rb +37 -0
- data/lib/cogworker/redis_keys.rb +32 -0
- data/lib/cogworker/scheduled.rb +67 -0
- data/lib/cogworker/signals.rb +15 -0
- data/lib/cogworker/stats.rb +47 -0
- data/lib/cogworker/status/client_middleware.rb +19 -0
- data/lib/cogworker/status/server_middleware.rb +31 -0
- data/lib/cogworker/status/storage.rb +30 -0
- data/lib/cogworker/status/worker.rb +27 -0
- data/lib/cogworker/status.rb +40 -0
- data/lib/cogworker/swarm.rb +169 -0
- data/lib/cogworker/testing.rb +109 -0
- data/lib/cogworker/unique_jobs/client_middleware.rb +31 -0
- data/lib/cogworker/unique_jobs/release_middleware.rb +30 -0
- data/lib/cogworker/unique_jobs.rb +32 -0
- data/lib/cogworker/version.rb +5 -0
- data/lib/cogworker/web/action.rb +63 -0
- data/lib/cogworker/web/application.rb +62 -0
- data/lib/cogworker/web/assets/ag-grid/ag-grid-community.min.js +1 -0
- data/lib/cogworker/web/assets/ag-grid/ag-grid.min.css +7 -0
- data/lib/cogworker/web/assets/ag-grid/ag-theme-alpine.min.css +2 -0
- data/lib/cogworker/web/assets/chart.umd.min.js +13 -0
- data/lib/cogworker/web/assets/htmx.min.js +1 -0
- data/lib/cogworker/web/assets/tailwind.css +1 -0
- data/lib/cogworker/web/layout.rb +352 -0
- data/lib/cogworker/web/router.rb +27 -0
- data/lib/cogworker/web/routes/busy.rb +99 -0
- data/lib/cogworker/web/routes/dead.rb +93 -0
- data/lib/cogworker/web/routes/history.rb +226 -0
- data/lib/cogworker/web/routes/periodic.rb +67 -0
- data/lib/cogworker/web/routes/queues.rb +101 -0
- data/lib/cogworker/web/routes/retries.rb +89 -0
- data/lib/cogworker/web/routes/save_session.rb +21 -0
- data/lib/cogworker/web/routes/scheduled.rb +49 -0
- data/lib/cogworker/web/routes/stats.rb +298 -0
- data/lib/cogworker/web/views.rb +25 -0
- data/lib/cogworker/web.rb +257 -0
- data/lib/cogworker/work.rb +19 -0
- data/lib/cogworker/work_set.rb +23 -0
- data/lib/cogworker/worker.rb +5 -0
- data/lib/cogworker/workers.rb +8 -0
- data/lib/cogworker.rb +114 -0
- metadata +300 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 5aad05a1deaf85505c91f0b6ebe11199fc918b6b3e3831efa5ddb753381cf824
|
|
4
|
+
data.tar.gz: 760b88ec2cc3075f4bbfe152062bc8f1e24b71ce2c71984f9702c0cbdb7da81f
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: fc050c6eff33d2fcc5f61629d46e88452912d3f798c49b49fdc53d21ca525403a575e89e13c9e34307cba312eab7307baebc80a187661a032592fb31eb797854
|
|
7
|
+
data.tar.gz: a9ef2e674633abe4085b6e6e21eb74c3ec735b3ab7c4f6305f274172049e41058477b03e5a5e8105cc6b0afdeadcafc9babba07bbdd81855b54532585a9a946e
|
data/exe/cogworker
ADDED
data/exe/cogworkerswarm
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
# Pops jobs off Redis lists. Queue names repeated in the process's queue
|
|
5
|
+
# list represent weight; since BRPOP itself scans its key list
|
|
6
|
+
# strictly left-to-right, weighting is implemented by shuffling the
|
|
7
|
+
# (already-expanded, so repeats survive) key list before every fetch cycle
|
|
8
|
+
# — the more often a name appears, the more likely it lands first.
|
|
9
|
+
class BasicFetch
|
|
10
|
+
TIMEOUT = 2 # seconds; also how often a stopped/quieted processor notices and exits its fetch loop.
|
|
11
|
+
|
|
12
|
+
def initialize(queues)
|
|
13
|
+
@queue_keys = Array(queues).map { |q| RedisKeys.queue(q) }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def retrieve_work
|
|
17
|
+
keys = @queue_keys.shuffle
|
|
18
|
+
result = Cogworker.config.redis { |c| c.brpop(*keys, timeout: TIMEOUT) }
|
|
19
|
+
return nil unless result
|
|
20
|
+
|
|
21
|
+
queue_key, raw_job = result
|
|
22
|
+
UnitOfWork.new(queue_key.delete_prefix(RedisKeys::QUEUE_PREFIX), raw_job)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
UnitOfWork = Struct.new(:queue, :raw_job)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
|
|
5
|
+
module Cogworker
|
|
6
|
+
# Shared flag parsing + boot sequence for both the single-process and
|
|
7
|
+
# swarm-child entry points: `-e/--environment`, `-c/--concurrency`,
|
|
8
|
+
# `-r/--require <file>`, `-C/--config <file>`, `-L/--logfile <file>`,
|
|
9
|
+
# `-q/--queue <name>[,<weight>]` (repeatable).
|
|
10
|
+
class CLI
|
|
11
|
+
def self.parse(argv)
|
|
12
|
+
options = { queues: [], require_path: nil, environment: ENV['APP_ENV'] || ENV['RACK_ENV'] || 'development' }
|
|
13
|
+
|
|
14
|
+
OptionParser.new do |o|
|
|
15
|
+
o.on('-e ENV', '--environment ENV') { |v| options[:environment] = v }
|
|
16
|
+
o.on('-c INT', '--concurrency INT', Integer) { |v| options[:concurrency] = v }
|
|
17
|
+
o.on('-r PATH', '--require PATH') { |v| options[:require_path] = v }
|
|
18
|
+
o.on('-C PATH', '--config PATH') { |v| options[:config_path] = v }
|
|
19
|
+
o.on('-L PATH', '--logfile PATH') { |v| options[:logfile] = v }
|
|
20
|
+
o.on('-q QUEUE', '--queue QUEUE') { |v| options[:queues].concat(parse_queue_weight(v)) }
|
|
21
|
+
end.parse!(argv.dup)
|
|
22
|
+
|
|
23
|
+
options
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def self.parse_queue_weight(value)
|
|
27
|
+
name, weight = value.split(',')
|
|
28
|
+
Array.new([weight.to_i, 1].max, name)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Boots one OS process fully: applies config, requires app code, and
|
|
32
|
+
# blocks running the Launcher loop until a stop signal is handled.
|
|
33
|
+
# `argv` is parsed independently per call so a swarm parent can boot each
|
|
34
|
+
# forked child with the exact same flags it was started with.
|
|
35
|
+
def run(argv)
|
|
36
|
+
options = self.class.parse(argv)
|
|
37
|
+
file_config = ConfigLoader.load(options[:config_path])
|
|
38
|
+
|
|
39
|
+
Cogworker.server_process!
|
|
40
|
+
Cogworker.config.concurrency = options[:concurrency] || file_config[:concurrency]&.to_i || Cogworker.config.concurrency
|
|
41
|
+
Cogworker.config.queues = options[:queues].any? ? options[:queues] : (file_config[:queues] || Cogworker.config.queues)
|
|
42
|
+
|
|
43
|
+
redirect_logfile(options[:logfile]) if options[:logfile]
|
|
44
|
+
require_app_code(options[:require_path]) if options[:require_path]
|
|
45
|
+
|
|
46
|
+
Launcher.new.run
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def redirect_logfile(path)
|
|
52
|
+
Cogworker.logger = Logging.default_logger(File.open(path, 'a'))
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def require_app_code(path)
|
|
56
|
+
expanded = File.expand_path(path)
|
|
57
|
+
if File.directory?(expanded)
|
|
58
|
+
require File.join(expanded, 'config/environment')
|
|
59
|
+
else
|
|
60
|
+
require expanded
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Cogworker
|
|
6
|
+
# Enqueues jobs: normalizes the job hash, runs the client middleware
|
|
7
|
+
# chain, then pushes onto the target queue or schedule ZSET.
|
|
8
|
+
class Client
|
|
9
|
+
class << self
|
|
10
|
+
# Cogworker::Client.push(hash) — the single funnel every enqueue path
|
|
11
|
+
# (perform_async/perform_in/perform_at, and direct calls) goes through.
|
|
12
|
+
def push(item)
|
|
13
|
+
job = JobUtil.normalize_item(item)
|
|
14
|
+
queue = job['queue']
|
|
15
|
+
|
|
16
|
+
Cogworker.config.client_chain.invoke(job['class'], job, queue, Cogworker.config.redis_pool) do
|
|
17
|
+
dispatch(job)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# UniqueJobs::ClientMiddleware sets this when the job was a
|
|
21
|
+
# duplicate of an already-enqueued/running `unique: :until_executed`
|
|
22
|
+
# job and never actually reached `dispatch` — nil signals "nothing
|
|
23
|
+
# was pushed", same as `job['jid']` would be meaningless otherwise.
|
|
24
|
+
job['unique_skipped'] ? nil : job['jid']
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def push_bulk(items)
|
|
28
|
+
items.map { |item| push(item) }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Testing.fake?/.inline? divert the enqueue entirely; both still ran
|
|
34
|
+
# the client middleware chain above them in #push, same as the real
|
|
35
|
+
# push would.
|
|
36
|
+
def dispatch(job)
|
|
37
|
+
if Testing.fake?
|
|
38
|
+
Testing.jobs_for(job['class']) << job
|
|
39
|
+
elsif Testing.inline?
|
|
40
|
+
Testing.perform_inline(job)
|
|
41
|
+
else
|
|
42
|
+
raw_push(job)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def raw_push(job)
|
|
47
|
+
Cogworker.config.redis do |conn|
|
|
48
|
+
if job['at']
|
|
49
|
+
conn.zadd(RedisKeys::SCHEDULE, job['at'].to_f, JSON.generate(job))
|
|
50
|
+
else
|
|
51
|
+
job['enqueued_at'] = Time.now.to_f
|
|
52
|
+
conn.multi do |pipeline|
|
|
53
|
+
pipeline.sadd(RedisKeys::QUEUES, job['queue'])
|
|
54
|
+
pipeline.lpush(RedisKeys.queue(job['queue']), JSON.generate(job))
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
# Mixin giving any class `#identity`/`#logger`, delegating to the
|
|
5
|
+
# process-global `Cogworker.identity`/`Cogworker.logger`. Deliberately global
|
|
6
|
+
# (not instance-scoped config injection): each OS process — including every
|
|
7
|
+
# cogworkerswarm child — has exactly one Cogworker::Config, so a singleton-backed
|
|
8
|
+
# mixin is sufficient and keeps `include Cogworker::Component` usable with zero
|
|
9
|
+
# wiring, as existing custom middleware (e.g. WorkerKiller) expects.
|
|
10
|
+
module Component
|
|
11
|
+
def identity
|
|
12
|
+
Cogworker.identity
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def logger
|
|
16
|
+
Cogworker.logger
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
# Process-global configuration, built once and mutated in place by
|
|
5
|
+
# (possibly several, per the real init code) `configure_server`/
|
|
6
|
+
# `configure_client` calls, so every block sees and extends the same
|
|
7
|
+
# server/client middleware chains.
|
|
8
|
+
class Config
|
|
9
|
+
attr_reader :server_chain, :client_chain, :periodic_manager, :redis_options
|
|
10
|
+
attr_accessor :concurrency, :queues, :periodic_catch_up, :unique_lock_ttl
|
|
11
|
+
|
|
12
|
+
def initialize
|
|
13
|
+
@server_chain = Middleware::Chain.new
|
|
14
|
+
@client_chain = Middleware::Chain.new
|
|
15
|
+
@periodic_manager = Periodic::Manager.new
|
|
16
|
+
@redis_options = {}
|
|
17
|
+
@concurrency = 10
|
|
18
|
+
@queues = ['default']
|
|
19
|
+
# Default true: an entry's most-recently-due slot fires immediately on
|
|
20
|
+
# a process's first tick, same as always. Set to false to skip that
|
|
21
|
+
# one-time catch-up fire instead — see Periodic::Ticker#priming_first_slot?
|
|
22
|
+
# for why a cold start against an empty/reset Redis otherwise fires
|
|
23
|
+
# every registered entry at once.
|
|
24
|
+
@periodic_catch_up = true
|
|
25
|
+
# Safety-net TTL (seconds) on a regular `unique: :until_executed`
|
|
26
|
+
# job's lock — `UniqueJobs::ReleaseMiddleware` is what actually clears
|
|
27
|
+
# it on success/terminal failure; this only bounds how long a lock
|
|
28
|
+
# can stay stuck if a process dies mid-job before that ever runs
|
|
29
|
+
# (same class of gap as a crashed process losing its in-flight job
|
|
30
|
+
# generally — see the Status section of CLAUDE.md). 24h by default;
|
|
31
|
+
# set higher/lower to match how long a unique job might legitimately
|
|
32
|
+
# run plus however long its retries can take.
|
|
33
|
+
@unique_lock_ttl = 24 * 60 * 60
|
|
34
|
+
register_default_middleware
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def redis=(options)
|
|
38
|
+
@redis_options = options
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def redis_pool
|
|
42
|
+
@redis_pool ||= RedisConnection.create(@redis_options.merge(size: concurrency + 5))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def redis(&block)
|
|
46
|
+
pool = redis_pool
|
|
47
|
+
if block_given?
|
|
48
|
+
pool.with(&block)
|
|
49
|
+
else
|
|
50
|
+
pool
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def server_middleware
|
|
55
|
+
yield server_chain if block_given?
|
|
56
|
+
server_chain
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def client_middleware
|
|
60
|
+
yield client_chain if block_given?
|
|
61
|
+
client_chain
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# `config.periodic(&PERIODIC_JOBS)` — the block receives the manager and
|
|
65
|
+
# registers cron entries synchronously as it runs. No-op-safe: an app
|
|
66
|
+
# that never calls this simply has an empty periodic_manager, and the
|
|
67
|
+
# Ticker (started later, per process) has nothing to do.
|
|
68
|
+
def periodic(&block)
|
|
69
|
+
block&.call(periodic_manager)
|
|
70
|
+
periodic_manager
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
# Always present, regardless of whether config.periodic/a unique job is
|
|
76
|
+
# ever used — each is a no-op for a job that doesn't carry its own
|
|
77
|
+
# marker (`periodic_pjid`/`unique: :until_executed`).
|
|
78
|
+
def register_default_middleware
|
|
79
|
+
@server_chain.add(Periodic::ReleaseMiddleware)
|
|
80
|
+
@client_chain.add(UniqueJobs::ClientMiddleware)
|
|
81
|
+
@server_chain.add(UniqueJobs::ReleaseMiddleware)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'yaml'
|
|
4
|
+
require 'erb'
|
|
5
|
+
|
|
6
|
+
module Cogworker
|
|
7
|
+
# Loads the process config file (`:concurrency:`/`:queues:` YAML, symbol
|
|
8
|
+
# keys, ERB-interpolated before parsing — e.g. `<%= ENV["CONCURRENCY"] %>`).
|
|
9
|
+
# A queue name repeated in `:queues:` is weight, not a duplicate: the
|
|
10
|
+
# repetition survives untouched here and is what BasicFetch's per-cycle
|
|
11
|
+
# shuffle uses to weight fetches.
|
|
12
|
+
module ConfigLoader
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def load(path)
|
|
16
|
+
return {} unless path
|
|
17
|
+
|
|
18
|
+
erb_result = ERB.new(File.read(path)).result
|
|
19
|
+
YAML.safe_load(erb_result, permitted_classes: [Symbol], aliases: true) || {}
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'redis'
|
|
5
|
+
|
|
6
|
+
module Cogworker
|
|
7
|
+
# Publishes this process's presence (for ProcessSet) and listens for
|
|
8
|
+
# remote quiet!/stop! requests (for Process#quiet!/#stop! issued by another
|
|
9
|
+
# process, e.g. a self-targeting WorkerKiller or the Web UI). Both the
|
|
10
|
+
# heartbeat loop and the pub/sub subscriber must only start *after* a
|
|
11
|
+
# cogworkerswarm fork, never before, in the parent — otherwise the thread
|
|
12
|
+
# simply doesn't exist in the child, and a lock that thread held could
|
|
13
|
+
# leave the child permanently deadlocked.
|
|
14
|
+
class Heartbeat
|
|
15
|
+
INTERVAL = 5
|
|
16
|
+
TTL = 60
|
|
17
|
+
|
|
18
|
+
def initialize(manager)
|
|
19
|
+
@manager = manager
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def start!
|
|
23
|
+
@stopping = false
|
|
24
|
+
@beat_thread = Thread.new { beat_loop }
|
|
25
|
+
@signal_thread = Thread.new { subscribe_loop }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Tears down both background threads, not just the Redis keys — leaving
|
|
29
|
+
# them running would leak a thread (and a dedicated pub/sub connection)
|
|
30
|
+
# per Heartbeat instance for the remaining life of the process.
|
|
31
|
+
def stop!
|
|
32
|
+
@stopping = true
|
|
33
|
+
@beat_thread&.kill
|
|
34
|
+
@subscribe_client&.unsubscribe
|
|
35
|
+
@signal_thread&.kill
|
|
36
|
+
cleanup_presence!
|
|
37
|
+
rescue StandardError
|
|
38
|
+
nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def cleanup_presence!
|
|
44
|
+
Cogworker.config.redis do |c|
|
|
45
|
+
c.del(RedisKeys.process(Cogworker.identity), RedisKeys.workers(Cogworker.identity))
|
|
46
|
+
c.srem(RedisKeys::PROCESSES, Cogworker.identity)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def beat_loop
|
|
51
|
+
loop do
|
|
52
|
+
beat
|
|
53
|
+
sleep(INTERVAL)
|
|
54
|
+
end
|
|
55
|
+
rescue StandardError => e
|
|
56
|
+
Cogworker.logger.error { "Heartbeat died: #{e.class}: #{e.message}" }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def beat
|
|
60
|
+
identity = Cogworker.identity
|
|
61
|
+
info = {
|
|
62
|
+
'hostname' => Cogworker.hostname,
|
|
63
|
+
'pid' => ::Process.pid,
|
|
64
|
+
'concurrency' => Cogworker.config.concurrency,
|
|
65
|
+
'queues' => @manager.queues,
|
|
66
|
+
'started_at' => (@started_at ||= Time.now.to_f),
|
|
67
|
+
'rss_kb' => current_rss_kb
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
Cogworker.config.redis do |c|
|
|
71
|
+
c.sadd(RedisKeys::PROCESSES, identity)
|
|
72
|
+
c.hset(RedisKeys.process(identity),
|
|
73
|
+
'info', JSON.generate(info),
|
|
74
|
+
'busy', @manager.busy_count.to_s,
|
|
75
|
+
'quiet', @manager.quiet?.to_s)
|
|
76
|
+
c.expire(RedisKeys.process(identity), TTL)
|
|
77
|
+
c.expire(RedisKeys.workers(identity), TTL)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Current resident set size, in KB — read straight from the kernel
|
|
82
|
+
# (`/proc/self/status`, present on any real Linux deployment target)
|
|
83
|
+
# when available, since that's a plain file read with no extra process;
|
|
84
|
+
# `ps` (a real fork+exec every heartbeat) is only the fallback for
|
|
85
|
+
# platforms without `/proc` (macOS in local dev). `nil` — not `0` — on
|
|
86
|
+
# any failure, so the Web UI can render "n/a" rather than a misleading
|
|
87
|
+
# "0M" if this ever can't be measured.
|
|
88
|
+
def current_rss_kb
|
|
89
|
+
proc_status = '/proc/self/status'
|
|
90
|
+
if File.readable?(proc_status)
|
|
91
|
+
matched = File.read(proc_status)[/^VmRSS:\s+(\d+)\s+kB/, 1]
|
|
92
|
+
return matched.to_i if matched
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
out = `ps -o rss= -p #{::Process.pid} 2>/dev/null`.strip
|
|
96
|
+
out.empty? ? nil : out.to_i
|
|
97
|
+
rescue StandardError
|
|
98
|
+
nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Uses its own dedicated (non-pooled) Redis client: SUBSCRIBE blocks the
|
|
102
|
+
# connection for as long as the subscription is open, which would starve
|
|
103
|
+
# the shared job-execution pool if it borrowed a connection from there.
|
|
104
|
+
def subscribe_loop
|
|
105
|
+
@subscribe_client = ::Redis.new(RedisConnection.client_options(Cogworker.config.redis_options))
|
|
106
|
+
@subscribe_client.subscribe(RedisKeys.signal(Cogworker.identity)) do |on|
|
|
107
|
+
on.message do |_channel, message|
|
|
108
|
+
dispatch(message)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
rescue StandardError => e
|
|
112
|
+
return if @stopping
|
|
113
|
+
|
|
114
|
+
Cogworker.logger.error { "Signal subscriber died: #{e.class}: #{e.message}" }
|
|
115
|
+
sleep(1)
|
|
116
|
+
retry
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def dispatch(message)
|
|
120
|
+
case message
|
|
121
|
+
when 'quiet' then @manager.quiet!
|
|
122
|
+
when 'stop' then remote_stop!
|
|
123
|
+
else Cogworker.logger.warn { "Unknown signal message: #{message}" }
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# A *remote* stop (published by another process — typically the Web
|
|
128
|
+
# UI's `Process#stop!` — as opposed to this process's own `TERM`/`INT`,
|
|
129
|
+
# handled by `Launcher#stop!`/`#watch_signals` on the main thread) has
|
|
130
|
+
# to actually end this OS process, not just quiesce the Manager
|
|
131
|
+
# forever: nothing here naturally returns from a running `Launcher#run`
|
|
132
|
+
# loop the way breaking out of `watch_signals` does — this dispatch
|
|
133
|
+
# runs on the pub/sub subscriber's own background thread, so without an
|
|
134
|
+
# explicit exit the process would otherwise sit there, still
|
|
135
|
+
# heartbeating, permanently "quiet", never picking up work again,
|
|
136
|
+
# until someone `kill`s it for real or the swarm is restarted.
|
|
137
|
+
# `::Process.exit!` (not `Kernel#exit`, which only raises `SystemExit`
|
|
138
|
+
# on the *calling* thread and wouldn't actually end the process from
|
|
139
|
+
# here) terminates immediately and unconditionally, from any thread —
|
|
140
|
+
# deliberately *after* `@manager.stop!` has already drained in-flight
|
|
141
|
+
# work and `cleanup_presence!` has already removed this process from
|
|
142
|
+
# Redis, so both happen before the process can vanish out from under
|
|
143
|
+
# them.
|
|
144
|
+
def remote_stop!
|
|
145
|
+
@manager.stop!
|
|
146
|
+
cleanup_presence!
|
|
147
|
+
::Process.exit!(true)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
module History
|
|
5
|
+
# Server middleware: times every run and records it via Storage,
|
|
6
|
+
# success or failure, regardless of what other middleware/worker code does.
|
|
7
|
+
class Middleware
|
|
8
|
+
def call(_worker, job, queue)
|
|
9
|
+
started_at = Time.now.to_f
|
|
10
|
+
yield
|
|
11
|
+
Storage.record(job, queue, started_at, Time.now.to_f, 'success')
|
|
12
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
13
|
+
Storage.record(job, queue, started_at, Time.now.to_f, 'failed', error: e)
|
|
14
|
+
raise e
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Cogworker
|
|
6
|
+
module History
|
|
7
|
+
# Three parallel ZSETs (score = finished_at), so any of the three
|
|
8
|
+
# filters (all/success/failed) is a plain `ZREVRANGE` — no per-request
|
|
9
|
+
# `ZUNIONSTORE`/scan needed. Each is trimmed to `History.max_entries`
|
|
10
|
+
# independently on every write, oldest first.
|
|
11
|
+
module Storage
|
|
12
|
+
LIST_KEYS = {
|
|
13
|
+
'all' => 'cogworker:history:all',
|
|
14
|
+
'success' => 'cogworker:history:success',
|
|
15
|
+
'failed' => 'cogworker:history:failed'
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
def record(job, queue, started_at, finished_at, status, error: nil)
|
|
21
|
+
entry = {
|
|
22
|
+
'jid' => job['jid'], 'class' => job['class'], 'queue' => queue, 'args' => job['args'],
|
|
23
|
+
'status' => status, 'started_at' => started_at, 'finished_at' => finished_at
|
|
24
|
+
}
|
|
25
|
+
if error
|
|
26
|
+
entry['error_class'] = error.class.name
|
|
27
|
+
entry['error_message'] = error.message.to_s[0, 10_000]
|
|
28
|
+
entry['backtrace'] = (error.backtrace || []).first(200)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
raw = JSON.generate(entry)
|
|
32
|
+
max = Cogworker::History.max_entries
|
|
33
|
+
Cogworker.config.redis do |c|
|
|
34
|
+
write_and_trim(c, LIST_KEYS.fetch('all'), raw, finished_at, max)
|
|
35
|
+
write_and_trim(c, LIST_KEYS.fetch(status), raw, finished_at, max)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Newest-first page of `count`/`status` entries: `[entries, total]`.
|
|
40
|
+
def page(status, page_number, per_page)
|
|
41
|
+
key = LIST_KEYS.fetch(status, LIST_KEYS.fetch('all'))
|
|
42
|
+
start = [(page_number - 1), 0].max * per_page
|
|
43
|
+
stop = start + per_page - 1
|
|
44
|
+
|
|
45
|
+
Cogworker.config.redis do |c|
|
|
46
|
+
total = c.zcard(key)
|
|
47
|
+
raw_entries = c.zrevrange(key, start, stop)
|
|
48
|
+
[raw_entries.map { |raw| JSON.parse(raw) }, total]
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def write_and_trim(conn, key, raw, score, max)
|
|
53
|
+
conn.zadd(key, score, raw)
|
|
54
|
+
conn.zremrangebyrank(key, 0, -(max + 1))
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Success/failed counts per UTC calendar day, for the last `days` days
|
|
58
|
+
# (today included) — `{ 'YYYY-MM-DD' => { 'success' => n, 'failed' => n } }`,
|
|
59
|
+
# a day with no entries simply absent from the Hash. Reads the shared
|
|
60
|
+
# "all" list once (rather than "success" and "failed" separately) since
|
|
61
|
+
# every entry already carries its own `status`. Bucketing uses UTC,
|
|
62
|
+
# not the viewer's browser timezone (unlike `Layout.time_tag`
|
|
63
|
+
# elsewhere) — this is a server-rendered daily aggregate, not a single
|
|
64
|
+
# instant, so there's no one "browser day" to convert into.
|
|
65
|
+
# Same caveat as any other read of this list: only entries still
|
|
66
|
+
# within `History.max_entries` are counted, so a very busy queue's
|
|
67
|
+
# oldest requested days may already have been trimmed away.
|
|
68
|
+
def daily_counts(days)
|
|
69
|
+
since = Time.now.to_f - (days * 86_400)
|
|
70
|
+
Cogworker.config.redis do |c|
|
|
71
|
+
c.zrangebyscore(LIST_KEYS.fetch('all'), since, '+inf').each_with_object({}) do |raw, counts|
|
|
72
|
+
entry = JSON.parse(raw)
|
|
73
|
+
day = Time.at(entry['finished_at']).utc.strftime('%Y-%m-%d')
|
|
74
|
+
bucket = (counts[day] ||= { 'success' => 0, 'failed' => 0 })
|
|
75
|
+
bucket[entry['status']] += 1 if bucket.key?(entry['status'])
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
# Execution history: every job run (success and failure alike) is recorded
|
|
5
|
+
# with its full args, timing, and — on failure — error class/message/
|
|
6
|
+
# backtrace. Distinct from the Status layer (`Cogworker::Status`), which
|
|
7
|
+
# tracks *current* per-jid state with a TTL; History is an append-only,
|
|
8
|
+
# length-capped log meant for browsing/auditing past runs.
|
|
9
|
+
module History
|
|
10
|
+
DEFAULT_MAX_ENTRIES = 1000
|
|
11
|
+
|
|
12
|
+
class << self
|
|
13
|
+
# How many entries are retained (oldest trimmed first), independently
|
|
14
|
+
# for the "all", "success", and "failed" lists — read fresh on every
|
|
15
|
+
# write, so changing it takes effect immediately, no need to rebuild
|
|
16
|
+
# the middleware chain. Configurable via
|
|
17
|
+
# `configure_server_middleware(config, max_entries: N)`, or directly:
|
|
18
|
+
# `Cogworker::History.max_entries = 5000`.
|
|
19
|
+
attr_writer :max_entries
|
|
20
|
+
|
|
21
|
+
def max_entries
|
|
22
|
+
@max_entries ||= DEFAULT_MAX_ENTRIES
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
def configure_server_middleware(config, max_entries: DEFAULT_MAX_ENTRIES)
|
|
29
|
+
self.max_entries = max_entries
|
|
30
|
+
config.server_middleware { |chain| chain.add(Middleware) }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Cogworker
|
|
4
|
+
# `include Cogworker::Worker` (alias `Cogworker::Job`) DSL.
|
|
5
|
+
module Job
|
|
6
|
+
def self.included(base)
|
|
7
|
+
base.include(Component)
|
|
8
|
+
base.extend(ClassMethods)
|
|
9
|
+
base.include(InstanceMethods)
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# `cogworker_options`/`perform_async`/`perform_in`/`perform_at`, extended
|
|
13
|
+
# onto the including class.
|
|
14
|
+
module ClassMethods
|
|
15
|
+
# Accepts a plain Hash (both `key: value` and `:key => value` call-site
|
|
16
|
+
# syntax already collect into the same Hash literal in Ruby — nothing
|
|
17
|
+
# special to implement). Merges into the class's option set rather than
|
|
18
|
+
# replacing it, and never filters keys: arbitrary custom options (e.g.
|
|
19
|
+
# `lock_run: :while_executing`) ride along unchanged into the job hash.
|
|
20
|
+
def cogworker_options(opts = {})
|
|
21
|
+
cogworker_options_hash.merge!(opts.transform_keys(&:to_sym))
|
|
22
|
+
cogworker_options_hash
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def cogworker_options_hash
|
|
26
|
+
@cogworker_options_hash ||= if superclass.respond_to?(:cogworker_options_hash)
|
|
27
|
+
superclass.cogworker_options_hash.dup
|
|
28
|
+
else
|
|
29
|
+
{}
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def perform_async(*args)
|
|
34
|
+
Client.push(job_payload('args' => args))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def perform_in(interval, *args)
|
|
38
|
+
ts = interval_to_ts(interval)
|
|
39
|
+
Client.push(job_payload('args' => args, 'at' => ts))
|
|
40
|
+
end
|
|
41
|
+
alias perform_at perform_in
|
|
42
|
+
|
|
43
|
+
# `SomeJob.jobs`/`SomeJob.clear` — reads/clears this class's entries in
|
|
44
|
+
# `Cogworker::Testing`'s fake queue (empty outside `Testing.fake!`).
|
|
45
|
+
def jobs
|
|
46
|
+
Testing.jobs_for(name)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def clear
|
|
50
|
+
Testing.jobs_for(name).clear
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def job_payload(extra)
|
|
56
|
+
cogworker_options_hash.transform_keys(&:to_s).merge('class' => name).merge(extra)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def interval_to_ts(interval)
|
|
60
|
+
numeric = interval.respond_to?(:to_f) ? interval.to_f : interval.to_time.to_f
|
|
61
|
+
# A value below a billion is treated as "seconds from now"
|
|
62
|
+
# (perform_in), anything
|
|
63
|
+
# at or above it as an absolute unix timestamp (perform_at, or a
|
|
64
|
+
# Time passed straight through — Time#to_f is already an epoch
|
|
65
|
+
# timestamp, always well above the threshold).
|
|
66
|
+
numeric < 1_000_000_000 ? Time.now.to_f + numeric : numeric
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# `jid`, included onto the including class's instances.
|
|
71
|
+
module InstanceMethods
|
|
72
|
+
attr_accessor :jid
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|