monkrb 0.23.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +53 -0
- data/README.md +3 -2
- data/lib/monk/generators/presence.rb +22 -0
- data/lib/monk/generators.rb +1 -1
- data/lib/monk/live/client/monk_live.js +159 -7
- data/lib/monk/live/client/protocol.js +37 -0
- data/lib/monk/live/session.rb +59 -2
- data/lib/monk/presence/counts.rb +118 -0
- data/lib/monk/presence/reactor.rb +126 -0
- data/lib/monk/presence/store.rb +271 -0
- data/lib/monk/presence/tracker.rb +111 -0
- data/lib/monk/presence.rb +129 -0
- data/lib/monk/templates/base/Gemfile +1 -1
- data/lib/monk/templates/base/config/load.rb +1 -1
- data/lib/monk/templates/base/config.ru +8 -1
- data/lib/monk/templates/presence/agents.md +29 -0
- data/lib/monk/templates/presence/app/views/presence/_dot.erb +1 -0
- data/lib/monk/templates/presence/config/presence.rb +52 -0
- data/lib/monk/templates/presence/db/migrate/00000000000005_create_presence_tables.down.sql +3 -0
- data/lib/monk/templates/presence/db/migrate/00000000000005_create_presence_tables.up.sql +36 -0
- data/lib/monk/templates/presence/setup.md +51 -0
- data/lib/monk/templates/presence/test/presence_test.rb +13 -0
- data/lib/monk/templates/websocket/bin/websocket_server +29 -13
- data/lib/monk/version.rb +1 -1
- data/lib/monk/websocket/connection.rb +108 -5
- data/lib/monk/websocket/server.rb +38 -12
- metadata +14 -1
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "pg"
|
|
3
|
+
require_relative "../log"
|
|
4
|
+
require_relative "../persistence/pg"
|
|
5
|
+
require_relative "store"
|
|
6
|
+
|
|
7
|
+
module Monk
|
|
8
|
+
module Presence
|
|
9
|
+
# bin/server's side of presence (ADR 0019): a Ractor that reacts to the
|
|
10
|
+
# WebSocket processes' signals by calling the app's on_change with the
|
|
11
|
+
# changed subjects' current status. Every web process starts one; only
|
|
12
|
+
# the one holding a session advisory lock LISTENs, the others retry, and
|
|
13
|
+
# take over when the holder's connection ends (Postgres frees the lock).
|
|
14
|
+
#
|
|
15
|
+
# A signal sent while no reactor listens is lost: on_change is at most
|
|
16
|
+
# once, and a page catches up on its next resync.
|
|
17
|
+
#
|
|
18
|
+
# Plain module methods, like PgFanout.subscribe: the Ractor has no
|
|
19
|
+
# instance to call into.
|
|
20
|
+
module Reactor
|
|
21
|
+
LOCK_SQL = "SELECT pg_try_advisory_lock(hashtext('monk_presence_reactor'))".freeze
|
|
22
|
+
|
|
23
|
+
# Postgres learns that a reactor's host vanished (a lost node, a cut
|
|
24
|
+
# network: no FIN, no RST) only when TCP keepalive gives up, after
|
|
25
|
+
# about 2 hours with Linux's defaults, and until then the dead session
|
|
26
|
+
# holds the lock and no web process reacts. These make it about 25 s:
|
|
27
|
+
# 10 s idle, then 3 probes 5 s apart. Server settings any user may
|
|
28
|
+
# set for their own session; the client-side ones in .connect do the
|
|
29
|
+
# same for a reactor waiting on a Postgres host that vanished.
|
|
30
|
+
KEEPALIVE_SQL = "SET tcp_keepalives_idle = 10; SET tcp_keepalives_interval = 5; " \
|
|
31
|
+
"SET tcp_keepalives_count = 3".freeze
|
|
32
|
+
CLIENT_KEEPALIVES = { keepalives: 1, keepalives_idle: 10, keepalives_interval: 5, keepalives_count: 3 }.freeze
|
|
33
|
+
|
|
34
|
+
# How often a waiting reactor checks whether it was stopped.
|
|
35
|
+
POLL = 0.5
|
|
36
|
+
|
|
37
|
+
Handle = Data.define(:ractor, :stop_port)
|
|
38
|
+
|
|
39
|
+
# pg_opts: the database's connection options; on_change: a shareable
|
|
40
|
+
# block. Returns a handle for .stop.
|
|
41
|
+
#
|
|
42
|
+
# One thread: a second one, waiting in Postgres, kept the Ractor
|
|
43
|
+
# from handling a stop message about half the time. So the reactor
|
|
44
|
+
# waits in short steps and stops once its stop port is closed.
|
|
45
|
+
def self.start(pg_opts, on_change, retry_interval:)
|
|
46
|
+
stop_port = Ractor::Port.new
|
|
47
|
+
ractor = Ractor.new(Ractor.make_shareable(pg_opts.dup), Ractor.make_shareable(on_change), retry_interval,
|
|
48
|
+
stop_port, name: "monk-presence-reactor",) do |opts, hook, interval, stop|
|
|
49
|
+
Monk::Presence::Reactor.run(opts, hook, interval, stop)
|
|
50
|
+
end
|
|
51
|
+
Handle.new(ractor: ractor, stop_port: stop_port)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Ends the reactor and closes its connection, which frees the lock.
|
|
55
|
+
def self.stop(handle)
|
|
56
|
+
handle.stop_port.close
|
|
57
|
+
handle.ractor.join
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.run(pg_opts, on_change, retry_interval, stop = nil)
|
|
61
|
+
stopped = -> { stop&.closed? }
|
|
62
|
+
until stopped.call
|
|
63
|
+
conn = nil
|
|
64
|
+
begin
|
|
65
|
+
conn = connect(pg_opts)
|
|
66
|
+
next unless lock_acquired?(conn, retry_interval, stopped)
|
|
67
|
+
|
|
68
|
+
conn.exec("LISTEN #{Store::CHANNEL}")
|
|
69
|
+
log(:info, "Monk::Presence: this process reacts to presence changes (pid #{Process.pid})")
|
|
70
|
+
conn.wait_for_notify(POLL) { |_channel, _pid, payload| react(on_change, payload) } until stopped.call
|
|
71
|
+
rescue PG::Error => e
|
|
72
|
+
log(:error, "Monk::Presence reactor: #{e.class}: #{e.message.lines.first&.chomp}; " \
|
|
73
|
+
"retrying in #{retry_interval} s",)
|
|
74
|
+
pause(retry_interval, stopped)
|
|
75
|
+
ensure
|
|
76
|
+
conn&.close
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The reactor's own connection, kept apart from Monk::Persistence's:
|
|
82
|
+
# the lock and the LISTEN belong to its session (so not through
|
|
83
|
+
# PgBouncer in transaction mode).
|
|
84
|
+
def self.connect(pg_opts)
|
|
85
|
+
conn = PG.connect(connect_timeout: Monk::Persistence::Pg::DEFAULT_CONNECT_TIMEOUT, **CLIENT_KEEPALIVES,
|
|
86
|
+
**pg_opts,)
|
|
87
|
+
conn.exec(KEEPALIVE_SQL)
|
|
88
|
+
conn
|
|
89
|
+
rescue PG::Error
|
|
90
|
+
conn&.close
|
|
91
|
+
raise
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# True once this connection holds the lock; false if stopped first.
|
|
95
|
+
def self.lock_acquired?(conn, retry_interval, stopped)
|
|
96
|
+
until conn.exec(LOCK_SQL).getvalue(0, 0) == "t"
|
|
97
|
+
pause(retry_interval, stopped)
|
|
98
|
+
return false if stopped.call
|
|
99
|
+
end
|
|
100
|
+
true
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def self.pause(seconds, stopped)
|
|
104
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
|
|
105
|
+
until stopped.call || (left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)) <= 0
|
|
106
|
+
sleep [left, POLL].min
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# One signal. Anything that goes wrong drops this signal only.
|
|
111
|
+
def self.react(on_change, payload)
|
|
112
|
+
on_change.call(Monk::Presence.status(JSON.parse(payload)))
|
|
113
|
+
rescue StandardError => e
|
|
114
|
+
log(:error, "Monk::Presence on_change failed: #{e.class}: #{e.message.lines.first&.chomp}")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Monk::Log writes from a Ractor once Monk has booted (bin/server
|
|
118
|
+
# starts the reactor after Monk.boot). Logging must never end it.
|
|
119
|
+
def self.log(level, message)
|
|
120
|
+
Monk::Log.public_send(level, message)
|
|
121
|
+
rescue StandardError
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require_relative "../log"
|
|
3
|
+
require_relative "../persistence/pg"
|
|
4
|
+
|
|
5
|
+
module Monk
|
|
6
|
+
module Presence
|
|
7
|
+
# One subject's change of status, as settling returned it: status is
|
|
8
|
+
# :online, :away or :offline, last_seen_at a Time when :offline.
|
|
9
|
+
Transition = Data.define(:subject, :status, :last_seen_at)
|
|
10
|
+
|
|
11
|
+
# Presence's statements on Postgres: monk_presence_processes,
|
|
12
|
+
# monk_presence and monk_presence_status
|
|
13
|
+
# (docs/adr/0019-presence-per-process-rows-in-postgres-settled-once-per-transition.md).
|
|
14
|
+
# Holds only settings and freezes itself, so one instance is shareable
|
|
15
|
+
# with every Ractor; each Ractor reaches the database through its own
|
|
16
|
+
# connection, via Monk::Persistence::Pg, as Monk::Jobs' adapter does.
|
|
17
|
+
#
|
|
18
|
+
# Every time is the database's now(), so hosts' clocks don't matter.
|
|
19
|
+
class Store
|
|
20
|
+
ARRAY = Ractor.make_shareable(PG::TextEncoder::Array.new)
|
|
21
|
+
|
|
22
|
+
# Whether a row counts toward its subject's status: an open one
|
|
23
|
+
# while its process heartbeats, a gone one for the grace after it
|
|
24
|
+
# went, whatever its process does since (a stopping process's rows
|
|
25
|
+
# carry its subjects through a deploy). A row whose process row is
|
|
26
|
+
# gone counts only if it's inside the grace. $grace and $timeout are
|
|
27
|
+
# replaced with each statement's own parameter numbers.
|
|
28
|
+
COUNTING = <<~SQL.freeze
|
|
29
|
+
CASE WHEN r.gone_at IS NULL
|
|
30
|
+
THEN COALESCE(p.heartbeat_at >= now() - $timeout::float8 * interval '1 second', false)
|
|
31
|
+
ELSE r.gone_at >= now() - $grace::float8 * interval '1 second'
|
|
32
|
+
END
|
|
33
|
+
SQL
|
|
34
|
+
|
|
35
|
+
# The best of the counting rows: online over away, offline without.
|
|
36
|
+
BEST = <<~SQL.freeze
|
|
37
|
+
CASE WHEN bool_or(r.counting AND r.status = 'online') THEN 'online'
|
|
38
|
+
WHEN bool_or(r.counting) THEN 'away'
|
|
39
|
+
ELSE 'offline'
|
|
40
|
+
END
|
|
41
|
+
SQL
|
|
42
|
+
|
|
43
|
+
REGISTER_PROCESS_SQL = <<~SQL.freeze
|
|
44
|
+
INSERT INTO monk_presence_processes (hostname, pid) VALUES ($1, $2) RETURNING id
|
|
45
|
+
SQL
|
|
46
|
+
|
|
47
|
+
# An upsert, as Monk::Jobs': a process that stalled long enough to be
|
|
48
|
+
# pruned gets its row back under the same id. Says whether it had to
|
|
49
|
+
# (xmax is 0 on a freshly inserted row): its own rows went with it,
|
|
50
|
+
# so it writes them all again.
|
|
51
|
+
HEARTBEAT_SQL = <<~SQL.freeze
|
|
52
|
+
INSERT INTO monk_presence_processes (id, hostname, pid) OVERRIDING SYSTEM VALUE VALUES ($1, $2, $3)
|
|
53
|
+
ON CONFLICT (id) DO UPDATE SET heartbeat_at = now()
|
|
54
|
+
RETURNING xmax = 0 AS restored
|
|
55
|
+
SQL
|
|
56
|
+
|
|
57
|
+
# One process's changes since its last flush, in one statement:
|
|
58
|
+
# subjects with sockets ($2, their statuses in $3) upserted back to
|
|
59
|
+
# open, subjects down to none ($4) marked gone. The lists never share
|
|
60
|
+
# a subject.
|
|
61
|
+
FLUSH_SQL = <<~SQL.freeze
|
|
62
|
+
WITH open AS (
|
|
63
|
+
INSERT INTO monk_presence (process_id, subject, status)
|
|
64
|
+
SELECT $1, subject, status FROM unnest($2::text[], $3::text[]) AS u(subject, status)
|
|
65
|
+
ON CONFLICT (process_id, subject) DO UPDATE SET status = excluded.status, gone_at = NULL
|
|
66
|
+
)
|
|
67
|
+
UPDATE monk_presence SET gone_at = now()
|
|
68
|
+
WHERE process_id = $1 AND subject = ANY($4::text[]) AND gone_at IS NULL
|
|
69
|
+
SQL
|
|
70
|
+
|
|
71
|
+
# This process's rows written again from its whole state ($2, $3):
|
|
72
|
+
# after an error lost a flush, or after its rows were pruned. A row
|
|
73
|
+
# not in the state is marked gone, so a close the lost flush carried
|
|
74
|
+
# isn't left open.
|
|
75
|
+
REWRITE_SQL = <<~SQL.freeze
|
|
76
|
+
WITH open AS (
|
|
77
|
+
INSERT INTO monk_presence (process_id, subject, status)
|
|
78
|
+
SELECT $1, subject, status FROM unnest($2::text[], $3::text[]) AS u(subject, status)
|
|
79
|
+
ON CONFLICT (process_id, subject) DO UPDATE SET status = excluded.status, gone_at = NULL
|
|
80
|
+
)
|
|
81
|
+
UPDATE monk_presence SET gone_at = now()
|
|
82
|
+
WHERE process_id = $1 AND gone_at IS NULL AND subject <> ALL($2::text[])
|
|
83
|
+
SQL
|
|
84
|
+
|
|
85
|
+
# A stopping process (TERM): its rows count for the grace instead of
|
|
86
|
+
# vanishing, while its pages reconnect elsewhere.
|
|
87
|
+
MARK_ALL_GONE_SQL = <<~SQL.freeze
|
|
88
|
+
UPDATE monk_presence SET gone_at = now() WHERE process_id = $1 AND gone_at IS NULL
|
|
89
|
+
SQL
|
|
90
|
+
|
|
91
|
+
# last_seen_at: rows that stopped counting but aren't settled yet
|
|
92
|
+
# (up to a tick) say it, as settling will; else the stored one.
|
|
93
|
+
STATUS_SQL = <<~SQL.freeze
|
|
94
|
+
WITH r AS (
|
|
95
|
+
SELECT r.subject, r.status, r.gone_at, p.heartbeat_at,
|
|
96
|
+
#{COUNTING.gsub("$timeout", "$3").gsub("$grace", "$2")} AS counting
|
|
97
|
+
FROM monk_presence r LEFT JOIN monk_presence_processes p ON p.id = r.process_id
|
|
98
|
+
WHERE r.subject = ANY($1::text[])
|
|
99
|
+
)
|
|
100
|
+
SELECT s.subject, #{BEST} AS status,
|
|
101
|
+
extract(epoch FROM COALESCE(
|
|
102
|
+
max(COALESCE(r.gone_at, r.heartbeat_at)) FILTER (WHERE NOT r.counting), st.last_seen_at
|
|
103
|
+
))::float8 AS last_seen_at
|
|
104
|
+
FROM unnest($1::text[]) AS s(subject)
|
|
105
|
+
LEFT JOIN r ON r.subject = s.subject
|
|
106
|
+
LEFT JOIN monk_presence_status st ON st.subject = s.subject
|
|
107
|
+
GROUP BY s.subject, st.last_seen_at
|
|
108
|
+
SQL
|
|
109
|
+
|
|
110
|
+
# The subjects given ($1), those with rows that stopped counting, and
|
|
111
|
+
# with $4 every subject stored as not offline (the minute's full
|
|
112
|
+
# reconcile), each settled into monk_presence_status where its status
|
|
113
|
+
# changed; the expired rows and dead processes deleted, all in one
|
|
114
|
+
# statement. The upsert's WHERE makes a second settling of the same
|
|
115
|
+
# change find nothing to write, so each transition is returned once.
|
|
116
|
+
# A subject never stored isn't stored as offline: there's nothing to
|
|
117
|
+
# announce. Going offline, last seen is when its last rows stopped:
|
|
118
|
+
# the latest gone_at, or a dead process's last heartbeat.
|
|
119
|
+
SETTLE_SQL = <<~SQL.freeze
|
|
120
|
+
WITH r AS (
|
|
121
|
+
SELECT r.process_id, r.subject, r.status, r.gone_at, p.heartbeat_at,
|
|
122
|
+
#{COUNTING.gsub("$timeout", "$3").gsub("$grace", "$2")} AS counting
|
|
123
|
+
FROM monk_presence r LEFT JOIN monk_presence_processes p ON p.id = r.process_id
|
|
124
|
+
), candidates AS (
|
|
125
|
+
SELECT unnest($1::text[]) AS subject
|
|
126
|
+
UNION SELECT subject FROM r WHERE NOT counting
|
|
127
|
+
UNION SELECT subject FROM monk_presence_status WHERE $4::boolean AND status <> 'offline'
|
|
128
|
+
), computed AS (
|
|
129
|
+
SELECT c.subject, #{BEST} AS status,
|
|
130
|
+
max(COALESCE(r.gone_at, r.heartbeat_at)) FILTER (WHERE NOT r.counting) AS last_seen_at
|
|
131
|
+
FROM candidates c LEFT JOIN r ON r.subject = c.subject
|
|
132
|
+
GROUP BY c.subject
|
|
133
|
+
), expired AS (
|
|
134
|
+
DELETE FROM monk_presence m USING r
|
|
135
|
+
WHERE NOT r.counting AND m.process_id = r.process_id AND m.subject = r.subject
|
|
136
|
+
), dead AS (
|
|
137
|
+
DELETE FROM monk_presence_processes WHERE heartbeat_at < now() - $3::float8 * interval '1 second'
|
|
138
|
+
)
|
|
139
|
+
INSERT INTO monk_presence_status AS s (subject, status, changed_at, last_seen_at)
|
|
140
|
+
SELECT c.subject, c.status, now(), CASE WHEN c.status = 'offline' THEN COALESCE(c.last_seen_at, now()) END
|
|
141
|
+
FROM computed c
|
|
142
|
+
WHERE c.status <> 'offline' OR EXISTS (SELECT 1 FROM monk_presence_status x WHERE x.subject = c.subject)
|
|
143
|
+
ON CONFLICT (subject) DO UPDATE SET
|
|
144
|
+
status = excluded.status, changed_at = excluded.changed_at,
|
|
145
|
+
last_seen_at = COALESCE(excluded.last_seen_at, s.last_seen_at)
|
|
146
|
+
WHERE s.status IS DISTINCT FROM excluded.status
|
|
147
|
+
RETURNING s.subject, s.status,
|
|
148
|
+
CASE WHEN s.status = 'offline' THEN extract(epoch FROM s.last_seen_at)::float8 END AS last_seen_at
|
|
149
|
+
SQL
|
|
150
|
+
|
|
151
|
+
# Settling processes take turns: each statement then sees what the
|
|
152
|
+
# one before it committed, so two can't settle one subject from
|
|
153
|
+
# snapshots taken before each other's flush.
|
|
154
|
+
SETTLE_LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtext('monk_presence_settle'))".freeze
|
|
155
|
+
|
|
156
|
+
# The channel bin/server's reactor LISTENs on (Reactor). A payload is
|
|
157
|
+
# a JSON array of subjects, under Postgres's 8000-byte NOTIFY limit.
|
|
158
|
+
CHANNEL = "monk_presence".freeze
|
|
159
|
+
NOTIFY_SQL = "SELECT pg_notify('#{CHANNEL}', $1)".freeze
|
|
160
|
+
MAX_SIGNAL_BYTES = 7999
|
|
161
|
+
|
|
162
|
+
# subjects as JSON arrays that each fit one notify. A subject too
|
|
163
|
+
# long to fit even alone is left out, and logged: sending it would
|
|
164
|
+
# fail the settle, every tick.
|
|
165
|
+
def self.signals(subjects)
|
|
166
|
+
chunks = []
|
|
167
|
+
current = []
|
|
168
|
+
size = 2 # "[]"
|
|
169
|
+
subjects.each do |subject|
|
|
170
|
+
json = JSON.generate(subject)
|
|
171
|
+
if json.bytesize + 2 > MAX_SIGNAL_BYTES
|
|
172
|
+
Monk::Log.error("Monk::Presence: subject too long to signal (#{json.bytesize} bytes), left out")
|
|
173
|
+
next
|
|
174
|
+
end
|
|
175
|
+
added = json.bytesize + (current.empty? ? 0 : 1)
|
|
176
|
+
if size + added > MAX_SIGNAL_BYTES
|
|
177
|
+
chunks << current
|
|
178
|
+
current = []
|
|
179
|
+
size = 2
|
|
180
|
+
added = json.bytesize
|
|
181
|
+
end
|
|
182
|
+
current << json
|
|
183
|
+
size += added
|
|
184
|
+
end
|
|
185
|
+
chunks << current unless current.empty?
|
|
186
|
+
chunks.map { |jsons| "[#{jsons.join(",")}]" }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
attr_reader :db_name, :grace, :process_timeout
|
|
190
|
+
|
|
191
|
+
def initialize(db_name:, grace:, process_timeout:)
|
|
192
|
+
@db_name = db_name
|
|
193
|
+
@grace = grace
|
|
194
|
+
@process_timeout = process_timeout
|
|
195
|
+
freeze
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# This process's own row; returns its id.
|
|
199
|
+
def register_process(hostname:, pid:)
|
|
200
|
+
with_connection { |c| Integer(c.exec_params(REGISTER_PROCESS_SQL, [hostname, pid]).getvalue(0, 0)) }
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# True when the row was gone and had to be written again: the
|
|
204
|
+
# process's presence rows were pruned with it.
|
|
205
|
+
def heartbeat(process_id, hostname:, pid:)
|
|
206
|
+
with_connection { |c| c.exec_params(HEARTBEAT_SQL, [process_id, hostname, pid]).getvalue(0, 0) == true }
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# online:/away: the subjects with sockets in this process, by status;
|
|
210
|
+
# gone: those down to none.
|
|
211
|
+
def flush(process_id, online: [], away: [], gone: [])
|
|
212
|
+
params = [process_id, *subjects_and_statuses(online, away), ARRAY.encode(gone)]
|
|
213
|
+
with_connection { |c| c.exec_params(FLUSH_SQL, params) }
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# online:/away: every subject with a socket in this process.
|
|
218
|
+
def rewrite(process_id, online:, away:)
|
|
219
|
+
params = [process_id, *subjects_and_statuses(online, away)]
|
|
220
|
+
with_connection { |c| c.exec_params(REWRITE_SQL, params) }
|
|
221
|
+
nil
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def mark_all_gone(process_id)
|
|
225
|
+
with_connection { |c| c.exec_params(MARK_ALL_GONE_SQL, [process_id]) }
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# {subject => {status:, last_seen_at:}} for each of subjects, from
|
|
230
|
+
# the rows: right even with every WebSocket process down.
|
|
231
|
+
# last_seen_at is a Time for an offline subject seen before, else nil.
|
|
232
|
+
def status(subjects)
|
|
233
|
+
rows = with_connection { |c| c.exec_params(STATUS_SQL, [ARRAY.encode(subjects), @grace, @process_timeout]) }
|
|
234
|
+
rows.to_h do |row|
|
|
235
|
+
status = row["status"].to_sym
|
|
236
|
+
seen = row["last_seen_at"] if status == :offline
|
|
237
|
+
[row["subject"], { status: status, last_seen_at: seen && Time.at(seen) }]
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# The transitions this call recorded, each one only ever returned to
|
|
242
|
+
# one caller. reconcile: also checks every subject stored as not
|
|
243
|
+
# offline. notify: also signals the changed subjects on CHANNEL, in
|
|
244
|
+
# the same transaction, so the signal is delivered with the commit.
|
|
245
|
+
def settle(subjects, reconcile: false, notify: false)
|
|
246
|
+
params = [ARRAY.encode(subjects), @grace, @process_timeout, reconcile]
|
|
247
|
+
rows = with_connection do |c|
|
|
248
|
+
c.transaction do
|
|
249
|
+
c.exec(SETTLE_LOCK_SQL)
|
|
250
|
+
settled = c.exec_params(SETTLE_SQL, params).to_a
|
|
251
|
+
Store.signals(settled.map { |row| row["subject"] }).each { |p| c.exec_params(NOTIFY_SQL, [p]) } if notify
|
|
252
|
+
settled
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
rows.map do |row|
|
|
256
|
+
seen = row["last_seen_at"]
|
|
257
|
+
Transition.new(subject: row["subject"], status: row["status"].to_sym, last_seen_at: seen && Time.at(seen))
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
private
|
|
262
|
+
|
|
263
|
+
def with_connection(&) = Monk::Persistence::Pg.checkout(@db_name, &)
|
|
264
|
+
|
|
265
|
+
def subjects_and_statuses(online, away)
|
|
266
|
+
statuses = Array.new(online.size, "online") + Array.new(away.size, "away")
|
|
267
|
+
[ARRAY.encode(online + away), ARRAY.encode(statuses)]
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
require "socket"
|
|
2
|
+
|
|
3
|
+
module Monk
|
|
4
|
+
module Presence
|
|
5
|
+
# A WebSocket process's presence tick, once a second on a thread of the
|
|
6
|
+
# main Ractor (ADR 0019): take what changed from the counts, write it
|
|
7
|
+
# as this process's rows, and settle transitions, signalling them with
|
|
8
|
+
# notify: for bin/server's reactor. The heartbeat rides the same tick.
|
|
9
|
+
# No app code runs here.
|
|
10
|
+
#
|
|
11
|
+
# Errors are logged and never end the process. A tick that failed may
|
|
12
|
+
# have lost the counts' changes it took, so the next one writes this
|
|
13
|
+
# process's whole state again instead.
|
|
14
|
+
class Tracker
|
|
15
|
+
TICK = 1
|
|
16
|
+
HEARTBEAT_INTERVAL = 5
|
|
17
|
+
RECONCILE_INTERVAL = 60
|
|
18
|
+
|
|
19
|
+
attr_reader :process_id
|
|
20
|
+
|
|
21
|
+
def initialize(store:, counts:, notify: false, hostname: Socket.gethostname, pid: Process.pid)
|
|
22
|
+
@store = store
|
|
23
|
+
@counts = counts
|
|
24
|
+
@notify = notify
|
|
25
|
+
@hostname = hostname
|
|
26
|
+
@pid = pid
|
|
27
|
+
@rewrite = false
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Registers this process. Raises if the database can't be reached,
|
|
31
|
+
# so a WS process with presence fails at boot, not silently.
|
|
32
|
+
def start
|
|
33
|
+
@process_id = @store.register_process(hostname: @hostname, pid: @pid)
|
|
34
|
+
@last_heartbeat = @last_reconcile = monotonic
|
|
35
|
+
self
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def run
|
|
39
|
+
@thread = Thread.new do
|
|
40
|
+
loop do
|
|
41
|
+
sleep TICK
|
|
42
|
+
tick
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
self
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# A stop (TERM): this process's rows count for the grace, while its
|
|
49
|
+
# pages reconnect to another process, so a deploy changes nobody's
|
|
50
|
+
# status.
|
|
51
|
+
def stop
|
|
52
|
+
@thread&.kill
|
|
53
|
+
@store.mark_all_gone(@process_id) if @process_id
|
|
54
|
+
rescue StandardError => e
|
|
55
|
+
Monk::Log.error("Monk::Presence: couldn't mark this process's rows gone: #{describe(e)}")
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Returns the transitions this tick settled (none after an error).
|
|
59
|
+
def tick
|
|
60
|
+
heartbeat if @rewrite || monotonic - @last_heartbeat >= HEARTBEAT_INTERVAL
|
|
61
|
+
subjects = @rewrite ? rewrite : flush
|
|
62
|
+
@rewrite = false
|
|
63
|
+
settle(subjects)
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
@rewrite = true
|
|
66
|
+
Monk::Log.error("Monk::Presence: tick failed, writing all of this process's rows next time: #{describe(e)}")
|
|
67
|
+
[]
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The next tick heartbeats whatever the interval: for tests.
|
|
71
|
+
def heartbeat_now!
|
|
72
|
+
@last_heartbeat = -Float::INFINITY
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def heartbeat
|
|
78
|
+
@rewrite = true if @store.heartbeat(@process_id, hostname: @hostname, pid: @pid)
|
|
79
|
+
@last_heartbeat = monotonic
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def flush
|
|
83
|
+
diff = @counts.take
|
|
84
|
+
return [] if diff.values.all?(&:empty?)
|
|
85
|
+
|
|
86
|
+
@store.flush(@process_id, **diff)
|
|
87
|
+
diff.values.flatten
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# take before all: a change between the two is in all's state and
|
|
91
|
+
# still marked changed, so the next take writes it again, harmlessly.
|
|
92
|
+
def rewrite
|
|
93
|
+
diff = @counts.take
|
|
94
|
+
state = @counts.all
|
|
95
|
+
@store.rewrite(@process_id, **state)
|
|
96
|
+
(state.values + [diff[:gone]]).flatten.uniq
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def settle(subjects)
|
|
100
|
+
reconcile = monotonic - @last_reconcile >= RECONCILE_INTERVAL
|
|
101
|
+
transitions = @store.settle(subjects, reconcile: reconcile, notify: @notify)
|
|
102
|
+
@last_reconcile = monotonic if reconcile
|
|
103
|
+
transitions
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def describe(error) = "#{error.class}: #{error.message.lines.first&.chomp}"
|
|
107
|
+
|
|
108
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
require_relative "errors"
|
|
2
|
+
require_relative "log"
|
|
3
|
+
require_relative "presence/store"
|
|
4
|
+
require_relative "presence/counts"
|
|
5
|
+
require_relative "presence/tracker"
|
|
6
|
+
require_relative "presence/reactor"
|
|
7
|
+
|
|
8
|
+
module Monk
|
|
9
|
+
# Whether a person is online, from the WebSocket processes' sockets
|
|
10
|
+
# (docs/adr/0019-presence-per-process-rows-in-postgres-settled-once-per-transition.md).
|
|
11
|
+
# Opt-in: require "monk/presence". Only Monk::Auth subjects are tracked.
|
|
12
|
+
#
|
|
13
|
+
# Monk::Presence.configure(db_name: :primary)
|
|
14
|
+
# Monk::Presence.status(["user:1", "user:2"])
|
|
15
|
+
# # => {"user:1" => {status: :online, last_seen_at: nil},
|
|
16
|
+
# # "user:2" => {status: :offline, last_seen_at: 2026-11-13 10:02:11 UTC}}
|
|
17
|
+
# Monk::Presence.on_change(&AppPresence::CHANGED) # bin/server reacts
|
|
18
|
+
#
|
|
19
|
+
# :online is a page open and visible, :away only hidden ones, :offline
|
|
20
|
+
# none, after the grace (20 s by default): a phone switching networks or
|
|
21
|
+
# a deploy reconnecting every page doesn't read as going offline.
|
|
22
|
+
module Presence
|
|
23
|
+
class NotConfiguredError < StandardError
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
DEFAULT_GRACE = 20
|
|
27
|
+
PROCESS_TIMEOUT = 30
|
|
28
|
+
# Seconds a web process's reactor waits before trying again for the
|
|
29
|
+
# lock, or after losing its connection.
|
|
30
|
+
REACTOR_RETRY = 5
|
|
31
|
+
|
|
32
|
+
class << self
|
|
33
|
+
# The status of each of subjects, from any process, in one query.
|
|
34
|
+
# last_seen_at is when an offline subject was last online or away
|
|
35
|
+
# (nil if never, or if not offline).
|
|
36
|
+
def status(subjects)
|
|
37
|
+
store.status(Array(subjects).map(&:to_s))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def online?(subject)
|
|
41
|
+
status([subject]).fetch(subject.to_s)[:status] == :online
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# grace: seconds a subject stays as it was after its last socket
|
|
45
|
+
# closed, before it counts as offline.
|
|
46
|
+
def configure(db_name:, grace: DEFAULT_GRACE)
|
|
47
|
+
unless grace.is_a?(Numeric) && grace >= 0
|
|
48
|
+
raise ArgumentError, "grace must be a number of seconds, 0 or more, got #{grace.inspect}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
@store = Ractor.make_shareable(Store.new(db_name: db_name, grace: grace, process_timeout: PROCESS_TIMEOUT))
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def configured? = !@store.nil?
|
|
55
|
+
|
|
56
|
+
# Called in bin/server with the subjects whose status changed, as
|
|
57
|
+
# #status returns them: {subject => {status:, last_seen_at:}}. At
|
|
58
|
+
# most once per change: a change signalled while no web process
|
|
59
|
+
# listens is missed, and a page catches up on its next resync. The
|
|
60
|
+
# block runs in the reactor's own Ractor, so it must be shareable:
|
|
61
|
+
# build it in a module, as Monk::Live.authorize rules are. An error
|
|
62
|
+
# in it is logged.
|
|
63
|
+
#
|
|
64
|
+
# Set in config/presence.rb, which the WebSocket processes load too:
|
|
65
|
+
# with an on_change, they signal each change.
|
|
66
|
+
def on_change(&block)
|
|
67
|
+
raise ArgumentError, "on_change needs a block: { |changes| ... }" unless block
|
|
68
|
+
|
|
69
|
+
@on_change = shareable(block)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# In a WebSocket process: starts counting its sockets and the tick
|
|
73
|
+
# that writes them, signalling changes when there's an on_change.
|
|
74
|
+
# Returns the counts, for Server.new(presence:). start_visible: false
|
|
75
|
+
# for Live, whose sockets count as hidden until their page's first
|
|
76
|
+
# ping.
|
|
77
|
+
def start!(start_visible:)
|
|
78
|
+
counts = Counts.new(start_visible: start_visible)
|
|
79
|
+
@tracker = Tracker.new(store: store, counts: counts, notify: !@on_change.nil?).start.run
|
|
80
|
+
counts
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# In bin/server, after Monk.boot: starts the reactor that calls
|
|
84
|
+
# on_change. Nil, and nothing started, without an on_change.
|
|
85
|
+
def start_reactor!(retry_interval: REACTOR_RETRY)
|
|
86
|
+
return unless @on_change
|
|
87
|
+
|
|
88
|
+
return @reactor if @reactor
|
|
89
|
+
|
|
90
|
+
@reactor = Reactor.start(Monk::Persistence::Pg.connection_options(store.db_name), @on_change,
|
|
91
|
+
retry_interval: retry_interval,)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def stop_reactor!
|
|
95
|
+
Reactor.stop(@reactor) if @reactor
|
|
96
|
+
@reactor = nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# On TERM: this process's subjects count for the grace while its
|
|
100
|
+
# pages reconnect elsewhere.
|
|
101
|
+
def stop!
|
|
102
|
+
@tracker&.stop
|
|
103
|
+
@tracker = nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Raises unless configured.
|
|
107
|
+
def store
|
|
108
|
+
@store || raise(NotConfiguredError, "Monk::Presence isn't configured -- call Monk::Presence.configure first")
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def shareable(block)
|
|
112
|
+
Ractor.make_shareable(block)
|
|
113
|
+
rescue ArgumentError, Ractor::IsolationError => e
|
|
114
|
+
raise Monk::UnshareableBlockError,
|
|
115
|
+
"Monk::Presence.on_change block is not Ractor-shareable: #{e.message} " \
|
|
116
|
+
"(build it where self is shareable, e.g. in a module body)"
|
|
117
|
+
end
|
|
118
|
+
private :shareable
|
|
119
|
+
|
|
120
|
+
# Tests only.
|
|
121
|
+
def reset!
|
|
122
|
+
stop!
|
|
123
|
+
stop_reactor!
|
|
124
|
+
@store = nil
|
|
125
|
+
@on_change = nil
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -6,7 +6,7 @@ require_relative "settings"
|
|
|
6
6
|
# Each Monk module's config, if this app has it (`monk add <module>` writes
|
|
7
7
|
# it), in Monk's fixed order: a module's config may rely on the ones
|
|
8
8
|
# before it. Nothing to edit here when a module is added.
|
|
9
|
-
%w[persistence redis auth mail jobs websocket live storage].each do |name|
|
|
9
|
+
%w[persistence redis auth mail jobs websocket live presence storage].each do |name|
|
|
10
10
|
path = File.expand_path("#{name}.rb", __dir__)
|
|
11
11
|
require path if File.exist?(path)
|
|
12
12
|
end
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
require_relative "config/load" # the configs and app/ (config/load.rb)
|
|
2
2
|
require_relative "app/app"
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
app = Monk.boot(App) # Boot: freezes the app
|
|
5
|
+
|
|
6
|
+
# With presence (monk add presence) and an on_change in
|
|
7
|
+
# config/presence.rb, this process competes to be the one that calls it.
|
|
8
|
+
# Here, not in config/load.rb: the tests and bin/console don't react.
|
|
9
|
+
Monk::Presence.start_reactor! if defined?(Monk::Presence) && Monk::Presence.configured?
|
|
10
|
+
|
|
11
|
+
run app # serves it
|