monkrb 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +85 -0
- data/README.md +3 -2
- data/lib/monk/base.rb +51 -3
- data/lib/monk/context.rb +6 -1
- data/lib/monk/generators/presence.rb +22 -0
- data/lib/monk/generators.rb +1 -1
- data/lib/monk/live/client/monk_live.js +159 -7
- data/lib/monk/live/client/protocol.js +37 -0
- data/lib/monk/live/session.rb +59 -2
- data/lib/monk/persistence/pg/model.rb +19 -6
- data/lib/monk/presence/counts.rb +118 -0
- data/lib/monk/presence/reactor.rb +126 -0
- data/lib/monk/presence/store.rb +271 -0
- data/lib/monk/presence/tracker.rb +111 -0
- data/lib/monk/presence.rb +129 -0
- data/lib/monk/templates/base/Gemfile +1 -1
- data/lib/monk/templates/base/config/load.rb +1 -1
- data/lib/monk/templates/base/config.ru +8 -1
- data/lib/monk/templates/presence/agents.md +29 -0
- data/lib/monk/templates/presence/app/views/presence/_dot.erb +1 -0
- data/lib/monk/templates/presence/config/presence.rb +52 -0
- data/lib/monk/templates/presence/db/migrate/00000000000005_create_presence_tables.down.sql +3 -0
- data/lib/monk/templates/presence/db/migrate/00000000000005_create_presence_tables.up.sql +36 -0
- data/lib/monk/templates/presence/setup.md +51 -0
- data/lib/monk/templates/presence/test/presence_test.rb +13 -0
- data/lib/monk/templates/websocket/bin/websocket_server +29 -13
- data/lib/monk/version.rb +1 -1
- data/lib/monk/views.rb +28 -13
- data/lib/monk/websocket/connection.rb +108 -5
- data/lib/monk/websocket/server.rb +38 -12
- metadata +14 -1
data/lib/monk/live/session.rb
CHANGED
|
@@ -2,6 +2,11 @@ require "json"
|
|
|
2
2
|
|
|
3
3
|
module Monk
|
|
4
4
|
module Live
|
|
5
|
+
# Seconds between the client's pings while its page is visible:
|
|
6
|
+
# protocol.js's HEARTBEAT, which a test keeps equal. What Away means
|
|
7
|
+
# follows from it (ADR 0019), so the two are changed together.
|
|
8
|
+
HEARTBEAT = 25
|
|
9
|
+
|
|
5
10
|
# One WebSocket connection's live state, running inside its own
|
|
6
11
|
# connection Ractor: the subscribe/unsubscribe protocol, the topic
|
|
7
12
|
# authorization check, and the relay that stamps each published
|
|
@@ -9,9 +14,21 @@ module Monk
|
|
|
9
14
|
#
|
|
10
15
|
# Client -> server: {"op":"subscribe"|"unsubscribe","topics":[...]}.
|
|
11
16
|
# Replies: subscribed (+ denied), unsubscribed, or error/bad_message.
|
|
17
|
+
# And {"op":"ping"} -> {"op":"pong"}: the client's heartbeat, since
|
|
18
|
+
# browser JS can't see WebSocket pings. {"op":"hidden"}, with no
|
|
19
|
+
# reply: the client's page was just hidden.
|
|
12
20
|
# A denial never says why or whether the topic exists.
|
|
13
21
|
#
|
|
14
|
-
# `connection` is anything with #read, #write and #subject.
|
|
22
|
+
# `connection` is anything with #read, #write and #subject. One that
|
|
23
|
+
# #tracks_presence? (Server.new(presence:)) is also told, through
|
|
24
|
+
# #visible=, whether its page is visible: while its client has pinged
|
|
25
|
+
# within visible_for, 2 x HEARTBEAT (ADR 0019). The client pings only
|
|
26
|
+
# while its page is visible, and at once on open, so this is Away. It
|
|
27
|
+
# also says when its page is hidden, so Away shows at once; the
|
|
28
|
+
# silence covers a page frozen before it could say so, checked on the
|
|
29
|
+
# connection's heartbeat (Connection#on_heartbeat), once per server
|
|
30
|
+
# ping: no thread of its own, which at 15K sockets cost more than
|
|
31
|
+
# everything else presence does (docs/plan-live-networks-presence.md).
|
|
15
32
|
class Session
|
|
16
33
|
# No `*`, no whitespace or NUL, ASCII only: a client can't smuggle a
|
|
17
34
|
# glob or a delimiter into a topic, and only a topic that passed this
|
|
@@ -19,17 +36,23 @@ module Monk
|
|
|
19
36
|
TOPIC_FORMAT = %r{\A[\w:.\-/@]{1,200}\z}
|
|
20
37
|
MAX_TOPICS_PER_MESSAGE = 100
|
|
21
38
|
|
|
22
|
-
|
|
39
|
+
VISIBLE_FOR = 2 * HEARTBEAT
|
|
40
|
+
|
|
41
|
+
def initialize(connection, registry:, rules:, max_topics:, visible_for: VISIBLE_FOR)
|
|
23
42
|
@connection = connection
|
|
24
43
|
@registry = registry
|
|
25
44
|
@rules = rules
|
|
26
45
|
@max_topics = max_topics
|
|
27
46
|
@topics = {} # String topic => the Symbol it is registered under
|
|
47
|
+
@visible_for = visible_for
|
|
48
|
+
@presence = connection.respond_to?(:tracks_presence?) && connection.tracks_presence?
|
|
49
|
+
@visibility = Mutex.new
|
|
28
50
|
end
|
|
29
51
|
|
|
30
52
|
def run
|
|
31
53
|
@port = Ractor::Port.new
|
|
32
54
|
@relay = Thread.new { relay }
|
|
55
|
+
@connection.on_heartbeat { check_visibility } if @presence
|
|
33
56
|
while (raw = @connection.read)
|
|
34
57
|
dispatch(raw)
|
|
35
58
|
end
|
|
@@ -41,6 +64,9 @@ module Monk
|
|
|
41
64
|
|
|
42
65
|
def dispatch(raw)
|
|
43
66
|
message = JSON.parse(raw, max_nesting: 5)
|
|
67
|
+
return pinged if message.is_a?(Hash) && message["op"] == "ping"
|
|
68
|
+
return hidden if message.is_a?(Hash) && message["op"] == "hidden"
|
|
69
|
+
|
|
44
70
|
topics = message.is_a?(Hash) ? message["topics"] : nil
|
|
45
71
|
return reply(op: "error", reason: "bad_message") unless valid_topics?(topics)
|
|
46
72
|
|
|
@@ -94,6 +120,37 @@ module Monk
|
|
|
94
120
|
@connection.write(JSON.generate(message))
|
|
95
121
|
end
|
|
96
122
|
|
|
123
|
+
def pinged
|
|
124
|
+
if @presence
|
|
125
|
+
@visibility.synchronize do
|
|
126
|
+
@last_ping = Monk::WebSocket::Connection.monotonic
|
|
127
|
+
@connection.visible = true
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
reply(op: "pong")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def hidden
|
|
134
|
+
return unless @presence
|
|
135
|
+
|
|
136
|
+
@visibility.synchronize do
|
|
137
|
+
@last_ping = nil
|
|
138
|
+
@connection.visible = false
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# On the connection's heartbeat, once per server ping: the page turns
|
|
143
|
+
# hidden once no ping has come for visible_for, so up to one ping
|
|
144
|
+
# interval late.
|
|
145
|
+
def check_visibility
|
|
146
|
+
@visibility.synchronize do
|
|
147
|
+
if @last_ping && Monk::WebSocket::Connection.monotonic - @last_ping >= @visible_for
|
|
148
|
+
@last_ping = nil
|
|
149
|
+
@connection.visible = false
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
97
154
|
# Runs on its own thread: each published envelope gets the next
|
|
98
155
|
# per-connection seq spliced in after its opening `{`. The
|
|
99
156
|
# publisher's frozen string is shared by every subscriber (ADR
|
|
@@ -19,6 +19,13 @@ module Monk
|
|
|
19
19
|
ne: "<>",
|
|
20
20
|
}.freeze
|
|
21
21
|
|
|
22
|
+
# An Array condition's values as one array parameter (#in_clause).
|
|
23
|
+
# The Integer one, for a list of ids, writes each number straight
|
|
24
|
+
# into the literal; the generic one makes a String of each item
|
|
25
|
+
# first. Shareable: worker Ractors encode with them.
|
|
26
|
+
ARRAY_ENCODER = Ractor.make_shareable(PG::TextEncoder::Array.new)
|
|
27
|
+
INTEGER_ARRAY_ENCODER = Ractor.make_shareable(PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::Integer.new))
|
|
28
|
+
|
|
22
29
|
class << self
|
|
23
30
|
def create(data)
|
|
24
31
|
Monk::Persistence::Pg.checkout(db_name) do |conn|
|
|
@@ -196,14 +203,17 @@ module Monk
|
|
|
196
203
|
"#{ident} #{operator} $#{values.size}"
|
|
197
204
|
end
|
|
198
205
|
|
|
206
|
+
# `= ANY($n)` with the whole list as one Postgres array literal,
|
|
207
|
+
# not `IN ($1, ..., $n)`: a placeholder String and an encoded
|
|
208
|
+
# parameter per item made a 40-id lookup ~160 objects. Postgres
|
|
209
|
+
# types the parameter from the column, as it typed each `$n`,
|
|
210
|
+
# and a nil item matches nothing in either form.
|
|
199
211
|
def in_clause(ident, list, values)
|
|
200
212
|
return "1 = 0" if list.empty?
|
|
201
213
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
end
|
|
206
|
-
"#{ident} IN (#{placeholders.join(", ")})"
|
|
214
|
+
integers = list.all? { |item| item.nil? || item.is_a?(Integer) }
|
|
215
|
+
values << (integers ? INTEGER_ARRAY_ENCODER : ARRAY_ENCODER).encode(list)
|
|
216
|
+
"#{ident} = ANY($#{values.size})"
|
|
207
217
|
end
|
|
208
218
|
|
|
209
219
|
def order_clause(conn, order)
|
|
@@ -219,8 +229,11 @@ module Monk
|
|
|
219
229
|
" ORDER BY #{clause}"
|
|
220
230
|
end
|
|
221
231
|
|
|
232
|
+
# Symbol keys straight from pg, not String keys transformed: one
|
|
233
|
+
# Hash per row instead of two.
|
|
222
234
|
def to_rows(result)
|
|
223
|
-
result.
|
|
235
|
+
result.field_name_type = :symbol
|
|
236
|
+
result.to_a
|
|
224
237
|
end
|
|
225
238
|
|
|
226
239
|
def to_row(result)
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
module Monk
|
|
2
|
+
module Presence
|
|
3
|
+
# One WebSocket process's sockets, per subject (ADR 0019, "Counting, no
|
|
4
|
+
# query per connect"): a Ractor holding them, told by each socket's own
|
|
5
|
+
# Ractor when it opens, closes, or turns visible or hidden, with a
|
|
6
|
+
# send, never an ask, so a connect never waits on it. The flusher reads
|
|
7
|
+
# what changed with #take, once a second.
|
|
8
|
+
#
|
|
9
|
+
# A socket is reported under a key of its own (Server.serve uses its
|
|
10
|
+
# connection's Ractor), so a late or repeated report changes nothing.
|
|
11
|
+
# start_visible: is what an opened socket counts as until told: true
|
|
12
|
+
# for app/sockets clients, which have no Away; false for Live, whose
|
|
13
|
+
# client pings at once when its page is visible.
|
|
14
|
+
class Counts
|
|
15
|
+
def initialize(start_visible:)
|
|
16
|
+
@ractor = Ractor.new(start_visible) { |visible_at_open| Counts.run(visible_at_open) }
|
|
17
|
+
freeze
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# The Ractor's loop. sockets: key => [subject, visible];
|
|
21
|
+
# subjects: subject => [open sockets, visible sockets]; changed:
|
|
22
|
+
# subjects to report at the next take.
|
|
23
|
+
def self.run(visible_at_open)
|
|
24
|
+
sockets = {}
|
|
25
|
+
subjects = Hash.new { |h, k| h[k] = [0, 0] }
|
|
26
|
+
changed = {}
|
|
27
|
+
loop do
|
|
28
|
+
op, key, arg = Ractor.receive
|
|
29
|
+
case op
|
|
30
|
+
when :open
|
|
31
|
+
next if sockets.key?(key)
|
|
32
|
+
|
|
33
|
+
sockets[key] = [arg, visible_at_open]
|
|
34
|
+
counts = subjects[arg]
|
|
35
|
+
counts[0] += 1
|
|
36
|
+
counts[1] += 1 if visible_at_open
|
|
37
|
+
changed[arg] = true
|
|
38
|
+
when :close
|
|
39
|
+
subject, visible = sockets.delete(key)
|
|
40
|
+
next unless subject
|
|
41
|
+
|
|
42
|
+
counts = subjects[subject]
|
|
43
|
+
counts[0] -= 1
|
|
44
|
+
counts[1] -= 1 if visible
|
|
45
|
+
changed[subject] = true
|
|
46
|
+
when :visible
|
|
47
|
+
socket = sockets[key]
|
|
48
|
+
next if socket.nil? || socket[1] == arg
|
|
49
|
+
|
|
50
|
+
socket[1] = arg
|
|
51
|
+
subjects[socket[0]][1] += arg ? 1 : -1
|
|
52
|
+
changed[socket[0]] = true
|
|
53
|
+
when :take
|
|
54
|
+
arg.send(take(subjects, changed))
|
|
55
|
+
when :all
|
|
56
|
+
arg.send(all(subjects))
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def self.take(subjects, changed)
|
|
62
|
+
result = { online: [], away: [], gone: [] }
|
|
63
|
+
changed.each_key do |subject|
|
|
64
|
+
open, visible = subjects[subject]
|
|
65
|
+
if open.zero?
|
|
66
|
+
subjects.delete(subject)
|
|
67
|
+
result[:gone] << subject
|
|
68
|
+
else
|
|
69
|
+
result[visible.positive? ? :online : :away] << subject
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
changed.clear
|
|
73
|
+
Ractor.make_shareable(result)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def self.all(subjects)
|
|
77
|
+
result = { online: [], away: [] }
|
|
78
|
+
subjects.each do |subject, (open, visible)|
|
|
79
|
+
result[visible.positive? ? :online : :away] << subject if open.positive?
|
|
80
|
+
end
|
|
81
|
+
Ractor.make_shareable(result)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def open(key, subject) = tell(:open, key, subject)
|
|
85
|
+
|
|
86
|
+
def close(key) = tell(:close, key, nil)
|
|
87
|
+
|
|
88
|
+
def visible(key, visible) = tell(:visible, key, visible)
|
|
89
|
+
|
|
90
|
+
# {online:, away:, gone:}: the subjects that changed since the last
|
|
91
|
+
# take, by where they are now. A subject can open and close between
|
|
92
|
+
# two takes and be reported gone without ever having been reported.
|
|
93
|
+
def take = ask(:take)
|
|
94
|
+
|
|
95
|
+
# {online:, away:}: every subject with a socket, for writing this
|
|
96
|
+
# process's rows again from scratch.
|
|
97
|
+
def all = ask(:all)
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def tell(message, key, arg)
|
|
102
|
+
@ractor.send([message, key, arg])
|
|
103
|
+
nil
|
|
104
|
+
rescue Ractor::ClosedError
|
|
105
|
+
# The process is stopping: nothing left to count for.
|
|
106
|
+
nil
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def ask(message)
|
|
110
|
+
port = Ractor::Port.new
|
|
111
|
+
@ractor.send([message, nil, port])
|
|
112
|
+
port.receive
|
|
113
|
+
ensure
|
|
114
|
+
port&.close
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "pg"
|
|
3
|
+
require_relative "../log"
|
|
4
|
+
require_relative "../persistence/pg"
|
|
5
|
+
require_relative "store"
|
|
6
|
+
|
|
7
|
+
module Monk
|
|
8
|
+
module Presence
|
|
9
|
+
# bin/server's side of presence (ADR 0019): a Ractor that reacts to the
|
|
10
|
+
# WebSocket processes' signals by calling the app's on_change with the
|
|
11
|
+
# changed subjects' current status. Every web process starts one; only
|
|
12
|
+
# the one holding a session advisory lock LISTENs, the others retry, and
|
|
13
|
+
# take over when the holder's connection ends (Postgres frees the lock).
|
|
14
|
+
#
|
|
15
|
+
# A signal sent while no reactor listens is lost: on_change is at most
|
|
16
|
+
# once, and a page catches up on its next resync.
|
|
17
|
+
#
|
|
18
|
+
# Plain module methods, like PgFanout.subscribe: the Ractor has no
|
|
19
|
+
# instance to call into.
|
|
20
|
+
module Reactor
|
|
21
|
+
LOCK_SQL = "SELECT pg_try_advisory_lock(hashtext('monk_presence_reactor'))".freeze
|
|
22
|
+
|
|
23
|
+
# Postgres learns that a reactor's host vanished (a lost node, a cut
|
|
24
|
+
# network: no FIN, no RST) only when TCP keepalive gives up, after
|
|
25
|
+
# about 2 hours with Linux's defaults, and until then the dead session
|
|
26
|
+
# holds the lock and no web process reacts. These make it about 25 s:
|
|
27
|
+
# 10 s idle, then 3 probes 5 s apart. Server settings any user may
|
|
28
|
+
# set for their own session; the client-side ones in .connect do the
|
|
29
|
+
# same for a reactor waiting on a Postgres host that vanished.
|
|
30
|
+
KEEPALIVE_SQL = "SET tcp_keepalives_idle = 10; SET tcp_keepalives_interval = 5; " \
|
|
31
|
+
"SET tcp_keepalives_count = 3".freeze
|
|
32
|
+
CLIENT_KEEPALIVES = { keepalives: 1, keepalives_idle: 10, keepalives_interval: 5, keepalives_count: 3 }.freeze
|
|
33
|
+
|
|
34
|
+
# How often a waiting reactor checks whether it was stopped.
|
|
35
|
+
POLL = 0.5
|
|
36
|
+
|
|
37
|
+
Handle = Data.define(:ractor, :stop_port)
|
|
38
|
+
|
|
39
|
+
# pg_opts: the database's connection options; on_change: a shareable
|
|
40
|
+
# block. Returns a handle for .stop.
|
|
41
|
+
#
|
|
42
|
+
# One thread: a second one, waiting in Postgres, kept the Ractor
|
|
43
|
+
# from handling a stop message about half the time. So the reactor
|
|
44
|
+
# waits in short steps and stops once its stop port is closed.
|
|
45
|
+
def self.start(pg_opts, on_change, retry_interval:)
|
|
46
|
+
stop_port = Ractor::Port.new
|
|
47
|
+
ractor = Ractor.new(Ractor.make_shareable(pg_opts.dup), Ractor.make_shareable(on_change), retry_interval,
|
|
48
|
+
stop_port, name: "monk-presence-reactor",) do |opts, hook, interval, stop|
|
|
49
|
+
Monk::Presence::Reactor.run(opts, hook, interval, stop)
|
|
50
|
+
end
|
|
51
|
+
Handle.new(ractor: ractor, stop_port: stop_port)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Ends the reactor and closes its connection, which frees the lock.
|
|
55
|
+
def self.stop(handle)
|
|
56
|
+
handle.stop_port.close
|
|
57
|
+
handle.ractor.join
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.run(pg_opts, on_change, retry_interval, stop = nil)
|
|
61
|
+
stopped = -> { stop&.closed? }
|
|
62
|
+
until stopped.call
|
|
63
|
+
conn = nil
|
|
64
|
+
begin
|
|
65
|
+
conn = connect(pg_opts)
|
|
66
|
+
next unless lock_acquired?(conn, retry_interval, stopped)
|
|
67
|
+
|
|
68
|
+
conn.exec("LISTEN #{Store::CHANNEL}")
|
|
69
|
+
log(:info, "Monk::Presence: this process reacts to presence changes (pid #{Process.pid})")
|
|
70
|
+
conn.wait_for_notify(POLL) { |_channel, _pid, payload| react(on_change, payload) } until stopped.call
|
|
71
|
+
rescue PG::Error => e
|
|
72
|
+
log(:error, "Monk::Presence reactor: #{e.class}: #{e.message.lines.first&.chomp}; " \
|
|
73
|
+
"retrying in #{retry_interval} s",)
|
|
74
|
+
pause(retry_interval, stopped)
|
|
75
|
+
ensure
|
|
76
|
+
conn&.close
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The reactor's own connection, kept apart from Monk::Persistence's:
|
|
82
|
+
# the lock and the LISTEN belong to its session (so not through
|
|
83
|
+
# PgBouncer in transaction mode).
|
|
84
|
+
def self.connect(pg_opts)
|
|
85
|
+
conn = PG.connect(connect_timeout: Monk::Persistence::Pg::DEFAULT_CONNECT_TIMEOUT, **CLIENT_KEEPALIVES,
|
|
86
|
+
**pg_opts,)
|
|
87
|
+
conn.exec(KEEPALIVE_SQL)
|
|
88
|
+
conn
|
|
89
|
+
rescue PG::Error
|
|
90
|
+
conn&.close
|
|
91
|
+
raise
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# True once this connection holds the lock; false if stopped first.
|
|
95
|
+
def self.lock_acquired?(conn, retry_interval, stopped)
|
|
96
|
+
until conn.exec(LOCK_SQL).getvalue(0, 0) == "t"
|
|
97
|
+
pause(retry_interval, stopped)
|
|
98
|
+
return false if stopped.call
|
|
99
|
+
end
|
|
100
|
+
true
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def self.pause(seconds, stopped)
|
|
104
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
|
|
105
|
+
until stopped.call || (left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)) <= 0
|
|
106
|
+
sleep [left, POLL].min
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# One signal. Anything that goes wrong drops this signal only.
|
|
111
|
+
def self.react(on_change, payload)
|
|
112
|
+
on_change.call(Monk::Presence.status(JSON.parse(payload)))
|
|
113
|
+
rescue StandardError => e
|
|
114
|
+
log(:error, "Monk::Presence on_change failed: #{e.class}: #{e.message.lines.first&.chomp}")
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Monk::Log writes from a Ractor once Monk has booted (bin/server
|
|
118
|
+
# starts the reactor after Monk.boot). Logging must never end it.
|
|
119
|
+
def self.log(level, message)
|
|
120
|
+
Monk::Log.public_send(level, message)
|
|
121
|
+
rescue StandardError
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require_relative "../log"
|
|
3
|
+
require_relative "../persistence/pg"
|
|
4
|
+
|
|
5
|
+
module Monk
|
|
6
|
+
module Presence
|
|
7
|
+
# One subject's change of status, as settling returned it: status is
|
|
8
|
+
# :online, :away or :offline, last_seen_at a Time when :offline.
|
|
9
|
+
Transition = Data.define(:subject, :status, :last_seen_at)
|
|
10
|
+
|
|
11
|
+
# Presence's statements on Postgres: monk_presence_processes,
|
|
12
|
+
# monk_presence and monk_presence_status
|
|
13
|
+
# (docs/adr/0019-presence-per-process-rows-in-postgres-settled-once-per-transition.md).
|
|
14
|
+
# Holds only settings and freezes itself, so one instance is shareable
|
|
15
|
+
# with every Ractor; each Ractor reaches the database through its own
|
|
16
|
+
# connection, via Monk::Persistence::Pg, as Monk::Jobs' adapter does.
|
|
17
|
+
#
|
|
18
|
+
# Every time is the database's now(), so hosts' clocks don't matter.
|
|
19
|
+
class Store
|
|
20
|
+
ARRAY = Ractor.make_shareable(PG::TextEncoder::Array.new)
|
|
21
|
+
|
|
22
|
+
# Whether a row counts toward its subject's status: an open one
|
|
23
|
+
# while its process heartbeats, a gone one for the grace after it
|
|
24
|
+
# went, whatever its process does since (a stopping process's rows
|
|
25
|
+
# carry its subjects through a deploy). A row whose process row is
|
|
26
|
+
# gone counts only if it's inside the grace. $grace and $timeout are
|
|
27
|
+
# replaced with each statement's own parameter numbers.
|
|
28
|
+
COUNTING = <<~SQL.freeze
|
|
29
|
+
CASE WHEN r.gone_at IS NULL
|
|
30
|
+
THEN COALESCE(p.heartbeat_at >= now() - $timeout::float8 * interval '1 second', false)
|
|
31
|
+
ELSE r.gone_at >= now() - $grace::float8 * interval '1 second'
|
|
32
|
+
END
|
|
33
|
+
SQL
|
|
34
|
+
|
|
35
|
+
# The best of the counting rows: online over away, offline without.
|
|
36
|
+
BEST = <<~SQL.freeze
|
|
37
|
+
CASE WHEN bool_or(r.counting AND r.status = 'online') THEN 'online'
|
|
38
|
+
WHEN bool_or(r.counting) THEN 'away'
|
|
39
|
+
ELSE 'offline'
|
|
40
|
+
END
|
|
41
|
+
SQL
|
|
42
|
+
|
|
43
|
+
REGISTER_PROCESS_SQL = <<~SQL.freeze
|
|
44
|
+
INSERT INTO monk_presence_processes (hostname, pid) VALUES ($1, $2) RETURNING id
|
|
45
|
+
SQL
|
|
46
|
+
|
|
47
|
+
# An upsert, as Monk::Jobs': a process that stalled long enough to be
|
|
48
|
+
# pruned gets its row back under the same id. Says whether it had to
|
|
49
|
+
# (xmax is 0 on a freshly inserted row): its own rows went with it,
|
|
50
|
+
# so it writes them all again.
|
|
51
|
+
HEARTBEAT_SQL = <<~SQL.freeze
|
|
52
|
+
INSERT INTO monk_presence_processes (id, hostname, pid) OVERRIDING SYSTEM VALUE VALUES ($1, $2, $3)
|
|
53
|
+
ON CONFLICT (id) DO UPDATE SET heartbeat_at = now()
|
|
54
|
+
RETURNING xmax = 0 AS restored
|
|
55
|
+
SQL
|
|
56
|
+
|
|
57
|
+
# One process's changes since its last flush, in one statement:
|
|
58
|
+
# subjects with sockets ($2, their statuses in $3) upserted back to
|
|
59
|
+
# open, subjects down to none ($4) marked gone. The lists never share
|
|
60
|
+
# a subject.
|
|
61
|
+
FLUSH_SQL = <<~SQL.freeze
|
|
62
|
+
WITH open AS (
|
|
63
|
+
INSERT INTO monk_presence (process_id, subject, status)
|
|
64
|
+
SELECT $1, subject, status FROM unnest($2::text[], $3::text[]) AS u(subject, status)
|
|
65
|
+
ON CONFLICT (process_id, subject) DO UPDATE SET status = excluded.status, gone_at = NULL
|
|
66
|
+
)
|
|
67
|
+
UPDATE monk_presence SET gone_at = now()
|
|
68
|
+
WHERE process_id = $1 AND subject = ANY($4::text[]) AND gone_at IS NULL
|
|
69
|
+
SQL
|
|
70
|
+
|
|
71
|
+
# This process's rows written again from its whole state ($2, $3):
|
|
72
|
+
# after an error lost a flush, or after its rows were pruned. A row
|
|
73
|
+
# not in the state is marked gone, so a close the lost flush carried
|
|
74
|
+
# isn't left open.
|
|
75
|
+
REWRITE_SQL = <<~SQL.freeze
|
|
76
|
+
WITH open AS (
|
|
77
|
+
INSERT INTO monk_presence (process_id, subject, status)
|
|
78
|
+
SELECT $1, subject, status FROM unnest($2::text[], $3::text[]) AS u(subject, status)
|
|
79
|
+
ON CONFLICT (process_id, subject) DO UPDATE SET status = excluded.status, gone_at = NULL
|
|
80
|
+
)
|
|
81
|
+
UPDATE monk_presence SET gone_at = now()
|
|
82
|
+
WHERE process_id = $1 AND gone_at IS NULL AND subject <> ALL($2::text[])
|
|
83
|
+
SQL
|
|
84
|
+
|
|
85
|
+
# A stopping process (TERM): its rows count for the grace instead of
|
|
86
|
+
# vanishing, while its pages reconnect elsewhere.
|
|
87
|
+
MARK_ALL_GONE_SQL = <<~SQL.freeze
|
|
88
|
+
UPDATE monk_presence SET gone_at = now() WHERE process_id = $1 AND gone_at IS NULL
|
|
89
|
+
SQL
|
|
90
|
+
|
|
91
|
+
# last_seen_at: rows that stopped counting but aren't settled yet
|
|
92
|
+
# (up to a tick) say it, as settling will; else the stored one.
|
|
93
|
+
STATUS_SQL = <<~SQL.freeze
|
|
94
|
+
WITH r AS (
|
|
95
|
+
SELECT r.subject, r.status, r.gone_at, p.heartbeat_at,
|
|
96
|
+
#{COUNTING.gsub("$timeout", "$3").gsub("$grace", "$2")} AS counting
|
|
97
|
+
FROM monk_presence r LEFT JOIN monk_presence_processes p ON p.id = r.process_id
|
|
98
|
+
WHERE r.subject = ANY($1::text[])
|
|
99
|
+
)
|
|
100
|
+
SELECT s.subject, #{BEST} AS status,
|
|
101
|
+
extract(epoch FROM COALESCE(
|
|
102
|
+
max(COALESCE(r.gone_at, r.heartbeat_at)) FILTER (WHERE NOT r.counting), st.last_seen_at
|
|
103
|
+
))::float8 AS last_seen_at
|
|
104
|
+
FROM unnest($1::text[]) AS s(subject)
|
|
105
|
+
LEFT JOIN r ON r.subject = s.subject
|
|
106
|
+
LEFT JOIN monk_presence_status st ON st.subject = s.subject
|
|
107
|
+
GROUP BY s.subject, st.last_seen_at
|
|
108
|
+
SQL
|
|
109
|
+
|
|
110
|
+
# The subjects given ($1), those with rows that stopped counting, and
|
|
111
|
+
# with $4 every subject stored as not offline (the minute's full
|
|
112
|
+
# reconcile), each settled into monk_presence_status where its status
|
|
113
|
+
# changed; the expired rows and dead processes deleted, all in one
|
|
114
|
+
# statement. The upsert's WHERE makes a second settling of the same
|
|
115
|
+
# change find nothing to write, so each transition is returned once.
|
|
116
|
+
# A subject never stored isn't stored as offline: there's nothing to
|
|
117
|
+
# announce. Going offline, last seen is when its last rows stopped:
|
|
118
|
+
# the latest gone_at, or a dead process's last heartbeat.
|
|
119
|
+
SETTLE_SQL = <<~SQL.freeze
|
|
120
|
+
WITH r AS (
|
|
121
|
+
SELECT r.process_id, r.subject, r.status, r.gone_at, p.heartbeat_at,
|
|
122
|
+
#{COUNTING.gsub("$timeout", "$3").gsub("$grace", "$2")} AS counting
|
|
123
|
+
FROM monk_presence r LEFT JOIN monk_presence_processes p ON p.id = r.process_id
|
|
124
|
+
), candidates AS (
|
|
125
|
+
SELECT unnest($1::text[]) AS subject
|
|
126
|
+
UNION SELECT subject FROM r WHERE NOT counting
|
|
127
|
+
UNION SELECT subject FROM monk_presence_status WHERE $4::boolean AND status <> 'offline'
|
|
128
|
+
), computed AS (
|
|
129
|
+
SELECT c.subject, #{BEST} AS status,
|
|
130
|
+
max(COALESCE(r.gone_at, r.heartbeat_at)) FILTER (WHERE NOT r.counting) AS last_seen_at
|
|
131
|
+
FROM candidates c LEFT JOIN r ON r.subject = c.subject
|
|
132
|
+
GROUP BY c.subject
|
|
133
|
+
), expired AS (
|
|
134
|
+
DELETE FROM monk_presence m USING r
|
|
135
|
+
WHERE NOT r.counting AND m.process_id = r.process_id AND m.subject = r.subject
|
|
136
|
+
), dead AS (
|
|
137
|
+
DELETE FROM monk_presence_processes WHERE heartbeat_at < now() - $3::float8 * interval '1 second'
|
|
138
|
+
)
|
|
139
|
+
INSERT INTO monk_presence_status AS s (subject, status, changed_at, last_seen_at)
|
|
140
|
+
SELECT c.subject, c.status, now(), CASE WHEN c.status = 'offline' THEN COALESCE(c.last_seen_at, now()) END
|
|
141
|
+
FROM computed c
|
|
142
|
+
WHERE c.status <> 'offline' OR EXISTS (SELECT 1 FROM monk_presence_status x WHERE x.subject = c.subject)
|
|
143
|
+
ON CONFLICT (subject) DO UPDATE SET
|
|
144
|
+
status = excluded.status, changed_at = excluded.changed_at,
|
|
145
|
+
last_seen_at = COALESCE(excluded.last_seen_at, s.last_seen_at)
|
|
146
|
+
WHERE s.status IS DISTINCT FROM excluded.status
|
|
147
|
+
RETURNING s.subject, s.status,
|
|
148
|
+
CASE WHEN s.status = 'offline' THEN extract(epoch FROM s.last_seen_at)::float8 END AS last_seen_at
|
|
149
|
+
SQL
|
|
150
|
+
|
|
151
|
+
# Settling processes take turns: each statement then sees what the
|
|
152
|
+
# one before it committed, so two can't settle one subject from
|
|
153
|
+
# snapshots taken before each other's flush.
|
|
154
|
+
SETTLE_LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtext('monk_presence_settle'))".freeze
|
|
155
|
+
|
|
156
|
+
# The channel bin/server's reactor LISTENs on (Reactor). A payload is
|
|
157
|
+
# a JSON array of subjects, under Postgres's 8000-byte NOTIFY limit.
|
|
158
|
+
CHANNEL = "monk_presence".freeze
|
|
159
|
+
NOTIFY_SQL = "SELECT pg_notify('#{CHANNEL}', $1)".freeze
|
|
160
|
+
MAX_SIGNAL_BYTES = 7999
|
|
161
|
+
|
|
162
|
+
# subjects as JSON arrays that each fit one notify. A subject too
|
|
163
|
+
# long to fit even alone is left out, and logged: sending it would
|
|
164
|
+
# fail the settle, every tick.
|
|
165
|
+
def self.signals(subjects)
|
|
166
|
+
chunks = []
|
|
167
|
+
current = []
|
|
168
|
+
size = 2 # "[]"
|
|
169
|
+
subjects.each do |subject|
|
|
170
|
+
json = JSON.generate(subject)
|
|
171
|
+
if json.bytesize + 2 > MAX_SIGNAL_BYTES
|
|
172
|
+
Monk::Log.error("Monk::Presence: subject too long to signal (#{json.bytesize} bytes), left out")
|
|
173
|
+
next
|
|
174
|
+
end
|
|
175
|
+
added = json.bytesize + (current.empty? ? 0 : 1)
|
|
176
|
+
if size + added > MAX_SIGNAL_BYTES
|
|
177
|
+
chunks << current
|
|
178
|
+
current = []
|
|
179
|
+
size = 2
|
|
180
|
+
added = json.bytesize
|
|
181
|
+
end
|
|
182
|
+
current << json
|
|
183
|
+
size += added
|
|
184
|
+
end
|
|
185
|
+
chunks << current unless current.empty?
|
|
186
|
+
chunks.map { |jsons| "[#{jsons.join(",")}]" }
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
attr_reader :db_name, :grace, :process_timeout
|
|
190
|
+
|
|
191
|
+
def initialize(db_name:, grace:, process_timeout:)
|
|
192
|
+
@db_name = db_name
|
|
193
|
+
@grace = grace
|
|
194
|
+
@process_timeout = process_timeout
|
|
195
|
+
freeze
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# This process's own row; returns its id.
|
|
199
|
+
def register_process(hostname:, pid:)
|
|
200
|
+
with_connection { |c| Integer(c.exec_params(REGISTER_PROCESS_SQL, [hostname, pid]).getvalue(0, 0)) }
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# True when the row was gone and had to be written again: the
|
|
204
|
+
# process's presence rows were pruned with it.
|
|
205
|
+
def heartbeat(process_id, hostname:, pid:)
|
|
206
|
+
with_connection { |c| c.exec_params(HEARTBEAT_SQL, [process_id, hostname, pid]).getvalue(0, 0) == true }
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# online:/away: the subjects with sockets in this process, by status;
|
|
210
|
+
# gone: those down to none.
|
|
211
|
+
def flush(process_id, online: [], away: [], gone: [])
|
|
212
|
+
params = [process_id, *subjects_and_statuses(online, away), ARRAY.encode(gone)]
|
|
213
|
+
with_connection { |c| c.exec_params(FLUSH_SQL, params) }
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# online:/away: every subject with a socket in this process.
|
|
218
|
+
def rewrite(process_id, online:, away:)
|
|
219
|
+
params = [process_id, *subjects_and_statuses(online, away)]
|
|
220
|
+
with_connection { |c| c.exec_params(REWRITE_SQL, params) }
|
|
221
|
+
nil
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def mark_all_gone(process_id)
|
|
225
|
+
with_connection { |c| c.exec_params(MARK_ALL_GONE_SQL, [process_id]) }
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# {subject => {status:, last_seen_at:}} for each of subjects, from
|
|
230
|
+
# the rows: right even with every WebSocket process down.
|
|
231
|
+
# last_seen_at is a Time for an offline subject seen before, else nil.
|
|
232
|
+
def status(subjects)
|
|
233
|
+
rows = with_connection { |c| c.exec_params(STATUS_SQL, [ARRAY.encode(subjects), @grace, @process_timeout]) }
|
|
234
|
+
rows.to_h do |row|
|
|
235
|
+
status = row["status"].to_sym
|
|
236
|
+
seen = row["last_seen_at"] if status == :offline
|
|
237
|
+
[row["subject"], { status: status, last_seen_at: seen && Time.at(seen) }]
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# The transitions this call recorded, each one only ever returned to
|
|
242
|
+
# one caller. reconcile: also checks every subject stored as not
|
|
243
|
+
# offline. notify: also signals the changed subjects on CHANNEL, in
|
|
244
|
+
# the same transaction, so the signal is delivered with the commit.
|
|
245
|
+
def settle(subjects, reconcile: false, notify: false)
|
|
246
|
+
params = [ARRAY.encode(subjects), @grace, @process_timeout, reconcile]
|
|
247
|
+
rows = with_connection do |c|
|
|
248
|
+
c.transaction do
|
|
249
|
+
c.exec(SETTLE_LOCK_SQL)
|
|
250
|
+
settled = c.exec_params(SETTLE_SQL, params).to_a
|
|
251
|
+
Store.signals(settled.map { |row| row["subject"] }).each { |p| c.exec_params(NOTIFY_SQL, [p]) } if notify
|
|
252
|
+
settled
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
rows.map do |row|
|
|
256
|
+
seen = row["last_seen_at"]
|
|
257
|
+
Transition.new(subject: row["subject"], status: row["status"].to_sym, last_seen_at: seen && Time.at(seen))
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
private
|
|
262
|
+
|
|
263
|
+
def with_connection(&) = Monk::Persistence::Pg.checkout(@db_name, &)
|
|
264
|
+
|
|
265
|
+
def subjects_and_statuses(online, away)
|
|
266
|
+
statuses = Array.new(online.size, "online") + Array.new(away.size, "away")
|
|
267
|
+
[ARRAY.encode(online + away), ARRAY.encode(statuses)]
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
end
|