@rbxts/scribe 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +21 -0
- package/README.md +100 -0
- package/Scribe-source-map.json +1 -0
- package/licenses/ProfileStore-Apache-2.0.txt +201 -0
- package/package.json +32 -0
- package/src/Client/DebugHook.luau +457 -0
- package/src/Client/init.luau +1607 -0
- package/src/Internal/Addons.luau +68 -0
- package/src/Internal/Big.luau +582 -0
- package/src/Internal/Budget.luau +377 -0
- package/src/Internal/Clock.luau +75 -0
- package/src/Internal/Config.luau +17 -0
- package/src/Internal/Conform.luau +400 -0
- package/src/Internal/Datatypes.luau +653 -0
- package/src/Internal/Defaults.luau +12 -0
- package/src/Internal/Derived.luau +428 -0
- package/src/Internal/Exchange.luau +556 -0
- package/src/Internal/ExchangeLeg.luau +1368 -0
- package/src/Internal/ExchangeVerdict.luau +179 -0
- package/src/Internal/Failure.luau +148 -0
- package/src/Internal/Health.luau +320 -0
- package/src/Internal/LeaderboardDiagnostics.luau +36 -0
- package/src/Internal/Log.luau +378 -0
- package/src/Internal/Marker.luau +76 -0
- package/src/Internal/Metrics.luau +152 -0
- package/src/Internal/Node.luau +3954 -0
- package/src/Internal/Profile.luau +69 -0
- package/src/Internal/Rng.luau +83 -0
- package/src/Internal/Sched.luau +56 -0
- package/src/Internal/Signal.luau +199 -0
- package/src/Internal/Store/ProfileStore.luau +3019 -0
- package/src/Internal/Store/init.luau +18 -0
- package/src/Internal/StudioCodec.luau +155 -0
- package/src/Internal/Template.luau +1848 -0
- package/src/Internal/Transport.luau +169 -0
- package/src/Internal/Util.luau +517 -0
- package/src/Internal/Version.luau +4 -0
- package/src/Internal/Wire.luau +1283 -0
- package/src/Server/Analytics.luau +223 -0
- package/src/Server/Commands.luau +728 -0
- package/src/Server/DebugHook.luau +1490 -0
- package/src/Server/Exchanges.luau +737 -0
- package/src/Server/Leaderboards.luau +2143 -0
- package/src/Server/Monetization.luau +3212 -0
- package/src/Server/Persistence.luau +3775 -0
- package/src/Server/Replication.luau +1253 -0
- package/src/Server/ServerStore.luau +81 -0
- package/src/Server/Timed.luau +579 -0
- package/src/Server/init.luau +672 -0
- package/src/Types.luau +480 -0
- package/src/init.luau +2231 -0
- package/types/api.d.ts +252 -0
- package/types/common.d.ts +355 -0
- package/types/index.d.ts +226 -0
- package/types/schema.d.ts +276 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- The verdict key: the single point of no return for an exchange. Two players are two
|
|
3
|
+
-- DataStore keys with no multi-key transaction, so ONE key decides and everything after
|
|
4
|
+
-- it is infallible or infinitely retryable. The only property owed: every participant,
|
|
5
|
+
-- on every server, on every retry, reads the SAME answer.
|
|
6
|
+
--
|
|
7
|
+
-- First writer wins and the loser learns from the return value. The transform never
|
|
8
|
+
-- returns nil: that aborts the write and yields (nil, nil), which the caller cannot tell
|
|
9
|
+
-- from "nothing happened". Indeterminate is not special: a proposer that never learned
|
|
10
|
+
-- its proposal won has not settled, and the next proposal reads the committed answer.
|
|
11
|
+
local Clock = require(script.Parent.Clock)
|
|
12
|
+
|
|
13
|
+
local ExchangeVerdict = {}
|
|
14
|
+
|
|
15
|
+
ExchangeVerdict.STORE_NAME = "ClaimExchangeVerdicts"
|
|
16
|
+
|
|
17
|
+
export type Verdict = {
|
|
18
|
+
Id: string?, -- original id for a recovered legacy key
|
|
19
|
+
V: string, -- "Commit" or "Abort"
|
|
20
|
+
A: number, -- userId of role A
|
|
21
|
+
B: number, -- userId of role B
|
|
22
|
+
D: string, -- basket digest
|
|
23
|
+
HA: string?, -- content hash of A's take payload
|
|
24
|
+
HB: string?,
|
|
25
|
+
T: number, -- when the winner decided
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
local function fail(message: string): ()
|
|
29
|
+
error(`ExchangeVerdict: {message}`, 0)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
local function storeKey(exchangeId: string): (string?, string?)
|
|
33
|
+
if utf8.len(exchangeId) == nil then
|
|
34
|
+
return nil, "exchange id must be valid UTF-8"
|
|
35
|
+
end
|
|
36
|
+
if #exchangeId <= 50 then
|
|
37
|
+
return exchangeId, nil
|
|
38
|
+
end
|
|
39
|
+
local guid = string.match(exchangeId, "^%d+%-%d+%-(.+)$")
|
|
40
|
+
if guid ~= nil and #guid == 36 then
|
|
41
|
+
local groups = string.split(guid, "-")
|
|
42
|
+
local sizes = { 8, 4, 4, 4, 12 }
|
|
43
|
+
local valid = #groups == #sizes
|
|
44
|
+
for index, size in sizes do
|
|
45
|
+
local group = groups[index]
|
|
46
|
+
if group == nil or #group ~= size or string.match(group, "^%x+$") == nil then
|
|
47
|
+
valid = false
|
|
48
|
+
break
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
if valid then
|
|
52
|
+
-- Overlong legacy keys never reached Roblox. Preserve their profile records.
|
|
53
|
+
return `legacy:{guid}`, nil
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
return nil, "exchange id exceeds the 50 character limit and is not a recognized legacy id"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
--- Is this a verdict record, rather than something else that happens to be stored?
|
|
60
|
+
local function wellFormed(value: any): boolean
|
|
61
|
+
return type(value) == "table"
|
|
62
|
+
and (value.V == "Commit" or value.V == "Abort")
|
|
63
|
+
and type(value.D) == "string"
|
|
64
|
+
and type(value.A) == "number"
|
|
65
|
+
and type(value.B) == "number"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
--- Propose a verdict for an exchange, and return whatever verdict is now in force.
|
|
69
|
+
---
|
|
70
|
+
--- Returns the committed verdict and whether THIS call decided it. A caller almost never
|
|
71
|
+
--- needs the second value, since winner and loser take the identical branch; it is there
|
|
72
|
+
--- for metrics and for the abort path, which logs differently when it lost to a Commit.
|
|
73
|
+
---
|
|
74
|
+
--- Yields. Never call it with a transaction open.
|
|
75
|
+
function ExchangeVerdict.Propose(store: any, exchangeId: string, proposal: any): (any, boolean, string?)
|
|
76
|
+
if type(store) ~= "table" and type(store) ~= "userdata" then
|
|
77
|
+
fail("Propose needs a DataStore")
|
|
78
|
+
end
|
|
79
|
+
if type(exchangeId) ~= "string" or #exchangeId == 0 then
|
|
80
|
+
fail("Propose needs an exchange id")
|
|
81
|
+
end
|
|
82
|
+
if not wellFormed(proposal) then
|
|
83
|
+
fail("Propose needs a well-formed verdict of Commit or Abort")
|
|
84
|
+
end
|
|
85
|
+
local key, keyError = storeKey(exchangeId)
|
|
86
|
+
if key == nil then
|
|
87
|
+
return nil, false, keyError
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
local decided = false
|
|
91
|
+
local ok, committed = pcall(function()
|
|
92
|
+
return store:UpdateAsync(key, function(old)
|
|
93
|
+
-- NEVER nil. See the header: a nil return aborts the write and the caller
|
|
94
|
+
-- gets (nil, nil), which is indistinguishable from a failure.
|
|
95
|
+
if old ~= nil then
|
|
96
|
+
return old
|
|
97
|
+
end
|
|
98
|
+
decided = true
|
|
99
|
+
return {
|
|
100
|
+
Id = if key ~= exchangeId then exchangeId else nil,
|
|
101
|
+
V = proposal.V,
|
|
102
|
+
A = proposal.A,
|
|
103
|
+
B = proposal.B,
|
|
104
|
+
D = proposal.D,
|
|
105
|
+
HA = proposal.HA,
|
|
106
|
+
HB = proposal.HB,
|
|
107
|
+
T = Clock.Now(),
|
|
108
|
+
}
|
|
109
|
+
end)
|
|
110
|
+
end)
|
|
111
|
+
|
|
112
|
+
if not ok then
|
|
113
|
+
-- Includes the indeterminate case, where the write may well have landed. The
|
|
114
|
+
-- caller must NOT treat this as "no verdict exists"; it means "no verdict is
|
|
115
|
+
-- KNOWN", and the next proposal settles which.
|
|
116
|
+
return nil, false, tostring(committed)
|
|
117
|
+
end
|
|
118
|
+
if committed == nil then
|
|
119
|
+
return nil, false, "verdict store returned nothing, which should be unreachable"
|
|
120
|
+
end
|
|
121
|
+
if not wellFormed(committed) then
|
|
122
|
+
return nil, false, "verdict key holds a malformed record"
|
|
123
|
+
end
|
|
124
|
+
if key ~= exchangeId and committed.Id ~= exchangeId then
|
|
125
|
+
return nil, false, "verdict key belongs to another exchange"
|
|
126
|
+
end
|
|
127
|
+
-- `decided` is set inside the transform, which can run more than once under
|
|
128
|
+
-- contention; the return value is the authority on WHAT was decided, and this
|
|
129
|
+
-- flag only reports whether the last transform pass wrote.
|
|
130
|
+
return committed, decided and committed.T ~= nil, nil
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
--- Read the verdict in force, without proposing one.
|
|
134
|
+
---
|
|
135
|
+
--- Nil means no verdict this read can see, which is not the same as "no verdict exists":
|
|
136
|
+
--- a proposal that landed indeterminately may still be unread. Only a proposal settles it.
|
|
137
|
+
function ExchangeVerdict.Read(store: any, exchangeId: string): (any, string?)
|
|
138
|
+
if type(exchangeId) ~= "string" or #exchangeId == 0 then
|
|
139
|
+
fail("Read needs an exchange id")
|
|
140
|
+
end
|
|
141
|
+
local key, keyError = storeKey(exchangeId)
|
|
142
|
+
if key == nil then
|
|
143
|
+
return nil, keyError
|
|
144
|
+
end
|
|
145
|
+
local ok, value = pcall(function()
|
|
146
|
+
return store:GetAsync(key)
|
|
147
|
+
end)
|
|
148
|
+
if not ok then
|
|
149
|
+
return nil, tostring(value)
|
|
150
|
+
end
|
|
151
|
+
if value == nil then
|
|
152
|
+
return nil, nil
|
|
153
|
+
end
|
|
154
|
+
if not wellFormed(value) then
|
|
155
|
+
return nil, "verdict key holds a malformed record"
|
|
156
|
+
end
|
|
157
|
+
if key ~= exchangeId and value.Id ~= exchangeId then
|
|
158
|
+
return nil, "verdict key belongs to another exchange"
|
|
159
|
+
end
|
|
160
|
+
return value, nil
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
--- Does this verdict govern this exchange, for this pair, on this basket? A mismatch is
|
|
164
|
+
--- corruption, not an expected branch, and the caller must refuse to settle rather than
|
|
165
|
+
--- settle against the wrong basket.
|
|
166
|
+
function ExchangeVerdict.Matches(verdict: any, digest: string, userIdA: number, userIdB: number): (boolean, string?)
|
|
167
|
+
if not wellFormed(verdict) then
|
|
168
|
+
return false, "malformed verdict"
|
|
169
|
+
end
|
|
170
|
+
if verdict.D ~= digest then
|
|
171
|
+
return false, "digest mismatch"
|
|
172
|
+
end
|
|
173
|
+
if verdict.A ~= userIdA or verdict.B ~= userIdB then
|
|
174
|
+
return false, "participant mismatch"
|
|
175
|
+
end
|
|
176
|
+
return true, nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
return ExchangeVerdict
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- Classifies a DataStore failure and reports it. Failure.Report is the single entry
|
|
3
|
+
-- point every failure site uses.
|
|
4
|
+
-- Throttled dropped; back off, retrying makes it worse
|
|
5
|
+
-- Failed transient service-side; a retry may work
|
|
6
|
+
-- Unresolved timed out and the write may have landed; never retry blindly
|
|
7
|
+
-- Rejected permanently invalid; every retry fails identically
|
|
8
|
+
-- Only the numeric prefix of the error string is read; the prose is not a contract.
|
|
9
|
+
-- An unrecognised code classifies as Failed, with the code in the log context.
|
|
10
|
+
|
|
11
|
+
local Health = require(script.Parent.Health)
|
|
12
|
+
local Log = require(script.Parent.Log)
|
|
13
|
+
local Metrics = require(script.Parent.Metrics)
|
|
14
|
+
local Types = require(script.Parent.Parent.Types)
|
|
15
|
+
|
|
16
|
+
local Failure = {}
|
|
17
|
+
|
|
18
|
+
export type Class = "Throttled" | "Failed" | "Unresolved" | "Rejected"
|
|
19
|
+
|
|
20
|
+
-- Codes whose class does not follow from its family. Everything else falls back
|
|
21
|
+
-- to the family rule below, so a code Roblox adds inside a known family is
|
|
22
|
+
-- classified sensibly without this table being updated.
|
|
23
|
+
local EXPLICIT: { [number]: Class } = {
|
|
24
|
+
-- A timeout is not a failure: the write may have committed before the error
|
|
25
|
+
-- came back.
|
|
26
|
+
[504] = "Unresolved",
|
|
27
|
+
-- Metadata caps sit in the 5xx family but are permanent misuse, not service
|
|
28
|
+
-- trouble: the same call will fail the same way forever.
|
|
29
|
+
[511] = "Rejected",
|
|
30
|
+
[512] = "Rejected",
|
|
31
|
+
[513] = "Rejected",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
--- Class for a whole code family, used when EXPLICIT has no entry.
|
|
35
|
+
---
|
|
36
|
+
--- 1xx is input validation, which Roblox rejects on the same input every time.
|
|
37
|
+
--- 3xx is throttling. 5xx is the service having trouble.
|
|
38
|
+
local function byFamily(code: number): Class
|
|
39
|
+
if code >= 100 and code < 200 then
|
|
40
|
+
return "Rejected"
|
|
41
|
+
elseif code >= 300 and code < 400 then
|
|
42
|
+
return "Throttled"
|
|
43
|
+
elseif code >= 500 and code < 600 then
|
|
44
|
+
return "Failed"
|
|
45
|
+
end
|
|
46
|
+
return "Failed"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
--- The numeric code a DataStore error message carries, or nil if it carries none.
|
|
50
|
+
--- Anchored on purpose: a loose search for digits-then-colon would eventually match a
|
|
51
|
+
--- number inside a key name and report a confident wrong class. nil classifies as
|
|
52
|
+
--- Failed, the safe default.
|
|
53
|
+
function Failure.CodeOf(message: any): number?
|
|
54
|
+
if type(message) ~= "string" then
|
|
55
|
+
return nil
|
|
56
|
+
end
|
|
57
|
+
return tonumber(string.match(message, "^%s*(%d+):"))
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
--- Classify a DataStore error message.
|
|
61
|
+
function Failure.Classify(message: any): Class
|
|
62
|
+
local code = Failure.CodeOf(message)
|
|
63
|
+
if code == nil then
|
|
64
|
+
return "Failed"
|
|
65
|
+
end
|
|
66
|
+
return EXPLICIT[code] or byFamily(code)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
-- One failed operation, reported everywhere it belongs -----------------------
|
|
70
|
+
--
|
|
71
|
+
-- Health collapses retries of one operation into a single problem. The log follows
|
|
72
|
+
-- Health's own verdict rather than approximating it, so the identity rule (subsystem
|
|
73
|
+
-- namespacing, the shared throttle subject, the retry cooldown) stays in one place.
|
|
74
|
+
--
|
|
75
|
+
-- Collapsing is keyed on (subsystem, subject), so a report with no Key is never
|
|
76
|
+
-- collapsed and always logs. A burst that ends never prints its tail, because the tally
|
|
77
|
+
-- dies with its identity; `FailuresCollapsed` always moves, and the Repeats field on the
|
|
78
|
+
-- next line covers the rest.
|
|
79
|
+
export type Report = {
|
|
80
|
+
-- Health subsystem: a fixed internal literal, never a value from outside.
|
|
81
|
+
Subsystem: string,
|
|
82
|
+
-- The subject of the operation, a profile key or a user id. Omit it only where
|
|
83
|
+
-- there genuinely is none: omitting it means retries count and log separately.
|
|
84
|
+
Key: string?,
|
|
85
|
+
-- The raw error, classified. Supplying it is what enables the per-class metric
|
|
86
|
+
-- dimension, because a class derived from nothing is not a measurement.
|
|
87
|
+
Error: any?,
|
|
88
|
+
-- Already classified by the caller (the ProfileStore OnError path classifies
|
|
89
|
+
-- before it knows whether it has an entry to attribute to).
|
|
90
|
+
Class: Class?,
|
|
91
|
+
-- One counter family. Bumped as-is, plus `<Metric>_<Class>` when a real error
|
|
92
|
+
-- was classified. Most sites need nothing here.
|
|
93
|
+
Metric: string?,
|
|
94
|
+
Level: ("Warn" | "Error")?,
|
|
95
|
+
Category: Types.LogCategory,
|
|
96
|
+
Code: Types.LogCode,
|
|
97
|
+
Message: string,
|
|
98
|
+
Context: { [string]: any }?,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
--- Report one failed operation: classify it, count it, and say so once.
|
|
102
|
+
---
|
|
103
|
+
--- Returns `(class, logged)`. `logged` is false when this report was folded into
|
|
104
|
+
--- a live problem for the same subject.
|
|
105
|
+
function Failure.Report(report: Report): (Class, boolean)
|
|
106
|
+
local classified = report.Error ~= nil or report.Class ~= nil
|
|
107
|
+
local class: Class = report.Class or Failure.Classify(report.Error)
|
|
108
|
+
|
|
109
|
+
local counted, folded = Health.ReportFailure(report.Subsystem, report.Key, class)
|
|
110
|
+
|
|
111
|
+
local metric = report.Metric
|
|
112
|
+
if metric ~= nil then
|
|
113
|
+
Metrics.Add(metric)
|
|
114
|
+
-- Only where a real error message was read: a class defaulted from nothing
|
|
115
|
+
-- reads as a measurement and is not one.
|
|
116
|
+
if classified then
|
|
117
|
+
Metrics.Add(metric .. "_" .. class)
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
if not counted then
|
|
122
|
+
Metrics.Add("FailuresCollapsed")
|
|
123
|
+
return class, false
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
local context = report.Context
|
|
127
|
+
if folded > 0 then
|
|
128
|
+
-- Copied rather than mutated: the caller's table may be a literal it also
|
|
129
|
+
-- uses elsewhere.
|
|
130
|
+
local merged: { [string]: any } = {}
|
|
131
|
+
if context then
|
|
132
|
+
for key, value in context do
|
|
133
|
+
merged[key] = value
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
merged.Repeats = folded
|
|
137
|
+
context = merged
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
if report.Level == "Error" then
|
|
141
|
+
Log.Error(report.Category, report.Code, report.Message, context)
|
|
142
|
+
else
|
|
143
|
+
Log.Warn(report.Category, report.Code, report.Message, context)
|
|
144
|
+
end
|
|
145
|
+
return class, true
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
return Failure
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- Health status machine with hysteresis.
|
|
3
|
+
-- Healthy -> Degraded (N failures within a window, or critical state on)
|
|
4
|
+
-- -> Outage (sustained) ; recovers on a success streak.
|
|
5
|
+
-- Library-level singleton. On the client this only stores the broadcast status.
|
|
6
|
+
|
|
7
|
+
local Signal = require(script.Parent.Signal)
|
|
8
|
+
local Log = require(script.Parent.Log)
|
|
9
|
+
local Metrics = require(script.Parent.Metrics)
|
|
10
|
+
local Clock = require(script.Parent.Clock)
|
|
11
|
+
|
|
12
|
+
export type Status = "Healthy" | "Degraded" | "Outage"
|
|
13
|
+
|
|
14
|
+
local Health = {}
|
|
15
|
+
|
|
16
|
+
Health.OnStatusChanged = Signal.new() :: Signal.Signal<Status, Status> -- (newStatus: Status, oldStatus: Status)
|
|
17
|
+
|
|
18
|
+
local thresholds = {
|
|
19
|
+
FailWindow = 60, -- seconds
|
|
20
|
+
FailCount = 3, -- failures within window: Healthy -> Degraded
|
|
21
|
+
RecoverStreak = 5, -- consecutive successes to step one state down
|
|
22
|
+
-- Failures within window -> Outage. Derived by the same formula Configure uses, so a
|
|
23
|
+
-- StatusThresholds table that leaves FailCount alone cannot silently shift it.
|
|
24
|
+
OutageFailCount = math.max(3 * 2, 3 + 1),
|
|
25
|
+
CriticalOutageDelay = 30, -- seconds of sustained ProfileStore critical state -> Outage
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
-- A throttle is one problem no matter how many keys hit it, so every throttled failure
|
|
29
|
+
-- in a subsystem shares this subject. Being throttled is not the store being broken, and
|
|
30
|
+
-- counting a join burst as N problems drives the server past Outage, which refuses Robux
|
|
31
|
+
-- receipts while nothing is wrong. Not namespaced per key: the burst is simultaneous, so
|
|
32
|
+
-- per-key dedup would let N through at once.
|
|
33
|
+
local THROTTLE_SUBJECT = "~throttled"
|
|
34
|
+
|
|
35
|
+
-- Smallest gap between two failures of the same operation that counts as two separate
|
|
36
|
+
-- problems. ProfileStore retries a failing key on its own backoff and every attempt
|
|
37
|
+
-- arrives here, so without this one stuck profile reaches Outage alone. Set above
|
|
38
|
+
-- ProfileStore's 20 s backoff ceiling so a genuinely new failure of the same key counts.
|
|
39
|
+
local RETRY_COOLDOWN = 25
|
|
40
|
+
|
|
41
|
+
local status: Status = "Healthy"
|
|
42
|
+
-- Sorted by At: entries are only ever appended with a non-decreasing monotonic clock,
|
|
43
|
+
-- and pruneFailures depends on that to stop at the first survivor.
|
|
44
|
+
local failures: { { At: number, Id: string? } } = {}
|
|
45
|
+
-- Newest in-window failure per identity, for the retry cooldown. The cooldown is shorter
|
|
46
|
+
-- than the window, so pruneFailures is enough to bound this.
|
|
47
|
+
local lastAt: { [string]: number } = {}
|
|
48
|
+
local counts: { [string]: number } = {}
|
|
49
|
+
-- How many reports have been folded into the identity's current live problem, so the
|
|
50
|
+
-- line eventually logged can say how many it stands for. Kept beside `counts` because
|
|
51
|
+
-- `forget` retires an identity at the right moment; a tally that outlives its identity
|
|
52
|
+
-- is a leak on a long-lived server.
|
|
53
|
+
local folded: { [string]: number } = {}
|
|
54
|
+
-- Maintained incrementally so evaluate() stays O(1) on a path that only runs while
|
|
55
|
+
-- things are already going wrong.
|
|
56
|
+
local distinctIds = 0
|
|
57
|
+
local unkeyed = 0
|
|
58
|
+
local successStreak = 0
|
|
59
|
+
local criticalSince: number? = nil
|
|
60
|
+
|
|
61
|
+
local function setStatus(new: Status)
|
|
62
|
+
if new == status then
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
local old = status
|
|
66
|
+
status = new
|
|
67
|
+
Metrics.Set("HealthStatus", if new == "Healthy" then 0 elseif new == "Degraded" then 1 else 2)
|
|
68
|
+
if new == "Healthy" then
|
|
69
|
+
Log.Info("Persistence", "STATUS_CHANGED", `service status {old} -> {new}`, { From = old, To = new })
|
|
70
|
+
else
|
|
71
|
+
Log.Warn("Persistence", "STATUS_CHANGED", `service status {old} -> {new}`, { From = old, To = new })
|
|
72
|
+
end
|
|
73
|
+
Health.OnStatusChanged:Fire(new, old)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
local function forget(entry: { At: number, Id: string? })
|
|
77
|
+
local id = entry.Id
|
|
78
|
+
if id == nil then
|
|
79
|
+
unkeyed -= 1
|
|
80
|
+
return
|
|
81
|
+
end
|
|
82
|
+
local left = (counts[id] or 1) - 1
|
|
83
|
+
if left <= 0 then
|
|
84
|
+
counts[id] = nil
|
|
85
|
+
lastAt[id] = nil
|
|
86
|
+
folded[id] = nil
|
|
87
|
+
distinctIds -= 1
|
|
88
|
+
else
|
|
89
|
+
counts[id] = left
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
local function pruneFailures(now: number)
|
|
94
|
+
local cutoff = now - thresholds.FailWindow
|
|
95
|
+
while failures[1] and failures[1].At < cutoff do
|
|
96
|
+
forget(table.remove(failures, 1) :: any)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
-- Lifting Outage once the failures stop --------------------------------------
|
|
101
|
+
-- evaluate() only escalates and a quiet server never calls it, so a burst that ends
|
|
102
|
+
-- would leave Outage forever. The step condition is an empty window, not a count under
|
|
103
|
+
-- the threshold: with the sparse stream the retry cooldown guarantees, the count dips
|
|
104
|
+
-- between retries and the status would flap, opening the money gate mid-outage. It
|
|
105
|
+
-- stops at Degraded, because no reports is not evidence anything worked.
|
|
106
|
+
local DECAY_MIN_DELAY = 1 -- seconds; the horizon can land on the prune boundary, where an entry survives
|
|
107
|
+
local decayPending = false
|
|
108
|
+
local armDecay: () -> ()
|
|
109
|
+
|
|
110
|
+
local function decayCheck()
|
|
111
|
+
decayPending = false
|
|
112
|
+
if status ~= "Outage" then
|
|
113
|
+
return
|
|
114
|
+
end
|
|
115
|
+
pruneFailures(Clock.Monotonic())
|
|
116
|
+
-- Re-armed on every path, not just the one that steps: a check that declines and then
|
|
117
|
+
-- dies is a decay that never happens.
|
|
118
|
+
if criticalSince or #failures > 0 then
|
|
119
|
+
armDecay()
|
|
120
|
+
return
|
|
121
|
+
end
|
|
122
|
+
successStreak = 0
|
|
123
|
+
setStatus("Degraded")
|
|
124
|
+
armDecay()
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
function armDecay()
|
|
128
|
+
if decayPending or status ~= "Outage" then
|
|
129
|
+
return
|
|
130
|
+
end
|
|
131
|
+
decayPending = true
|
|
132
|
+
local newest = failures[#failures]
|
|
133
|
+
local now = Clock.Monotonic()
|
|
134
|
+
local delay = if newest then (newest.At + thresholds.FailWindow) - now else thresholds.FailWindow
|
|
135
|
+
Clock.Delay(math.max(delay, DECAY_MIN_DELAY), decayCheck)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
local function evaluate()
|
|
139
|
+
local now = Clock.Monotonic()
|
|
140
|
+
pruneFailures(now)
|
|
141
|
+
local recent = #failures
|
|
142
|
+
|
|
143
|
+
if criticalSince and now - criticalSince >= thresholds.CriticalOutageDelay then
|
|
144
|
+
-- ProfileStore's critical flag has no key dimension, so one permanently-failing
|
|
145
|
+
-- profile trips it unaided. Treat it as corroboration, not an independent trigger:
|
|
146
|
+
-- it forces Outage only when Scribe's own failures span more than one operation.
|
|
147
|
+
if distinctIds + unkeyed >= 2 then
|
|
148
|
+
setStatus("Outage")
|
|
149
|
+
-- Without this re-arm, the critical-state path is the one entry to Outage that
|
|
150
|
+
-- never schedules a decay, so a server that escalated here and then went quiet
|
|
151
|
+
-- stays in Outage forever.
|
|
152
|
+
armDecay()
|
|
153
|
+
return
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
if recent >= thresholds.OutageFailCount then
|
|
158
|
+
setStatus("Outage")
|
|
159
|
+
elseif recent >= thresholds.FailCount or criticalSince then
|
|
160
|
+
if status == "Healthy" then
|
|
161
|
+
setStatus("Degraded")
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
armDecay()
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
-- What has been explicitly configured, and which conflicts have been reported. This
|
|
168
|
+
-- machine is a library-level singleton, so a second bundle in the same process retunes
|
|
169
|
+
-- the first's Outage gate, the gate that refuses Robux grants during a DataStore
|
|
170
|
+
-- failure. The later value still wins, matching SAVE_INTERVAL_CONFLICT, but it warns.
|
|
171
|
+
local configured: { [string]: number } = {}
|
|
172
|
+
local conflictWarned: { [string]: true } = {}
|
|
173
|
+
|
|
174
|
+
local CONFIGURABLE = { "FailWindow", "FailCount", "RecoverStreak" }
|
|
175
|
+
|
|
176
|
+
function Health.Configure(config: { FailWindow: number?, FailCount: number?, RecoverStreak: number? }?)
|
|
177
|
+
if not config then
|
|
178
|
+
return
|
|
179
|
+
end
|
|
180
|
+
for _, field in CONFIGURABLE do
|
|
181
|
+
local requested = (config :: any)[field]
|
|
182
|
+
if requested ~= nil then
|
|
183
|
+
local existing = configured[field]
|
|
184
|
+
-- Once per field, not once per call: a process that rebuilds bundles would
|
|
185
|
+
-- otherwise turn one real signal into a line per build.
|
|
186
|
+
if existing ~= nil and existing ~= requested and not conflictWarned[field] then
|
|
187
|
+
conflictWarned[field] = true
|
|
188
|
+
-- "Persistence" because the health machine has no subsystem of its own in
|
|
189
|
+
-- the Types.LogSubsystem union.
|
|
190
|
+
Log.Warn(
|
|
191
|
+
"Persistence",
|
|
192
|
+
"HEALTH_THRESHOLDS_CONFLICT",
|
|
193
|
+
"the health thresholds are a process global shared by every Scribe bundle and were already set "
|
|
194
|
+
.. "to a different value; the later value wins. Set them once, identically, in every bundle.",
|
|
195
|
+
{ Field = field, Existing = existing, Requested = requested }
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
local live: any = thresholds
|
|
199
|
+
live[field] = requested
|
|
200
|
+
configured[field] = requested
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
thresholds.OutageFailCount = math.max(thresholds.FailCount * 2, thresholds.FailCount + 1)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
-- The thresholds as they currently stand, copied so a reader cannot edit the singleton
|
|
207
|
+
-- by accident. OutageFailCount is the interesting member: it gates money, it is rebuilt
|
|
208
|
+
-- from FailCount on every Configure, and there is otherwise no way to see which won.
|
|
209
|
+
function Health.Thresholds(): { [string]: number }
|
|
210
|
+
return table.clone(thresholds)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
-- Put the singleton back to its declared defaults, including the record of what has
|
|
214
|
+
-- been configured, so the first Configure after a teardown is not compared against a
|
|
215
|
+
-- bundle that no longer exists.
|
|
216
|
+
function Health.ResetThresholds()
|
|
217
|
+
thresholds.FailWindow = 60
|
|
218
|
+
thresholds.FailCount = 3
|
|
219
|
+
thresholds.RecoverStreak = 5
|
|
220
|
+
thresholds.OutageFailCount = math.max(3 * 2, 3 + 1)
|
|
221
|
+
thresholds.CriticalOutageDelay = 30
|
|
222
|
+
table.clear(configured)
|
|
223
|
+
table.clear(conflictWarned)
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
--- Report a failed DataStore/OrderedDataStore/MessageAsync operation. `key` names the
|
|
227
|
+
--- subject so repeated failures of one thing collapse into one problem; omit it only
|
|
228
|
+
--- where there is no subject. Of the `class` verdicts only "Throttled" changes anything.
|
|
229
|
+
--- Returns `(counted, folded)`: whether this became a distinct problem rather than being
|
|
230
|
+
--- absorbed into a live one, and how many reports that problem has absorbed.
|
|
231
|
+
function Health.ReportFailure(subsystem: string, key: string?, class: string?): (boolean, number)
|
|
232
|
+
successStreak = 0
|
|
233
|
+
-- The metric still counts every report; only the state machine below dedupes.
|
|
234
|
+
Metrics.Add("HealthFailures")
|
|
235
|
+
Metrics.Add("HealthFailures_" .. subsystem)
|
|
236
|
+
|
|
237
|
+
local now = Clock.Monotonic()
|
|
238
|
+
-- Namespaced by subsystem so a version read and a save failure on the same key stay
|
|
239
|
+
-- independent problems. Subsystem names are fixed internal literals and never contain
|
|
240
|
+
-- the separator, so the pair cannot be ambiguous.
|
|
241
|
+
local id = if class == "Throttled"
|
|
242
|
+
then subsystem .. "|" .. THROTTLE_SUBJECT
|
|
243
|
+
elseif key ~= nil then subsystem .. "|" .. key
|
|
244
|
+
else nil
|
|
245
|
+
local flushed = 0
|
|
246
|
+
if id ~= nil then
|
|
247
|
+
local seen = lastAt[id]
|
|
248
|
+
if seen ~= nil and now - seen < RETRY_COOLDOWN then
|
|
249
|
+
-- Same operation, still inside its retry window: already counted.
|
|
250
|
+
local tally = (folded[id] or 0) + 1
|
|
251
|
+
folded[id] = tally
|
|
252
|
+
evaluate()
|
|
253
|
+
return false, tally
|
|
254
|
+
end
|
|
255
|
+
lastAt[id] = now
|
|
256
|
+
flushed = folded[id] or 0
|
|
257
|
+
folded[id] = nil
|
|
258
|
+
local n = (counts[id] or 0) + 1
|
|
259
|
+
counts[id] = n
|
|
260
|
+
if n == 1 then
|
|
261
|
+
distinctIds += 1
|
|
262
|
+
end
|
|
263
|
+
else
|
|
264
|
+
unkeyed += 1
|
|
265
|
+
end
|
|
266
|
+
table.insert(failures, { At = now, Id = id })
|
|
267
|
+
evaluate()
|
|
268
|
+
return true, flushed
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
-- Report a successful operation; a streak steps the status back down.
|
|
272
|
+
function Health.ReportSuccess(_subsystem: string?)
|
|
273
|
+
if status == "Healthy" then
|
|
274
|
+
successStreak = 0
|
|
275
|
+
return
|
|
276
|
+
end
|
|
277
|
+
successStreak += 1
|
|
278
|
+
if successStreak >= thresholds.RecoverStreak and not criticalSince then
|
|
279
|
+
successStreak = 0
|
|
280
|
+
table.clear(failures)
|
|
281
|
+
table.clear(lastAt)
|
|
282
|
+
table.clear(counts)
|
|
283
|
+
table.clear(folded)
|
|
284
|
+
distinctIds = 0
|
|
285
|
+
unkeyed = 0
|
|
286
|
+
setStatus(if status == "Outage" then "Degraded" else "Healthy")
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
-- Bound to ProfileStore.OnCriticalToggle.
|
|
291
|
+
function Health.SetCritical(isCritical: boolean)
|
|
292
|
+
if isCritical then
|
|
293
|
+
if not criticalSince then
|
|
294
|
+
criticalSince = Clock.Monotonic()
|
|
295
|
+
-- evaluate() runs only on ReportFailure/SetCritical, so a sustained critical
|
|
296
|
+
-- window with no further failures would never escalate. It re-reads elapsed
|
|
297
|
+
-- time, so a cleared or freshly-reset critical state won't escalate early.
|
|
298
|
+
Clock.Delay(thresholds.CriticalOutageDelay, function()
|
|
299
|
+
if criticalSince then
|
|
300
|
+
evaluate()
|
|
301
|
+
end
|
|
302
|
+
end)
|
|
303
|
+
end
|
|
304
|
+
evaluate()
|
|
305
|
+
else
|
|
306
|
+
criticalSince = nil
|
|
307
|
+
-- Leave recovery to the success streak.
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
function Health.GetStatus(): Status
|
|
312
|
+
return status
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
-- Client side: adopt a status broadcast from the server.
|
|
316
|
+
function Health.AdoptBroadcast(new: Status)
|
|
317
|
+
setStatus(new)
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
return Health
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- Passive registry; each running bundle owns its provider.
|
|
3
|
+
|
|
4
|
+
local Types = require(script.Parent.Parent.Types)
|
|
5
|
+
|
|
6
|
+
type Provider = () -> { [string]: Types.LeaderboardStatusInfo }
|
|
7
|
+
|
|
8
|
+
local Diagnostics = {}
|
|
9
|
+
local nextId = 0
|
|
10
|
+
local providers: { [number]: Provider } = setmetatable({}, { __mode = "v" }) :: any
|
|
11
|
+
|
|
12
|
+
function Diagnostics.Register(provider: Provider): () -> ()
|
|
13
|
+
nextId += 1
|
|
14
|
+
local id = nextId
|
|
15
|
+
providers[id] = provider
|
|
16
|
+
return function()
|
|
17
|
+
providers[id] = nil
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
function Diagnostics.Snapshot(): { Types.LeaderboardSnapshotEntry }
|
|
22
|
+
local out: { Types.LeaderboardSnapshotEntry } = {}
|
|
23
|
+
for id, provider in providers do
|
|
24
|
+
for name, status in provider() do
|
|
25
|
+
local entry = table.clone(status) :: any
|
|
26
|
+
entry.BundleId, entry.Name = id, name
|
|
27
|
+
table.insert(out, entry)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
table.sort(out, function(a: Types.LeaderboardSnapshotEntry, b: Types.LeaderboardSnapshotEntry)
|
|
31
|
+
return if a.BundleId == b.BundleId then a.Name < b.Name else a.BundleId < b.BundleId
|
|
32
|
+
end)
|
|
33
|
+
return out
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
return Diagnostics
|