@rbxts/scribe 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/LICENSE +21 -0
  2. package/NOTICE +21 -0
  3. package/README.md +100 -0
  4. package/Scribe-source-map.json +1 -0
  5. package/licenses/ProfileStore-Apache-2.0.txt +201 -0
  6. package/package.json +32 -0
  7. package/src/Client/DebugHook.luau +457 -0
  8. package/src/Client/init.luau +1607 -0
  9. package/src/Internal/Addons.luau +68 -0
  10. package/src/Internal/Big.luau +582 -0
  11. package/src/Internal/Budget.luau +377 -0
  12. package/src/Internal/Clock.luau +75 -0
  13. package/src/Internal/Config.luau +17 -0
  14. package/src/Internal/Conform.luau +400 -0
  15. package/src/Internal/Datatypes.luau +653 -0
  16. package/src/Internal/Defaults.luau +12 -0
  17. package/src/Internal/Derived.luau +428 -0
  18. package/src/Internal/Exchange.luau +556 -0
  19. package/src/Internal/ExchangeLeg.luau +1368 -0
  20. package/src/Internal/ExchangeVerdict.luau +179 -0
  21. package/src/Internal/Failure.luau +148 -0
  22. package/src/Internal/Health.luau +320 -0
  23. package/src/Internal/LeaderboardDiagnostics.luau +36 -0
  24. package/src/Internal/Log.luau +378 -0
  25. package/src/Internal/Marker.luau +76 -0
  26. package/src/Internal/Metrics.luau +152 -0
  27. package/src/Internal/Node.luau +3954 -0
  28. package/src/Internal/Profile.luau +69 -0
  29. package/src/Internal/Rng.luau +83 -0
  30. package/src/Internal/Sched.luau +56 -0
  31. package/src/Internal/Signal.luau +199 -0
  32. package/src/Internal/Store/ProfileStore.luau +3019 -0
  33. package/src/Internal/Store/init.luau +18 -0
  34. package/src/Internal/StudioCodec.luau +155 -0
  35. package/src/Internal/Template.luau +1848 -0
  36. package/src/Internal/Transport.luau +169 -0
  37. package/src/Internal/Util.luau +517 -0
  38. package/src/Internal/Version.luau +4 -0
  39. package/src/Internal/Wire.luau +1283 -0
  40. package/src/Server/Analytics.luau +223 -0
  41. package/src/Server/Commands.luau +728 -0
  42. package/src/Server/DebugHook.luau +1490 -0
  43. package/src/Server/Exchanges.luau +737 -0
  44. package/src/Server/Leaderboards.luau +2143 -0
  45. package/src/Server/Monetization.luau +3212 -0
  46. package/src/Server/Persistence.luau +3775 -0
  47. package/src/Server/Replication.luau +1253 -0
  48. package/src/Server/ServerStore.luau +81 -0
  49. package/src/Server/Timed.luau +579 -0
  50. package/src/Server/init.luau +672 -0
  51. package/src/Types.luau +480 -0
  52. package/src/init.luau +2231 -0
  53. package/types/api.d.ts +252 -0
  54. package/types/common.d.ts +355 -0
  55. package/types/index.d.ts +226 -0
  56. package/types/schema.d.ts +276 -0
@@ -0,0 +1,179 @@
1
+ --!strict
2
+ -- The verdict key: the single point of no return for an exchange. Two players are two
3
+ -- DataStore keys with no multi-key transaction, so ONE key decides and everything after
4
+ -- it is infallible or infinitely retryable. The only property owed: every participant,
5
+ -- on every server, on every retry, reads the SAME answer.
6
+ --
7
+ -- First writer wins and the loser learns from the return value. The transform never
8
+ -- returns nil: that aborts the write and yields (nil, nil), which the caller cannot tell
9
+ -- from "nothing happened". Indeterminate is not special: a proposer that never learned
10
+ -- its proposal won has not settled, and the next proposal reads the committed answer.
11
+ local Clock = require(script.Parent.Clock)
12
+
13
+ local ExchangeVerdict = {}
14
+
15
+ ExchangeVerdict.STORE_NAME = "ClaimExchangeVerdicts"
16
+
17
+ export type Verdict = {
18
+ Id: string?, -- original id for a recovered legacy key
19
+ V: string, -- "Commit" or "Abort"
20
+ A: number, -- userId of role A
21
+ B: number, -- userId of role B
22
+ D: string, -- basket digest
23
+ HA: string?, -- content hash of A's take payload
24
+ HB: string?,
25
+ T: number, -- when the winner decided
26
+ }
27
+
28
+ local function fail(message: string): ()
29
+ error(`ExchangeVerdict: {message}`, 0)
30
+ end
31
+
32
+ local function storeKey(exchangeId: string): (string?, string?)
33
+ if utf8.len(exchangeId) == nil then
34
+ return nil, "exchange id must be valid UTF-8"
35
+ end
36
+ if #exchangeId <= 50 then
37
+ return exchangeId, nil
38
+ end
39
+ local guid = string.match(exchangeId, "^%d+%-%d+%-(.+)$")
40
+ if guid ~= nil and #guid == 36 then
41
+ local groups = string.split(guid, "-")
42
+ local sizes = { 8, 4, 4, 4, 12 }
43
+ local valid = #groups == #sizes
44
+ for index, size in sizes do
45
+ local group = groups[index]
46
+ if group == nil or #group ~= size or string.match(group, "^%x+$") == nil then
47
+ valid = false
48
+ break
49
+ end
50
+ end
51
+ if valid then
52
+ -- Overlong legacy keys never reached Roblox. Preserve their profile records.
53
+ return `legacy:{guid}`, nil
54
+ end
55
+ end
56
+ return nil, "exchange id exceeds the 50 character limit and is not a recognized legacy id"
57
+ end
58
+
59
+ --- Is this a verdict record, rather than something else that happens to be stored?
60
+ local function wellFormed(value: any): boolean
61
+ return type(value) == "table"
62
+ and (value.V == "Commit" or value.V == "Abort")
63
+ and type(value.D) == "string"
64
+ and type(value.A) == "number"
65
+ and type(value.B) == "number"
66
+ end
67
+
68
+ --- Propose a verdict for an exchange, and return whatever verdict is now in force.
69
+ ---
70
+ --- Returns the committed verdict and whether THIS call decided it. A caller almost never
71
+ --- needs the second value, since winner and loser take the identical branch; it is there
72
+ --- for metrics and for the abort path, which logs differently when it lost to a Commit.
73
+ ---
74
+ --- Yields. Never call it with a transaction open.
75
+ function ExchangeVerdict.Propose(store: any, exchangeId: string, proposal: any): (any, boolean, string?)
76
+ if type(store) ~= "table" and type(store) ~= "userdata" then
77
+ fail("Propose needs a DataStore")
78
+ end
79
+ if type(exchangeId) ~= "string" or #exchangeId == 0 then
80
+ fail("Propose needs an exchange id")
81
+ end
82
+ if not wellFormed(proposal) then
83
+ fail("Propose needs a well-formed verdict of Commit or Abort")
84
+ end
85
+ local key, keyError = storeKey(exchangeId)
86
+ if key == nil then
87
+ return nil, false, keyError
88
+ end
89
+
90
+ local decided = false
91
+ local ok, committed = pcall(function()
92
+ return store:UpdateAsync(key, function(old)
93
+ -- NEVER nil. See the header: a nil return aborts the write and the caller
94
+ -- gets (nil, nil), which is indistinguishable from a failure.
95
+ if old ~= nil then
96
+ return old
97
+ end
98
+ decided = true
99
+ return {
100
+ Id = if key ~= exchangeId then exchangeId else nil,
101
+ V = proposal.V,
102
+ A = proposal.A,
103
+ B = proposal.B,
104
+ D = proposal.D,
105
+ HA = proposal.HA,
106
+ HB = proposal.HB,
107
+ T = Clock.Now(),
108
+ }
109
+ end)
110
+ end)
111
+
112
+ if not ok then
113
+ -- Includes the indeterminate case, where the write may well have landed. The
114
+ -- caller must NOT treat this as "no verdict exists"; it means "no verdict is
115
+ -- KNOWN", and the next proposal settles which.
116
+ return nil, false, tostring(committed)
117
+ end
118
+ if committed == nil then
119
+ return nil, false, "verdict store returned nothing, which should be unreachable"
120
+ end
121
+ if not wellFormed(committed) then
122
+ return nil, false, "verdict key holds a malformed record"
123
+ end
124
+ if key ~= exchangeId and committed.Id ~= exchangeId then
125
+ return nil, false, "verdict key belongs to another exchange"
126
+ end
127
+ -- `decided` is set inside the transform, which can run more than once under
128
+ -- contention; the return value is the authority on WHAT was decided, and this
129
+ -- flag only reports whether the last transform pass wrote.
130
+ return committed, decided and committed.T ~= nil, nil
131
+ end
132
+
133
+ --- Read the verdict in force, without proposing one.
134
+ ---
135
+ --- Nil means no verdict this read can see, which is not the same as "no verdict exists":
136
+ --- a proposal that landed indeterminately may still be unread. Only a proposal settles it.
137
+ function ExchangeVerdict.Read(store: any, exchangeId: string): (any, string?)
138
+ if type(exchangeId) ~= "string" or #exchangeId == 0 then
139
+ fail("Read needs an exchange id")
140
+ end
141
+ local key, keyError = storeKey(exchangeId)
142
+ if key == nil then
143
+ return nil, keyError
144
+ end
145
+ local ok, value = pcall(function()
146
+ return store:GetAsync(key)
147
+ end)
148
+ if not ok then
149
+ return nil, tostring(value)
150
+ end
151
+ if value == nil then
152
+ return nil, nil
153
+ end
154
+ if not wellFormed(value) then
155
+ return nil, "verdict key holds a malformed record"
156
+ end
157
+ if key ~= exchangeId and value.Id ~= exchangeId then
158
+ return nil, "verdict key belongs to another exchange"
159
+ end
160
+ return value, nil
161
+ end
162
+
163
+ --- Does this verdict govern this exchange, for this pair, on this basket? A mismatch is
164
+ --- corruption, not an expected branch, and the caller must refuse to settle rather than
165
+ --- settle against the wrong basket.
166
+ function ExchangeVerdict.Matches(verdict: any, digest: string, userIdA: number, userIdB: number): (boolean, string?)
167
+ if not wellFormed(verdict) then
168
+ return false, "malformed verdict"
169
+ end
170
+ if verdict.D ~= digest then
171
+ return false, "digest mismatch"
172
+ end
173
+ if verdict.A ~= userIdA or verdict.B ~= userIdB then
174
+ return false, "participant mismatch"
175
+ end
176
+ return true, nil
177
+ end
178
+
179
+ return ExchangeVerdict
@@ -0,0 +1,148 @@
1
+ --!strict
2
+ -- Classifies a DataStore failure and reports it. Failure.Report is the single entry
3
+ -- point every failure site uses.
4
+ -- Throttled dropped; back off, retrying makes it worse
5
+ -- Failed transient service-side; a retry may work
6
+ -- Unresolved timed out and the write may have landed; never retry blindly
7
+ -- Rejected permanently invalid; every retry fails identically
8
+ -- Only the numeric prefix of the error string is read; the prose is not a contract.
9
+ -- An unrecognised code classifies as Failed, with the code in the log context.
10
+
11
+ local Health = require(script.Parent.Health)
12
+ local Log = require(script.Parent.Log)
13
+ local Metrics = require(script.Parent.Metrics)
14
+ local Types = require(script.Parent.Parent.Types)
15
+
16
+ local Failure = {}
17
+
18
+ export type Class = "Throttled" | "Failed" | "Unresolved" | "Rejected"
19
+
20
+ -- Codes whose class does not follow from its family. Everything else falls back
21
+ -- to the family rule below, so a code Roblox adds inside a known family is
22
+ -- classified sensibly without this table being updated.
23
+ local EXPLICIT: { [number]: Class } = {
24
+ -- A timeout is not a failure: the write may have committed before the error
25
+ -- came back.
26
+ [504] = "Unresolved",
27
+ -- Metadata caps sit in the 5xx family but are permanent misuse, not service
28
+ -- trouble: the same call will fail the same way forever.
29
+ [511] = "Rejected",
30
+ [512] = "Rejected",
31
+ [513] = "Rejected",
32
+ }
33
+
34
+ --- Class for a whole code family, used when EXPLICIT has no entry.
35
+ ---
36
+ --- 1xx is input validation, which Roblox rejects on the same input every time.
37
+ --- 3xx is throttling. 5xx is the service having trouble.
38
+ local function byFamily(code: number): Class
39
+ if code >= 100 and code < 200 then
40
+ return "Rejected"
41
+ elseif code >= 300 and code < 400 then
42
+ return "Throttled"
43
+ elseif code >= 500 and code < 600 then
44
+ return "Failed"
45
+ end
46
+ return "Failed"
47
+ end
48
+
49
+ --- The numeric code a DataStore error message carries, or nil if it carries none.
50
+ --- Anchored on purpose: a loose search for digits-then-colon would eventually match a
51
+ --- number inside a key name and report a confident wrong class. nil classifies as
52
+ --- Failed, the safe default.
53
+ function Failure.CodeOf(message: any): number?
54
+ if type(message) ~= "string" then
55
+ return nil
56
+ end
57
+ return tonumber(string.match(message, "^%s*(%d+):"))
58
+ end
59
+
60
+ --- Classify a DataStore error message.
61
+ function Failure.Classify(message: any): Class
62
+ local code = Failure.CodeOf(message)
63
+ if code == nil then
64
+ return "Failed"
65
+ end
66
+ return EXPLICIT[code] or byFamily(code)
67
+ end
68
+
69
+ -- One failed operation, reported everywhere it belongs -----------------------
70
+ --
71
+ -- Health collapses retries of one operation into a single problem. The log follows
72
+ -- Health's own verdict rather than approximating it, so the identity rule (subsystem
73
+ -- namespacing, the shared throttle subject, the retry cooldown) stays in one place.
74
+ --
75
+ -- Collapsing is keyed on (subsystem, subject), so a report with no Key is never
76
+ -- collapsed and always logs. A burst that ends never prints its tail, because the tally
77
+ -- dies with its identity; `FailuresCollapsed` always moves, and the Repeats field on the
78
+ -- next line covers the rest.
79
+ export type Report = {
80
+ -- Health subsystem: a fixed internal literal, never a value from outside.
81
+ Subsystem: string,
82
+ -- The subject of the operation, a profile key or a user id. Omit it only where
83
+ -- there genuinely is none: omitting it means retries count and log separately.
84
+ Key: string?,
85
+ -- The raw error, classified. Supplying it is what enables the per-class metric
86
+ -- dimension, because a class derived from nothing is not a measurement.
87
+ Error: any?,
88
+ -- Already classified by the caller (the ProfileStore OnError path classifies
89
+ -- before it knows whether it has an entry to attribute to).
90
+ Class: Class?,
91
+ -- One counter family. Bumped as-is, plus `<Metric>_<Class>` when a real error
92
+ -- was classified. Most sites need nothing here.
93
+ Metric: string?,
94
+ Level: ("Warn" | "Error")?,
95
+ Category: Types.LogCategory,
96
+ Code: Types.LogCode,
97
+ Message: string,
98
+ Context: { [string]: any }?,
99
+ }
100
+
101
+ --- Report one failed operation: classify it, count it, and say so once.
102
+ ---
103
+ --- Returns `(class, logged)`. `logged` is false when this report was folded into
104
+ --- a live problem for the same subject.
105
+ function Failure.Report(report: Report): (Class, boolean)
106
+ local classified = report.Error ~= nil or report.Class ~= nil
107
+ local class: Class = report.Class or Failure.Classify(report.Error)
108
+
109
+ local counted, folded = Health.ReportFailure(report.Subsystem, report.Key, class)
110
+
111
+ local metric = report.Metric
112
+ if metric ~= nil then
113
+ Metrics.Add(metric)
114
+ -- Only where a real error message was read: a class defaulted from nothing
115
+ -- reads as a measurement and is not one.
116
+ if classified then
117
+ Metrics.Add(metric .. "_" .. class)
118
+ end
119
+ end
120
+
121
+ if not counted then
122
+ Metrics.Add("FailuresCollapsed")
123
+ return class, false
124
+ end
125
+
126
+ local context = report.Context
127
+ if folded > 0 then
128
+ -- Copied rather than mutated: the caller's table may be a literal it also
129
+ -- uses elsewhere.
130
+ local merged: { [string]: any } = {}
131
+ if context then
132
+ for key, value in context do
133
+ merged[key] = value
134
+ end
135
+ end
136
+ merged.Repeats = folded
137
+ context = merged
138
+ end
139
+
140
+ if report.Level == "Error" then
141
+ Log.Error(report.Category, report.Code, report.Message, context)
142
+ else
143
+ Log.Warn(report.Category, report.Code, report.Message, context)
144
+ end
145
+ return class, true
146
+ end
147
+
148
+ return Failure
@@ -0,0 +1,320 @@
1
+ --!strict
2
+ -- Health status machine with hysteresis.
3
+ -- Healthy -> Degraded (N failures within a window, or critical state on)
4
+ -- -> Outage (sustained) ; recovers on a success streak.
5
+ -- Library-level singleton. On the client this only stores the broadcast status.
6
+
7
+ local Signal = require(script.Parent.Signal)
8
+ local Log = require(script.Parent.Log)
9
+ local Metrics = require(script.Parent.Metrics)
10
+ local Clock = require(script.Parent.Clock)
11
+
12
+ export type Status = "Healthy" | "Degraded" | "Outage"
13
+
14
+ local Health = {}
15
+
16
+ Health.OnStatusChanged = Signal.new() :: Signal.Signal<Status, Status> -- (newStatus: Status, oldStatus: Status)
17
+
18
+ local thresholds = {
19
+ FailWindow = 60, -- seconds
20
+ FailCount = 3, -- failures within window: Healthy -> Degraded
21
+ RecoverStreak = 5, -- consecutive successes to step one state down
22
+ -- Failures within window -> Outage. Derived by the same formula Configure uses, so a
23
+ -- StatusThresholds table that leaves FailCount alone cannot silently shift it.
24
+ OutageFailCount = math.max(3 * 2, 3 + 1),
25
+ CriticalOutageDelay = 30, -- seconds of sustained ProfileStore critical state -> Outage
26
+ }
27
+
28
+ -- A throttle is one problem no matter how many keys hit it, so every throttled failure
29
+ -- in a subsystem shares this subject. Being throttled is not the store being broken, and
30
+ -- counting a join burst as N problems drives the server past Outage, which refuses Robux
31
+ -- receipts while nothing is wrong. Not namespaced per key: the burst is simultaneous, so
32
+ -- per-key dedup would let N through at once.
33
+ local THROTTLE_SUBJECT = "~throttled"
34
+
35
+ -- Smallest gap between two failures of the same operation that counts as two separate
36
+ -- problems. ProfileStore retries a failing key on its own backoff and every attempt
37
+ -- arrives here, so without this one stuck profile reaches Outage alone. Set above
38
+ -- ProfileStore's 20 s backoff ceiling so a genuinely new failure of the same key counts.
39
+ local RETRY_COOLDOWN = 25
40
+
41
+ local status: Status = "Healthy"
42
+ -- Sorted by At: entries are only ever appended with a non-decreasing monotonic clock,
43
+ -- and pruneFailures depends on that to stop at the first survivor.
44
+ local failures: { { At: number, Id: string? } } = {}
45
+ -- Newest in-window failure per identity, for the retry cooldown. The cooldown is shorter
46
+ -- than the window, so pruneFailures is enough to bound this.
47
+ local lastAt: { [string]: number } = {}
48
+ local counts: { [string]: number } = {}
49
+ -- How many reports have been folded into the identity's current live problem, so the
50
+ -- line eventually logged can say how many it stands for. Kept beside `counts` because
51
+ -- `forget` retires an identity at the right moment; a tally that outlives its identity
52
+ -- is a leak on a long-lived server.
53
+ local folded: { [string]: number } = {}
54
+ -- Maintained incrementally so evaluate() stays O(1) on a path that only runs while
55
+ -- things are already going wrong.
56
+ local distinctIds = 0
57
+ local unkeyed = 0
58
+ local successStreak = 0
59
+ local criticalSince: number? = nil
60
+
61
+ local function setStatus(new: Status)
62
+ if new == status then
63
+ return
64
+ end
65
+ local old = status
66
+ status = new
67
+ Metrics.Set("HealthStatus", if new == "Healthy" then 0 elseif new == "Degraded" then 1 else 2)
68
+ if new == "Healthy" then
69
+ Log.Info("Persistence", "STATUS_CHANGED", `service status {old} -> {new}`, { From = old, To = new })
70
+ else
71
+ Log.Warn("Persistence", "STATUS_CHANGED", `service status {old} -> {new}`, { From = old, To = new })
72
+ end
73
+ Health.OnStatusChanged:Fire(new, old)
74
+ end
75
+
76
+ local function forget(entry: { At: number, Id: string? })
77
+ local id = entry.Id
78
+ if id == nil then
79
+ unkeyed -= 1
80
+ return
81
+ end
82
+ local left = (counts[id] or 1) - 1
83
+ if left <= 0 then
84
+ counts[id] = nil
85
+ lastAt[id] = nil
86
+ folded[id] = nil
87
+ distinctIds -= 1
88
+ else
89
+ counts[id] = left
90
+ end
91
+ end
92
+
93
+ local function pruneFailures(now: number)
94
+ local cutoff = now - thresholds.FailWindow
95
+ while failures[1] and failures[1].At < cutoff do
96
+ forget(table.remove(failures, 1) :: any)
97
+ end
98
+ end
99
+
100
+ -- Lifting Outage once the failures stop --------------------------------------
101
+ -- evaluate() only escalates and a quiet server never calls it, so a burst that ends
102
+ -- would leave Outage forever. The step condition is an empty window, not a count under
103
+ -- the threshold: with the sparse stream the retry cooldown guarantees, the count dips
104
+ -- between retries and the status would flap, opening the money gate mid-outage. It
105
+ -- stops at Degraded, because no reports is not evidence anything worked.
106
+ local DECAY_MIN_DELAY = 1 -- seconds; the horizon can land on the prune boundary, where an entry survives
107
+ local decayPending = false
108
+ local armDecay: () -> ()
109
+
110
+ local function decayCheck()
111
+ decayPending = false
112
+ if status ~= "Outage" then
113
+ return
114
+ end
115
+ pruneFailures(Clock.Monotonic())
116
+ -- Re-armed on every path, not just the one that steps: a check that declines and then
117
+ -- dies is a decay that never happens.
118
+ if criticalSince or #failures > 0 then
119
+ armDecay()
120
+ return
121
+ end
122
+ successStreak = 0
123
+ setStatus("Degraded")
124
+ armDecay()
125
+ end
126
+
127
+ function armDecay()
128
+ if decayPending or status ~= "Outage" then
129
+ return
130
+ end
131
+ decayPending = true
132
+ local newest = failures[#failures]
133
+ local now = Clock.Monotonic()
134
+ local delay = if newest then (newest.At + thresholds.FailWindow) - now else thresholds.FailWindow
135
+ Clock.Delay(math.max(delay, DECAY_MIN_DELAY), decayCheck)
136
+ end
137
+
138
+ local function evaluate()
139
+ local now = Clock.Monotonic()
140
+ pruneFailures(now)
141
+ local recent = #failures
142
+
143
+ if criticalSince and now - criticalSince >= thresholds.CriticalOutageDelay then
144
+ -- ProfileStore's critical flag has no key dimension, so one permanently-failing
145
+ -- profile trips it unaided. Treat it as corroboration, not an independent trigger:
146
+ -- it forces Outage only when Scribe's own failures span more than one operation.
147
+ if distinctIds + unkeyed >= 2 then
148
+ setStatus("Outage")
149
+ -- Without this re-arm, the critical-state path is the one entry to Outage that
150
+ -- never schedules a decay, so a server that escalated here and then went quiet
151
+ -- stays in Outage forever.
152
+ armDecay()
153
+ return
154
+ end
155
+ end
156
+
157
+ if recent >= thresholds.OutageFailCount then
158
+ setStatus("Outage")
159
+ elseif recent >= thresholds.FailCount or criticalSince then
160
+ if status == "Healthy" then
161
+ setStatus("Degraded")
162
+ end
163
+ end
164
+ armDecay()
165
+ end
166
+
167
+ -- What has been explicitly configured, and which conflicts have been reported. This
168
+ -- machine is a library-level singleton, so a second bundle in the same process retunes
169
+ -- the first's Outage gate, the gate that refuses Robux grants during a DataStore
170
+ -- failure. The later value still wins, matching SAVE_INTERVAL_CONFLICT, but it warns.
171
+ local configured: { [string]: number } = {}
172
+ local conflictWarned: { [string]: true } = {}
173
+
174
+ local CONFIGURABLE = { "FailWindow", "FailCount", "RecoverStreak" }
175
+
176
+ function Health.Configure(config: { FailWindow: number?, FailCount: number?, RecoverStreak: number? }?)
177
+ if not config then
178
+ return
179
+ end
180
+ for _, field in CONFIGURABLE do
181
+ local requested = (config :: any)[field]
182
+ if requested ~= nil then
183
+ local existing = configured[field]
184
+ -- Once per field, not once per call: a process that rebuilds bundles would
185
+ -- otherwise turn one real signal into a line per build.
186
+ if existing ~= nil and existing ~= requested and not conflictWarned[field] then
187
+ conflictWarned[field] = true
188
+ -- "Persistence" because the health machine has no subsystem of its own in
189
+ -- the Types.LogSubsystem union.
190
+ Log.Warn(
191
+ "Persistence",
192
+ "HEALTH_THRESHOLDS_CONFLICT",
193
+ "the health thresholds are a process global shared by every Scribe bundle and were already set "
194
+ .. "to a different value; the later value wins. Set them once, identically, in every bundle.",
195
+ { Field = field, Existing = existing, Requested = requested }
196
+ )
197
+ end
198
+ local live: any = thresholds
199
+ live[field] = requested
200
+ configured[field] = requested
201
+ end
202
+ end
203
+ thresholds.OutageFailCount = math.max(thresholds.FailCount * 2, thresholds.FailCount + 1)
204
+ end
205
+
206
+ -- The thresholds as they currently stand, copied so a reader cannot edit the singleton
207
+ -- by accident. OutageFailCount is the interesting member: it gates money, it is rebuilt
208
+ -- from FailCount on every Configure, and there is otherwise no way to see which won.
209
+ function Health.Thresholds(): { [string]: number }
210
+ return table.clone(thresholds)
211
+ end
212
+
213
+ -- Put the singleton back to its declared defaults, including the record of what has
214
+ -- been configured, so the first Configure after a teardown is not compared against a
215
+ -- bundle that no longer exists.
216
+ function Health.ResetThresholds()
217
+ thresholds.FailWindow = 60
218
+ thresholds.FailCount = 3
219
+ thresholds.RecoverStreak = 5
220
+ thresholds.OutageFailCount = math.max(3 * 2, 3 + 1)
221
+ thresholds.CriticalOutageDelay = 30
222
+ table.clear(configured)
223
+ table.clear(conflictWarned)
224
+ end
225
+
226
+ --- Report a failed DataStore/OrderedDataStore/MessageAsync operation. `key` names the
227
+ --- subject so repeated failures of one thing collapse into one problem; omit it only
228
+ --- where there is no subject. Of the `class` verdicts only "Throttled" changes anything.
229
+ --- Returns `(counted, folded)`: whether this became a distinct problem rather than being
230
+ --- absorbed into a live one, and how many reports that problem has absorbed.
231
+ function Health.ReportFailure(subsystem: string, key: string?, class: string?): (boolean, number)
232
+ successStreak = 0
233
+ -- The metric still counts every report; only the state machine below dedupes.
234
+ Metrics.Add("HealthFailures")
235
+ Metrics.Add("HealthFailures_" .. subsystem)
236
+
237
+ local now = Clock.Monotonic()
238
+ -- Namespaced by subsystem so a version read and a save failure on the same key stay
239
+ -- independent problems. Subsystem names are fixed internal literals and never contain
240
+ -- the separator, so the pair cannot be ambiguous.
241
+ local id = if class == "Throttled"
242
+ then subsystem .. "|" .. THROTTLE_SUBJECT
243
+ elseif key ~= nil then subsystem .. "|" .. key
244
+ else nil
245
+ local flushed = 0
246
+ if id ~= nil then
247
+ local seen = lastAt[id]
248
+ if seen ~= nil and now - seen < RETRY_COOLDOWN then
249
+ -- Same operation, still inside its retry window: already counted.
250
+ local tally = (folded[id] or 0) + 1
251
+ folded[id] = tally
252
+ evaluate()
253
+ return false, tally
254
+ end
255
+ lastAt[id] = now
256
+ flushed = folded[id] or 0
257
+ folded[id] = nil
258
+ local n = (counts[id] or 0) + 1
259
+ counts[id] = n
260
+ if n == 1 then
261
+ distinctIds += 1
262
+ end
263
+ else
264
+ unkeyed += 1
265
+ end
266
+ table.insert(failures, { At = now, Id = id })
267
+ evaluate()
268
+ return true, flushed
269
+ end
270
+
271
+ -- Report a successful operation; a streak steps the status back down.
272
+ function Health.ReportSuccess(_subsystem: string?)
273
+ if status == "Healthy" then
274
+ successStreak = 0
275
+ return
276
+ end
277
+ successStreak += 1
278
+ if successStreak >= thresholds.RecoverStreak and not criticalSince then
279
+ successStreak = 0
280
+ table.clear(failures)
281
+ table.clear(lastAt)
282
+ table.clear(counts)
283
+ table.clear(folded)
284
+ distinctIds = 0
285
+ unkeyed = 0
286
+ setStatus(if status == "Outage" then "Degraded" else "Healthy")
287
+ end
288
+ end
289
+
290
+ -- Bound to ProfileStore.OnCriticalToggle.
291
+ function Health.SetCritical(isCritical: boolean)
292
+ if isCritical then
293
+ if not criticalSince then
294
+ criticalSince = Clock.Monotonic()
295
+ -- evaluate() runs only on ReportFailure/SetCritical, so a sustained critical
296
+ -- window with no further failures would never escalate. It re-reads elapsed
297
+ -- time, so a cleared or freshly-reset critical state won't escalate early.
298
+ Clock.Delay(thresholds.CriticalOutageDelay, function()
299
+ if criticalSince then
300
+ evaluate()
301
+ end
302
+ end)
303
+ end
304
+ evaluate()
305
+ else
306
+ criticalSince = nil
307
+ -- Leave recovery to the success streak.
308
+ end
309
+ end
310
+
311
+ function Health.GetStatus(): Status
312
+ return status
313
+ end
314
+
315
+ -- Client side: adopt a status broadcast from the server.
316
+ function Health.AdoptBroadcast(new: Status)
317
+ setStatus(new)
318
+ end
319
+
320
+ return Health
@@ -0,0 +1,36 @@
1
+ --!strict
2
+ -- Passive registry; each running bundle owns its provider.
3
+
4
+ local Types = require(script.Parent.Parent.Types)
5
+
6
+ type Provider = () -> { [string]: Types.LeaderboardStatusInfo }
7
+
8
+ local Diagnostics = {}
9
+ local nextId = 0
10
+ local providers: { [number]: Provider } = setmetatable({}, { __mode = "v" }) :: any
11
+
12
+ function Diagnostics.Register(provider: Provider): () -> ()
13
+ nextId += 1
14
+ local id = nextId
15
+ providers[id] = provider
16
+ return function()
17
+ providers[id] = nil
18
+ end
19
+ end
20
+
21
+ function Diagnostics.Snapshot(): { Types.LeaderboardSnapshotEntry }
22
+ local out: { Types.LeaderboardSnapshotEntry } = {}
23
+ for id, provider in providers do
24
+ for name, status in provider() do
25
+ local entry = table.clone(status) :: any
26
+ entry.BundleId, entry.Name = id, name
27
+ table.insert(out, entry)
28
+ end
29
+ end
30
+ table.sort(out, function(a: Types.LeaderboardSnapshotEntry, b: Types.LeaderboardSnapshotEntry)
31
+ return if a.BundleId == b.BundleId then a.Name < b.Name else a.BundleId < b.BundleId
32
+ end)
33
+ return out
34
+ end
35
+
36
+ return Diagnostics