@rbxts/scribe-telemetry 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/init.luau ADDED
@@ -0,0 +1,1120 @@
1
+ --!strict
2
+ -- ScribeTelemetry: Scribe's health, failures, selected warnings, performance rules and
3
+ -- periodic summaries, reported to Discord webhooks through the public diagnostics API.
4
+
5
+ local RunService = game:GetService("RunService")
6
+
7
+ local Config = require(script.Config)
8
+ local Diagnostics = require(script.Diagnostics)
9
+ local Format = require(script.Format)
10
+ local LeaderboardMonitor = require(script.LeaderboardMonitor)
11
+ local Preview = require(script.Preview)
12
+ local Router = require(script.Router)
13
+ local Sanitize = require(script.Sanitize)
14
+ local Transport = require(script.Transport)
15
+
16
+ export type Options = Config.Options
17
+ export type PreviewKind = Preview.Kind
18
+ export type Stats = Transport.Stats & {
19
+ State: string,
20
+ Reports: number,
21
+ Previews: number,
22
+ Skipped: { [string]: number },
23
+ Filtered: { [string]: number },
24
+ Groups: number,
25
+ EncodeFailures: number,
26
+ LastTickError: string?,
27
+ Leaderboards: { State: string, Monitored: number, Omitted: number, SnapshotAge: number?, LastError: string? },
28
+ }
29
+ export type Handle = {
30
+ Test: (self: Handle, destination: string) -> (boolean, string?),
31
+ Preview: (self: Handle, destination: string, kind: PreviewKind?) -> (boolean, string?),
32
+ Flush: (self: Handle, timeout: number?) -> boolean,
33
+ GetStats: (self: Handle) -> Stats,
34
+ Stop: (self: Handle) -> (),
35
+ }
36
+ export type LogEntry = {
37
+ At: number,
38
+ Level: string,
39
+ Category: string,
40
+ Code: string,
41
+ Message: string,
42
+ Context: { [string]: any }?,
43
+ }
44
+ --- The slice of the Scribe module the add-on reads; the real module satisfies it.
45
+ export type ScribeModule = {
46
+ Version: string?,
47
+ AddLogSink: (sink: (entry: LogEntry) -> (), options: { Level: string? }?) -> () -> (),
48
+ OnStatusChanged: { Connect: (self: any, fn: (new: string, old: string?) -> ()) -> any },
49
+ GetStatus: () -> string,
50
+ GetMetrics: () -> { [string]: any },
51
+ GetPercentiles: () -> { [string]: any },
52
+ GetBudgetSnapshot: (() -> { [string]: any })?,
53
+ GetLeaderboardSnapshot: (() -> { LeaderboardMonitor.Entry })?,
54
+ GetProfileKeyPrefix: (() -> string)?,
55
+ RegisterAddon: ((name: string, actions: { [string]: { Run: (args: any) -> any, Writes: boolean? } }) -> () -> ())?,
56
+ }
57
+
58
+ local ScribeTelemetry = {}
59
+ ScribeTelemetry.Version = "1.0.0"
60
+ ScribeTelemetry.LogoUrl = Config.LOGO_URL
61
+ ScribeTelemetry.DefaultWarningCodes = table.freeze(table.clone(Config.DEFAULT_WARNING_CODES))
62
+
63
+ local PRIORITY = { Health = 3, Issues = 3, Warnings = 2, Leaderboards = 2, Performance = 2, Summaries = 1, Test = 1 }
64
+ local TICK = 1
65
+ local MESSAGE_MAX = 1500
66
+
67
+ -- One running add-on per Scribe module, weakly keyed.
68
+ local active: { [any]: boolean } = setmetatable({}, { __mode = "k" }) :: any
69
+
70
+ local function defaultHttp()
71
+ return {
72
+ Post = function(url: string, body: string): Transport.Response
73
+ local HttpService = game:GetService("HttpService")
74
+ local ok, result = pcall(function()
75
+ return HttpService:RequestAsync({
76
+ Url = Config.PostUrl(url),
77
+ Method = "POST",
78
+ Headers = { ["Content-Type"] = "application/json" },
79
+ Body = body,
80
+ })
81
+ end)
82
+ if not ok then
83
+ return { Ok = false, Error = tostring(result) }
84
+ end
85
+ return {
86
+ Ok = result.Success == true,
87
+ Status = result.StatusCode,
88
+ Headers = result.Headers,
89
+ Body = result.Body,
90
+ Error = if result.Success then nil else result.StatusMessage,
91
+ }
92
+ end,
93
+ }
94
+ end
95
+
96
+ local function defaultScheduler()
97
+ return {
98
+ Spawn = function(fn: () -> ())
99
+ task.spawn(fn)
100
+ end,
101
+ Wait = function(seconds: number)
102
+ task.wait(seconds)
103
+ end,
104
+ }
105
+ end
106
+
107
+ local function defaultClock()
108
+ return { Now = os.time, Monotonic = os.clock }
109
+ end
110
+
111
+ local function defaultJson()
112
+ local HttpService = game:GetService("HttpService")
113
+ return {
114
+ Encode = function(value: any): string
115
+ return HttpService:JSONEncode(value)
116
+ end,
117
+ Decode = function(text: string): any
118
+ return HttpService:JSONDecode(text)
119
+ end,
120
+ }
121
+ end
122
+
123
+ local function gameMeta(environment: string, scribeVersion: string): Format.Meta
124
+ local function read(name: string, fallback: any): any
125
+ local ok, value = pcall(function()
126
+ return (game :: any)[name]
127
+ end)
128
+ return if ok and value ~= nil then value else fallback
129
+ end
130
+ return {
131
+ PlaceId = read("PlaceId", 0),
132
+ PlaceVersion = read("PlaceVersion", 0),
133
+ JobId = tostring(read("JobId", "")),
134
+ ScribeVersion = scribeVersion,
135
+ Environment = environment,
136
+ }
137
+ end
138
+
139
+ local function duration(seconds: number): string
140
+ if seconds < 10 then
141
+ return string.format("%.2fs", seconds)
142
+ elseif seconds < 60 then
143
+ return `{math.floor(seconds)}s`
144
+ elseif seconds < 3600 then
145
+ return `{math.floor(seconds / 60)}m {math.floor(seconds % 60)}s`
146
+ end
147
+ return `{math.floor(seconds / 3600)}h {math.floor((seconds % 3600) / 60)}m`
148
+ end
149
+
150
+ local function clockText(at: number): string
151
+ local ok, text = pcall(function()
152
+ return os.date("!%H:%M:%S UTC", at)
153
+ end)
154
+ return if ok then text else tostring(at)
155
+ end
156
+
157
+ local function bytesText(n: number): string
158
+ if n >= 1048576 then
159
+ return string.format("%.1f MB", n / 1048576)
160
+ elseif n >= 1024 then
161
+ return string.format("%.1f KB", n / 1024)
162
+ end
163
+ return `{math.floor(n)} B`
164
+ end
165
+
166
+ local Handle = {}
167
+ Handle.__index = Handle
168
+
169
+ function ScribeTelemetry.Start(Scribe: ScribeModule, options: Options): Handle
170
+ for _, name in { "AddLogSink", "GetStatus", "GetMetrics", "GetPercentiles" } do
171
+ if type(Scribe) ~= "table" or type((Scribe :: any)[name]) ~= "function" then
172
+ error(`ScribeTelemetry: the first argument must be the Scribe module (missing {name})`, 0)
173
+ end
174
+ end
175
+ if type(Scribe.OnStatusChanged) ~= "table" or type(Scribe.OnStatusChanged.Connect) ~= "function" then
176
+ error("ScribeTelemetry: the first argument must be the Scribe module (missing OnStatusChanged)", 0)
177
+ end
178
+ local config = Config.Resolve(options)
179
+ if active[Scribe] then
180
+ error("ScribeTelemetry: already started for this Scribe module; Stop() the running one first", 0)
181
+ end
182
+
183
+ local self: any = setmetatable({}, Handle)
184
+ self.scribe = Scribe
185
+ self.config = config
186
+ self.deps = {
187
+ Http = options.Http or defaultHttp(),
188
+ Clock = options.Clock or defaultClock(),
189
+ Scheduler = options.Scheduler or defaultScheduler(),
190
+ Json = options.Json or defaultJson(),
191
+ Jitter = options.Jitter or math.random,
192
+ }
193
+ self.meta = gameMeta(config.Environment, tostring(Scribe.Version or "unknown"))
194
+ self.brand = { Name = config.BrandName, IconUrl = config.IconUrl }
195
+ self.running = false
196
+ self.skipped = {} :: { [string]: number }
197
+ self.filtered = { Issues = 0, Warnings = 0, Ignored = 0 }
198
+ self.reports = 0
199
+ self.previews = 0
200
+ self.mentionedAt = {} :: { [string]: number }
201
+ self.nextId = 0
202
+
203
+ local register = Scribe.RegisterAddon
204
+ if register ~= nil then
205
+ self.unregister = register("Telemetry", {
206
+ Status = {
207
+ Run = function()
208
+ return self:studioStatus()
209
+ end,
210
+ },
211
+ Preview = {
212
+ Run = function(args: any)
213
+ local ok, why = self:Preview(tostring(args.Destination), args.Kind)
214
+ return { Ok = ok, Reason = why }
215
+ end,
216
+ Writes = true,
217
+ },
218
+ })
219
+ end
220
+
221
+ -- `options.Studio` is a test seam.
222
+ local studio = false
223
+ if options.Studio ~= nil then
224
+ studio = options.Studio == true
225
+ else
226
+ pcall(function()
227
+ studio = RunService:IsStudio()
228
+ end)
229
+ end
230
+ if studio and not config.AllowStudio then
231
+ self.state = "DisabledInStudio"
232
+ return self
233
+ end
234
+
235
+ active[Scribe] = true
236
+ self.state = "Running"
237
+ self.running = true
238
+ self.transport = Transport.new({
239
+ MaxQueued = config.MaxQueued,
240
+ MaxQueuedBytes = config.MaxQueuedBytes,
241
+ PerDestinationQueued = config.PerDestinationQueued,
242
+ MaxAge = config.MaxAge,
243
+ MaxAttempts = config.MaxAttempts,
244
+ MaxRetryDelay = config.MaxRetryDelay,
245
+ }, self.deps)
246
+ for name, dest in config.Destinations do
247
+ self.transport:AddDestination(name, dest.Url)
248
+ end
249
+ self.router = Router.new(config.GroupInterval, config.GroupExpiry, config.MaxGroups, self.deps.Clock.Monotonic, {
250
+ First = function(record)
251
+ self:reportRecord(record)
252
+ end,
253
+ Summary = function(group)
254
+ self:reportGroup(group)
255
+ end,
256
+ })
257
+
258
+ -- Health first: an incident already underway is reported before anything else.
259
+ local status = Scribe.GetStatus()
260
+ self.status = status
261
+ self.statusSince = nil :: number?
262
+ self.statusChanged = (Scribe.OnStatusChanged :: any):Connect(function(new: string, old: string?)
263
+ if self.running then
264
+ self:onStatus(new, old)
265
+ end
266
+ end)
267
+ if status ~= "Healthy" then
268
+ self:reportHealth(nil, status, nil)
269
+ elseif config.ReportHealthyStartup then
270
+ self:reportStartup()
271
+ end
272
+
273
+ self.removeSink = Scribe.AddLogSink(function(entry)
274
+ if self.running then
275
+ self:onLog(entry)
276
+ end
277
+ end, { Level = "Warn" })
278
+
279
+ self.lastSummary = { At = self.deps.Clock.Monotonic(), Metrics = Scribe.GetMetrics() }
280
+ self.lastPerformance = { At = self.deps.Clock.Monotonic(), Metrics = self.lastSummary.Metrics }
281
+ local startMetrics = self.lastSummary.Metrics
282
+ local savesSoFar = if type(startMetrics) == "table" and type(startMetrics.SaveDuration) == "table"
283
+ then tonumber(startMetrics.SaveDuration.Count) or 0
284
+ else 0
285
+ self.rules = {
286
+ SlowSaves = { Streak = 0, Active = false, LastCount = savesSoFar },
287
+ SaveFailures = { Streak = 0, Active = false },
288
+ Traffic = { Streak = 0, Active = false },
289
+ }
290
+ self.summaryDue = self.deps.Clock.Monotonic() + config.SummaryInterval
291
+ self.performanceDue = self.deps.Clock.Monotonic() + config.PerformanceInterval
292
+ self.leaderboards = LeaderboardMonitor.new(
293
+ Scribe,
294
+ config,
295
+ self.deps.Clock.Monotonic,
296
+ function(entry, previous, observedFor)
297
+ self:reportLeaderboard(entry, previous, observedFor)
298
+ end
299
+ )
300
+ self.leaderboards:Poll()
301
+ self.deps.Scheduler.Spawn(function()
302
+ while self.running do
303
+ self.deps.Scheduler.Wait(TICK)
304
+ if not self.running then
305
+ break
306
+ end
307
+ local ok, err = pcall(self.tick, self)
308
+ if not ok then
309
+ self.lastTickError = tostring(err)
310
+ end
311
+ end
312
+ end)
313
+ return self
314
+ end
315
+
316
+ -- Reports ----------------------------------------------------------------------
317
+
318
+ function Handle.newId(self: any): string
319
+ self.nextId += 1
320
+ local server = if self.meta.JobId == "" then "studio" else string.sub(self.meta.JobId, 1, 8)
321
+ return string.format("%s-%04x", server, self.nextId % 65536)
322
+ end
323
+
324
+ function Handle.mentionFor(self: any, dest: Config.Destination, kind: string?): string?
325
+ if kind == nil or dest.MentionRoleId == nil or not dest.MentionOn[kind] then
326
+ return nil
327
+ end
328
+ local now = self.deps.Clock.Monotonic()
329
+ local last = self.mentionedAt[dest.Name]
330
+ if last ~= nil and now - last < dest.MentionCooldown then
331
+ return nil
332
+ end
333
+ self.mentionedAt[dest.Name] = now
334
+ return dest.MentionRoleId
335
+ end
336
+
337
+ local PREVIEW_NOTE = "Synthetic data from telemetry:Preview(); nothing here happened on this server."
338
+
339
+ --- Routes one report to its category's destinations, or records why it went nowhere.
340
+ --- Under a preview it goes to that destination alone, below every real report.
341
+ function Handle.report(self: any, draft: any, groupKey: string?, preview: any?)
342
+ if not self.running then
343
+ return
344
+ end
345
+ if preview ~= nil then
346
+ local fields = table.clone(draft.Fields)
347
+ table.insert(fields, { Name = "Preview", Value = PREVIEW_NOTE, Inline = false })
348
+ local report: Format.Report = {
349
+ Id = self:newId(),
350
+ Kind = draft.Kind,
351
+ Category = draft.Category,
352
+ Severity = draft.Severity,
353
+ Title = `PREVIEW: {draft.Title}`,
354
+ Code = draft.Code,
355
+ Description = draft.Description,
356
+ Fields = fields,
357
+ At = self.deps.Clock.Now(),
358
+ MentionKind = nil,
359
+ Priority = 0,
360
+ }
361
+ self.previews += 1
362
+ local ok, why = self:deliver(preview.Destination, report, nil)
363
+ if not ok and preview.Failure == nil then
364
+ preview.Failure = why
365
+ end
366
+ return
367
+ end
368
+ local resolution = Config.Route(self.config, draft.Category)
369
+ if resolution.Skipped ~= nil then
370
+ self.skipped[draft.Category] = (self.skipped[draft.Category] or 0) + 1
371
+ return
372
+ end
373
+ local report: Format.Report = {
374
+ Id = self:newId(),
375
+ Kind = draft.Kind,
376
+ Category = draft.Category,
377
+ Severity = draft.Severity,
378
+ Title = draft.Title,
379
+ Code = draft.Code,
380
+ Description = draft.Description,
381
+ Fields = draft.Fields,
382
+ At = self.deps.Clock.Now(),
383
+ MentionKind = draft.MentionKind,
384
+ Priority = PRIORITY[draft.Category] or 1,
385
+ }
386
+ self.reports += 1
387
+ for _, dest in resolution.Destinations do
388
+ self:deliver(dest, report, groupKey)
389
+ end
390
+ end
391
+
392
+ function Handle.reportStartup(self: any, preview: any?)
393
+ self:report({
394
+ Category = "Health",
395
+ Kind = "Health",
396
+ Severity = "Info",
397
+ Title = "Scribe Telemetry started",
398
+ Description = `Scribe reports **Healthy** on this server.`,
399
+ Fields = {},
400
+ MentionKind = nil,
401
+ }, nil, preview)
402
+ end
403
+
404
+ local REFUSALS = {
405
+ ["dead-destination"] = "destination is dead",
406
+ overflow = "queue is full",
407
+ stopped = "telemetry is stopped",
408
+ }
409
+
410
+ function Handle.deliver(
411
+ self: any,
412
+ dest: Config.Destination,
413
+ report: Format.Report,
414
+ groupKey: string?
415
+ ): (boolean, string?)
416
+ local payload = Format.Payload(report, self.meta, self.brand, self:mentionFor(dest, report.MentionKind))
417
+ local ok, encoded = pcall(self.deps.Json.Encode, payload)
418
+ if not ok then
419
+ self.encodeFailures = (self.encodeFailures or 0) + 1
420
+ return false, "the report could not be encoded"
421
+ end
422
+ local queued, why =
423
+ self.transport:Enqueue(dest.Name, report.Id, report.Category, groupKey, report.Priority, encoded)
424
+ if queued then
425
+ return true, nil
426
+ end
427
+ local reason = REFUSALS[why] or tostring(why)
428
+ if why == "dead-destination" then
429
+ local stats = self.transport:Stats().Destinations[dest.Name]
430
+ if stats and stats.LastError then
431
+ reason ..= `: {stats.LastError}`
432
+ end
433
+ end
434
+ return false, reason
435
+ end
436
+
437
+ local function contextFields(context: { [string]: any }?): { Format.Field }
438
+ local fields = {}
439
+ if context == nil then
440
+ return fields
441
+ end
442
+ local keys = {}
443
+ for key in context do
444
+ table.insert(keys, key)
445
+ end
446
+ table.sort(keys)
447
+ for _, key in keys do
448
+ local value = context[key]
449
+ local text = if type(value) == "table" then "" else tostring(value)
450
+ if type(value) == "table" then
451
+ local parts: { string } = {}
452
+ for k, v in value :: { [any]: any } do
453
+ table.insert(parts, `{k}={tostring(v)}`)
454
+ end
455
+ table.sort(parts)
456
+ text = table.concat(parts, ", ")
457
+ end
458
+ table.insert(fields, { Name = key, Value = text, Inline = #text < 24 })
459
+ if #fields >= 8 then
460
+ break
461
+ end
462
+ end
463
+ return fields
464
+ end
465
+
466
+ function Handle.reportRecord(self: any, record: Router.Record, preview: any?)
467
+ local fields = contextFields(record.Context)
468
+ table.insert(fields, 1, { Name = "Code", Value = `{record.LogCategory}/{record.Code}`, Inline = true })
469
+ table.insert(fields, 2, { Name = "At", Value = clockText(record.At), Inline = true })
470
+ if record.Repeats > 0 then
471
+ table.insert(fields, 3, {
472
+ Name = "Upstream",
473
+ Value = `Scribe folded {record.Repeats} earlier repeat(s) into this entry before it reached telemetry`,
474
+ Inline = false,
475
+ })
476
+ end
477
+ self:report({
478
+ Category = record.Category,
479
+ Kind = record.Kind,
480
+ Severity = record.Severity,
481
+ Title = `{record.Severity}: {record.Code}`,
482
+ Code = record.Code,
483
+ Description = record.Message,
484
+ Fields = fields,
485
+ MentionKind = if record.Severity == "Fatal" or record.Severity == "Error" then record.Severity else nil,
486
+ }, nil, preview)
487
+ end
488
+
489
+ function Handle.reportGroup(self: any, group: Router.Group, preview: any?)
490
+ local sample = group.Sample
491
+ local description =
492
+ `**{group.WindowCount}** more occurrence(s) of {sample.Code} since the last report; **{group.Count}** observed by telemetry since the group opened.`
493
+ if group.Upstream > 0 then
494
+ description ..= ` Scribe itself folded a further {group.Upstream} repeat(s) before they reached telemetry, so the underlying total is higher than the count here.`
495
+ end
496
+ self:report({
497
+ Category = sample.Category,
498
+ Kind = "Repeat",
499
+ Severity = sample.Severity,
500
+ Title = `Still occurring: {sample.Code}`,
501
+ Code = sample.Code,
502
+ Description = description,
503
+ Fields = {
504
+ { Name = "First", Value = clockText(group.FirstAt), Inline = true },
505
+ { Name = "Latest", Value = clockText(group.LatestAt), Inline = true },
506
+ { Name = "Subject", Value = sample.Subject or "none", Inline = true },
507
+ { Name = "Last message", Value = sample.Message, Inline = false },
508
+ },
509
+ MentionKind = nil,
510
+ }, group.Key, preview)
511
+ end
512
+
513
+ function Handle.reportHealth(self: any, previous: string?, current: string, observedFor: number?, preview: any?)
514
+ local severity: string
515
+ local title: string
516
+ local mention: string?
517
+ if current == "Healthy" then
518
+ severity, title, mention = "Recovery", `Recovered: {previous or "?"} \u{2192} Healthy`, "Recovery"
519
+ elseif current == "Degraded" and previous == "Outage" then
520
+ severity, title, mention = "Degraded", "Partial recovery: Outage \u{2192} Degraded", "Recovery"
521
+ elseif current == "Outage" then
522
+ severity, title, mention = "Outage", `Outage{if previous then ` (was {previous})` else ""}`, "Outage"
523
+ else
524
+ severity, title, mention = "Degraded", `Degraded{if previous then ` (was {previous})` else ""}`, "Degraded"
525
+ end
526
+ local observed = if observedFor ~= nil
527
+ then `{duration(observedFor)} in {previous or "the previous state"} before this change`
528
+ else "already active when telemetry started; its start time is not known here"
529
+ self:report({
530
+ Category = "Health",
531
+ Kind = "Health",
532
+ Severity = severity,
533
+ Title = title,
534
+ Description = "Scribe's observed data-service condition on this server, from its own DataStore traffic. It says nothing about other servers or Roblox as a whole.",
535
+ Fields = {
536
+ { Name = "Previous", Value = previous or "unknown", Inline = true },
537
+ { Name = "Current", Value = current, Inline = true },
538
+ { Name = "At", Value = clockText(self.deps.Clock.Now()), Inline = true },
539
+ { Name = "Observed", Value = observed, Inline = false },
540
+ },
541
+ MentionKind = mention,
542
+ }, nil, preview)
543
+ end
544
+
545
+ function Handle.reportLeaderboard(
546
+ self: any,
547
+ entry: LeaderboardMonitor.Entry,
548
+ previous: string?,
549
+ observedFor: number?,
550
+ preview: any?
551
+ )
552
+ local safe = self:recordFrom("Leaderboards", "Warn", "LEADERBOARD_HEALTH", "Leaderboards", entry.Name, {
553
+ LastReadError = entry.LastReadError,
554
+ LastWriteError = entry.LastWriteError,
555
+ }, 0)
556
+ local recovered = entry.Status == "Healthy"
557
+ local label = `{Sanitize.Truncate(safe.Message, 96)} (bundle {entry.BundleId})`
558
+ local fields: { Format.Field } = {
559
+ { Name = "Board", Value = label, Inline = false },
560
+ { Name = "Status", Value = `{previous or "unknown"} → {entry.Status}`, Inline = true },
561
+ {
562
+ Name = "Cache age",
563
+ Value = if entry.ReadAge then duration(entry.ReadAge) else "no successful refresh",
564
+ Inline = true,
565
+ },
566
+ { Name = "Pending writes", Value = tostring(entry.PendingWrites), Inline = true },
567
+ { Name = "Retrying writes", Value = tostring(entry.RetryingWrites), Inline = true },
568
+ { Name = "Rejected writes", Value = tostring(entry.RejectedWrites), Inline = true },
569
+ {
570
+ Name = "Observed",
571
+ Value = if observedFor
572
+ then `{duration(observedFor)} since the previous state was observed`
573
+ else "already degraded when first observed",
574
+ Inline = false,
575
+ },
576
+ }
577
+ for _, field in contextFields(safe.Context) do
578
+ table.insert(fields, field)
579
+ end
580
+ self:report({
581
+ Category = "Leaderboards",
582
+ Kind = "Health",
583
+ Severity = if recovered then "Recovery" else "Degraded",
584
+ Title = `{if recovered then "Leaderboard recovered" else "Leaderboard degraded"}: {label}`,
585
+ Description = "Cached leaderboard health on this server. Profile-saving health is reported separately.",
586
+ Fields = fields,
587
+ MentionKind = if recovered then "Recovery" else "Degraded",
588
+ }, nil, preview)
589
+ end
590
+
591
+ function Handle.leaderboardFields(self: any): { Format.Field }
592
+ local monitor = self.leaderboards
593
+ if monitor == nil or monitor.State ~= "Available" then
594
+ return {
595
+ {
596
+ Name = "Leaderboard health",
597
+ Value = if monitor
598
+ then `{monitor.State}{if monitor.LastError then `: {monitor.LastError}` else ""}`
599
+ else "Unavailable",
600
+ Inline = false,
601
+ },
602
+ }
603
+ end
604
+ local healthy, degraded, starting, pending, retrying, rejected = 0, 0, 0, 0, 0, 0
605
+ for _, entry in monitor.Entries do
606
+ healthy += if entry.Status == "Healthy" then 1 else 0
607
+ degraded += if entry.Status == "Degraded" then 1 else 0
608
+ starting += if entry.Status == "Starting" then 1 else 0
609
+ pending += entry.PendingWrites
610
+ retrying += entry.RetryingWrites
611
+ rejected += entry.RejectedWrites
612
+ end
613
+ local overview = if #monitor.Entries == 0
614
+ then "No active leaderboards"
615
+ else `{healthy} healthy · {degraded} degraded · {starting} starting\n{pending} pending · {retrying} retrying · {rejected} rejected writes`
616
+ if monitor.Omitted > 0 then
617
+ overview ..= `\n{monitor.Omitted} boards omitted by MaxBoards; counts cover monitored boards only`
618
+ end
619
+ if monitor.ObservedAt ~= nil then
620
+ overview ..= `\nSnapshot {duration(math.max(0, self.deps.Clock.Monotonic() - monitor.ObservedAt))} ago`
621
+ end
622
+ local fields: { Format.Field } = { { Name = "Leaderboard health", Value = overview, Inline = false } }
623
+ local entries: { LeaderboardMonitor.Entry } = table.clone(monitor.Entries)
624
+ local rank: { [string]: number } = { Degraded = 1, Starting = 2, Healthy = 3 }
625
+ table.sort(entries, function(a: LeaderboardMonitor.Entry, b: LeaderboardMonitor.Entry)
626
+ if rank[a.Status] ~= rank[b.Status] then
627
+ return rank[a.Status] < rank[b.Status]
628
+ end
629
+ if a.PendingWrites ~= b.PendingWrites then
630
+ return a.PendingWrites > b.PendingWrites
631
+ end
632
+ if a.BundleId ~= b.BundleId then
633
+ return a.BundleId < b.BundleId
634
+ end
635
+ return a.Name < b.Name
636
+ end)
637
+ local lines: { string } = {}
638
+ for i = 1, math.min(5, #entries) do
639
+ local entry = entries[i]
640
+ local safe = self:recordFrom("Leaderboards", "Info", "LEADERBOARD_HEALTH", "Leaderboards", entry.Name, nil, 0)
641
+ local age = if entry.ReadAge then duration(entry.ReadAge) else "no refresh"
642
+ table.insert(
643
+ lines,
644
+ `{Sanitize.Truncate(safe.Message, 64)} (bundle {entry.BundleId}): {entry.Status}; {entry.PendingWrites} pending, {entry.RetryingWrites} retrying, {entry.RejectedWrites} rejected; cache {age}`
645
+ )
646
+ end
647
+ if #entries > 5 then
648
+ table.insert(lines, `+{#entries - 5} more monitored boards`)
649
+ end
650
+ if #lines > 0 then
651
+ table.insert(fields, { Name = "Leaderboard details", Value = table.concat(lines, "\n"), Inline = false })
652
+ end
653
+ return fields
654
+ end
655
+
656
+ -- Collection ---------------------------------------------------------------------
657
+
658
+ function Handle.onStatus(self: any, new: string, old: string?)
659
+ local previous = old or self.status
660
+ local now = self.deps.Clock.Monotonic()
661
+ local observedFor = if self.statusSince ~= nil then now - self.statusSince else nil
662
+ self.status = new
663
+ self.statusSince = now
664
+ if new == previous then
665
+ return
666
+ end
667
+ self:reportHealth(previous, new, observedFor)
668
+ end
669
+
670
+ --- One log entry as the router sees it, sanitised.
671
+ function Handle.recordFrom(
672
+ self: any,
673
+ category: string,
674
+ level: string,
675
+ code: string,
676
+ logCategory: string,
677
+ message: string,
678
+ context: any,
679
+ repeats: number,
680
+ at: number?
681
+ ): Router.Record
682
+ local config: Config.Resolved = self.config
683
+ -- Asked per entry: a bundle started after the add-on may use another prefix.
684
+ local prefix: string? = nil
685
+ if type(self.scribe.GetProfileKeyPrefix) == "function" then
686
+ local ok, value = pcall(self.scribe.GetProfileKeyPrefix)
687
+ if ok and type(value) == "string" then
688
+ prefix = value
689
+ end
690
+ end
691
+ local options: Sanitize.Options =
692
+ { IncludePlayerIds = config.IncludePlayerIds, Urls = config.Urls, KeyPrefix = prefix }
693
+ return {
694
+ Category = category,
695
+ Kind = if category == "Issues" then "Issue" else "Warning",
696
+ Severity = level,
697
+ Code = code,
698
+ LogCategory = logCategory,
699
+ Subject = Sanitize.Subject(context, options),
700
+ Message = Sanitize.Text(message, MESSAGE_MAX, options),
701
+ Context = Sanitize.Snapshot(context, options),
702
+ Repeats = repeats,
703
+ At = at or self.deps.Clock.Now(),
704
+ }
705
+ end
706
+
707
+ function Handle.onLog(self: any, entry: any)
708
+ local config: Config.Resolved = self.config
709
+ local level = tostring(entry.Level)
710
+ local code = tostring(entry.Code)
711
+ local logCategory = tostring(entry.Category)
712
+ local category: string? = nil
713
+ if level == "Error" or level == "Fatal" then
714
+ if not (config.ExcludeCodes[code] or config.ExcludeCategories[logCategory]) then
715
+ category = "Issues"
716
+ end
717
+ elseif level == "Warn" then
718
+ if (config.WarningCodes[code] or config.WarningCategories[logCategory]) and not config.ExcludeCodes[code] then
719
+ category = "Warnings"
720
+ end
721
+ end
722
+ if category == nil then
723
+ self.filtered.Ignored += 1
724
+ return
725
+ end
726
+ self.filtered[category] += 1
727
+ local context = entry.Context
728
+ local repeats = 0
729
+ if type(context) == "table" then
730
+ repeats = (tonumber(context.Repeats) or 0) + (tonumber(context.Suppressed) or 0)
731
+ end
732
+ self.router:Offer(
733
+ self:recordFrom(category, level, code, logCategory, entry.Message, context, repeats, tonumber(entry.At))
734
+ )
735
+ end
736
+
737
+ local function counter(metrics: { [string]: any }, name: string): number?
738
+ local value = metrics[name]
739
+ return if type(value) == "number" then value else nil
740
+ end
741
+
742
+ local function delta(current: { [string]: any }, previous: { [string]: any }, name: string): number?
743
+ local a, b = counter(current, name), counter(previous, name)
744
+ if a == nil then
745
+ return nil
746
+ end
747
+ return math.max(0, a - (b or 0))
748
+ end
749
+
750
+ local function percentileText(p: any, format: (number) -> string): string
751
+ if type(p) ~= "table" then
752
+ return "no samples"
753
+ end
754
+ local window = if type(p.Window) == "number" then duration(p.Window) else "?"
755
+ local age = if type(p.Age) == "number" then duration(p.Age) else "?"
756
+ return `p50 {format(p.P50)} \u{00B7} p90 {format(p.P90)} \u{00B7} p99 {format(p.P99)} ({p.Samples or "?"} samples over {window}, newest {age} ago)`
757
+ end
758
+
759
+ type SummaryInputs = {
760
+ Status: string,
761
+ Elapsed: number,
762
+ Metrics: { [string]: any },
763
+ Previous: { [string]: any },
764
+ Percentiles: { [string]: any },
765
+ Budget: { [string]: any }?,
766
+ LeaderboardFields: { Format.Field }?,
767
+ }
768
+
769
+ function Handle.summarise(self: any)
770
+ local now = self.deps.Clock.Monotonic()
771
+ local metrics = self.scribe.GetMetrics()
772
+ local previous = self.lastSummary
773
+ self.lastSummary = { At = now, Metrics = metrics }
774
+ local budget: { [string]: any }? = nil
775
+ if type(self.scribe.GetBudgetSnapshot) == "function" then
776
+ local ok, snapshot = pcall(self.scribe.GetBudgetSnapshot)
777
+ if ok and type(snapshot) == "table" then
778
+ budget = snapshot
779
+ end
780
+ end
781
+ self:reportSummary({
782
+ Status = tostring(self.scribe.GetStatus()),
783
+ Elapsed = math.max(1, now - previous.At),
784
+ Metrics = metrics,
785
+ Previous = previous.Metrics,
786
+ Percentiles = self.scribe.GetPercentiles(),
787
+ Budget = budget,
788
+ LeaderboardFields = self:leaderboardFields(),
789
+ })
790
+ end
791
+
792
+ function Handle.reportSummary(self: any, inputs: SummaryInputs, preview: any?)
793
+ local metrics, percentiles, elapsed = inputs.Metrics, inputs.Percentiles, inputs.Elapsed
794
+ local function count(name: string): string
795
+ local d = delta(metrics, inputs.Previous, name)
796
+ return if d == nil then "n/a" else tostring(d)
797
+ end
798
+ local bytes = delta(metrics, inputs.Previous, "BytesOut")
799
+ local fields: { Format.Field } = {
800
+ { Name = "Profile health", Value = inputs.Status, Inline = true },
801
+ { Name = "Sessions", Value = tostring(counter(metrics, "ActiveSessions") or "n/a"), Inline = true },
802
+ { Name = "Interval", Value = duration(elapsed), Inline = true },
803
+ {
804
+ Name = "Loads",
805
+ Value = `{count("ProfilesLoaded")} ok, {count("ProfileLoadFailures")} failed`,
806
+ Inline = true,
807
+ },
808
+ { Name = "Saves", Value = `{count("SavesOk")} ok, {count("SavesFailed")} failed`, Inline = true },
809
+ { Name = "Bytes out", Value = if bytes == nil then "n/a" else `{bytesText(bytes / elapsed)}/s`, Inline = true },
810
+ { Name = "Save duration", Value = percentileText(percentiles.SaveDuration, duration), Inline = false },
811
+ { Name = "Profile size", Value = percentileText(percentiles.ProfileSize, bytesText), Inline = false },
812
+ }
813
+ for _, field in inputs.LeaderboardFields or {} do
814
+ table.insert(fields, field)
815
+ end
816
+ for _, field in Diagnostics.Fields(metrics, inputs.Previous, percentiles) do
817
+ table.insert(fields, field)
818
+ end
819
+ local snapshot = inputs.Budget
820
+ if snapshot ~= nil then
821
+ if snapshot.Available then
822
+ local parts = {}
823
+ for _, name in { "UpdateAsync", "GetAsync", "GetSortedAsync", "SetIncrementSortedAsync", "OrderedRemove" } do
824
+ local remaining = snapshot.Budgets[name]
825
+ if remaining ~= nil then
826
+ table.insert(parts, `{name} {remaining}`)
827
+ end
828
+ end
829
+ table.insert(fields, {
830
+ Name = "DataStore budget",
831
+ Value = if #parts > 0 then table.concat(parts, " \u{00B7} ") else "n/a",
832
+ Inline = false,
833
+ })
834
+ else
835
+ table.insert(fields, {
836
+ Name = "DataStore budget",
837
+ Value = `not readable here ({snapshot.Reason or "no reason"})`,
838
+ Inline = false,
839
+ })
840
+ end
841
+ end
842
+ self:report({
843
+ Category = "Summaries",
844
+ Kind = "Summary",
845
+ Severity = "Info",
846
+ Title = `Summary: last {duration(elapsed)} on this server`,
847
+ Code = nil,
848
+ Description = "Counts are the change over the interval; percentiles are over Scribe's rolling sample window, which is not the interval.",
849
+ Fields = fields,
850
+ MentionKind = nil,
851
+ }, nil, preview)
852
+ end
853
+
854
+ function Handle.evaluatePerformance(self: any)
855
+ local config: Config.Resolved = self.config
856
+ local now = self.deps.Clock.Monotonic()
857
+ local metrics = self.scribe.GetMetrics()
858
+ local percentiles = self.scribe.GetPercentiles()
859
+ local previous = self.lastPerformance
860
+ local elapsed = math.max(1, now - previous.At)
861
+ self.lastPerformance = { At = now, Metrics = metrics }
862
+
863
+ local function evaluate(name: string, rule: Config.Rule, evidence: boolean, qualifies: boolean, detail: string)
864
+ if not rule.Enabled then
865
+ return
866
+ end
867
+ local state = self.rules[name]
868
+ if not evidence then
869
+ return
870
+ end
871
+ if qualifies then
872
+ state.Streak += 1
873
+ if not state.Active and state.Streak >= config.PerformanceChecks then
874
+ state.Active = true
875
+ self:reportPerformance(name, detail, state.Streak, false)
876
+ end
877
+ else
878
+ state.Streak = 0
879
+ if state.Active then
880
+ state.Active = false
881
+ self:reportPerformance(name, detail, 0, true)
882
+ end
883
+ end
884
+ end
885
+
886
+ local slow = config.Rules.SlowSaves
887
+ local saveP = percentiles.SaveDuration
888
+ local saveCount = if type(metrics.SaveDuration) == "table" then tonumber(metrics.SaveDuration.Count) or 0 else 0
889
+ local fresh = saveCount - self.rules.SlowSaves.LastCount >= slow.MinSamples
890
+ self.rules.SlowSaves.LastCount = saveCount
891
+ local slowEvidence = fresh and type(saveP) == "table" and (tonumber(saveP.Samples) or 0) >= slow.MinSamples
892
+ evaluate(
893
+ "SlowSaves",
894
+ slow,
895
+ slowEvidence,
896
+ slowEvidence and saveP.P99 > slow.Threshold,
897
+ if slowEvidence
898
+ then `Save p99 is {duration(saveP.P99)} against a threshold of {duration(slow.Threshold)}`
899
+ else ""
900
+ )
901
+
902
+ local failures = delta(metrics, previous.Metrics, "SavesFailed")
903
+ local perMinute = if failures ~= nil then failures / (elapsed / 60) else 0
904
+ evaluate(
905
+ "SaveFailures",
906
+ config.Rules.SaveFailures,
907
+ failures ~= nil,
908
+ failures ~= nil and failures > 0 and perMinute > config.Rules.SaveFailures.Threshold,
909
+ string.format(
910
+ "Save failures at %.1f per minute against a threshold of %.1f",
911
+ perMinute,
912
+ config.Rules.SaveFailures.Threshold
913
+ )
914
+ )
915
+
916
+ local bytes = delta(metrics, previous.Metrics, "BytesOut")
917
+ local perSecond = if bytes ~= nil then bytes / elapsed else 0
918
+ evaluate(
919
+ "Traffic",
920
+ config.Rules.Traffic,
921
+ bytes ~= nil,
922
+ bytes ~= nil and bytes > 0 and perSecond > config.Rules.Traffic.Threshold,
923
+ `Outbound traffic at {bytesText(perSecond)}/s against a threshold of {bytesText(config.Rules.Traffic.Threshold)}/s`
924
+ )
925
+ end
926
+
927
+ function Handle.reportPerformance(
928
+ self: any,
929
+ name: string,
930
+ detail: string,
931
+ streak: number,
932
+ cleared: boolean,
933
+ preview: any?
934
+ )
935
+ if cleared then
936
+ self:report({
937
+ Category = "Performance",
938
+ Kind = "Performance",
939
+ Severity = "Recovery",
940
+ Title = `Performance cleared: {name}`,
941
+ Code = name,
942
+ Description = detail,
943
+ Fields = {},
944
+ MentionKind = nil,
945
+ }, nil, preview)
946
+ return
947
+ end
948
+ self:report({
949
+ Category = "Performance",
950
+ Kind = "Performance",
951
+ Severity = "Performance",
952
+ Title = `Performance: {name}`,
953
+ Code = name,
954
+ Description = `{detail} for {streak} consecutive checks of {duration(self.config.PerformanceInterval)}.`,
955
+ Fields = {},
956
+ MentionKind = "Performance",
957
+ }, nil, preview)
958
+ end
959
+
960
+ function Handle.tick(self: any)
961
+ local now = self.deps.Clock.Monotonic()
962
+ self.router:Tick()
963
+ self.leaderboards:Tick()
964
+ if now >= self.performanceDue then
965
+ self.performanceDue = now + self.config.PerformanceInterval
966
+ self:evaluatePerformance()
967
+ end
968
+ if self.config.SummariesEnabled and now >= self.summaryDue then
969
+ self.summaryDue = now + self.config.SummaryInterval
970
+ self:summarise()
971
+ end
972
+ end
973
+
974
+ -- Lifecycle ----------------------------------------------------------------------
975
+
976
+ --- Sends a test message to one destination, outside the routes. Returns false with a
977
+ --- reason when it could not be queued.
978
+ function Handle.Test(self: any, destinationName: string): (boolean, string?)
979
+ if self.state ~= "Running" then
980
+ return false, self.state
981
+ end
982
+ local dest = self.config.Destinations[destinationName]
983
+ if dest == nil then
984
+ return false, `no destination named "{tostring(destinationName)}"`
985
+ end
986
+ if not dest.Enabled then
987
+ return false, "destination is disabled"
988
+ end
989
+ local report: Format.Report = {
990
+ Id = self:newId(),
991
+ Kind = "Test",
992
+ Category = "Test",
993
+ Severity = "Info",
994
+ Title = "Test message",
995
+ Code = nil,
996
+ Description = `Scribe Telemetry can reach **{dest.Name}** from this server.`,
997
+ Fields = {
998
+ { Name = "Scribe", Value = self.meta.ScribeVersion, Inline = true },
999
+ { Name = "Telemetry", Value = ScribeTelemetry.Version, Inline = true },
1000
+ },
1001
+ At = self.deps.Clock.Now(),
1002
+ MentionKind = nil,
1003
+ Priority = 1,
1004
+ }
1005
+ return self:deliver(dest, report, nil)
1006
+ end
1007
+
1008
+ --- Synthetic examples of the add-on's reports to one destination: the real builders, a
1009
+ --- PREVIEW title, no mention, the normal queue. `kind` is "All", a group, or one scenario.
1010
+ function Handle.Preview(self: any, destinationName: string, kind: PreviewKind?): (boolean, string?)
1011
+ if self.state ~= "Running" then
1012
+ return false, self.state
1013
+ end
1014
+ local dest = self.config.Destinations[destinationName]
1015
+ if dest == nil then
1016
+ return false, `no destination named "{tostring(destinationName)}"`
1017
+ end
1018
+ if not dest.Enabled then
1019
+ return false, "destination is disabled"
1020
+ end
1021
+ local context = { Destination = dest, Failure = nil :: string? }
1022
+ local chosen: string = if kind == nil then "All" else kind
1023
+ local ok, why = pcall(Preview.Run, self, chosen, context)
1024
+ if not ok then
1025
+ return false, tostring(why)
1026
+ end
1027
+ if why ~= nil then
1028
+ return false, why
1029
+ end
1030
+ if self.state ~= "Running" then
1031
+ return false, self.state
1032
+ end
1033
+ if context.Failure ~= nil then
1034
+ return false, context.Failure
1035
+ end
1036
+ return true, nil
1037
+ end
1038
+
1039
+ --- The state, enabled destinations and preview kinds, for the plugin.
1040
+ function Handle.studioStatus(self: any): { [string]: any }
1041
+ local destinations = {}
1042
+ for name, dest in self.config.Destinations do
1043
+ if dest.Enabled then
1044
+ table.insert(destinations, name)
1045
+ end
1046
+ end
1047
+ table.sort(destinations)
1048
+ return { State = self.state, Destinations = destinations, Kinds = Preview.KINDS }
1049
+ end
1050
+
1051
+ --- Waits up to `timeout` seconds for the queues to drain.
1052
+ function Handle.Flush(self: any, timeout: number?): boolean
1053
+ if self.state ~= "Running" then
1054
+ return true
1055
+ end
1056
+ return self.transport:Flush(timeout or 5)
1057
+ end
1058
+
1059
+ function Handle.GetStats(self: any): Stats
1060
+ local stats: { [string]: any } = if self.transport
1061
+ then self.transport:Stats()
1062
+ else {
1063
+ Queued = 0,
1064
+ QueuedBytes = 0,
1065
+ Delivered = 0,
1066
+ Retries = 0,
1067
+ Uncertain = 0,
1068
+ Dropped = Transport.EmptyDropped(),
1069
+ Destinations = {},
1070
+ }
1071
+ stats.State = self.state
1072
+ stats.Reports = self.reports
1073
+ stats.Previews = self.previews
1074
+ stats.Skipped = table.clone(self.skipped)
1075
+ stats.Filtered = table.clone(self.filtered)
1076
+ stats.Groups = if self.router then self.router.count else 0
1077
+ stats.EncodeFailures = self.encodeFailures or 0
1078
+ stats.LastTickError = self.lastTickError
1079
+ local monitor = self.leaderboards
1080
+ stats.Leaderboards = {
1081
+ State = if monitor then monitor.State else self.state,
1082
+ Monitored = if monitor then #monitor.Entries else 0,
1083
+ Omitted = if monitor then monitor.Omitted else 0,
1084
+ SnapshotAge = if monitor and monitor.ObservedAt
1085
+ then math.max(0, self.deps.Clock.Monotonic() - monitor.ObservedAt)
1086
+ else nil,
1087
+ LastError = if monitor then monitor.LastError else nil,
1088
+ }
1089
+ return stats
1090
+ end
1091
+
1092
+ --- Stops everything: the sink, the status connection, the workers and the queues. Safe
1093
+ --- to call twice.
1094
+ function Handle.Stop(self: any)
1095
+ local unregister: (() -> ())? = self.unregister
1096
+ if unregister ~= nil then
1097
+ unregister()
1098
+ self.unregister = nil
1099
+ end
1100
+ if self.state ~= "Running" then
1101
+ self.state = "Stopped"
1102
+ return
1103
+ end
1104
+ self.state = "Stopped"
1105
+ self.running = false
1106
+ self.leaderboards:Stop()
1107
+ if self.removeSink then
1108
+ (self.removeSink :: () -> ())()
1109
+ self.removeSink = nil
1110
+ end
1111
+ if self.statusChanged then
1112
+ self.statusChanged:Disconnect()
1113
+ self.statusChanged = nil
1114
+ end
1115
+ self.transport:Stop()
1116
+ self.router:Clear()
1117
+ active[self.scribe] = nil
1118
+ end
1119
+
1120
+ return ScribeTelemetry