@rbxts/scribe-telemetry 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Config.luau +446 -0
- package/Diagnostics.luau +104 -0
- package/Format.luau +152 -0
- package/LICENSE +21 -0
- package/LeaderboardMonitor.luau +146 -0
- package/NOTICE +21 -0
- package/Preview.luau +396 -0
- package/README.md +5 -0
- package/Router.luau +152 -0
- package/Sanitize.luau +265 -0
- package/Scribe-source-map.json +1 -0
- package/Transport.luau +498 -0
- package/index.d.ts +150 -0
- package/init.luau +1120 -0
- package/package.json +20 -0
package/init.luau
ADDED
|
@@ -0,0 +1,1120 @@
|
|
|
1
|
+
--!strict
|
|
2
|
+
-- ScribeTelemetry: Scribe's health, failures, selected warnings, performance rules and
|
|
3
|
+
-- periodic summaries, reported to Discord webhooks through the public diagnostics API.
|
|
4
|
+
|
|
5
|
+
local RunService = game:GetService("RunService")
|
|
6
|
+
|
|
7
|
+
local Config = require(script.Config)
|
|
8
|
+
local Diagnostics = require(script.Diagnostics)
|
|
9
|
+
local Format = require(script.Format)
|
|
10
|
+
local LeaderboardMonitor = require(script.LeaderboardMonitor)
|
|
11
|
+
local Preview = require(script.Preview)
|
|
12
|
+
local Router = require(script.Router)
|
|
13
|
+
local Sanitize = require(script.Sanitize)
|
|
14
|
+
local Transport = require(script.Transport)
|
|
15
|
+
|
|
16
|
+
export type Options = Config.Options
|
|
17
|
+
export type PreviewKind = Preview.Kind
|
|
18
|
+
export type Stats = Transport.Stats & {
|
|
19
|
+
State: string,
|
|
20
|
+
Reports: number,
|
|
21
|
+
Previews: number,
|
|
22
|
+
Skipped: { [string]: number },
|
|
23
|
+
Filtered: { [string]: number },
|
|
24
|
+
Groups: number,
|
|
25
|
+
EncodeFailures: number,
|
|
26
|
+
LastTickError: string?,
|
|
27
|
+
Leaderboards: { State: string, Monitored: number, Omitted: number, SnapshotAge: number?, LastError: string? },
|
|
28
|
+
}
|
|
29
|
+
export type Handle = {
|
|
30
|
+
Test: (self: Handle, destination: string) -> (boolean, string?),
|
|
31
|
+
Preview: (self: Handle, destination: string, kind: PreviewKind?) -> (boolean, string?),
|
|
32
|
+
Flush: (self: Handle, timeout: number?) -> boolean,
|
|
33
|
+
GetStats: (self: Handle) -> Stats,
|
|
34
|
+
Stop: (self: Handle) -> (),
|
|
35
|
+
}
|
|
36
|
+
export type LogEntry = {
|
|
37
|
+
At: number,
|
|
38
|
+
Level: string,
|
|
39
|
+
Category: string,
|
|
40
|
+
Code: string,
|
|
41
|
+
Message: string,
|
|
42
|
+
Context: { [string]: any }?,
|
|
43
|
+
}
|
|
44
|
+
--- The slice of the Scribe module the add-on reads; the real module satisfies it.
|
|
45
|
+
export type ScribeModule = {
|
|
46
|
+
Version: string?,
|
|
47
|
+
AddLogSink: (sink: (entry: LogEntry) -> (), options: { Level: string? }?) -> () -> (),
|
|
48
|
+
OnStatusChanged: { Connect: (self: any, fn: (new: string, old: string?) -> ()) -> any },
|
|
49
|
+
GetStatus: () -> string,
|
|
50
|
+
GetMetrics: () -> { [string]: any },
|
|
51
|
+
GetPercentiles: () -> { [string]: any },
|
|
52
|
+
GetBudgetSnapshot: (() -> { [string]: any })?,
|
|
53
|
+
GetLeaderboardSnapshot: (() -> { LeaderboardMonitor.Entry })?,
|
|
54
|
+
GetProfileKeyPrefix: (() -> string)?,
|
|
55
|
+
RegisterAddon: ((name: string, actions: { [string]: { Run: (args: any) -> any, Writes: boolean? } }) -> () -> ())?,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
local ScribeTelemetry = {}
|
|
59
|
+
ScribeTelemetry.Version = "1.0.0"
|
|
60
|
+
ScribeTelemetry.LogoUrl = Config.LOGO_URL
|
|
61
|
+
ScribeTelemetry.DefaultWarningCodes = table.freeze(table.clone(Config.DEFAULT_WARNING_CODES))
|
|
62
|
+
|
|
63
|
+
local PRIORITY = { Health = 3, Issues = 3, Warnings = 2, Leaderboards = 2, Performance = 2, Summaries = 1, Test = 1 }
|
|
64
|
+
local TICK = 1
|
|
65
|
+
local MESSAGE_MAX = 1500
|
|
66
|
+
|
|
67
|
+
-- One running add-on per Scribe module, weakly keyed.
|
|
68
|
+
local active: { [any]: boolean } = setmetatable({}, { __mode = "k" }) :: any
|
|
69
|
+
|
|
70
|
+
local function defaultHttp()
|
|
71
|
+
return {
|
|
72
|
+
Post = function(url: string, body: string): Transport.Response
|
|
73
|
+
local HttpService = game:GetService("HttpService")
|
|
74
|
+
local ok, result = pcall(function()
|
|
75
|
+
return HttpService:RequestAsync({
|
|
76
|
+
Url = Config.PostUrl(url),
|
|
77
|
+
Method = "POST",
|
|
78
|
+
Headers = { ["Content-Type"] = "application/json" },
|
|
79
|
+
Body = body,
|
|
80
|
+
})
|
|
81
|
+
end)
|
|
82
|
+
if not ok then
|
|
83
|
+
return { Ok = false, Error = tostring(result) }
|
|
84
|
+
end
|
|
85
|
+
return {
|
|
86
|
+
Ok = result.Success == true,
|
|
87
|
+
Status = result.StatusCode,
|
|
88
|
+
Headers = result.Headers,
|
|
89
|
+
Body = result.Body,
|
|
90
|
+
Error = if result.Success then nil else result.StatusMessage,
|
|
91
|
+
}
|
|
92
|
+
end,
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
local function defaultScheduler()
|
|
97
|
+
return {
|
|
98
|
+
Spawn = function(fn: () -> ())
|
|
99
|
+
task.spawn(fn)
|
|
100
|
+
end,
|
|
101
|
+
Wait = function(seconds: number)
|
|
102
|
+
task.wait(seconds)
|
|
103
|
+
end,
|
|
104
|
+
}
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
local function defaultClock()
|
|
108
|
+
return { Now = os.time, Monotonic = os.clock }
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
local function defaultJson()
|
|
112
|
+
local HttpService = game:GetService("HttpService")
|
|
113
|
+
return {
|
|
114
|
+
Encode = function(value: any): string
|
|
115
|
+
return HttpService:JSONEncode(value)
|
|
116
|
+
end,
|
|
117
|
+
Decode = function(text: string): any
|
|
118
|
+
return HttpService:JSONDecode(text)
|
|
119
|
+
end,
|
|
120
|
+
}
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
local function gameMeta(environment: string, scribeVersion: string): Format.Meta
|
|
124
|
+
local function read(name: string, fallback: any): any
|
|
125
|
+
local ok, value = pcall(function()
|
|
126
|
+
return (game :: any)[name]
|
|
127
|
+
end)
|
|
128
|
+
return if ok and value ~= nil then value else fallback
|
|
129
|
+
end
|
|
130
|
+
return {
|
|
131
|
+
PlaceId = read("PlaceId", 0),
|
|
132
|
+
PlaceVersion = read("PlaceVersion", 0),
|
|
133
|
+
JobId = tostring(read("JobId", "")),
|
|
134
|
+
ScribeVersion = scribeVersion,
|
|
135
|
+
Environment = environment,
|
|
136
|
+
}
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
local function duration(seconds: number): string
|
|
140
|
+
if seconds < 10 then
|
|
141
|
+
return string.format("%.2fs", seconds)
|
|
142
|
+
elseif seconds < 60 then
|
|
143
|
+
return `{math.floor(seconds)}s`
|
|
144
|
+
elseif seconds < 3600 then
|
|
145
|
+
return `{math.floor(seconds / 60)}m {math.floor(seconds % 60)}s`
|
|
146
|
+
end
|
|
147
|
+
return `{math.floor(seconds / 3600)}h {math.floor((seconds % 3600) / 60)}m`
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
local function clockText(at: number): string
|
|
151
|
+
local ok, text = pcall(function()
|
|
152
|
+
return os.date("!%H:%M:%S UTC", at)
|
|
153
|
+
end)
|
|
154
|
+
return if ok then text else tostring(at)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
local function bytesText(n: number): string
|
|
158
|
+
if n >= 1048576 then
|
|
159
|
+
return string.format("%.1f MB", n / 1048576)
|
|
160
|
+
elseif n >= 1024 then
|
|
161
|
+
return string.format("%.1f KB", n / 1024)
|
|
162
|
+
end
|
|
163
|
+
return `{math.floor(n)} B`
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
local Handle = {}
|
|
167
|
+
Handle.__index = Handle
|
|
168
|
+
|
|
169
|
+
function ScribeTelemetry.Start(Scribe: ScribeModule, options: Options): Handle
|
|
170
|
+
for _, name in { "AddLogSink", "GetStatus", "GetMetrics", "GetPercentiles" } do
|
|
171
|
+
if type(Scribe) ~= "table" or type((Scribe :: any)[name]) ~= "function" then
|
|
172
|
+
error(`ScribeTelemetry: the first argument must be the Scribe module (missing {name})`, 0)
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
if type(Scribe.OnStatusChanged) ~= "table" or type(Scribe.OnStatusChanged.Connect) ~= "function" then
|
|
176
|
+
error("ScribeTelemetry: the first argument must be the Scribe module (missing OnStatusChanged)", 0)
|
|
177
|
+
end
|
|
178
|
+
local config = Config.Resolve(options)
|
|
179
|
+
if active[Scribe] then
|
|
180
|
+
error("ScribeTelemetry: already started for this Scribe module; Stop() the running one first", 0)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
local self: any = setmetatable({}, Handle)
|
|
184
|
+
self.scribe = Scribe
|
|
185
|
+
self.config = config
|
|
186
|
+
self.deps = {
|
|
187
|
+
Http = options.Http or defaultHttp(),
|
|
188
|
+
Clock = options.Clock or defaultClock(),
|
|
189
|
+
Scheduler = options.Scheduler or defaultScheduler(),
|
|
190
|
+
Json = options.Json or defaultJson(),
|
|
191
|
+
Jitter = options.Jitter or math.random,
|
|
192
|
+
}
|
|
193
|
+
self.meta = gameMeta(config.Environment, tostring(Scribe.Version or "unknown"))
|
|
194
|
+
self.brand = { Name = config.BrandName, IconUrl = config.IconUrl }
|
|
195
|
+
self.running = false
|
|
196
|
+
self.skipped = {} :: { [string]: number }
|
|
197
|
+
self.filtered = { Issues = 0, Warnings = 0, Ignored = 0 }
|
|
198
|
+
self.reports = 0
|
|
199
|
+
self.previews = 0
|
|
200
|
+
self.mentionedAt = {} :: { [string]: number }
|
|
201
|
+
self.nextId = 0
|
|
202
|
+
|
|
203
|
+
local register = Scribe.RegisterAddon
|
|
204
|
+
if register ~= nil then
|
|
205
|
+
self.unregister = register("Telemetry", {
|
|
206
|
+
Status = {
|
|
207
|
+
Run = function()
|
|
208
|
+
return self:studioStatus()
|
|
209
|
+
end,
|
|
210
|
+
},
|
|
211
|
+
Preview = {
|
|
212
|
+
Run = function(args: any)
|
|
213
|
+
local ok, why = self:Preview(tostring(args.Destination), args.Kind)
|
|
214
|
+
return { Ok = ok, Reason = why }
|
|
215
|
+
end,
|
|
216
|
+
Writes = true,
|
|
217
|
+
},
|
|
218
|
+
})
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
-- `options.Studio` is a test seam.
|
|
222
|
+
local studio = false
|
|
223
|
+
if options.Studio ~= nil then
|
|
224
|
+
studio = options.Studio == true
|
|
225
|
+
else
|
|
226
|
+
pcall(function()
|
|
227
|
+
studio = RunService:IsStudio()
|
|
228
|
+
end)
|
|
229
|
+
end
|
|
230
|
+
if studio and not config.AllowStudio then
|
|
231
|
+
self.state = "DisabledInStudio"
|
|
232
|
+
return self
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
active[Scribe] = true
|
|
236
|
+
self.state = "Running"
|
|
237
|
+
self.running = true
|
|
238
|
+
self.transport = Transport.new({
|
|
239
|
+
MaxQueued = config.MaxQueued,
|
|
240
|
+
MaxQueuedBytes = config.MaxQueuedBytes,
|
|
241
|
+
PerDestinationQueued = config.PerDestinationQueued,
|
|
242
|
+
MaxAge = config.MaxAge,
|
|
243
|
+
MaxAttempts = config.MaxAttempts,
|
|
244
|
+
MaxRetryDelay = config.MaxRetryDelay,
|
|
245
|
+
}, self.deps)
|
|
246
|
+
for name, dest in config.Destinations do
|
|
247
|
+
self.transport:AddDestination(name, dest.Url)
|
|
248
|
+
end
|
|
249
|
+
self.router = Router.new(config.GroupInterval, config.GroupExpiry, config.MaxGroups, self.deps.Clock.Monotonic, {
|
|
250
|
+
First = function(record)
|
|
251
|
+
self:reportRecord(record)
|
|
252
|
+
end,
|
|
253
|
+
Summary = function(group)
|
|
254
|
+
self:reportGroup(group)
|
|
255
|
+
end,
|
|
256
|
+
})
|
|
257
|
+
|
|
258
|
+
-- Health first: an incident already underway is reported before anything else.
|
|
259
|
+
local status = Scribe.GetStatus()
|
|
260
|
+
self.status = status
|
|
261
|
+
self.statusSince = nil :: number?
|
|
262
|
+
self.statusChanged = (Scribe.OnStatusChanged :: any):Connect(function(new: string, old: string?)
|
|
263
|
+
if self.running then
|
|
264
|
+
self:onStatus(new, old)
|
|
265
|
+
end
|
|
266
|
+
end)
|
|
267
|
+
if status ~= "Healthy" then
|
|
268
|
+
self:reportHealth(nil, status, nil)
|
|
269
|
+
elseif config.ReportHealthyStartup then
|
|
270
|
+
self:reportStartup()
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
self.removeSink = Scribe.AddLogSink(function(entry)
|
|
274
|
+
if self.running then
|
|
275
|
+
self:onLog(entry)
|
|
276
|
+
end
|
|
277
|
+
end, { Level = "Warn" })
|
|
278
|
+
|
|
279
|
+
self.lastSummary = { At = self.deps.Clock.Monotonic(), Metrics = Scribe.GetMetrics() }
|
|
280
|
+
self.lastPerformance = { At = self.deps.Clock.Monotonic(), Metrics = self.lastSummary.Metrics }
|
|
281
|
+
local startMetrics = self.lastSummary.Metrics
|
|
282
|
+
local savesSoFar = if type(startMetrics) == "table" and type(startMetrics.SaveDuration) == "table"
|
|
283
|
+
then tonumber(startMetrics.SaveDuration.Count) or 0
|
|
284
|
+
else 0
|
|
285
|
+
self.rules = {
|
|
286
|
+
SlowSaves = { Streak = 0, Active = false, LastCount = savesSoFar },
|
|
287
|
+
SaveFailures = { Streak = 0, Active = false },
|
|
288
|
+
Traffic = { Streak = 0, Active = false },
|
|
289
|
+
}
|
|
290
|
+
self.summaryDue = self.deps.Clock.Monotonic() + config.SummaryInterval
|
|
291
|
+
self.performanceDue = self.deps.Clock.Monotonic() + config.PerformanceInterval
|
|
292
|
+
self.leaderboards = LeaderboardMonitor.new(
|
|
293
|
+
Scribe,
|
|
294
|
+
config,
|
|
295
|
+
self.deps.Clock.Monotonic,
|
|
296
|
+
function(entry, previous, observedFor)
|
|
297
|
+
self:reportLeaderboard(entry, previous, observedFor)
|
|
298
|
+
end
|
|
299
|
+
)
|
|
300
|
+
self.leaderboards:Poll()
|
|
301
|
+
self.deps.Scheduler.Spawn(function()
|
|
302
|
+
while self.running do
|
|
303
|
+
self.deps.Scheduler.Wait(TICK)
|
|
304
|
+
if not self.running then
|
|
305
|
+
break
|
|
306
|
+
end
|
|
307
|
+
local ok, err = pcall(self.tick, self)
|
|
308
|
+
if not ok then
|
|
309
|
+
self.lastTickError = tostring(err)
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end)
|
|
313
|
+
return self
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
-- Reports ----------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
function Handle.newId(self: any): string
|
|
319
|
+
self.nextId += 1
|
|
320
|
+
local server = if self.meta.JobId == "" then "studio" else string.sub(self.meta.JobId, 1, 8)
|
|
321
|
+
return string.format("%s-%04x", server, self.nextId % 65536)
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
function Handle.mentionFor(self: any, dest: Config.Destination, kind: string?): string?
|
|
325
|
+
if kind == nil or dest.MentionRoleId == nil or not dest.MentionOn[kind] then
|
|
326
|
+
return nil
|
|
327
|
+
end
|
|
328
|
+
local now = self.deps.Clock.Monotonic()
|
|
329
|
+
local last = self.mentionedAt[dest.Name]
|
|
330
|
+
if last ~= nil and now - last < dest.MentionCooldown then
|
|
331
|
+
return nil
|
|
332
|
+
end
|
|
333
|
+
self.mentionedAt[dest.Name] = now
|
|
334
|
+
return dest.MentionRoleId
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
local PREVIEW_NOTE = "Synthetic data from telemetry:Preview(); nothing here happened on this server."
|
|
338
|
+
|
|
339
|
+
--- Routes one report to its category's destinations, or records why it went nowhere.
|
|
340
|
+
--- Under a preview it goes to that destination alone, below every real report.
|
|
341
|
+
function Handle.report(self: any, draft: any, groupKey: string?, preview: any?)
|
|
342
|
+
if not self.running then
|
|
343
|
+
return
|
|
344
|
+
end
|
|
345
|
+
if preview ~= nil then
|
|
346
|
+
local fields = table.clone(draft.Fields)
|
|
347
|
+
table.insert(fields, { Name = "Preview", Value = PREVIEW_NOTE, Inline = false })
|
|
348
|
+
local report: Format.Report = {
|
|
349
|
+
Id = self:newId(),
|
|
350
|
+
Kind = draft.Kind,
|
|
351
|
+
Category = draft.Category,
|
|
352
|
+
Severity = draft.Severity,
|
|
353
|
+
Title = `PREVIEW: {draft.Title}`,
|
|
354
|
+
Code = draft.Code,
|
|
355
|
+
Description = draft.Description,
|
|
356
|
+
Fields = fields,
|
|
357
|
+
At = self.deps.Clock.Now(),
|
|
358
|
+
MentionKind = nil,
|
|
359
|
+
Priority = 0,
|
|
360
|
+
}
|
|
361
|
+
self.previews += 1
|
|
362
|
+
local ok, why = self:deliver(preview.Destination, report, nil)
|
|
363
|
+
if not ok and preview.Failure == nil then
|
|
364
|
+
preview.Failure = why
|
|
365
|
+
end
|
|
366
|
+
return
|
|
367
|
+
end
|
|
368
|
+
local resolution = Config.Route(self.config, draft.Category)
|
|
369
|
+
if resolution.Skipped ~= nil then
|
|
370
|
+
self.skipped[draft.Category] = (self.skipped[draft.Category] or 0) + 1
|
|
371
|
+
return
|
|
372
|
+
end
|
|
373
|
+
local report: Format.Report = {
|
|
374
|
+
Id = self:newId(),
|
|
375
|
+
Kind = draft.Kind,
|
|
376
|
+
Category = draft.Category,
|
|
377
|
+
Severity = draft.Severity,
|
|
378
|
+
Title = draft.Title,
|
|
379
|
+
Code = draft.Code,
|
|
380
|
+
Description = draft.Description,
|
|
381
|
+
Fields = draft.Fields,
|
|
382
|
+
At = self.deps.Clock.Now(),
|
|
383
|
+
MentionKind = draft.MentionKind,
|
|
384
|
+
Priority = PRIORITY[draft.Category] or 1,
|
|
385
|
+
}
|
|
386
|
+
self.reports += 1
|
|
387
|
+
for _, dest in resolution.Destinations do
|
|
388
|
+
self:deliver(dest, report, groupKey)
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
function Handle.reportStartup(self: any, preview: any?)
|
|
393
|
+
self:report({
|
|
394
|
+
Category = "Health",
|
|
395
|
+
Kind = "Health",
|
|
396
|
+
Severity = "Info",
|
|
397
|
+
Title = "Scribe Telemetry started",
|
|
398
|
+
Description = `Scribe reports **Healthy** on this server.`,
|
|
399
|
+
Fields = {},
|
|
400
|
+
MentionKind = nil,
|
|
401
|
+
}, nil, preview)
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
local REFUSALS = {
|
|
405
|
+
["dead-destination"] = "destination is dead",
|
|
406
|
+
overflow = "queue is full",
|
|
407
|
+
stopped = "telemetry is stopped",
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function Handle.deliver(
|
|
411
|
+
self: any,
|
|
412
|
+
dest: Config.Destination,
|
|
413
|
+
report: Format.Report,
|
|
414
|
+
groupKey: string?
|
|
415
|
+
): (boolean, string?)
|
|
416
|
+
local payload = Format.Payload(report, self.meta, self.brand, self:mentionFor(dest, report.MentionKind))
|
|
417
|
+
local ok, encoded = pcall(self.deps.Json.Encode, payload)
|
|
418
|
+
if not ok then
|
|
419
|
+
self.encodeFailures = (self.encodeFailures or 0) + 1
|
|
420
|
+
return false, "the report could not be encoded"
|
|
421
|
+
end
|
|
422
|
+
local queued, why =
|
|
423
|
+
self.transport:Enqueue(dest.Name, report.Id, report.Category, groupKey, report.Priority, encoded)
|
|
424
|
+
if queued then
|
|
425
|
+
return true, nil
|
|
426
|
+
end
|
|
427
|
+
local reason = REFUSALS[why] or tostring(why)
|
|
428
|
+
if why == "dead-destination" then
|
|
429
|
+
local stats = self.transport:Stats().Destinations[dest.Name]
|
|
430
|
+
if stats and stats.LastError then
|
|
431
|
+
reason ..= `: {stats.LastError}`
|
|
432
|
+
end
|
|
433
|
+
end
|
|
434
|
+
return false, reason
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
local function contextFields(context: { [string]: any }?): { Format.Field }
|
|
438
|
+
local fields = {}
|
|
439
|
+
if context == nil then
|
|
440
|
+
return fields
|
|
441
|
+
end
|
|
442
|
+
local keys = {}
|
|
443
|
+
for key in context do
|
|
444
|
+
table.insert(keys, key)
|
|
445
|
+
end
|
|
446
|
+
table.sort(keys)
|
|
447
|
+
for _, key in keys do
|
|
448
|
+
local value = context[key]
|
|
449
|
+
local text = if type(value) == "table" then "" else tostring(value)
|
|
450
|
+
if type(value) == "table" then
|
|
451
|
+
local parts: { string } = {}
|
|
452
|
+
for k, v in value :: { [any]: any } do
|
|
453
|
+
table.insert(parts, `{k}={tostring(v)}`)
|
|
454
|
+
end
|
|
455
|
+
table.sort(parts)
|
|
456
|
+
text = table.concat(parts, ", ")
|
|
457
|
+
end
|
|
458
|
+
table.insert(fields, { Name = key, Value = text, Inline = #text < 24 })
|
|
459
|
+
if #fields >= 8 then
|
|
460
|
+
break
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
return fields
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
function Handle.reportRecord(self: any, record: Router.Record, preview: any?)
|
|
467
|
+
local fields = contextFields(record.Context)
|
|
468
|
+
table.insert(fields, 1, { Name = "Code", Value = `{record.LogCategory}/{record.Code}`, Inline = true })
|
|
469
|
+
table.insert(fields, 2, { Name = "At", Value = clockText(record.At), Inline = true })
|
|
470
|
+
if record.Repeats > 0 then
|
|
471
|
+
table.insert(fields, 3, {
|
|
472
|
+
Name = "Upstream",
|
|
473
|
+
Value = `Scribe folded {record.Repeats} earlier repeat(s) into this entry before it reached telemetry`,
|
|
474
|
+
Inline = false,
|
|
475
|
+
})
|
|
476
|
+
end
|
|
477
|
+
self:report({
|
|
478
|
+
Category = record.Category,
|
|
479
|
+
Kind = record.Kind,
|
|
480
|
+
Severity = record.Severity,
|
|
481
|
+
Title = `{record.Severity}: {record.Code}`,
|
|
482
|
+
Code = record.Code,
|
|
483
|
+
Description = record.Message,
|
|
484
|
+
Fields = fields,
|
|
485
|
+
MentionKind = if record.Severity == "Fatal" or record.Severity == "Error" then record.Severity else nil,
|
|
486
|
+
}, nil, preview)
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
function Handle.reportGroup(self: any, group: Router.Group, preview: any?)
|
|
490
|
+
local sample = group.Sample
|
|
491
|
+
local description =
|
|
492
|
+
`**{group.WindowCount}** more occurrence(s) of {sample.Code} since the last report; **{group.Count}** observed by telemetry since the group opened.`
|
|
493
|
+
if group.Upstream > 0 then
|
|
494
|
+
description ..= ` Scribe itself folded a further {group.Upstream} repeat(s) before they reached telemetry, so the underlying total is higher than the count here.`
|
|
495
|
+
end
|
|
496
|
+
self:report({
|
|
497
|
+
Category = sample.Category,
|
|
498
|
+
Kind = "Repeat",
|
|
499
|
+
Severity = sample.Severity,
|
|
500
|
+
Title = `Still occurring: {sample.Code}`,
|
|
501
|
+
Code = sample.Code,
|
|
502
|
+
Description = description,
|
|
503
|
+
Fields = {
|
|
504
|
+
{ Name = "First", Value = clockText(group.FirstAt), Inline = true },
|
|
505
|
+
{ Name = "Latest", Value = clockText(group.LatestAt), Inline = true },
|
|
506
|
+
{ Name = "Subject", Value = sample.Subject or "none", Inline = true },
|
|
507
|
+
{ Name = "Last message", Value = sample.Message, Inline = false },
|
|
508
|
+
},
|
|
509
|
+
MentionKind = nil,
|
|
510
|
+
}, group.Key, preview)
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
function Handle.reportHealth(self: any, previous: string?, current: string, observedFor: number?, preview: any?)
|
|
514
|
+
local severity: string
|
|
515
|
+
local title: string
|
|
516
|
+
local mention: string?
|
|
517
|
+
if current == "Healthy" then
|
|
518
|
+
severity, title, mention = "Recovery", `Recovered: {previous or "?"} \u{2192} Healthy`, "Recovery"
|
|
519
|
+
elseif current == "Degraded" and previous == "Outage" then
|
|
520
|
+
severity, title, mention = "Degraded", "Partial recovery: Outage \u{2192} Degraded", "Recovery"
|
|
521
|
+
elseif current == "Outage" then
|
|
522
|
+
severity, title, mention = "Outage", `Outage{if previous then ` (was {previous})` else ""}`, "Outage"
|
|
523
|
+
else
|
|
524
|
+
severity, title, mention = "Degraded", `Degraded{if previous then ` (was {previous})` else ""}`, "Degraded"
|
|
525
|
+
end
|
|
526
|
+
local observed = if observedFor ~= nil
|
|
527
|
+
then `{duration(observedFor)} in {previous or "the previous state"} before this change`
|
|
528
|
+
else "already active when telemetry started; its start time is not known here"
|
|
529
|
+
self:report({
|
|
530
|
+
Category = "Health",
|
|
531
|
+
Kind = "Health",
|
|
532
|
+
Severity = severity,
|
|
533
|
+
Title = title,
|
|
534
|
+
Description = "Scribe's observed data-service condition on this server, from its own DataStore traffic. It says nothing about other servers or Roblox as a whole.",
|
|
535
|
+
Fields = {
|
|
536
|
+
{ Name = "Previous", Value = previous or "unknown", Inline = true },
|
|
537
|
+
{ Name = "Current", Value = current, Inline = true },
|
|
538
|
+
{ Name = "At", Value = clockText(self.deps.Clock.Now()), Inline = true },
|
|
539
|
+
{ Name = "Observed", Value = observed, Inline = false },
|
|
540
|
+
},
|
|
541
|
+
MentionKind = mention,
|
|
542
|
+
}, nil, preview)
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
function Handle.reportLeaderboard(
|
|
546
|
+
self: any,
|
|
547
|
+
entry: LeaderboardMonitor.Entry,
|
|
548
|
+
previous: string?,
|
|
549
|
+
observedFor: number?,
|
|
550
|
+
preview: any?
|
|
551
|
+
)
|
|
552
|
+
local safe = self:recordFrom("Leaderboards", "Warn", "LEADERBOARD_HEALTH", "Leaderboards", entry.Name, {
|
|
553
|
+
LastReadError = entry.LastReadError,
|
|
554
|
+
LastWriteError = entry.LastWriteError,
|
|
555
|
+
}, 0)
|
|
556
|
+
local recovered = entry.Status == "Healthy"
|
|
557
|
+
local label = `{Sanitize.Truncate(safe.Message, 96)} (bundle {entry.BundleId})`
|
|
558
|
+
local fields: { Format.Field } = {
|
|
559
|
+
{ Name = "Board", Value = label, Inline = false },
|
|
560
|
+
{ Name = "Status", Value = `{previous or "unknown"} → {entry.Status}`, Inline = true },
|
|
561
|
+
{
|
|
562
|
+
Name = "Cache age",
|
|
563
|
+
Value = if entry.ReadAge then duration(entry.ReadAge) else "no successful refresh",
|
|
564
|
+
Inline = true,
|
|
565
|
+
},
|
|
566
|
+
{ Name = "Pending writes", Value = tostring(entry.PendingWrites), Inline = true },
|
|
567
|
+
{ Name = "Retrying writes", Value = tostring(entry.RetryingWrites), Inline = true },
|
|
568
|
+
{ Name = "Rejected writes", Value = tostring(entry.RejectedWrites), Inline = true },
|
|
569
|
+
{
|
|
570
|
+
Name = "Observed",
|
|
571
|
+
Value = if observedFor
|
|
572
|
+
then `{duration(observedFor)} since the previous state was observed`
|
|
573
|
+
else "already degraded when first observed",
|
|
574
|
+
Inline = false,
|
|
575
|
+
},
|
|
576
|
+
}
|
|
577
|
+
for _, field in contextFields(safe.Context) do
|
|
578
|
+
table.insert(fields, field)
|
|
579
|
+
end
|
|
580
|
+
self:report({
|
|
581
|
+
Category = "Leaderboards",
|
|
582
|
+
Kind = "Health",
|
|
583
|
+
Severity = if recovered then "Recovery" else "Degraded",
|
|
584
|
+
Title = `{if recovered then "Leaderboard recovered" else "Leaderboard degraded"}: {label}`,
|
|
585
|
+
Description = "Cached leaderboard health on this server. Profile-saving health is reported separately.",
|
|
586
|
+
Fields = fields,
|
|
587
|
+
MentionKind = if recovered then "Recovery" else "Degraded",
|
|
588
|
+
}, nil, preview)
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
function Handle.leaderboardFields(self: any): { Format.Field }
|
|
592
|
+
local monitor = self.leaderboards
|
|
593
|
+
if monitor == nil or monitor.State ~= "Available" then
|
|
594
|
+
return {
|
|
595
|
+
{
|
|
596
|
+
Name = "Leaderboard health",
|
|
597
|
+
Value = if monitor
|
|
598
|
+
then `{monitor.State}{if monitor.LastError then `: {monitor.LastError}` else ""}`
|
|
599
|
+
else "Unavailable",
|
|
600
|
+
Inline = false,
|
|
601
|
+
},
|
|
602
|
+
}
|
|
603
|
+
end
|
|
604
|
+
local healthy, degraded, starting, pending, retrying, rejected = 0, 0, 0, 0, 0, 0
|
|
605
|
+
for _, entry in monitor.Entries do
|
|
606
|
+
healthy += if entry.Status == "Healthy" then 1 else 0
|
|
607
|
+
degraded += if entry.Status == "Degraded" then 1 else 0
|
|
608
|
+
starting += if entry.Status == "Starting" then 1 else 0
|
|
609
|
+
pending += entry.PendingWrites
|
|
610
|
+
retrying += entry.RetryingWrites
|
|
611
|
+
rejected += entry.RejectedWrites
|
|
612
|
+
end
|
|
613
|
+
local overview = if #monitor.Entries == 0
|
|
614
|
+
then "No active leaderboards"
|
|
615
|
+
else `{healthy} healthy · {degraded} degraded · {starting} starting\n{pending} pending · {retrying} retrying · {rejected} rejected writes`
|
|
616
|
+
if monitor.Omitted > 0 then
|
|
617
|
+
overview ..= `\n{monitor.Omitted} boards omitted by MaxBoards; counts cover monitored boards only`
|
|
618
|
+
end
|
|
619
|
+
if monitor.ObservedAt ~= nil then
|
|
620
|
+
overview ..= `\nSnapshot {duration(math.max(0, self.deps.Clock.Monotonic() - monitor.ObservedAt))} ago`
|
|
621
|
+
end
|
|
622
|
+
local fields: { Format.Field } = { { Name = "Leaderboard health", Value = overview, Inline = false } }
|
|
623
|
+
local entries: { LeaderboardMonitor.Entry } = table.clone(monitor.Entries)
|
|
624
|
+
local rank: { [string]: number } = { Degraded = 1, Starting = 2, Healthy = 3 }
|
|
625
|
+
table.sort(entries, function(a: LeaderboardMonitor.Entry, b: LeaderboardMonitor.Entry)
|
|
626
|
+
if rank[a.Status] ~= rank[b.Status] then
|
|
627
|
+
return rank[a.Status] < rank[b.Status]
|
|
628
|
+
end
|
|
629
|
+
if a.PendingWrites ~= b.PendingWrites then
|
|
630
|
+
return a.PendingWrites > b.PendingWrites
|
|
631
|
+
end
|
|
632
|
+
if a.BundleId ~= b.BundleId then
|
|
633
|
+
return a.BundleId < b.BundleId
|
|
634
|
+
end
|
|
635
|
+
return a.Name < b.Name
|
|
636
|
+
end)
|
|
637
|
+
local lines: { string } = {}
|
|
638
|
+
for i = 1, math.min(5, #entries) do
|
|
639
|
+
local entry = entries[i]
|
|
640
|
+
local safe = self:recordFrom("Leaderboards", "Info", "LEADERBOARD_HEALTH", "Leaderboards", entry.Name, nil, 0)
|
|
641
|
+
local age = if entry.ReadAge then duration(entry.ReadAge) else "no refresh"
|
|
642
|
+
table.insert(
|
|
643
|
+
lines,
|
|
644
|
+
`{Sanitize.Truncate(safe.Message, 64)} (bundle {entry.BundleId}): {entry.Status}; {entry.PendingWrites} pending, {entry.RetryingWrites} retrying, {entry.RejectedWrites} rejected; cache {age}`
|
|
645
|
+
)
|
|
646
|
+
end
|
|
647
|
+
if #entries > 5 then
|
|
648
|
+
table.insert(lines, `+{#entries - 5} more monitored boards`)
|
|
649
|
+
end
|
|
650
|
+
if #lines > 0 then
|
|
651
|
+
table.insert(fields, { Name = "Leaderboard details", Value = table.concat(lines, "\n"), Inline = false })
|
|
652
|
+
end
|
|
653
|
+
return fields
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
-- Collection ---------------------------------------------------------------------
|
|
657
|
+
|
|
658
|
+
function Handle.onStatus(self: any, new: string, old: string?)
|
|
659
|
+
local previous = old or self.status
|
|
660
|
+
local now = self.deps.Clock.Monotonic()
|
|
661
|
+
local observedFor = if self.statusSince ~= nil then now - self.statusSince else nil
|
|
662
|
+
self.status = new
|
|
663
|
+
self.statusSince = now
|
|
664
|
+
if new == previous then
|
|
665
|
+
return
|
|
666
|
+
end
|
|
667
|
+
self:reportHealth(previous, new, observedFor)
|
|
668
|
+
end
|
|
669
|
+
|
|
670
|
+
--- One log entry as the router sees it, sanitised.
|
|
671
|
+
function Handle.recordFrom(
|
|
672
|
+
self: any,
|
|
673
|
+
category: string,
|
|
674
|
+
level: string,
|
|
675
|
+
code: string,
|
|
676
|
+
logCategory: string,
|
|
677
|
+
message: string,
|
|
678
|
+
context: any,
|
|
679
|
+
repeats: number,
|
|
680
|
+
at: number?
|
|
681
|
+
): Router.Record
|
|
682
|
+
local config: Config.Resolved = self.config
|
|
683
|
+
-- Asked per entry: a bundle started after the add-on may use another prefix.
|
|
684
|
+
local prefix: string? = nil
|
|
685
|
+
if type(self.scribe.GetProfileKeyPrefix) == "function" then
|
|
686
|
+
local ok, value = pcall(self.scribe.GetProfileKeyPrefix)
|
|
687
|
+
if ok and type(value) == "string" then
|
|
688
|
+
prefix = value
|
|
689
|
+
end
|
|
690
|
+
end
|
|
691
|
+
local options: Sanitize.Options =
|
|
692
|
+
{ IncludePlayerIds = config.IncludePlayerIds, Urls = config.Urls, KeyPrefix = prefix }
|
|
693
|
+
return {
|
|
694
|
+
Category = category,
|
|
695
|
+
Kind = if category == "Issues" then "Issue" else "Warning",
|
|
696
|
+
Severity = level,
|
|
697
|
+
Code = code,
|
|
698
|
+
LogCategory = logCategory,
|
|
699
|
+
Subject = Sanitize.Subject(context, options),
|
|
700
|
+
Message = Sanitize.Text(message, MESSAGE_MAX, options),
|
|
701
|
+
Context = Sanitize.Snapshot(context, options),
|
|
702
|
+
Repeats = repeats,
|
|
703
|
+
At = at or self.deps.Clock.Now(),
|
|
704
|
+
}
|
|
705
|
+
end
|
|
706
|
+
|
|
707
|
+
function Handle.onLog(self: any, entry: any)
|
|
708
|
+
local config: Config.Resolved = self.config
|
|
709
|
+
local level = tostring(entry.Level)
|
|
710
|
+
local code = tostring(entry.Code)
|
|
711
|
+
local logCategory = tostring(entry.Category)
|
|
712
|
+
local category: string? = nil
|
|
713
|
+
if level == "Error" or level == "Fatal" then
|
|
714
|
+
if not (config.ExcludeCodes[code] or config.ExcludeCategories[logCategory]) then
|
|
715
|
+
category = "Issues"
|
|
716
|
+
end
|
|
717
|
+
elseif level == "Warn" then
|
|
718
|
+
if (config.WarningCodes[code] or config.WarningCategories[logCategory]) and not config.ExcludeCodes[code] then
|
|
719
|
+
category = "Warnings"
|
|
720
|
+
end
|
|
721
|
+
end
|
|
722
|
+
if category == nil then
|
|
723
|
+
self.filtered.Ignored += 1
|
|
724
|
+
return
|
|
725
|
+
end
|
|
726
|
+
self.filtered[category] += 1
|
|
727
|
+
local context = entry.Context
|
|
728
|
+
local repeats = 0
|
|
729
|
+
if type(context) == "table" then
|
|
730
|
+
repeats = (tonumber(context.Repeats) or 0) + (tonumber(context.Suppressed) or 0)
|
|
731
|
+
end
|
|
732
|
+
self.router:Offer(
|
|
733
|
+
self:recordFrom(category, level, code, logCategory, entry.Message, context, repeats, tonumber(entry.At))
|
|
734
|
+
)
|
|
735
|
+
end
|
|
736
|
+
|
|
737
|
+
local function counter(metrics: { [string]: any }, name: string): number?
|
|
738
|
+
local value = metrics[name]
|
|
739
|
+
return if type(value) == "number" then value else nil
|
|
740
|
+
end
|
|
741
|
+
|
|
742
|
+
local function delta(current: { [string]: any }, previous: { [string]: any }, name: string): number?
|
|
743
|
+
local a, b = counter(current, name), counter(previous, name)
|
|
744
|
+
if a == nil then
|
|
745
|
+
return nil
|
|
746
|
+
end
|
|
747
|
+
return math.max(0, a - (b or 0))
|
|
748
|
+
end
|
|
749
|
+
|
|
750
|
+
local function percentileText(p: any, format: (number) -> string): string
|
|
751
|
+
if type(p) ~= "table" then
|
|
752
|
+
return "no samples"
|
|
753
|
+
end
|
|
754
|
+
local window = if type(p.Window) == "number" then duration(p.Window) else "?"
|
|
755
|
+
local age = if type(p.Age) == "number" then duration(p.Age) else "?"
|
|
756
|
+
return `p50 {format(p.P50)} \u{00B7} p90 {format(p.P90)} \u{00B7} p99 {format(p.P99)} ({p.Samples or "?"} samples over {window}, newest {age} ago)`
|
|
757
|
+
end
|
|
758
|
+
|
|
759
|
+
type SummaryInputs = {
|
|
760
|
+
Status: string,
|
|
761
|
+
Elapsed: number,
|
|
762
|
+
Metrics: { [string]: any },
|
|
763
|
+
Previous: { [string]: any },
|
|
764
|
+
Percentiles: { [string]: any },
|
|
765
|
+
Budget: { [string]: any }?,
|
|
766
|
+
LeaderboardFields: { Format.Field }?,
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
function Handle.summarise(self: any)
|
|
770
|
+
local now = self.deps.Clock.Monotonic()
|
|
771
|
+
local metrics = self.scribe.GetMetrics()
|
|
772
|
+
local previous = self.lastSummary
|
|
773
|
+
self.lastSummary = { At = now, Metrics = metrics }
|
|
774
|
+
local budget: { [string]: any }? = nil
|
|
775
|
+
if type(self.scribe.GetBudgetSnapshot) == "function" then
|
|
776
|
+
local ok, snapshot = pcall(self.scribe.GetBudgetSnapshot)
|
|
777
|
+
if ok and type(snapshot) == "table" then
|
|
778
|
+
budget = snapshot
|
|
779
|
+
end
|
|
780
|
+
end
|
|
781
|
+
self:reportSummary({
|
|
782
|
+
Status = tostring(self.scribe.GetStatus()),
|
|
783
|
+
Elapsed = math.max(1, now - previous.At),
|
|
784
|
+
Metrics = metrics,
|
|
785
|
+
Previous = previous.Metrics,
|
|
786
|
+
Percentiles = self.scribe.GetPercentiles(),
|
|
787
|
+
Budget = budget,
|
|
788
|
+
LeaderboardFields = self:leaderboardFields(),
|
|
789
|
+
})
|
|
790
|
+
end
|
|
791
|
+
|
|
792
|
+
function Handle.reportSummary(self: any, inputs: SummaryInputs, preview: any?)
|
|
793
|
+
local metrics, percentiles, elapsed = inputs.Metrics, inputs.Percentiles, inputs.Elapsed
|
|
794
|
+
local function count(name: string): string
|
|
795
|
+
local d = delta(metrics, inputs.Previous, name)
|
|
796
|
+
return if d == nil then "n/a" else tostring(d)
|
|
797
|
+
end
|
|
798
|
+
local bytes = delta(metrics, inputs.Previous, "BytesOut")
|
|
799
|
+
local fields: { Format.Field } = {
|
|
800
|
+
{ Name = "Profile health", Value = inputs.Status, Inline = true },
|
|
801
|
+
{ Name = "Sessions", Value = tostring(counter(metrics, "ActiveSessions") or "n/a"), Inline = true },
|
|
802
|
+
{ Name = "Interval", Value = duration(elapsed), Inline = true },
|
|
803
|
+
{
|
|
804
|
+
Name = "Loads",
|
|
805
|
+
Value = `{count("ProfilesLoaded")} ok, {count("ProfileLoadFailures")} failed`,
|
|
806
|
+
Inline = true,
|
|
807
|
+
},
|
|
808
|
+
{ Name = "Saves", Value = `{count("SavesOk")} ok, {count("SavesFailed")} failed`, Inline = true },
|
|
809
|
+
{ Name = "Bytes out", Value = if bytes == nil then "n/a" else `{bytesText(bytes / elapsed)}/s`, Inline = true },
|
|
810
|
+
{ Name = "Save duration", Value = percentileText(percentiles.SaveDuration, duration), Inline = false },
|
|
811
|
+
{ Name = "Profile size", Value = percentileText(percentiles.ProfileSize, bytesText), Inline = false },
|
|
812
|
+
}
|
|
813
|
+
for _, field in inputs.LeaderboardFields or {} do
|
|
814
|
+
table.insert(fields, field)
|
|
815
|
+
end
|
|
816
|
+
for _, field in Diagnostics.Fields(metrics, inputs.Previous, percentiles) do
|
|
817
|
+
table.insert(fields, field)
|
|
818
|
+
end
|
|
819
|
+
local snapshot = inputs.Budget
|
|
820
|
+
if snapshot ~= nil then
|
|
821
|
+
if snapshot.Available then
|
|
822
|
+
local parts = {}
|
|
823
|
+
for _, name in { "UpdateAsync", "GetAsync", "GetSortedAsync", "SetIncrementSortedAsync", "OrderedRemove" } do
|
|
824
|
+
local remaining = snapshot.Budgets[name]
|
|
825
|
+
if remaining ~= nil then
|
|
826
|
+
table.insert(parts, `{name} {remaining}`)
|
|
827
|
+
end
|
|
828
|
+
end
|
|
829
|
+
table.insert(fields, {
|
|
830
|
+
Name = "DataStore budget",
|
|
831
|
+
Value = if #parts > 0 then table.concat(parts, " \u{00B7} ") else "n/a",
|
|
832
|
+
Inline = false,
|
|
833
|
+
})
|
|
834
|
+
else
|
|
835
|
+
table.insert(fields, {
|
|
836
|
+
Name = "DataStore budget",
|
|
837
|
+
Value = `not readable here ({snapshot.Reason or "no reason"})`,
|
|
838
|
+
Inline = false,
|
|
839
|
+
})
|
|
840
|
+
end
|
|
841
|
+
end
|
|
842
|
+
self:report({
|
|
843
|
+
Category = "Summaries",
|
|
844
|
+
Kind = "Summary",
|
|
845
|
+
Severity = "Info",
|
|
846
|
+
Title = `Summary: last {duration(elapsed)} on this server`,
|
|
847
|
+
Code = nil,
|
|
848
|
+
Description = "Counts are the change over the interval; percentiles are over Scribe's rolling sample window, which is not the interval.",
|
|
849
|
+
Fields = fields,
|
|
850
|
+
MentionKind = nil,
|
|
851
|
+
}, nil, preview)
|
|
852
|
+
end
|
|
853
|
+
|
|
854
|
+
function Handle.evaluatePerformance(self: any)
|
|
855
|
+
local config: Config.Resolved = self.config
|
|
856
|
+
local now = self.deps.Clock.Monotonic()
|
|
857
|
+
local metrics = self.scribe.GetMetrics()
|
|
858
|
+
local percentiles = self.scribe.GetPercentiles()
|
|
859
|
+
local previous = self.lastPerformance
|
|
860
|
+
local elapsed = math.max(1, now - previous.At)
|
|
861
|
+
self.lastPerformance = { At = now, Metrics = metrics }
|
|
862
|
+
|
|
863
|
+
local function evaluate(name: string, rule: Config.Rule, evidence: boolean, qualifies: boolean, detail: string)
|
|
864
|
+
if not rule.Enabled then
|
|
865
|
+
return
|
|
866
|
+
end
|
|
867
|
+
local state = self.rules[name]
|
|
868
|
+
if not evidence then
|
|
869
|
+
return
|
|
870
|
+
end
|
|
871
|
+
if qualifies then
|
|
872
|
+
state.Streak += 1
|
|
873
|
+
if not state.Active and state.Streak >= config.PerformanceChecks then
|
|
874
|
+
state.Active = true
|
|
875
|
+
self:reportPerformance(name, detail, state.Streak, false)
|
|
876
|
+
end
|
|
877
|
+
else
|
|
878
|
+
state.Streak = 0
|
|
879
|
+
if state.Active then
|
|
880
|
+
state.Active = false
|
|
881
|
+
self:reportPerformance(name, detail, 0, true)
|
|
882
|
+
end
|
|
883
|
+
end
|
|
884
|
+
end
|
|
885
|
+
|
|
886
|
+
local slow = config.Rules.SlowSaves
|
|
887
|
+
local saveP = percentiles.SaveDuration
|
|
888
|
+
local saveCount = if type(metrics.SaveDuration) == "table" then tonumber(metrics.SaveDuration.Count) or 0 else 0
|
|
889
|
+
local fresh = saveCount - self.rules.SlowSaves.LastCount >= slow.MinSamples
|
|
890
|
+
self.rules.SlowSaves.LastCount = saveCount
|
|
891
|
+
local slowEvidence = fresh and type(saveP) == "table" and (tonumber(saveP.Samples) or 0) >= slow.MinSamples
|
|
892
|
+
evaluate(
|
|
893
|
+
"SlowSaves",
|
|
894
|
+
slow,
|
|
895
|
+
slowEvidence,
|
|
896
|
+
slowEvidence and saveP.P99 > slow.Threshold,
|
|
897
|
+
if slowEvidence
|
|
898
|
+
then `Save p99 is {duration(saveP.P99)} against a threshold of {duration(slow.Threshold)}`
|
|
899
|
+
else ""
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
local failures = delta(metrics, previous.Metrics, "SavesFailed")
|
|
903
|
+
local perMinute = if failures ~= nil then failures / (elapsed / 60) else 0
|
|
904
|
+
evaluate(
|
|
905
|
+
"SaveFailures",
|
|
906
|
+
config.Rules.SaveFailures,
|
|
907
|
+
failures ~= nil,
|
|
908
|
+
failures ~= nil and failures > 0 and perMinute > config.Rules.SaveFailures.Threshold,
|
|
909
|
+
string.format(
|
|
910
|
+
"Save failures at %.1f per minute against a threshold of %.1f",
|
|
911
|
+
perMinute,
|
|
912
|
+
config.Rules.SaveFailures.Threshold
|
|
913
|
+
)
|
|
914
|
+
)
|
|
915
|
+
|
|
916
|
+
local bytes = delta(metrics, previous.Metrics, "BytesOut")
|
|
917
|
+
local perSecond = if bytes ~= nil then bytes / elapsed else 0
|
|
918
|
+
evaluate(
|
|
919
|
+
"Traffic",
|
|
920
|
+
config.Rules.Traffic,
|
|
921
|
+
bytes ~= nil,
|
|
922
|
+
bytes ~= nil and bytes > 0 and perSecond > config.Rules.Traffic.Threshold,
|
|
923
|
+
`Outbound traffic at {bytesText(perSecond)}/s against a threshold of {bytesText(config.Rules.Traffic.Threshold)}/s`
|
|
924
|
+
)
|
|
925
|
+
end
|
|
926
|
+
|
|
927
|
+
function Handle.reportPerformance(
|
|
928
|
+
self: any,
|
|
929
|
+
name: string,
|
|
930
|
+
detail: string,
|
|
931
|
+
streak: number,
|
|
932
|
+
cleared: boolean,
|
|
933
|
+
preview: any?
|
|
934
|
+
)
|
|
935
|
+
if cleared then
|
|
936
|
+
self:report({
|
|
937
|
+
Category = "Performance",
|
|
938
|
+
Kind = "Performance",
|
|
939
|
+
Severity = "Recovery",
|
|
940
|
+
Title = `Performance cleared: {name}`,
|
|
941
|
+
Code = name,
|
|
942
|
+
Description = detail,
|
|
943
|
+
Fields = {},
|
|
944
|
+
MentionKind = nil,
|
|
945
|
+
}, nil, preview)
|
|
946
|
+
return
|
|
947
|
+
end
|
|
948
|
+
self:report({
|
|
949
|
+
Category = "Performance",
|
|
950
|
+
Kind = "Performance",
|
|
951
|
+
Severity = "Performance",
|
|
952
|
+
Title = `Performance: {name}`,
|
|
953
|
+
Code = name,
|
|
954
|
+
Description = `{detail} for {streak} consecutive checks of {duration(self.config.PerformanceInterval)}.`,
|
|
955
|
+
Fields = {},
|
|
956
|
+
MentionKind = "Performance",
|
|
957
|
+
}, nil, preview)
|
|
958
|
+
end
|
|
959
|
+
|
|
960
|
+
function Handle.tick(self: any)
|
|
961
|
+
local now = self.deps.Clock.Monotonic()
|
|
962
|
+
self.router:Tick()
|
|
963
|
+
self.leaderboards:Tick()
|
|
964
|
+
if now >= self.performanceDue then
|
|
965
|
+
self.performanceDue = now + self.config.PerformanceInterval
|
|
966
|
+
self:evaluatePerformance()
|
|
967
|
+
end
|
|
968
|
+
if self.config.SummariesEnabled and now >= self.summaryDue then
|
|
969
|
+
self.summaryDue = now + self.config.SummaryInterval
|
|
970
|
+
self:summarise()
|
|
971
|
+
end
|
|
972
|
+
end
|
|
973
|
+
|
|
974
|
+
-- Lifecycle ----------------------------------------------------------------------
|
|
975
|
+
|
|
976
|
+
--- Sends a test message to one destination, outside the routes. Returns false with a
|
|
977
|
+
--- reason when it could not be queued.
|
|
978
|
+
function Handle.Test(self: any, destinationName: string): (boolean, string?)
|
|
979
|
+
if self.state ~= "Running" then
|
|
980
|
+
return false, self.state
|
|
981
|
+
end
|
|
982
|
+
local dest = self.config.Destinations[destinationName]
|
|
983
|
+
if dest == nil then
|
|
984
|
+
return false, `no destination named "{tostring(destinationName)}"`
|
|
985
|
+
end
|
|
986
|
+
if not dest.Enabled then
|
|
987
|
+
return false, "destination is disabled"
|
|
988
|
+
end
|
|
989
|
+
local report: Format.Report = {
|
|
990
|
+
Id = self:newId(),
|
|
991
|
+
Kind = "Test",
|
|
992
|
+
Category = "Test",
|
|
993
|
+
Severity = "Info",
|
|
994
|
+
Title = "Test message",
|
|
995
|
+
Code = nil,
|
|
996
|
+
Description = `Scribe Telemetry can reach **{dest.Name}** from this server.`,
|
|
997
|
+
Fields = {
|
|
998
|
+
{ Name = "Scribe", Value = self.meta.ScribeVersion, Inline = true },
|
|
999
|
+
{ Name = "Telemetry", Value = ScribeTelemetry.Version, Inline = true },
|
|
1000
|
+
},
|
|
1001
|
+
At = self.deps.Clock.Now(),
|
|
1002
|
+
MentionKind = nil,
|
|
1003
|
+
Priority = 1,
|
|
1004
|
+
}
|
|
1005
|
+
return self:deliver(dest, report, nil)
|
|
1006
|
+
end
|
|
1007
|
+
|
|
1008
|
+
--- Synthetic examples of the add-on's reports to one destination: the real builders, a
|
|
1009
|
+
--- PREVIEW title, no mention, the normal queue. `kind` is "All", a group, or one scenario.
|
|
1010
|
+
function Handle.Preview(self: any, destinationName: string, kind: PreviewKind?): (boolean, string?)
|
|
1011
|
+
if self.state ~= "Running" then
|
|
1012
|
+
return false, self.state
|
|
1013
|
+
end
|
|
1014
|
+
local dest = self.config.Destinations[destinationName]
|
|
1015
|
+
if dest == nil then
|
|
1016
|
+
return false, `no destination named "{tostring(destinationName)}"`
|
|
1017
|
+
end
|
|
1018
|
+
if not dest.Enabled then
|
|
1019
|
+
return false, "destination is disabled"
|
|
1020
|
+
end
|
|
1021
|
+
local context = { Destination = dest, Failure = nil :: string? }
|
|
1022
|
+
local chosen: string = if kind == nil then "All" else kind
|
|
1023
|
+
local ok, why = pcall(Preview.Run, self, chosen, context)
|
|
1024
|
+
if not ok then
|
|
1025
|
+
return false, tostring(why)
|
|
1026
|
+
end
|
|
1027
|
+
if why ~= nil then
|
|
1028
|
+
return false, why
|
|
1029
|
+
end
|
|
1030
|
+
if self.state ~= "Running" then
|
|
1031
|
+
return false, self.state
|
|
1032
|
+
end
|
|
1033
|
+
if context.Failure ~= nil then
|
|
1034
|
+
return false, context.Failure
|
|
1035
|
+
end
|
|
1036
|
+
return true, nil
|
|
1037
|
+
end
|
|
1038
|
+
|
|
1039
|
+
--- The state, enabled destinations and preview kinds, for the plugin.
|
|
1040
|
+
function Handle.studioStatus(self: any): { [string]: any }
|
|
1041
|
+
local destinations = {}
|
|
1042
|
+
for name, dest in self.config.Destinations do
|
|
1043
|
+
if dest.Enabled then
|
|
1044
|
+
table.insert(destinations, name)
|
|
1045
|
+
end
|
|
1046
|
+
end
|
|
1047
|
+
table.sort(destinations)
|
|
1048
|
+
return { State = self.state, Destinations = destinations, Kinds = Preview.KINDS }
|
|
1049
|
+
end
|
|
1050
|
+
|
|
1051
|
+
--- Waits up to `timeout` seconds for the queues to drain.
|
|
1052
|
+
function Handle.Flush(self: any, timeout: number?): boolean
|
|
1053
|
+
if self.state ~= "Running" then
|
|
1054
|
+
return true
|
|
1055
|
+
end
|
|
1056
|
+
return self.transport:Flush(timeout or 5)
|
|
1057
|
+
end
|
|
1058
|
+
|
|
1059
|
+
function Handle.GetStats(self: any): Stats
|
|
1060
|
+
local stats: { [string]: any } = if self.transport
|
|
1061
|
+
then self.transport:Stats()
|
|
1062
|
+
else {
|
|
1063
|
+
Queued = 0,
|
|
1064
|
+
QueuedBytes = 0,
|
|
1065
|
+
Delivered = 0,
|
|
1066
|
+
Retries = 0,
|
|
1067
|
+
Uncertain = 0,
|
|
1068
|
+
Dropped = Transport.EmptyDropped(),
|
|
1069
|
+
Destinations = {},
|
|
1070
|
+
}
|
|
1071
|
+
stats.State = self.state
|
|
1072
|
+
stats.Reports = self.reports
|
|
1073
|
+
stats.Previews = self.previews
|
|
1074
|
+
stats.Skipped = table.clone(self.skipped)
|
|
1075
|
+
stats.Filtered = table.clone(self.filtered)
|
|
1076
|
+
stats.Groups = if self.router then self.router.count else 0
|
|
1077
|
+
stats.EncodeFailures = self.encodeFailures or 0
|
|
1078
|
+
stats.LastTickError = self.lastTickError
|
|
1079
|
+
local monitor = self.leaderboards
|
|
1080
|
+
stats.Leaderboards = {
|
|
1081
|
+
State = if monitor then monitor.State else self.state,
|
|
1082
|
+
Monitored = if monitor then #monitor.Entries else 0,
|
|
1083
|
+
Omitted = if monitor then monitor.Omitted else 0,
|
|
1084
|
+
SnapshotAge = if monitor and monitor.ObservedAt
|
|
1085
|
+
then math.max(0, self.deps.Clock.Monotonic() - monitor.ObservedAt)
|
|
1086
|
+
else nil,
|
|
1087
|
+
LastError = if monitor then monitor.LastError else nil,
|
|
1088
|
+
}
|
|
1089
|
+
return stats
|
|
1090
|
+
end
|
|
1091
|
+
|
|
1092
|
+
--- Stops everything: the sink, the status connection, the workers and the queues. Safe
|
|
1093
|
+
--- to call twice.
|
|
1094
|
+
function Handle.Stop(self: any)
|
|
1095
|
+
local unregister: (() -> ())? = self.unregister
|
|
1096
|
+
if unregister ~= nil then
|
|
1097
|
+
unregister()
|
|
1098
|
+
self.unregister = nil
|
|
1099
|
+
end
|
|
1100
|
+
if self.state ~= "Running" then
|
|
1101
|
+
self.state = "Stopped"
|
|
1102
|
+
return
|
|
1103
|
+
end
|
|
1104
|
+
self.state = "Stopped"
|
|
1105
|
+
self.running = false
|
|
1106
|
+
self.leaderboards:Stop()
|
|
1107
|
+
if self.removeSink then
|
|
1108
|
+
(self.removeSink :: () -> ())()
|
|
1109
|
+
self.removeSink = nil
|
|
1110
|
+
end
|
|
1111
|
+
if self.statusChanged then
|
|
1112
|
+
self.statusChanged:Disconnect()
|
|
1113
|
+
self.statusChanged = nil
|
|
1114
|
+
end
|
|
1115
|
+
self.transport:Stop()
|
|
1116
|
+
self.router:Clear()
|
|
1117
|
+
active[self.scribe] = nil
|
|
1118
|
+
end
|
|
1119
|
+
|
|
1120
|
+
return ScribeTelemetry
|