@tsplc/debug 0.2.16 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/debugger.ts CHANGED
@@ -1,56 +1,63 @@
1
1
  import { join } from "node:path";
2
2
  import { setTimeout as delay } from "node:timers/promises";
3
- import { telemetryBatch, type TelemetryReply } from "@tsplc/server/telemetry";
4
- import { z } from "zod";
3
+ import { requestsReply } from "@tsplc/server/requests";
4
+ import { telemetryReply } from "@tsplc/server/telemetry";
5
5
  import type { TargetAdapter, TargetRead, TargetSample } from "./adapter.ts";
6
6
  import type {
7
- Interest,
8
- JobState,
9
- RequestTrace,
10
- Sample,
11
- Session,
12
- StreamEvents,
13
- TimelineEntry,
14
- WatchGroups,
7
+ Interest,
8
+ JobState,
9
+ Sample,
10
+ Session,
11
+ StreamEvents,
12
+ TimelineEntry,
13
+ WatchGroups,
15
14
  } from "./api.ts";
15
+ import { errorMessage } from "./errors.ts";
16
16
  import { deviceReads, deviceState, healthReads } from "./hardware.ts";
17
- import { busyOf, busyRead, detailReads, jobState, type TracedJob } from "./jobs.ts";
18
- import type { DebugJob, DebugManifest } from "./manifest.ts";
17
+ import {
18
+ busyOf,
19
+ busyRead,
20
+ detailReads,
21
+ jobState,
22
+ type TracedJob,
23
+ } from "./jobs.ts";
24
+ import type { DebugDevice, DebugJob, DebugManifest } from "./manifest.ts";
19
25
  import { createPaths } from "./paths.ts";
26
+ import { createRequestFollower } from "./requests.ts";
20
27
  import { createTelemetryFollower, type ReceivedRecord } from "./telemetry.ts";
21
- import { createTimeline, type Entry, timelineEntries } from "./timeline.ts";
28
+ import { createTimeline, timelineEntries } from "./timeline.ts";
22
29
  import { createTraceDecoder } from "./trace.ts";
23
30
  import { loadWatchGroups } from "./watch.ts";
24
31
 
25
32
  export interface DebuggerOptions {
26
- manifest: DebugManifest;
27
- /** Owned by the debugger from here on: closing the debugger closes it. */
28
- adapter: TargetAdapter;
29
- /** Saved watch groups live here. */
30
- stateDirectory: string;
31
- /** `debug.watch` from the config. */
32
- watch: Record<string, string[]>;
33
- /** The app server, whose telemetry and request trace are pulled every second. */
34
- server?: { url: string; token: string };
35
- release?: Session["release"];
36
- /** Entries from before the debugger started, such as the agent's deployments. */
37
- timeline?: readonly TimelineEntry[];
38
- projectRoot?: string;
39
- log: (message: string) => void;
33
+ manifest: DebugManifest;
34
+ /** Owned by the debugger from here on: closing the debugger closes it. */
35
+ adapter: TargetAdapter;
36
+ /** Saved watch groups live here. */
37
+ stateDirectory: string;
38
+ /** `debug.watch` from the config. */
39
+ watch: Record<string, string[]>;
40
+ /** The app server, whose telemetry and request trace are pulled every second. */
41
+ server?: { url: string; token: string };
42
+ release?: Session["release"];
43
+ /** Entries from before the debugger started, such as the agent's deployments. */
44
+ timeline?: readonly TimelineEntry[];
45
+ projectRoot?: string;
46
+ log: (message: string) => void;
40
47
  }
41
48
 
42
49
  /** A request the API refuses, with its HTTP status. */
43
50
  export class DebugError extends Error {
44
- constructor(
45
- message: string,
46
- readonly status: number,
47
- ) {
48
- super(message);
49
- }
51
+ constructor(
52
+ message: string,
53
+ readonly status: number,
54
+ ) {
55
+ super(message);
56
+ }
50
57
  }
51
58
 
52
59
  export type StreamMessage = {
53
- [Name in keyof StreamEvents]: { event: Name; data: StreamEvents[Name] };
60
+ [Name in keyof StreamEvents]: { event: Name; data: StreamEvents[Name] };
54
61
  }[keyof StreamEvents];
55
62
 
56
63
  const pullMs = 1000;
@@ -60,45 +67,28 @@ const buildCheckMs = 10_000;
60
67
  const hardwareMs = 5000;
61
68
  /** Busy jobs whose details each poll reads; the rest show as busy only. */
62
69
  const detailedJobs = 16;
63
- const requestsRemembered = 1000;
64
-
65
- const requestTrace = z.object({
66
- seq: z.number().int(),
67
- service: z.string(),
68
- job: z.number().int().optional(),
69
- sequence: z.number().int().optional(),
70
- startedAt: z.string(),
71
- durationMs: z.number().optional(),
72
- status: z.enum(["pending", "ok", "error", "timeout"]),
73
- httpStatus: z.number().int().optional(),
74
- error: z.string().optional(),
75
- input: z.unknown().optional(),
76
- output: z.unknown().optional(),
77
- }) satisfies z.ZodType<RequestTrace>;
78
- const requestsReply = z.object({ requests: z.array(requestTrace), latest: z.number().int() });
79
- const telemetryReply = z.object({
80
- batches: z.array(telemetryBatch.extend({ seq: z.number().int(), receivedAt: z.string() })),
81
- latest: z.number().int(),
82
- }) satisfies z.ZodType<TelemetryReply>;
83
-
84
- const message = (error: unknown) => (error instanceof Error ? error.message : String(error));
85
70
 
86
71
  /** Counts how many streams show each key. */
87
72
  const createShown = <Key>() => {
88
- const counts = new Map<Key, number>();
89
- return {
90
- has: (key: Key) => counts.has(key),
91
- add: (keys: readonly Key[]) => {
92
- for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1);
93
- return () => {
94
- for (const key of keys) {
95
- const count = counts.get(key)! - 1;
96
- if (count) counts.set(key, count);
97
- else counts.delete(key);
98
- }
99
- };
100
- },
101
- };
73
+ const counts = new Map<Key, number>();
74
+ return {
75
+ has: (key: Key) => counts.has(key),
76
+ add: (keys: readonly Key[]) => {
77
+ for (const key of keys) {
78
+ counts.set(key, (counts.get(key) ?? 0) + 1);
79
+ }
80
+ return () => {
81
+ for (const key of keys) {
82
+ const count = counts.get(key)! - 1;
83
+ if (count) {
84
+ counts.set(key, count);
85
+ } else {
86
+ counts.delete(key);
87
+ }
88
+ }
89
+ };
90
+ },
91
+ };
102
92
  };
103
93
 
104
94
  /**
@@ -107,403 +97,530 @@ const createShown = <Key>() => {
107
97
  * recent window of both. Everything it offers is read-only.
108
98
  */
109
99
  export const createDebugger = async (options: DebuggerOptions) => {
110
- const { manifest, adapter } = options;
111
- const paths = createPaths(manifest);
112
- const trace = createTraceDecoder(manifest, paths);
113
- const telemetry = createTelemetryFollower(manifest);
114
- const timeline = createTimeline();
115
- const watchGroups = await loadWatchGroups(
116
- join(options.stateDirectory, "watch.json"),
117
- options.watch,
118
- );
119
- const listeners = new Set<(message: StreamMessage) => void>();
120
- const controller = new AbortController();
121
- const { signal } = controller;
122
- let controllerBuild: string | null = null;
123
-
124
- const emit = (message: StreamMessage) => {
125
- for (const listener of listeners) listener(message);
126
- };
127
- const record = (entry: TimelineEntry) => {
128
- timeline.add(entry);
129
- emit({ event: "timeline", data: entry });
130
- };
131
- for (const entry of options.timeline ?? []) timeline.add(entry);
132
-
133
- const session = (): Session => ({
134
- build: {
135
- manifest: manifest.build.id,
136
- controller: controllerBuild,
137
- matches: controllerBuild === manifest.build.id,
138
- },
139
- connection: { ...adapter.connection(), endpoint: adapter.endpoint },
140
- reading: { method: adapter.method, intervalMs: adapter.samplingMs },
141
- tasks: telemetry.tasks(),
142
- clock: telemetry.clock(),
143
- ...(options.release && { release: options.release }),
144
- ...(options.projectRoot && { projectRoot: options.projectRoot }),
145
- });
146
-
147
- const setBuild = (build: string) => {
148
- if (build === controllerBuild) return;
149
- controllerBuild = build;
150
- emit({ event: "session", data: session() });
151
- };
152
-
153
- // --- Values by TypeScript path ---------------------------------------------------------------
154
-
155
- const unavailable = (path: string, status: string): Sample => ({
156
- path,
157
- value: null,
158
- quality: "unavailable",
159
- status,
160
- receivedAt: new Date().toISOString(),
161
- });
162
-
163
- /** A path's target, or the sample it has without reading the controller. */
164
- const locate = (path: string): { target: string; type: string } | Sample => {
165
- const resolved = paths.resolve(path);
166
- if (!resolved) return unavailable(path, "Unknown path");
167
- if (resolved.kind === "constant")
168
- return {
169
- path,
170
- value: resolved.value,
171
- quality: "good",
172
- status: "Constant",
173
- receivedAt: new Date().toISOString(),
174
- };
175
- if (!resolved.readable) return unavailable(path, "Not readable");
176
- return resolved;
177
- };
178
-
179
- const targetRead = ({ target, type }: { target: string; type: string }): TargetRead => ({
180
- target,
181
- type: paths.typeOf(type),
182
- });
183
-
184
- const sampleOf = (path: string, type: string, sample: TargetSample): Sample => ({
185
- path,
186
- ...sample,
187
- value: paths.decode(type, sample.value),
188
- });
189
-
190
- /** Reads the leaves `patterns` match each sampling interval, until the returned function is called. */
191
- const watch = (patterns: readonly string[], listener: (sample: Sample) => void) => {
192
- const watched = new Map<string, { path: string; type: string }[]>();
193
- const reads: TargetRead[] = [];
194
- for (const path of paths.expand(patterns)) {
195
- const located = locate(path);
196
- if (!("target" in located)) {
197
- listener(located);
198
- continue;
199
- }
200
- if (!watched.has(located.target)) reads.push(targetRead(located));
201
- watched.set(located.target, [...(watched.get(located.target) ?? []), { path, ...located }]);
202
- }
203
- if (!reads.length) return () => {};
204
- return adapter.subscribe(reads, (target, received) => {
205
- for (const { path, type } of watched.get(target) ?? [])
206
- listener(sampleOf(path, type, received));
207
- });
208
- };
209
-
210
- const values = async (patterns: readonly string[]) => {
211
- const connection = adapter.connection();
212
- if (connection.state !== "connected")
213
- throw new DebugError(connection.error ?? "The controller is not connected", 503);
214
- const located = paths.expand(patterns).map((path) => ({ path, located: locate(path) }));
215
- const read = await adapter.read(
216
- located.flatMap(({ located }) => ("target" in located ? [targetRead(located)] : [])),
217
- );
218
- return located.map(({ path, located }) => {
219
- if (!("target" in located)) return located;
220
- const sample = read.get(located.target);
221
- return sample ? sampleOf(path, located.type, sample) : unavailable(path, "Not read");
222
- });
223
- };
224
-
225
- const stopBuild = adapter.subscribe(
226
- [{ target: manifest.build.target, type: { kind: "scalar", iec: "STRING" } }],
227
- (_, sample) => {
228
- if (sample.quality === "good" && typeof sample.value === "string") setBuild(sample.value);
229
- },
230
- buildCheckMs,
231
- );
232
-
233
- // --- Jobs and devices, read only while shown -------------------------------------------------
234
-
235
- const latest = new Map<string, TargetSample>();
236
- const traced = new Map<number, TracedJob>();
237
- const failures = new Map<number, NonNullable<JobState["failure"]>>();
238
- /** The server's traces by `<job id>:<service>:<sequence>`, newest last. */
239
- const requests = new Map<string, RequestTrace>();
240
- let unread = new Set<number>();
241
- const shownJobs = createShown<number>();
242
- const shownHealth = createShown<string>();
243
- const shownDevices = createShown<string>();
244
- /** The JSON of what each stream last got, so unchanged states aren't sent again. */
245
- const sent = new Map<string, string>();
246
-
247
- const jobOf = (job: DebugJob) =>
248
- jobState(job, {
249
- value: (target) => latest.get(target),
250
- traced: traced.get(job.id),
251
- failure: failures.get(job.id),
252
- unread: unread.has(job.id),
253
- clock: telemetry.clock(),
254
- now: Date.now(),
255
- request: ({ job, service, sequence }) => requests.get(`${job}:${service}:${sequence}`),
256
- });
257
- const deviceOf = (id: string) =>
258
- deviceState(
259
- manifest.devices.find((device) => device.id === id)!,
260
- manifest.io,
261
- (target) => latest.get(target),
262
- );
263
- const send = (key: string, message: StreamMessage) => {
264
- const json = JSON.stringify(message.data);
265
- if (sent.get(key) === json) return;
266
- sent.set(key, json);
267
- emit(message);
268
- };
269
- const updateJob = (job: DebugJob) => send(`job:${job.id}`, { event: "job", data: jobOf(job) });
270
- const updateDevice = (id: string) =>
271
- send(`device:${id}`, { event: "device", data: deviceOf(id) });
272
-
273
- /** Re-subscribes `reads` when they change, telling `update` which owners each sample is for. */
274
- const reader = <Owner>(intervalMs: number, update: (owner: Owner) => void) => {
275
- let current = { key: "", stop: () => {} };
276
- return (reads: { owner: Owner; read: TargetRead }[]) => {
277
- const key = reads.map(({ read }) => read.target).join("\n");
278
- if (key === current.key) return;
279
- current.stop();
280
- const owners = new Map<string, Owner[]>();
281
- for (const { owner, read } of reads)
282
- owners.set(read.target, [...(owners.get(read.target) ?? []), owner]);
283
- current = {
284
- key,
285
- stop: reads.length
286
- ? adapter.subscribe(
287
- [...new Map(reads.map(({ read }) => [read.target, read])).values()],
288
- (target, sample) => {
289
- latest.set(target, sample);
290
- for (const owner of owners.get(target) ?? []) update(owner);
291
- schedule();
292
- },
293
- intervalMs,
294
- )
295
- : () => {},
296
- };
297
- };
298
- };
299
- const readJobs = reader<DebugJob>(adapter.samplingMs, updateJob);
300
- const readDevices = reader<string>(hardwareMs, updateDevice);
301
-
302
- /** Reads `busy` of the shown jobs, and the details of the busy ones. */
303
- const plan = () => {
304
- const shown = manifest.jobs.filter((job) => shownJobs.has(job.id));
305
- const busy = shown.filter((job) =>
306
- busyOf(job, (target) => latest.get(target), traced.get(job.id)),
307
- );
308
- unread = new Set(busy.slice(detailedJobs).map(({ id }) => id));
309
- readJobs([
310
- ...shown.map((job) => ({ owner: job, read: busyRead(job) })),
311
- ...busy
312
- .slice(0, detailedJobs)
313
- .flatMap((job) => detailReads(job).map((read) => ({ owner: job, read }))),
314
- ]);
315
- readDevices(
316
- manifest.devices.flatMap((device) =>
317
- (shownDevices.has(device.id)
318
- ? deviceReads(device, manifest.io, paths.typeOf)
319
- : shownHealth.has(device.id)
320
- ? healthReads(device)
321
- : []
322
- ).map((read) => ({ owner: device.id, read })),
323
- ),
324
- );
325
- };
326
- let planned = false;
327
- const schedule = () => {
328
- if (planned) return;
329
- planned = true;
330
- queueMicrotask(() => {
331
- planned = false;
332
- plan();
333
- });
334
- };
335
-
336
- /** Reads what `interest` shows until the returned function is called. */
337
- const show = (interest: Omit<Interest, "watch">) => {
338
- const stops = [
339
- shownJobs.add(interest.jobs),
340
- shownHealth.add(interest.health),
341
- shownDevices.add(interest.devices),
342
- ];
343
- schedule();
344
- return () => {
345
- for (const stop of stops) stop();
346
- schedule();
347
- };
348
- };
349
-
350
- // --- The app server ---------------------------------------------------------------------------
351
-
352
- const receive = ({ build, task, sequence, at, record: received }: ReceivedRecord) => {
353
- if (received.kind === "trace") {
354
- // Event sites of another build would decode to the wrong source.
355
- if (build !== manifest.build.id) return;
356
- const event = trace.decode(received, task, sequence);
357
- if (event) record(timelineEntries.trace(at, event));
358
- const site = trace.job(received);
359
- if (!site) return;
360
- const startedAt = site.started ? at : traced.get(site.job.id)?.startedAt;
361
- traced.set(site.job.id, { busy: site.started, ...(startedAt && { startedAt }) });
362
- updateJob(site.job);
363
- schedule();
364
- return;
365
- }
366
- record(timelineEntries.app(at, task, sequence, received));
367
- if (received.kind !== "exception" || received.handled) return;
368
- const job = manifest.jobs.find(
369
- ({ name, instance }) => name === received.function && instance === received.instance,
370
- );
371
- if (!job) return;
372
- failures.set(job.id, {
373
- at,
374
- code: received.code,
375
- ...(received.name && { name: received.name }),
376
- source: received.source,
377
- });
378
- updateJob(job);
379
- };
380
-
381
- const pull = async (url: URL, token: string) => {
382
- const response = await fetch(url, { headers: { authorization: `Bearer ${token}` }, signal });
383
- if (!response.ok) throw new Error(`The server answered ${response.status}`);
384
- return response.json();
385
- };
386
-
387
- const pullTelemetry = async ({ url, token }: { url: string; token: string }) => {
388
- const reply = telemetryReply.parse(
389
- await pull(new URL(`/tsplc/telemetry?since=${telemetry.since()}`, url), token),
390
- );
391
- const received = telemetry.accept(reply);
392
- for (const item of received) receive(item);
393
- const build = telemetry.build();
394
- if (build) setBuild(build);
395
- // Scan rates and the clock estimate change with every batch.
396
- if (reply.batches.length) emit({ event: "session", data: session() });
397
- };
398
-
399
- /** Follows the server's ring, asking again from before the oldest call still pending. */
400
- const pullRequests = ({ url, token }: { url: string; token: string }) => {
401
- let newest = 0;
402
- const pending = new Map<number, Entry<"request">>();
403
- return async () => {
404
- const since = pending.size ? Math.min(...pending.keys()) - 1 : newest;
405
- const body = requestsReply.parse(
406
- await pull(new URL(`/tsplc/debug/requests?since=${since}`, url), token),
407
- );
408
- // A restarted server counts from 1 again.
409
- if (body.latest < newest) {
410
- newest = 0;
411
- pending.clear();
412
- return;
413
- }
414
- const oldest = body.requests[0]?.seq;
415
- if (oldest !== undefined)
416
- for (const seq of pending.keys()) if (seq < oldest) pending.delete(seq);
417
- for (const request of body.requests) {
418
- const waiting = pending.get(request.seq);
419
- if (waiting && request.status === "pending") continue;
420
- if (!waiting && request.seq <= newest) continue;
421
- const entry = waiting ?? timelineEntries.request(request);
422
- entry.request = request;
423
- newest = Math.max(newest, request.seq);
424
- if (request.status === "pending") pending.set(request.seq, entry);
425
- else pending.delete(request.seq);
426
- if (!waiting) timeline.add(entry);
427
- emit({ event: "request", data: request });
428
- emit({ event: "timeline", data: entry });
429
- if (request.job === undefined || request.sequence === undefined) continue;
430
- requests.set(`${request.job}:${request.service}:${request.sequence}`, request);
431
- for (const key of [...requests.keys()].slice(0, -requestsRemembered)) requests.delete(key);
432
- const job = manifest.jobs.find(({ id }) => id === request.job);
433
- if (job) updateJob(job);
434
- }
435
- };
436
- };
437
-
438
- // --- Background work -------------------------------------------------------------------------
439
-
440
- const stopConnection = adapter.onConnection((connection) => {
441
- record(timelineEntries.connection(connection.since, connection.state, connection.error));
442
- emit({ event: "session", data: session() });
443
- if (connection.state !== "connected") {
444
- for (const [target, sample] of latest)
445
- latest.set(target, {
446
- ...sample,
447
- quality: "unavailable",
448
- status: connection.error ?? "The controller is not connected",
449
- });
450
- for (const job of manifest.jobs) updateJob(job);
451
- for (const device of manifest.devices) updateDevice(device.id);
452
- }
453
- });
454
-
455
- const repeat = async (name: string, intervalMs: number, work: () => Promise<void>) => {
456
- let failure: string | undefined;
457
- while (!signal.aborted) {
458
- try {
459
- await work();
460
- failure = undefined;
461
- } catch (error) {
462
- if (signal.aborted) return;
463
- if (message(error) !== failure) options.log(`${name} failed: ${message(error)}`);
464
- failure = message(error);
465
- }
466
- await delay(intervalMs, undefined, { signal }).catch(() => undefined);
467
- }
468
- };
469
-
470
- const { server } = options;
471
- const running = server
472
- ? [
473
- repeat("Reading the server's telemetry", pullMs, () => pullTelemetry(server)),
474
- repeat("Reading the server's requests", pullMs, pullRequests(server)),
475
- ]
476
- : [];
477
-
478
- return {
479
- manifest,
480
- paths,
481
- session,
482
- values,
483
- jobs: () => manifest.jobs.map(jobOf),
484
- hardware: () => manifest.devices.map(({ id }) => deviceOf(id)),
485
- timeline: (since?: string) => timeline.since(since),
486
- watchGroups: () => watchGroups.groups(),
487
- saveWatchGroups: (groups: WatchGroups) => watchGroups.save(groups),
488
- /** Adds an entry from another source, such as the agent's audit log. */
489
- record,
490
- watch,
491
- show,
492
- /** Calls `listener` with every other stream event until the returned function is called. */
493
- subscribe: (listener: (message: StreamMessage) => void) => {
494
- listeners.add(listener);
495
- return () => listeners.delete(listener);
496
- },
497
- close: async () => {
498
- controller.abort();
499
- stopBuild();
500
- readJobs([]);
501
- readDevices([]);
502
- stopConnection();
503
- await Promise.all(running);
504
- await adapter.close();
505
- },
506
- };
100
+ const { manifest, adapter } = options;
101
+ const paths = createPaths(manifest);
102
+ const trace = createTraceDecoder(manifest, paths);
103
+ const telemetry = createTelemetryFollower(manifest);
104
+ const requests = createRequestFollower();
105
+ const timeline = createTimeline();
106
+ const watchGroups = await loadWatchGroups(
107
+ join(options.stateDirectory, "watch.json"),
108
+ options.watch,
109
+ );
110
+ const listeners = new Set<(message: StreamMessage) => void>();
111
+ const controller = new AbortController();
112
+ const { signal } = controller;
113
+ let controllerBuild: string | null = null;
114
+
115
+ const emit = (message: StreamMessage) => {
116
+ for (const listener of listeners) {
117
+ listener(message);
118
+ }
119
+ };
120
+ const record = (entry: TimelineEntry) => {
121
+ timeline.add(entry);
122
+ emit({ event: "timeline", data: entry });
123
+ };
124
+ for (const entry of options.timeline ?? []) {
125
+ timeline.add(entry);
126
+ }
127
+
128
+ const session = (): Session => ({
129
+ build: {
130
+ manifest: manifest.build.id,
131
+ controller: controllerBuild,
132
+ matches: controllerBuild === manifest.build.id,
133
+ },
134
+ connection: { ...adapter.connection(), endpoint: adapter.endpoint },
135
+ reading: { method: adapter.method, intervalMs: adapter.samplingMs },
136
+ tasks: telemetry.tasks(),
137
+ clock: telemetry.clock(),
138
+ ...(options.release && { release: options.release }),
139
+ ...(options.projectRoot && { projectRoot: options.projectRoot }),
140
+ });
141
+
142
+ const setBuild = (build: string) => {
143
+ if (build === controllerBuild) {
144
+ return;
145
+ }
146
+ controllerBuild = build;
147
+ emit({ event: "session", data: session() });
148
+ };
149
+
150
+ // --- Values by TypeScript path ---------------------------------------------------------------
151
+
152
+ const unavailable = (path: string, status: string): Sample => ({
153
+ path,
154
+ value: null,
155
+ quality: "unavailable",
156
+ status,
157
+ receivedAt: new Date().toISOString(),
158
+ });
159
+
160
+ /** A path's target, or the sample it has without reading the controller. */
161
+ const locate = (path: string): { target: string; type: string } | Sample => {
162
+ const resolved = paths.resolve(path);
163
+ if (!resolved) {
164
+ return unavailable(path, "Unknown path");
165
+ }
166
+ if (resolved.kind === "constant") {
167
+ return {
168
+ path,
169
+ value: resolved.value,
170
+ quality: "good",
171
+ status: "Constant",
172
+ receivedAt: new Date().toISOString(),
173
+ };
174
+ }
175
+ if (!resolved.readable) {
176
+ return unavailable(path, "Not readable");
177
+ }
178
+ return resolved;
179
+ };
180
+
181
+ const targetRead = ({
182
+ target,
183
+ type,
184
+ }: {
185
+ target: string;
186
+ type: string;
187
+ }): TargetRead => ({
188
+ target,
189
+ type: paths.typeOf(type),
190
+ });
191
+
192
+ const sampleOf = (
193
+ path: string,
194
+ type: string,
195
+ sample: TargetSample,
196
+ ): Sample => ({
197
+ path,
198
+ ...sample,
199
+ value: paths.decode(type, sample.value),
200
+ });
201
+
202
+ /** Reads the leaves `patterns` match each sampling interval, until the returned function is called. */
203
+ const watch = (
204
+ patterns: readonly string[],
205
+ listener: (sample: Sample) => void,
206
+ ) => {
207
+ const watched = new Map<string, { path: string; type: string }[]>();
208
+ const reads: TargetRead[] = [];
209
+ for (const path of paths.expand(patterns)) {
210
+ const located = locate(path);
211
+ if (!("target" in located)) {
212
+ listener(located);
213
+ continue;
214
+ }
215
+ if (!watched.has(located.target)) {
216
+ reads.push(targetRead(located));
217
+ }
218
+ watched.set(located.target, [
219
+ ...(watched.get(located.target) ?? []),
220
+ { path, ...located },
221
+ ]);
222
+ }
223
+ if (!reads.length) {
224
+ return () => {};
225
+ }
226
+ return adapter.subscribe(reads, (target, received) => {
227
+ for (const { path, type } of watched.get(target) ?? []) {
228
+ listener(sampleOf(path, type, received));
229
+ }
230
+ });
231
+ };
232
+
233
+ const values = async (patterns: readonly string[]) => {
234
+ const connection = adapter.connection();
235
+ if (connection.state !== "connected") {
236
+ throw new DebugError(
237
+ connection.error ?? "The controller is not connected",
238
+ 503,
239
+ );
240
+ }
241
+ const located = paths
242
+ .expand(patterns)
243
+ .map((path) => ({ path, located: locate(path) }));
244
+ const read = await adapter.read(
245
+ located.flatMap(({ located }) =>
246
+ "target" in located ? [targetRead(located)] : [],
247
+ ),
248
+ );
249
+ return located.map(({ path, located }) => {
250
+ if (!("target" in located)) {
251
+ return located;
252
+ }
253
+ const sample = read.get(located.target);
254
+ return sample
255
+ ? sampleOf(path, located.type, sample)
256
+ : unavailable(path, "Not read");
257
+ });
258
+ };
259
+
260
+ const stopBuild = adapter.subscribe(
261
+ [
262
+ {
263
+ target: manifest.build.target,
264
+ type: { kind: "scalar", iec: "STRING" },
265
+ },
266
+ ],
267
+ (_, sample) => {
268
+ if (sample.quality === "good" && typeof sample.value === "string") {
269
+ setBuild(sample.value);
270
+ }
271
+ },
272
+ buildCheckMs,
273
+ );
274
+
275
+ // --- Jobs and devices, read only while shown -------------------------------------------------
276
+
277
+ const latest = new Map<string, TargetSample>();
278
+ const traced = new Map<number, TracedJob>();
279
+ const failures = new Map<number, NonNullable<JobState["failure"]>>();
280
+ let unread = new Set<number>();
281
+ const shownJobs = createShown<number>();
282
+ const shownHealth = createShown<string>();
283
+ const shownDevices = createShown<string>();
284
+ /** The JSON of what each stream last got, so unchanged states aren't sent again. */
285
+ const sent = new Map<string, string>();
286
+
287
+ const jobOf = (job: DebugJob) =>
288
+ jobState(job, {
289
+ value: (target) => latest.get(target),
290
+ traced: traced.get(job.id),
291
+ failure: failures.get(job.id),
292
+ unread: unread.has(job.id),
293
+ clock: telemetry.clock(),
294
+ now: Date.now(),
295
+ request: requests.call,
296
+ });
297
+ const deviceOf = (device: DebugDevice) =>
298
+ deviceState(device, manifest.io, (target) => latest.get(target));
299
+ const send = (key: string, message: StreamMessage) => {
300
+ const json = JSON.stringify(message.data);
301
+ if (sent.get(key) === json) {
302
+ return;
303
+ }
304
+ sent.set(key, json);
305
+ emit(message);
306
+ };
307
+ const updateJob = (job: DebugJob) =>
308
+ send(`job:${job.id}`, { event: "job", data: jobOf(job) });
309
+ const updateDevice = (device: DebugDevice) =>
310
+ send(`device:${device.id}`, { event: "device", data: deviceOf(device) });
311
+
312
+ /** Re-subscribes `reads` when they change, telling `update` which owners each sample is for. */
313
+ const reader = <Owner>(
314
+ intervalMs: number,
315
+ update: (owner: Owner) => void,
316
+ ) => {
317
+ let current = { key: "", stop: () => {} };
318
+ return (reads: { owner: Owner; read: TargetRead }[]) => {
319
+ const key = reads.map(({ read }) => read.target).join("\n");
320
+ if (key === current.key) {
321
+ return;
322
+ }
323
+ current.stop();
324
+ const owners = new Map<string, Owner[]>();
325
+ for (const { owner, read } of reads) {
326
+ owners.set(read.target, [...(owners.get(read.target) ?? []), owner]);
327
+ }
328
+ current = {
329
+ key,
330
+ stop: reads.length
331
+ ? adapter.subscribe(
332
+ [
333
+ ...new Map(
334
+ reads.map(({ read }) => [read.target, read]),
335
+ ).values(),
336
+ ],
337
+ (target, sample) => {
338
+ latest.set(target, sample);
339
+ for (const owner of owners.get(target) ?? []) {
340
+ update(owner);
341
+ }
342
+ schedule();
343
+ },
344
+ intervalMs,
345
+ )
346
+ : () => {},
347
+ };
348
+ };
349
+ };
350
+ const readJobs = reader<DebugJob>(adapter.samplingMs, updateJob);
351
+ const readDevices = reader<DebugDevice>(hardwareMs, updateDevice);
352
+
353
+ /** Everything of a device whose page is shown, else its health while a list shows it. */
354
+ const shownDeviceReads = (device: DebugDevice) => {
355
+ if (shownDevices.has(device.id)) {
356
+ return deviceReads(device, manifest.io, paths.typeOf);
357
+ }
358
+ if (shownHealth.has(device.id)) {
359
+ return healthReads(device);
360
+ }
361
+ return [];
362
+ };
363
+
364
+ /** Reads `busy` of the shown jobs, and the details of the busy ones. */
365
+ const plan = () => {
366
+ const shown = manifest.jobs.filter((job) => shownJobs.has(job.id));
367
+ const busy = shown.filter((job) =>
368
+ busyOf(job, (target) => latest.get(target), traced.get(job.id)),
369
+ );
370
+ unread = new Set(busy.slice(detailedJobs).map(({ id }) => id));
371
+ readJobs([
372
+ ...shown.map((job) => ({ owner: job, read: busyRead(job) })),
373
+ ...busy
374
+ .slice(0, detailedJobs)
375
+ .flatMap((job) =>
376
+ detailReads(job).map((read) => ({ owner: job, read })),
377
+ ),
378
+ ]);
379
+ readDevices(
380
+ manifest.devices.flatMap((device) =>
381
+ shownDeviceReads(device).map((read) => ({ owner: device, read })),
382
+ ),
383
+ );
384
+ };
385
+ let planned = false;
386
+ const schedule = () => {
387
+ if (planned) {
388
+ return;
389
+ }
390
+ planned = true;
391
+ queueMicrotask(() => {
392
+ planned = false;
393
+ plan();
394
+ });
395
+ };
396
+
397
+ /** Reads what `interest` shows until the returned function is called. */
398
+ const show = (interest: Omit<Interest, "watch">) => {
399
+ const stops = [
400
+ shownJobs.add(interest.jobs),
401
+ shownHealth.add(interest.health),
402
+ shownDevices.add(interest.devices),
403
+ ];
404
+ schedule();
405
+ return () => {
406
+ for (const stop of stops) {
407
+ stop();
408
+ }
409
+ schedule();
410
+ };
411
+ };
412
+
413
+ // --- The app server ---------------------------------------------------------------------------
414
+
415
+ const receive = ({
416
+ build,
417
+ task,
418
+ sequence,
419
+ at,
420
+ record: received,
421
+ }: ReceivedRecord) => {
422
+ if (received.kind === "trace") {
423
+ // Event sites of another build would decode to the wrong source.
424
+ if (build !== manifest.build.id) {
425
+ return;
426
+ }
427
+ const event = trace.decode(received, task, sequence);
428
+ if (event) {
429
+ record(timelineEntries.trace(at, event));
430
+ }
431
+ const site = trace.job(received);
432
+ if (!site) {
433
+ return;
434
+ }
435
+ const startedAt = site.started ? at : traced.get(site.job.id)?.startedAt;
436
+ traced.set(site.job.id, {
437
+ busy: site.started,
438
+ ...(startedAt && { startedAt }),
439
+ });
440
+ updateJob(site.job);
441
+ schedule();
442
+ return;
443
+ }
444
+ record(timelineEntries.app(at, task, sequence, received));
445
+ if (received.kind !== "exception" || received.handled) {
446
+ return;
447
+ }
448
+ const job = manifest.jobs.find(
449
+ ({ name, instance }) =>
450
+ name === received.function && instance === received.instance,
451
+ );
452
+ if (!job) {
453
+ return;
454
+ }
455
+ failures.set(job.id, {
456
+ at,
457
+ code: received.code,
458
+ ...(received.name && { name: received.name }),
459
+ source: received.source,
460
+ });
461
+ updateJob(job);
462
+ };
463
+
464
+ const pull = async (url: URL, token: string) => {
465
+ const response = await fetch(url, {
466
+ headers: { authorization: `Bearer ${token}` },
467
+ signal,
468
+ });
469
+ if (!response.ok) {
470
+ throw new Error(`The server answered ${response.status}`);
471
+ }
472
+ return response.json();
473
+ };
474
+
475
+ const pullTelemetry = async ({
476
+ url,
477
+ token,
478
+ }: {
479
+ url: string;
480
+ token: string;
481
+ }) => {
482
+ const reply = telemetryReply.parse(
483
+ await pull(
484
+ new URL(`/tsplc/telemetry?since=${telemetry.since()}`, url),
485
+ token,
486
+ ),
487
+ );
488
+ const received = telemetry.accept(reply);
489
+ for (const item of received) {
490
+ receive(item);
491
+ }
492
+ const build = telemetry.build();
493
+ if (build) {
494
+ setBuild(build);
495
+ }
496
+ // Scan rates and the clock estimate change with every batch.
497
+ if (reply.batches.length) {
498
+ emit({ event: "session", data: session() });
499
+ }
500
+ };
501
+
502
+ const pullRequests = async ({
503
+ url,
504
+ token,
505
+ }: {
506
+ url: string;
507
+ token: string;
508
+ }) => {
509
+ const reply = requestsReply.parse(
510
+ await pull(
511
+ new URL(`/tsplc/debug/requests?since=${requests.since()}`, url),
512
+ token,
513
+ ),
514
+ );
515
+ requests.accept(reply, ({ request, entry, added }) => {
516
+ if (added) {
517
+ timeline.add(entry);
518
+ }
519
+ emit({ event: "request", data: request });
520
+ emit({ event: "timeline", data: entry });
521
+ if (request.job === undefined || request.sequence === undefined) {
522
+ return;
523
+ }
524
+ const job = manifest.jobs.find(({ id }) => id === request.job);
525
+ if (job) {
526
+ updateJob(job);
527
+ }
528
+ });
529
+ };
530
+
531
+ // --- Background work -------------------------------------------------------------------------
532
+
533
+ const stopConnection = adapter.onConnection((connection) => {
534
+ record(
535
+ timelineEntries.connection(
536
+ connection.since,
537
+ connection.state,
538
+ connection.error,
539
+ ),
540
+ );
541
+ emit({ event: "session", data: session() });
542
+ if (connection.state !== "connected") {
543
+ for (const [target, sample] of latest) {
544
+ latest.set(target, {
545
+ ...sample,
546
+ quality: "unavailable",
547
+ status: connection.error ?? "The controller is not connected",
548
+ });
549
+ }
550
+ for (const job of manifest.jobs) {
551
+ updateJob(job);
552
+ }
553
+ for (const device of manifest.devices) {
554
+ updateDevice(device);
555
+ }
556
+ }
557
+ });
558
+
559
+ const repeat = async (
560
+ name: string,
561
+ intervalMs: number,
562
+ work: () => Promise<void>,
563
+ ) => {
564
+ let failure: string | undefined;
565
+ while (!signal.aborted) {
566
+ try {
567
+ await work();
568
+ failure = undefined;
569
+ } catch (error) {
570
+ if (signal.aborted) {
571
+ return;
572
+ }
573
+ const message = errorMessage(error);
574
+ if (message !== failure) {
575
+ options.log(`${name} failed: ${message}`);
576
+ }
577
+ failure = message;
578
+ }
579
+ await delay(intervalMs, undefined, { signal }).catch(() => undefined);
580
+ }
581
+ };
582
+
583
+ const { server } = options;
584
+ const running = server
585
+ ? [
586
+ repeat("Reading the server's telemetry", pullMs, () =>
587
+ pullTelemetry(server),
588
+ ),
589
+ repeat("Reading the server's requests", pullMs, () =>
590
+ pullRequests(server),
591
+ ),
592
+ ]
593
+ : [];
594
+
595
+ return {
596
+ manifest,
597
+ paths,
598
+ session,
599
+ values,
600
+ jobs: () => manifest.jobs.map(jobOf),
601
+ hardware: () => manifest.devices.map(deviceOf),
602
+ timeline: (since?: string) => timeline.since(since),
603
+ watchGroups: () => watchGroups.groups(),
604
+ saveWatchGroups: (groups: WatchGroups) => watchGroups.save(groups),
605
+ /** Adds an entry from another source, such as the agent's audit log. */
606
+ record,
607
+ watch,
608
+ show,
609
+ /** Calls `listener` with every other stream event until the returned function is called. */
610
+ subscribe: (listener: (message: StreamMessage) => void) => {
611
+ listeners.add(listener);
612
+ return () => listeners.delete(listener);
613
+ },
614
+ close: async () => {
615
+ controller.abort();
616
+ stopBuild();
617
+ readJobs([]);
618
+ readDevices([]);
619
+ stopConnection();
620
+ await Promise.all(running);
621
+ await adapter.close();
622
+ },
623
+ };
507
624
  };
508
625
 
509
626
  export type Debugger = Awaited<ReturnType<typeof createDebugger>>;