@tsplc/debug 0.0.0-stage → 0.2.17

package/src/debugger.ts

@@ -0,0 +1,509 @@
+import { join } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { telemetryBatch, type TelemetryReply } from "@tsplc/server/telemetry";
+import { z } from "zod";
+import type { TargetAdapter, TargetRead, TargetSample } from "./adapter.ts";
+import type {
+  Interest,
+  JobState,
+  RequestTrace,
+  Sample,
+  Session,
+  StreamEvents,
+  TimelineEntry,
+  WatchGroups,
+} from "./api.ts";
+import { deviceReads, deviceState, healthReads } from "./hardware.ts";
+import { busyOf, busyRead, detailReads, jobState, type TracedJob } from "./jobs.ts";
+import type { DebugJob, DebugManifest } from "./manifest.ts";
+import { createPaths } from "./paths.ts";
+import { createTelemetryFollower, type ReceivedRecord } from "./telemetry.ts";
+import { createTimeline, type Entry, timelineEntries } from "./timeline.ts";
+import { createTraceDecoder } from "./trace.ts";
+import { loadWatchGroups } from "./watch.ts";
+
+export interface DebuggerOptions {
+  manifest: DebugManifest;
+  /** Owned by the debugger from here on: closing the debugger closes it. */
+  adapter: TargetAdapter;
+  /** Saved watch groups live here. */
+  stateDirectory: string;
+  /** `debug.watch` from the config. */
+  watch: Record<string, string[]>;
+  /** The app server, whose telemetry and request trace are pulled every second. */
+  server?: { url: string; token: string };
+  release?: Session["release"];
+  /** Entries from before the debugger started, such as the agent's deployments. */
+  timeline?: readonly TimelineEntry[];
+  projectRoot?: string;
+  log: (message: string) => void;
+}
+
+/** A request the API refuses, with its HTTP status. */
+export class DebugError extends Error {
+  constructor(
+    message: string,
+    readonly status: number,
+  ) {
+    super(message);
+  }
+}
+
+export type StreamMessage = {
+  [Name in keyof StreamEvents]: { event: Name; data: StreamEvents[Name] };
+}[keyof StreamEvents];
+
+const pullMs = 1000;
+/** Without telemetry, the build id is read through the controller this often. */
+const buildCheckMs = 10_000;
+/** Device health, raw bytes and channels change rarely enough for this. */
+const hardwareMs = 5000;
+/** Busy jobs whose details each poll reads; the rest show as busy only. */
+const detailedJobs = 16;
+const requestsRemembered = 1000;
+
+const requestTrace = z.object({
+  seq: z.number().int(),
+  service: z.string(),
+  job: z.number().int().optional(),
+  sequence: z.number().int().optional(),
+  startedAt: z.string(),
+  durationMs: z.number().optional(),
+  status: z.enum(["pending", "ok", "error", "timeout"]),
+  httpStatus: z.number().int().optional(),
+  error: z.string().optional(),
+  input: z.unknown().optional(),
+  output: z.unknown().optional(),
+}) satisfies z.ZodType<RequestTrace>;
+const requestsReply = z.object({ requests: z.array(requestTrace), latest: z.number().int() });
+const telemetryReply = z.object({
+  batches: z.array(telemetryBatch.extend({ seq: z.number().int(), receivedAt: z.string() })),
+  latest: z.number().int(),
+}) satisfies z.ZodType<TelemetryReply>;
+
+const message = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+/** Counts how many streams show each key. */
+const createShown = <Key>() => {
+  const counts = new Map<Key, number>();
+  return {
+    has: (key: Key) => counts.has(key),
+    add: (keys: readonly Key[]) => {
+      for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1);
+      return () => {
+        for (const key of keys) {
+          const count = counts.get(key)! - 1;
+          if (count) counts.set(key, count);
+          else counts.delete(key);
+        }
+      };
+    },
+  };
+};
+
+/**
+ * The debugger for one build on one controller (docs/design/debugging.md). It reads what the
+ * pages show through the adapter, follows the app server's telemetry and requests, and keeps the
+ * recent window of both. Everything it offers is read-only.
+ */
+export const createDebugger = async (options: DebuggerOptions) => {
+  const { manifest, adapter } = options;
+  const paths = createPaths(manifest);
+  const trace = createTraceDecoder(manifest, paths);
+  const telemetry = createTelemetryFollower(manifest);
+  const timeline = createTimeline();
+  const watchGroups = await loadWatchGroups(
+    join(options.stateDirectory, "watch.json"),
+    options.watch,
+  );
+  const listeners = new Set<(message: StreamMessage) => void>();
+  const controller = new AbortController();
+  const { signal } = controller;
+  let controllerBuild: string | null = null;
+
+  const emit = (message: StreamMessage) => {
+    for (const listener of listeners) listener(message);
+  };
+  const record = (entry: TimelineEntry) => {
+    timeline.add(entry);
+    emit({ event: "timeline", data: entry });
+  };
+  for (const entry of options.timeline ?? []) timeline.add(entry);
+
+  const session = (): Session => ({
+    build: {
+      manifest: manifest.build.id,
+      controller: controllerBuild,
+      matches: controllerBuild === manifest.build.id,
+    },
+    connection: { ...adapter.connection(), endpoint: adapter.endpoint },
+    reading: { method: adapter.method, intervalMs: adapter.samplingMs },
+    tasks: telemetry.tasks(),
+    clock: telemetry.clock(),
+    ...(options.release && { release: options.release }),
+    ...(options.projectRoot && { projectRoot: options.projectRoot }),
+  });
+
+  const setBuild = (build: string) => {
+    if (build === controllerBuild) return;
+    controllerBuild = build;
+    emit({ event: "session", data: session() });
+  };
+
+  // --- Values by TypeScript path ---------------------------------------------------------------
+
+  const unavailable = (path: string, status: string): Sample => ({
+    path,
+    value: null,
+    quality: "unavailable",
+    status,
+    receivedAt: new Date().toISOString(),
+  });
+
+  /** A path's target, or the sample it has without reading the controller. */
+  const locate = (path: string): { target: string; type: string } | Sample => {
+    const resolved = paths.resolve(path);
+    if (!resolved) return unavailable(path, "Unknown path");
+    if (resolved.kind === "constant")
+      return {
+        path,
+        value: resolved.value,
+        quality: "good",
+        status: "Constant",
+        receivedAt: new Date().toISOString(),
+      };
+    if (!resolved.readable) return unavailable(path, "Not readable");
+    return resolved;
+  };
+
+  const targetRead = ({ target, type }: { target: string; type: string }): TargetRead => ({
+    target,
+    type: paths.typeOf(type),
+  });
+
+  const sampleOf = (path: string, type: string, sample: TargetSample): Sample => ({
+    path,
+    ...sample,
+    value: paths.decode(type, sample.value),
+  });
+
+  /** Reads the leaves `patterns` match each sampling interval, until the returned function is called. */
+  const watch = (patterns: readonly string[], listener: (sample: Sample) => void) => {
+    const watched = new Map<string, { path: string; type: string }[]>();
+    const reads: TargetRead[] = [];
+    for (const path of paths.expand(patterns)) {
+      const located = locate(path);
+      if (!("target" in located)) {
+        listener(located);
+        continue;
+      }
+      if (!watched.has(located.target)) reads.push(targetRead(located));
+      watched.set(located.target, [...(watched.get(located.target) ?? []), { path, ...located }]);
+    }
+    if (!reads.length) return () => {};
+    return adapter.subscribe(reads, (target, received) => {
+      for (const { path, type } of watched.get(target) ?? [])
+        listener(sampleOf(path, type, received));
+    });
+  };
+
+  const values = async (patterns: readonly string[]) => {
+    const connection = adapter.connection();
+    if (connection.state !== "connected")
+      throw new DebugError(connection.error ?? "The controller is not connected", 503);
+    const located = paths.expand(patterns).map((path) => ({ path, located: locate(path) }));
+    const read = await adapter.read(
+      located.flatMap(({ located }) => ("target" in located ? [targetRead(located)] : [])),
+    );
+    return located.map(({ path, located }) => {
+      if (!("target" in located)) return located;
+      const sample = read.get(located.target);
+      return sample ? sampleOf(path, located.type, sample) : unavailable(path, "Not read");
+    });
+  };
+
+  const stopBuild = adapter.subscribe(
+    [{ target: manifest.build.target, type: { kind: "scalar", iec: "STRING" } }],
+    (_, sample) => {
+      if (sample.quality === "good" && typeof sample.value === "string") setBuild(sample.value);
+    },
+    buildCheckMs,
+  );
+
+  // --- Jobs and devices, read only while shown -------------------------------------------------
+
+  const latest = new Map<string, TargetSample>();
+  const traced = new Map<number, TracedJob>();
+  const failures = new Map<number, NonNullable<JobState["failure"]>>();
+  /** The server's traces by `<job id>:<service>:<sequence>`, newest last. */
+  const requests = new Map<string, RequestTrace>();
+  let unread = new Set<number>();
+  const shownJobs = createShown<number>();
+  const shownHealth = createShown<string>();
+  const shownDevices = createShown<string>();
+  /** The JSON of what each stream last got, so unchanged states aren't sent again. */
+  const sent = new Map<string, string>();
+
+  const jobOf = (job: DebugJob) =>
+    jobState(job, {
+      value: (target) => latest.get(target),
+      traced: traced.get(job.id),
+      failure: failures.get(job.id),
+      unread: unread.has(job.id),
+      clock: telemetry.clock(),
+      now: Date.now(),
+      request: ({ job, service, sequence }) => requests.get(`${job}:${service}:${sequence}`),
+    });
+  const deviceOf = (id: string) =>
+    deviceState(
+      manifest.devices.find((device) => device.id === id)!,
+      manifest.io,
+      (target) => latest.get(target),
+    );
+  const send = (key: string, message: StreamMessage) => {
+    const json = JSON.stringify(message.data);
+    if (sent.get(key) === json) return;
+    sent.set(key, json);
+    emit(message);
+  };
+  const updateJob = (job: DebugJob) => send(`job:${job.id}`, { event: "job", data: jobOf(job) });
+  const updateDevice = (id: string) =>
+    send(`device:${id}`, { event: "device", data: deviceOf(id) });
+
+  /** Re-subscribes `reads` when they change, telling `update` which owners each sample is for. */
+  const reader = <Owner>(intervalMs: number, update: (owner: Owner) => void) => {
+    let current = { key: "", stop: () => {} };
+    return (reads: { owner: Owner; read: TargetRead }[]) => {
+      const key = reads.map(({ read }) => read.target).join("\n");
+      if (key === current.key) return;
+      current.stop();
+      const owners = new Map<string, Owner[]>();
+      for (const { owner, read } of reads)
+        owners.set(read.target, [...(owners.get(read.target) ?? []), owner]);
+      current = {
+        key,
+        stop: reads.length
+          ? adapter.subscribe(
+              [...new Map(reads.map(({ read }) => [read.target, read])).values()],
+              (target, sample) => {
+                latest.set(target, sample);
+                for (const owner of owners.get(target) ?? []) update(owner);
+                schedule();
+              },
+              intervalMs,
+            )
+          : () => {},
+      };
+    };
+  };
+  const readJobs = reader<DebugJob>(adapter.samplingMs, updateJob);
+  const readDevices = reader<string>(hardwareMs, updateDevice);
+
+  /** Reads `busy` of the shown jobs, and the details of the busy ones. */
+  const plan = () => {
+    const shown = manifest.jobs.filter((job) => shownJobs.has(job.id));
+    const busy = shown.filter((job) =>
+      busyOf(job, (target) => latest.get(target), traced.get(job.id)),
+    );
+    unread = new Set(busy.slice(detailedJobs).map(({ id }) => id));
+    readJobs([
+      ...shown.map((job) => ({ owner: job, read: busyRead(job) })),
+      ...busy
+        .slice(0, detailedJobs)
+        .flatMap((job) => detailReads(job).map((read) => ({ owner: job, read }))),
+    ]);
+    readDevices(
+      manifest.devices.flatMap((device) =>
+        (shownDevices.has(device.id)
+          ? deviceReads(device, manifest.io, paths.typeOf)
+          : shownHealth.has(device.id)
+            ? healthReads(device)
+            : []
+        ).map((read) => ({ owner: device.id, read })),
+      ),
+    );
+  };
+  let planned = false;
+  const schedule = () => {
+    if (planned) return;
+    planned = true;
+    queueMicrotask(() => {
+      planned = false;
+      plan();
+    });
+  };
+
+  /** Reads what `interest` shows until the returned function is called. */
+  const show = (interest: Omit<Interest, "watch">) => {
+    const stops = [
+      shownJobs.add(interest.jobs),
+      shownHealth.add(interest.health),
+      shownDevices.add(interest.devices),
+    ];
+    schedule();
+    return () => {
+      for (const stop of stops) stop();
+      schedule();
+    };
+  };
+
+  // --- The app server ---------------------------------------------------------------------------
+
+  const receive = ({ build, task, sequence, at, record: received }: ReceivedRecord) => {
+    if (received.kind === "trace") {
+      // Event sites of another build would decode to the wrong source.
+      if (build !== manifest.build.id) return;
+      const event = trace.decode(received, task, sequence);
+      if (event) record(timelineEntries.trace(at, event));
+      const site = trace.job(received);
+      if (!site) return;
+      const startedAt = site.started ? at : traced.get(site.job.id)?.startedAt;
+      traced.set(site.job.id, { busy: site.started, ...(startedAt && { startedAt }) });
+      updateJob(site.job);
+      schedule();
+      return;
+    }
+    record(timelineEntries.app(at, task, sequence, received));
+    if (received.kind !== "exception" || received.handled) return;
+    const job = manifest.jobs.find(
+      ({ name, instance }) => name === received.function && instance === received.instance,
+    );
+    if (!job) return;
+    failures.set(job.id, {
+      at,
+      code: received.code,
+      ...(received.name && { name: received.name }),
+      source: received.source,
+    });
+    updateJob(job);
+  };
+
+  const pull = async (url: URL, token: string) => {
+    const response = await fetch(url, { headers: { authorization: `Bearer ${token}` }, signal });
+    if (!response.ok) throw new Error(`The server answered ${response.status}`);
+    return response.json();
+  };
+
+  const pullTelemetry = async ({ url, token }: { url: string; token: string }) => {
+    const reply = telemetryReply.parse(
+      await pull(new URL(`/tsplc/telemetry?since=${telemetry.since()}`, url), token),
+    );
+    const received = telemetry.accept(reply);
+    for (const item of received) receive(item);
+    const build = telemetry.build();
+    if (build) setBuild(build);
+    // Scan rates and the clock estimate change with every batch.
+    if (reply.batches.length) emit({ event: "session", data: session() });
+  };
+
+  /** Follows the server's ring, asking again from before the oldest call still pending. */
+  const pullRequests = ({ url, token }: { url: string; token: string }) => {
+    let newest = 0;
+    const pending = new Map<number, Entry<"request">>();
+    return async () => {
+      const since = pending.size ? Math.min(...pending.keys()) - 1 : newest;
+      const body = requestsReply.parse(
+        await pull(new URL(`/tsplc/debug/requests?since=${since}`, url), token),
+      );
+      // A restarted server counts from 1 again.
+      if (body.latest < newest) {
+        newest = 0;
+        pending.clear();
+        return;
+      }
+      const oldest = body.requests[0]?.seq;
+      if (oldest !== undefined)
+        for (const seq of pending.keys()) if (seq < oldest) pending.delete(seq);
+      for (const request of body.requests) {
+        const waiting = pending.get(request.seq);
+        if (waiting && request.status === "pending") continue;
+        if (!waiting && request.seq <= newest) continue;
+        const entry = waiting ?? timelineEntries.request(request);
+        entry.request = request;
+        newest = Math.max(newest, request.seq);
+        if (request.status === "pending") pending.set(request.seq, entry);
+        else pending.delete(request.seq);
+        if (!waiting) timeline.add(entry);
+        emit({ event: "request", data: request });
+        emit({ event: "timeline", data: entry });
+        if (request.job === undefined || request.sequence === undefined) continue;
+        requests.set(`${request.job}:${request.service}:${request.sequence}`, request);
+        for (const key of [...requests.keys()].slice(0, -requestsRemembered)) requests.delete(key);
+        const job = manifest.jobs.find(({ id }) => id === request.job);
+        if (job) updateJob(job);
+      }
+    };
+  };
+
+  // --- Background work -------------------------------------------------------------------------
+
+  const stopConnection = adapter.onConnection((connection) => {
+    record(timelineEntries.connection(connection.since, connection.state, connection.error));
+    emit({ event: "session", data: session() });
+    if (connection.state !== "connected") {
+      for (const [target, sample] of latest)
+        latest.set(target, {
+          ...sample,
+          quality: "unavailable",
+          status: connection.error ?? "The controller is not connected",
+        });
+      for (const job of manifest.jobs) updateJob(job);
+      for (const device of manifest.devices) updateDevice(device.id);
+    }
+  });
+
+  const repeat = async (name: string, intervalMs: number, work: () => Promise<void>) => {
+    let failure: string | undefined;
+    while (!signal.aborted) {
+      try {
+        await work();
+        failure = undefined;
+      } catch (error) {
+        if (signal.aborted) return;
+        if (message(error) !== failure) options.log(`${name} failed: ${message(error)}`);
+        failure = message(error);
+      }
+      await delay(intervalMs, undefined, { signal }).catch(() => undefined);
+    }
+  };
+
+  const { server } = options;
+  const running = server
+    ? [
+        repeat("Reading the server's telemetry", pullMs, () => pullTelemetry(server)),
+        repeat("Reading the server's requests", pullMs, pullRequests(server)),
+      ]
+    : [];
+
+  return {
+    manifest,
+    paths,
+    session,
+    values,
+    jobs: () => manifest.jobs.map(jobOf),
+    hardware: () => manifest.devices.map(({ id }) => deviceOf(id)),
+    timeline: (since?: string) => timeline.since(since),
+    watchGroups: () => watchGroups.groups(),
+    saveWatchGroups: (groups: WatchGroups) => watchGroups.save(groups),
+    /** Adds an entry from another source, such as the agent's audit log. */
+    record,
+    watch,
+    show,
+    /** Calls `listener` with every other stream event until the returned function is called. */
+    subscribe: (listener: (message: StreamMessage) => void) => {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    close: async () => {
+      controller.abort();
+      stopBuild();
+      readJobs([]);
+      readDevices([]);
+      stopConnection();
+      await Promise.all(running);
+      await adapter.close();
+    },
+  };
+};
+
+export type Debugger = Awaited<ReturnType<typeof createDebugger>>;

package/src/hardware.ts

@@ -0,0 +1,125 @@
+import { type TargetRead, type TargetSample, worstQuality } from "./adapter.ts";
+import type { DeviceState } from "./api.ts";
+import type { DebugDevice, DebugIo, DebugType } from "./manifest.ts";
+
+/** Driver state names that mean the connection isn't working. */
+const faultyState = /error|fault|fail|timeout|lost|closed/i;
+
+/** Hand-configured status symbols have no type in the manifest; they show as read. */
+const untyped: DebugType = { kind: "scalar", iec: "ANY" };
+
+/** What a device's health is read from: the driver's own state, when the build names it. */
+export const healthReads = (device: DebugDevice): TargetRead[] => {
+  const { state, states, diagnostic, message } = device.health ?? {};
+  return [
+    ...(state
+      ? [
+          {
+            target: state,
+            type: states
+              ? { kind: "enum" as const, name: "state", members: states }
+              : { kind: "scalar" as const, iec: "DINT" },
+          },
+        ]
+      : []),
+    ...(diagnostic ? [{ target: diagnostic, type: { kind: "scalar" as const, iec: "BOOL" } }] : []),
+    ...(message ? [{ target: message, type: { kind: "scalar" as const, iec: "STRING" } }] : []),
+  ];
+};
+
+/** Everything a device's page shows: its health, raw assembly bytes, bound channels and status symbols. */
+export const deviceReads = (
+  device: DebugDevice,
+  io: readonly DebugIo[],
+  typeOf: (id: string) => DebugType,
+): TargetRead[] => [
+  ...healthReads(device),
+  ...device.assemblies.flatMap(({ raw, bytes }) =>
+    raw
+      ? [
+          {
+            target: raw,
+            type: { kind: "array" as const, element: "BYTE", length: bytes, lower: 0 },
+          },
+        ]
+      : [],
+  ),
+  ...io
+    .filter((binding) => binding.device === device.id)
+    .map(({ target, type }) => ({ target, type: typeOf(type) })),
+  ...Object.values(device.status ?? {}).map((target) => ({ target, type: untyped })),
+];
+
+const health = (
+  device: DebugDevice,
+  value: (target: string) => TargetSample | undefined,
+): DeviceState["health"] => {
+  const { state, states, diagnostic, message } = device.health ?? {};
+  if (!state && !diagnostic) return { state: "unknown", quality: "unavailable" };
+  const stateSample = state ? value(state) : undefined;
+  const diagnosticSample = diagnostic ? value(diagnostic) : undefined;
+  const messageSample = message ? value(message) : undefined;
+  const quality = worstQuality([
+    ...(state ? [stateSample] : []),
+    ...(diagnostic ? [diagnosticSample] : []),
+  ]);
+  const number = typeof stateSample?.value === "number" ? stateSample.value : undefined;
+  // The login names enum members the manifest may not list.
+  const name =
+    Object.entries(states ?? {}).find(([, member]) => member === number)?.[0] ??
+    (typeof stateSample?.value === "string" ? stateSample.value : undefined);
+  const faulted =
+    diagnosticSample?.value === true || (name !== undefined && faultyState.test(name));
+  return {
+    state: quality !== "good" ? "stale" : faulted ? "fault" : "ok",
+    ...(number !== undefined && { value: number }),
+    ...(name && { name }),
+    ...(typeof diagnosticSample?.value === "boolean" && { diagnostic: diagnosticSample.value }),
+    ...(typeof messageSample?.value === "string" && { message: messageSample.value }),
+    quality,
+  };
+};
+
+/**
+ * A device's driver health, raw assembly bytes, bound channel values and status symbols. Health
+ * is "unknown" unless the manifest names the driver's own state.
+ */
+export const deviceState = (
+  device: DebugDevice,
+  io: readonly DebugIo[],
+  value: (target: string) => TargetSample | undefined,
+): DeviceState => ({
+  id: device.id,
+  health: health(device, value),
+  assemblies: device.assemblies.map(({ direction, address, raw }) => {
+    const sample = raw ? value(raw) : undefined;
+    return {
+      direction,
+      ...(address && { address }),
+      bytes:
+        Array.isArray(sample?.value) &&
+        sample.value.every((byte): byte is number => typeof byte === "number")
+          ? sample.value
+          : null,
+      quality: sample?.quality ?? "unavailable",
+    };
+  }),
+  channels: io.flatMap((binding) =>
+    binding.device === device.id && binding.channel
+      ? [
+          {
+            id: binding.channel,
+            path: binding.path,
+            value: value(binding.target)?.value ?? null,
+            quality: value(binding.target)?.quality ?? "unavailable",
+          },
+        ]
+      : [],
+  ),
+  status: Object.fromEntries(
+    Object.entries(device.status ?? {}).map(([label, target]) => [
+      label,
+      { value: value(target)?.value ?? null, quality: value(target)?.quality ?? "unavailable" },
+    ]),
+  ),
+});