@agoric/telemetry 0.6.3-u19.2 → 0.6.3-u20.0

package/CHANGELOG.md

@@ -3,45 +3,35 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
-### [0.6.3-u19.2](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.3-u19.1...@agoric/telemetry@0.6.3-u19.2) (2025-03-13)
-
-**Note:** Version bump only for package @agoric/telemetry
-
-
-
-
-
-### [0.6.3-u19.1](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.3-u19.0...@agoric/telemetry@0.6.3-u19.1) (2025-03-03)
-
-
-### Bug Fixes
-
-* Properly synchronize slog sender termination ([2fc342c](https://github.com/Agoric/agoric-sdk/commit/2fc342c180e296208d077a1d4799da139d3b7848))
-
-
-
-### [0.6.3-u19.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.2...@agoric/telemetry@0.6.3-u19.0) (2025-02-24)
+### [0.6.3-u20.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.2...@agoric/telemetry@0.6.3-u20.0) (2025-04-16)
 
 
 ### Features
 
+* Add the Prometheus slog sender module and load it per OTEL_EXPORTER_PROMETHEUS_PORT ([1dc1827](https://github.com/Agoric/agoric-sdk/commit/1dc182783ce191f0ba2131cb1f7b3042f287737a)), closes [#11045](https://github.com/Agoric/agoric-sdk/issues/11045)
 * **cosmic-swingset:** add JS upgrade plan handler stub ([655133e](https://github.com/Agoric/agoric-sdk/commit/655133ed909b5d632dc033e992214a7b6a1b5ab1))
+* **internal:** Add helper `unprefixedProperties` for environment variable consumption ([878fecf](https://github.com/Agoric/agoric-sdk/commit/878fecf4f5153fa80f48a27a8b79e67943b2d199))
 * simple CircularBuffer with fs offsets ([8d9cb7a](https://github.com/Agoric/agoric-sdk/commit/8d9cb7abe96e8905f5aaa0927e02914ef09279c4))
 * **telemetry:** context aware slog support new triggers ([03965d9](https://github.com/Agoric/agoric-sdk/commit/03965d90b86cf75ce7f6677861e3a0aa8ac70710))
 * **telemetry:** ingest-slog explicitly supports `-` for stdin ([63367c4](https://github.com/Agoric/agoric-sdk/commit/63367c4aaf9bafbd6553a1f4cb808c96bc90845a))
 * **telemetry:** ingest-slog throttle and flush per block ([2134944](https://github.com/Agoric/agoric-sdk/commit/21349448b3b9379a9da43218a59a7e7eaf4f5a9e))
+* **telemetry:** Update slog sender JSON serialization of error instances ([5db996d](https://github.com/Agoric/agoric-sdk/commit/5db996d99830e61fad6eed373e2fb2dc810d662e))
 * use writeSync slogSender ([47a2add](https://github.com/Agoric/agoric-sdk/commit/47a2adda72a5377eda181a425130cdc5a7fd7ff5))
 
 
 ### Bug Fixes
 
 * ensure script main rejections exit with error ([abdab87](https://github.com/Agoric/agoric-sdk/commit/abdab879014a5c3124ebd0e9246995ac6b1ce6e5))
+* Properly synchronize slog sender termination ([f83c01d](https://github.com/Agoric/agoric-sdk/commit/f83c01d89d80798e0922acdb498fcc7250560977))
 * **telemetry:** add missing slog type ([1aec8d0](https://github.com/Agoric/agoric-sdk/commit/1aec8d05036f6b3c3e3730339d1829da6b4a9051))
 * **telemetry:** avoid polluting stdout in ingest-slog ([d4b8dfa](https://github.com/Agoric/agoric-sdk/commit/d4b8dfa91155789f7ceda5cc3cef06019b9527e7))
 * **telemetry:** Empty context persisted when remaining beans are negative after run finish ([#10635](https://github.com/Agoric/agoric-sdk/issues/10635)) ([ad4e83e](https://github.com/Agoric/agoric-sdk/commit/ad4e83e0b6dff9716da91fd65d367d3acad1772e))
 * **telemetry:** event name typo ([9e19321](https://github.com/Agoric/agoric-sdk/commit/9e19321ea8fed32d445d44169b32f5d94a93d61e))
+* **telemetry:** flight-recorder ignores write after shutdown ([3d2bcb3](https://github.com/Agoric/agoric-sdk/commit/3d2bcb3c56ac24a0f991200b223e6af8514dc5b8))
 * **telemetry:** handle new trigger slog events ([d32cb7e](https://github.com/Agoric/agoric-sdk/commit/d32cb7e9f406c25399321dc32e827b5018c38b69))
 * **telemetry:** ingest-slog avoid writing progress file for stdin ([62589ca](https://github.com/Agoric/agoric-sdk/commit/62589ca7b6d4aaa9eb7042f95ec7aec633db27f9))
+* **telemetry:** initialize empty flight-recorders ([0908258](https://github.com/Agoric/agoric-sdk/commit/0908258c159a18f2bace0f76fa25c485c0460d15))
+* **telemetry:** Launch a slog sender subprocess with the correct environment ([1a60955](https://github.com/Agoric/agoric-sdk/commit/1a60955181f4e8b02b3b0d5a2f213d4cb051d7d3))
 * **telemetry:** otel correctly pop upgrade span ([0ffdf00](https://github.com/Agoric/agoric-sdk/commit/0ffdf001bc8cbdc94081fedfeb4d2376902f4ffc)), closes [#8272](https://github.com/Agoric/agoric-sdk/issues/8272) [#9569](https://github.com/Agoric/agoric-sdk/issues/9569)
 * **telemetry:** silence slogfile write errors ([91089d7](https://github.com/Agoric/agoric-sdk/commit/91089d7273ef3d41555b34d84471120d45602497))
 * **telemetry:** timer-poll run.id ([#10672](https://github.com/Agoric/agoric-sdk/issues/10672)) ([3b478fb](https://github.com/Agoric/agoric-sdk/commit/3b478fb9e3fe7ded8dec1e83bab68760571f9071)), closes [#10357](https://github.com/Agoric/agoric-sdk/issues/10357) [#10357](https://github.com/Agoric/agoric-sdk/issues/10357)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/telemetry",
-  "version": "0.6.3-u19.2",
+  "version": "0.6.3-u20.0",
   "description": "Agoric's telemetry implementation",
   "type": "module",
   "repository": "https://github.com/Agoric/agoric-sdk",
@@ -22,12 +22,12 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "dependencies": {
-    "@agoric/internal": "^0.4.0-u19.2",
-    "@agoric/store": "^0.9.3-u19.0",
-    "@endo/errors": "^1.2.9",
-    "@endo/init": "^1.1.8",
-    "@endo/marshal": "^1.6.3",
-    "@endo/stream": "^1.2.9",
+    "@agoric/internal": "^0.4.0-u20.0",
+    "@agoric/store": "^0.9.3-u20.0",
+    "@endo/errors": "^1.2.10",
+    "@endo/init": "^1.1.9",
+    "@endo/marshal": "^1.6.4",
+    "@endo/stream": "^1.2.10",
     "@opentelemetry/api": "~1.9.0",
     "@opentelemetry/api-logs": "0.57.1",
     "@opentelemetry/exporter-logs-otlp-http": "0.57.1",
@@ -43,8 +43,8 @@
     "tmp": "^0.2.1"
   },
   "devDependencies": {
-    "@endo/lockdown": "^1.0.14",
-    "@endo/ses-ava": "^1.2.9",
+    "@endo/lockdown": "^1.0.15",
+    "@endo/ses-ava": "^1.2.10",
     "ava": "^5.3.0",
     "c8": "^10.1.2",
     "tmp": "^0.2.1"
@@ -66,7 +66,7 @@
     "workerThreads": false
   },
   "typeCoverage": {
-    "atLeast": 88.88
+    "atLeast": 88.87
   },
-  "gitHead": "f0ae74b84cb6de3724bfdcd18b4bea7e8199dee1"
+  "gitHead": "8e4207fa19dabf76c1f91f8779b5b5b93570ecea"
 }

package/src/flight-recorder.js

@@ -39,6 +39,9 @@ const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
     }
     throw e;
   });
+
+  // Use the default size if not provided and file doesn't exist.
+  circularBufferSize = circularBufferSize || stbuf?.size || DEFAULT_CBUF_SIZE;
   const arenaSize = BigInt(circularBufferSize - I_ARENA_START);
 
   if (stbuf && stbuf.size >= I_ARENA_START) {
@@ -280,20 +283,22 @@ export const makeSimpleCircularBuffer = async ({
  * @param {Pick<CircularBuffer, 'fileHandle' | 'writeCircBuf'>} circBuf
  */
 export const makeSlogSenderFromBuffer = ({ fileHandle, writeCircBuf }) => {
-  /** @type {Promise<void>} */
+  /** @type {Promise<void> | undefined} */
   let toWrite = Promise.resolve();
   const writeJSON = (obj, serialized = serializeSlogObj(obj)) => {
     // Prepend a newline so that the file can be more easily manipulated.
     const data = new TextEncoder().encode(`\n${serialized}`);
     // console.log('have obj', obj, data);
-    toWrite = toWrite.then(() => writeCircBuf(data));
+    toWrite = toWrite?.then(() => writeCircBuf(data));
   };
   return Object.assign(writeJSON, {
     forceFlush: async () => {
       await toWrite;
     },
     shutdown: async () => {
-      await toWrite;
+      const lastWritten = toWrite;
+      toWrite = undefined;
+      await lastWritten;
       await fileHandle.close();
     },
     usesJsonObject: true,

package/src/index.js

@@ -84,44 +84,37 @@ export const getResourceAttributes = ({
 
 /**
  * @typedef {object} Powers
- * @property {{ warn: Console['warn'] }} console
+ * @property {Pick<Console, 'warn'>} console
  * @property {NodeJS.ProcessEnv} env
  * @property {import('@opentelemetry/sdk-metrics').View[]} views
  * @property {string} [serviceName]
  */
 
 /**
- * @param {Partial<Powers>} param0
+ * @param {Partial<Powers>} powers
  */
-const getPrometheusMeterProvider = ({
+export const getPrometheusMeterProvider = ({
   console = globalThis.console,
   env = process.env,
   views,
   ...rest
 } = {}) => {
-  const { OTEL_EXPORTER_PROMETHEUS_PORT } = env;
-  if (!OTEL_EXPORTER_PROMETHEUS_PORT) {
-    // No Prometheus config, so don't install.
-    return undefined;
-  }
+  const { OTEL_EXPORTER_PROMETHEUS_HOST, OTEL_EXPORTER_PROMETHEUS_PORT } = env;
+
+  // The opt-in signal is a non-empty OTEL_EXPORTER_PROMETHEUS_PORT.
+  if (!OTEL_EXPORTER_PROMETHEUS_PORT) return;
 
   const resource = new Resource(getResourceAttributes({ env, ...rest }));
 
-  const port =
-    parseInt(OTEL_EXPORTER_PROMETHEUS_PORT || '', 10) ||
-    PrometheusExporter.DEFAULT_OPTIONS.port;
+  const { DEFAULT_OPTIONS } = PrometheusExporter;
+  const host = OTEL_EXPORTER_PROMETHEUS_HOST || DEFAULT_OPTIONS.host;
+  const port = +OTEL_EXPORTER_PROMETHEUS_PORT || DEFAULT_OPTIONS.port;
+  const url = `http://${host || '0.0.0.0'}:${port}${DEFAULT_OPTIONS.endpoint}`;
 
-  const exporter = new PrometheusExporter(
-    {
-      port,
-      appendTimestamp: true,
-    },
-    () => {
-      console.warn(
-        `Prometheus scrape endpoint: http://0.0.0.0:${port}${PrometheusExporter.DEFAULT_OPTIONS.endpoint}`,
-      );
-    },
-  );
+  const options = { host, port, appendTimestamp: true };
+  const exporter = new PrometheusExporter(options, () => {
+    console.warn(`Prometheus scrape endpoint: ${url}`);
+  });
 
   const provider = new MeterProvider({ resource, views });
   provider.addMetricReader(exporter);

package/src/make-slog-sender.js

@@ -1,13 +1,13 @@
 import path from 'path';
 import tmp from 'tmp';
-import { PromiseAllOrErrors } from '@agoric/internal';
+import { PromiseAllOrErrors, unprefixedProperties } from '@agoric/internal';
 import { serializeSlogObj } from './serialize-slog-obj.js';
 
+export const DEFAULT_SLOGSENDER_AGENT = 'self';
 export const DEFAULT_SLOGSENDER_MODULE =
   '@agoric/telemetry/src/flight-recorder.js';
 export const SLOGFILE_SENDER_MODULE = '@agoric/telemetry/src/slog-file.js';
-
-export const DEFAULT_SLOGSENDER_AGENT = 'self';
+export const PROMETHEUS_SENDER_MODULE = '@agoric/telemetry/src/prometheus.js';
 
 /** @import {SlogSender} from './index.js' */
 
@@ -19,6 +19,23 @@ export const DEFAULT_SLOGSENDER_AGENT = 'self';
 const filterTruthy = arr => /** @type {any[]} */ (arr.filter(Boolean));
 
 /**
+ * Create an aggregate slog sender that fans out inbound slog entries to modules
+ * as indicated by variables in the supplied `env` option. The SLOGSENDER value
+ * (or a default DEFAULT_SLOGSENDER_MODULE defined above) is split on commas
+ * into a list of module identifiers and adjusted by automatic insertions (a
+ * non-empty SLOGFILE value inserts DEFAULT_SLOGSENDER_AGENT defined above), and
+ * then each identifier is dynamically `import`ed for its own `makeSlogSender`
+ * export, which is invoked with a non-empty `stateDir` option and a modified
+ * `env` in which SLOGSENDER_AGENT_* variables have overridden their unprefixed
+ * equivalents to produce a subordinate slog sender.
+ * Subordinate slog senders remain isolated from each other, and any errors from
+ * them are caught and held until the next `forceFlush()` without disrupting
+ * any remaining slog entry fanout.
+ * If SLOGSENDER_AGENT is 'process', 'slog-sender-pipe.js' is used to load the
+ * subordinates in a child process rather than the main process.
+ * When there are no subordinates, the return value will be `undefined` rather
+ * than a slog sender function.
+ *
  * @type {import('./index.js').MakeSlogSender}
  */
 export const makeSlogSender = async (opts = {}) => {
@@ -26,95 +43,95 @@ export const makeSlogSender = async (opts = {}) => {
   const {
     SLOGSENDER = DEFAULT_SLOGSENDER_MODULE,
     SLOGSENDER_AGENT = DEFAULT_SLOGSENDER_AGENT,
+    // While cosmic-swingset/kernel code includes its own Prometheus metrics
+    // export, that trumps a slog sender module doing so.
+    // This extraction can be removed when that changes, but in the meantime,
+    // opt-in is only by SLOGSENDER_AGENT_OTEL_EXPORTER_PROMETHEUS_PORT.
+    OTEL_EXPORTER_PROMETHEUS_PORT: _prometheusExportPort,
     ...otherEnv
   } = env;
 
   const agentEnv = {
     ...otherEnv,
-    ...Object.fromEntries(
-      Object.entries(otherEnv)
-        .filter(([k]) => k.match(/^(?:SLOGSENDER_AGENT_)+/)) // narrow to SLOGSENDER_AGENT_ prefixes.
-        .map(([k, v]) => [k.replace(/^(?:SLOGSENDER_AGENT_)+/, ''), v]), // Rewrite SLOGSENDER_AGENT_ to un-prefixed version.
-    ),
+    ...unprefixedProperties(otherEnv, 'SLOGSENDER_AGENT_'),
   };
 
-  const slogSenderModules = [
-    ...new Set([
-      ...(agentEnv.SLOGFILE ? [SLOGFILE_SENDER_MODULE] : []),
-      ...SLOGSENDER.split(',')
-        .filter(Boolean)
-        .map(modulePath =>
-          modulePath.startsWith('.')
-            ? // Resolve relative to the current working directory.
-              path.resolve(modulePath)
-            : modulePath,
-        ),
-    ]),
-  ];
-
-  if (!slogSenderModules.length) {
+  const slogSenderModules = new Set();
+  if (agentEnv.OTEL_EXPORTER_PROMETHEUS_PORT) {
+    slogSenderModules.add(PROMETHEUS_SENDER_MODULE);
+  }
+  if (agentEnv.SLOGFILE) {
+    slogSenderModules.add(SLOGFILE_SENDER_MODULE);
+  }
+  for (const moduleIdentifier of filterTruthy(SLOGSENDER.split(','))) {
+    if (moduleIdentifier.startsWith('-')) {
+      // Opt out of an automatically-included sender.
+      slogSenderModules.delete(moduleIdentifier.slice(1));
+    } else if (moduleIdentifier.startsWith('.')) {
+      // Resolve relative to the current working directory.
+      slogSenderModules.add(path.resolve(moduleIdentifier));
+    } else {
+      slogSenderModules.add(moduleIdentifier);
+    }
+  }
+
+  if (!slogSenderModules.size) {
     return undefined;
   }
 
-  switch (SLOGSENDER_AGENT) {
-    case '':
-    case 'self':
-      break;
-    case 'process': {
-      console.warn('Loading slog sender in subprocess');
-      return import('./slog-sender-pipe.js').then(
-        async ({ makeSlogSender: makeSogSenderPipe }) =>
-          makeSogSenderPipe({
-            env: {
-              ...agentEnv,
-              SLOGSENDER,
-              SLOGSENDER_AGENT: 'self',
-            },
-            stateDir: stateDirOption,
-            ...otherOpts,
-          }),
-      );
-    }
-    case 'worker':
-    default:
-      console.warn(`Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}`);
+  if (SLOGSENDER_AGENT === 'process') {
+    console.warn('Loading slog sender in subprocess');
+    return import('./slog-sender-pipe.js').then(async module =>
+      module.makeSlogSender({
+        env: {
+          ...agentEnv,
+          SLOGSENDER,
+          SLOGSENDER_AGENT: 'self',
+        },
+        stateDir: stateDirOption,
+        ...otherOpts,
+      }),
+    );
+  } else if (SLOGSENDER_AGENT && SLOGSENDER_AGENT !== 'self') {
+    console.warn(
+      `Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}; defaulting to 'self'`,
+    );
   }
 
   if (SLOGSENDER) {
     console.warn('Loading slog sender modules:', ...slogSenderModules);
   }
 
-  const makersInfo = await Promise.all(
-    slogSenderModules.map(async moduleIdentifier =>
-      import(moduleIdentifier)
-        .then(
-          /** @param {{makeSlogSender: import('./index.js').MakeSlogSender}} module */ ({
-            makeSlogSender: maker,
-          }) => {
-            if (typeof maker !== 'function') {
-              return Promise.reject(
-                Error(`No 'makeSlogSender' function exported by module`),
-              );
-            } else if (maker === makeSlogSender) {
-              return Promise.reject(
-                Error(`Cannot recursively load 'makeSlogSender' aggregator`),
-              );
-            }
-
-            return /** @type {const} */ ([maker, moduleIdentifier]);
-          },
-        )
-        .catch(err => {
+  /** @type {Map<import('./index.js').MakeSlogSender, string>} */
+  const makerMap = new Map();
+  await Promise.all(
+    [...slogSenderModules].map(async moduleIdentifier => {
+      await null;
+      try {
+        const module = await import(moduleIdentifier);
+        const { makeSlogSender: maker } = module;
+        if (typeof maker !== 'function') {
+          throw Error(`No 'makeSlogSender' function exported by module`);
+        } else if (maker === makeSlogSender) {
+          throw Error(`Cannot recursively load 'makeSlogSender' aggregator`);
+        }
+        const isReplacing = makerMap.get(maker);
+        if (isReplacing) {
           console.warn(
-            `Failed to load slog sender from ${moduleIdentifier}.`,
-            err,
+            `The slog sender from ${moduleIdentifier} matches the one from ${isReplacing}.`,
           );
-          return undefined;
-        }),
-    ),
-  ).then(makerEntries => [...new Map(filterTruthy(makerEntries)).entries()]);
+        }
+        makerMap.set(maker, moduleIdentifier);
+      } catch (err) {
+        console.warn(
+          `Failed to load slog sender from ${moduleIdentifier}.`,
+          err,
+        );
+      }
+    }),
+  );
 
-  if (!makersInfo.length) {
+  if (!makerMap.size) {
     return undefined;
   }
 
@@ -122,11 +139,11 @@ export const makeSlogSender = async (opts = {}) => {
 
   if (stateDir === undefined) {
     stateDir = tmp.dirSync().name;
-    console.warn(`Using ${stateDir} for stateDir`);
+    console.warn(`Using ${stateDir} for slog sender stateDir`);
   }
 
   const senders = await Promise.all(
-    makersInfo.map(async ([maker, moduleIdentifier]) =>
+    [...makerMap.entries()].map(async ([maker, moduleIdentifier]) =>
       maker({
         ...otherOpts,
         stateDir,
@@ -137,37 +154,37 @@ export const makeSlogSender = async (opts = {}) => {
 
   if (!senders.length) {
     return undefined;
-  } else {
-    // Optimize creating a JSON serialization only if needed
-    // by any of the sender modules
-    const hasSenderUsingJsonObj = senders.some(
-      ({ usesJsonObject = true }) => usesJsonObject,
-    );
-    const getJsonObj = hasSenderUsingJsonObj
-      ? serializeSlogObj
-      : () => undefined;
-    const sendErrors = [];
-    /** @type {SlogSender} */
-    const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
-      for (const sender of senders) {
-        try {
-          sender(slogObj, jsonObj);
-        } catch (err) {
-          sendErrors.push(err);
-        }
-      }
-    };
-    return Object.assign(slogSender, {
-      forceFlush: async () => {
-        await PromiseAllOrErrors([
-          ...senders.map(sender => sender.forceFlush?.()),
-          ...sendErrors.splice(0).map(err => Promise.reject(err)),
-        ]);
-      },
-      shutdown: async () => {
-        await PromiseAllOrErrors(senders.map(sender => sender.shutdown?.()));
-      },
-      usesJsonObject: hasSenderUsingJsonObj,
-    });
   }
+
+  // Optimize creating a JSON serialization only if needed
+  // by at least one of the senders.
+  const hasSenderUsingJsonObj = senders.some(
+    ({ usesJsonObject = true }) => usesJsonObject,
+  );
+  const getJsonObj = hasSenderUsingJsonObj ? serializeSlogObj : () => undefined;
+
+  const sendErrors = [];
+
+  /** @type {SlogSender} */
+  const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
+    for (const sender of senders) {
+      try {
+        sender(slogObj, jsonObj);
+      } catch (err) {
+        sendErrors.push(err);
+      }
+    }
+  };
+  return Object.assign(slogSender, {
+    forceFlush: async () => {
+      await PromiseAllOrErrors([
+        ...senders.map(sender => sender.forceFlush?.()),
+        ...sendErrors.splice(0).map(err => Promise.reject(err)),
+      ]);
+    },
+    shutdown: async () => {
+      await PromiseAllOrErrors(senders.map(sender => sender.shutdown?.()));
+    },
+    usesJsonObject: hasSenderUsingJsonObj,
+  });
 };

package/src/otel-metrics.js

@@ -0,0 +1,223 @@
+import { q, Fail } from '@endo/errors';
+
+import * as ActionType from '@agoric/internal/src/action-types.js';
+import { objectMapMutable } from '@agoric/internal/src/js-utils.js';
+import {
+  HISTOGRAM_METRICS,
+  BLOCK_HISTOGRAM_METRICS,
+  KERNEL_STATS_METRICS,
+  makeQueueMetrics,
+} from '@agoric/internal/src/metrics.js';
+
+/**
+ * @import {MeterProvider, MetricOptions, ObservableCounter, ObservableUpDownCounter} from '@opentelemetry/api';
+ * @import {TotalMap} from '@agoric/internal';
+ */
+
+const knownActionTypes = new Set(Object.values(ActionType.QueuedActionType));
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName: string, otelMeterProvider?: MeterProvider}} opts */
+export const makeSlogSender = async (opts = /** @type {any} */ ({})) => {
+  const { otelMeterName, otelMeterProvider } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  if (!otelMeterProvider) return;
+
+  const otelMeter = otelMeterProvider.getMeter(otelMeterName);
+
+  const processedInboundActionCounter = otelMeter.createCounter(
+    'cosmic_swingset_inbound_actions',
+    { description: 'Processed inbound action counts by type' },
+  );
+  const histograms = {
+    ...objectMapMutable(HISTOGRAM_METRICS, (desc, name) => {
+      const { boundaries, ...options } = desc;
+      const advice = boundaries && { explicitBucketBoundaries: boundaries };
+      return otelMeter.createHistogram(name, { ...options, advice });
+    }),
+    ...objectMapMutable(BLOCK_HISTOGRAM_METRICS, (desc, name) =>
+      otelMeter.createHistogram(name, desc),
+    ),
+  };
+
+  const inboundQueueMetrics = makeQueueMetrics({
+    otelMeter,
+    namePrefix: 'cosmic_swingset_inbound_queue',
+    descPrefix: 'inbound queue',
+    console,
+  });
+
+  // Values for KERNEL_STATS_METRICS could be built up locally by observing slog
+  // entries, but they are all collectively reported in "kernel-stats"
+  // (@see {@link ../../cosmic-swingset/src/kernel-stats.js exportKernelStats})
+  // and for now we just reflect that, which requires implementation as async
+  // ("observable") instruments rather than synchronous ones.
+  /** @typedef {string} KernelStatsKey */
+  /** @typedef {string} KernelMetricName */
+  /** @type {TotalMap<KernelStatsKey, number>} */
+  const kernelStats = new Map();
+  /** @type {Map<KernelMetricName, ObservableCounter | ObservableUpDownCounter>} */
+  const kernelStatsCounters = new Map();
+  for (const meta of KERNEL_STATS_METRICS) {
+    const { key, name, sub, metricType, ...options } = meta;
+    kernelStats.set(key, 0);
+    if (metricType === 'gauge') {
+      kernelStats.set(`${key}Up`, 0);
+      kernelStats.set(`${key}Down`, 0);
+      kernelStats.set(`${key}Max`, 0);
+    } else if (metricType !== 'counter') {
+      Fail`Unknown metric type ${q(metricType)} for key ${q(key)} name ${q(name)}`;
+    }
+    let counter = kernelStatsCounters.get(name);
+    if (!counter) {
+      counter =
+        metricType === 'counter'
+          ? otelMeter.createObservableCounter(name, options)
+          : otelMeter.createObservableUpDownCounter(name, options);
+      kernelStatsCounters.set(name, counter);
+    }
+    const attributes = sub ? { [sub.dimension]: sub.value } : {};
+    counter.addCallback(observer => {
+      observer.observe(kernelStats.get(key), attributes);
+    });
+  }
+  const expectedKernelStats = new Set(kernelStats.keys());
+
+  /**
+   * @typedef {object} LazyStats
+   * @property {string} namePrefix
+   * @property {MetricOptions} options
+   * @property {Set<string>} keys
+   * @property {Record<string, number>} data
+   */
+  /** @type {(namePrefix: string, description: string) => LazyStats} */
+  const makeLazyStats = (namePrefix, description) => {
+    return { namePrefix, options: { description }, keys: new Set(), data: {} };
+  };
+  const dynamicAfterCommitStatsCounters = {
+    memoryUsage: makeLazyStats(
+      'memoryUsage_',
+      'kernel process memory statistic',
+    ),
+    heapStats: makeLazyStats('heapStats_', 'v8 kernel heap statistic'),
+  };
+
+  const slogSender = ({ type: slogType, ...slogObj }) => {
+    switch (slogType) {
+      // Consume cosmic-swingset block lifecycle slog entries.
+      case 'cosmic-swingset-init': {
+        const { inboundQueueInitialLengths: lengths } = slogObj;
+        inboundQueueMetrics.initLengths(lengths);
+        break;
+      }
+      case 'cosmic-swingset-begin-block': {
+        const {
+          interBlockSeconds,
+          afterCommitHangoverSeconds,
+          blockLagSeconds,
+        } = slogObj;
+
+        Number.isFinite(interBlockSeconds) &&
+          histograms.interBlockSeconds.record(interBlockSeconds);
+        histograms.afterCommitHangoverSeconds.record(
+          afterCommitHangoverSeconds,
+        );
+        Number.isFinite(blockLagSeconds) &&
+          histograms.blockLagSeconds.record(blockLagSeconds);
+        break;
+      }
+      case 'cosmic-swingset-run-finish': {
+        histograms.swingset_block_processing_seconds.record(slogObj.seconds);
+        break;
+      }
+      case 'cosmic-swingset-end-block-finish': {
+        const { inboundQueueStartLengths, processedActionCounts } = slogObj;
+        inboundQueueMetrics.updateLengths(inboundQueueStartLengths);
+        for (const processedActionRecord of processedActionCounts) {
+          const { count, phase, type: actionType } = processedActionRecord;
+          if (!knownActionTypes.has(actionType)) {
+            console.warn('Unknown inbound action type', actionType);
+          }
+          processedInboundActionCounter.add(count, { actionType });
+          inboundQueueMetrics.decLength(phase);
+        }
+        break;
+      }
+      case 'cosmic-swingset-commit-block-finish': {
+        const {
+          runSeconds,
+          chainTime,
+          saveTime,
+          cosmosCommitSeconds,
+          fullSaveTime,
+        } = slogObj;
+        histograms.swingsetRunSeconds.record(runSeconds);
+        histograms.swingsetChainSaveSeconds.record(chainTime);
+        histograms.swingsetCommitSeconds.record(saveTime);
+        histograms.cosmosCommitSeconds.record(cosmosCommitSeconds);
+        histograms.fullCommitSeconds.record(fullSaveTime);
+        break;
+      }
+
+      // Consume Swingset kernel slog entries.
+      case 'vat-startup-finish': {
+        histograms.swingset_vat_startup.record(slogObj.seconds * 1000);
+        break;
+      }
+      case 'crank-finish': {
+        const { crankType, messageType, seconds } = slogObj;
+        // TODO: Reflect crankType/messageType as proper dimensional attributes.
+        // For now, we're going for parity with direct metrics.
+        if (crankType !== 'routing' && messageType !== 'create-vat') {
+          histograms.swingset_crank_processing_time.record(seconds * 1000);
+        }
+        break;
+      }
+
+      // Consume miscellaneous slog entries.
+      case 'kernel-stats': {
+        const { stats } = slogObj;
+        const notYetFoundKernelStats = new Set(expectedKernelStats);
+        for (const [key, value] of Object.entries(stats)) {
+          notYetFoundKernelStats.delete(key);
+          if (!kernelStats.has(key)) {
+            console.warn('Unexpected SwingSet kernel statistic', key);
+          }
+          kernelStats.set(key, value);
+        }
+        if (notYetFoundKernelStats.size) {
+          console.warn('Expected SwingSet kernel statistics not found', [
+            ...notYetFoundKernelStats,
+          ]);
+        }
+        break;
+      }
+      case 'cosmic-swingset-after-commit-stats': {
+        const dynamicCounterEntries = Object.entries(
+          dynamicAfterCommitStatsCounters,
+        );
+        for (const [slogKey, meta] of dynamicCounterEntries) {
+          const { namePrefix, options, keys } = meta;
+          meta.data = slogObj[slogKey] || {};
+          const newKeys = Object.keys(meta.data).filter(key => !keys.has(key));
+          for (const key of newKeys) {
+            keys.add(key);
+            const name = `${namePrefix}${key}`;
+            const gauge = otelMeter.createObservableUpDownCounter(
+              name,
+              options,
+            );
+            gauge.addCallback(observer => {
+              observer.observe(meta.data[key]);
+            });
+          }
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  };
+  return Object.assign(slogSender, {
+    usesJsonObject: false,
+  });
+};

package/src/prometheus.js

@@ -0,0 +1,18 @@
+import { Fail } from '@endo/errors';
+
+import { getPrometheusMeterProvider } from './index.js';
+import { makeSlogSender as makeOtelMetricsSender } from './otel-metrics.js';
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName?: string}} opts */
+export const makeSlogSender = async (opts = {}) => {
+  const { env, otelMeterName, serviceName } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  const otelMeterProvider = getPrometheusMeterProvider({
+    console,
+    env,
+    serviceName,
+  });
+  if (!otelMeterProvider) return;
+
+  return makeOtelMetricsSender({ ...opts, otelMeterName, otelMeterProvider });
+};

package/src/serialize-slog-obj.js

@@ -1,4 +1,32 @@
-export const serializeSlogObj = slogObj =>
-  JSON.stringify(slogObj, (_, arg) =>
-    typeof arg === 'bigint' ? Number(arg) : arg,
-  );
+const { hasOwn } = Object;
+
+const replacer = (_key, value) => {
+  switch (typeof value) {
+    case 'object': {
+      if (value instanceof Error) {
+        // Represent each error as a serialization-friendly
+        // { errorType, message, cause?, errors?, stack? } object
+        // (itself subject to recursive replacement, particularly in `cause` and
+        // `errors`).
+        const obj = { errorType: value.name, message: value.message };
+        if (hasOwn(value, 'cause')) obj.cause = value.cause;
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore TS2339 property "errors" is only on AggregateError
+        if (hasOwn(value, 'errors')) obj.errors = value.errors;
+        const stack = value.stack;
+        if (stack) obj.stack = stack;
+        return obj;
+      }
+      break;
+    }
+    case 'bigint':
+      // Represent each bigint as a JSON-serializable number, accepting the
+      // possible loss of precision.
+      return Number(value);
+    default:
+      break;
+  }
+  return value;
+};
+
+export const serializeSlogObj = slogObj => JSON.stringify(slogObj, replacer);

package/src/slog-sender-pipe-entrypoint.js

@@ -1,32 +1,34 @@
 /* eslint-env node */
+/**
+ * @file Run as a child process of {@link ./slog-sender-pipe.js} to isolate an
+ *   aggregate slog sender (@see {@link ./make-slog-sender.js}). Communicates
+ *   with its parent via Node.js IPC with advanced (structured clone)
+ *   serialization.
+ *   https://nodejs.org/docs/latest/api/child_process.html#advanced-serialization
+ */
+
 import '@endo/init';
 
 import anylogger from 'anylogger';
+import { Fail } from '@endo/errors';
 import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
 
 import { makeSlogSender } from './make-slog-sender.js';
 
 const logger = anylogger('slog-sender-pipe-entrypoint');
 
-/** @type {(msg: import('./slog-sender-pipe.js').SlogSenderPipeWaitReplies) => void} */
+/** @type {(msg: import('./slog-sender-pipe.js').PipeAPIReply) => void} */
 const send = Function.prototype.bind.call(process.send, process);
 
 /**
- * @typedef {object} InitMessage
- * @property {'init'} type
- * @property {import('./index.js').MakeSlogSenderOptions} options
- */
-/**
- * @typedef {object} SendMessage
- * @property {'send'} type
- * @property {object} obj
+ * @typedef {{type: 'init', options: import('./index.js').MakeSlogSenderOptions }} InitMessage
+ * @typedef {{type: 'flush' }} FlushMessage
+ * @typedef {{type: 'send', obj: Record<string, unknown> }} SendMessage
+ *
+ * @typedef {InitMessage | FlushMessage} PipeAPIResponsefulMessage
+ * @typedef {SendMessage} PipeAPIResponselessMessage
+ * @typedef {PipeAPIResponsefulMessage | PipeAPIResponselessMessage} PipeAPIMessage
  */
-/**
- * @typedef {object} FlushMessage
- * @property {'flush'} type
- */
-/** @typedef {InitMessage | FlushMessage} SlogSenderPipeWaitMessages */
-/** @typedef {SlogSenderPipeWaitMessages | SendMessage } SlogSenderPipeMessages */
 
 const main = async () => {
   /** @type {import('./index.js').SlogSender | undefined} */
@@ -44,9 +46,7 @@ const main = async () => {
 
   /** @param {import('./index.js').MakeSlogSenderOptions} opts */
   const init = async ({ env, ...otherOpts } = {}) => {
-    if (slogSender) {
-      assert.fail('Already initialized');
-    }
+    !slogSender || Fail`Already initialized`;
 
     slogSender = await makeSlogSender({
       ...otherOpts,
@@ -57,9 +57,7 @@ const main = async () => {
   };
 
   const flush = async () => {
-    if (!slogSender) {
-      assert.fail('No sender available');
-    }
+    if (!slogSender) throw Fail`No sender available`;
 
     await slogSender.forceFlush?.();
   };
@@ -77,56 +75,55 @@ const main = async () => {
     return AggregateError(sendErrors.splice(0));
   };
 
-  process.on(
-    'message',
-    /** @param {SlogSenderPipeMessages} msg */ msg => {
-      if (!msg || typeof msg !== 'object') {
-        logger.warn('received invalid message', msg);
-        return;
-      }
+  /** @param {PipeAPIMessage} msg */
+  const onMessage = msg => {
+    if (!msg || typeof msg !== 'object') {
+      logger.warn('Received invalid message', msg);
+      return;
+    }
 
-      switch (msg.type) {
-        case 'init': {
-          void init(msg.options).then(
-            hasSender => {
-              send({ type: 'initReply', hasSender });
-            },
-            error => {
-              send({ type: 'initReply', hasSender: false, error });
-            },
-          );
-          break;
-        }
-        case 'flush': {
-          void flush().then(
-            () => {
-              send({ type: 'flushReply', error: generateFlushError() });
-            },
-            error => {
-              send({ type: 'flushReply', error: generateFlushError(error) });
-            },
-          );
-          break;
-        }
-        case 'send': {
-          if (!slogSender) {
-            logger.warn('received send with no sender available');
-          } else {
-            try {
-              slogSender(msg.obj);
-            } catch (e) {
-              sendErrors.push(e);
-            }
+    switch (msg.type) {
+      case 'init': {
+        void init(msg.options).then(
+          hasSender => {
+            send({ type: 'initReply', hasSender });
+          },
+          error => {
+            send({ type: 'initReply', hasSender: false, error });
+          },
+        );
+        break;
+      }
+      case 'flush': {
+        void flush().then(
+          () => {
+            send({ type: 'flushReply', error: generateFlushError() });
+          },
+          error => {
+            send({ type: 'flushReply', error: generateFlushError(error) });
+          },
+        );
+        break;
+      }
+      case 'send': {
+        if (!slogSender) {
+          logger.warn('Received send with no sender available');
+        } else {
+          try {
+            slogSender(harden(msg.obj));
+          } catch (e) {
+            sendErrors.push(e);
           }
-          break;
-        }
-        default: {
-          // @ts-expect-error exhaustive type check
-          logger.warn('received unknown message type', msg.type);
         }
+        break;
+      }
+      default: {
+        // @ts-expect-error exhaustive type check
+        logger.warn('Received unknown message type', msg.type);
       }
-    },
-  );
+    }
+  };
+  process.on('message', onMessage);
 };
 
 process.exitCode = 1;

package/src/slog-sender-pipe.js

@@ -1,7 +1,17 @@
+/**
+ * @file Export a `makeSlogSender` that spawns a
+ *   {@link ./slog-sender-pipe-entrypoint.js} child process to which it forwards
+ *   all slog entries via Node.js IPC with advanced (structured clone)
+ *   serialization.
+ *   https://nodejs.org/docs/latest/api/child_process.html#advanced-serialization
+ */
+
 import { fork } from 'child_process';
 import path from 'path';
+import { promisify } from 'util';
 import anylogger from 'anylogger';
 
+import { q, Fail } from '@endo/errors';
 import { makeQueue } from '@endo/stream';
 
 import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
@@ -10,6 +20,8 @@ const dirname = path.dirname(new URL(import.meta.url).pathname);
 
 const logger = anylogger('slog-sender-pipe');
 
+const sink = () => {};
+
 /**
  * @template {any[]} T
  * @template R
@@ -23,168 +35,120 @@ const withMutex = operation => {
   return async (...args) => {
     await mutex.get();
     const result = operation(...args);
-    mutex.put(
-      result.then(
-        () => {},
-        () => {},
-      ),
-    );
+    mutex.put(result.then(sink, sink));
     return result;
   };
 };
 
 /**
- * @typedef {object} SlogSenderInitReply
- * @property {'initReply'} type
- * @property {boolean} hasSender
- * @property {Error} [error]
+ * @template [P=unknown]
+ * @typedef {{ type: string, error?: Error } & P} PipeReply
  */
+
 /**
- * @typedef {object} SlogSenderFlushReply
- * @property {'flushReply'} type
- * @property {Error} [error]
+ * @typedef {{
+ *   init: {
+ *     message: import('./slog-sender-pipe-entrypoint.js').InitMessage;
+ *     reply: PipeReply<{ hasSender: boolean }>;
+ *   };
+ *   flush: {
+ *     message: import('./slog-sender-pipe-entrypoint.js').FlushMessage;
+ *     reply: PipeReply<{}>;
+ *   };
+ * }} SlogSenderPipeAPI
+ *
+ * @typedef {keyof SlogSenderPipeAPI} PipeAPICommand
+ * @typedef {SlogSenderPipeAPI[PipeAPICommand]["reply"]} PipeAPIReply
  */
-/** @typedef {SlogSenderInitReply | SlogSenderFlushReply} SlogSenderPipeWaitReplies */
 
-/** @param {import('.').MakeSlogSenderOptions} opts */
-export const makeSlogSender = async opts => {
+/** @param {import('.').MakeSlogSenderOptions} options */
+export const makeSlogSender = async options => {
+  const { env = {} } = options;
   const { registerShutdown } = makeShutdown();
+
   const cp = fork(path.join(dirname, 'slog-sender-pipe-entrypoint.js'), [], {
-    stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+    stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
     serialization: 'advanced',
+    env,
   });
   // logger.log('done fork');
+  /** @type {(msg: Record<string, unknown> & {type: string}) => Promise<void>} */
+  const rawSend = promisify(cp.send.bind(cp));
+  const pipeSend = withMutex(rawSend);
 
-  const pipeSend = withMutex(
-    /**
-     * @template {{type: string}} T
-     * @param {T} msg
-     */
-    msg =>
-      /** @type {Promise<void>} */ (
-        new Promise((resolve, reject) => {
-          cp.send(msg, err => {
-            if (err) {
-              reject(err);
-            } else {
-              resolve();
-            }
-          });
-        })
-      ),
-  );
-
-  /**
-   * @typedef {{
-   *   init: {
-   *     message: import('./slog-sender-pipe-entrypoint.js').InitMessage;
-   *     reply: SlogSenderInitReply;
-   *   };
-   *   flush: {
-   *     message: import('./slog-sender-pipe-entrypoint.js').FlushMessage;
-   *     reply: SlogSenderFlushReply;
-   *   };
-   * }} SlogSenderWaitMessagesAndReplies
-   */
-
-  /** @typedef {keyof SlogSenderWaitMessagesAndReplies} SendWaitCommands */
-  /**
-   * @template {SlogSenderPipeWaitReplies} T
-   * @typedef {Omit<T, 'type' | 'error'>} ReplyPayload
-   */
-
-  /** @type {import('@endo/stream').AsyncQueue<SlogSenderPipeWaitReplies>} */
+  /** @type {import('@endo/stream').AsyncQueue<PipeAPIReply>} */
   const sendWaitQueue = makeQueue();
-  /** @type {SendWaitCommands | undefined} */
+  /** @type {PipeAPICommand | undefined} */
   let sendWaitType;
 
   const sendWaitReply = withMutex(
     /**
-     * @template {SendWaitCommands} T
+     * @template {PipeAPICommand} T
      * @param {T} type
-     * @param {Omit<SlogSenderWaitMessagesAndReplies[T]["message"], 'type'>} payload
-     * @returns {Promise<ReplyPayload<SlogSenderWaitMessagesAndReplies[T]["reply"]>>}
+     * @param {Omit<SlogSenderPipeAPI[T]["message"], 'type'>} payload
+     * @returns {Promise<Omit<SlogSenderPipeAPI[T]["reply"], keyof PipeReply>>}
      */
     async (type, payload) => {
-      !sendWaitType || assert.fail('Invalid mutex state');
+      !sendWaitType || Fail`Invalid mutex state`;
 
       const msg = { ...payload, type };
 
       sendWaitType = type;
-      return pipeSend(msg)
-        .then(async () => sendWaitQueue.get())
-        .then(
-          /** @param {SlogSenderWaitMessagesAndReplies[T]["reply"]} reply */ ({
-            type: replyType,
-            error,
-            ...rest
-          }) => {
-            replyType === `${type}Reply` ||
-              assert.fail(`Unexpected reply ${replyType}`);
-            if (error) {
-              throw error;
-            }
-            return rest;
-          },
-        )
-        .finally(() => {
-          sendWaitType = undefined;
-        });
-    },
-  );
-
-  cp.on(
-    'message',
-    /** @param { SlogSenderPipeWaitReplies } msg */
-    msg => {
-      // logger.log('received', msg);
-      if (
-        !msg ||
-        typeof msg !== 'object' ||
-        msg.type !== `${sendWaitType}Reply`
-      ) {
-        logger.warn('Received unexpected message', msg);
-        return;
+      await null;
+      try {
+        await pipeSend(msg);
+        /** @type {SlogSenderPipeAPI[T]["reply"]} */
+        const reply = await sendWaitQueue.get();
+        const { type: replyType, error, ...rest } = reply;
+        replyType === `${type}Reply` ||
+          Fail`Unexpected reply type ${q(replyType)}`;
+        if (error) throw error;
+        return rest;
+      } finally {
+        sendWaitType = undefined;
       }
-
-      sendWaitQueue.put(msg);
     },
   );
 
-  const flush = async () => sendWaitReply('flush', {});
-  /** @param {import('./index.js').MakeSlogSenderOptions} options */
-  const init = async options => sendWaitReply('init', { options });
+  /** @param {PipeReply} msg */
+  const onMessage = msg => {
+    // logger.log('received', msg);
+    if (!msg || msg.type !== `${sendWaitType}Reply`) {
+      logger.warn('Received unexpected message', msg);
+      return;
+    }
+
+    sendWaitQueue.put(msg);
+  };
+  cp.on('message', onMessage);
 
-  const send = obj => {
-    void pipeSend({ type: 'send', obj }).catch(() => {});
+  const flush = async () => {
+    await sendWaitReply('flush', {});
   };
 
   const shutdown = async () => {
     // logger.log('shutdown');
-    if (!cp.connected) {
-      return;
-    }
+    if (!cp.connected) return;
 
     await flush();
     cp.disconnect();
   };
   registerShutdown(shutdown);
 
-  const { hasSender } = await init(opts).catch(err => {
+  const { hasSender } = await sendWaitReply('init', { options }).catch(err => {
     cp.disconnect();
     throw err;
   });
-
   if (!hasSender) {
     cp.disconnect();
     return undefined;
   }
 
-  const slogSender = send;
+  const slogSender = obj => {
+    void pipeSend({ type: 'send', obj }).catch(sink);
+  };
   return Object.assign(slogSender, {
-    forceFlush: async () => {
-      await flush();
-    },
+    forceFlush: flush,
     shutdown,
     usesJsonObject: false,
   });

package/test/flight-recorder.test.js

@@ -31,8 +31,25 @@ const bufferTests = test.macro(
       circularBufferSize: BUFFER_SIZE,
       circularBufferFilename: tmpFile,
     });
-    const slogSender = makeSlogSenderFromBuffer({ fileHandle, writeCircBuf });
-    t.teardown(slogSender.shutdown);
+    const realSlogSender = makeSlogSenderFromBuffer({
+      fileHandle,
+      writeCircBuf,
+    });
+    let wasShutdown = false;
+    const shutdown = () => {
+      if (wasShutdown) return;
+      wasShutdown = true;
+
+      return realSlogSender.shutdown();
+    };
+    t.teardown(shutdown);
+    // To verify lack of attempted mutation by the consumer, send only hardened
+    // entries.
+    /** @type {typeof realSlogSender} */
+    const slogSender = Object.assign(
+      (obj, serialized) => realSlogSender(harden(obj), serialized),
+      realSlogSender,
+    );
     slogSender({ type: 'start' });
     await slogSender.forceFlush();
     t.is(fs.readFileSync(tmpFile, { encoding: 'utf8' }).length, BUFFER_SIZE);
@@ -83,6 +100,18 @@ const bufferTests = test.macro(
     slogSender(null, 'PRE-SERIALIZED');
     await slogSender.forceFlush();
     t.truthy(fs.readFileSync(tmpFile).includes('PRE-SERIALIZED'));
+
+    slogSender(null, 'PRE_SHUTDOWN');
+    const shutdownP = shutdown();
+    slogSender(null, 'POST_SHUTDOWN');
+    await shutdownP;
+    slogSender(null, 'SHUTDOWN_COMPLETED');
+
+    const finalContent = fs.readFileSync(tmpFile);
+
+    t.truthy(finalContent.includes('PRE_SHUTDOWN'));
+    t.falsy(finalContent.includes('POST_SHUTDOWN'));
+    t.falsy(finalContent.includes('SHUTDOWN_COMPLETED'));
   },
 );