@agoric/telemetry 0.6.3-u19.2 → 0.6.3-u21.0

package/src/otel-metrics.js

@@ -0,0 +1,229 @@
+import { q, Fail } from '@endo/errors';
+
+import * as ActionType from '@agoric/internal/src/action-types.js';
+import { objectMapMutable } from '@agoric/internal/src/js-utils.js';
+import {
+  HISTOGRAM_METRICS,
+  BLOCK_HISTOGRAM_METRICS,
+  KERNEL_STATS_METRICS,
+  makeQueueMetrics,
+} from '@agoric/internal/src/metrics.js';
+
+/**
+ * @import {MetricOptions, ObservableCounter, ObservableUpDownCounter} from '@opentelemetry/api';
+ * @import {MeterProvider} from '@opentelemetry/sdk-metrics';
+ * @import {TotalMap} from '@agoric/internal';
+ */
+
+const knownActionTypes = new Set(Object.values(ActionType.QueuedActionType));
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName: string, otelMeterProvider?: MeterProvider}} opts */
+export const makeSlogSender = async (opts = /** @type {any} */ ({})) => {
+  const { otelMeterName, otelMeterProvider } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  if (!otelMeterProvider) return;
+
+  const shutdown = async () => {
+    await otelMeterProvider.shutdown();
+  };
+
+  const otelMeter = otelMeterProvider.getMeter(otelMeterName);
+
+  const processedInboundActionCounter = otelMeter.createCounter(
+    'cosmic_swingset_inbound_actions',
+    { description: 'Processed inbound action counts by type' },
+  );
+  const histograms = {
+    ...objectMapMutable(HISTOGRAM_METRICS, (desc, name) => {
+      const { boundaries, ...options } = desc;
+      const advice = boundaries && { explicitBucketBoundaries: boundaries };
+      return otelMeter.createHistogram(name, { ...options, advice });
+    }),
+    ...objectMapMutable(BLOCK_HISTOGRAM_METRICS, (desc, name) =>
+      otelMeter.createHistogram(name, desc),
+    ),
+  };
+
+  const inboundQueueMetrics = makeQueueMetrics({
+    otelMeter,
+    namePrefix: 'cosmic_swingset_inbound_queue',
+    descPrefix: 'inbound queue',
+    console,
+  });
+
+  // Values for KERNEL_STATS_METRICS could be built up locally by observing slog
+  // entries, but they are all collectively reported in "kernel-stats"
+  // (@see {@link ../../cosmic-swingset/src/kernel-stats.js exportKernelStats})
+  // and for now we just reflect that, which requires implementation as async
+  // ("observable") instruments rather than synchronous ones.
+  /** @typedef {string} KernelStatsKey */
+  /** @typedef {string} KernelMetricName */
+  /** @type {TotalMap<KernelStatsKey, number>} */
+  const kernelStats = new Map();
+  /** @type {Map<KernelMetricName, ObservableCounter | ObservableUpDownCounter>} */
+  const kernelStatsCounters = new Map();
+  for (const meta of KERNEL_STATS_METRICS) {
+    const { key, name, sub, metricType, ...options } = meta;
+    kernelStats.set(key, 0);
+    if (metricType === 'gauge') {
+      kernelStats.set(`${key}Up`, 0);
+      kernelStats.set(`${key}Down`, 0);
+      kernelStats.set(`${key}Max`, 0);
+    } else if (metricType !== 'counter') {
+      Fail`Unknown metric type ${q(metricType)} for key ${q(key)} name ${q(name)}`;
+    }
+    let counter = kernelStatsCounters.get(name);
+    if (!counter) {
+      counter =
+        metricType === 'counter'
+          ? otelMeter.createObservableCounter(name, options)
+          : otelMeter.createObservableUpDownCounter(name, options);
+      kernelStatsCounters.set(name, counter);
+    }
+    const attributes = sub ? { [sub.dimension]: sub.value } : {};
+    counter.addCallback(observer => {
+      observer.observe(kernelStats.get(key), attributes);
+    });
+  }
+  const expectedKernelStats = new Set(kernelStats.keys());
+
+  /**
+   * @typedef {object} LazyStats
+   * @property {string} namePrefix
+   * @property {MetricOptions} options
+   * @property {Set<string>} keys
+   * @property {Record<string, number>} data
+   */
+  /** @type {(namePrefix: string, description: string) => LazyStats} */
+  const makeLazyStats = (namePrefix, description) => {
+    return { namePrefix, options: { description }, keys: new Set(), data: {} };
+  };
+  const dynamicAfterCommitStatsCounters = {
+    memoryUsage: makeLazyStats(
+      'memoryUsage_',
+      'kernel process memory statistic',
+    ),
+    heapStats: makeLazyStats('heapStats_', 'v8 kernel heap statistic'),
+  };
+
+  const slogSender = ({ type: slogType, ...slogObj }) => {
+    switch (slogType) {
+      // Consume cosmic-swingset block lifecycle slog entries.
+      case 'cosmic-swingset-init': {
+        const { inboundQueueInitialLengths: lengths } = slogObj;
+        inboundQueueMetrics.initLengths(lengths);
+        break;
+      }
+      case 'cosmic-swingset-begin-block': {
+        const {
+          interBlockSeconds,
+          afterCommitHangoverSeconds,
+          blockLagSeconds,
+        } = slogObj;
+
+        Number.isFinite(interBlockSeconds) &&
+          histograms.interBlockSeconds.record(interBlockSeconds);
+        histograms.afterCommitHangoverSeconds.record(
+          afterCommitHangoverSeconds,
+        );
+        Number.isFinite(blockLagSeconds) &&
+          histograms.blockLagSeconds.record(blockLagSeconds);
+        break;
+      }
+      case 'cosmic-swingset-run-finish': {
+        histograms.swingset_block_processing_seconds.record(slogObj.seconds);
+        break;
+      }
+      case 'cosmic-swingset-end-block-finish': {
+        const { inboundQueueStartLengths, processedActionCounts } = slogObj;
+        inboundQueueMetrics.updateLengths(inboundQueueStartLengths);
+        for (const processedActionRecord of processedActionCounts) {
+          const { count, phase, type: actionType } = processedActionRecord;
+          if (!knownActionTypes.has(actionType)) {
+            console.warn('Unknown inbound action type', actionType);
+          }
+          processedInboundActionCounter.add(count, { actionType });
+          inboundQueueMetrics.decLength(phase);
+        }
+        break;
+      }
+      case 'cosmic-swingset-commit-block-finish': {
+        const {
+          runSeconds,
+          chainTime,
+          saveTime,
+          cosmosCommitSeconds,
+          fullSaveTime,
+        } = slogObj;
+        histograms.swingsetRunSeconds.record(runSeconds);
+        histograms.swingsetChainSaveSeconds.record(chainTime);
+        histograms.swingsetCommitSeconds.record(saveTime);
+        histograms.cosmosCommitSeconds.record(cosmosCommitSeconds);
+        histograms.fullCommitSeconds.record(fullSaveTime);
+        break;
+      }
+
+      // Consume Swingset kernel slog entries.
+      case 'vat-startup-finish': {
+        histograms.swingset_vat_startup.record(slogObj.seconds * 1000);
+        break;
+      }
+      case 'crank-finish': {
+        const { crankType, messageType, seconds } = slogObj;
+        // TODO: Reflect crankType/messageType as proper dimensional attributes.
+        // For now, we're going for parity with direct metrics.
+        if (crankType !== 'routing' && messageType !== 'create-vat') {
+          histograms.swingset_crank_processing_time.record(seconds * 1000);
+        }
+        break;
+      }
+
+      // Consume miscellaneous slog entries.
+      case 'kernel-stats': {
+        const { stats } = slogObj;
+        const notYetFoundKernelStats = new Set(expectedKernelStats);
+        for (const [key, value] of Object.entries(stats)) {
+          notYetFoundKernelStats.delete(key);
+          if (!kernelStats.has(key)) {
+            console.warn('Unexpected SwingSet kernel statistic', key);
+          }
+          kernelStats.set(key, value);
+        }
+        if (notYetFoundKernelStats.size) {
+          console.warn('Expected SwingSet kernel statistics not found', [
+            ...notYetFoundKernelStats,
+          ]);
+        }
+        break;
+      }
+      case 'cosmic-swingset-after-commit-stats': {
+        const dynamicCounterEntries = Object.entries(
+          dynamicAfterCommitStatsCounters,
+        );
+        for (const [slogKey, meta] of dynamicCounterEntries) {
+          const { namePrefix, options, keys } = meta;
+          meta.data = slogObj[slogKey] || {};
+          const newKeys = Object.keys(meta.data).filter(key => !keys.has(key));
+          for (const key of newKeys) {
+            keys.add(key);
+            const name = `${namePrefix}${key}`;
+            const gauge = otelMeter.createObservableUpDownCounter(
+              name,
+              options,
+            );
+            gauge.addCallback(observer => {
+              observer.observe(meta.data[key]);
+            });
+          }
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  };
+  return Object.assign(slogSender, {
+    shutdown,
+    usesJsonObject: false,
+  });
+};

package/src/prometheus.js

@@ -0,0 +1,18 @@
+import { Fail } from '@endo/errors';
+
+import { getPrometheusMeterProvider } from './index.js';
+import { makeSlogSender as makeOtelMetricsSender } from './otel-metrics.js';
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName?: string}} opts */
+export const makeSlogSender = async (opts = {}) => {
+  const { env, otelMeterName, serviceName } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  const otelMeterProvider = getPrometheusMeterProvider({
+    console,
+    env,
+    serviceName,
+  });
+  if (!otelMeterProvider) return;
+
+  return makeOtelMetricsSender({ ...opts, otelMeterName, otelMeterProvider });
+};

package/src/serialize-slog-obj.js

@@ -1,4 +1,32 @@
-export const serializeSlogObj = slogObj =>
-  JSON.stringify(slogObj, (_, arg) =>
-    typeof arg === 'bigint' ? Number(arg) : arg,
-  );
+const { hasOwn } = Object;
+
+const replacer = (_key, value) => {
+  switch (typeof value) {
+    case 'object': {
+      if (value instanceof Error) {
+        // Represent each error as a serialization-friendly
+        // { errorType, message, cause?, errors?, stack? } object
+        // (itself subject to recursive replacement, particularly in `cause` and
+        // `errors`).
+        const obj = { errorType: value.name, message: value.message };
+        if (hasOwn(value, 'cause')) obj.cause = value.cause;
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore TS2339 property "errors" is only on AggregateError
+        if (hasOwn(value, 'errors')) obj.errors = value.errors;
+        const stack = value.stack;
+        if (stack) obj.stack = stack;
+        return obj;
+      }
+      break;
+    }
+    case 'bigint':
+      // Represent each bigint as a JSON-serializable number, accepting the
+      // possible loss of precision.
+      return Number(value);
+    default:
+      break;
+  }
+  return value;
+};
+
+export const serializeSlogObj = slogObj => JSON.stringify(slogObj, replacer);

package/src/slog-sender-pipe-entrypoint.js

@@ -1,32 +1,34 @@
 /* eslint-env node */
+/**
+ * @file Run as a child process of {@link ./slog-sender-pipe.js} to isolate an
+ *   aggregate slog sender (@see {@link ./make-slog-sender.js}). Communicates
+ *   with its parent via Node.js IPC with advanced (structured clone)
+ *   serialization.
+ *   https://nodejs.org/docs/latest/api/child_process.html#advanced-serialization
+ */
+
 import '@endo/init';
 
 import anylogger from 'anylogger';
+import { Fail } from '@endo/errors';
 import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
 
 import { makeSlogSender } from './make-slog-sender.js';
 
 const logger = anylogger('slog-sender-pipe-entrypoint');
 
-/** @type {(msg: import('./slog-sender-pipe.js').SlogSenderPipeWaitReplies) => void} */
+/** @type {(msg: import('./slog-sender-pipe.js').PipeAPIReply) => void} */
 const send = Function.prototype.bind.call(process.send, process);
 
 /**
- * @typedef {object} InitMessage
- * @property {'init'} type
- * @property {import('./index.js').MakeSlogSenderOptions} options
- */
-/**
- * @typedef {object} SendMessage
- * @property {'send'} type
- * @property {object} obj
+ * @typedef {{type: 'init', options: import('./index.js').MakeSlogSenderOptions }} InitMessage
+ * @typedef {{type: 'flush' }} FlushMessage
+ * @typedef {{type: 'send', obj: Record<string, unknown> }} SendMessage
+ *
+ * @typedef {InitMessage | FlushMessage} PipeAPIResponsefulMessage
+ * @typedef {SendMessage} PipeAPIResponselessMessage
+ * @typedef {PipeAPIResponsefulMessage | PipeAPIResponselessMessage} PipeAPIMessage
  */
-/**
- * @typedef {object} FlushMessage
- * @property {'flush'} type
- */
-/** @typedef {InitMessage | FlushMessage} SlogSenderPipeWaitMessages */
-/** @typedef {SlogSenderPipeWaitMessages | SendMessage } SlogSenderPipeMessages */
 
 const main = async () => {
   /** @type {import('./index.js').SlogSender | undefined} */
@@ -44,9 +46,7 @@ const main = async () => {
 
   /** @param {import('./index.js').MakeSlogSenderOptions} opts */
   const init = async ({ env, ...otherOpts } = {}) => {
-    if (slogSender) {
-      assert.fail('Already initialized');
-    }
+    !slogSender || Fail`Already initialized`;
 
     slogSender = await makeSlogSender({
       ...otherOpts,
@@ -57,9 +57,7 @@ const main = async () => {
   };
 
   const flush = async () => {
-    if (!slogSender) {
-      assert.fail('No sender available');
-    }
+    if (!slogSender) throw Fail`No sender available`;
 
     await slogSender.forceFlush?.();
   };
@@ -77,56 +75,55 @@ const main = async () => {
     return AggregateError(sendErrors.splice(0));
   };
 
-  process.on(
-    'message',
-    /** @param {SlogSenderPipeMessages} msg */ msg => {
-      if (!msg || typeof msg !== 'object') {
-        logger.warn('received invalid message', msg);
-        return;
-      }
+  /** @param {PipeAPIMessage} msg */
+  const onMessage = msg => {
+    if (!msg || typeof msg !== 'object') {
+      logger.warn('Received invalid message', msg);
+      return;
+    }
 
-      switch (msg.type) {
-        case 'init': {
-          void init(msg.options).then(
-            hasSender => {
-              send({ type: 'initReply', hasSender });
-            },
-            error => {
-              send({ type: 'initReply', hasSender: false, error });
-            },
-          );
-          break;
-        }
-        case 'flush': {
-          void flush().then(
-            () => {
-              send({ type: 'flushReply', error: generateFlushError() });
-            },
-            error => {
-              send({ type: 'flushReply', error: generateFlushError(error) });
-            },
-          );
-          break;
-        }
-        case 'send': {
-          if (!slogSender) {
-            logger.warn('received send with no sender available');
-          } else {
-            try {
-              slogSender(msg.obj);
-            } catch (e) {
-              sendErrors.push(e);
-            }
+    switch (msg.type) {
+      case 'init': {
+        void init(msg.options).then(
+          hasSender => {
+            send({ type: 'initReply', hasSender });
+          },
+          error => {
+            send({ type: 'initReply', hasSender: false, error });
+          },
+        );
+        break;
+      }
+      case 'flush': {
+        void flush().then(
+          () => {
+            send({ type: 'flushReply', error: generateFlushError() });
+          },
+          error => {
+            send({ type: 'flushReply', error: generateFlushError(error) });
+          },
+        );
+        break;
+      }
+      case 'send': {
+        if (!slogSender) {
+          logger.warn('Received send with no sender available');
+        } else {
+          try {
+            slogSender(harden(msg.obj));
+          } catch (e) {
+            sendErrors.push(e);
           }
-          break;
-        }
-        default: {
-          // @ts-expect-error exhaustive type check
-          logger.warn('received unknown message type', msg.type);
         }
+        break;
+      }
+      default: {
+        // @ts-expect-error exhaustive type check
+        logger.warn('Received unknown message type', msg.type);
       }
-    },
-  );
+    }
+  };
+  process.on('message', onMessage);
 };
 
 process.exitCode = 1;