@agoric/telemetry 0.6.3-other-dev-8f8782b.0 → 0.6.3-other-dev-fbe72e7.0.fbe72e7

package/src/otel-trace.js

@@ -22,16 +22,26 @@ export const SPAN_EXPORT_DELAY_MS = 1_000;
 export const makeOtelTracingProvider = opts => {
   const { env = process.env } = opts || {};
 
+  // https://opentelemetry.io/docs/concepts/signals/
+  // https://opentelemetry.io/docs/specs/otel/protocol/exporter/#endpoint-urls-for-otlphttp
+  // https://github.com/open-telemetry/opentelemetry-js/blob/experimental/v0.57.1/experimental/packages/exporter-trace-otlp-http/README.md#configuration-options-as-environment-variables
   const { OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT } =
     env;
   if (!OTEL_EXPORTER_OTLP_ENDPOINT && !OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) {
+    console.debug(
+      'Not enabling OTLP Traces Exporter; enable with OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=<target URL> or OTEL_EXPORTER_OTLP_ENDPOINT=<target URL prefix>',
+    );
     return undefined;
   }
 
   const resource = new Resource(getResourceAttributes(opts));
 
   const exporter = new OTLPTraceExporter();
-  console.info('Enabling OTLP Traces Exporter to', exporter.getDefaultUrl({}));
+  console.info(
+    'Enabling OTLP Traces Exporter to',
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||
+      `${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
+  );
 
   const provider = new BasicTracerProvider({ resource });
   provider.addSpanProcessor(

package/src/prometheus.js

@@ -0,0 +1,18 @@
+import { Fail } from '@endo/errors';
+
+import { getPrometheusMeterProvider } from './index.js';
+import { makeSlogSender as makeOtelMetricsSender } from './otel-metrics.js';
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName?: string}} opts */
+export const makeSlogSender = async (opts = {}) => {
+  const { env, otelMeterName, serviceName } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  const otelMeterProvider = getPrometheusMeterProvider({
+    console,
+    env,
+    serviceName,
+  });
+  if (!otelMeterProvider) return;
+
+  return makeOtelMetricsSender({ ...opts, otelMeterName, otelMeterProvider });
+};

package/src/serialize-slog-obj.js

@@ -1,4 +1,32 @@
-export const serializeSlogObj = slogObj =>
-  JSON.stringify(slogObj, (_, arg) =>
-    typeof arg === 'bigint' ? Number(arg) : arg,
-  );
+const { hasOwn } = Object;
+
+const replacer = (_key, value) => {
+  switch (typeof value) {
+    case 'object': {
+      if (value instanceof Error) {
+        // Represent each error as a serialization-friendly
+        // { errorType, message, cause?, errors?, stack? } object
+        // (itself subject to recursive replacement, particularly in `cause` and
+        // `errors`).
+        const obj = { errorType: value.name, message: value.message };
+        if (hasOwn(value, 'cause')) obj.cause = value.cause;
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore TS2339 property "errors" is only on AggregateError
+        if (hasOwn(value, 'errors')) obj.errors = value.errors;
+        const stack = value.stack;
+        if (stack) obj.stack = stack;
+        return obj;
+      }
+      break;
+    }
+    case 'bigint':
+      // Represent each bigint as a JSON-serializable number, accepting the
+      // possible loss of precision.
+      return Number(value);
+    default:
+      break;
+  }
+  return value;
+};
+
+export const serializeSlogObj = slogObj => JSON.stringify(slogObj, replacer);

package/src/slog-file.js

@@ -11,7 +11,7 @@ export const makeSlogSender = async ({ env: { SLOGFILE } = {} } = {}) => {
 
   const slogSender = (slogObj, jsonObj = serializeSlogObj(slogObj)) => {
     // eslint-disable-next-line prefer-template
-    void stream.write(jsonObj + '\n');
+    stream.write(jsonObj + '\n').catch(() => {});
   };
 
   return Object.assign(slogSender, {

package/src/slog-sender-pipe-entrypoint.js

@@ -1,33 +1,34 @@
-/* global process */
+/* eslint-env node */
+/**
+ * @file Run as a child process of {@link ./slog-sender-pipe.js} to isolate an
+ *   aggregate slog sender (@see {@link ./make-slog-sender.js}). Communicates
+ *   with its parent via Node.js IPC with advanced (structured clone)
+ *   serialization.
+ *   https://nodejs.org/docs/latest/api/child_process.html#advanced-serialization
+ */
+
 import '@endo/init';
 
-import { makeAggregateError } from '@agoric/internal';
 import anylogger from 'anylogger';
+import { Fail } from '@endo/errors';
 import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
 
 import { makeSlogSender } from './make-slog-sender.js';
 
 const logger = anylogger('slog-sender-pipe-entrypoint');
 
-/** @type {(msg: import('./slog-sender-pipe.js').SlogSenderPipeWaitReplies) => void} */
+/** @type {(msg: import('./slog-sender-pipe.js').PipeAPIReply) => void} */
 const send = Function.prototype.bind.call(process.send, process);
 
 /**
- * @typedef {object} InitMessage
- * @property {'init'} type
- * @property {import('./index.js').MakeSlogSenderOptions} options
- */
-/**
- * @typedef {object} SendMessage
- * @property {'send'} type
- * @property {object} obj
+ * @typedef {{type: 'init', options: import('./index.js').MakeSlogSenderOptions }} InitMessage
+ * @typedef {{type: 'flush' }} FlushMessage
+ * @typedef {{type: 'send', obj: Record<string, unknown> }} SendMessage
+ *
+ * @typedef {InitMessage | FlushMessage} PipeAPIResponsefulMessage
+ * @typedef {SendMessage} PipeAPIResponselessMessage
+ * @typedef {PipeAPIResponsefulMessage | PipeAPIResponselessMessage} PipeAPIMessage
  */
-/**
- * @typedef {object} FlushMessage
- * @property {'flush'} type
- */
-/** @typedef {InitMessage | FlushMessage} SlogSenderPipeWaitMessages */
-/** @typedef {SlogSenderPipeWaitMessages | SendMessage } SlogSenderPipeMessages */
 
 const main = async () => {
   /** @type {import('./index.js').SlogSender | undefined} */
@@ -45,9 +46,7 @@ const main = async () => {
 
   /** @param {import('./index.js').MakeSlogSenderOptions} opts */
   const init = async ({ env, ...otherOpts } = {}) => {
-    if (slogSender) {
-      assert.fail('Already initialized');
-    }
+    !slogSender || Fail`Already initialized`;
 
     slogSender = await makeSlogSender({
       ...otherOpts,
@@ -58,9 +57,7 @@ const main = async () => {
   };
 
   const flush = async () => {
-    if (!slogSender) {
-      assert.fail('No sender available');
-    }
+    if (!slogSender) throw Fail`No sender available`;
 
     await slogSender.forceFlush?.();
   };
@@ -75,62 +72,67 @@ const main = async () => {
       sendErrors.unshift(actualFlushError);
     }
 
-    return makeAggregateError(sendErrors.splice(0));
+    return AggregateError(sendErrors.splice(0));
   };
 
-  process.on(
-    'message',
-    /** @param {SlogSenderPipeMessages} msg */ msg => {
-      if (!msg || typeof msg !== 'object') {
-        logger.warn('received invalid message', msg);
-        return;
-      }
+  /** @param {PipeAPIMessage} msg */
+  const onMessage = msg => {
+    if (!msg || typeof msg !== 'object') {
+      logger.warn('Received invalid message', msg);
+      return;
+    }
 
-      switch (msg.type) {
-        case 'init': {
-          void init(msg.options).then(
-            hasSender => {
-              send({ type: 'initReply', hasSender });
-            },
-            error => {
-              send({ type: 'initReply', hasSender: false, error });
-            },
-          );
-          break;
-        }
-        case 'flush': {
-          void flush().then(
-            () => {
-              send({ type: 'flushReply', error: generateFlushError() });
-            },
-            error => {
-              send({ type: 'flushReply', error: generateFlushError(error) });
-            },
-          );
-          break;
-        }
-        case 'send': {
-          if (!slogSender) {
-            logger.warn('received send with no sender available');
-          } else {
-            try {
-              slogSender(msg.obj);
-            } catch (e) {
-              sendErrors.push(e);
-            }
+    switch (msg.type) {
+      case 'init': {
+        void init(msg.options).then(
+          hasSender => {
+            send({ type: 'initReply', hasSender });
+          },
+          error => {
+            send({ type: 'initReply', hasSender: false, error });
+          },
+        );
+        break;
+      }
+      case 'flush': {
+        void flush().then(
+          () => {
+            send({ type: 'flushReply', error: generateFlushError() });
+          },
+          error => {
+            send({ type: 'flushReply', error: generateFlushError(error) });
+          },
+        );
+        break;
+      }
+      case 'send': {
+        if (!slogSender) {
+          logger.warn('Received send with no sender available');
+        } else {
+          try {
+            slogSender(harden(msg.obj));
+          } catch (e) {
+            sendErrors.push(e);
           }
-          break;
-        }
-        default: {
-          // @ts-expect-error exhaustive type check
-          logger.warn('received unknown message type', msg.type);
         }
+        break;
+      }
+      default: {
+        // @ts-expect-error exhaustive type check
+        logger.warn('Received unknown message type', msg.type);
       }
-    },
-  );
+    }
+  };
+  process.on('message', onMessage);
 };
 
-main().catch(e => {
-  logger.error(e);
-  process.exitCode = 1;
-});
+process.exitCode = 1;
+main().then(
+  () => {
+    process.exitCode = 0;
+  },
+  err => {
+    logger.error('Failed with', err);
+    process.exit(process.exitCode || 1);
+  },
+);

package/src/slog-sender-pipe.js

@@ -1,16 +1,28 @@
+/**
+ * @file Export a `makeSlogSender` that spawns a
+ *   {@link ./slog-sender-pipe-entrypoint.js} child process to which it forwards
+ *   all slog entries via Node.js IPC with advanced (structured clone)
+ *   serialization.
+ *   https://nodejs.org/docs/latest/api/child_process.html#advanced-serialization
+ */
+
 import { fork } from 'child_process';
 import path from 'path';
+import { promisify } from 'util';
 import anylogger from 'anylogger';
 
+import { q, Fail } from '@endo/errors';
+import { makePromiseKit } from '@endo/promise-kit';
 import { makeQueue } from '@endo/stream';
 
 import { makeShutdown } from '@agoric/internal/src/node/shutdown.js';
 
-const filename = new URL(import.meta.url).pathname;
-const dirname = path.dirname(filename);
+const dirname = path.dirname(new URL(import.meta.url).pathname);
 
 const logger = anylogger('slog-sender-pipe');
 
+const sink = () => {};
+
 /**
  * @template {any[]} T
  * @template R
@@ -24,168 +36,131 @@ const withMutex = operation => {
   return async (...args) => {
     await mutex.get();
     const result = operation(...args);
-    mutex.put(
-      result.then(
-        () => {},
-        () => {},
-      ),
-    );
+    mutex.put(result.then(sink, sink));
     return result;
   };
 };
 
 /**
- * @typedef {object} SlogSenderInitReply
- * @property {'initReply'} type
- * @property {boolean} hasSender
- * @property {Error} [error]
+ * @template [P=unknown]
+ * @typedef {{ type: string, error?: Error } & P} PipeReply
  */
+
 /**
- * @typedef {object} SlogSenderFlushReply
- * @property {'flushReply'} type
- * @property {Error} [error]
+ * @typedef {{
+ *   init: {
+ *     message: import('./slog-sender-pipe-entrypoint.js').InitMessage;
+ *     reply: PipeReply<{ hasSender: boolean }>;
+ *   };
+ *   flush: {
+ *     message: import('./slog-sender-pipe-entrypoint.js').FlushMessage;
+ *     reply: PipeReply<{}>;
+ *   };
+ * }} SlogSenderPipeAPI
+ *
+ * @typedef {keyof SlogSenderPipeAPI} PipeAPICommand
+ * @typedef {SlogSenderPipeAPI[PipeAPICommand]["reply"]} PipeAPIReply
  */
-/** @typedef {SlogSenderInitReply | SlogSenderFlushReply} SlogSenderPipeWaitReplies */
 
-/** @param {import('.').MakeSlogSenderOptions} opts */
-export const makeSlogSender = async opts => {
+/** @param {import('.').MakeSlogSenderOptions} options */
+export const makeSlogSender = async options => {
+  const { env = {} } = options;
   const { registerShutdown } = makeShutdown();
+
   const cp = fork(path.join(dirname, 'slog-sender-pipe-entrypoint.js'), [], {
-    stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
+    stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
     serialization: 'advanced',
+    env,
   });
   // logger.log('done fork');
 
-  const pipeSend = withMutex(
-    /**
-     * @template {{type: string}} T
-     * @param {T} msg
-     */
-    msg =>
-      /** @type {Promise<void>} */ (
-        new Promise((resolve, reject) => {
-          cp.send(msg, err => {
-            if (err) {
-              reject(err);
-            } else {
-              resolve();
-            }
-          });
-        })
-      ),
-  );
+  const exitKit = makePromiseKit();
+  cp.on('error', error => {
+    // An exit event *might* be coming, so wait a tick.
+    setImmediate(() => exitKit.resolve({ error }));
+  });
+  cp.on('exit', (exitCode, signal) => {
+    exitKit.resolve({ exitCode, signal });
+  });
+
+  /** @type {(msg: Record<string, unknown> & {type: string}) => Promise<void>} */
+  const rawSend = promisify(cp.send.bind(cp));
+  const pipeSend = withMutex(rawSend);
 
-  /**
-   * @typedef {{
-   *   init: {
-   *     message: import('./slog-sender-pipe-entrypoint').InitMessage;
-   *     reply: SlogSenderInitReply;
-   *   };
-   *   flush: {
-   *     message: import('./slog-sender-pipe-entrypoint').FlushMessage;
-   *     reply: SlogSenderFlushReply;
-   *   };
-   * }} SlogSenderWaitMessagesAndReplies
-   */
-
-  /** @typedef {keyof SlogSenderWaitMessagesAndReplies} SendWaitCommands */
-  /**
-   * @template {SlogSenderPipeWaitReplies} T
-   * @typedef {Omit<T, 'type' | 'error'>} ReplyPayload
-   */
-
-  /** @type {import('@endo/stream').AsyncQueue<SlogSenderPipeWaitReplies>} */
+  /** @type {import('@endo/stream').AsyncQueue<PipeAPIReply>} */
   const sendWaitQueue = makeQueue();
-  /** @type {SendWaitCommands | undefined} */
+  /** @type {PipeAPICommand | undefined} */
   let sendWaitType;
 
   const sendWaitReply = withMutex(
     /**
-     * @template {SendWaitCommands} T
+     * @template {PipeAPICommand} T
      * @param {T} type
-     * @param {Omit<SlogSenderWaitMessagesAndReplies[T]["message"], 'type'>} payload
-     * @returns {Promise<ReplyPayload<SlogSenderWaitMessagesAndReplies[T]["reply"]>>}
+     * @param {Omit<SlogSenderPipeAPI[T]["message"], 'type'>} payload
+     * @returns {Promise<Omit<SlogSenderPipeAPI[T]["reply"], keyof PipeReply>>}
      */
     async (type, payload) => {
-      !sendWaitType || assert.fail('Invalid mutex state');
+      !sendWaitType || Fail`Invalid mutex state`;
 
       const msg = { ...payload, type };
 
       sendWaitType = type;
-      return pipeSend(msg)
-        .then(async () => sendWaitQueue.get())
-        .then(
-          /** @param {SlogSenderWaitMessagesAndReplies[T]["reply"]} reply */ ({
-            type: replyType,
-            error,
-            ...rest
-          }) => {
-            replyType === `${type}Reply` ||
-              assert.fail(`Unexpected reply ${replyType}`);
-            if (error) {
-              throw error;
-            }
-            return rest;
-          },
-        )
-        .finally(() => {
-          sendWaitType = undefined;
-        });
-    },
-  );
-
-  cp.on(
-    'message',
-    /** @param { SlogSenderPipeWaitReplies } msg */
-    msg => {
-      // logger.log('received', msg);
-      if (
-        !msg ||
-        typeof msg !== 'object' ||
-        msg.type !== `${sendWaitType}Reply`
-      ) {
-        logger.warn('Received unexpected message', msg);
-        return;
+      await null;
+      try {
+        await pipeSend(msg);
+        /** @type {SlogSenderPipeAPI[T]["reply"]} */
+        const reply = await sendWaitQueue.get();
+        const { type: replyType, error, ...rest } = reply;
+        replyType === `${type}Reply` ||
+          Fail`Unexpected reply type ${q(replyType)}`;
+        if (error) throw error;
+        return rest;
+      } finally {
+        sendWaitType = undefined;
       }
-
-      sendWaitQueue.put(msg);
     },
   );
 
-  const flush = async () => sendWaitReply('flush', {});
-  /** @param {import('./index.js').MakeSlogSenderOptions} options */
-  const init = async options => sendWaitReply('init', { options });
+  /** @param {PipeReply} msg */
+  const onMessage = msg => {
+    // logger.log('received', msg);
+    if (!msg || msg.type !== `${sendWaitType}Reply`) {
+      logger.warn('Received unexpected message', msg);
+      return;
+    }
+
+    sendWaitQueue.put(msg);
+  };
+  cp.on('message', onMessage);
 
-  const send = obj => {
-    void pipeSend({ type: 'send', obj }).catch(() => {});
+  const flush = async () => {
+    await sendWaitReply('flush', {});
   };
 
   const shutdown = async () => {
     // logger.log('shutdown');
-    if (!cp.connected) {
-      return;
-    }
+    if (!cp.connected) return;
 
     await flush();
     cp.disconnect();
+    await exitKit.promise;
   };
   registerShutdown(shutdown);
 
-  const { hasSender } = await init(opts).catch(err => {
+  const { hasSender } = await sendWaitReply('init', { options }).catch(err => {
     cp.disconnect();
     throw err;
   });
-
   if (!hasSender) {
     cp.disconnect();
     return undefined;
   }
 
-  const slogSender = send;
+  const slogSender = obj => {
+    void pipeSend({ type: 'send', obj }).catch(sink);
+  };
   return Object.assign(slogSender, {
-    forceFlush: async () => {
-      await flush();
-    },
+    forceFlush: flush,
     shutdown,
     usesJsonObject: false,
   });

package/src/slog-to-otel.js

@@ -1,7 +1,7 @@
 import otel, { SpanStatusCode } from '@opentelemetry/api';
 
+import { Fail, q } from '@endo/errors';
 import { makeMarshal, Remotable } from '@endo/marshal';
-import { Fail, q } from '@agoric/assert';
 
 import { makeLegacyMap } from '@agoric/store';
 import {
@@ -14,11 +14,8 @@ import {
 
 // diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.VERBOSE);
 
-/** @typedef {import('@opentelemetry/api').Span} Span */
-/** @typedef {import('@opentelemetry/api').Link} SpanLink */
-/** @typedef {import('@opentelemetry/api').SpanContext} SpanContext */
-/** @typedef {import('@opentelemetry/api').SpanOptions} SpanOptions */
-/** @typedef {import('@opentelemetry/api').SpanAttributes} SpanAttributes */
+/** @import {Span, Link as SpanLink} from '@opentelemetry/api' */
+/** @import {SpanContext, SpanOptions} from '@opentelemetry/api' */
 
 const { assign } = Object;
 
@@ -54,9 +51,9 @@ const serializeInto = (value, prefix, target = {}, depth = 3) => {
       } else {
         const proto = Object.getPrototypeOf(value);
         if (proto == null || proto === Object.prototype) {
-          Object.entries(value).forEach(([key, nested]) =>
-            serializeInto(nested, `${prefix}.${key}`, target, depth),
-          );
+          for (const [key, nested] of Object.entries(value)) {
+            serializeInto(nested, `${prefix}.${key}`, target, depth);
+          }
           return target;
         }
       }
@@ -142,7 +139,10 @@ export const makeSlogToOtelKit = (tracer, overrideAttrs = {}) => {
     serializeBodyFormat: 'smallcaps',
   });
 
-  /** @param {import('@agoric/swingset-vat').SwingSetCapData} data */
+  /**
+   * @param {import('@agoric/swingset-vat').SwingSetCapData} data
+   * @returns {any}
+   */
   const unserialize = data => {
     try {
       const body = rawUnserialize(data);
@@ -915,7 +915,7 @@ export const makeSlogToOtelKit = (tracer, overrideAttrs = {}) => {
         break;
       }
       case 'cosmic-swingset-upgrade-finish': {
-        spans.pop(['slogAttrs.blockHeight', slogAttrs.blockHeight]);
+        spans.pop(['upgrade', slogAttrs.blockHeight]);
         dbTransactionManager.end();
         break;
       }
@@ -971,6 +971,21 @@ export const makeSlogToOtelKit = (tracer, overrideAttrs = {}) => {
         spans.pop('bridge-inbound');
         break;
       }
+      case 'cosmic-swingset-timer-poll': {
+        spans.push(['timer-poll', slogAttrs.blockTime]);
+        spans.pop('timer-poll');
+        break;
+      }
+      case 'cosmic-swingset-inject-kernel-upgrade-events': {
+        spans.push('kernel-upgrade-events');
+        spans.pop('kernel-upgrade-events');
+        break;
+      }
+      case 'cosmic-swingset-install-bundle': {
+        spans.push(['install-bundle', slogAttrs.endoZipBase64Sha512]);
+        spans.pop('install-bundle');
+        break;
+      }
       case 'cosmic-swingset-end-block-start': {
         // Add `end-block` as an event onto the encompassing `block` span
         spans.top()?.addEvent('end-block-action', cleanAttrs(slogAttrs), now);