@agoric/telemetry 0.6.3-other-dev-8f8782b.0 → 0.6.3-other-dev-fbe72e7.0.fbe72e7

package/src/make-slog-sender.js

@@ -1,15 +1,15 @@
 import path from 'path';
 import tmp from 'tmp';
-import { PromiseAllOrErrors } from '@agoric/internal';
+import { PromiseAllOrErrors, unprefixedProperties } from '@agoric/internal';
 import { serializeSlogObj } from './serialize-slog-obj.js';
 
+export const DEFAULT_SLOGSENDER_AGENT = 'self';
 export const DEFAULT_SLOGSENDER_MODULE =
   '@agoric/telemetry/src/flight-recorder.js';
 export const SLOGFILE_SENDER_MODULE = '@agoric/telemetry/src/slog-file.js';
+export const PROMETHEUS_SENDER_MODULE = '@agoric/telemetry/src/prometheus.js';
 
-export const DEFAULT_SLOGSENDER_AGENT = 'self';
-
-/** @typedef {import('./index.js').SlogSender} SlogSender */
+/** @import {SlogSender} from './index.js' */
 
 /**
  * @template T
@@ -19,102 +19,119 @@ export const DEFAULT_SLOGSENDER_AGENT = 'self';
 const filterTruthy = arr => /** @type {any[]} */ (arr.filter(Boolean));
 
 /**
- * @param {import('./index.js').MakeSlogSenderOptions} opts
+ * Create an aggregate slog sender that fans out inbound slog entries to modules
+ * as indicated by variables in the supplied `env` option. The SLOGSENDER value
+ * (or a default DEFAULT_SLOGSENDER_MODULE defined above) is split on commas
+ * into a list of module identifiers and adjusted by automatic insertions (a
+ * non-empty SLOGFILE value inserts DEFAULT_SLOGSENDER_AGENT defined above), and
+ * then each identifier is dynamically `import`ed for its own `makeSlogSender`
+ * export, which is invoked with a non-empty `stateDir` option and a modified
+ * `env` in which SLOGSENDER_AGENT_* variables have overridden their unprefixed
+ * equivalents to produce a subordinate slog sender.
+ * Subordinate slog senders remain isolated from each other, and any errors from
+ * them are caught and held until the next `forceFlush()` without disrupting
+ * any remaining slog entry fanout.
+ * If SLOGSENDER_AGENT is 'process', 'slog-sender-pipe.js' is used to load the
+ * subordinates in a child process rather than the main process.
+ * When there are no subordinates, the return value will be `undefined` rather
+ * than a slog sender function.
+ *
+ * @type {import('./index.js').MakeSlogSender}
  */
 export const makeSlogSender = async (opts = {}) => {
   const { env = {}, stateDir: stateDirOption, ...otherOpts } = opts;
   const {
     SLOGSENDER = DEFAULT_SLOGSENDER_MODULE,
     SLOGSENDER_AGENT = DEFAULT_SLOGSENDER_AGENT,
+    // While cosmic-swingset/kernel code includes its own Prometheus metrics
+    // export, that trumps a slog sender module doing so.
+    // This extraction can be removed when that changes, but in the meantime,
+    // opt-in is only by SLOGSENDER_AGENT_OTEL_EXPORTER_PROMETHEUS_PORT.
+    OTEL_EXPORTER_PROMETHEUS_PORT: _prometheusExportPort,
     ...otherEnv
   } = env;
 
   const agentEnv = {
     ...otherEnv,
-    ...Object.fromEntries(
-      Object.entries(otherEnv)
-        .filter(([k]) => k.match(/^(?:SLOGSENDER_AGENT_)+/)) // narrow to SLOGSENDER_AGENT_ prefixes.
-        .map(([k, v]) => [k.replace(/^(?:SLOGSENDER_AGENT_)+/, ''), v]), // Rewrite SLOGSENDER_AGENT_ to un-prefixed version.
-    ),
+    ...unprefixedProperties(otherEnv, 'SLOGSENDER_AGENT_'),
   };
 
-  const slogSenderModules = [
-    ...new Set([
-      ...(agentEnv.SLOGFILE ? [SLOGFILE_SENDER_MODULE] : []),
-      ...SLOGSENDER.split(',')
-        .filter(Boolean)
-        .map(modulePath =>
-          modulePath.startsWith('.')
-            ? // Resolve relative to the current working directory.
-              path.resolve(modulePath)
-            : modulePath,
-        ),
-    ]),
-  ];
-
-  if (!slogSenderModules.length) {
+  const slogSenderModules = new Set();
+  if (agentEnv.OTEL_EXPORTER_PROMETHEUS_PORT) {
+    slogSenderModules.add(PROMETHEUS_SENDER_MODULE);
+  }
+  if (agentEnv.SLOGFILE) {
+    slogSenderModules.add(SLOGFILE_SENDER_MODULE);
+  }
+  for (const moduleIdentifier of filterTruthy(SLOGSENDER.split(','))) {
+    if (moduleIdentifier.startsWith('-')) {
+      // Opt out of an automatically-included sender.
+      slogSenderModules.delete(moduleIdentifier.slice(1));
+    } else if (moduleIdentifier.startsWith('.')) {
+      // Resolve relative to the current working directory.
+      slogSenderModules.add(path.resolve(moduleIdentifier));
+    } else {
+      slogSenderModules.add(moduleIdentifier);
+    }
+  }
+
+  if (!slogSenderModules.size) {
     return undefined;
   }
 
-  switch (SLOGSENDER_AGENT) {
-    case '':
-    case 'self':
-      break;
-    case 'process': {
-      console.warn('Loading slog sender in subprocess');
-      return import('./slog-sender-pipe.js').then(
-        async ({ makeSlogSender: makeSogSenderPipe }) =>
-          makeSogSenderPipe({
-            env: {
-              ...agentEnv,
-              SLOGSENDER,
-              SLOGSENDER_AGENT: 'self',
-            },
-            stateDir: stateDirOption,
-            ...otherOpts,
-          }),
-      );
-    }
-    case 'worker':
-    default:
-      console.warn(`Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}`);
+  if (SLOGSENDER_AGENT === 'process') {
+    console.warn('Loading slog sender in subprocess');
+    return import('./slog-sender-pipe.js').then(async module =>
+      module.makeSlogSender({
+        env: {
+          ...agentEnv,
+          SLOGSENDER,
+          SLOGSENDER_AGENT: 'self',
+        },
+        stateDir: stateDirOption,
+        ...otherOpts,
+      }),
+    );
+  } else if (SLOGSENDER_AGENT && SLOGSENDER_AGENT !== 'self') {
+    console.warn(
+      `Unknown SLOGSENDER_AGENT=${SLOGSENDER_AGENT}; defaulting to 'self'`,
+    );
   }
 
   if (SLOGSENDER) {
     console.warn('Loading slog sender modules:', ...slogSenderModules);
   }
 
-  const makersInfo = await Promise.all(
-    slogSenderModules.map(async moduleIdentifier =>
-      import(moduleIdentifier)
-        .then(
-          /** @param {{makeSlogSender: (opts: {}) => Promise<SlogSender | undefined>}} module */ ({
-            makeSlogSender: maker,
-          }) => {
-            if (typeof maker !== 'function') {
-              return Promise.reject(
-                Error(`No 'makeSlogSender' function exported by module`),
-              );
-            } else if (maker === makeSlogSender) {
-              return Promise.reject(
-                Error(`Cannot recursively load 'makeSlogSender' aggregator`),
-              );
-            }
-
-            return /** @type {const} */ ([maker, moduleIdentifier]);
-          },
-        )
-        .catch(err => {
+  /** @type {Map<import('./index.js').MakeSlogSender, string>} */
+  const makerMap = new Map();
+  await Promise.all(
+    [...slogSenderModules].map(async moduleIdentifier => {
+      await null;
+      try {
+        const module = await import(moduleIdentifier);
+        const { makeSlogSender: maker } = module;
+        if (typeof maker !== 'function') {
+          throw Error(`No 'makeSlogSender' function exported by module`);
+        } else if (maker === makeSlogSender) {
+          throw Error(`Cannot recursively load 'makeSlogSender' aggregator`);
+        }
+        const isReplacing = makerMap.get(maker);
+        if (isReplacing) {
           console.warn(
-            `Failed to load slog sender from ${moduleIdentifier}.`,
-            err,
+            `The slog sender from ${moduleIdentifier} matches the one from ${isReplacing}.`,
           );
-          return undefined;
-        }),
-    ),
-  ).then(makerEntries => [...new Map(filterTruthy(makerEntries)).entries()]);
+        }
+        makerMap.set(maker, moduleIdentifier);
+      } catch (err) {
+        console.warn(
+          `Failed to load slog sender from ${moduleIdentifier}.`,
+          err,
+        );
+      }
+    }),
+  );
 
-  if (!makersInfo.length) {
+  if (!makerMap.size) {
     return undefined;
   }
 
@@ -122,11 +139,11 @@ export const makeSlogSender = async (opts = {}) => {
 
   if (stateDir === undefined) {
     stateDir = tmp.dirSync().name;
-    console.warn(`Using ${stateDir} for stateDir`);
+    console.warn(`Using ${stateDir} for slog sender stateDir`);
   }
 
   const senders = await Promise.all(
-    makersInfo.map(async ([maker, moduleIdentifier]) =>
+    [...makerMap.entries()].map(async ([maker, moduleIdentifier]) =>
       maker({
         ...otherOpts,
         stateDir,
@@ -137,37 +154,37 @@ export const makeSlogSender = async (opts = {}) => {
 
   if (!senders.length) {
     return undefined;
-  } else {
-    // Optimize creating a JSON serialization only if needed
-    // by any of the sender modules
-    const hasSenderUsingJsonObj = senders.some(
-      ({ usesJsonObject = true }) => usesJsonObject,
-    );
-    const getJsonObj = hasSenderUsingJsonObj
-      ? serializeSlogObj
-      : () => undefined;
-    const sendErrors = [];
-    /** @type {SlogSender} */
-    const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
-      for (const sender of senders) {
-        try {
-          sender(slogObj, jsonObj);
-        } catch (err) {
-          sendErrors.push(err);
-        }
-      }
-    };
-    return Object.assign(slogSender, {
-      forceFlush: async () =>
-        PromiseAllOrErrors([
-          ...senders.map(sender => sender.forceFlush?.()),
-          ...sendErrors.splice(0).map(err => Promise.reject(err)),
-        ]).then(() => {}),
-      shutdown: async () =>
-        PromiseAllOrErrors(senders.map(sender => sender.shutdown?.())).then(
-          () => {},
-        ),
-      usesJsonObject: hasSenderUsingJsonObj,
-    });
   }
+
+  // Optimize creating a JSON serialization only if needed
+  // by at least one of the senders.
+  const hasSenderUsingJsonObj = senders.some(
+    ({ usesJsonObject = true }) => usesJsonObject,
+  );
+  const getJsonObj = hasSenderUsingJsonObj ? serializeSlogObj : () => undefined;
+
+  const sendErrors = [];
+
+  /** @type {SlogSender} */
+  const slogSender = (slogObj, jsonObj = getJsonObj(slogObj)) => {
+    for (const sender of senders) {
+      try {
+        sender(slogObj, jsonObj);
+      } catch (err) {
+        sendErrors.push(err);
+      }
+    }
+  };
+  return Object.assign(slogSender, {
+    forceFlush: async () => {
+      await PromiseAllOrErrors([
+        ...senders.map(sender => sender.forceFlush?.()),
+        ...sendErrors.splice(0).map(err => Promise.reject(err)),
+      ]);
+    },
+    shutdown: async () => {
+      await PromiseAllOrErrors(senders.map(sender => sender.shutdown?.()));
+    },
+    usesJsonObject: hasSenderUsingJsonObj,
+  });
 };

package/src/otel-and-flight-recorder.js

@@ -1,6 +1,9 @@
-import { NonNullish } from '@agoric/assert';
+import { NonNullish } from '@agoric/internal';
 import { makeSlogSender as makeSlogSenderFromEnv } from './make-slog-sender.js';
 
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} opts
+ */
 export const makeSlogSender = async opts => {
   const { SLOGFILE: _1, SLOGSENDER: _2, ...otherEnv } = opts.env || {};
 

package/src/otel-context-aware-slog.js

@@ -0,0 +1,131 @@
+/* eslint-env node */
+import { logs, SeverityNumber } from '@opentelemetry/api-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
+import { Resource } from '@opentelemetry/resources';
+import {
+  LoggerProvider,
+  SimpleLogRecordProcessor,
+} from '@opentelemetry/sdk-logs';
+import { readFileSync, writeFileSync } from 'fs';
+import { makeContextualSlogProcessor } from './context-aware-slog.js';
+import { getResourceAttributes } from './index.js';
+import { serializeSlogObj } from './serialize-slog-obj.js';
+
+const DEFAULT_CONTEXT_FILE = 'slog-context.json';
+const FILE_ENCODING = 'utf8';
+
+/**
+ * @param {string} filePath
+ */
+export const getContextFilePersistenceUtils = filePath => {
+  console.warn(`Using file ${filePath} for slogger context`);
+
+  return {
+    /**
+     * @param {import('./context-aware-slog.js').Context} context
+     */
+    persistContext: context => {
+      try {
+        writeFileSync(filePath, serializeSlogObj(context), FILE_ENCODING);
+      } catch (err) {
+        console.error('Error writing context to file: ', err);
+      }
+    },
+
+    /**
+     * @returns {import('./context-aware-slog.js').Context | null}
+     */
+    restoreContext: () => {
+      try {
+        return JSON.parse(readFileSync(filePath, FILE_ENCODING));
+      } catch (parseErr) {
+        console.error('Error reading context from file: ', parseErr);
+        return null;
+      }
+    },
+  };
+};
+
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} options
+ */
+export const makeSlogSender = async options => {
+  const { CHAIN_ID, OTEL_EXPORTER_OTLP_ENDPOINT } = options.env || {};
+  if (!(OTEL_EXPORTER_OTLP_ENDPOINT && options.stateDir))
+    return console.error(
+      'Ignoring invocation of slogger "context-aware-slog" without the presence of "OTEL_EXPORTER_OTLP_ENDPOINT" and "stateDir"',
+    );
+
+  const loggerProvider = new LoggerProvider({
+    resource: new Resource(getResourceAttributes(options)),
+  });
+
+  const otelLogExporter = new OTLPLogExporter({ keepAlive: true });
+  const logRecordProcessor = new SimpleLogRecordProcessor(otelLogExporter);
+
+  loggerProvider.addLogRecordProcessor(logRecordProcessor);
+
+  logs.setGlobalLoggerProvider(loggerProvider);
+  const logger = logs.getLogger('default');
+
+  const persistenceUtils = getContextFilePersistenceUtils(
+    process.env.SLOG_CONTEXT_FILE_PATH ||
+      `${options.stateDir}/${DEFAULT_CONTEXT_FILE}`,
+  );
+
+  const contextualSlogProcessor = makeContextualSlogProcessor(
+    { 'chain-id': CHAIN_ID },
+    persistenceUtils,
+  );
+
+  /**
+   * @param {import('./context-aware-slog.js').Slog} slog
+   */
+  const slogSender = slog => {
+    const { time, ...logRecord } = contextualSlogProcessor(slog);
+
+    const [secondsStr, fractionStr] = String(time).split('.');
+    const seconds = parseInt(secondsStr, 10);
+    const nanoSeconds = parseInt(
+      (fractionStr || String(0)).padEnd(9, String(0)).slice(0, 9),
+      10,
+    );
+
+    logger.emit({
+      ...JSON.parse(serializeSlogObj(logRecord)),
+      severityNumber: SeverityNumber.INFO,
+      timestamp: [seconds, nanoSeconds],
+    });
+  };
+
+  const shutdown = async () => {
+    await Promise.resolve();
+    const errors = [];
+
+    try {
+      await logRecordProcessor.shutdown();
+    } catch (err) {
+      errors.push(err);
+    }
+
+    try {
+      await otelLogExporter.forceFlush();
+    } catch (err) {
+      errors.push(err);
+    }
+
+    switch (errors.length) {
+      case 0:
+        return;
+      case 1:
+        throw errors[0];
+      default:
+        throw AggregateError(errors);
+    }
+  };
+
+  return Object.assign(slogSender, {
+    forceFlush: () => otelLogExporter.forceFlush(),
+    shutdown,
+  });
+};

package/src/otel-metrics.js

@@ -0,0 +1,229 @@
+import { q, Fail } from '@endo/errors';
+
+import * as ActionType from '@agoric/internal/src/action-types.js';
+import { objectMapMutable } from '@agoric/internal/src/js-utils.js';
+import {
+  HISTOGRAM_METRICS,
+  BLOCK_HISTOGRAM_METRICS,
+  KERNEL_STATS_METRICS,
+  makeQueueMetrics,
+} from '@agoric/internal/src/metrics.js';
+
+/**
+ * @import {MetricOptions, ObservableCounter, ObservableUpDownCounter} from '@opentelemetry/api';
+ * @import {MeterProvider} from '@opentelemetry/sdk-metrics';
+ * @import {TotalMap} from '@agoric/internal';
+ */
+
+const knownActionTypes = new Set(Object.values(ActionType.QueuedActionType));
+
+/** @param {import('./index.js').MakeSlogSenderOptions & {otelMeterName: string, otelMeterProvider?: MeterProvider}} opts */
+export const makeSlogSender = async (opts = /** @type {any} */ ({})) => {
+  const { otelMeterName, otelMeterProvider } = opts;
+  if (!otelMeterName) throw Fail`OTel meter name is required`;
+  if (!otelMeterProvider) return;
+
+  const shutdown = async () => {
+    await otelMeterProvider.shutdown();
+  };
+
+  const otelMeter = otelMeterProvider.getMeter(otelMeterName);
+
+  const processedInboundActionCounter = otelMeter.createCounter(
+    'cosmic_swingset_inbound_actions',
+    { description: 'Processed inbound action counts by type' },
+  );
+  const histograms = {
+    ...objectMapMutable(HISTOGRAM_METRICS, (desc, name) => {
+      const { boundaries, ...options } = desc;
+      const advice = boundaries && { explicitBucketBoundaries: boundaries };
+      return otelMeter.createHistogram(name, { ...options, advice });
+    }),
+    ...objectMapMutable(BLOCK_HISTOGRAM_METRICS, (desc, name) =>
+      otelMeter.createHistogram(name, desc),
+    ),
+  };
+
+  const inboundQueueMetrics = makeQueueMetrics({
+    otelMeter,
+    namePrefix: 'cosmic_swingset_inbound_queue',
+    descPrefix: 'inbound queue',
+    console,
+  });
+
+  // Values for KERNEL_STATS_METRICS could be built up locally by observing slog
+  // entries, but they are all collectively reported in "kernel-stats"
+  // (@see {@link ../../cosmic-swingset/src/kernel-stats.js exportKernelStats})
+  // and for now we just reflect that, which requires implementation as async
+  // ("observable") instruments rather than synchronous ones.
+  /** @typedef {string} KernelStatsKey */
+  /** @typedef {string} KernelMetricName */
+  /** @type {TotalMap<KernelStatsKey, number>} */
+  const kernelStats = new Map();
+  /** @type {Map<KernelMetricName, ObservableCounter | ObservableUpDownCounter>} */
+  const kernelStatsCounters = new Map();
+  for (const meta of KERNEL_STATS_METRICS) {
+    const { key, name, sub, metricType, ...options } = meta;
+    kernelStats.set(key, 0);
+    if (metricType === 'gauge') {
+      kernelStats.set(`${key}Up`, 0);
+      kernelStats.set(`${key}Down`, 0);
+      kernelStats.set(`${key}Max`, 0);
+    } else if (metricType !== 'counter') {
+      Fail`Unknown metric type ${q(metricType)} for key ${q(key)} name ${q(name)}`;
+    }
+    let counter = kernelStatsCounters.get(name);
+    if (!counter) {
+      counter =
+        metricType === 'counter'
+          ? otelMeter.createObservableCounter(name, options)
+          : otelMeter.createObservableUpDownCounter(name, options);
+      kernelStatsCounters.set(name, counter);
+    }
+    const attributes = sub ? { [sub.dimension]: sub.value } : {};
+    counter.addCallback(observer => {
+      observer.observe(kernelStats.get(key), attributes);
+    });
+  }
+  const expectedKernelStats = new Set(kernelStats.keys());
+
+  /**
+   * @typedef {object} LazyStats
+   * @property {string} namePrefix
+   * @property {MetricOptions} options
+   * @property {Set<string>} keys
+   * @property {Record<string, number>} data
+   */
+  /** @type {(namePrefix: string, description: string) => LazyStats} */
+  const makeLazyStats = (namePrefix, description) => {
+    return { namePrefix, options: { description }, keys: new Set(), data: {} };
+  };
+  const dynamicAfterCommitStatsCounters = {
+    memoryUsage: makeLazyStats(
+      'memoryUsage_',
+      'kernel process memory statistic',
+    ),
+    heapStats: makeLazyStats('heapStats_', 'v8 kernel heap statistic'),
+  };
+
+  const slogSender = ({ type: slogType, ...slogObj }) => {
+    switch (slogType) {
+      // Consume cosmic-swingset block lifecycle slog entries.
+      case 'cosmic-swingset-init': {
+        const { inboundQueueInitialLengths: lengths } = slogObj;
+        inboundQueueMetrics.initLengths(lengths);
+        break;
+      }
+      case 'cosmic-swingset-begin-block': {
+        const {
+          interBlockSeconds,
+          afterCommitHangoverSeconds,
+          blockLagSeconds,
+        } = slogObj;
+
+        Number.isFinite(interBlockSeconds) &&
+          histograms.interBlockSeconds.record(interBlockSeconds);
+        histograms.afterCommitHangoverSeconds.record(
+          afterCommitHangoverSeconds,
+        );
+        Number.isFinite(blockLagSeconds) &&
+          histograms.blockLagSeconds.record(blockLagSeconds);
+        break;
+      }
+      case 'cosmic-swingset-run-finish': {
+        histograms.swingset_block_processing_seconds.record(slogObj.seconds);
+        break;
+      }
+      case 'cosmic-swingset-end-block-finish': {
+        const { inboundQueueStartLengths, processedActionCounts } = slogObj;
+        inboundQueueMetrics.updateLengths(inboundQueueStartLengths);
+        for (const processedActionRecord of processedActionCounts) {
+          const { count, phase, type: actionType } = processedActionRecord;
+          if (!knownActionTypes.has(actionType)) {
+            console.warn('Unknown inbound action type', actionType);
+          }
+          processedInboundActionCounter.add(count, { actionType });
+          inboundQueueMetrics.decLength(phase);
+        }
+        break;
+      }
+      case 'cosmic-swingset-commit-block-finish': {
+        const {
+          runSeconds,
+          chainTime,
+          saveTime,
+          cosmosCommitSeconds,
+          fullSaveTime,
+        } = slogObj;
+        histograms.swingsetRunSeconds.record(runSeconds);
+        histograms.swingsetChainSaveSeconds.record(chainTime);
+        histograms.swingsetCommitSeconds.record(saveTime);
+        histograms.cosmosCommitSeconds.record(cosmosCommitSeconds);
+        histograms.fullCommitSeconds.record(fullSaveTime);
+        break;
+      }
+
+      // Consume Swingset kernel slog entries.
+      case 'vat-startup-finish': {
+        histograms.swingset_vat_startup.record(slogObj.seconds * 1000);
+        break;
+      }
+      case 'crank-finish': {
+        const { crankType, messageType, seconds } = slogObj;
+        // TODO: Reflect crankType/messageType as proper dimensional attributes.
+        // For now, we're going for parity with direct metrics.
+        if (crankType !== 'routing' && messageType !== 'create-vat') {
+          histograms.swingset_crank_processing_time.record(seconds * 1000);
+        }
+        break;
+      }
+
+      // Consume miscellaneous slog entries.
+      case 'kernel-stats': {
+        const { stats } = slogObj;
+        const notYetFoundKernelStats = new Set(expectedKernelStats);
+        for (const [key, value] of Object.entries(stats)) {
+          notYetFoundKernelStats.delete(key);
+          if (!kernelStats.has(key)) {
+            console.warn('Unexpected SwingSet kernel statistic', key);
+          }
+          kernelStats.set(key, value);
+        }
+        if (notYetFoundKernelStats.size) {
+          console.warn('Expected SwingSet kernel statistics not found', [
+            ...notYetFoundKernelStats,
+          ]);
+        }
+        break;
+      }
+      case 'cosmic-swingset-after-commit-stats': {
+        const dynamicCounterEntries = Object.entries(
+          dynamicAfterCommitStatsCounters,
+        );
+        for (const [slogKey, meta] of dynamicCounterEntries) {
+          const { namePrefix, options, keys } = meta;
+          meta.data = slogObj[slogKey] || {};
+          const newKeys = Object.keys(meta.data).filter(key => !keys.has(key));
+          for (const key of newKeys) {
+            keys.add(key);
+            const name = `${namePrefix}${key}`;
+            const gauge = otelMeter.createObservableUpDownCounter(
+              name,
+              options,
+            );
+            gauge.addCallback(observer => {
+              observer.observe(meta.data[key]);
+            });
+          }
+        }
+        break;
+      }
+      default:
+        break;
+    }
+  };
+  return Object.assign(slogSender, {
+    shutdown,
+    usesJsonObject: false,
+  });
+};