@agoric/telemetry 0.6.3-dev-7ae1f27.0 → 0.6.3-dev-acbd3ae.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/telemetry",
-  "version": "0.6.3-dev-7ae1f27.0+7ae1f27",
+  "version": "0.6.3-dev-acbd3ae.0+acbd3ae",
   "description": "Agoric's telemetry implementation",
   "type": "module",
   "repository": "https://github.com/Agoric/agoric-sdk",
@@ -22,16 +22,19 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-7ae1f27.0+7ae1f27",
-    "@agoric/store": "0.9.3-dev-7ae1f27.0+7ae1f27",
+    "@agoric/internal": "0.3.3-dev-acbd3ae.0+acbd3ae",
+    "@agoric/store": "0.9.3-dev-acbd3ae.0+acbd3ae",
     "@endo/errors": "^1.2.7",
     "@endo/init": "^1.1.6",
     "@endo/marshal": "^1.6.1",
     "@endo/stream": "^1.2.7",
     "@opentelemetry/api": "~1.3.0",
+    "@opentelemetry/api-logs": "0.53.0",
+    "@opentelemetry/exporter-logs-otlp-http": "0.53.0",
     "@opentelemetry/exporter-prometheus": "~0.35.0",
     "@opentelemetry/exporter-trace-otlp-http": "~0.35.0",
     "@opentelemetry/resources": "~1.9.0",
+    "@opentelemetry/sdk-logs": "0.53.0",
     "@opentelemetry/sdk-metrics": "~1.9.0",
     "@opentelemetry/sdk-trace-base": "~1.9.0",
     "@opentelemetry/semantic-conventions": "~1.27.0",
@@ -65,5 +68,5 @@
   "typeCoverage": {
     "atLeast": 87.03
   },
-  "gitHead": "7ae1f278fa8cbeb0cfc777b7cebf507b1f07c958"
+  "gitHead": "acbd3ae486c3749b66bf068dcca83585a34f9110"
 }

package/src/context-aware-slog-file.js

@@ -0,0 +1,42 @@
+/* eslint-env node */
+
+import { makeFsStreamWriter } from '@agoric/internal/src/node/fs-stream.js';
+import { makeContextualSlogProcessor } from './context-aware-slog.js';
+import { serializeSlogObj } from './serialize-slog-obj.js';
+
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} options
+ */
+export const makeSlogSender = async options => {
+  const { CHAIN_ID, CONTEXTUAL_SLOGFILE } = options.env || {};
+  if (!CONTEXTUAL_SLOGFILE)
+    return console.warn(
+      'Ignoring invocation of slogger "context-aware-slog-file" without the presence of "CONTEXTUAL_SLOGFILE"',
+    );
+
+  const stream = await makeFsStreamWriter(CONTEXTUAL_SLOGFILE);
+
+  if (!stream)
+    return console.error(
+      `Couldn't create a write stream on file "${CONTEXTUAL_SLOGFILE}"`,
+    );
+
+  const contextualSlogProcessor = makeContextualSlogProcessor({
+    'chain-id': CHAIN_ID,
+  });
+
+  /**
+   * @param {import('./context-aware-slog.js').Slog} slog
+   */
+  const slogSender = slog => {
+    const contextualizedSlog = contextualSlogProcessor(slog);
+
+    // eslint-disable-next-line prefer-template
+    stream.write(serializeSlogObj(contextualizedSlog) + '\n').catch(() => {});
+  };
+
+  return Object.assign(slogSender, {
+    forceFlush: () => stream.flush(),
+    shutdown: () => stream.close(),
+  });
+};

package/src/context-aware-slog.js

@@ -0,0 +1,345 @@
+/* eslint-env node */
+
+/**
+ * @typedef {Partial<{
+ *    'block.height': Slog['blockHeight'];
+ *    'block.time': Slog['blockTime'];
+ *    'crank.deliveryNum': Slog['deliveryNum'];
+ *    'crank.num': Slog['crankNum'];
+ *    'crank.type': Slog['crankType'];
+ *    'crank.vatID': Slog['vatID'];
+ *    init: boolean;
+ *    replay: boolean;
+ *    'run.id': string;
+ *    'run.num': string | null;
+ *    'run.trigger.blockHeight': Slog['blockHeight'];
+ *    'run.trigger.msgIdx': number;
+ *    'run.trigger.sender': Slog['sender'];
+ *    'run.trigger.source': Slog['source'];
+ *    'run.trigger.time': Slog['blockTime'];
+ *    'run.trigger.txHash': string;
+ *    'run.trigger.type': string;
+ *  }>
+ * } Context
+ *
+ * @typedef {{
+ *  'crank.syscallNum'?: Slog['syscallNum'];
+ *  'process.uptime': Slog['monotime'];
+ * } & Context} LogAttributes
+ *
+ * @typedef {{
+ *  blockHeight?: number;
+ *  blockTime?: number;
+ *  crankNum?: bigint;
+ *  crankType?: string;
+ *  deliveryNum?: bigint;
+ *  inboundNum?: string;
+ *  monotime: number;
+ *  remainingBeans?: bigint;
+ *  replay?: boolean;
+ *  runNum?: number;
+ *  sender?: string;
+ *  source?: string;
+ *  syscallNum?: number;
+ *  time: number;
+ *  type: string;
+ *  vatID?: string;
+ * }} Slog
+ */
+
+const SLOG_TYPES = {
+  CLIST: 'clist',
+  CONSOLE: 'console',
+  COSMIC_SWINGSET: {
+    AFTER_COMMIT_STATS: 'cosmic-swingset-after-commit-stats',
+    BEGIN_BLOCK: 'cosmic-swingset-begin-block',
+    BOOTSTRAP_BLOCK: {
+      FINISH: 'cosmic-swingset-bootstrap-block-finish',
+      START: 'cosmic-swingset-bootstrap-block-start',
+    },
+    BRIDGE_INBOUND: 'cosmic-swingset-bridge-inbound',
+    COMMIT: {
+      FINISH: 'cosmic-swingset-commit-finish',
+      START: 'cosmic-swingset-commit-start',
+    },
+    DELIVER_INBOUND: 'cosmic-swingset-deliver-inbound',
+    END_BLOCK: {
+      FINISH: 'cosmic-swingset-end-block-finish',
+      START: 'cosmic-swingset-end-block-start',
+    },
+    // eslint-disable-next-line no-restricted-syntax
+    RUN: {
+      FINISH: 'cosmic-swingset-run-finish',
+      START: 'cosmic-swingset-run-start',
+    },
+  },
+  CRANK: {
+    FINISH: 'crank-finish',
+    START: 'crank-start',
+  },
+  DELIVER: 'deliver',
+  DELIVER_RESULT: 'deliver-result',
+  KERNEL: {
+    INIT: {
+      FINISH: 'kernel-init-finish',
+      START: 'kernel-init-start',
+    },
+  },
+  REPLAY: {
+    FINISH: 'finish-replay',
+    START: 'start-replay',
+  },
+  SYSCALL: 'syscall',
+  SYSCALL_RESULT: 'syscall-result',
+};
+
+/**
+ * @template {Record<string, any>} [T={}]
+ * @param {T} [staticContext]
+ * @param {Partial<{ persistContext: (context: Context) => void; restoreContext: () => Context | null; }>} [persistenceUtils]
+ */
+export const makeContextualSlogProcessor = (
+  staticContext,
+  persistenceUtils = {},
+) => {
+  /** @type Array<Context | null> */
+  let [
+    blockContext,
+    crankContext,
+    initContext,
+    lastPersistedTriggerContext,
+    replayContext,
+    triggerContext,
+  ] = [null, null, null, null, null, null];
+
+  /**
+   * @param {Context} context
+   */
+  const persistContext = context => {
+    lastPersistedTriggerContext = context;
+    return persistenceUtils?.persistContext?.(context);
+  };
+
+  const restoreContext = () => {
+    if (!lastPersistedTriggerContext)
+      lastPersistedTriggerContext =
+        persistenceUtils?.restoreContext?.() || null;
+    return lastPersistedTriggerContext;
+  };
+
+  /**
+   * @param {Slog} slog
+   * @returns {{ attributes: T & LogAttributes, body: Partial<Slog>; timestamp: Slog['time'] }}
+   */
+  const slogProcessor = ({ monotime, time: timestamp, ...body }) => {
+    const finalBody = { ...body };
+
+    /** @type {{'crank.syscallNum'?: Slog['syscallNum']}} */
+    const eventLogAttributes = {};
+
+    /**
+     * Add any before report operations here
+     * like setting context data
+     */
+    switch (body.type) {
+      case SLOG_TYPES.KERNEL.INIT.START: {
+        initContext = { init: true };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BEGIN_BLOCK: {
+        blockContext = {
+          'block.height': finalBody.blockHeight,
+          'block.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.START: {
+        blockContext = {
+          'block.height': finalBody.blockHeight || 0,
+          'block.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.END_BLOCK.START:
+      case SLOG_TYPES.COSMIC_SWINGSET.END_BLOCK.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.COMMIT.START:
+      case SLOG_TYPES.COSMIC_SWINGSET.COMMIT.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.AFTER_COMMIT_STATS: {
+        assert(!!blockContext && !triggerContext);
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BRIDGE_INBOUND:
+      case SLOG_TYPES.COSMIC_SWINGSET.DELIVER_INBOUND: {
+        const [blockHeight, txHash, msgIdx] = (
+          finalBody.inboundNum || ''
+        ).split('-');
+        const [, triggerType] =
+          /cosmic-swingset-([^-]+)-inbound/.exec(body.type) || [];
+
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `${triggerType}-${finalBody.inboundNum}`,
+          'run.trigger.type': triggerType,
+          'run.trigger.source': finalBody.source,
+          'run.trigger.sender': finalBody.sender,
+          'run.trigger.blockHeight': Number(blockHeight),
+          'run.trigger.txHash': txHash,
+          'run.trigger.msgIdx': Number(msgIdx),
+        };
+        break;
+      }
+      // eslint-disable-next-line no-restricted-syntax
+      case SLOG_TYPES.COSMIC_SWINGSET.RUN.START: {
+        if (!finalBody.runNum) {
+          assert(!triggerContext);
+          triggerContext = restoreContext(); // Restore persisted context if any
+        } else if (!triggerContext) {
+          assert(!!blockContext);
+          // TODO: add explicit slog events of both timer poll and install bundle
+          // https://github.com/Agoric/agoric-sdk/issues/10332
+          triggerContext = {
+            'run.num': undefined,
+            'run.id': `unknown-${finalBody.blockHeight}-${finalBody.runNum}`,
+            'run.trigger.type': 'unknown',
+            'run.trigger.blockHeight': finalBody.blockHeight,
+          };
+        }
+
+        if (!triggerContext) triggerContext = {};
+        triggerContext['run.num'] = `${finalBody.runNum}`;
+
+        break;
+      }
+      case SLOG_TYPES.CRANK.START: {
+        crankContext = {
+          'crank.num': finalBody.crankNum,
+          'crank.type': finalBody.crankType,
+        };
+        break;
+      }
+      case SLOG_TYPES.CLIST: {
+        assert(!!crankContext);
+        crankContext['crank.vatID'] = finalBody.vatID;
+        break;
+      }
+      case SLOG_TYPES.REPLAY.START:
+      case SLOG_TYPES.REPLAY.FINISH: {
+        replayContext = { replay: true, 'crank.vatID': finalBody.vatID };
+        break;
+      }
+      case SLOG_TYPES.DELIVER: {
+        if (replayContext) {
+          assert(finalBody.replay);
+          replayContext = {
+            ...replayContext,
+            'crank.vatID': finalBody.vatID,
+            'crank.deliveryNum': finalBody.deliveryNum,
+          };
+        } else {
+          assert(!!crankContext && !finalBody.replay);
+          crankContext = {
+            ...crankContext,
+            'crank.vatID': finalBody.vatID,
+            'crank.deliveryNum': finalBody.deliveryNum,
+          };
+        }
+
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+
+        break;
+      }
+      case SLOG_TYPES.DELIVER_RESULT: {
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+
+        break;
+      }
+      case SLOG_TYPES.SYSCALL:
+      case SLOG_TYPES.SYSCALL_RESULT: {
+        eventLogAttributes['crank.syscallNum'] = finalBody.syscallNum;
+
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+        delete finalBody.syscallNum;
+
+        break;
+      }
+      case SLOG_TYPES.CONSOLE: {
+        delete finalBody.crankNum;
+        delete finalBody.deliveryNum;
+
+        break;
+      }
+      default:
+        // All other log types are logged as is (using existing contexts) without
+        // any change to the slogs or any contributions to the contexts. This also
+        // means that any unexpected slog type will pass through. To fix that, add
+        // all remaining cases of expected slog types above with a simple break
+        // statement and log a warning here
+        break;
+    }
+
+    const logAttributes = {
+      ...staticContext,
+      'process.uptime': monotime,
+      ...initContext, // Optional prelude
+      ...blockContext, // Block is the first level of execution nesting
+      ...triggerContext, // run and trigger info is nested next
+      ...crankContext, // Finally cranks are the last level of nesting
+      ...replayContext, // Replay is a substitute for crank context during vat page in
+      ...eventLogAttributes,
+    };
+
+    /**
+     * Add any after report operations here
+     * like resetting context data
+     */
+    switch (body.type) {
+      case SLOG_TYPES.KERNEL.INIT.FINISH: {
+        initContext = null;
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.START: {
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `bootstrap-${finalBody.blockTime}`,
+          'run.trigger.type': 'bootstrap',
+          'run.trigger.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.AFTER_COMMIT_STATS:
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.FINISH: {
+        blockContext = null;
+        break;
+      }
+      // eslint-disable-next-line no-restricted-syntax
+      case SLOG_TYPES.COSMIC_SWINGSET.RUN.FINISH: {
+        assert(!!triggerContext);
+        persistContext(finalBody.remainingBeans ? {} : triggerContext);
+        triggerContext = null;
+        break;
+      }
+      case SLOG_TYPES.CRANK.FINISH: {
+        crankContext = null;
+        break;
+      }
+      case SLOG_TYPES.REPLAY.FINISH: {
+        replayContext = null;
+        break;
+      }
+      default:
+        break;
+    }
+
+    return {
+      attributes: /** @type {T & LogAttributes} */ (logAttributes),
+      body: finalBody,
+      timestamp,
+    };
+  };
+
+  return slogProcessor;
+};

package/src/otel-context-aware-slog.js

@@ -0,0 +1,131 @@
+/* eslint-env node */
+import { logs, SeverityNumber } from '@opentelemetry/api-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
+import { Resource } from '@opentelemetry/resources';
+import {
+  LoggerProvider,
+  SimpleLogRecordProcessor,
+} from '@opentelemetry/sdk-logs';
+import { readFileSync, writeFileSync } from 'fs';
+import { makeContextualSlogProcessor } from './context-aware-slog.js';
+import { getResourceAttributes } from './index.js';
+import { serializeSlogObj } from './serialize-slog-obj.js';
+
+const DEFAULT_CONTEXT_FILE = 'slog-context.json';
+const FILE_ENCODING = 'utf8';
+
+/**
+ * @param {string} filePath
+ */
+export const getContextFilePersistenceUtils = filePath => {
+  console.warn(`Using file ${filePath} for slogger context`);
+
+  return {
+    /**
+     * @param {import('./context-aware-slog.js').Context} context
+     */
+    persistContext: context => {
+      try {
+        writeFileSync(filePath, serializeSlogObj(context), FILE_ENCODING);
+      } catch (err) {
+        console.error('Error writing context to file: ', err);
+      }
+    },
+
+    /**
+     * @returns {import('./context-aware-slog.js').Context | null}
+     */
+    restoreContext: () => {
+      try {
+        return JSON.parse(readFileSync(filePath, FILE_ENCODING));
+      } catch (parseErr) {
+        console.error('Error reading context from file: ', parseErr);
+        return null;
+      }
+    },
+  };
+};
+
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} options
+ */
+export const makeSlogSender = async options => {
+  const { CHAIN_ID, OTEL_EXPORTER_OTLP_ENDPOINT } = options.env || {};
+  if (!(OTEL_EXPORTER_OTLP_ENDPOINT && options.stateDir))
+    return console.error(
+      'Ignoring invocation of slogger "context-aware-slog" without the presence of "OTEL_EXPORTER_OTLP_ENDPOINT" and "stateDir"',
+    );
+
+  const loggerProvider = new LoggerProvider({
+    resource: new Resource(getResourceAttributes(options)),
+  });
+
+  const otelLogExporter = new OTLPLogExporter({ keepAlive: true });
+  const logRecordProcessor = new SimpleLogRecordProcessor(otelLogExporter);
+
+  loggerProvider.addLogRecordProcessor(logRecordProcessor);
+
+  logs.setGlobalLoggerProvider(loggerProvider);
+  const logger = logs.getLogger('default');
+
+  const persistenceUtils = getContextFilePersistenceUtils(
+    process.env.SLOG_CONTEXT_FILE_PATH ||
+      `${options.stateDir}/${DEFAULT_CONTEXT_FILE}`,
+  );
+
+  const contextualSlogProcessor = makeContextualSlogProcessor(
+    { 'chain-id': CHAIN_ID },
+    persistenceUtils,
+  );
+
+  /**
+   * @param {import('./context-aware-slog.js').Slog} slog
+   */
+  const slogSender = slog => {
+    const { timestamp, ...logRecord } = contextualSlogProcessor(slog);
+
+    const [secondsStr, fractionStr] = String(timestamp).split('.');
+    const seconds = parseInt(secondsStr, 10);
+    const nanoSeconds = parseInt(
+      (fractionStr || String(0)).padEnd(9, String(0)).slice(0, 9),
+      10,
+    );
+
+    logger.emit({
+      ...JSON.parse(serializeSlogObj(logRecord)),
+      severityNumber: SeverityNumber.INFO,
+      timestamp: [seconds, nanoSeconds],
+    });
+  };
+
+  const shutdown = async () => {
+    await Promise.resolve();
+    const errors = [];
+
+    try {
+      await logRecordProcessor.shutdown();
+    } catch (err) {
+      errors.push(err);
+    }
+
+    try {
+      await otelLogExporter.forceFlush();
+    } catch (err) {
+      errors.push(err);
+    }
+
+    switch (errors.length) {
+      case 0:
+        return;
+      case 1:
+        throw errors[0];
+      default:
+        throw AggregateError(errors);
+    }
+  };
+
+  return Object.assign(slogSender, {
+    forceFlush: () => otelLogExporter.forceFlush(),
+    shutdown,
+  });
+};