@agoric/telemetry 0.6.3-other-dev-1f26562.0 → 0.6.3-other-dev-3eb1a1d.0

package/CHANGELOG.md

@@ -3,15 +3,6 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
-### [0.6.3-u11.0](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.2...@agoric/telemetry@0.6.3-u11.0) (2023-08-24)
-
-
-### Features
-
-* **cosmic-swingset:** add JS upgrade plan handler stub ([7803d3d](https://github.com/Agoric/agoric-sdk/commit/7803d3de8e0cba681dfd27dacfc3577eed0bf2f8))
-
-
-
 ### [0.6.2](https://github.com/Agoric/agoric-sdk/compare/@agoric/telemetry@0.6.1...@agoric/telemetry@0.6.2) (2023-06-02)
 
 **Note:** Version bump only for package @agoric/telemetry

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/telemetry",
-  "version": "0.6.3-other-dev-1f26562.0+1f26562",
+  "version": "0.6.3-other-dev-3eb1a1d.0+3eb1a1d",
   "description": "Agoric's telemetry implementation",
   "type": "module",
   "repository": "https://github.com/Agoric/agoric-sdk",
@@ -8,11 +8,11 @@
   "scripts": {
     "build": "exit 0",
     "test": "ava",
-    "test:c8": "c8 $C8_OPTIONS ava --config=ava-nesm.config.js",
+    "test:c8": "c8 --all $C8_OPTIONS ava",
     "test:xs": "exit 0",
     "lint-fix": "yarn lint:eslint --fix",
     "lint": "run-s --continue-on-error lint:*",
-    "lint:types": "tsc -p jsconfig.json",
+    "lint:types": "tsc",
     "lint:eslint": "eslint ."
   },
   "bin": {
@@ -22,43 +22,51 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "dependencies": {
-    "@agoric/assert": "0.6.1-other-dev-1f26562.0+1f26562",
-    "@agoric/internal": "0.3.3-other-dev-1f26562.0+1f26562",
-    "@agoric/store": "0.9.3-other-dev-1f26562.0+1f26562",
-    "@endo/init": "0.5.56",
-    "@endo/marshal": "0.8.5",
-    "@endo/stream": "0.3.25",
+    "@agoric/internal": "0.3.3-other-dev-3eb1a1d.0+3eb1a1d",
+    "@agoric/store": "0.9.3-other-dev-3eb1a1d.0+3eb1a1d",
+    "@endo/errors": "^1.2.8",
+    "@endo/init": "^1.1.7",
+    "@endo/marshal": "^1.6.2",
+    "@endo/stream": "^1.2.8",
     "@opentelemetry/api": "~1.3.0",
+    "@opentelemetry/api-logs": "0.53.0",
+    "@opentelemetry/exporter-logs-otlp-http": "0.53.0",
     "@opentelemetry/exporter-prometheus": "~0.35.0",
     "@opentelemetry/exporter-trace-otlp-http": "~0.35.0",
     "@opentelemetry/resources": "~1.9.0",
+    "@opentelemetry/sdk-logs": "0.53.0",
     "@opentelemetry/sdk-metrics": "~1.9.0",
     "@opentelemetry/sdk-trace-base": "~1.9.0",
-    "@opentelemetry/semantic-conventions": "~1.9.0",
+    "@opentelemetry/semantic-conventions": "~1.27.0",
     "anylogger": "^0.21.0",
-    "better-sqlite3": "^8.2.0",
-    "bufferfromfile": "agoric-labs/BufferFromFile#Agoric-built",
+    "better-sqlite3": "^9.1.1",
     "tmp": "^0.2.1"
   },
   "devDependencies": {
-    "@endo/lockdown": "0.1.28",
-    "@endo/ses-ava": "0.2.40",
-    "ava": "^5.2.0",
-    "c8": "^7.13.0",
+    "@endo/lockdown": "^1.0.13",
+    "@endo/ses-ava": "^1.2.8",
+    "ava": "^5.3.0",
+    "c8": "^10.1.2",
     "tmp": "^0.2.1"
   },
   "publishConfig": {
     "access": "public"
   },
   "engines": {
-    "node": ">=14.15.0"
+    "node": "^18.12 || ^20.9"
   },
   "ava": {
     "files": [
-      "test/**/test-*.js"
+      "test/**/*.test.*"
+    ],
+    "require": [
+      "@endo/init/debug.js"
     ],
     "timeout": "20m",
     "workerThreads": false
   },
-  "gitHead": "1f265627270d514ab48943f86c668eb58a280552"
+  "typeCoverage": {
+    "atLeast": 88.87
+  },
+  "gitHead": "3eb1a1d2d75b2b4a94807cd3bf759bc9fc531f05"
 }

package/scripts/ingest.sh

@@ -18,8 +18,8 @@ if [[ ! -s "$PROGRESS" ]]; then
   fi
   if [[ -n "$INGEST_START" ]]; then
     case "$1" in
-    *.Z | *.gz) firstline=$(zcat "$1" | head -1) ;;
-    *) firstline=$(head -1 "$1") ;;
+      *.Z | *.gz) firstline=$(zcat "$1" | head -1) ;;
+      *) firstline=$(head -1 "$1") ;;
     esac
     echo "$firstline" | jq --arg targetStart "$INGEST_START" \
       '{virtualTimeOffset: (($targetStart | fromdate) - .time), lastSlogTime: 0}' \

package/src/context-aware-slog-file.js

@@ -0,0 +1,42 @@
+/* eslint-env node */
+
+import { makeFsStreamWriter } from '@agoric/internal/src/node/fs-stream.js';
+import { makeContextualSlogProcessor } from './context-aware-slog.js';
+import { serializeSlogObj } from './serialize-slog-obj.js';
+
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} options
+ */
+export const makeSlogSender = async options => {
+  const { CHAIN_ID, CONTEXTUAL_SLOGFILE } = options.env || {};
+  if (!CONTEXTUAL_SLOGFILE)
+    return console.warn(
+      'Ignoring invocation of slogger "context-aware-slog-file" without the presence of "CONTEXTUAL_SLOGFILE"',
+    );
+
+  const stream = await makeFsStreamWriter(CONTEXTUAL_SLOGFILE);
+
+  if (!stream)
+    return console.error(
+      `Couldn't create a write stream on file "${CONTEXTUAL_SLOGFILE}"`,
+    );
+
+  const contextualSlogProcessor = makeContextualSlogProcessor({
+    'chain-id': CHAIN_ID,
+  });
+
+  /**
+   * @param {import('./context-aware-slog.js').Slog} slog
+   */
+  const slogSender = slog => {
+    const contextualizedSlog = contextualSlogProcessor(slog);
+
+    // eslint-disable-next-line prefer-template
+    stream.write(serializeSlogObj(contextualizedSlog) + '\n').catch(() => {});
+  };
+
+  return Object.assign(slogSender, {
+    forceFlush: () => stream.flush(),
+    shutdown: () => stream.close(),
+  });
+};

package/src/context-aware-slog.js

@@ -0,0 +1,383 @@
+/* eslint-env node */
+
+/**
+ * @typedef {Partial<{
+ *    'block.height': Slog['blockHeight'];
+ *    'block.time': Slog['blockTime'];
+ *    'crank.deliveryNum': Slog['deliveryNum'];
+ *    'crank.num': Slog['crankNum'];
+ *    'crank.type': Slog['crankType'];
+ *    'crank.vatID': Slog['vatID'];
+ *    init: boolean;
+ *    replay: boolean;
+ *    'run.id': string;
+ *    'run.num': string | null;
+ *    'run.trigger.blockHeight': Slog['blockHeight'];
+ *    'run.trigger.msgIdx': number;
+ *    'run.trigger.sender': Slog['sender'];
+ *    'run.trigger.source': Slog['source'];
+ *    'run.trigger.bundleHash': Slog['endoZipBase64Sha512'];
+ *    'run.trigger.time': Slog['blockTime'];
+ *    'run.trigger.txHash': string;
+ *    'run.trigger.type': string;
+ *  }>
+ * } Context
+ *
+ * @typedef {{
+ *  'crank.syscallNum'?: Slog['syscallNum'];
+ *  'process.uptime': Slog['monotime'];
+ * } & Context} LogAttributes
+ *
+ * @typedef {{
+ *  blockHeight?: number;
+ *  blockTime?: number;
+ *  crankNum?: bigint;
+ *  crankType?: string;
+ *  deliveryNum?: bigint;
+ *  inboundNum?: string;
+ *  monotime: number;
+ *  remainingBeans?: bigint;
+ *  replay?: boolean;
+ *  runNum?: number;
+ *  sender?: string;
+ *  source?: string;
+ *  endoZipBase64Sha512?: string;
+ *  syscallNum?: number;
+ *  time: number;
+ *  type: string;
+ *  vatID?: string;
+ * }} Slog
+ */
+
+const SLOG_TYPES = {
+  CLIST: 'clist',
+  CONSOLE: 'console',
+  COSMIC_SWINGSET: {
+    AFTER_COMMIT_STATS: 'cosmic-swingset-after-commit-stats',
+    BEGIN_BLOCK: 'cosmic-swingset-begin-block',
+    BOOTSTRAP_BLOCK: {
+      FINISH: 'cosmic-swingset-bootstrap-block-finish',
+      START: 'cosmic-swingset-bootstrap-block-start',
+    },
+    COMMIT: {
+      FINISH: 'cosmic-swingset-commit-finish',
+      START: 'cosmic-swingset-commit-start',
+    },
+    END_BLOCK: {
+      FINISH: 'cosmic-swingset-end-block-finish',
+      START: 'cosmic-swingset-end-block-start',
+    },
+    // eslint-disable-next-line no-restricted-syntax
+    RUN: {
+      FINISH: 'cosmic-swingset-run-finish',
+      START: 'cosmic-swingset-run-start',
+    },
+  },
+  COSMIC_SWINGSET_TRIGGERS: {
+    BRIDGE_INBOUND: 'cosmic-swingset-bridge-inbound',
+    DELIVER_INBOUND: 'cosmic-swingset-deliver-inbound',
+    TIMER_POLL: 'cosmic-swingset-timer-poll',
+    INSTALL_BUNDLE: 'cosmic-swingset-install-bundle',
+  },
+  CRANK: {
+    FINISH: 'crank-finish',
+    START: 'crank-start',
+  },
+  DELIVER: 'deliver',
+  DELIVER_RESULT: 'deliver-result',
+  KERNEL: {
+    INIT: {
+      FINISH: 'kernel-init-finish',
+      START: 'kernel-init-start',
+    },
+  },
+  REPLAY: {
+    FINISH: 'finish-replay',
+    START: 'start-replay',
+  },
+  SYSCALL: 'syscall',
+  SYSCALL_RESULT: 'syscall-result',
+};
+
+/**
+ * @template {Record<string, any>} [T={}]
+ * @param {T} [staticContext]
+ * @param {Partial<{ persistContext: (context: Context) => void; restoreContext: () => Context | null; }>} [persistenceUtils]
+ */
+export const makeContextualSlogProcessor = (
+  staticContext,
+  persistenceUtils = {},
+) => {
+  /** @type Array<Context | null> */
+  let [
+    blockContext,
+    crankContext,
+    initContext,
+    lastPersistedTriggerContext,
+    replayContext,
+    triggerContext,
+  ] = [null, null, null, null, null, null];
+
+  /**
+   * @param {Context} context
+   */
+  const persistContext = context => {
+    lastPersistedTriggerContext = context;
+    return persistenceUtils?.persistContext?.(context);
+  };
+
+  const restoreContext = () => {
+    if (!lastPersistedTriggerContext)
+      lastPersistedTriggerContext =
+        persistenceUtils?.restoreContext?.() || null;
+    return lastPersistedTriggerContext;
+  };
+
+  /**
+   * @param {Slog} slog
+   * @returns {{ attributes: T & LogAttributes, body: Partial<Slog>; timestamp: Slog['time'] }}
+   */
+  const slogProcessor = ({ monotime, time: timestamp, ...body }) => {
+    const finalBody = { ...body };
+
+    /** @type {{'crank.syscallNum'?: Slog['syscallNum']}} */
+    const eventLogAttributes = {};
+
+    /**
+     * Add any before report operations here
+     * like setting context data
+     */
+    switch (body.type) {
+      case SLOG_TYPES.KERNEL.INIT.START: {
+        initContext = { init: true };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BEGIN_BLOCK: {
+        blockContext = {
+          'block.height': finalBody.blockHeight,
+          'block.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.START: {
+        blockContext = {
+          'block.height': finalBody.blockHeight || 0,
+          'block.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.END_BLOCK.START:
+      case SLOG_TYPES.COSMIC_SWINGSET.END_BLOCK.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.COMMIT.START:
+      case SLOG_TYPES.COSMIC_SWINGSET.COMMIT.FINISH:
+      case SLOG_TYPES.COSMIC_SWINGSET.AFTER_COMMIT_STATS: {
+        assert(!!blockContext && !triggerContext);
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET_TRIGGERS.BRIDGE_INBOUND:
+      case SLOG_TYPES.COSMIC_SWINGSET_TRIGGERS.DELIVER_INBOUND: {
+        const [blockHeight, txHash, msgIdx] = (
+          finalBody.inboundNum || ''
+        ).split('-');
+        const [, triggerType] =
+          /cosmic-swingset-([^-]+)-inbound/.exec(body.type) || [];
+
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `${triggerType}-${finalBody.inboundNum}`,
+          'run.trigger.type': triggerType,
+          'run.trigger.source': finalBody.source,
+          'run.trigger.sender': finalBody.sender,
+          'run.trigger.blockHeight': Number(blockHeight),
+          'run.trigger.txHash': txHash,
+          'run.trigger.msgIdx': Number(msgIdx),
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET_TRIGGERS.INSTALL_BUNDLE: {
+        const [blockHeight, txHash, msgIdx] = (
+          finalBody.inboundNum || ''
+        ).split('-');
+
+        const triggerType = 'install-bundle';
+
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `${triggerType}-${finalBody.inboundNum}`,
+          'run.trigger.type': triggerType,
+          'run.trigger.bundleHash': finalBody.endoZipBase64Sha512,
+          'run.trigger.blockHeight': Number(blockHeight),
+          'run.trigger.txHash': txHash,
+          'run.trigger.msgIdx': Number(msgIdx),
+        };
+
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET_TRIGGERS.TIMER_POLL: {
+        const triggerType = 'timer-poll';
+
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `${triggerType}-${finalBody.inboundNum}`,
+          'run.trigger.type': triggerType,
+          'run.trigger.time': finalBody.blockTime,
+          'run.trigger.blockHeight': finalBody.blockHeight,
+        };
+
+        break;
+      }
+      // eslint-disable-next-line no-restricted-syntax
+      case SLOG_TYPES.COSMIC_SWINGSET.RUN.START: {
+        if (!finalBody.runNum) {
+          assert(!triggerContext);
+          triggerContext = restoreContext(); // Restore persisted context if any
+        } else if (!triggerContext) {
+          assert(!!blockContext);
+          // TODO: add explicit slog events of both timer poll and install bundle
+          // https://github.com/Agoric/agoric-sdk/issues/10332
+          triggerContext = {
+            'run.num': undefined,
+            'run.id': `unknown-${finalBody.blockHeight}-${finalBody.runNum}`,
+            'run.trigger.type': 'unknown',
+            'run.trigger.blockHeight': finalBody.blockHeight,
+          };
+        }
+
+        if (!triggerContext) triggerContext = {};
+        triggerContext['run.num'] = `${finalBody.runNum}`;
+
+        break;
+      }
+      case SLOG_TYPES.CRANK.START: {
+        crankContext = {
+          'crank.num': finalBody.crankNum,
+          'crank.type': finalBody.crankType,
+        };
+        break;
+      }
+      case SLOG_TYPES.CLIST: {
+        assert(!!crankContext);
+        crankContext['crank.vatID'] = finalBody.vatID;
+        break;
+      }
+      case SLOG_TYPES.REPLAY.START:
+      case SLOG_TYPES.REPLAY.FINISH: {
+        replayContext = { replay: true, 'crank.vatID': finalBody.vatID };
+        break;
+      }
+      case SLOG_TYPES.DELIVER: {
+        if (replayContext) {
+          assert(finalBody.replay);
+          replayContext = {
+            ...replayContext,
+            'crank.vatID': finalBody.vatID,
+            'crank.deliveryNum': finalBody.deliveryNum,
+          };
+        } else {
+          assert(!!crankContext && !finalBody.replay);
+          crankContext = {
+            ...crankContext,
+            'crank.vatID': finalBody.vatID,
+            'crank.deliveryNum': finalBody.deliveryNum,
+          };
+        }
+
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+
+        break;
+      }
+      case SLOG_TYPES.DELIVER_RESULT: {
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+
+        break;
+      }
+      case SLOG_TYPES.SYSCALL:
+      case SLOG_TYPES.SYSCALL_RESULT: {
+        eventLogAttributes['crank.syscallNum'] = finalBody.syscallNum;
+
+        delete finalBody.deliveryNum;
+        delete finalBody.replay;
+        delete finalBody.syscallNum;
+
+        break;
+      }
+      case SLOG_TYPES.CONSOLE: {
+        delete finalBody.crankNum;
+        delete finalBody.deliveryNum;
+
+        break;
+      }
+      default:
+        // All other log types are logged as is (using existing contexts) without
+        // any change to the slogs or any contributions to the contexts. This also
+        // means that any unexpected slog type will pass through. To fix that, add
+        // all remaining cases of expected slog types above with a simple break
+        // statement and log a warning here
+        break;
+    }
+
+    const logAttributes = {
+      ...staticContext,
+      'process.uptime': monotime,
+      ...initContext, // Optional prelude
+      ...blockContext, // Block is the first level of execution nesting
+      ...triggerContext, // run and trigger info is nested next
+      ...crankContext, // Finally cranks are the last level of nesting
+      ...replayContext, // Replay is a substitute for crank context during vat page in
+      ...eventLogAttributes,
+    };
+
+    /**
+     * Add any after report operations here
+     * like resetting context data
+     */
+    switch (body.type) {
+      case SLOG_TYPES.KERNEL.INIT.FINISH: {
+        initContext = null;
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.START: {
+        triggerContext = {
+          'run.num': undefined,
+          'run.id': `bootstrap-${finalBody.blockTime}`,
+          'run.trigger.type': 'bootstrap',
+          'run.trigger.time': finalBody.blockTime,
+        };
+        break;
+      }
+      case SLOG_TYPES.COSMIC_SWINGSET.AFTER_COMMIT_STATS:
+      case SLOG_TYPES.COSMIC_SWINGSET.BOOTSTRAP_BLOCK.FINISH: {
+        blockContext = null;
+        break;
+      }
+      // eslint-disable-next-line no-restricted-syntax
+      case SLOG_TYPES.COSMIC_SWINGSET.RUN.FINISH: {
+        assert(!!triggerContext);
+        persistContext(finalBody.remainingBeans ? {} : triggerContext);
+        triggerContext = null;
+        break;
+      }
+      case SLOG_TYPES.CRANK.FINISH: {
+        crankContext = null;
+        break;
+      }
+      case SLOG_TYPES.REPLAY.FINISH: {
+        replayContext = null;
+        break;
+      }
+      default:
+        break;
+    }
+
+    return {
+      attributes: /** @type {T & LogAttributes} */ (logAttributes),
+      body: finalBody,
+      timestamp,
+    };
+  };
+
+  return slogProcessor;
+};