@agoric/telemetry 0.6.3-other-dev-1f26562.0 → 0.6.3-other-dev-3eb1a1d.0

package/src/flight-recorder.js

@@ -1,24 +1,13 @@
+// @ts-check
+/* eslint-env node */
 /// <reference types="ses" />
 
-// https://github.com/Agoric/agoric-sdk/issues/3742#issuecomment-1028451575
-// I'd mmap() a 100MB file, reserve a few bytes for offsets, then use the rest
-// as a circular buffer to hold length-prefixed records. The agd process would
-// keep writing new events into the RAM window and updating the start/end
-// pointers, with some sequencing to make sure the record gets written before
-// the pointer is updated. Then, no mattter how abruptly the process is
-// terminated, as long as the host computer itself is still running, the on-disk
-// file would contain the most recent state, and anybody who reads the file will
-// get the most recent state. The host kernel (linux) is under no obligation to
-// flush it to disk any particular time, but knows when reads happen, so there's
-// no coherency problem, and the speed is unaffected by disk write speeds.
-
-import BufferFromFile from 'bufferfromfile';
-import { promises as fsPromises } from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { Fail } from '@endo/errors';
 import { serializeSlogObj } from './serialize-slog-obj.js';
 
-const { Fail } = assert;
-
 export const DEFAULT_CBUF_SIZE = 100 * 1024 * 1024;
 export const DEFAULT_CBUF_FILE = 'flight-recorder.bin';
 export const SLOG_MAGIC = 0x41472d534c4f4721n; // 'AG-SLOG!'
@@ -31,12 +20,16 @@ const I_ARENA_START = 4 * BigUint64Array.BYTES_PER_ELEMENT;
 
 const RECORD_HEADER_SIZE = BigUint64Array.BYTES_PER_ELEMENT;
 
+/**
+ * Initializes a circular buffer with the given size, creating the buffer file if it doesn't exist or is not large enough.
+ *
+ * @param {string} bufferFile - the file path for the circular buffer
+ * @param {number} circularBufferSize - the size of the circular buffer
+ * @returns {Promise<bigint>} the size of the initialized circular buffer
+ */
 const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
-  if (!circularBufferSize) {
-    return undefined;
-  }
   // If the file doesn't exist, or is not large enough, create it.
-  const stbuf = await fsPromises.stat(bufferFile).catch(e => {
+  const stbuf = await fsp.stat(bufferFile).catch(e => {
     if (e.code === 'ENOENT') {
       return undefined;
     }
@@ -57,8 +50,8 @@ const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
   header.setBigUint64(I_CIRC_START, 0n);
   header.setBigUint64(I_CIRC_END, 0n);
 
-  await fsPromises.mkdir(path.dirname(bufferFile), { recursive: true });
-  await fsPromises.writeFile(bufferFile, headerBuf);
+  await fsp.mkdir(path.dirname(bufferFile), { recursive: true });
+  await fsp.writeFile(bufferFile, headerBuf);
 
   if (stbuf && stbuf.size >= circularBufferSize) {
     // File is big enough.
@@ -66,50 +59,20 @@ const initializeCircularBuffer = async (bufferFile, circularBufferSize) => {
   }
 
   // Increase the file size.
-  await fsPromises.truncate(bufferFile, circularBufferSize);
+  await fsp.truncate(bufferFile, circularBufferSize);
   return arenaSize;
 };
 
-export const makeMemoryMappedCircularBuffer = async ({
-  circularBufferSize = DEFAULT_CBUF_SIZE,
-  stateDir = '/tmp',
-  circularBufferFilename,
-}) => {
-  const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`;
-  // console.log({ circularBufferFilename, filename });
+/** @typedef {Awaited<ReturnType<typeof makeSimpleCircularBuffer>>} CircularBuffer */
 
-  const newArenaSize = await initializeCircularBuffer(
-    filename,
-    circularBufferSize,
-  );
-
-  /**
-   * @type {Uint8Array}
-   * BufferFromFile mmap()s the file into the process address space.
-   */
-  const fileBuf = BufferFromFile(filename).Uint8Array();
-  const header = new DataView(fileBuf.buffer, 0, I_ARENA_START);
-
-  // Detect the arena size from the header, if not initialized.
-  const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE);
-  const arenaSize = newArenaSize || hdrArenaSize;
-
-  const hdrMagic = header.getBigUint64(I_MAGIC);
-  SLOG_MAGIC === hdrMagic ||
-    Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`;
-  arenaSize === hdrArenaSize ||
-    Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`;
-  const arena = new Uint8Array(
-    fileBuf.buffer,
-    header.byteLength,
-    Number(arenaSize),
-  );
-
-  /**
-   * @param {Uint8Array} outbuf
-   * @param {number} [offset] offset relative to the current trailing edge (circStart) of the data
-   * @returns {IteratorResult<Uint8Array, void>}
-   */
+/**
+ *
+ * @param {bigint} arenaSize
+ * @param {DataView} header
+ * @param {(outbuf: Uint8Array, readStart: number, firstReadLength: number) => void} readRecord
+ * @param {(record: Uint8Array, firstWriteLength: number, circEnd: bigint) => Promise<void>} writeRecord
+ */
+function finishCircularBuffer(arenaSize, header, readRecord, writeRecord) {
   const readCircBuf = (outbuf, offset = 0) => {
     offset + outbuf.byteLength <= arenaSize ||
       Fail`Reading past end of circular buffer`;
@@ -132,19 +95,13 @@ export const makeMemoryMappedCircularBuffer = async ({
       // The data is contiguous, like ---AAABBB---
       return { done: true, value: undefined };
     }
-    outbuf.set(arena.subarray(readStart, readStart + firstReadLength));
-    if (firstReadLength < outbuf.byteLength) {
-      outbuf.set(
-        arena.subarray(0, outbuf.byteLength - firstReadLength),
-        firstReadLength,
-      );
-    }
+    readRecord(outbuf, readStart, firstReadLength);
     return { done: false, value: outbuf };
   };
 
-  /** @param {Uint8Array} data */
-  const writeCircBuf = data => {
-    if (RECORD_HEADER_SIZE + data.byteLength > arena.byteLength) {
+  /** @type {(data: Uint8Array) => Promise<void>} */
+  const writeCircBuf = async data => {
+    if (RECORD_HEADER_SIZE + data.byteLength > arenaSize) {
       // The data is too big to fit in the arena, so skip it.
       const tooBigRecord = JSON.stringify({
         type: 'slog-record-too-big',
@@ -153,14 +110,17 @@ export const makeMemoryMappedCircularBuffer = async ({
       data = new TextEncoder().encode(tooBigRecord);
     }
 
-    if (RECORD_HEADER_SIZE + data.byteLength > arena.byteLength) {
+    if (RECORD_HEADER_SIZE + data.byteLength > arenaSize) {
       // Silently drop, it just doesn't fit.
       return;
     }
 
+    // Allocate for the data and a header
     const record = new Uint8Array(RECORD_HEADER_SIZE + data.byteLength);
+    // Set the data, after the header
     record.set(data, RECORD_HEADER_SIZE);
 
+    // Set the size in the header
     const lengthPrefix = new DataView(record.buffer);
     lengthPrefix.setBigUint64(0, BigInt(data.byteLength));
 
@@ -206,31 +166,135 @@ export const makeMemoryMappedCircularBuffer = async ({
       );
     }
 
-    arena.set(record.subarray(0, firstWriteLength), Number(circEnd));
-    if (firstWriteLength < record.byteLength) {
-      // Write to the beginning of the arena.
-      arena.set(record.subarray(firstWriteLength, record.byteLength), 0);
-    }
     header.setBigUint64(
       I_CIRC_END,
       (circEnd + BigInt(record.byteLength)) % arenaSize,
     );
+
+    return writeRecord(record, firstWriteLength, circEnd);
   };
 
-  const writeJSON = (obj, jsonObj = serializeSlogObj(obj)) => {
-    // Prepend a newline so that the file can be more easily manipulated.
-    const data = new TextEncoder().encode(`\n${jsonObj}`);
-    // console.log('have obj', obj, data);
-    writeCircBuf(data);
+  return { readCircBuf, writeCircBuf };
+}
+
+/**
+ * @param {{
+ *   circularBufferSize?: number,
+ *   stateDir?: string,
+ *   circularBufferFilename?: string
+ * }} opts
+ */
+export const makeSimpleCircularBuffer = async ({
+  circularBufferSize = DEFAULT_CBUF_SIZE,
+  stateDir = '/tmp',
+  circularBufferFilename,
+}) => {
+  const filename = circularBufferFilename || `${stateDir}/${DEFAULT_CBUF_FILE}`;
+
+  const newArenaSize = await initializeCircularBuffer(
+    filename,
+    circularBufferSize,
+  );
+
+  const file = await fsp.open(filename, 'r+');
+
+  const headerBuffer = Buffer.alloc(I_ARENA_START);
+
+  await file.read({
+    buffer: headerBuffer,
+    length: I_ARENA_START,
+    position: 0,
+  });
+  const header = new DataView(headerBuffer.buffer);
+
+  // Detect the arena size from the header, if not initialized.
+  const hdrArenaSize = header.getBigUint64(I_ARENA_SIZE);
+  const arenaSize = newArenaSize || hdrArenaSize;
+
+  const hdrMagic = header.getBigUint64(I_MAGIC);
+  SLOG_MAGIC === hdrMagic ||
+    Fail`${filename} is not a slog buffer; wanted magic ${SLOG_MAGIC}, got ${hdrMagic}`;
+  arenaSize === hdrArenaSize ||
+    Fail`${filename} arena size mismatch; wanted ${arenaSize}, got ${hdrArenaSize}`;
+
+  /** @type {(outbuf: Uint8Array, readStart: number, firstReadLength: number) => void} */
+  const readRecord = (outbuf, readStart, firstReadLength) => {
+    const bytesRead = fs.readSync(file.fd, outbuf, {
+      length: firstReadLength,
+      position: Number(readStart) + I_ARENA_START,
+    });
+    assert.equal(bytesRead, firstReadLength, 'Too few bytes read');
+
+    if (bytesRead < outbuf.byteLength) {
+      fs.readSync(file.fd, outbuf, {
+        offset: firstReadLength,
+        length: outbuf.byteLength - firstReadLength,
+        position: I_ARENA_START,
+      });
+    }
+  };
+
+  /**
+   * Writes to the file, offset by the header size. Also updates the file header.
+   *
+   * @param {Uint8Array} record
+   * @param {number} firstWriteLength
+   * @param {bigint} circEnd
+   */
+  const writeRecord = async (record, firstWriteLength, circEnd) => {
+    await file.write(
+      record,
+      // TS saying options bag not available
+      0,
+      firstWriteLength,
+      I_ARENA_START + Number(circEnd),
+    );
+    if (firstWriteLength < record.byteLength) {
+      // Write to the beginning of the arena.
+      await file.write(
+        record,
+        firstWriteLength,
+        record.byteLength - firstWriteLength,
+        I_ARENA_START,
+      );
+    }
+
+    // Write out the updated file header.
+    // This is somewhat independent of writing the record itself, but it needs
+    // updating each time a record is written.
+    await file.write(headerBuffer, undefined, undefined, 0);
   };
 
-  return { readCircBuf, writeCircBuf, writeJSON };
+  return finishCircularBuffer(arenaSize, header, readRecord, writeRecord);
 };
 
-export const makeSlogSender = async opts => {
-  const { writeJSON } = await makeMemoryMappedCircularBuffer(opts);
+/**
+ *
+ * @param {Pick<Awaited<ReturnType<typeof makeSimpleCircularBuffer>>, 'writeCircBuf'>} circBuf
+ */
+export const makeSlogSenderFromBuffer = ({ writeCircBuf }) => {
+  /** @type {Promise<void>} */
+  let toWrite = Promise.resolve();
+  const writeJSON = (obj, serialized = serializeSlogObj(obj)) => {
+    // Prepend a newline so that the file can be more easily manipulated.
+    const data = new TextEncoder().encode(`\n${serialized}`);
+    // console.log('have obj', obj, data);
+    toWrite = toWrite.then(() => writeCircBuf(data));
+  };
   return Object.assign(writeJSON, {
-    forceFlush: async () => {},
+    forceFlush: async () => {
+      await toWrite;
+    },
     usesJsonObject: true,
   });
 };
+
+/**
+ * Loaded dynamically by makeSlogSender()
+ *
+ * @type {import('./index.js').MakeSlogSender}
+ */
+export const makeSlogSender = async opts => {
+  const { writeCircBuf } = await makeSimpleCircularBuffer(opts);
+  return makeSlogSenderFromBuffer({ writeCircBuf });
+};

package/src/frcat-entrypoint.js

@@ -1,11 +1,11 @@
 #! /usr/bin/env node
-/* global process */
+/* eslint-env node */
 // frcat - print out the contents of a flight recorder
 // NOTE: this only works on inactive recorder files where the writer has terminated
 
 import '@endo/init';
 
-import { makeMemoryMappedCircularBuffer } from './flight-recorder.js';
+import { makeSimpleCircularBuffer } from './flight-recorder.js';
 
 const main = async () => {
   const files = process.argv.slice(2);
@@ -14,8 +14,7 @@ const main = async () => {
   }
 
   for await (const file of files) {
-    // eslint-disable-next-line @jessie.js/no-nested-await
-    const { readCircBuf } = await makeMemoryMappedCircularBuffer({
+    const { readCircBuf } = await makeSimpleCircularBuffer({
       circularBufferFilename: file,
       circularBufferSize: 0,
     });
@@ -50,13 +49,18 @@ const main = async () => {
       }
 
       // If the buffer is full, wait for stdout to drain.
-      // eslint-disable-next-line no-await-in-loop, @jessie.js/no-nested-await
       await new Promise(resolve => process.stdout.once('drain', resolve));
     }
   }
 };
 
-main().catch(err => {
-  console.error(err);
-  process.exitCode = 1;
-});
+process.exitCode = 1;
+main().then(
+  () => {
+    process.exitCode = 0;
+  },
+  err => {
+    console.error('Failed with', err);
+    process.exit(process.exitCode || 1);
+  },
+);

package/src/index.js

@@ -13,6 +13,9 @@ export * from './make-slog-sender.js';
  *  shutdown?: () => Promise<void>;
  * }} SlogSender
  */
+/**
+ * @typedef {(opts: import('./index.js').MakeSlogSenderOptions) => SlogSender | undefined} MakeSlogSender
+ */
 /**
  * @typedef {MakeSlogSenderCommonOptions & Record<string, unknown>} MakeSlogSenderOptions
  * @typedef {object} MakeSlogSenderCommonOptions
@@ -34,9 +37,9 @@ export const tryFlushSlogSender = async (
   await Promise.resolve(slogSender?.forceFlush?.()).catch(err => {
     log?.('Failed to flush slog sender', err);
     if (err.errors) {
-      err.errors.forEach(error => {
+      for (const error of err.errors) {
         log?.('nested error:', error);
-      });
+      }
     }
     if (env.SLOGSENDER_FAIL_ON_ERROR) {
       throw err;
@@ -58,21 +61,20 @@ export const getResourceAttributes = ({
       SDK_REVISION;
   }
   if (!resourceAttributes[SemanticResourceAttributes.SERVICE_INSTANCE_ID]) {
-    resourceAttributes[
-      SemanticResourceAttributes.SERVICE_INSTANCE_ID
-    ] = `${Math.random()}`;
+    resourceAttributes[SemanticResourceAttributes.SERVICE_INSTANCE_ID] =
+      `${Math.random()}`;
   }
   if (serviceName) {
     resourceAttributes[SemanticResourceAttributes.SERVICE_NAME] = serviceName;
   }
   if (OTEL_RESOURCE_ATTRIBUTES) {
     // Allow overriding resource attributes.
-    OTEL_RESOURCE_ATTRIBUTES.split(',').forEach(kv => {
+    for (const kv of OTEL_RESOURCE_ATTRIBUTES.split(',')) {
       const match = kv.match(/^([^=]*)=(.*)$/);
       if (match) {
         resourceAttributes[match[1]] = match[2];
       }
-    });
+    }
   }
   return resourceAttributes;
 };

package/src/ingest-slog-entrypoint.js

@@ -1,5 +1,5 @@
 #! /usr/bin/env node
-/* global setTimeout */
+/* eslint-env node */
 import '@endo/init';
 
 import fs from 'fs';
@@ -11,7 +11,8 @@ import { makeSlogSender } from './make-slog-sender.js';
 
 const LINE_COUNT_TO_FLUSH = 10000;
 const ELAPSED_MS_TO_FLUSH = 3000;
-const MAX_LINE_COUNT_PER_PERIOD = 1000;
+const MAX_LINE_COUNT_PER_PERIOD = 10000;
+const MAX_BLOCKS_PER_PERIOD = 10;
 const PROCESSING_PERIOD = 1000;
 
 async function run() {
@@ -29,7 +30,7 @@ async function run() {
     return;
   }
 
-  const [slogFile] = args;
+  const slogFile = args[0] === '-' ? undefined : args[0];
   const slogSender = await makeSlogSender({
     serviceName,
     stateDir: '.',
@@ -56,14 +57,21 @@ async function run() {
   const lines = readline.createInterface({ input: slogF });
   const slogFileName = slogFile || '*stdin*';
 
-  const progressFileName = `${slogFileName}.ingest-progress`;
-  if (!fs.existsSync(progressFileName)) {
-    const progress = { virtualTimeOffset: 0, lastSlogTime: 0 };
-    fs.writeFileSync(progressFileName, JSON.stringify(progress));
+  const progressFileName = slogFile && `${slogFileName}.ingest-progress`;
+  const progress = { virtualTimeOffset: 0, lastSlogTime: 0 };
+  if (progressFileName) {
+    if (!fs.existsSync(progressFileName)) {
+      fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    } else {
+      Object.assign(
+        progress,
+        JSON.parse(fs.readFileSync(progressFileName).toString()),
+      );
+    }
   }
-  const progress = JSON.parse(fs.readFileSync(progressFileName).toString());
 
   let linesProcessedThisPeriod = 0;
+  let blocksInThisPeriod = 0;
   let startOfLastPeriod = 0;
 
   let lastTime = Date.now();
@@ -74,11 +82,13 @@ async function run() {
     if (!flush) {
       return;
     }
-    await slogSender.forceFlush();
-    fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    await slogSender.forceFlush?.();
+    if (progressFileName) {
+      fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    }
   };
 
-  console.log(`parsing`, slogFileName);
+  console.warn(`parsing`, slogFileName);
 
   let update = false;
   const maybeUpdateStats = async now => {
@@ -106,9 +116,14 @@ async function run() {
       continue;
     }
 
+    const isAfterCommit = obj.type === 'cosmic-swingset-after-commit-stats';
+
     // Maybe wait for the next period to process a bunch of lines.
     let maybeWait;
-    if (linesProcessedThisPeriod >= MAX_LINE_COUNT_PER_PERIOD) {
+    if (
+      linesProcessedThisPeriod >= MAX_LINE_COUNT_PER_PERIOD ||
+      blocksInThisPeriod >= MAX_BLOCKS_PER_PERIOD
+    ) {
       const delayMS = PROCESSING_PERIOD - (now - startOfLastPeriod);
       maybeWait = new Promise(resolve => setTimeout(resolve, delayMS));
     }
@@ -118,8 +133,8 @@ async function run() {
     if (now - startOfLastPeriod >= PROCESSING_PERIOD) {
       startOfLastPeriod = now;
       linesProcessedThisPeriod = 0;
+      blocksInThisPeriod = 0;
     }
-    linesProcessedThisPeriod += 1;
 
     if (progress.virtualTimeOffset) {
       const virtualTime = obj.time + progress.virtualTimeOffset;
@@ -133,14 +148,30 @@ async function run() {
       // Use the original.
       slogSender(obj);
     }
+
+    linesProcessedThisPeriod += 1;
+    if (isAfterCommit) {
+      blocksInThisPeriod += 1;
+      lastTime = Date.now();
+      await stats(true);
+    }
   }
 
   await stats(true);
-  console.log(
+  console.warn(
     `done parsing`,
     slogFileName,
     `(${lineCount} lines, ${byteCount} bytes)`,
   );
 }
 
-run().catch(err => console.log('err', err));
+process.exitCode = 1;
+run().then(
+  () => {
+    process.exitCode = 0;
+  },
+  err => {
+    console.error('Failed with', err);
+    process.exit(process.exitCode || 1);
+  },
+);

package/src/make-slog-sender.js

@@ -9,7 +9,7 @@ export const SLOGFILE_SENDER_MODULE = '@agoric/telemetry/src/slog-file.js';
 
 export const DEFAULT_SLOGSENDER_AGENT = 'self';
 
-/** @typedef {import('./index.js').SlogSender} SlogSender */
+/** @import {SlogSender} from './index.js' */
 
 /**
  * @template T
@@ -19,7 +19,7 @@ export const DEFAULT_SLOGSENDER_AGENT = 'self';
 const filterTruthy = arr => /** @type {any[]} */ (arr.filter(Boolean));
 
 /**
- * @param {import('./index.js').MakeSlogSenderOptions} opts
+ * @type {import('./index.js').MakeSlogSender}
  */
 export const makeSlogSender = async (opts = {}) => {
   const { env = {}, stateDir: stateDirOption, ...otherOpts } = opts;
@@ -88,7 +88,7 @@ export const makeSlogSender = async (opts = {}) => {
     slogSenderModules.map(async moduleIdentifier =>
       import(moduleIdentifier)
         .then(
-          /** @param {{makeSlogSender: (opts: {}) => Promise<SlogSender | undefined>}} module */ ({
+          /** @param {{makeSlogSender: import('./index.js').MakeSlogSender}} module */ ({
             makeSlogSender: maker,
           }) => {
             if (typeof maker !== 'function') {

package/src/otel-and-flight-recorder.js

@@ -1,6 +1,9 @@
-import { NonNullish } from '@agoric/assert';
+import { NonNullish } from '@agoric/internal';
 import { makeSlogSender as makeSlogSenderFromEnv } from './make-slog-sender.js';
 
+/**
+ * @param {import('./index.js').MakeSlogSenderOptions} opts
+ */
 export const makeSlogSender = async opts => {
   const { SLOGFILE: _1, SLOGSENDER: _2, ...otherEnv } = opts.env || {};