@agoric/telemetry 0.6.3-dev-2ede8c1.0 → 0.6.3-dev-7ae1f27.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/telemetry",
-  "version": "0.6.3-dev-2ede8c1.0+2ede8c1",
+  "version": "0.6.3-dev-7ae1f27.0+7ae1f27",
   "description": "Agoric's telemetry implementation",
   "type": "module",
   "repository": "https://github.com/Agoric/agoric-sdk",
@@ -22,8 +22,8 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-2ede8c1.0+2ede8c1",
-    "@agoric/store": "0.9.3-dev-2ede8c1.0+2ede8c1",
+    "@agoric/internal": "0.3.3-dev-7ae1f27.0+7ae1f27",
+    "@agoric/store": "0.9.3-dev-7ae1f27.0+7ae1f27",
     "@endo/errors": "^1.2.7",
     "@endo/init": "^1.1.6",
     "@endo/marshal": "^1.6.1",
@@ -65,5 +65,5 @@
   "typeCoverage": {
     "atLeast": 87.03
   },
-  "gitHead": "2ede8c1cb2234c3e35cc78d88501aa840af3a903"
+  "gitHead": "7ae1f278fa8cbeb0cfc777b7cebf507b1f07c958"
 }

package/src/ingest-slog-entrypoint.js

@@ -11,7 +11,8 @@ import { makeSlogSender } from './make-slog-sender.js';
 
 const LINE_COUNT_TO_FLUSH = 10000;
 const ELAPSED_MS_TO_FLUSH = 3000;
-const MAX_LINE_COUNT_PER_PERIOD = 1000;
+const MAX_LINE_COUNT_PER_PERIOD = 10000;
+const MAX_BLOCKS_PER_PERIOD = 10;
 const PROCESSING_PERIOD = 1000;
 
 async function run() {
@@ -29,7 +30,7 @@ async function run() {
     return;
   }
 
-  const [slogFile] = args;
+  const slogFile = args[0] === '-' ? undefined : args[0];
   const slogSender = await makeSlogSender({
     serviceName,
     stateDir: '.',
@@ -56,14 +57,21 @@ async function run() {
   const lines = readline.createInterface({ input: slogF });
   const slogFileName = slogFile || '*stdin*';
 
-  const progressFileName = `${slogFileName}.ingest-progress`;
-  if (!fs.existsSync(progressFileName)) {
-    const progress = { virtualTimeOffset: 0, lastSlogTime: 0 };
-    fs.writeFileSync(progressFileName, JSON.stringify(progress));
+  const progressFileName = slogFile && `${slogFileName}.ingest-progress`;
+  const progress = { virtualTimeOffset: 0, lastSlogTime: 0 };
+  if (progressFileName) {
+    if (!fs.existsSync(progressFileName)) {
+      fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    } else {
+      Object.assign(
+        progress,
+        JSON.parse(fs.readFileSync(progressFileName).toString()),
+      );
+    }
   }
-  const progress = JSON.parse(fs.readFileSync(progressFileName).toString());
 
   let linesProcessedThisPeriod = 0;
+  let blocksInThisPeriod = 0;
   let startOfLastPeriod = 0;
 
   let lastTime = Date.now();
@@ -75,10 +83,12 @@ async function run() {
       return;
     }
     await slogSender.forceFlush?.();
-    fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    if (progressFileName) {
+      fs.writeFileSync(progressFileName, JSON.stringify(progress));
+    }
   };
 
-  console.log(`parsing`, slogFileName);
+  console.warn(`parsing`, slogFileName);
 
   let update = false;
   const maybeUpdateStats = async now => {
@@ -106,9 +116,14 @@ async function run() {
       continue;
     }
 
+    const isAfterCommit = obj.type === 'cosmic-swingset-after-commit-stats';
+
     // Maybe wait for the next period to process a bunch of lines.
     let maybeWait;
-    if (linesProcessedThisPeriod >= MAX_LINE_COUNT_PER_PERIOD) {
+    if (
+      linesProcessedThisPeriod >= MAX_LINE_COUNT_PER_PERIOD ||
+      blocksInThisPeriod >= MAX_BLOCKS_PER_PERIOD
+    ) {
       const delayMS = PROCESSING_PERIOD - (now - startOfLastPeriod);
       maybeWait = new Promise(resolve => setTimeout(resolve, delayMS));
     }
@@ -118,8 +133,8 @@ async function run() {
     if (now - startOfLastPeriod >= PROCESSING_PERIOD) {
       startOfLastPeriod = now;
       linesProcessedThisPeriod = 0;
+      blocksInThisPeriod = 0;
     }
-    linesProcessedThisPeriod += 1;
 
     if (progress.virtualTimeOffset) {
       const virtualTime = obj.time + progress.virtualTimeOffset;
@@ -133,10 +148,17 @@ async function run() {
       // Use the original.
       slogSender(obj);
     }
+
+    linesProcessedThisPeriod += 1;
+    if (isAfterCommit) {
+      blocksInThisPeriod += 1;
+      lastTime = Date.now();
+      await stats(true);
+    }
   }
 
   await stats(true);
-  console.log(
+  console.warn(
     `done parsing`,
     slogFileName,
     `(${lineCount} lines, ${byteCount} bytes)`,

package/src/slog-file.js

@@ -11,7 +11,7 @@ export const makeSlogSender = async ({ env: { SLOGFILE } = {} } = {}) => {
 
   const slogSender = (slogObj, jsonObj = serializeSlogObj(slogObj)) => {
     // eslint-disable-next-line prefer-template
-    void stream.write(jsonObj + '\n');
+    stream.write(jsonObj + '\n').catch(() => {});
   };
 
   return Object.assign(slogSender, {