@agoric/swingset-vat 0.32.3-dev-0eaf7cc.0 → 0.32.3-dev-1f816f9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swingset-vat",
-  "version": "0.32.3-dev-0eaf7cc.0+0eaf7cc",
+  "version": "0.32.3-dev-1f816f9.0+1f816f9",
   "description": "Vat/Container Launcher",
   "type": "module",
   "main": "src/index.js",
@@ -27,16 +27,16 @@
     "@types/yargs-parser": "^21.0.0"
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/kmarshal": "0.1.1-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/store": "0.9.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/swing-store": "0.9.2-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/swingset-liveslots": "0.10.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/time": "0.3.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/vat-data": "0.5.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/xsnap": "0.14.3-dev-0eaf7cc.0+0eaf7cc",
-    "@agoric/xsnap-lockdown": "0.14.1-dev-0eaf7cc.0+0eaf7cc",
+    "@agoric/internal": "0.3.3-dev-1f816f9.0+1f816f9",
+    "@agoric/kmarshal": "0.1.1-dev-1f816f9.0+1f816f9",
+    "@agoric/store": "0.9.3-dev-1f816f9.0+1f816f9",
+    "@agoric/swing-store": "0.9.2-dev-1f816f9.0+1f816f9",
+    "@agoric/swingset-liveslots": "0.10.3-dev-1f816f9.0+1f816f9",
+    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-1f816f9.0+1f816f9",
+    "@agoric/time": "0.3.3-dev-1f816f9.0+1f816f9",
+    "@agoric/vat-data": "0.5.3-dev-1f816f9.0+1f816f9",
+    "@agoric/xsnap": "0.14.3-dev-1f816f9.0+1f816f9",
+    "@agoric/xsnap-lockdown": "0.14.1-dev-1f816f9.0+1f816f9",
     "@endo/base64": "^1.0.6",
     "@endo/bundle-source": "^3.3.0",
     "@endo/captp": "^4.2.2",
@@ -102,5 +102,5 @@
   "typeCoverage": {
     "atLeast": 75.1
   },
-  "gitHead": "0eaf7cc8151b4e1966f3f33e43362d3d021457aa"
+  "gitHead": "1f816f92e8b4f79c43df9608d7a7b443a3ceca20"
 }

package/src/controller/upgradeSwingset.js

@@ -196,6 +196,14 @@ export const upgradeSwingset = kernelStorage => {
     version = 1;
   }
 
+  if (version < 2) {
+    // schema v2: add vats.terminated = []
+    assert(!kvStore.has('vats.terminated'));
+    kvStore.set('vats.terminated', JSON.stringify([]));
+    modified = true;
+    version = 2;
+  }
+
   if (modified) {
     kvStore.set('version', `${version}`);
   }

package/src/kernel/kernel.js

@@ -2,6 +2,7 @@
 
 import { assert, Fail } from '@endo/errors';
 import { isNat } from '@endo/nat';
+import { mustMatch, M } from '@endo/patterns';
 import { importBundle } from '@endo/import-bundle';
 import { makeUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js';
 import { kser, kslot, makeError } from '@agoric/kmarshal';
@@ -275,12 +276,16 @@ export default function buildKernel(
       // (#9157). The fix will add .critical to CrankResults, populated by a
       // getOptions query in deliveryCrankResults() or copied from
       // dynamicOptions in processCreateVat.
-      critical = kernelKeeper.provideVatKeeper(vatID).getOptions().critical;
+      const vatKeeper = kernelKeeper.provideVatKeeper(vatID);
+      critical = vatKeeper.getOptions().critical;
 
       // Reject all promises decided by the vat, making sure to capture the list
       // of kpids before that data is deleted.
       const deadPromises = [...kernelKeeper.enumeratePromisesByDecider(vatID)];
-      kernelKeeper.cleanupAfterTerminatedVat(vatID);
+      // remove vatID from the list of live vats, and mark for deletion
+      kernelKeeper.deleteVatID(vatID);
+      kernelKeeper.markVatAsTerminated(vatID);
+      kernelKeeper.removeVatFromSwingStoreExports(vatID);
       for (const kpid of deadPromises) {
         resolveToError(kpid, makeError('vat terminated'), vatID);
       }
@@ -387,7 +392,8 @@ export default function buildKernel(
    *    abort?: boolean, // changes should be discarded, not committed
    *    consumeMessage?: boolean, // discard the aborted delivery
    *    didDelivery?: VatID, // we made a delivery to a vat, for run policy and save-snapshot
-   *    computrons?: BigInt, // computron count for run policy
+   *    computrons?: bigint, // computron count for run policy
+   *    cleanups?: number, // cleanup budget spent
    *    meterID?: string, // deduct those computrons from a meter
    *    measureDirt?: { vatID: VatID, dirt: Dirt }, // dirt counters should increment
    *    terminate?: { vatID: VatID, reject: boolean, info: SwingSetCapData }, // terminate vat, notify vat-admin
@@ -651,16 +657,40 @@ export default function buildKernel(
     if (!vatWarehouse.lookup(vatID)) {
       return NO_DELIVERY_CRANK_RESULTS; // can't collect from the dead
     }
-    const vatKeeper = kernelKeeper.provideVatKeeper(vatID);
     /** @type { KernelDeliveryBringOutYourDead } */
     const kd = harden([type]);
     const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd);
     const status = await deliverAndLogToVat(vatID, kd, vd);
-    vatKeeper.clearReapDirt(); // BOYD zeros out the when-to-BOYD counters
     // no gcKrefs, BOYD clears them anyways
     return deliveryCrankResults(vatID, status, false); // no meter, BOYD clears dirt
   }
 
+  /**
+   * Perform a small (budget-limited) amount of dead-vat cleanup work.
+   *
+   * @param {RunQueueEventCleanupTerminatedVat} message
+   *     'message' is the run-queue cleanup action, which includes a vatID and budget.
+   *     A budget of 'undefined' allows unlimited work. Otherwise, the budget is a Number,
+   *     and cleanup should not touch more than maybe 5*budget DB rows.
+   * @returns {Promise<CrankResults>}
+   */
+  async function processCleanupTerminatedVat(message) {
+    const { vatID, budget } = message;
+    const { done, cleanups } = kernelKeeper.cleanupAfterTerminatedVat(
+      vatID,
+      budget,
+    );
+    if (done) {
+      kernelKeeper.forgetTerminatedVat(vatID);
+      kernelSlog.write({ type: 'vat-cleanup-complete', vatID });
+    }
+    // We don't perform any deliveries here, so there are no computrons to
+    // report, but we do tell the runPolicy know how much kernel-side DB
+    // work we did, so it can decide how much was too much.
+    const computrons = 0n;
+    return harden({ computrons, cleanups });
+  }
+
   /**
    * The 'startVat' event is queued by `initializeKernel` for all static vats,
    * so that we execute their bundle imports and call their `buildRootObject`
@@ -916,7 +946,6 @@ export default function buildKernel(
     const boydVD = vatWarehouse.kernelDeliveryToVatDelivery(vatID, boydKD);
     const boydStatus = await deliverAndLogToVat(vatID, boydKD, boydVD);
     const boydResults = deliveryCrankResults(vatID, boydStatus, false);
-    vatKeeper.clearReapDirt();
 
     // we don't meter bringOutYourDead since no user code is running, but we
     // still report computrons to the runPolicy
@@ -1172,6 +1201,7 @@ export default function buildKernel(
    * @typedef { import('../types-internal.js').RunQueueEventRetireImports } RunQueueEventRetireImports
    * @typedef { import('../types-internal.js').RunQueueEventNegatedGCAction } RunQueueEventNegatedGCAction
    * @typedef { import('../types-internal.js').RunQueueEventBringOutYourDead } RunQueueEventBringOutYourDead
+   * @typedef { import('../types-internal.js').RunQueueEventCleanupTerminatedVat } RunQueueEventCleanupTerminatedVat
    * @typedef { import('../types-internal.js').RunQueueEvent } RunQueueEvent
    */
 
@@ -1239,6 +1269,8 @@ export default function buildKernel(
     } else if (message.type === 'negated-gc-action') {
       // processGCActionSet pruned some negated actions, but had no GC
       // action to perform. Record the DB changes in their own crank.
+    } else if (message.type === 'cleanup-terminated-vat') {
+      deliverP = processCleanupTerminatedVat(message);
     } else if (gcMessages.includes(message.type)) {
       deliverP = processGCMessage(message);
     } else {
@@ -1289,18 +1321,13 @@ export default function buildKernel(
     const crankResults = await deliverRunQueueEvent(message);
     // { abort/commit, deduct, terminate+notify, consumeMessage }
 
-    if (crankResults.didDelivery) {
-      if (message.type === 'create-vat') {
-        // TODO: create-vat now gets metering, at least for the
-        // dispatch.startVat . We should probably tell the policy about
-        // the creation too since there's extra overhead (we're
-        // launching a new child process, at least, although that
-        // sometimes happens randomly because of vat eviction policy
-        // which should not affect the in-consensus policyInput)
-        policyInput = ['create-vat', {}];
-      } else {
-        policyInput = ['crank', {}];
-      }
+    if (message.type === 'cleanup-terminated-vat') {
+      const { cleanups } = crankResults;
+      assert(cleanups !== undefined);
+      policyInput = ['cleanup', { cleanups }];
+    } else if (crankResults.didDelivery) {
+      const tag = message.type === 'create-vat' ? 'create-vat' : 'crank';
+      policyInput = [tag, {}];
     }
 
     // Deliveries cause syscalls, syscalls might cause errors
@@ -1758,16 +1785,24 @@ export default function buildKernel(
     }
   }
 
+  const allowCleanupShape = M.or(
+    // 'false' will prohibit cleanup
+    false,
+    // otherwise allow cleanup, optionally with a limiting budget
+    M.splitRecord({}, { budget: M.number() }, M.record()),
+  );
+
   /**
    * Pulls the next message from the highest-priority queue and returns it
    * along with a corresponding processor.
    *
+   * @param {RunPolicy} [policy] - a RunPolicy to limit the work being done
    * @returns {{
    *   message: RunQueueEvent | undefined,
    *   processor: (message: RunQueueEvent) => Promise<PolicyInput>,
    * }}
    */
-  function getNextMessageAndProcessor() {
+  function getNextMessageAndProcessor(policy) {
     const acceptanceMessage = kernelKeeper.getNextAcceptanceQueueMsg();
     if (acceptanceMessage) {
       return {
@@ -1775,7 +1810,12 @@ export default function buildKernel(
         processor: processAcceptanceMessage,
       };
     }
+    // Absent specific configuration, allow unlimited cleanup.
+    const allowCleanup = policy?.allowCleanup?.() ?? {};
+    mustMatch(harden(allowCleanup), allowCleanupShape);
+
     const message =
+      kernelKeeper.nextCleanupTerminatedVatAction(allowCleanup) ||
       processGCActionSet(kernelKeeper) ||
       kernelKeeper.nextReapAction() ||
       kernelKeeper.getNextRunQueueMsg();
@@ -1885,7 +1925,7 @@ export default function buildKernel(
       kernelKeeper.startCrank();
       try {
         kernelKeeper.establishCrankSavepoint('start');
-        const { processor, message } = getNextMessageAndProcessor();
+        const { processor, message } = getNextMessageAndProcessor(policy);
         if (!message) {
           break;
         }
@@ -1907,6 +1947,13 @@ export default function buildKernel(
           case 'crank-failed':
             policyOutput = policy.crankFailed(policyInput[1]);
             break;
+          case 'cleanup': {
+            // Give the policy a chance to interrupt kernel execution,
+            // but default to continuing.
+            const { didCleanup } = policy;
+            policyOutput = didCleanup ? didCleanup(policyInput[1]) : true;
+            break;
+          }
           case 'none':
             policyOutput = policy.emptyCrank();
             break;

package/src/kernel/state/kernelKeeper.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-use-before-define */
 import { Nat, isNat } from '@endo/nat';
 import { assert, Fail } from '@endo/errors';
+import { objectMetaMap } from '@agoric/internal';
 import {
   initializeVatState,
   makeVatKeeper,
@@ -50,7 +51,7 @@ const enableKernelGC = true;
 export { DEFAULT_REAP_DIRT_THRESHOLD_KEY };
 
 // most recent DB schema version
-export const CURRENT_SCHEMA_VERSION = 1;
+export const CURRENT_SCHEMA_VERSION = 2;
 
 // Kernel state lives in a key-value store supporting key retrieval by
 // lexicographic range. All keys and values are strings.
@@ -69,14 +70,15 @@ export const CURRENT_SCHEMA_VERSION = 1;
 // only modified by a call to upgradeSwingset(). See below for
 // deltas/upgrades from one version to the next.
 //
-// The current ("v1") schema keys/values are:
+// The current ("v2") schema keys/values are:
 //
-// version = '1'
+// version = '2'
 // vat.names = JSON([names..])
 // vat.dynamicIDs = JSON([vatIDs..])
 // vat.name.$NAME = $vatID = v$NN
 // vat.nextID = $NN
 // vat.nextUpgradeID = $NN
+// vats.terminated = JSON([vatIDs..])
 // device.names = JSON([names..])
 // device.name.$NAME = $deviceID = d$NN
 // device.nextID = $NN
@@ -172,6 +174,9 @@ export const CURRENT_SCHEMA_VERSION = 1;
 //   * replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold`
 //   * replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt`
 //             and a `vNN.reapDirtThreshold` in `vNN.options`
+// v2:
+//   * change `version` to `'2'`
+//   * add `vats.terminated` with `[]` as initial value
 
 export function commaSplit(s) {
   if (s === '') {
@@ -180,6 +185,11 @@ export function commaSplit(s) {
   return s.split(',');
 }
 
+export function stripPrefix(prefix, str) {
+  assert(str.startsWith(prefix), str);
+  return str.slice(prefix.length);
+}
+
 function insistMeterID(m) {
   assert.typeof(m, 'string');
   assert.equal(m[0], 'm');
@@ -248,6 +258,12 @@ export default function makeKernelKeeper(
 
   insistStorageAPI(kvStore);
 
+  // the terminated-vats cache is normally populated from
+  // 'vats.terminated', but for initialization purposes we need give
+  // it a value here, and then populate it for real if we're dealing
+  // with an up-to-date DB
+  let terminatedVats = [];
+
   const versionString = kvStore.get('version');
   const version = Number(versionString || '0');
   if (expectedVersion === 'uninitialized') {
@@ -261,6 +277,9 @@ export default function makeKernelKeeper(
     throw Error(
       `kernel DB is too old: has version v${version}, but expected v${expectedVersion}`,
     );
+  } else {
+    // DB is up-to-date, so populate any caches we use
+    terminatedVats = JSON.parse(getRequired('vats.terminated'));
   }
 
   /**
@@ -399,6 +418,7 @@ export default function makeKernelKeeper(
     kvStore.set('vat.dynamicIDs', '[]');
     kvStore.set('vat.nextID', `${FIRST_VAT_ID}`);
     kvStore.set('vat.nextUpgradeID', `1`);
+    kvStore.set('vats.terminated', '[]');
     kvStore.set('device.names', '[]');
     kvStore.set('device.nextID', `${FIRST_DEVICE_ID}`);
     kvStore.set('ko.nextID', `${FIRST_OBJECT_ID}`);
@@ -679,8 +699,12 @@ export default function makeKernelKeeper(
   function ownerOfKernelObject(kernelSlot) {
     insistKernelType('object', kernelSlot);
     const owner = kvStore.get(`${kernelSlot}.owner`);
-    if (owner) {
-      insistVatID(owner);
+    if (!owner) {
+      return undefined;
+    }
+    insistVatID(owner);
+    if (terminatedVats.includes(owner)) {
+      return undefined;
     }
     return owner;
   }
@@ -885,13 +909,70 @@ export default function makeKernelKeeper(
     kvStore.set(`${kernelSlot}.data.slots`, capdata.slots.join(','));
   }
 
-  function cleanupAfterTerminatedVat(vatID) {
+  function removeVatFromSwingStoreExports(vatID) {
+    // Delete primary swingstore records for this vat, in preparation
+    // for (slow) deletion. After this, swingstore exports will omit
+    // this vat. This is called from the kernel's terminateVat, which
+    // initiates (but does not complete) deletion.
+    snapStore.stopUsingLastSnapshot(vatID);
+    transcriptStore.stopUsingTranscript(vatID);
+  }
+
+  /**
+   * Perform some cleanup work for a specific (terminated but not
+   * fully-deleted) vat, possibly limited by a budget. Returns 'done'
+   * (where false means "please call me again", and true means "you
+   * can delete the vatID now"), and a count of how much work was done
+   * (so the runPolicy can decide when to stop).
+   *
+   * @param {string} vatID
+   * @param {number} [budget]
+   * @returns {{ done: boolean, cleanups: number }}
+   *
+   */
+  function cleanupAfterTerminatedVat(vatID, budget = undefined) {
     insistVatID(vatID);
-    const vatKeeper = provideVatKeeper(vatID);
-    const exportPrefix = `${vatID}.c.o+`;
-    const importPrefix = `${vatID}.c.o-`;
+    let cleanups = 0;
+    const work = {
+      exports: 0,
+      imports: 0,
+      kv: 0,
+      snapshots: 0,
+      transcripts: 0,
+    };
+    let spend = _did => false; // returns "stop now"
+    if (budget !== undefined) {
+      assert.typeof(budget, 'number');
+      spend = (did = 1) => {
+        assert(budget !== undefined); // hush TSC
+        cleanups += did;
+        budget -= did;
+        return budget <= 0;
+      };
+    }
+    const logWork = () => {
+      const w = objectMetaMap(work, desc => (desc.value ? desc : undefined));
+      kernelSlog?.write({ type: 'vat-cleanup', vatID, work: w });
+    };
+
+    // TODO: it would be slightly cheaper to walk all kvStore keys in
+    // order, and act on each one according to its category (c-list
+    // export, c-list import, vatstore, other), so we use a single
+    // enumeratePrefixedKeys() call each time. Until we do that, the
+    // last phase of the cleanup (where we've deleted all the exports
+    // and imports, and are working on the remaining keys) will waste
+    // two DB queries on each call. OTOH, those queries will probably
+    // hit the same SQLite index page as the successful one, so it
+    // probably won't cause any extra disk IO. So we can defer this
+    // optimization for a while. Note: when we implement it, be
+    // prepared to encounter the clist entries in eiher order (kref
+    // first or vref first), and delete the other one in the same
+    // call, so we don't wind up with half an entry.
 
-    vatKeeper.deleteSnapshotsAndTranscript();
+    const vatKeeper = provideVatKeeper(vatID);
+    const clistPrefix = `${vatID}.c.`;
+    const exportPrefix = `${clistPrefix}o+`;
+    const importPrefix = `${clistPrefix}o-`;
 
     // Note: ASCII order is "+,-./", and we rely upon this to split the
     // keyspace into the various o+NN/o-NN/etc spaces. If we were using a
@@ -915,8 +996,27 @@ export default function makeKernelKeeper(
       // begin with `vMM.c.o+`.  In addition to deleting the c-list entry, we
       // must also delete the corresponding kernel owner entry for the object,
       // since the object will no longer be accessible.
+      const vref = stripPrefix(clistPrefix, k);
+      assert(vref.startsWith('o+'), vref);
       const kref = kvStore.get(k);
-      orphanKernelObject(kref, vatID);
+      // we must delete the c-list entry, but we don't need to
+      // manipulate refcounts like the way vatKeeper/deleteCListEntry
+      // does, so just delete the keys directly
+      // vatKeeper.deleteCListEntry(kref, vref);
+      kvStore.delete(`${vatID}.c.${kref}`);
+      kvStore.delete(`${vatID}.c.${vref}`);
+      // if this object became unreferenced, processRefcounts will
+      // delete it, so don't be surprised. TODO: this results in an
+      // extra get() for each delete(), see if there's a way to avoid
+      // this. TODO: think carefully about other such races.
+      if (kvStore.has(`${kref}.owner`)) {
+        orphanKernelObject(kref, vatID);
+      }
+      work.exports += 1;
+      if (spend()) {
+        logWork();
+        return { done: false, cleanups };
+      }
     }
 
     // then scan for imported objects, which must be decrefed
@@ -924,9 +1024,14 @@ export default function makeKernelKeeper(
       // abandoned imports: delete the clist entry as if the vat did a
       // drop+retire
       const kref = kvStore.get(k) || Fail`getNextKey ensures get`;
-      const vref = k.slice(`${vatID}.c.`.length);
+      const vref = stripPrefix(clistPrefix, k);
       vatKeeper.deleteCListEntry(kref, vref);
       // that will also delete both db keys
+      work.imports += 1;
+      if (spend()) {
+        logWork();
+        return { done: false, cleanups };
+      }
     }
 
     // the caller used enumeratePromisesByDecider() before calling us,
@@ -935,8 +1040,31 @@ export default function makeKernelKeeper(
     // now loop back through everything and delete it all
     for (const k of enumeratePrefixedKeys(kvStore, `${vatID}.`)) {
       kvStore.delete(k);
+      work.kv += 1;
+      if (spend()) {
+        logWork();
+        return { done: false, cleanups };
+      }
+    }
+
+    // this will internally loop through 'budget' deletions
+    const dsc = vatKeeper.deleteSnapshots(budget);
+    work.snapshots += dsc.cleanups;
+    if (spend(dsc.cleanups)) {
+      logWork();
+      return { done: false, cleanups };
     }
 
+    // same
+    const dts = vatKeeper.deleteTranscripts(budget);
+    work.transcripts += dts.cleanups;
+    // last task, so increment cleanups, but dc.done is authoritative
+    spend(dts.cleanups);
+    logWork();
+    return { done: dts.done, cleanups };
+  }
+
+  function deleteVatID(vatID) {
     // TODO: deleting entries from the dynamic vat IDs list requires a linear
     // scan of the list; arguably this collection ought to be represented in a
     // different way that makes it efficient to remove an entry from it, though
@@ -1221,6 +1349,41 @@ export default function makeKernelKeeper(
     return makeUpgradeID(nextID);
   }
 
+  function markVatAsTerminated(vatID) {
+    if (!terminatedVats.includes(vatID)) {
+      terminatedVats.push(vatID);
+      kvStore.set(`vats.terminated`, JSON.stringify(terminatedVats));
+    }
+  }
+
+  function getFirstTerminatedVat() {
+    if (terminatedVats.length) {
+      return terminatedVats[0];
+    }
+    return undefined;
+  }
+
+  function forgetTerminatedVat(vatID) {
+    terminatedVats = terminatedVats.filter(id => id !== vatID);
+    kvStore.set(`vats.terminated`, JSON.stringify(terminatedVats));
+  }
+
+  function nextCleanupTerminatedVatAction(allowCleanup) {
+    if (!allowCleanup) {
+      return undefined;
+    }
+    // budget === undefined means "unlimited"
+    const { budget } = allowCleanup;
+    // if (getGCActions().size) {
+    //  return undefined;
+    // }
+    const vatID = getFirstTerminatedVat();
+    if (vatID) {
+      return { type: 'cleanup-terminated-vat', vatID, budget };
+    }
+    return undefined;
+  }
+
   // As refcounts are decremented, we accumulate a set of krefs for which
   // action might need to be taken:
   //   * promises which are now resolved and unreferenced can be deleted
@@ -1413,7 +1576,7 @@ export default function makeKernelKeeper(
 
   function vatIsAlive(vatID) {
     insistVatID(vatID);
-    return kvStore.has(`${vatID}.o.nextID`);
+    return kvStore.has(`${vatID}.o.nextID`) && !terminatedVats.includes(vatID);
   }
 
   /**
@@ -1705,11 +1868,18 @@ export default function makeKernelKeeper(
     provideVatKeeper,
     vatIsAlive,
     evictVatKeeper,
+    removeVatFromSwingStoreExports,
     cleanupAfterTerminatedVat,
     addDynamicVatID,
     getDynamicVats,
     getStaticVats,
     getDevices,
+    deleteVatID,
+
+    markVatAsTerminated,
+    getFirstTerminatedVat,
+    forgetTerminatedVat,
+    nextCleanupTerminatedVatAction,
 
     allocateUpgradeID,
 

package/src/kernel/state/vatKeeper.js

@@ -675,11 +675,44 @@ export function makeVatKeeper(
     });
   }
 
-  function deleteSnapshotsAndTranscript() {
-    if (snapStore) {
-      snapStore.deleteVatSnapshots(vatID);
+  /**
+   * Perform some (possibly-limited) cleanup work for a vat. Returns
+   * 'done' (where false means "please call me again", and true means
+   * "you can delete the vatID now"), and a count of how much work was
+   * done (so the runPolicy can decide when to stop).
+   *
+   * @param {number} [budget]
+   * @returns {{ done: boolean, cleanups: number }}
+   *
+   */
+  function deleteSnapshots(budget = undefined) {
+    // Each budget=1 allows us to delete one snapshot entry.
+    if (!snapStore) {
+      return { done: true, cleanups: 0 };
     }
-    transcriptStore.deleteVatTranscripts(vatID);
+    // initially uses 2+2*budget DB statements, then just 1 when done
+    return snapStore.deleteVatSnapshots(vatID, budget);
+  }
+
+  /**
+   * Perform some (possibly-limited) cleanup work for a vat. Returns
+   * 'done' (where false means "please call me again", and true means
+   * "you can delete the vatID now"), and a count of how much work was
+   * done (so the runPolicy can decide when to stop).
+   *
+   * @param {number} [budget]
+   * @returns {{ done: boolean, cleanups: number }}
+   *
+   */
+  function deleteTranscripts(budget = undefined) {
+    // Each budget=1 allows us to delete one transcript span and any
+    // transcript items associated with that span. Some nodes will
+    // have historical transcript items, some will not. Using budget=5
+    // and snapshotInterval=200 means we delete 5 span records and
+    // maybe 1000 span items.
+
+    // initially uses 2+3*budget DB statements, then just 1 when done
+    return transcriptStore.deleteVatTranscripts(vatID, budget);
   }
 
   function beginNewIncarnation() {
@@ -761,7 +794,8 @@ export function makeVatKeeper(
     dumpState,
     saveSnapshot,
     getSnapshotInfo,
-    deleteSnapshotsAndTranscript,
+    deleteSnapshots,
+    deleteTranscripts,
     beginNewIncarnation,
   });
 }

package/src/kernel/vat-warehouse.js

@@ -544,6 +544,10 @@ export function makeVatWarehouse({
       vatKeeper.addToTranscript(getTranscriptEntry(vd, deliveryResult));
     }
 
+    if (kd[0] === 'bringOutYourDead') {
+      vatKeeper.clearReapDirt(); // BOYD zeros out the when-to-BOYD counters
+    }
+
     // TODO: if per-vat policy decides it wants a BOYD or heap snapshot,
     // now is the time to do it, or to ask the kernel to schedule it
 

package/src/lib/runPolicies.js

@@ -3,6 +3,9 @@ import { assert } from '@endo/errors';
 export function foreverPolicy() {
   /** @type { RunPolicy } */
   return harden({
+    allowCleanup() {
+      return {}; // unlimited budget
+    },
     vatCreated(_details) {
       return true;
     },
@@ -27,6 +30,9 @@ export function crankCounter(
   let vats = 0;
   /** @type { RunPolicy } */
   const policy = harden({
+    allowCleanup() {
+      return { budget: 100 }; // limited budget
+    },
     vatCreated() {
       vats += 1;
       return vats < maxCreateVats;
@@ -47,13 +53,21 @@ export function crankCounter(
   return policy;
 }
 
-export function computronCounter(limit) {
+export function computronCounter(limit, options = {}) {
   assert.typeof(limit, 'bigint');
+  const {
+    cleanupBudget = 100,
+    vatCreatedComputrons = 100_000n, // pretend that's the cost
+    crankFailedComputrons = 1_000_000n,
+  } = options;
   let total = 0n;
   /** @type { RunPolicy } */
   const policy = harden({
+    allowCleanup() {
+      return { budget: cleanupBudget }; // limited budget
+    },
     vatCreated() {
-      total += 100000n; // pretend vat creation takes 100k computrons
+      total += vatCreatedComputrons;
       return total < limit;
     },
     crankComplete(details = {}) {
@@ -65,7 +79,7 @@ export function computronCounter(limit) {
       return total < limit;
     },
     crankFailed() {
-      total += 1000000n; // who knows, 1M is as good as anything
+      total += crankFailedComputrons;
       return total < limit;
     },
     emptyCrank() {
@@ -79,6 +93,7 @@ export function wallClockWaiter(seconds) {
   const timeout = Date.now() + 1000 * seconds;
   /** @type { RunPolicy } */
   const policy = harden({
+    allowCleanup: () => ({}), // unlimited budget
     vatCreated: () => Date.now() < timeout,
     crankComplete: () => Date.now() < timeout,
     crankFailed: () => Date.now() < timeout,
@@ -86,3 +101,34 @@ export function wallClockWaiter(seconds) {
   });
   return policy;
 }
+
+export function noCleanup() {
+  /** @type { RunPolicy } */
+  const policy = harden({
+    allowCleanup: () => false,
+    vatCreated: () => true,
+    crankComplete: () => true,
+    crankFailed: () => true,
+    emptyCrank: () => true,
+  });
+  return policy;
+}
+
+export function someCleanup(budget) {
+  let once = true;
+  /** @type { RunPolicy } */
+  const policy = harden({
+    allowCleanup: () => {
+      if (once) {
+        once = false;
+        return { budget };
+      }
+      return false;
+    },
+    vatCreated: () => true,
+    crankComplete: () => true,
+    crankFailed: () => true,
+    emptyCrank: () => true,
+  });
+  return policy;
+}

package/src/types-external.js

@@ -224,12 +224,17 @@ export {};
  * @typedef { [tag: 'create-vat', details: PolicyInputDetails  ]} PolicyInputCreateVat
  * @typedef { [tag: 'crank', details: PolicyInputDetails ] } PolicyInputCrankComplete
  * @typedef { [tag: 'crank-failed', details: PolicyInputDetails ]} PolicyInputCrankFailed
- * @typedef { PolicyInputNone | PolicyInputCreateVat | PolicyInputCrankComplete | PolicyInputCrankFailed } PolicyInput
+ * @typedef { [tag: 'cleanup', details: { cleanups: number, computrons?: number }] } PolicyInputCleanup
+ * @typedef { PolicyInputNone | PolicyInputCreateVat | PolicyInputCrankComplete |
+ *            PolicyInputCrankFailed | PolicyInputCleanup } PolicyInput
  * @typedef { boolean } PolicyOutput
- * @typedef { { vatCreated: (details: {}) => PolicyOutput,
+ * @typedef { {
+ *              allowCleanup?: () => false | { budget?: number },
+ *              vatCreated: (details: {}) => PolicyOutput,
  *              crankComplete: (details: { computrons?: bigint }) => PolicyOutput,
  *              crankFailed: (details: {}) => PolicyOutput,
  *              emptyCrank: () => PolicyOutput,
+ *              didCleanup?: (details: { cleanups: number }) => PolicyOutput,
  *             } } RunPolicy
  *
  * @typedef {object} VatWarehousePolicy

package/src/types-internal.js

@@ -143,10 +143,12 @@ export {};
  * @typedef { { type: 'retireImports', vatID: VatID, krefs: string[] } } RunQueueEventRetireImports
  * @typedef { { type: 'negated-gc-action', vatID?: VatID } } RunQueueEventNegatedGCAction
  * @typedef { { type: 'bringOutYourDead', vatID: VatID } } RunQueueEventBringOutYourDead
+ * @typedef { { type: 'cleanup-terminated-vat', vatID: VatID,
+ *              budget: number | undefined } } RunQueueEventCleanupTerminatedVat
  * @typedef { RunQueueEventNotify | RunQueueEventSend | RunQueueEventCreateVat |
  *            RunQueueEventUpgradeVat | RunQueueEventChangeVatOptions | RunQueueEventStartVat |
  *            RunQueueEventDropExports | RunQueueEventRetireExports | RunQueueEventRetireImports |
- *            RunQueueEventNegatedGCAction | RunQueueEventBringOutYourDead
+ *            RunQueueEventNegatedGCAction | RunQueueEventBringOutYourDead | RunQueueEventCleanupTerminatedVat
  *          } RunQueueEvent
  */