@agoric/swingset-vat 0.32.3-dev-b4480d8.0 → 0.32.3-dev-ad7ac3c.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agoric/swingset-vat",
-  "version": "0.32.3-dev-b4480d8.0+b4480d8",
+  "version": "0.32.3-dev-ad7ac3c.0+ad7ac3c",
   "description": "Vat/Container Launcher",
   "type": "module",
   "main": "src/index.js",
@@ -27,16 +27,16 @@
     "@types/yargs-parser": "^21.0.0"
   },
   "dependencies": {
-    "@agoric/internal": "0.3.3-dev-b4480d8.0+b4480d8",
-    "@agoric/kmarshal": "0.1.1-dev-b4480d8.0+b4480d8",
-    "@agoric/store": "0.9.3-dev-b4480d8.0+b4480d8",
-    "@agoric/swing-store": "0.9.2-dev-b4480d8.0+b4480d8",
-    "@agoric/swingset-liveslots": "0.10.3-dev-b4480d8.0+b4480d8",
-    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-b4480d8.0+b4480d8",
-    "@agoric/time": "0.3.3-dev-b4480d8.0+b4480d8",
-    "@agoric/vat-data": "0.5.3-dev-b4480d8.0+b4480d8",
-    "@agoric/xsnap": "0.14.3-dev-b4480d8.0+b4480d8",
-    "@agoric/xsnap-lockdown": "0.14.1-dev-b4480d8.0+b4480d8",
+    "@agoric/internal": "0.3.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/kmarshal": "0.1.1-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/store": "0.9.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/swing-store": "0.9.2-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/swingset-liveslots": "0.10.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/swingset-xsnap-supervisor": "0.10.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/time": "0.3.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/vat-data": "0.5.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/xsnap": "0.14.3-dev-ad7ac3c.0+ad7ac3c",
+    "@agoric/xsnap-lockdown": "0.14.1-dev-ad7ac3c.0+ad7ac3c",
     "@endo/base64": "^1.0.8",
     "@endo/bundle-source": "^3.4.2",
     "@endo/captp": "^4.4.2",
@@ -103,5 +103,5 @@
   "typeCoverage": {
     "atLeast": 75.82
   },
-  "gitHead": "b4480d8443ec4864a2deda7f67e186a7ff8e73ea"
+  "gitHead": "ad7ac3c0b5722cecd8bed5c3c554499a39a1d519"
 }

package/src/controller/controller.js

@@ -366,6 +366,10 @@ export async function makeSwingsetController(
       return kref;
     },
 
+    kpRegisterInterest(kpid) {
+      return kernel.kpRegisterInterest(kpid);
+    },
+
     kpStatus(kpid) {
       return kernel.kpStatus(kpid);
     },
@@ -384,6 +388,8 @@ export async function makeSwingsetController(
       return kernel.deviceNameToID(deviceName);
     },
 
+    injectQueuedUpgradeEvents: () => kernel.injectQueuedUpgradeEvents(),
+
     /**
      * Queue a method call into the named vat
      *

package/src/controller/upgradeSwingset.js

@@ -1,9 +1,17 @@
+import { Fail } from '@endo/errors';
 import {
   DEFAULT_REAP_DIRT_THRESHOLD_KEY,
   DEFAULT_GC_KREFS_PER_BOYD,
   getAllDynamicVats,
   getAllStaticVats,
+  incrementReferenceCount,
+  readQueue,
 } from '../kernel/state/kernelKeeper.js';
+import { enumeratePrefixedKeys } from '../kernel/state/storageHelper.js';
+
+/**
+ * @import {RunQueueEvent} from '../types-internal.js';
+ */
 
 const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => {
   // This is called, once per vat, when upgradeSwingset migrates from
@@ -92,11 +100,13 @@ const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => {
  * `hostStorage.commit()` afterwards.
  *
  * @param {SwingStoreKernelStorage} kernelStorage
- * @returns {boolean} true if any changes were made
+ * @returns {{ modified: boolean }}
  */
 export const upgradeSwingset = kernelStorage => {
   const { kvStore } = kernelStorage;
   let modified = false;
+  /** @type {RunQueueEvent[]} */
+  const upgradeEvents = [];
   let vstring = kvStore.get('version');
   if (vstring === undefined) {
     vstring = '0';
@@ -204,9 +214,129 @@ export const upgradeSwingset = kernelStorage => {
     version = 2;
   }
 
+  if (version < 3) {
+    // v3 means that we've completed remediation for bug #9039
+    console.log(`Starting remediation of bug #9039`);
+
+    // find all terminated vats
+    const terminated = new Set(JSON.parse(getRequired('vats.terminated')));
+
+    // find all live vats
+    const allVatIDs = [];
+    for (const [_name, vatID] of getAllStaticVats(kvStore)) {
+      if (!terminated.has(vatID)) {
+        allVatIDs.push(vatID);
+      }
+    }
+    for (const vatID of getAllDynamicVats(getRequired)) {
+      if (!terminated.has(vatID)) {
+        allVatIDs.push(vatID);
+      }
+    }
+
+    // find all pending notifies
+    const notifies = new Map(); // .get(kpid) = [vatIDs..];
+    for (const name of ['runQueue', 'acceptanceQueue']) {
+      for (const rq of readQueue(name, getRequired)) {
+        if (rq.type === 'notify') {
+          const { vatID, kpid } = rq;
+          assert(vatID);
+          assert(kpid);
+          let vats = notifies.get(kpid);
+          if (!vats) {
+            vats = [];
+            notifies.set(kpid, vats);
+          }
+          vats.push(vatID);
+        }
+      }
+    }
+    console.log(` - pending notifies:`, notifies);
+
+    // cache of known-settled kpids: will grow to num(kpids)
+    const KPIDStatus = new Map();
+    const isSettled = kpid => {
+      if (KPIDStatus.has(kpid)) {
+        return KPIDStatus.get(kpid);
+      }
+      const state = kvStore.get(`${kpid}.state`);
+      // missing state means the kpid is deleted somehow, shouldn't happen
+      state || Fail`${kpid}.state is missing`;
+      const settled = state !== 'unresolved';
+      KPIDStatus.set(kpid, settled);
+      return settled;
+    };
+
+    // walk vNN.c.kpNN for all vats, for each one check the
+    // kpNN.state, for the settled ones check for a pending notify,
+    // record the ones without a pending notify
+
+    const buggyKPIDs = []; // [kpid, vatID]
+    for (const vatID of allVatIDs) {
+      const prefix = `${vatID}.c.`;
+      const len = prefix.length;
+      const ckpPrefix = `${vatID}.c.kp`;
+      for (const key of enumeratePrefixedKeys(kvStore, ckpPrefix)) {
+        const kpid = key.slice(len);
+        if (isSettled(kpid)) {
+          const n = notifies.get(kpid);
+          if (!n || !n.includes(vatID)) {
+            // there is no pending notify
+            buggyKPIDs.push([kpid, vatID]);
+          }
+        }
+      }
+    }
+    console.log(` - found ${buggyKPIDs.length} buggy kpids, enqueueing fixes`);
+
+    // now fix it. The bug means we failed to delete the c-list entry
+    // and decref it back when the promise was rejected. That decref
+    // would have pushed the kpid onto maybeFreeKrefs, which would
+    // have triggered a refcount check at end-of-crank, which might
+    // have deleted the promise records (if nothing else was
+    // referencing the promise, like arguments in messages enqueued to
+    // unresolved promises, or something transient on the
+    // run-queue). Deleting those promise records might have decreffed
+    // krefs in the rejection data (although in general 9039 rejects
+    // those promises with non-slot-bearing DisconnectionObjects).
+    //
+    // To avoid duplicating a lot of kernel code inside this upgrade
+    // handler, we do the simplest possible thing: enqueue a notify to
+    // the upgraded vat for all these leftover promises. The new vat
+    // incarnation will ignore it (they don't recognize the vpid), but
+    // the dispatch.notify() delivery will clear the c-list and decref
+    // the kpid, and will trigger all the usual GC work. Note that
+    // these notifies will be delivered before any activity the host
+    // app might trigger for e.g. a chain upgrade, but they should not
+    // cause userspace-visible behavior (non-slot-bearing rejection
+    // data means no other vat will even get a gc-action delivery:
+    // only the upgraded vat will see anything, and those deliveries
+    // won't make it past liveslots).
+
+    let count = 0;
+    for (const [kpid, vatID] of buggyKPIDs) {
+      // account for the reference to this kpid in upgradeEvents
+      incrementReferenceCount(getRequired, kvStore, kpid, `enq|notify`);
+      upgradeEvents.push({ type: 'notify', vatID, kpid });
+      count += 1;
+    }
+
+    console.log(` - #9039 remediation complete, ${count} notifies to inject`);
+    modified = true;
+    version = 3;
+  }
+
+  if (upgradeEvents.length) {
+    assert(modified);
+    // stash until host calls controller.injectQueuedUpgradeEvents()
+    const oldEvents = JSON.parse(kvStore.get('upgradeEvents') || '[]');
+    const events = [...oldEvents, ...upgradeEvents];
+    kvStore.set('upgradeEvents', JSON.stringify(events));
+  }
+
   if (modified) {
     kvStore.set('version', `${version}`);
   }
-  return modified;
+  return harden({ modified });
 };
 harden(upgradeSwingset);

package/src/kernel/kernel.js

@@ -215,6 +215,10 @@ export default function buildKernel(
     return deviceID;
   }
 
+  function injectQueuedUpgradeEvents() {
+    kernelKeeper.injectQueuedUpgradeEvents();
+  }
+
   function addImport(forVatID, what) {
     if (!started) {
       throw Error('must do kernel.start() before addImport()');
@@ -287,15 +291,16 @@ export default function buildKernel(
       const vatKeeper = kernelKeeper.provideVatKeeper(vatID);
       critical = vatKeeper.getOptions().critical;
 
-      // Reject all promises decided by the vat, making sure to capture the list
-      // of kpids before that data is deleted.
-      const deadPromises = [...kernelKeeper.enumeratePromisesByDecider(vatID)];
-      // remove vatID from the list of live vats, and mark for deletion
+      // remove vatID from the list of live vats, and mark for
+      // deletion (which will happen later, in vat-cleanup events)
       kernelKeeper.deleteVatID(vatID);
       kernelKeeper.markVatAsTerminated(vatID);
       deferred.push(kernelKeeper.removeVatFromSwingStoreExports(vatID));
-      for (const kpid of deadPromises) {
-        resolveToError(kpid, makeError('vat terminated'), vatID);
+
+      // Reject all promises decided by the vat
+      const errdata = makeError('vat terminated');
+      for (const [kpid, _p] of kernelKeeper.enumeratePromisesByDecider(vatID)) {
+        resolveToError(kpid, errdata, vatID);
       }
     }
     if (critical) {
@@ -597,7 +602,9 @@ export default function buildKernel(
     const p = kernelKeeper.getKernelPromise(kpid);
     kernelKeeper.incStat('dispatchNotify');
     const vatKeeper = kernelKeeper.provideVatKeeper(vatID);
-    p.state !== 'unresolved' || Fail`spurious notification ${kpid}`;
+    if (p.state === 'unresolved') {
+      throw Fail`spurious notification ${kpid}`;
+    }
     /** @type { KernelDeliveryOneNotify[] } */
     const resolutions = [];
     if (!vatKeeper.hasCListEntry(kpid)) {
@@ -612,7 +619,9 @@ export default function buildKernel(
       return NO_DELIVERY_CRANK_RESULTS;
     }
     for (const toResolve of targets) {
-      const { state, data } = kernelKeeper.getKernelPromise(toResolve);
+      const tp = kernelKeeper.getKernelPromise(toResolve);
+      assert(tp.state !== 'unresolved');
+      const { state, data } = tp;
       resolutions.push([toResolve, { state, data }]);
     }
     /** @type { KernelDeliveryNotify } */
@@ -995,9 +1004,30 @@ export default function buildKernel(
       return results;
     }
 
-    // reject all promises for which the vat was decider
-    for (const kpid of kernelKeeper.enumeratePromisesByDecider(vatID)) {
-      resolveToError(kpid, disconnectionCapData, vatID);
+    // We are homesick for a future in which most promises are
+    // durable, and vats do not need to subscribe to their own
+    // promises to make promise-watchers work. In that world, vats
+    // somehow mark the non-durable promises, which we must
+    // reject/disconnect on their behalf during upgrade.
+    //
+    // To handle the present reality, without durable promises, we
+    // pretend that all promises are so marked.
+
+    // reject all ephemeral promises for which the vat was decider
+    for (const [kpid, p] of kernelKeeper.enumeratePromisesByDecider(vatID)) {
+      const isEphemeral = true; // future vats will mark these explicitly
+      const selfSubscribed =
+        p.state === 'unresolved' && p.subscribers.includes(vatID);
+      if (isEphemeral) {
+        resolveToError(kpid, disconnectionCapData, vatID);
+        if (!selfSubscribed) {
+          // If the vat was subscribed to its own promise, the
+          // resolveToError will enqueue a dispatch.notify, whose delivery
+          // will delete the c-list entry. If it is *not* subscribed, we
+          // should delete the c-list entry now, because nobody else will.
+          vatKeeper.deleteCListEntriesForKernelSlots([kpid]);
+        }
+      }
     }
 
     // simulate an abandonExports syscall from the vat,
@@ -1186,6 +1216,7 @@ export default function buildKernel(
         }
       }
       default:
+        // @ts-expect-error
         throw Fail`unknown promise resolution '${kp.state}'`;
     }
   }
@@ -2081,6 +2112,7 @@ export default function buildKernel(
         return p.data;
       }
       default: {
+        // @ts-expect-error
         throw Fail`invalid state for ${kpid}: ${p.state}`;
       }
     }
@@ -2179,6 +2211,7 @@ export default function buildKernel(
     pinObject,
     vatNameToID,
     deviceNameToID,
+    injectQueuedUpgradeEvents,
     queueToKref,
     kpRegisterInterest,
     kpStatus,

package/src/kernel/state/kernelKeeper.js

@@ -45,14 +45,16 @@ const enableKernelGC = true;
  * @typedef { import('../../types-external.js').VatKeeper } VatKeeper
  * @typedef { import('../../types-internal.js').InternalKernelOptions } InternalKernelOptions
  * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold
+ * @import {PromiseRecord} from '../../types-internal.js';
  * @import {CleanupBudget, CleanupWork, PolicyOutputCleanupBudget} from '../../types-external.js';
  * @import {RunQueueEventCleanupTerminatedVat} from '../../types-internal.js';
+ * @import {SwingStoreKernelStorage} from '@agoric/swing-store';
  */
 
 export { DEFAULT_REAP_DIRT_THRESHOLD_KEY };
 
 // most recent DB schema version
-export const CURRENT_SCHEMA_VERSION = 2;
+export const CURRENT_SCHEMA_VERSION = 3;
 
 // Kernel state lives in a key-value store supporting key retrieval by
 // lexicographic range. All keys and values are strings.
@@ -71,9 +73,9 @@ export const CURRENT_SCHEMA_VERSION = 2;
 // only modified by a call to upgradeSwingset(). See below for
 // deltas/upgrades from one version to the next.
 //
-// The current ("v2") schema keys/values are:
+// The current ("v3") schema keys/values are:
 //
-// version = '2'
+// version = '3'
 // vat.names = JSON([names..])
 // vat.dynamicIDs = JSON([vatIDs..])
 // vat.name.$NAME = $vatID = v$NN
@@ -141,6 +143,7 @@ export const CURRENT_SCHEMA_VERSION = 2;
 // gcActions = JSON(gcActions)
 // reapQueue = JSON([vatIDs...])
 // pinnedObjects = ko$NN[,ko$NN..]
+// upgradeEvents = JSON([events..])
 
 // ko.nextID = $NN
 // ko$NN.owner = $vatID
@@ -177,6 +180,11 @@ export const CURRENT_SCHEMA_VERSION = 2;
 // v2:
 //   * change `version` to `'2'`
 //   * add `vats.terminated` with `[]` as initial value
+// v3:
+//   * change `version` to `'3'`
+//   * perform remediation for bug #9039
+// (after v3, does not get its own version)
+//   * `upgradeEvents` recognized, but omitted if empty
 
 /** @type {(s: string) => string[]} s */
 export function commaSplit(s) {
@@ -212,6 +220,77 @@ export const getAllDynamicVats = getRequired => {
   return JSON.parse(getRequired('vat.dynamicIDs'));
 };
 
+const getObjectReferenceCount = (kvStore, kref) => {
+  const data = kvStore.get(`${kref}.refCount`);
+  if (!data) {
+    return { reachable: 0, recognizable: 0 };
+  }
+  const [reachable, recognizable] = commaSplit(data).map(Number);
+  reachable <= recognizable ||
+    Fail`refmismatch(get) ${kref} ${reachable},${recognizable}`;
+  return { reachable, recognizable };
+};
+
+const setObjectReferenceCount = (kvStore, kref, counts) => {
+  const { reachable, recognizable } = counts;
+  assert.typeof(reachable, 'number');
+  assert.typeof(recognizable, 'number');
+  (reachable >= 0 && recognizable >= 0) ||
+    Fail`${kref} underflow ${reachable},${recognizable}`;
+  reachable <= recognizable ||
+    Fail`refmismatch(set) ${kref} ${reachable},${recognizable}`;
+  kvStore.set(`${kref}.refCount`, `${reachable},${recognizable}`);
+};
+
+/**
+ * Increment the reference count associated with some kernel object.
+ *
+ * We track references to promises and objects, but not devices. Promises
+ * have only a "reachable" count, whereas objects track both "reachable"
+ * and "recognizable" counts.
+ *
+ * @param { (key: string) => string} getRequired
+ * @param { import('@agoric/swing-store').KVStore } kvStore
+ * @param {string} kref  The kernel slot whose refcount is to be incremented.
+ * @param {string?} tag  Debugging note with rough source of the reference.
+ * @param {{ isExport?: boolean, onlyRecognizable?: boolean }} options
+ * 'isExport' means the reference comes from a clist export, which counts
+ * for promises but not objects. 'onlyRecognizable' means the reference
+ * provides only recognition, not reachability
+ */
+export const incrementReferenceCount = (
+  getRequired,
+  kvStore,
+  kref,
+  tag,
+  options = {},
+) => {
+  const { isExport = false, onlyRecognizable = false } = options;
+  kref || Fail`incrementRefCount called with empty kref, tag=${tag}`;
+  const { type } = parseKernelSlot(kref);
+  if (type === 'promise') {
+    const refCount = Number(getRequired(`${kref}.refCount`)) + 1;
+    // kdebug(`++ ${kref}  ${tag} ${refCount}`);
+    kvStore.set(`${kref}.refCount`, `${refCount}`);
+  }
+  if (type === 'object' && !isExport) {
+    let { reachable, recognizable } = getObjectReferenceCount(kvStore, kref);
+    if (!onlyRecognizable) {
+      reachable += 1;
+    }
+    recognizable += 1;
+    // kdebug(`++ ${kref}  ${tag} ${reachable},${recognizable}`);
+    setObjectReferenceCount(kvStore, kref, { reachable, recognizable });
+  }
+};
+
+export function* readQueue(queue, getRequired) {
+  const [head, tail] = JSON.parse(getRequired(`${queue}`));
+  for (let i = head; i < tail; i += 1) {
+    yield JSON.parse(getRequired(`${queue}.${i}`));
+  }
+}
+
 // we use different starting index values for the various vNN/koNN/kdNN/kpNN
 // slots, to reduce confusing overlap when looking at debug messages (e.g.
 // seeing both kp1 and ko1, which are completely unrelated despite having the
@@ -389,14 +468,7 @@ export default function makeKernelKeeper(
     return tail - head;
   }
 
-  function dumpQueue(queue) {
-    const [head, tail] = JSON.parse(getRequired(`${queue}`));
-    const result = [];
-    for (let i = head; i < tail; i += 1) {
-      result.push(JSON.parse(getRequired(`${queue}.${i}`)));
-    }
-    return result;
-  }
+  const dumpQueue = queue => [...readQueue(queue, getRequired)];
 
   /**
    * @param {InternalKernelOptions} kernelOptions
@@ -620,26 +692,9 @@ export default function makeKernelKeeper(
     return parseReachableAndVatSlot(kvStore.get(kernelKey));
   }
 
-  function getObjectRefCount(kernelSlot) {
-    const data = kvStore.get(`${kernelSlot}.refCount`);
-    if (!data) {
-      return { reachable: 0, recognizable: 0 };
-    }
-    const [reachable, recognizable] = commaSplit(data).map(Number);
-    reachable <= recognizable ||
-      Fail`refmismatch(get) ${kernelSlot} ${reachable},${recognizable}`;
-    return { reachable, recognizable };
-  }
-
-  function setObjectRefCount(kernelSlot, { reachable, recognizable }) {
-    assert.typeof(reachable, 'number');
-    assert.typeof(recognizable, 'number');
-    (reachable >= 0 && recognizable >= 0) ||
-      Fail`${kernelSlot} underflow ${reachable},${recognizable}`;
-    reachable <= recognizable ||
-      Fail`refmismatch(set) ${kernelSlot} ${reachable},${recognizable}`;
-    kvStore.set(`${kernelSlot}.refCount`, `${reachable},${recognizable}`);
-  }
+  const getObjectRefCount = kref => getObjectReferenceCount(kvStore, kref);
+  const setObjectRefCount = (kref, counts) =>
+    setObjectReferenceCount(kvStore, kref, counts);
 
   /**
    * Iterate over non-durable objects exported by a vat.
@@ -791,45 +846,41 @@ export default function makeKernelKeeper(
     return kpid;
   }
 
+  /**
+   * @param {string} kernelSlot
+   * @returns {PromiseRecord}
+   */
   function getKernelPromise(kernelSlot) {
     insistKernelType('promise', kernelSlot);
-    const p = { state: getRequired(`${kernelSlot}.state`) };
-    switch (p.state) {
-      case undefined: {
-        throw Fail`unknown kernelPromise '${kernelSlot}'`;
-      }
+    const state = getRequired(`${kernelSlot}.state`);
+    const refCount = Number(kvStore.get(`${kernelSlot}.refCount`));
+    switch (state) {
       case 'unresolved': {
-        p.refCount = Number(kvStore.get(`${kernelSlot}.refCount`));
-        p.decider = kvStore.get(`${kernelSlot}.decider`);
-        if (p.decider === '') {
-          p.decider = undefined;
-        }
-        p.policy = kvStore.get(`${kernelSlot}.policy`) || 'ignore';
-        // @ts-expect-error get() may fail
-        p.subscribers = commaSplit(kvStore.get(`${kernelSlot}.subscribers`));
-        p.queue = Array.from(
+        const decider = kvStore.get(`${kernelSlot}.decider`) || undefined;
+        const policy = kvStore.get(`${kernelSlot}.policy`) || 'ignore';
+        const subscribers = commaSplit(
+          kvStore.get(`${kernelSlot}.subscribers`) || '',
+        );
+        const queue = Array.from(
           getPrefixedValues(kvStore, `${kernelSlot}.queue.`),
         ).map(s => JSON.parse(s));
-        break;
+        return harden({ state, refCount, decider, policy, subscribers, queue });
       }
       case 'fulfilled':
       case 'rejected': {
-        p.refCount = Number(kvStore.get(`${kernelSlot}.refCount`));
-        p.data = {
-          body: kvStore.get(`${kernelSlot}.data.body`),
-          // @ts-expect-error get() may fail
-          slots: commaSplit(kvStore.get(`${kernelSlot}.data.slots`)),
+        const data = {
+          body: getRequired(`${kernelSlot}.data.body`),
+          slots: commaSplit(getRequired(`${kernelSlot}.data.slots`)),
         };
-        for (const s of p.data.slots) {
+        for (const s of data.slots) {
           parseKernelSlot(s);
         }
-        break;
+        return harden({ state, refCount, data });
       }
       default: {
-        throw Fail`unknown state for ${kernelSlot}: ${p.state}`;
+        throw Fail`unknown state for ${kernelSlot}: ${state}`;
       }
     }
-    return harden(p);
   }
 
   function getResolveablePromise(kpid, expectedDecider) {
@@ -838,7 +889,9 @@ export default function makeKernelKeeper(
       insistVatID(expectedDecider);
     }
     const p = getKernelPromise(kpid);
-    p.state === 'unresolved' || Fail`${kpid} was already resolved`;
+    if (p.state !== 'unresolved') {
+      throw Fail`${kpid} was already resolved`;
+    }
     if (expectedDecider) {
       p.decider === expectedDecider ||
         Fail`${kpid} is decided by ${p.decider}, not ${expectedDecider}`;
@@ -897,6 +950,7 @@ export default function makeKernelKeeper(
     // up the resolution *now* and set the correct target early. Doing that
     // might make it easier to remove the Promise Table entry earlier.
     const p = getKernelPromise(kernelSlot);
+    assert.equal(p.state, 'unresolved');
     for (const msg of p.queue) {
       const entry = harden({ type: 'send', target: kernelSlot, msg });
       enqueue('acceptanceQueue', entry);
@@ -1142,18 +1196,22 @@ export default function makeKernelKeeper(
   function setDecider(kpid, decider) {
     insistVatID(decider);
     const p = getKernelPromise(kpid);
-    p.state === 'unresolved' || Fail`${kpid} was already resolved`;
-    !p.decider || Fail`${kpid} has decider ${p.decider}, not empty`;
+    assert.equal(p.state, 'unresolved', `${kpid} was already resolved`);
+    assert(!p.decider, `${kpid} has decider ${p.decider}, not empty`);
     kvStore.set(`${kpid}.decider`, decider);
   }
 
   function clearDecider(kpid) {
     const p = getKernelPromise(kpid);
-    p.state === 'unresolved' || Fail`${kpid} was already resolved`;
-    p.decider || Fail`${kpid} does not have a decider`;
+    assert.equal(p.state, 'unresolved', `${kpid} was already resolved`);
+    assert(p.decider, `${kpid} does not have a decider`);
     kvStore.set(`${kpid}.decider`, '');
   }
 
+  /**
+   * @param {string} vatID
+   * @returns {IterableIterator<[kpid: string, p: PromiseRecord]>}
+   */
   function* enumeratePromisesByDecider(vatID) {
     insistVatID(vatID);
     const promisePrefix = `${vatID}.c.p`;
@@ -1167,10 +1225,10 @@ export default function makeKernelKeeper(
       // whether the vat is the decider or not.  If it is, we add the promise
       // to the list of promises that must be rejected because the dead vat
       // will never be able to act upon them.
-      const kpid = kvStore.get(k);
+      const kpid = getRequired(k);
       const p = getKernelPromise(kpid);
       if (p.state === 'unresolved' && p.decider === vatID) {
-        yield kpid;
+        yield [kpid, p];
       }
     }
   }
@@ -1180,6 +1238,7 @@ export default function makeKernelKeeper(
     insistKernelType('promise', kernelSlot);
     insistVatID(vatID);
     const p = getKernelPromise(kernelSlot);
+    assert.equal(p.state, 'unresolved');
     const s = new Set(p.subscribers);
     s.add(vatID);
     const v = Array.from(s).sort().join(',');
@@ -1210,6 +1269,27 @@ export default function makeKernelKeeper(
     return dequeue('acceptanceQueue');
   }
 
+  function injectQueuedUpgradeEvents() {
+    // refcounts: Any krefs in `upgradeEvents` must have a refcount to
+    // represent the list's hold on those objects. When
+    // upgradeSwingset() creates these events, it must also
+    // incref(kref), otherwise we run the risk of dropping the kref by
+    // the time injectQueuedUpgradeEvents() is called. We're nominally
+    // removing each event from upgradeEvents (decref), then pushing
+    // it onto the run-queue (incref), but since those two cancel each
+    // other out, we don't actually need to modify any reference
+    // counts from within this function. Note that
+    // addToAcceptanceQueue does not increment refcounts, just kernel
+    // queue-length stats.
+
+    const events = JSON.parse(kvStore.get('upgradeEvents') || '[]');
+    kvStore.delete('upgradeEvents');
+    for (const e of events) {
+      assert(e.type, `not an event`);
+      addToAcceptanceQueue(e);
+    }
+  }
+
   function allocateMeter(remaining, threshold) {
     if (remaining !== 'unlimited') {
       assert.typeof(remaining, 'bigint');
@@ -1417,40 +1497,8 @@ export default function makeKernelKeeper(
     maybeFreeKrefs.add(kref);
   }
 
-  /**
-   * Increment the reference count associated with some kernel object.
-   *
-   * We track references to promises and objects, but not devices. Promises
-   * have only a "reachable" count, whereas objects track both "reachable"
-   * and "recognizable" counts.
-   *
-   * @param {unknown} kernelSlot  The kernel slot whose refcount is to be incremented.
-   * @param {string?} tag  Debugging note with rough source of the reference.
-   * @param {{ isExport?: boolean, onlyRecognizable?: boolean }} options
-   * 'isExport' means the reference comes from a clist export, which counts
-   * for promises but not objects. 'onlyRecognizable' means the reference
-   * provides only recognition, not reachability
-   */
-  function incrementRefCount(kernelSlot, tag, options = {}) {
-    const { isExport = false, onlyRecognizable = false } = options;
-    kernelSlot ||
-      Fail`incrementRefCount called with empty kernelSlot, tag=${tag}`;
-    const { type } = parseKernelSlot(kernelSlot);
-    if (type === 'promise') {
-      const refCount = Nat(BigInt(getRequired(`${kernelSlot}.refCount`))) + 1n;
-      // kdebug(`++ ${kernelSlot}  ${tag} ${refCount}`);
-      kvStore.set(`${kernelSlot}.refCount`, `${refCount}`);
-    }
-    if (type === 'object' && !isExport) {
-      let { reachable, recognizable } = getObjectRefCount(kernelSlot);
-      if (!onlyRecognizable) {
-        reachable += 1;
-      }
-      recognizable += 1;
-      // kdebug(`++ ${kernelSlot}  ${tag} ${reachable},${recognizable}`);
-      setObjectRefCount(kernelSlot, { reachable, recognizable });
-    }
-  }
+  const incrementRefCount = (kref, tag, options = {}) =>
+    incrementReferenceCount(getRequired, kvStore, kref, tag, options);
 
   /**
    * Decrement the reference count associated with some kernel object.
@@ -1542,13 +1590,15 @@ export default function makeKernelKeeper(
           const kp = getKernelPromise(kpid);
           if (kp.refCount === 0) {
             let idx = 0;
-            // TODO (#9889) don't assume promise is settled
-            for (const slot of kp.data.slots) {
-              // Note: the following decrement can result in an addition to the
-              // maybeFreeKrefs set, which we are in the midst of iterating.
-              // TC39 went to a lot of trouble to ensure that this is kosher.
-              decrementRefCount(slot, `gc|${kpid}|s${idx}`);
-              idx += 1;
+            if (kp.state === 'fulfilled' || kp.state === 'rejected') {
+              // #9889 don't assume promise is settled
+              for (const slot of kp.data.slots) {
+                // Note: the following decrement can result in an addition to the
+                // maybeFreeKrefs set, which we are in the midst of iterating.
+                // TC39 went to a lot of trouble to ensure that this is kosher.
+                decrementRefCount(slot, `gc|${kpid}|s${idx}`);
+                idx += 1;
+              }
             }
             deleteKernelPromise(kpid);
           }
@@ -1939,6 +1989,8 @@ export default function makeKernelKeeper(
     getAcceptanceQueueLength,
     getNextAcceptanceQueueMsg,
 
+    injectQueuedUpgradeEvents,
+
     allocateMeter,
     addMeterRemaining,
     setMeterThreshold,

package/src/kernel/vatTranslator.js

@@ -50,6 +50,7 @@ function makeTranslateKernelDeliveryToVatDelivery(vatID, kernelKeeper) {
       parseVatSlot(targetSlot).allocatedByVat || Fail`deliver() to wrong vat`;
     } else if (type === 'promise') {
       const p = kernelKeeper.getKernelPromise(target);
+      assert(p.state === 'unresolved');
       p.decider === vatID || Fail`wrong decider`;
     }
     const inputSlots = msg.methargs.slots.map(slot =>
@@ -59,7 +60,9 @@ function makeTranslateKernelDeliveryToVatDelivery(vatID, kernelKeeper) {
     if (msg.result) {
       insistKernelType('promise', msg.result);
       const p = kernelKeeper.getKernelPromise(msg.result);
-      p.state === 'unresolved' || Fail`result ${msg.result} already resolved`;
+      if (p.state !== 'unresolved') {
+        throw Fail`result ${msg.result} already resolved`;
+      }
       !p.decider || Fail`result ${msg.result} already has decider ${p.decider}`;
       resultSlot = vatKeeper.mapKernelSlotToVatSlot(msg.result);
       insistVatType('promise', resultSlot);
@@ -318,8 +321,9 @@ function makeTranslateVatSyscallToKernelSyscall(vatID, kernelKeeper) {
       // In the case of non-pipelining vats these checks are redundant since
       // we're guaranteed to have a promise newly allocated by the vat.
       const p = kernelKeeper.getKernelPromise(result);
-      p.state === 'unresolved' ||
-        Fail`send() result ${result} is already resolved`;
+      if (p.state !== 'unresolved') {
+        throw Fail`send() result ${result} is already resolved`;
+      }
       p.decider === vatID ||
         Fail`send() result ${result} is decided by ${p.decider} not ${vatID}`;
       kernelKeeper.clearDecider(result);

package/src/types-internal.js

@@ -92,6 +92,17 @@ export {};
  * @property {number} [computrons]
  */
 
+/**
+ * @typedef {{ state: 'unresolved', refCount: number,
+ *             decider: string | undefined, policy: string,
+ *             subscribers: string[], queue: string[],
+ *          }} UnresolvedPromiseRecord
+ * @typedef {{ state: 'fulfilled' | 'rejected', refCount: number,
+ *             data: SwingSetCapData,
+ *          }} SettledPromiseRecord
+ * @typedef {UnresolvedPromiseRecord | SettledPromiseRecord} PromiseRecord
+ */
+
 /**
  * @typedef {{
  *   enablePipelining: boolean,