@agoric/swingset-liveslots 0.10.3-other-dev-8f8782b.0 → 0.10.3-other-dev-3eb1a1d.0

package/src/virtualReferences.js

@@ -1,6 +1,6 @@
-/* eslint-disable no-use-before-define, jsdoc/require-returns-type */
+/* eslint-disable jsdoc/require-returns-type */
 
-import { assert, Fail } from '@agoric/assert';
+import { assert, Fail } from '@endo/errors';
 import { Nat } from '@endo/nat';
 import { parseVatSlot } from './parseVatSlots.js';
 import {
@@ -290,7 +290,10 @@ export function makeVirtualReferenceManager(
    */
   function isDurable(vref) {
     const { type, id, virtual, durable, allocatedByVat } = parseVatSlot(vref);
-    if (relaxDurabilityRules) {
+    if (type === 'promise') {
+      // promises are not durable even if `relaxDurabilityRules === true`
+      return false;
+    } else if (relaxDurabilityRules) {
       // we'll pretend an object is durable if running with relaxed rules
       return true;
     } else if (type === 'device') {
@@ -497,26 +500,51 @@ export function makeVirtualReferenceManager(
   }
 
   /**
-   * A vref is "recognizable" when it is used as the key of a weak Map
-   * or Set: that Map/Set can be used to query whether a future
-   * specimen matches the original or not, without holding onto the
-   * original.
+   * A vref is "recognizable" when it is used as the key of a weak
+   * collection, like a virtual/durable WeakMapStore or WeakSetStore,
+   * or the ephemeral voAwareWeakMap/Set that we impose upon userspace
+   * as "WeakMap/WeakSet". The collection can be used to query whether
+   * a future specimen matches the original or not, without holding
+   * onto the original.
+   *
+   * We need "recognition records" to map from the vref to the
+   * collection that can recognize it. When the vref is retired, we
+   * use the record to find all the collections from which we need to
+   * delete entries, so we can release the matching values. This might
+   * happen because the vref was for a Presence and the kernel just
+   * told us the upstream vat has deleted it (dispatch.retireImports),
+   * or because it was for a locally-managed object (an ephemeral
+   * Remotable or a virtual/durable Representative) and we decided to
+   * delete it.
+   *
+   * The virtual/durable collections track their "recognition records"
+   * in the vatstore, in keys like "vom.ir.${vref}|${collectionID}".
+   * These records do not contribute to our RAM usage.
+   *
+   * voAwareWeakMap and voAwareWeakSet store their recognition records
+   * in RAM, using this Map named 'vrefRecognizers'. Each key is a
+   * vref, and the value is a Set of recognizers. Each recognizer is
+   * the internal 'virtualObjectMap' in which the collection maps from
+   * vref to value. These in-RAM collections only use virtualObjectMap
+   * to track Presence-style (imports) and Representative-style
+   * (virtual/durable) vrefs: any Remotable-style keys are stored in
+   * the collection's internal (real) WeakMap under the Remotable
+   * object itself (because the engine handles the bookkeeping, and
+   * there is no virtual data in the value that we need to clean up at
+   * deletion time).
    *
-   * This 'vrefRecognizers' is a Map from those vrefs to the set of
-   * recognizing weak collections, for virtual keys and non-virtual
-   * collections. Specifically, the vrefs correspond to imported
-   * Presences or virtual-object Representatives (Remotables do not
-   * participate: they are keyed by the actual Remotable object, not
-   * its vref). The collections are either a VirtualObjectAwareWeakMap
-   * or a VirtualObjectAwareWeakSet. We remove the entry when the key
-   * is removed from the collection, and when the entire collection is
-   * deleted.
+   * Each voAwareWeakMap/Set must have a distinct recognizer, so we
+   * can remove the key from the right ones. The recognizer is held
+   * strongly by the recognition record, so it must not be the
+   * voAwareWeakMap/Set itself (which would inhibit GC).
    *
-   * It is critical that each collection have exactly one recognizer that is
-   * unique to that collection, because the recognizers themselves will be
-   * tracked by their object identities, but the recognizer cannot be the
-   * collection itself else it would prevent the collection from being garbage
-   * collected.
+   * When an individual entry is deleted from the weak collection, we
+   * must also delete the recognition record. When the collection
+   * itself is deleted (i.e. because nothing was referencing it), we
+   * must both delete all recognition records and also notify the
+   * kernel about any Presence-style vrefs that we can no longer
+   * recognize (syscall.retireImports). The kernel doesn't care about
+   * Remotable- or Representative- style vrefs, only the imports.
    *
    * TODO: all the "recognizers" in principle could be, and probably should be,
    * reduced to deleter functions.  However, since the VirtualObjectAware
@@ -532,14 +560,24 @@ export function makeVirtualReferenceManager(
   /** @type {Map<string, Set<Recognizer>>} */
   const vrefRecognizers = new Map();
 
+  /**
+   * @param {*} value  The vref-bearing object used as the collection key
+   * @param {string|Recognizer} recognizer  The collectionID or virtualObjectMap for the collection
+   * @param {boolean} [recognizerIsVirtual]  true for virtual/durable Stores, false for voAwareWeakMap/Set
+   */
   function addRecognizableValue(value, recognizer, recognizerIsVirtual) {
     const vref = getSlotForVal(value);
     if (vref) {
       const { type, allocatedByVat, virtual, durable } = parseVatSlot(vref);
-      if (type === 'object' && (!allocatedByVat || virtual || durable)) {
+      if (type === 'object') {
+        // recognizerSet (voAwareWeakMap/Set) doesn't track Remotables
+        const notRemotable = !allocatedByVat || virtual || durable;
+
         if (recognizerIsVirtual) {
+          assert.typeof(recognizer, 'string');
           syscall.vatstoreSet(`vom.ir.${vref}|${recognizer}`, '1');
-        } else {
+        } else if (notRemotable) {
+          assert.typeof(recognizer, 'object');
           let recognizerSet = vrefRecognizers.get(vref);
           if (!recognizerSet) {
             recognizerSet = new Set();
@@ -551,18 +589,33 @@ export function makeVirtualReferenceManager(
     }
   }
 
+  /**
+   * @param {string} vref  The vref or the object used as the collection key
+   * @param {string|Recognizer} recognizer  The collectionID or virtualObjectMap for the collection
+   * @param {boolean} [recognizerIsVirtual]  true for virtual/durable Stores, false for voAwareWeakMap/Set
+   */
   function removeRecognizableVref(vref, recognizer, recognizerIsVirtual) {
     const { type, allocatedByVat, virtual, durable } = parseVatSlot(vref);
-    if (type === 'object' && (!allocatedByVat || virtual || durable)) {
+    if (type === 'object') {
+      // addToPossiblyDeadSet only needs Presence-style vrefs
+      const isPresence = !allocatedByVat;
+      // recognizerSet (voAwareWeakMap/Set) doesn't track Remotables
+      const notRemotable = !allocatedByVat || virtual || durable;
+
       if (recognizerIsVirtual) {
+        assert.typeof(recognizer, 'string');
         syscall.vatstoreDelete(`vom.ir.${vref}|${recognizer}`);
-      } else {
+        if (isPresence) {
+          addToPossiblyRetiredSet(vref);
+        }
+      } else if (notRemotable) {
+        assert.typeof(recognizer, 'object');
         const recognizerSet = vrefRecognizers.get(vref);
         assert(recognizerSet && recognizerSet.has(recognizer));
         recognizerSet.delete(recognizer);
         if (recognizerSet.size === 0) {
           vrefRecognizers.delete(vref);
-          if (!allocatedByVat) {
+          if (isPresence) {
             addToPossiblyRetiredSet(vref);
           }
         }
@@ -570,6 +623,11 @@ export function makeVirtualReferenceManager(
     }
   }
 
+  /**
+   * @param {*} value  The vref-bearing object used as the collection key
+   * @param {string|Recognizer} recognizer  The collectionID or virtualObjectMap for the collection
+   * @param {boolean} [recognizerIsVirtual]  true for virtual/durable Stores, false for voAwareWeakMap/Set
+   */
   function removeRecognizableValue(value, recognizer, recognizerIsVirtual) {
     const vref = getSlotForVal(value);
     if (vref) {
@@ -682,6 +740,54 @@ export function makeVirtualReferenceManager(
     return size;
   }
 
+  /**
+   * Counters to track the next number for various categories of allocation.
+   * `exportID` starts at 1 because 'o+0' is always automatically
+   * pre-assigned to the root object.
+   * `promiseID` starts at 5 as a very minor aid to debugging: when puzzling
+   * over trace logs and the like, it helps for the numbers in various species
+   * of IDs that are jumbled together to be a little out of sync and thus a
+   * little less similar to each other.
+   */
+  const initialIDCounters = { exportID: 1, collectionID: 1, promiseID: 5 };
+  /** @type {Record<string, number>} */
+  let idCounters;
+  let idCountersAreDirty = false;
+
+  function initializeIDCounters() {
+    if (!idCounters) {
+      // the saved value might be missing, or from an older liveslots
+      // (with fewer counters), so merge it with our initial values
+      const saved = JSON.parse(syscall.vatstoreGet('idCounters') || '{}');
+      idCounters = { ...initialIDCounters, ...saved };
+      idCountersAreDirty = true;
+    }
+  }
+
+  function allocateNextID(name) {
+    if (!idCounters) {
+      // Normally `initializeIDCounters` would be called from startVat, but some
+      // tests bypass that so this is a backstop.  Note that the invocation from
+      // startVat is there to make vatStore access patterns a bit more
+      // consistent from one vat to another, principally as a confusion
+      // reduction measure in service of debugging; it is not a correctness
+      // issue.
+      initializeIDCounters();
+    }
+    const result = idCounters[name];
+    result !== undefined || Fail`unknown idCounters[${name}]`;
+    idCounters[name] += 1;
+    idCountersAreDirty = true;
+    return result;
+  }
+
+  function flushIDCounters() {
+    if (idCountersAreDirty) {
+      syscall.vatstoreSet('idCounters', JSON.stringify(idCounters));
+      idCountersAreDirty = false;
+    }
+  }
+
   const testHooks = {
     getReachableRefCount,
     countCollectionsForWeakKey,
@@ -726,6 +832,9 @@ export function makeVirtualReferenceManager(
     ceaseRecognition,
     setDeleteCollectionEntry,
     getRetentionStats,
+    initializeIDCounters,
+    allocateNextID,
+    flushIDCounters,
     testHooks,
   });
 }

package/src/watchedPromises.js

@@ -1,11 +1,18 @@
+// @ts-check
 // no-lonely-if is a stupid rule that really should be disabled globally
 /* eslint-disable no-lonely-if */
 
-import { assert } from '@agoric/assert';
-import { initEmpty, M } from '@agoric/store';
+import { Fail, assert } from '@endo/errors';
 import { E } from '@endo/eventual-send';
+import { initEmpty, M } from '@agoric/store';
 import { parseVatSlot } from './parseVatSlots.js';
 
+/**
+ * @template V
+ * @template {any[]} [A=unknown[]]
+ * @typedef {[watcher: import('./types.js').PromiseWatcher<V, A>, ...args: A]} PromiseWatcherTuple
+ */
+
 /**
  * @param {object} options
  * @param {*} options.syscall
@@ -28,17 +35,28 @@ export function makeWatchedPromiseManager({
   const { makeScalarBigMapStore } = collectionManager;
   const { defineDurableKind } = vom;
 
-  // virtual Store (not durable) mapping vpid to Promise objects, to
-  // maintain the slotToVal registration until resolution. Without
-  // this, slotToVal would forget local Promises that aren't exported.
+  /**
+   * virtual Store (not durable) mapping vpid to Promise objects, to
+   * maintain the slotToVal registration until resolution. Without
+   * this, slotToVal would forget local Promises that aren't exported.
+   *
+   * @type {MapStore<string, Promise<unknown>>}
+   */
   let promiseRegistrations;
 
-  // watched promises by vpid: each entry is an array of watches on the
-  // corresponding vpid; each of these is in turn an array of a watcher object
-  // and the arguments associated with it by `watchPromise`.
+  /**
+   * watched promises by vpid: each entry is an array of watches on the
+   * corresponding vpid; each of these is in turn an array of a watcher object
+   * and the arguments associated with it by `watchPromise`.
+   * @type {MapStore<string, PromiseWatcherTuple<unknown>[]>}
+   */
   let watchedPromiseTable;
 
-  // defined promise watcher objects indexed by kindHandle
+  /**
+   * defined promise watcher objects indexed by kindHandle
+   *
+   * @type {MapStore<import('./vatDataTypes.js').DurableKindHandle, import('./types.js').PromiseWatcher<unknown>>}
+   */
   let promiseWatcherByKindTable;
 
   function preparePromiseWatcherTables() {
@@ -73,11 +91,17 @@ export function makeWatchedPromiseManager({
   }
 
   /**
-   *
-   * @param {Promise<unknown>} p
+   * @template T
+   * @param {Promise<T>} p
    * @param {string} vpid
+   * @returns {void}
    */
   function pseudoThen(p, vpid) {
+    /**
+     *
+     * @param {T} value
+     * @param {boolean} wasFulfilled
+     */
     function settle(value, wasFulfilled) {
       const watches = watchedPromiseTable.get(vpid);
       watchedPromiseTable.delete(vpid);
@@ -115,26 +139,46 @@ export function makeWatchedPromiseManager({
    */
   function loadWatchedPromiseTable(revivePromise) {
     for (const vpid of watchedPromiseTable.keys()) {
+      if (promiseRegistrations.has(vpid)) {
+        // We're only interested in reconnecting the promises from the previous
+        // incarnation. Any promise watched during buildRootObject would have
+        // already created a registration.
+        continue;
+      }
       const p = revivePromise(vpid);
       promiseRegistrations.init(vpid, p);
       pseudoThen(p, vpid);
     }
   }
 
+  /**
+   * @template V
+   * @template {any[]} A]
+   * @param {import('./vatDataTypes.js').DurableKindHandle} kindHandle
+   * @param {(value: V, ...args: A) => void} fulfillHandler
+   * @param {(reason: any, ...args: A) => void} rejectHandler
+   * @returns {import('./types.js').PromiseWatcher<V, A>}
+   */
   function providePromiseWatcher(
     kindHandle,
-    fulfillHandler = x => x,
-    rejectHandler = x => {
-      throw x;
+    // @ts-expect-error xxx rest params in typedef
+    fulfillHandler = _value => {
+      // It's fine to not pass the value through since promise watchers are not chainable
+    },
+    // @ts-expect-error xxx rest params in typedef
+    rejectHandler = reason => {
+      // Replicate the unhandled rejection that would have happened if the
+      // watcher had not implemented an `onRejected` method. See `settle` above
+      throw reason;
     },
   ) {
     assert.typeof(fulfillHandler, 'function');
     assert.typeof(rejectHandler, 'function');
 
     const makeWatcher = defineDurableKind(kindHandle, initEmpty, {
-      // @ts-expect-error  TS is confused by the spread operator
+      /** @type {(context: unknown, res: V, ...args: A) => void} */
       onFulfilled: (_context, res, ...args) => fulfillHandler(res, ...args),
-      // @ts-expect-error
+      /** @type {(context: unknown, rej: unknown, ...args: A) => void} */
       onRejected: (_context, rej, ...args) => rejectHandler(rej, ...args),
     });
 
@@ -147,6 +191,9 @@ export function makeWatchedPromiseManager({
     }
   }
 
+  /**
+   * @type {<P extends Promise<any>, A extends any[]>(p: P, watcher: import('./types.js').PromiseWatcher<Awaited<P>, A>, ...args: A) => void}
+   */
   function watchPromise(p, watcher, ...args) {
     // The following wrapping defers setting up the promise watcher itself to a
     // later turn so that if the promise to be watched was the return value from
@@ -163,7 +210,10 @@ export function makeWatchedPromiseManager({
       const watcherVref = convertValToSlot(watcher);
       assert(watcherVref, 'invalid watcher');
       const { virtual, durable } = parseVatSlot(watcherVref);
-      assert(virtual || durable, 'promise watcher must be a virtual object');
+      virtual ||
+        durable ||
+        // separate line so easy to breakpoint on
+        Fail`promise watcher must be a virtual object`;
       if (watcher.onFulfilled) {
         assert.typeof(watcher.onFulfilled, 'function');
       }

package/test/{test-baggage.js → baggage.test.js}

@@ -1,10 +1,9 @@
 import test from 'ava';
-import '@endo/init/debug.js';
 
 import { Far } from '@endo/marshal';
+import { kunser } from '@agoric/kmarshal';
 import { setupTestLiveslots } from './liveslots-helpers.js';
 import { vstr } from './util.js';
-import { kunser } from './kmarshal.js';
 import { parseVatSlot } from '../src/parseVatSlots.js';
 
 function buildRootObject(vatPowers, vatParameters, baggage) {

package/test/{test-cache.js → cache.test.js}

@@ -1,5 +1,4 @@
 import test from 'ava';
-import '@endo/init/debug.js';
 
 import { makeCache } from '../src/cache.js';