@agoric/swingset-liveslots 0.10.3-other-dev-1f26562.0 → 0.10.3-other-dev-3eb1a1d.0

package/src/collectionManager.js

@@ -1,10 +1,9 @@
-import { assert, q, Fail } from '@agoric/assert';
+import { assert, q, Fail } from '@endo/errors';
 import { Far, passStyleOf } from '@endo/far';
 import {
   zeroPad,
   makeEncodePassable,
   makeDecodePassable,
-  isEncodedRemotable,
   compareRank,
 } from '@endo/marshal';
 import {
@@ -15,7 +14,10 @@ import {
   makeCopySet,
   makeCopyMap,
   getRankCover,
+  getCopyMapEntries,
+  getCopySetKeys,
 } from '@endo/patterns';
+import { isCopyMap, isCopySet } from '@agoric/store';
 import { makeBaseRef, parseVatSlot } from './parseVatSlots.js';
 import {
   enumerateKeysStartEnd,
@@ -23,6 +25,10 @@ import {
 } from './vatstore-iterators.js';
 import { makeCache } from './cache.js';
 
+/**
+ * @import {ToCapData, FromCapData} from '@endo/marshal';
+ */
+
 // XXX TODO: The following key length limit was put in place due to limitations
 // in LMDB.  With the move away from LMDB, it is no longer relevant, but I'm
 // leaving it in place for the time being as a general defensive measure against
@@ -47,10 +53,24 @@ function throwNotDurable(value, slotIndex, serializedValue) {
   Fail`value is not durable: ${value} at slot ${q(slotIndex)} of ${serializedValue.body}`;
 }
 
+function failNotFound(key, label) {
+  Fail`key ${key} not found in collection ${q(label)}`;
+}
+
+function failNotIterable(value) {
+  Fail`provided data source is not iterable: ${value}`;
+}
+
 function prefixc(collectionID, dbEntryKey) {
   return `vc.${collectionID}.${dbEntryKey}`;
 }
 
+export const collectionMetaKeys = new Set([
+  '|entryCount',
+  '|nextOrdinal',
+  '|schemata',
+]);
+
 /**
  * @typedef {object} SchemaCacheValue
  * @property {Pattern} keyShape
@@ -101,8 +121,8 @@ function makeSchemaCache(syscall, unserialize) {
  * @param {(val: any) => string | undefined} convertValToSlot
  * @param {*} convertSlotToVal
  * @param {*} registerValue
- * @param {import('@endo/marshal').ToCapData<string>} serialize
- * @param {import('@endo/marshal').FromCapData<string>} unserialize
+ * @param {ToCapData<string>} serialize
+ * @param {FromCapData<string>} unserialize
  * @param {(capDatas: any) => void} assertAcceptableSyscallCapdataSize
  */
 export function makeCollectionManager(
@@ -126,56 +146,48 @@ export function makeCollectionManager(
     scalarMapStore: {
       hasWeakKeys: false,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateMapStore,
       durable: false,
     },
     scalarWeakMapStore: {
       hasWeakKeys: true,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateWeakMapStore,
       durable: false,
     },
     scalarSetStore: {
       hasWeakKeys: false,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateSetStore,
       durable: false,
     },
     scalarWeakSetStore: {
       hasWeakKeys: true,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateWeakSetStore,
       durable: false,
     },
     scalarDurableMapStore: {
       hasWeakKeys: false,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateMapStore,
       durable: true,
     },
     scalarDurableWeakMapStore: {
       hasWeakKeys: true,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateWeakMapStore,
       durable: true,
     },
     scalarDurableSetStore: {
       hasWeakKeys: false,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateSetStore,
       durable: true,
     },
     scalarDurableWeakSetStore: {
       hasWeakKeys: true,
       kindID: 0,
-      // eslint-disable-next-line no-use-before-define
       reanimator: reanimateWeakSetStore,
       durable: true,
     },
@@ -198,7 +210,6 @@ export function makeCollectionManager(
       vrm.registerKind(
         kindID,
         storeKindInfo[kind].reanimator,
-        // eslint-disable-next-line no-use-before-define
         deleteCollection,
         storeKindInfo[kind].durable,
       );
@@ -244,7 +255,11 @@ export function makeCollectionManager(
     const kindInfo = storeKindInfo[kindName];
     kindInfo || Fail`unknown collection kind ${kindName}`;
     const { hasWeakKeys, durable } = kindInfo;
-    const getSchema = () => schemaCache.get(collectionID);
+    const getSchema = () => {
+      const result = schemaCache.get(collectionID);
+      assert(result !== undefined);
+      return result;
+    };
     const dbKeyPrefix = `vc.${collectionID}.`;
     let currentGenerationNumber = 0;
 
@@ -274,8 +289,21 @@ export function makeCollectionManager(
       return `${dbKeyPrefix}${dbEntryKey}`;
     }
 
+    // A "vref" is a string like "o-4" or "o+d44/2:0"
+    // An "EncodedKey" is the output of encode-passable:
+    // * strings become `s${string}`, like "foo" -> "sfoo"
+    // * small positive BigInts become `p${len}:${digits}`, like 47n -> "p2:47"
+    // * refs are assigned an "ordinal" and use `r${fixedLengthOrdinal}:${vref}`
+    //   * e.g. vref(o-4) becomes "r0000000001:o-4"
+    // A "DBKey" is used to index the vatstore. DBKeys for collection
+    // entries join a collection prefix and an EncodedKey. Some
+    // possible DBKeys for entries of collection "5", using collection
+    // prefix "vc.5.", are:
+    // * "foo" -> "vc.5.sfoo"
+    // * 47n -> "vc.5.p2:47"
+    // * vref(o-4) -> "vc.5.r0000000001:o-4"
+
     const encodeRemotable = remotable => {
-      // eslint-disable-next-line no-use-before-define
       const ordinal = getOrdinal(remotable);
       ordinal !== undefined || Fail`no ordinal for ${remotable}`;
       const ordinalTag = zeroPad(ordinal, BIGINT_TAG_LEN);
@@ -289,10 +317,11 @@ export function makeCollectionManager(
     // the resulting function will encode only `Key` arguments.
     const encodeKey = makeEncodePassable({ encodeRemotable });
 
-    const vrefFromDBKey = dbKey => dbKey.substring(BIGINT_TAG_LEN + 2);
+    const vrefFromEncodedKey = encodedKey =>
+      encodedKey.substring(1 + BIGINT_TAG_LEN + 1);
 
     const decodeRemotable = encodedKey =>
-      convertSlotToVal(vrefFromDBKey(encodedKey));
+      convertSlotToVal(vrefFromEncodedKey(encodedKey));
 
     // `makeDecodePassable` has three named options:
     // `decodeRemotable`, `decodeError`, and `decodePromise`.
@@ -328,32 +357,40 @@ export function makeCollectionManager(
     }
 
     function dbKeyToKey(dbKey) {
+      // convert e.g. vc.5.r0000000001:o+v10/1 to r0000000001:o+v10/1
       const dbEntryKey = dbKey.substring(dbKeyPrefix.length);
       return decodeKey(dbEntryKey);
     }
 
+    function dbKeyToEncodedKey(dbKey) {
+      assert(dbKey.startsWith(dbKeyPrefix), dbKey);
+      return dbKey.substring(dbKeyPrefix.length);
+    }
+
     function has(key) {
       const { keyShape } = getSchema();
       if (!matches(key, keyShape)) {
         return false;
-      }
-      if (passStyleOf(key) === 'remotable') {
+      } else if (passStyleOf(key) === 'remotable') {
         return getOrdinal(key) !== undefined;
       } else {
         return syscall.vatstoreGet(keyToDBKey(key)) !== undefined;
       }
     }
 
+    function mustGet(key, label) {
+      if (passStyleOf(key) === 'remotable' && getOrdinal(key) === undefined) {
+        failNotFound(key, label);
+      }
+      const dbKey = keyToDBKey(key);
+      const result = syscall.vatstoreGet(dbKey) || failNotFound(key, label);
+      return { dbKey, result };
+    }
+
     function get(key) {
       const { keyShape, label } = getSchema();
       mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label));
-      if (passStyleOf(key) === 'remotable' && getOrdinal(key) === undefined) {
-        throw Fail`key ${key} not found in collection ${q(label)}`;
-      }
-      const result = syscall.vatstoreGet(keyToDBKey(key));
-      if (!result) {
-        throw Fail`key ${key} not found in collection ${q(label)}`;
-      }
+      const { result } = mustGet(key, label);
       return unserializeValue(JSON.parse(result));
     }
 
@@ -377,11 +414,11 @@ export function makeCollectionManager(
       currentGenerationNumber += 1;
       assertAcceptableSyscallCapdataSize([serializedValue]);
       if (durable) {
-        serializedValue.slots.forEach((vref, slotIndex) => {
+        for (const [slotIndex, vref] of serializedValue.slots.entries()) {
           if (!vrm.isDurable(vref)) {
             throwNotDurable(value, slotIndex, serializedValue);
           }
-        });
+        }
       }
       if (passStyleOf(key) === 'remotable') {
         /** @type {string} */
@@ -397,7 +434,9 @@ export function makeCollectionManager(
           vrm.addReachableVref(vref);
         }
       }
-      serializedValue.slots.forEach(vrm.addReachableVref);
+      for (const vref of serializedValue.slots) {
+        vrm.addReachableVref(vref);
+      }
       syscall.vatstoreSet(keyToDBKey(key), JSON.stringify(serializedValue));
       updateEntryCount(1);
     };
@@ -416,15 +455,13 @@ export function makeCollectionManager(
       const after = serializeValue(harden(value));
       assertAcceptableSyscallCapdataSize([after]);
       if (durable) {
-        after.slots.forEach((vref, i) => {
+        for (const [i, vref] of after.slots.entries()) {
           if (!vrm.isDurable(vref)) {
             throwNotDurable(value, i, after);
           }
-        });
+        }
       }
-      const dbKey = keyToDBKey(key);
-      const rawBefore = syscall.vatstoreGet(dbKey);
-      rawBefore || Fail`key ${key} not found in collection ${q(label)}`;
+      const { dbKey, result: rawBefore } = mustGet(key, label);
       const before = JSON.parse(rawBefore);
       vrm.updateReferenceCounts(before.slots, after.slots);
       syscall.vatstoreSet(dbKey, JSON.stringify(after));
@@ -433,12 +470,7 @@ export function makeCollectionManager(
     function deleteInternal(key) {
       const { keyShape, label } = getSchema();
       mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label));
-      if (passStyleOf(key) === 'remotable' && getOrdinal(key) === undefined) {
-        throw Fail`key ${key} not found in collection ${q(label)}`;
-      }
-      const dbKey = keyToDBKey(key);
-      const rawValue = syscall.vatstoreGet(dbKey);
-      rawValue || Fail`key ${key} not found in collection ${q(label)}`;
+      const { dbKey, result: rawValue } = mustGet(key, label);
       const value = JSON.parse(rawValue);
       const doMoreGC1 = value.slots.map(vrm.removeReachableVref).some(b => b);
       syscall.vatstoreDelete(dbKey);
@@ -474,18 +506,15 @@ export function makeCollectionManager(
       const end = prefix(coverEnd); // exclusive
 
       const generationAtStart = currentGenerationNumber;
-      function checkGen() {
-        if (generationAtStart !== currentGenerationNumber) {
-          Fail`keys in store cannot be added to during iteration`;
-        }
-      }
+      const checkGen = () =>
+        currentGenerationNumber === generationAtStart ||
+        Fail`keys in store cannot be added to during iteration`;
 
       const needToMatchKey = !matchAny(keyPatt);
       const needToMatchValue = !matchAny(valuePatt);
 
-      // we always get the dbKey, but we might not need to unserialize it
+      // we might not need to unserialize the dbKey or get the dbValue
       const needKeys = yieldKeys || needToMatchKey;
-      // we don't always need the dbValue
       const needValues = yieldValues || needToMatchValue;
 
       /**
@@ -513,7 +542,7 @@ export function makeCollectionManager(
           yield [yieldKeys ? key : undefined, yieldValues ? value : undefined];
         }
       }
-
+      harden(iter);
       return iter();
     }
 
@@ -523,6 +552,7 @@ export function makeCollectionManager(
           yield entry[0];
         }
       }
+      harden(iter);
       return iter();
     }
 
@@ -539,40 +569,58 @@ export function makeCollectionManager(
      */
     function clearInternalFull() {
       let doMoreGC = false;
-      const [coverStart, coverEnd] = getRankCover(M.any(), encodeKey);
-      const start = prefix(coverStart);
-      const end = prefix(coverEnd);
-
-      // this yields all keys for which (start <= key < end)
-      for (const dbKey of enumerateKeysStartEnd(syscall, start, end)) {
-        const value = JSON.parse(syscall.vatstoreGet(dbKey));
-        doMoreGC =
-          value.slots.map(vrm.removeReachableVref).some(b => b) || doMoreGC;
-        syscall.vatstoreDelete(dbKey);
-        if (isEncodedRemotable(dbKey)) {
-          const keyVref = vrefFromDBKey(dbKey);
+
+      // visit every DB entry associated with the collection, which
+      // (due to sorting) will be collection entries first, and then a
+      // mixture of ordinal-assignment mappings and size-independent
+      // metadata (both of which start with "|").
+      for (const dbKey of enumerateKeysWithPrefix(syscall, dbKeyPrefix)) {
+        const encodedKey = dbKeyToEncodedKey(dbKey);
+
+        // preserve general metadata ("|entryCount" and friends are
+        // cleared by our caller)
+        if (collectionMetaKeys.has(encodedKey)) continue;
+
+        if (encodedKey.startsWith('|')) {
+          // ordinal assignment; decref or de-recognize its vref
+          const keyVref = encodedKey.substring(1);
+          parseVatSlot(keyVref);
           if (hasWeakKeys) {
             vrm.removeRecognizableVref(keyVref, `${collectionID}`, true);
           } else {
             doMoreGC = vrm.removeReachableVref(keyVref) || doMoreGC;
           }
-          syscall.vatstoreDelete(prefix(`|${keyVref}`));
+        } else {
+          // a collection entry; decref slots from its value
+          const value = JSON.parse(syscall.vatstoreGet(dbKey));
+          doMoreGC =
+            value.slots.map(vrm.removeReachableVref).some(b => b) || doMoreGC;
         }
+
+        // in either case, delete the DB entry
+        syscall.vatstoreDelete(dbKey);
       }
+
       return doMoreGC;
     }
 
     function clearInternal(isDeleting, keyPatt, valuePatt) {
       let doMoreGC = false;
-      if (isDeleting || (matchAny(keyPatt) && matchAny(valuePatt))) {
+      if (isDeleting) {
         doMoreGC = clearInternalFull();
+        // |entryCount will be deleted along with metadata keys
+      } else if (matchAny(keyPatt) && matchAny(valuePatt)) {
+        doMoreGC = clearInternalFull();
+        if (!hasWeakKeys) {
+          syscall.vatstoreSet(prefix('|entryCount'), '0');
+        }
       } else {
+        let numDeleted = 0;
         for (const k of keys(keyPatt, valuePatt)) {
+          numDeleted += 1;
           doMoreGC = deleteInternal(k) || doMoreGC;
         }
-      }
-      if (!hasWeakKeys && !isDeleting) {
-        syscall.vatstoreSet(prefix('|entryCount'), '0');
+        updateEntryCount(-numDeleted);
       }
       return doMoreGC;
     }
@@ -587,6 +635,7 @@ export function makeCollectionManager(
           yield entry[1];
         }
       }
+      harden(iter);
       return iter();
     }
 
@@ -596,12 +645,13 @@ export function makeCollectionManager(
           yield entry;
         }
       }
+      harden(iter);
       return iter();
     }
 
     function countEntries(keyPatt, valuePatt) {
       let count = 0;
-      // eslint-disable-next-line no-use-before-define, no-unused-vars
+      // eslint-disable-next-line no-unused-vars
       for (const k of keys(keyPatt, valuePatt)) {
         count += 1;
       }
@@ -624,6 +674,34 @@ export function makeCollectionManager(
     const snapshotMap = (keyPatt, valuePatt) =>
       makeCopyMap(entries(keyPatt, valuePatt));
 
+    const addAllToSet = elems => {
+      if (typeof elems[Symbol.iterator] !== 'function') {
+        elems =
+          Object.isFrozen(elems) && isCopySet(elems)
+            ? getCopySetKeys(elems)
+            : failNotIterable(elems);
+      }
+      for (const elem of elems) {
+        addToSet(elem);
+      }
+    };
+
+    const addAllToMap = mapEntries => {
+      if (typeof mapEntries[Symbol.iterator] !== 'function') {
+        mapEntries =
+          Object.isFrozen(mapEntries) && isCopyMap(mapEntries)
+            ? getCopyMapEntries(mapEntries)
+            : failNotIterable(mapEntries);
+      }
+      for (const [key, value] of mapEntries) {
+        if (has(key)) {
+          set(key, value);
+        } else {
+          doInit(key, value, true);
+        }
+      }
+    };
+
     return {
       has,
       get,
@@ -635,6 +713,8 @@ export function makeCollectionManager(
       keys,
       values,
       entries,
+      addAllToSet,
+      addAllToMap,
       snapshotSet,
       snapshotMap,
       sizeInternal,
@@ -647,12 +727,23 @@ export function makeCollectionManager(
     const hasWeakKeys = storeKindInfo[kindName].hasWeakKeys;
     const raw = summonCollectionInternal(initial, collectionID, kindName);
 
-    const { has, get, init, addToSet, set, delete: del } = raw;
+    const {
+      has,
+      get,
+      init,
+      addToSet,
+      addAllToMap,
+      addAllToSet,
+      set,
+      delete: del,
+    } = raw;
     const weakMethods = {
       has,
       get,
       init,
       addToSet,
+      addAllToSet,
+      addAllToMap,
       set,
       delete: del,
     };
@@ -700,7 +791,9 @@ export function makeCollectionManager(
     const collection = summonCollectionInternal(false, collectionID, kindName);
 
     let doMoreGC = collection.clearInternal(true);
-    const { schemataCapData } = schemaCache.get(collectionID);
+    const record = schemaCache.get(collectionID);
+    assert(record !== undefined);
+    const { schemataCapData } = record;
     doMoreGC =
       schemataCapData.slots.map(vrm.removeReachableVref).some(b => b) ||
       doMoreGC;
@@ -747,13 +840,15 @@ export function makeCollectionManager(
     }
     const schemataCapData = serialize(harden(schemata));
     if (isDurable) {
-      schemataCapData.slots.forEach((vref, slotIndex) => {
+      for (const [slotIndex, vref] of schemataCapData.slots.entries()) {
         if (!vrm.isDurable(vref)) {
           throwNotDurable(vref, slotIndex, schemataCapData);
         }
-      });
+      }
+    }
+    for (const vref of schemataCapData.slots) {
+      vrm.addReachableVref(vref);
     }
-    schemataCapData.slots.forEach(vrm.addReachableVref);
 
     schemaCache.set(
       collectionID,
@@ -764,12 +859,48 @@ export function makeCollectionManager(
   }
 
   function collectionToMapStore(collection) {
-    const { snapshotSet: _, snapshotMap, ...rest } = collection;
-    return Far('mapStore', { snapshot: snapshotMap, ...rest });
+    const {
+      has,
+      get,
+      init,
+      set,
+      delete: del,
+      addAllToMap,
+      keys,
+      values,
+      entries,
+      snapshotMap,
+      getSize,
+      clear,
+    } = collection;
+    const mapStore = {
+      has,
+      get,
+      init,
+      set,
+      delete: del,
+      addAll: addAllToMap,
+      keys,
+      values,
+      entries,
+      snapshot: snapshotMap,
+      getSize,
+      clear,
+    };
+    return Far('mapStore', mapStore);
   }
 
   function collectionToWeakMapStore(collection) {
-    return Far('weakMapStore', collection);
+    const { has, get, init, set, delete: del, addAllToMap } = collection;
+    const weakMapStore = {
+      has,
+      get,
+      init,
+      set,
+      delete: del,
+      addAll: addAllToMap,
+    };
+    return Far('weakMapStore', weakMapStore);
   }
 
   function collectionToSetStore(collection) {
@@ -777,50 +908,35 @@ export function makeCollectionManager(
       has,
       addToSet,
       delete: del,
+      addAllToSet,
       keys,
-      getSize,
       snapshotSet,
+      getSize,
       clear,
     } = collection;
-    function* entries(patt) {
-      for (const k of keys(patt)) {
-        yield [k, k];
-      }
-    }
-    function addAll(elems) {
-      for (const elem of elems) {
-        addToSet(elem, null);
-      }
-    }
 
     const setStore = {
       has,
       add: addToSet,
-      addAll,
       delete: del,
+      addAll: addAllToSet,
       keys: patt => keys(patt),
       values: patt => keys(patt),
-      entries,
-      getSize: patt => getSize(patt),
       snapshot: snapshotSet,
+      getSize: patt => getSize(patt),
       clear,
     };
     return Far('setStore', setStore);
   }
 
   function collectionToWeakSetStore(collection) {
-    const { has, addToSet, delete: del } = collection;
-    function addAll(elems) {
-      for (const elem of elems) {
-        addToSet(elem);
-      }
-    }
+    const { has, addToSet, delete: del, addAllToSet } = collection;
 
     const weakSetStore = {
       has,
       add: addToSet,
-      addAll,
       delete: del,
+      addAll: addAllToSet,
     };
     return Far('weakSetStore', weakSetStore);
   }
@@ -829,7 +945,7 @@ export function makeCollectionManager(
    * Produce a big map.
    *
    * @template K,V
-   * @param {string} [label='map'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {MapStore<K,V>}
    */
@@ -873,7 +989,7 @@ export function makeCollectionManager(
    * Produce a weak big map.
    *
    * @template K,V
-   * @param {string} [label='weakMap'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {WeakMapStore<K,V>}
    */
@@ -902,7 +1018,7 @@ export function makeCollectionManager(
    * Produce a big set.
    *
    * @template K
-   * @param {string} [label='set'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {SetStore<K>}
    */
@@ -929,7 +1045,7 @@ export function makeCollectionManager(
    * Produce a weak big set.
    *
    * @template K
-   * @param {string} [label='weakSet'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {WeakSetStore<K>}
    */
@@ -1006,7 +1122,7 @@ export function makeCollectionManager(
    * remotables.
    *
    * @template K,V
-   * @param {string} [label='map'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {MapStore<K,V>}
    */
@@ -1018,7 +1134,7 @@ export function makeCollectionManager(
    * primitives, or remotables.
    *
    * @template K,V
-   * @param {string} [label='weakMap'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {WeakMapStore<K,V>}
    */
@@ -1030,7 +1146,7 @@ export function makeCollectionManager(
    * remotables.
    *
    * @template K
-   * @param {string} [label='set'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {SetStore<K>}
    */
@@ -1042,7 +1158,7 @@ export function makeCollectionManager(
    * primitives, or remotables.
    *
    * @template K
-   * @param {string} [label='weakSet'] - diagnostic label for the store
+   * @param {string} [label] - diagnostic label for the store
    * @param {StoreOptions} [options]
    * @returns {WeakSetStore<K>}
    */