@agoric/inter-protocol 0.13.2-dev-606cbd2.0 → 0.13.2-dev-8d17cce.0

package/src/vaultFactory/vaultManager.js

@@ -31,7 +31,6 @@ import {
   M,
   makeScalarBigMapStore,
   makeScalarBigSetStore,
-  makeScalarMapStore,
   prepareExoClassKit,
   provide,
 } from '@agoric/vat-data';
@@ -64,7 +63,7 @@ import { Phase, prepareVault } from './vault.js';
 
 const { details: X, Fail } = assert;
 
-const trace = makeTracer('VM', false);
+const trace = makeTracer('VM');
 
 /** @typedef {import('./storeUtils.js').NormalizedDebt} NormalizedDebt */
 /** @typedef {import('@agoric/time/src/types').RelativeTime} RelativeTime */
@@ -275,7 +274,7 @@ export const prepareVaultManagerKit = (
           AmountShape,
           M.arrayOf(TransferPartShape),
         ).returns(),
-        burnAndRecord: M.call(AmountShape, SeatShape).returns(),
+        burn: M.call(AmountShape, SeatShape).returns(),
         getAssetSubscriber: M.call().returns(SubscriberShape),
         getCollateralBrand: M.call().returns(BrandShape),
         getDebtBrand: M.call().returns(BrandShape),
@@ -355,6 +354,7 @@ export const prepareVaultManagerKit = (
          */
         start() {
           const { state, facets } = this;
+          trace(state.collateralBrand, 'helper.start()', state.vaultCounter);
           const {
             collateralBrand,
             collateralUnit,
@@ -366,6 +366,7 @@ export const prepareVaultManagerKit = (
           const ephemera = collateralEphemera(collateralBrand);
           ephemera.prioritizedVaults = makePrioritizedVaults(unsettledVaults);
 
+          trace('helper.start() making periodNotifier');
           const periodNotifier = E(timerService).makeNotifier(
             0n,
             factoryPowers
@@ -373,6 +374,7 @@ export const prepareVaultManagerKit = (
               .getChargingPeriod(),
           );
 
+          trace('helper.start() starting observe periodNotifier');
           void observeNotifier(periodNotifier, {
             updateState: updateTime =>
               facets.helper
@@ -392,6 +394,7 @@ export const prepareVaultManagerKit = (
             },
           });
 
+          trace('helper.start() making quoteNotifier from', priceAuthority);
           const quoteNotifier = E(priceAuthority).makeQuoteNotifier(
             collateralUnit,
             debtBrand,
@@ -401,14 +404,20 @@ export const prepareVaultManagerKit = (
             E(storageNode).makeChildNode('quotes'),
             marshaller,
           );
+          trace('helper.start() awaiting observe storedQuotesNotifier');
+          // NB: upon restart, there may not be a price for a while. If manager
+          // operations are permitted, ones the depend on price information will
+          // throw. See https://github.com/Agoric/agoric-sdk/issues/4317
           void observeNotifier(quoteNotifier, {
             updateState(value) {
+              trace('vaultManager got new collateral quote', value);
               ephemera.storedCollateralQuote = value;
             },
             fail(reason) {
               console.error('quoteNotifier failed to iterate', reason);
             },
           });
+          trace('helper.start() done');
         },
         /**
          * @param {Timestamp} updateTime
@@ -416,7 +425,7 @@ export const prepareVaultManagerKit = (
         async chargeAllVaults(updateTime) {
           const { state, facets } = this;
           const { collateralBrand, debtMint, poolIncrementSeat } = state;
-          trace('chargeAllVaults', collateralBrand, {
+          trace(collateralBrand, 'chargeAllVaults', {
             updateTime,
           });
 
@@ -486,28 +495,6 @@ export const prepareVaultManagerKit = (
             proceeds,
           );
         },
-
-        /** @type {(accounting: { overage: Amount<'nat'>, shortfall: Amount<'nat'> }) => void} */
-        recordShortfallAndProceeds(accounting) {
-          trace('recordShortfallAndProceeds', accounting);
-          const { state } = this;
-
-          const { overage, shortfall } = accounting;
-          // cumulative values
-          state.totalOverageReceived = AmountMath.add(
-            state.totalOverageReceived,
-            overage,
-          );
-          state.totalShortfallReceived = AmountMath.add(
-            state.totalShortfallReceived,
-            shortfall,
-          );
-          state.totalDebt = AmountMath.subtract(state.totalDebt, shortfall);
-
-          void E.when(factoryPowers.getShortfallReporter(), reporter =>
-            E(reporter).increaseLiquidationShortfall(shortfall),
-          );
-        },
         sendToReserve(penalty, seat, seatKeyword = 'Collateral') {
           const invitation =
             E(reservePublicFacet).makeAddCollateralInvitation();
@@ -545,9 +532,17 @@ export const prepareVaultManagerKit = (
 
           state.liquidatingDebt = AmountMath.add(state.liquidatingDebt, debt);
         },
-        markDoneLiquidating(debt, collateral) {
+        /**
+         *
+         * @param {Amount<'nat'>} debt
+         * @param {Amount<'nat'>} collateral
+         * @param {{ overage: Amount<'nat'>, shortfall: Amount<'nat'> }} accounting
+         */
+        markDoneLiquidating(debt, collateral, accounting) {
           const { state } = this;
 
+          // update liquidation state
+
           state.liquidatingCollateral = AmountMath.subtract(
             state.liquidatingCollateral,
             collateral,
@@ -556,6 +551,32 @@ export const prepareVaultManagerKit = (
             state.liquidatingDebt,
             debt,
           );
+
+          // record shortfall and proceeds
+
+          const { overage, shortfall } = accounting;
+          // cumulative values
+          state.totalOverageReceived = AmountMath.add(
+            state.totalOverageReceived,
+            overage,
+          );
+          state.totalShortfallReceived = AmountMath.add(
+            state.totalShortfallReceived,
+            shortfall,
+          );
+          state.totalDebt = AmountMath.subtract(state.totalDebt, shortfall);
+
+          E.when(
+            factoryPowers.getShortfallReporter(),
+            reporter => E(reporter).increaseLiquidationShortfall(shortfall),
+            err =>
+              console.error(
+                '🛠️ getShortfallReporter() failed during liquidation; repair by updating governance',
+                err,
+              ),
+          ).catch(err => {
+            console.error('🚨 failed to report liquidation shortfall', err);
+          });
         },
         /**
          * If interest was charged between liquidating and liquidated, erase it.
@@ -576,7 +597,7 @@ export const prepareVaultManagerKit = (
 
           state.totalDebt = AmountMath.add(state.totalDebt, debt);
         },
-        updateMetrics() {
+        writeMetrics() {
           const { state } = this;
           const { collateralBrand, retainedCollateralSeat, metricsTopicKit } =
             state;
@@ -628,10 +649,13 @@ export const prepareVaultManagerKit = (
           const accounting = liquidationResults(totalDebt, mintedProceeds);
           const { Collateral: collateralProceeds } = proceeds;
 
-          /** @type {(v: Vault) => void} */
-          const recordLiquidation = v => {
-            v.liquidated();
-            liquidatingVaults.delete(v);
+          /** @type {Array<Vault>} */
+          const vaultsToLiquidate = [];
+          const liquidateAll = () => {
+            for (const vault of vaultsToLiquidate) {
+              vault.liquidated();
+              liquidatingVaults.delete(vault);
+            }
           };
 
           const penaltyRate = facets.self
@@ -689,7 +713,7 @@ export const prepareVaultManagerKit = (
                   { Collateral: collat },
                 ]);
               }
-              recordLiquidation(vault);
+              vaultsToLiquidate.push(vault);
             }
             if (transfers.length > 0) {
               atomicRearrange(zcf, harden(transfers));
@@ -705,7 +729,11 @@ export const prepareVaultManagerKit = (
               vaultData.getSize(),
             );
 
-            facets.helper.markDoneLiquidating(totalDebt, totalCollateral);
+            facets.helper.markDoneLiquidating(
+              totalDebt,
+              totalCollateral,
+              accounting,
+            );
           } else if (AmountMath.isEmpty(collateralProceeds)) {
             // Flow #2a
 
@@ -760,7 +788,7 @@ export const prepareVaultManagerKit = (
                 const seat = vault.getVaultSeat();
                 transfers.push([liqSeat, seat, { Minted: mintedToReturn }]);
               }
-              recordLiquidation(vault);
+              vaultsToLiquidate.push(vault);
             }
 
             if (transfers.length > 0) {
@@ -772,7 +800,11 @@ export const prepareVaultManagerKit = (
               vaultData.getSize(),
             );
 
-            facets.helper.markDoneLiquidating(totalDebt, totalCollateral);
+            facets.helper.markDoneLiquidating(
+              totalDebt,
+              totalCollateral,
+              accounting,
+            );
           } else {
             // Flow #2b: There's unsold collateral; some vaults may be revived.
 
@@ -791,13 +823,22 @@ export const prepareVaultManagerKit = (
               : AmountMath.makeEmptyFromAmount(collateralProceeds);
 
             let collatRemaining = distributableCollateral;
-            let debtRemaining = totalDebt;
             /** @type {import('@agoric/zoe/src/contractSupport/atomicTransfer.js').TransferPart[]} */
             const transfers = [];
             let liquidated = 0;
-            /** @type {MapStore<string, Vault>} */
-            const vaultsToReinstate = makeScalarMapStore();
+            /** @type {Array<Vault>} */
+            const vaultsToReinstate = [];
+            const reinstateAll = () => {
+              const { prioritizedVaults } = collateralEphemera(collateralBrand);
+              for (const vault of vaultsToReinstate) {
+                const vaultId = vault.abortLiquidation();
+                prioritizedVaults.addVault(vaultId, vault);
+                liquidatingVaults.delete(vault);
+              }
+            };
+
             let collateralReduction = AmountMath.makeEmpty(collateralBrand);
+            let shortfallToReserve = accounting.shortfall;
 
             const reduceCollateral = amount =>
               (collateralReduction = AmountMath.add(
@@ -815,18 +856,19 @@ export const prepareVaultManagerKit = (
               if (
                 reconstituteVaults &&
                 AmountMath.isGTE(collatRemaining, collatPostDebt) &&
-                AmountMath.isGTE(debtRemaining, debtAmount)
+                AmountMath.isGTE(totalDebt, debtAmount)
               ) {
                 collatRemaining = AmountMath.subtract(
                   collatRemaining,
                   collatPostDebt,
                 );
-                debtRemaining = AmountMath.subtract(debtRemaining, debtAmount);
+                shortfallToReserve = AmountMath.subtract(
+                  shortfallToReserve,
+                  debtAmount,
+                );
                 const seat = vault.getVaultSeat();
-                const vaultId = vault.abortLiquidation();
-                liquidatingVaults.delete(vault);
                 // must reinstate after atomicRearrange(), so we record them.
-                vaultsToReinstate.init(vaultId, vault);
+                vaultsToReinstate.push(vault);
                 reduceCollateral(vaultDebt);
                 transfers.push([liqSeat, seat, { Collateral: collatPostDebt }]);
               } else {
@@ -838,7 +880,7 @@ export const prepareVaultManagerKit = (
                 );
 
                 reduceCollateral(vCollat);
-                recordLiquidation(vault);
+                vaultsToLiquidate.push(vault);
               }
             }
 
@@ -848,10 +890,7 @@ export const prepareVaultManagerKit = (
             if (transfers.length > 0) {
               atomicRearrange(zcf, harden(transfers));
             }
-            const { prioritizedVaults } = collateralEphemera(collateralBrand);
-            for (const [vaultId, vault] of vaultsToReinstate.entries()) {
-              prioritizedVaults.addVault(vaultId, vault);
-            }
+            reinstateAll();
 
             facets.helper.markCollateralLiquidated(
               collateralReduction,
@@ -859,14 +898,14 @@ export const prepareVaultManagerKit = (
               transfers.length,
             );
 
-            facets.helper.markRestoreDebt(
-              AmountMath.subtract(totalDebt, debtRemaining),
-            );
             facets.helper.sendToReserve(collatRemaining, liqSeat);
-            facets.helper.markDoneLiquidating(totalDebt, totalCollateral);
+            facets.helper.markDoneLiquidating(totalDebt, totalCollateral, {
+              ...accounting,
+              shortfall: shortfallToReserve,
+            });
           }
-          facets.helper.recordShortfallAndProceeds(accounting);
-          return facets.helper.updateMetrics();
+          liquidateAll();
+          return facets.helper.writeMetrics();
         },
       },
 
@@ -915,22 +954,20 @@ export const prepareVaultManagerKit = (
             toMint,
           );
           factoryPowers.mintAndTransfer(mintReceiver, toMint, fee, transfers);
-          state.totalDebt = AmountMath.add(state.totalDebt, toMint);
         },
         /**
          * @param {Amount<'nat'>} toBurn
          * @param {ZCFSeat} seat
          */
-        burnAndRecord(toBurn, seat) {
+        burn(toBurn, seat) {
           const { state } = this;
           const { collateralBrand } = this.state;
 
-          trace('burnAndRecord', collateralBrand, {
+          trace(collateralBrand, 'burn', {
             toBurn,
             totalDebt: state.totalDebt,
           });
           factoryPowers.burnDebt(toBurn, seat);
-          state.totalDebt = AmountMath.subtract(state.totalDebt, toBurn);
         },
         getAssetSubscriber() {
           return this.state.assetTopicKit.subscriber;
@@ -966,6 +1003,7 @@ export const prepareVaultManagerKit = (
          * @param {VaultId} vaultId
          * @param {import('./vault.js').VaultPhase} vaultPhase at the end of whatever change updated balances
          * @param {Vault} vault
+         * @returns {void}
          */
         handleBalanceChange(
           oldDebtNormalized,
@@ -1004,41 +1042,42 @@ export const prepareVaultManagerKit = (
               ),
               'Settled vaults must not be retained in storage',
             );
-          } else {
-            const isNew = AmountMath.isEmpty(oldDebtNormalized);
-            trace({ isNew });
-            if (!isNew) {
-              // its position in the queue is no longer valid
-
-              const vaultInStore = prioritizedVaults.removeVaultByAttributes(
-                oldDebtNormalized,
-                oldCollateral,
-                vaultId,
-              );
-              assert(
-                vault === vaultInStore,
-                'handleBalanceChange for two different vaults',
-              );
-              trace('removed', vault, vaultId);
-            }
+            return;
+          }
 
-            // replace in queue, but only if it can accrue interest or be liquidated (i.e. has debt).
-            // getCurrentDebt() would also work (0x = 0) but require more computation.
-            if (!AmountMath.isEmpty(vault.getNormalizedDebt())) {
-              prioritizedVaults.addVault(vaultId, vault);
-            }
+          const isNew = AmountMath.isEmpty(oldDebtNormalized);
+          trace(state.collateralBrand, { isNew });
+          if (!isNew) {
+            // its position in the queue is no longer valid
 
-            // totalCollateral += vault's collateral delta (post — pre)
-            state.totalCollateral = AmountMath.subtract(
-              AmountMath.add(
-                state.totalCollateral,
-                vault.getCollateralAmount(),
-              ),
+            const vaultInStore = prioritizedVaults.removeVaultByAttributes(
+              oldDebtNormalized,
               oldCollateral,
+              vaultId,
+            );
+            assert(
+              vault === vaultInStore,
+              'handleBalanceChange for two different vaults',
             );
-            // debt accounting managed through minting and burning
-            facets.helper.updateMetrics();
+            trace('removed', vault, vaultId);
           }
+
+          // replace in queue, but only if it can accrue interest or be liquidated (i.e. has debt).
+          // getCurrentDebt() would also work (0x = 0) but require more computation.
+          if (!AmountMath.isEmpty(vault.getNormalizedDebt())) {
+            prioritizedVaults.addVault(vaultId, vault);
+          }
+
+          // total += vault's delta (post — pre)
+          state.totalCollateral = AmountMath.subtract(
+            AmountMath.add(state.totalCollateral, vault.getCollateralAmount()),
+            oldCollateral,
+          );
+          state.totalDebt = AmountMath.subtract(
+            AmountMath.add(state.totalDebt, vault.getCurrentDebt()),
+            oldDebtNormalized,
+          );
+          void facets.helper.writeMetrics();
         },
       },
       self: {
@@ -1051,11 +1090,11 @@ export const prepareVaultManagerKit = (
          * @param {ZCFSeat} seat
          */
         async makeVaultKit(seat) {
-          trace('makeVaultKit');
           const {
             state,
             facets: { manager },
           } = this;
+          trace(state.collateralBrand, 'makeVaultKit');
           const { storageNode } = this.state;
           assert(marshaller, 'makeVaultKit missing marshaller');
           assert(storageNode, 'makeVaultKit missing storageNode');
@@ -1069,7 +1108,7 @@ export const prepareVaultManagerKit = (
           ).makeChildNode(`vault${vaultId}`);
 
           const { self: vault } = makeVault(manager, vaultId, vaultStorageNode);
-          trace('makeVaultKit made vault', vault);
+          trace(state.collateralBrand, 'makeVaultKit made vault', vault);
 
           try {
             // TODO `await` is allowed until the above ordering is fixed
@@ -1168,7 +1207,7 @@ export const prepareVaultManagerKit = (
             liquidatingVaults,
             lockedQuote,
           } = state;
-          trace('considering liquidation');
+          trace(collateralBrand, 'considering liquidation');
 
           const { prioritizedVaults } = collateralEphemera(collateralBrand);
           assert(factoryPowers && prioritizedVaults && zcf);
@@ -1191,6 +1230,9 @@ export const prepareVaultManagerKit = (
               debtBrand,
               collateralBrand,
             );
+          // reset lockedQuote after we've used it for the liquidation decision
+          state.lockedQuote = undefined;
+
           if (vaultData.getSize() === 0) {
             return;
           }
@@ -1201,7 +1243,7 @@ export const prepareVaultManagerKit = (
           );
 
           helper.markLiquidating(totalDebt, totalCollateral);
-          helper.updateMetrics();
+          void helper.writeMetrics();
 
           const { userSeatPromise, deposited } = await E.when(
             E(auctionPF).makeDepositInvitation(),
@@ -1227,7 +1269,7 @@ export const prepareVaultManagerKit = (
           );
 
           trace(`LiqV after long wait`, proceeds);
-          helper.distributeProceeds(
+          return helper.distributeProceeds(
             proceeds,
             totalDebt,
             storedCollateralQuote,
@@ -1253,7 +1295,7 @@ export const prepareVaultManagerKit = (
         );
 
         // push initial state of metrics
-        helper.updateMetrics();
+        void helper.writeMetrics();
       },
     },
   );
@@ -1289,6 +1331,7 @@ export const prepareVaultManagerKit = (
  * @param {import('@agoric/vat-data').Baggage} baggage
  */
 export const provideAndStartVaultManagerKits = baggage => {
+  trace('provideAndStartVaultManagerKits start');
   const key = 'vaultManagerKits';
 
   const noKits = /** @type {VaultManagerKit[]} */ (harden([]));
@@ -1297,6 +1340,7 @@ export const provideAndStartVaultManagerKits = baggage => {
     kit.helper.start();
   }
 
+  trace('provideAndStartVaultManagerKits returning');
   return {
     /** @type {(kit: VaultManagerKit) => void} */
     add: kit => {