@ledgerhq/live-common 27.0.0 → 27.0.1-next.0

package/src/account/serialization.ts

@@ -794,20 +794,19 @@ export function fromAccountRaw(rawAccount: AccountRaw): Account {
           fromCosmosResourcesRaw(cosmosResourcesRaw);
       break;
     }
-    case "tezos":
-      {
-        const tezosResourcesRaw = (rawAccount as TezosAccountRaw)
-          .tezosResources;
-        if (tezosResourcesRaw)
-          (res as TezosAccount).tezosResources =
-            fromTezosResourcesRaw(tezosResourcesRaw);
-      }
+    case "tezos": {
+      const tezosResourcesRaw = (rawAccount as TezosAccountRaw).tezosResources;
+      if (tezosResourcesRaw)
+        (res as TezosAccount).tezosResources =
+          fromTezosResourcesRaw(tezosResourcesRaw);
       break;
+    }
     case "bitcoin": {
       const bitcoinResourcesRaw = (rawAccount as BitcoinAccountRaw)
         .bitcoinResources;
-      (res as BitcoinAccount).bitcoinResources =
-        fromBitcoinResourcesRaw(bitcoinResourcesRaw);
+      if (bitcoinResourcesRaw)
+        (res as BitcoinAccount).bitcoinResources =
+          fromBitcoinResourcesRaw(bitcoinResourcesRaw);
       break;
     }
     case "algorand": {
@@ -818,20 +817,17 @@ export function fromAccountRaw(rawAccount: AccountRaw): Account {
           fromAlgorandResourcesRaw(algoResourcesRaw);
       break;
     }
-    case "polkadot":
-      {
-        const polkadotResourcesRaw = (rawAccount as PolkadotAccountRaw)
-          .polkadotResources;
-
-        if (polkadotResourcesRaw)
-          (res as PolkadotAccount).polkadotResources =
-            fromPolkadotResourcesRaw(polkadotResourcesRaw);
-      }
+    case "polkadot": {
+      const polkadotResourcesRaw = (rawAccount as PolkadotAccountRaw)
+        .polkadotResources;
+      if (polkadotResourcesRaw)
+        (res as PolkadotAccount).polkadotResources =
+          fromPolkadotResourcesRaw(polkadotResourcesRaw);
       break;
+    }
     case "elrond": {
       const elrondResourcesRaw = (rawAccount as ElrondAccountRaw)
         .elrondResources;
-
       if (elrondResourcesRaw)
         (res as ElrondAccount).elrondResources =
           fromElrondResourcesRaw(elrondResourcesRaw);
@@ -942,61 +938,104 @@ export function toAccountRaw(account: Account): AccountRaw {
   }
 
   switch (account.currency.family) {
-    case "tron":
-      (res as TronAccountRaw).tronResources = toTronResourcesRaw(
-        (account as TronAccount).tronResources
-      );
+    case "tron": {
+      const tronAccount = account as TronAccount;
+      if (tronAccount.tronResources) {
+        (res as TronAccountRaw).tronResources = toTronResourcesRaw(
+          tronAccount.tronResources
+        );
+      }
       break;
-    case "cosmos":
-      (res as CosmosAccountRaw).cosmosResources = toCosmosResourcesRaw(
-        (account as CosmosAccount).cosmosResources
-      );
+    }
+    case "osmosis": {
+      const cosmosAccount = account as CosmosAccount;
+      if (cosmosAccount.cosmosResources) {
+        (res as CosmosAccountRaw).cosmosResources = toCosmosResourcesRaw(
+          cosmosAccount.cosmosResources
+        );
+      }
       break;
-    case "osmosis":
-      (res as CosmosAccountRaw).cosmosResources = toCosmosResourcesRaw(
-        (account as CosmosAccount).cosmosResources
-      );
+    }
+    case "cosmos": {
+      const cosmosAccount = account as CosmosAccount;
+      if (cosmosAccount.cosmosResources) {
+        (res as CosmosAccountRaw).cosmosResources = toCosmosResourcesRaw(
+          cosmosAccount.cosmosResources
+        );
+      }
       break;
-    case "tezos":
-      (res as TezosAccountRaw).tezosResources = toTezosResourcesRaw(
-        (account as TezosAccount).tezosResources
-      );
+    }
+    case "tezos": {
+      const tezosAccount = account as TezosAccount;
+      if (tezosAccount.tezosResources) {
+        (res as TezosAccountRaw).tezosResources = toTezosResourcesRaw(
+          tezosAccount.tezosResources
+        );
+      }
       break;
-    case "bitcoin":
-      (res as BitcoinAccountRaw).bitcoinResources = toBitcoinResourcesRaw(
-        (account as BitcoinAccount).bitcoinResources
-      );
+    }
+    case "bitcoin": {
+      const bitcoinAccount = account as BitcoinAccount;
+      if (bitcoinAccount.bitcoinResources) {
+        (res as BitcoinAccountRaw).bitcoinResources = toBitcoinResourcesRaw(
+          bitcoinAccount.bitcoinResources
+        );
+      }
       break;
-    case "algorand":
-      (res as AlgorandAccountRaw).algorandResources = toAlgorandResourcesRaw(
-        (account as AlgorandAccount).algorandResources
-      );
+    }
+    case "algorand": {
+      const algorandAccount = account as AlgorandAccount;
+      if (algorandAccount.algorandResources) {
+        (res as AlgorandAccountRaw).algorandResources = toAlgorandResourcesRaw(
+          algorandAccount.algorandResources
+        );
+      }
       break;
-    case "polkadot":
-      (res as PolkadotAccountRaw).polkadotResources = toPolkadotResourcesRaw(
-        (account as PolkadotAccount).polkadotResources
-      );
+    }
+    case "polkadot": {
+      const polkadotAccount = account as PolkadotAccount;
+      if (polkadotAccount.polkadotResources) {
+        (res as PolkadotAccountRaw).polkadotResources = toPolkadotResourcesRaw(
+          polkadotAccount.polkadotResources
+        );
+      }
       break;
-    case "elrond":
-      (res as ElrondAccountRaw).elrondResources = toElrondResourcesRaw(
-        (account as ElrondAccount).elrondResources
-      );
+    }
+    case "elrond": {
+      const elrondAccount = account as ElrondAccount;
+      if (elrondAccount.elrondResources) {
+        (res as ElrondAccountRaw).elrondResources = toElrondResourcesRaw(
+          elrondAccount.elrondResources
+        );
+      }
       break;
-    case "cardano":
-      (res as CardanoAccountRaw).cardanoResources = toCardanoResourceRaw(
-        (account as CardanoAccount).cardanoResources
-      );
+    }
+    case "cardano": {
+      const cardanoAccount = account as CardanoAccount;
+      if (cardanoAccount.cardanoResources) {
+        (res as CardanoAccountRaw).cardanoResources = toCardanoResourceRaw(
+          cardanoAccount.cardanoResources
+        );
+      }
       break;
-    case "solana":
-      (res as SolanaAccountRaw).solanaResources = toSolanaResourcesRaw(
-        (account as SolanaAccount).solanaResources
-      );
+    }
+    case "solana": {
+      const solanaAccount = account as SolanaAccount;
+      if (solanaAccount.solanaResources) {
+        (res as SolanaAccountRaw).solanaResources = toSolanaResourcesRaw(
+          solanaAccount.solanaResources
+        );
+      }
       break;
-    case "crypto_org":
-      (res as CryptoOrgAccountRaw).cryptoOrgResources = toCryptoOrgResourcesRaw(
-        (account as CryptoOrgAccount).cryptoOrgResources
-      );
+    }
+    case "crypto_org": {
+      const crytpoOrgAccount = account as CryptoOrgAccount;
+      if (crytpoOrgAccount.cryptoOrgResources) {
+        (res as CryptoOrgAccountRaw).cryptoOrgResources =
+          toCryptoOrgResourcesRaw(crytpoOrgAccount.cryptoOrgResources);
+      }
       break;
+    }
   }
 
   if (swapHistory) {

package/src/bot/bot-test-context.test.ts

@@ -0,0 +1,37 @@
+import { botTest, getContext } from "./bot-test-context";
+import { formatError } from "./formatters";
+
+describe("test the botTest context itself", () => {
+  test("botTest", () => {
+    const f = () => {
+      try {
+        botTest("CTX", () => expect(true).toEqual(false));
+      } catch (e) {
+        return getContext(e);
+      }
+    };
+    expect(f()).toEqual("CTX");
+  });
+
+  test("double botTest", () => {
+    const f = () => {
+      try {
+        botTest("1", () => botTest("2", () => expect(true).toEqual(false)));
+      } catch (e) {
+        return getContext(e);
+      }
+    };
+    expect(f()).toEqual("1 > 2");
+  });
+
+  test("formatError", () => {
+    const f = () => {
+      try {
+        botTest("CTX", () => expect(true).toEqual(false));
+      } catch (e) {
+        return formatError(e);
+      }
+    };
+    expect(f()?.slice(0, 8)).toEqual("TEST CTX");
+  });
+});

package/src/bot/bot-test-context.ts

@@ -0,0 +1,24 @@
+type EnhancedError = Error & {
+  _bot_context?: string[];
+};
+
+// This provides a botTest("description", () => {}) helper in order to give context in bot expect tests
+export function botTest(description: string, f: () => void): void {
+  try {
+    f();
+  } catch (e) {
+    if (e instanceof Error) {
+      const err = e as EnhancedError;
+      err._bot_context = [description].concat(err._bot_context || []);
+      throw err;
+    }
+    throw e;
+  }
+}
+
+// retrieve the text context of a given error that was thrown from inside a botTest(...)
+export function getContext(error: any): string | undefined {
+  if (error instanceof Error) {
+    return ((error as EnhancedError)._bot_context || []).join(" > ");
+  }
+}

package/src/bot/cli.ts

@@ -0,0 +1,18 @@
+import "../__tests__/test-helpers/environment";
+import { bot } from ".";
+
+const { BOT_FILTER_CURRENCY, BOT_FILTER_FAMILY } = process.env;
+const arg: {
+  currency?: string;
+  family?: string;
+} = {};
+
+if (BOT_FILTER_CURRENCY) {
+  arg.currency = BOT_FILTER_CURRENCY;
+}
+
+if (BOT_FILTER_FAMILY) {
+  arg.family = BOT_FILTER_FAMILY;
+}
+
+bot(arg);

package/src/bot/engine.ts

@@ -2,7 +2,7 @@ import expect from "expect";
 import invariant from "invariant";
 import now from "performance-now";
 import sample from "lodash/sample";
-import { throwError, of, Observable } from "rxjs";
+import { throwError, of, Observable, OperatorFunction } from "rxjs";
 import {
   first,
   filter,
@@ -25,7 +25,6 @@ import {
   releaseSpeculosDevice,
   findAppCandidate,
 } from "../load/speculos";
-import deviceActions from "../generated/speculos-deviceActions";
 import type { AppCandidate } from "../load/speculos";
 import {
   formatReportForConsole,
@@ -50,8 +49,8 @@ import type {
   Operation,
   SignOperationEvent,
 } from "@ledgerhq/types-live";
-import type { CryptoCurrency } from "@ledgerhq/types-cryptoassets";
 import type { Transaction, TransactionStatus } from "../generated/types";
+import { botTest } from "./bot-test-context";
 
 let appCandidates;
 const localCache = {};
@@ -120,17 +119,18 @@ export async function runWithAppSpec<T extends Transaction>(
 
   try {
     device = await createSpeculosDevice(deviceParams);
+    appReport.appPath = device.appPath;
     const bridge = getCurrencyBridge(currency);
     const syncConfig = {
       paginationConfig: {},
     };
     let t = now();
     const preloadedData = await cache.prepareCurrency(currency);
-    const preloadTime = now() - t;
+    const preloadDuration = now() - t;
+    appReport.preloadDuration = preloadDuration;
     // Scan all existing accounts
+    const beforeScanTime = now();
     t = now();
-    let scanTime = 0;
-    const firstSyncDurations = {};
     let accounts = await bridge
       .scanAccounts({
         currency,
@@ -140,12 +140,6 @@ export async function runWithAppSpec<T extends Transaction>(
       .pipe(
         filter((e) => e.type === "discovered"),
         map((e) => e.account),
-        tap((account) => {
-          const dt = now() - t;
-          firstSyncDurations[account.id] = dt;
-          t = now();
-          scanTime += dt;
-        }),
         reduce<Account, Account[]>((all, a) => all.concat(a), []),
         timeoutWith(
           getEnv("BOT_TIMEOUT_SCAN_ACCOUNTS"),
@@ -155,7 +149,7 @@ export async function runWithAppSpec<T extends Transaction>(
         )
       )
       .toPromise();
-    appReport.scanTime = scanTime;
+    appReport.scanDuration = now() - beforeScanTime;
     // "Migrate" the FIRST and every {crossAccountFrequency} account to simulate an export/import (same logic as export to mobile) – default to every 10
     // this is made a subset of the accounts to help identify problem that would be specific to the "cross" or not.
     for (
@@ -171,21 +165,13 @@ export async function runWithAppSpec<T extends Transaction>(
       "unexpected empty accounts for " + currency.name
     );
     const preloadStats =
-      preloadTime > 10 ? ` (preload: ${formatTime(preloadTime)})` : "";
+      preloadDuration > 10 ? ` (preload: ${formatTime(preloadDuration)})` : "";
     reportLog(
       `Spec ${spec.name} found ${accounts.length} ${
         currency.name
       } accounts${preloadStats}. Will use ${formatAppCandidate(
         appCandidate as AppCandidate
-      )}\n${accounts
-        .map(
-          (a) =>
-            "(" +
-            formatTime(firstSyncDurations[a.id] || 0) +
-            ") " +
-            formatAccount(a, "head")
-        )
-        .join("\n")}\n`
+      )}\n${accounts.map((a) => formatAccount(a, "head")).join("\n")}\n`
     );
 
     if (accounts.every(isAccountEmpty)) {
@@ -204,6 +190,8 @@ export async function runWithAppSpec<T extends Transaction>(
     // we sequentially iterate on the initial account set to perform mutations
     const length = accounts.length;
     const totalTries = spec.multipleRuns || 1;
+    // dynamic buffer with ids of accounts that need a resync (between runs)
+    let accountIdsNeedResync: string[] = [];
 
     for (let j = 0; j < totalTries; j++) {
       for (let i = 0; i < length; i++) {
@@ -211,15 +199,24 @@ export async function runWithAppSpec<T extends Transaction>(
           "engine",
           `spec ${spec.name} sync all accounts (try ${j} run ${i})`
         );
-        // resync all accounts (necessary between mutations)
+        // resync all accounts that needs to be resynced
         t = now();
-        accounts = await promiseAllBatched(
+
+        const resynced = await promiseAllBatched(
           getEnv("SYNC_MAX_CONCURRENT"),
-          accounts,
+          accounts.filter((a) => accountIdsNeedResync.includes(a.id)),
           syncAccount
         );
+
+        accounts = accounts.map((a: Account) => {
+          const i = accountIdsNeedResync.indexOf(a.id);
+          return i !== -1 ? resynced[i] : a;
+        });
+
+        accountIdsNeedResync = [];
+
         appReport.accountsAfter = accounts;
-        const syncAllAccountsTime = now() - t;
+        const resyncAccountsDuration = now() - t;
         const account = accounts[i];
         const report = await runOnAccount({
           appCandidate,
@@ -228,9 +225,20 @@ export async function runWithAppSpec<T extends Transaction>(
           account,
           accounts,
           mutationsCount,
-          syncAllAccountsTime,
+          resyncAccountsDuration,
+          accountIdsNeedResync,
           preloadedData,
         });
+        if (report.finalAccount) {
+          // optim: no need to resync if all went well with finalAccount
+          const finalAccount: Account = report.finalAccount;
+          accountIdsNeedResync = accountIdsNeedResync.filter(
+            (id) => id !== finalAccount.id
+          );
+          accounts = accounts.map((a: Account) =>
+            a.id === finalAccount.id ? finalAccount : a
+          );
+        }
         // eslint-disable-next-line no-console
         console.log(formatReportForConsole(report));
         mutationReports.push(report);
@@ -261,7 +269,7 @@ export async function runWithAppSpec<T extends Transaction>(
     appReport.mutations = mutationReports;
     appReport.accountsAfter = accounts;
   } catch (e: any) {
-    console.error(e);
+    if (process.env.CI) console.error(e);
     appReport.fatalError = e;
     log("engine", `spec ${spec.name} failed with ${String(e)}`);
   } finally {
@@ -277,8 +285,9 @@ export async function runOnAccount<T extends Transaction>({
   device,
   account,
   accounts,
+  accountIdsNeedResync,
   mutationsCount,
-  syncAllAccountsTime,
+  resyncAccountsDuration,
   preloadedData,
 }: {
   appCandidate: any;
@@ -286,8 +295,9 @@ export async function runOnAccount<T extends Transaction>({
   device: any;
   account: any;
   accounts: any;
+  accountIdsNeedResync: string[];
   mutationsCount: Record<string, number>;
-  syncAllAccountsTime: number;
+  resyncAccountsDuration: number;
   preloadedData: any;
 }): Promise<MutationReport<T>> {
   const { mutations } = spec;
@@ -295,7 +305,7 @@ export async function runOnAccount<T extends Transaction>({
   const report: MutationReport<T> = {
     spec,
     appCandidate,
-    syncAllAccountsTime,
+    resyncAccountsDuration,
   };
 
   try {
@@ -348,7 +358,7 @@ export async function runOnAccount<T extends Transaction>({
           updates: r.updates,
         });
       } catch (error: any) {
-        console.error(error);
+        if (process.env.CI) console.error(error);
         unavailableMutationReasons.push({
           mutation,
           error,
@@ -425,7 +435,11 @@ export async function runOnAccount<T extends Transaction>({
 
     // without recovering mechanism, we simply assume an error is a failure
     if (errors.length) {
-      throw errors[0];
+      botTest("mutation must not have tx status errors", () => {
+        // all mutation must express transaction that are POSSIBLE
+        // recoveredFromTransactionStatus can also be used to solve this for tricky cases
+        throw errors[0];
+      });
     }
 
     mutationsCount[mutation.name] = (mutationsCount[mutation.name] || 0) + 1;
@@ -444,8 +458,7 @@ export async function runOnAccount<T extends Transaction>({
         }),
         autoSignTransaction({
           transport: device.transport,
-          deviceAction:
-            mutation.deviceAction || getImplicitDeviceAction(account.currency),
+          deviceAction: mutation.deviceAction || spec.genericDeviceAction,
           appCandidate,
           account,
           transaction,
@@ -462,19 +475,34 @@ export async function runOnAccount<T extends Transaction>({
       .toPromise();
     report.signedOperation = signedOperation;
     report.signedTime = now();
+
+    // at this stage, we are about to broadcast we assume we will need to resync receiver account
+    if (report.destination) {
+      accountIdsNeedResync.push(report.destination.id);
+    }
+    accountIdsNeedResync.push(account.id);
+
     // broadcast the transaction
     const optimisticOperation = getEnv("DISABLE_TRANSACTION_BROADCAST")
       ? signedOperation.operation
-      : await accountBridge.broadcast({
-          account,
-          signedOperation,
-        });
+      : await accountBridge
+          .broadcast({
+            account,
+            signedOperation,
+          })
+          .catch((e) => {
+            // wrap the error into some bot test context
+            botTest("during broadcast", () => {
+              throw e;
+            });
+          });
     report.optimisticOperation = optimisticOperation;
     report.broadcastedTime = now();
     log(
       "engine",
       `spec ${spec.name}/${account.name}/${optimisticOperation.hash} broadcasted`
     );
+
     // wait the condition are good (operation confirmed)
     const testBefore = now();
 
@@ -487,9 +515,11 @@ export async function runOnAccount<T extends Transaction>({
       );
 
       if (timedOut && !operation) {
-        throw new Error(
-          "could not find optimisticOperation " + optimisticOperation.id
-        );
+        botTest("waiting operation id to appear after broadcast", () => {
+          throw new Error(
+            "could not find optimisticOperation " + optimisticOperation.id
+          );
+        });
       }
 
       if (operation) {
@@ -533,8 +563,8 @@ export async function runOnAccount<T extends Transaction>({
       `spec ${spec.name}/${account.name}/${optimisticOperation.hash} confirmed`
     );
   } catch (error: any) {
-    console.error(error);
-    log("mutation-error", spec.name + ": " + formatError(error));
+    if (process.env.CI) console.error(error);
+    log("mutation-error", spec.name + ": " + formatError(error, true));
     report.error = error;
   }
 
@@ -572,7 +602,7 @@ export function autoSignTransaction<T extends Transaction>({
   account: Account;
   transaction: T;
   status: TransactionStatus;
-}) {
+}): OperatorFunction<SignOperationEvent, SignOperationEvent> {
   let sub;
   let observer;
   let state;
@@ -643,16 +673,6 @@ export function autoSignTransaction<T extends Transaction>({
     return of<SignOperationEvent>(e);
   });
 }
-export function getImplicitDeviceAction(currency: CryptoCurrency) {
-  const actions = deviceActions[currency.family];
-  const accept = actions && actions.acceptTransaction;
-  invariant(
-    accept,
-    "a acceptTransaction is missing for family %s",
-    currency.family
-  );
-  return accept;
-}
 
 function awaitAccountOperation({
   account,
@@ -667,6 +687,9 @@ function awaitAccountOperation({
   log("engine", "awaitAccountOperation on " + account.name);
   let syncCounter = 0;
   let acc = account;
+  let lastSync = now();
+  const loopDebounce = 1000;
+  const targetInterval = getEnv("SYNC_PENDING_INTERVAL");
 
   async function loop() {
     const operation = step(acc);
@@ -678,8 +701,10 @@ function awaitAccountOperation({
         operation,
       };
     }
+    const spent = now() - lastSync;
+    await delay(Math.max(loopDebounce, targetInterval - spent));
 
-    await delay(5000);
+    lastSync = now();
     log("engine", "sync #" + syncCounter++ + " on " + account.name);
     acc = await syncAccount(acc);
     const r = await loop();
@@ -698,30 +723,41 @@ function transactionTest<T>({
   const timingThreshold = 30 * 60 * 1000;
   // FIXME: .valueOf to do arithmetic operations on date with typescript
   const dt = Date.now().valueOf() - operation.date.valueOf();
-  invariant(dt > 0, "operation.date must not be in in future");
-  expect(dt).toBeLessThan(timingThreshold);
-  invariant(!operation.hasFailed, "operation has failed");
+  botTest("operation.date must not be in future", () =>
+    expect(dt).toBeGreaterThanOrEqual(0)
+  );
+  botTest("operation.date less than 30mn ago", () =>
+    expect(dt).toBeLessThan(timingThreshold)
+  );
+  botTest("operation must not failed", () => {
+    expect(!operation.hasFailed).toBe(true);
+  });
+
   const { blockAvgTime } = account.currency;
 
   if (blockAvgTime && account.blockHeight) {
     const expected = getOperationConfirmationNumber(operation, account);
     const expectedMax = Math.ceil(timingThreshold / blockAvgTime);
-    invariant(
-      expected <= expectedMax,
-      "There are way too much operation confirmation for a small amount of time. %s < %s",
-      expected,
-      expectedMax
+    botTest("low amount of confirmations", () =>
+      invariant(
+        expected <= expectedMax,
+        "There are way too much operation confirmation for a small amount of time. %s < %s",
+        expected,
+        expectedMax
+      )
     );
   }
 
-  invariant(
-    !optimisticOperation.value.isNaN(),
-    "optimisticOperation.value must not be NaN"
+  botTest("optimisticOperation.value must not be NaN", () =>
+    expect(!optimisticOperation.value.isNaN()).toBe(true)
   );
-  invariant(
-    !optimisticOperation.fee.isNaN(),
-    "optimisticOperation.fee must not be NaN"
+  botTest("optimisticOperation.fee must not be NaN", () =>
+    expect(!optimisticOperation.fee.isNaN()).toBe(true)
+  );
+  botTest("operation.value must not be NaN", () =>
+    expect(!operation.value.isNaN()).toBe(true)
+  );
+  botTest("operation.fee must not be NaN", () =>
+    expect(!operation.fee.isNaN()).toBe(true)
   );
-  invariant(!operation.value.isNaN(), "operation.value must not be NaN");
-  invariant(!operation.fee.isNaN(), "operation.fee must not be NaN");
 }