@ledgerhq/live-common 27.2.0 → 27.3.0-next.0

package/src/apps/__snapshots__/logic.test.ts.snap

@@ -368,21 +368,21 @@ Object {
       "name": "Dogecoin",
     },
     Object {
-      "blocks": 1,
-      "bytes": 4096,
+      "blocks": 10,
+      "bytes": 40960,
       "currencyId": "zcash",
       "name": "Zcash",
     },
   ],
   "appsSpaceBlocks": 60,
   "appsSpaceBytes": 245760,
-  "freeSpaceBlocks": 27,
-  "freeSpaceBytes": 110592,
+  "freeSpaceBlocks": 18,
+  "freeSpaceBytes": 73728,
   "osBlocks": 20,
   "osBytes": 81920,
   "shouldWarnMemory": false,
-  "totalAppsBlocks": 33,
-  "totalAppsBytes": 135168,
+  "totalAppsBlocks": 42,
+  "totalAppsBytes": 172032,
   "totalBlocks": 80,
   "totalBytes": 327680,
 }
@@ -390,16 +390,23 @@ Object {
 
 exports[`Scenario: order is preserved in uninstall action plan 1`] = `
 Object {
-  "apps": Array [],
+  "apps": Array [
+    Object {
+      "blocks": 1,
+      "bytes": 4096,
+      "currencyId": "zcash",
+      "name": "Zcash",
+    },
+  ],
   "appsSpaceBlocks": 60,
   "appsSpaceBytes": 245760,
-  "freeSpaceBlocks": 60,
-  "freeSpaceBytes": 245760,
+  "freeSpaceBlocks": 59,
+  "freeSpaceBytes": 241664,
   "osBlocks": 20,
   "osBytes": 81920,
   "shouldWarnMemory": false,
-  "totalAppsBlocks": 0,
-  "totalAppsBytes": 0,
+  "totalAppsBlocks": 1,
+  "totalAppsBytes": 4096,
   "totalBlocks": 80,
   "totalBytes": 327680,
 }

package/src/apps/logic.test.ts

@@ -313,8 +313,7 @@ const scenarios = [
             name: "Bitcoin",
           },
         ],
-        expectPlan:
-          "-XRP, -Ethereum Classic, -Ethereum, -Dogecoin, -Zcash, -Bitcoin",
+        expectPlan: "-XRP, -Ethereum Classic, -Ethereum, -Dogecoin, -Bitcoin",
         expectInstalled: "",
       },
     ],

package/src/apps/polyfill.test.ts

@@ -0,0 +1,24 @@
+import { Application } from "@ledgerhq/types-live";
+import { polyfillApplication } from "./polyfill";
+
+test("polyfillApplication set ethereum currency for ethereum app", () => {
+  const app: Application = {
+    id: 1,
+    name: "Ethereum",
+    description: undefined,
+    application_versions: [],
+    providers: [],
+    category: 0,
+    date_creation: "",
+    date_last_modified: "",
+    publisher: undefined,
+    currencyId: undefined,
+    authorName: undefined,
+    supportURL: undefined,
+    contactURL: undefined,
+    sourceURL: undefined,
+    compatibleWalletsJSON: undefined,
+  };
+  const polyfilled = polyfillApplication(app);
+  expect(polyfilled.currencyId).toEqual("ethereum");
+});

package/src/apps/polyfill.ts

@@ -7,7 +7,14 @@ import {
 import { App, Application } from "@ledgerhq/types-live";
 const directDep = {};
 const reverseDep = {};
+
+// whitelist dependencies
+export const whitelistDependencies = ["Decred", "Decred Test", "Zcash"];
+
 export function declareDep(name: string, dep: string) {
+  if (whitelistDependencies.includes(name)) {
+    return;
+  }
   directDep[name] = (directDep[name] || []).concat(dep);
   reverseDep[dep] = (reverseDep[dep] || []).concat(name);
 }
@@ -25,8 +32,6 @@ listCryptoCurrencies(true, true).forEach((a) => {
     declareDep(a.managerAppName + " Test", dep.managerAppName);
   }
 });
-// whitelist dependencies
-export const whitelistDependencies = ["Decred", "Decred Testnet"];
 
 // extra dependencies
 [

package/src/apps/support.ts

@@ -33,10 +33,11 @@ const appVersionsRequired = {
   Algorand: ">= 1.2.9",
   Polkadot: ">= 13.9250.0",
   Elrond: ">= 1.0.11",
-  Ethereum: ">= 1.9.17",
+  Ethereum: ">= 1.9.20 || 1.9.20-dev",
   Solana: ">= 1.2.0",
   Celo: ">= 1.1.8",
   "Cardano ADA": ">= 4.1.0",
+  Zcash: "> 2.0.6",
 };
 export function mustUpgrade(
   deviceModel: DeviceModelId,

package/src/bot/engine.ts

@@ -41,6 +41,7 @@ import type {
   TransactionTestInput,
   TransactionArg,
   TransactionRes,
+  TransactionDestinationTestInput,
 } from "./types";
 import { makeBridgeCacheSystem } from "../bridge/cache";
 import { accountDataToAccount, accountToAccountData } from "../cross";
@@ -91,7 +92,9 @@ export async function runWithAppSpec<T extends Transaction>(
   const { appQuery, currency, dependency } = spec;
   const appCandidate = findAppCandidate(appCandidates, appQuery);
   if (!appCandidate) {
-    console.warn("no app found for " + spec.name, { appQuery, appCandidates });
+    console.warn("no app found for " + spec.name);
+    console.warn(appQuery);
+    console.warn(JSON.stringify(appCandidates, undefined, 2));
   }
   invariant(
     appCandidate,
@@ -113,9 +116,24 @@ export async function runWithAppSpec<T extends Transaction>(
     coinapps,
   };
   let device;
-  const appReport: SpecReport<T> = {
-    spec,
-  };
+  const hintWarnings: string[] = [];
+  const appReport: SpecReport<T> = { spec, hintWarnings };
+
+  // staticly check that all mutations declared a test too (if no generic spec test)
+  if (!spec.test) {
+    const list = spec.mutations.filter((m) => !m.test);
+    if (list.length > 0) {
+      hintWarnings.push(
+        "mutations should define a test(): " +
+          list.map((m) => m.name).join(", ")
+      );
+    }
+  }
+
+  // staticly assess if testDestination is necessary
+  const mutationThatProducedDestinationsWithoutTests: MutationSpec<any>[] = [];
+  const mutationWithDestinationTestsWithoutDestination: MutationSpec<any>[] =
+    [];
 
   try {
     device = await createSpeculosDevice(deviceParams);
@@ -150,6 +168,16 @@ export async function runWithAppSpec<T extends Transaction>(
       )
       .toPromise();
     appReport.scanDuration = now() - beforeScanTime;
+
+    // check if there are more accounts than mutation declared as a hint for the dev
+    if (accounts.length <= spec.mutations.length) {
+      hintWarnings.push(
+        "There are not enough accounts to cover all mutations. Please increase the account target to at least " +
+          (spec.mutations.length + 1) +
+          " accounts"
+      );
+    }
+
     // "Migrate" the FIRST and every {crossAccountFrequency} account to simulate an export/import (same logic as export to mobile) – default to every 10
     // this is made a subset of the accounts to help identify problem that would be specific to the "cross" or not.
     for (
@@ -239,6 +267,33 @@ export async function runWithAppSpec<T extends Transaction>(
             a.id === finalAccount.id ? finalAccount : a
           );
         }
+        if (report.finalDestination) {
+          // optim: no need to resync if all went well with finalDestination
+          const finalDestination: Account = report.finalDestination;
+          accountIdsNeedResync = accountIdsNeedResync.filter(
+            (id) => id !== finalDestination.id
+          );
+          accounts = accounts.map((a: Account) =>
+            a.id === finalDestination.id ? finalDestination : a
+          );
+        } else if (report.mutation) {
+          const { mutation } = report;
+          if (report.destination) {
+            if (
+              !mutation.testDestination &&
+              !mutationThatProducedDestinationsWithoutTests.includes(mutation)
+            ) {
+              mutationThatProducedDestinationsWithoutTests.push(mutation);
+            }
+          } else {
+            if (
+              mutation.testDestination &&
+              !mutationWithDestinationTestsWithoutDestination.includes(mutation)
+            ) {
+              mutationWithDestinationTestsWithoutDestination.push(mutation);
+            }
+          }
+        }
         // eslint-disable-next-line no-console
         console.log(formatReportForConsole(report));
         mutationReports.push(report);
@@ -261,11 +316,32 @@ export async function runWithAppSpec<T extends Transaction>(
       mutationsCount = {};
     }
 
-    accounts = await promiseAllBatched(
-      getEnv("SYNC_MAX_CONCURRENT"),
-      accounts,
-      syncAccount
-    );
+    if (
+      mutationReports.every((r) => !r.mutation) &&
+      accounts.some((a) => a.spendableBalance.gt(spec.minViableAmount || 0))
+    ) {
+      hintWarnings.push(
+        "No mutation were found possible. Yet there are funds in the accounts, please investigate."
+      );
+    }
+
+    if (mutationThatProducedDestinationsWithoutTests.length) {
+      hintWarnings.push(
+        "mutations should define a testDestination(): " +
+          mutationThatProducedDestinationsWithoutTests
+            .map((m) => m.name)
+            .join(", ")
+      );
+    }
+    if (mutationWithDestinationTestsWithoutDestination.length) {
+      hintWarnings.push(
+        "mutations should NOT define a testDestination() because there are no 'destination' sibling account found: " +
+          mutationWithDestinationTestsWithoutDestination
+            .map((m) => m.name)
+            .join(", ")
+      );
+    }
+
     appReport.mutations = mutationReports;
     appReport.accountsAfter = accounts;
   } catch (e: any) {
@@ -327,6 +403,7 @@ export async function runOnAccount<T extends Transaction>({
       mutation: MutationSpec<T>;
       tx: T;
       updates: Array<Partial<T> | null | undefined>;
+      destination: Account | undefined;
     }> = [];
     const unavailableMutationReasons: Array<{
       mutation: MutationSpec<T>;
@@ -356,6 +433,7 @@ export async function runOnAccount<T extends Transaction>({
           tx: r.transaction,
           // $FlowFixMe what the hell
           updates: r.updates,
+          destination: r.destination,
         });
       } catch (error: any) {
         if (process.env.CI) console.error(error);
@@ -398,9 +476,10 @@ export async function runOnAccount<T extends Transaction>({
     }
 
     report.transaction = transaction;
-    report.destination = accounts.find(
-      (a) => a.freshAddress === transaction.recipient
-    );
+    const destination =
+      candidate.destination ||
+      accounts.find((a) => a.freshAddress === transaction.recipient);
+    report.destination = destination;
     status = await accountBridge.getTransactionStatus(account, transaction);
     report.status = status;
     report.statusTime = now();
@@ -480,6 +559,8 @@ export async function runOnAccount<T extends Transaction>({
     if (report.destination) {
       accountIdsNeedResync.push(report.destination.id);
     }
+    // even if the test will actively sync the account, we need to pessimisticly assume it won't, we may not reach the final step of it.
+    // after the runOnAccount() call, we actively remove from accountIdsNeedResync the account.id if it is actually sucessful
     accountIdsNeedResync.push(account.id);
 
     // broadcast the transaction
@@ -504,12 +585,11 @@ export async function runOnAccount<T extends Transaction>({
     );
 
     // wait the condition are good (operation confirmed)
+    // test() is run over and over until either timeout is reach OR success
     const testBefore = now();
-
+    const timeOut = mutation.testTimeout || spec.testTimeout || 30 * 1000;
     const step = (account) => {
-      const timedOut =
-        now() - testBefore >
-        (mutation.testTimeout || spec.testTimeout || 30 * 1000);
+      const timedOut = now() - testBefore > timeOut;
       const operation = account.operations.find(
         (o) => o.id === optimisticOperation.id
       );
@@ -543,6 +623,7 @@ export async function runOnAccount<T extends Transaction>({
             throw e;
           }
 
+          log("bot", "failed confirm test. trying again. " + String(e));
           // We will try again
           return;
         }
@@ -551,17 +632,80 @@ export async function runOnAccount<T extends Transaction>({
       return operation;
     };
 
-    const result = await awaitAccountOperation({
-      account,
-      step,
-    });
-    report.finalAccount = result.account;
-    report.operation = result.operation;
+    const result = await awaitAccountOperation<Operation>({ account, step });
+    const { account: finalAccount, value: operation } = result;
+    report.finalAccount = finalAccount;
+    report.operation = operation;
     report.confirmedTime = now();
     log(
       "engine",
       `spec ${spec.name}/${account.name}/${optimisticOperation.hash} confirmed`
     );
+
+    const destinationBeforeTransaction = destination;
+    if (destination && mutation.testDestination) {
+      const { testDestination } = mutation;
+      // test() is run over and over until either timeout is reach OR success
+      const ntestBefore = now();
+      const newTimeOut = Math.max(10000, timeOut - (ntestBefore - testBefore));
+      log(
+        "bot",
+        "remaining time to test destination: " +
+          (newTimeOut / 1000).toFixed(0) +
+          "s"
+      );
+      const sendingOperation = operation;
+      const step = (account) => {
+        const timedOut = now() - ntestBefore > newTimeOut;
+        let operation;
+        try {
+          operation = account.operations.find(
+            (op) => op.hash === sendingOperation.hash
+          );
+          botTest(
+            "destination account should receive an operation (by tx hash)",
+            () =>
+              invariant(
+                operation,
+                "no operation found with hash %s",
+                operation.hash
+              )
+          );
+          if (!operation) throw new Error();
+          const arg: TransactionDestinationTestInput<T> = {
+            transaction,
+            status,
+            sendingAccount: finalAccount,
+            sendingOperation,
+            operation,
+            destinationBeforeTransaction,
+            destination: account,
+          };
+          botTest("destination", () => testDestination(arg));
+          report.testDestinationDuration = now() - ntestBefore;
+        } catch (e) {
+          // We never reach the final test success
+          if (timedOut) {
+            report.testDestinationDuration = now() - ntestBefore;
+            throw e;
+          }
+          log(
+            "bot",
+            "failed destination confirm test. trying again. " + String(e)
+          );
+          // We will try again
+          return;
+        }
+        return operation;
+      };
+      const result = await awaitAccountOperation<Operation>({
+        account: destination,
+        step,
+      });
+      report.finalDestination = result.account;
+      report.finalDestinationOperation = result.value;
+      report.destinationConfirmedTime = now();
+    }
   } catch (error: any) {
     if (process.env.CI) console.error(error);
     log("mutation-error", spec.name + ": " + formatError(error, true));
@@ -674,15 +818,15 @@ export function autoSignTransaction<T extends Transaction>({
   });
 }
 
-function awaitAccountOperation({
+function awaitAccountOperation<T>({
   account,
   step,
 }: {
   account: Account;
-  step: (arg0: Account) => Operation | null | undefined;
+  step: (arg0: Account) => T | null | undefined;
 }): Promise<{
   account: Account;
-  operation: Operation;
+  value: T;
 }> {
   log("engine", "awaitAccountOperation on " + account.name);
   let syncCounter = 0;
@@ -692,13 +836,12 @@ function awaitAccountOperation({
   const targetInterval = getEnv("SYNC_PENDING_INTERVAL");
 
   async function loop() {
-    const operation = step(acc);
+    const value = step(acc);
 
-    if (operation) {
-      log("engine", "found " + operation.id);
+    if (value) {
       return {
         account: acc,
-        operation,
+        value,
       };
     }
     const spent = now() - lastSync;
@@ -720,14 +863,14 @@ function transactionTest<T>({
   optimisticOperation,
   account,
 }: TransactionTestInput<T>) {
-  const timingThreshold = 30 * 60 * 1000;
-  // FIXME: .valueOf to do arithmetic operations on date with typescript
-  const dt = Date.now().valueOf() - operation.date.valueOf();
+  const dt = Date.now() - operation.date.getTime();
+  const lowerThreshold = -60 * 1000; // -1mn accepted
+  const upperThreshold = 30 * 60 * 1000; // 30mn up
   botTest("operation.date must not be in future", () =>
-    expect(dt).toBeGreaterThanOrEqual(0)
+    expect(dt).toBeGreaterThan(lowerThreshold)
   );
   botTest("operation.date less than 30mn ago", () =>
-    expect(dt).toBeLessThan(timingThreshold)
+    expect(dt).toBeLessThan(upperThreshold)
   );
   botTest("operation must not failed", () => {
     expect(!operation.hasFailed).toBe(true);
@@ -737,7 +880,7 @@ function transactionTest<T>({
 
   if (blockAvgTime && account.blockHeight) {
     const expected = getOperationConfirmationNumber(operation, account);
-    const expectedMax = Math.ceil(timingThreshold / blockAvgTime);
+    const expectedMax = Math.ceil(upperThreshold / blockAvgTime);
     botTest("low amount of confirmations", () =>
       invariant(
         expected <= expectedMax,

package/src/bot/formatters.ts

@@ -74,6 +74,9 @@ export function formatReportForConsole<T extends Transaction>({
   operation,
   confirmedTime,
   finalAccount,
+  finalDestination,
+  finalDestinationOperation,
+  testDestinationDuration,
   testDuration,
   error,
 }: MutationReport<T>): string {
@@ -167,11 +170,21 @@ export function formatReportForConsole<T extends Transaction>({
   }
 
   if (finalAccount) {
-    str += `✔️ ${formatAccount(finalAccount, "basic")}\n`;
+    str += `✔️ ${formatAccount(finalAccount, "basic")}`;
   }
 
   if (testDuration) {
-    str += `(final state reached in ${formatTime(testDuration)})\n`;
+    str += `(in ${formatTime(testDuration)})\n`;
+  }
+
+  if (finalDestination && finalDestinationOperation) {
+    str += `✔️ destination operation ${formatOperation(finalDestination)(
+      finalDestinationOperation
+    )}\n`;
+  }
+
+  if (testDestinationDuration) {
+    str += `(in ${formatTime(testDestinationDuration)})\n`;
   }
 
   if (error) {

package/src/bot/index.ts

@@ -9,7 +9,7 @@ import invariant from "invariant";
 import flatMap from "lodash/flatMap";
 import { getEnv } from "../env";
 import allSpecs from "../generated/specs";
-import type { MutationReport, SpecReport } from "./types";
+import type { AppSpec, MutationReport, SpecReport } from "./types";
 import { promiseAllBatched } from "../promise";
 import {
   findCryptoCurrencyByKeyword,
@@ -17,7 +17,7 @@ import {
   formatCurrencyUnit,
   getFiatCurrencyByTicker,
 } from "../currencies";
-import { isAccountEmpty, toAccountRaw } from "../account";
+import { formatAccount, isAccountEmpty, toAccountRaw } from "../account";
 import { runWithAppSpec } from "./engine";
 import { formatReportForConsole, formatError, formatTime } from "./formatters";
 import {
@@ -28,6 +28,7 @@ import {
 import { getPortfolio } from "../portfolio/v2";
 import { Account } from "@ledgerhq/types-live";
 import { getContext } from "./bot-test-context";
+
 type Arg = Partial<{
   currency: string;
   family: string;
@@ -71,7 +72,7 @@ export async function bot({
     }
 
     for (const key in familySpecs) {
-      let spec = familySpecs[key];
+      let spec: AppSpec<any> = familySpecs[key];
 
       if (!isCurrencySupported(spec.currency) || spec.disabled) {
         continue;
@@ -81,7 +82,7 @@ export async function bot({
         if (mutation) {
           spec = {
             ...spec,
-            mutations: spec.mutation.filter((m) =>
+            mutations: spec.mutations.filter((m) =>
               new RegExp(mutation).test(m.name)
             ),
           };
@@ -96,7 +97,7 @@ export async function bot({
   const results: Array<SpecReport<any>> = await promiseAllBatched(
     getEnv("BOT_MAX_CONCURRENT"),
     specs,
-    (spec) => {
+    (spec: AppSpec<any>) => {
       const logs: string[] = [];
       specsLogs.push(logs);
       return runWithAppSpec(spec, (message) => {
@@ -109,6 +110,7 @@ export async function bot({
         mutations: [],
         accountsBefore: [],
         accountsAfter: [],
+        hintWarnings: [],
       }));
     }
   );
@@ -373,13 +375,31 @@ export async function bot({
 
   if (errorCases.length) {
     appendBody("<details>\n");
-    appendBody(`<summary>${errorCases.length} mutation errors</summary>\n\n`);
+    appendBody(
+      `<summary>❌ ${errorCases.length} mutation errors</summary>\n\n`
+    );
     errorCases.forEach((c) => {
       appendBody("```\n" + formatReportForConsole(c) + "\n```\n\n");
     });
     appendBody("</details>\n\n");
   }
 
+  const specWithWarnings = results.filter((s) => s.hintWarnings.length > 0);
+  if (specWithWarnings.length > 0) {
+    appendBody("<details>\n");
+    appendBody(
+      `<summary>⚠️ ${specWithWarnings.reduce(
+        (sum, s) => s.hintWarnings.length + sum,
+        0
+      )} spec hints</summary>\n\n`
+    );
+    specWithWarnings.forEach((s) => {
+      appendBody(`- Spec ${s.spec.name}:\n`);
+      s.hintWarnings.forEach((txt) => appendBody(`  - ${txt}\n`));
+    });
+    appendBody("</details>\n\n");
+  }
+
   appendBodyFullOnly("<details>\n");
 
   appendBodyFullOnly(
@@ -508,12 +528,9 @@ export async function bot({
 
     const beforeOps = countOps(r.accountsBefore);
     const afterOps = countOps(r.accountsAfter);
-    const firstAccount = (r.accountsAfter || r.accountsBefore || [])[0];
-    appendBody(
-      `| ${r.spec.name} (${
-        (r.accountsBefore || []).filter((a) => a.used).length
-      }) `
-    );
+    const accounts = r.accountsAfter || r.accountsBefore || [];
+    const firstAccount = accounts[0];
+    appendBody(`| ${r.spec.name} (${accounts.filter((a) => a.used).length}) `);
     appendBody(
       `| ${afterOps || beforeOps}${
         afterOps > beforeOps ? ` (+${afterOps - beforeOps})` : ""
@@ -525,6 +542,11 @@ export async function bot({
     );
     appendBody("|\n");
   });
+
+  appendBody("\n```\n");
+  appendBody(allAccountsAfter.map((a) => formatAccount(a, "head")).join("\n"));
+  appendBody("\n```\n");
+
   appendBody("\n</details>\n\n");
 
   // Add performance details
@@ -545,9 +567,9 @@ export async function bot({
   }
 
   appendBody(
-    "| Spec (accounts) | preload | scan | re-sync | tx status | sign op | broadcast | mutation confirm |\n"
+    "| Spec (accounts) | preload | scan | re-sync | tx status | sign op | broadcast | test | destination test |\n"
   );
-  appendBody("|---|---|---|---|---|---|---|---|\n");
+  appendBody("|---|---|---|---|---|---|---|---|---|\n");
 
   appendBody("| **TOTAL** |");
   appendBody(`**${formatTime(sumResults((r) => r.preloadDuration))}** |`);
@@ -585,15 +607,17 @@ export async function bot({
           ? m.confirmedTime - m.broadcastedTime
           : 0
       )
+    )}** |`
+  );
+  appendBody(
+    `**${formatTime(
+      sumResultsMutation((m) => m.testDestinationDuration || 0)
     )}** |\n`
   );
 
   results.forEach((r) => {
-    appendBody(
-      `| ${r.spec.name} (${
-        (r.accountsBefore || []).filter((a) => a.used).length
-      }) |`
-    );
+    const accounts = r.accountsAfter || r.accountsBefore || [];
+    appendBody(`| ${r.spec.name} (${accounts.filter((a) => a.used).length}) |`);
     appendBody(`${formatTime(r.preloadDuration || 0)} |`);
     appendBody(`${formatTime(r.scanDuration || 0)} |`);
     appendBody(
@@ -631,6 +655,11 @@ export async function bot({
             ? m.confirmedTime - m.broadcastedTime
             : 0
         )
+      )} |`
+    );
+    appendBody(
+      `${formatTime(
+        sumMutation(r.mutations, (m) => m.testDestinationDuration || 0)
       )} |\n`
     );
   });