@dusk-network/app-core 0.3.2 → 0.4.0

package/src/duskevm-withdrawals.js

@@ -127,6 +127,12 @@ const U32_MAX = (1n << 32n) - 1n;
 /** The low 240 bits of a withdrawal's nonce: its count, under its version. */
 const NONCE_COUNT = (1n << 240n) - 1n;
 
+// The newest final blocks a sync reads again on each run, the new ones
+// among them, and how long after it first covered all the final blocks it
+// reads them all again: each pass after waits as long again as it has been
+// since.
+const L2_OVERLAP = 50_000;
+const L2_RECHECK_MS = 60 * 60_000;
 // Answers a read is sent again for, a quarter second, then half a second later.
 const RETRY_STATUSES = new Set([429, 502, 503, 504]);
 const ATTEMPTS = 3;
@@ -2086,20 +2092,41 @@ function withdrawalFromLog(log) {
 const tooWide = (error) => /range|limit|too many|exceed/i.test(asError(error).message);
 
 /**
- * The highest DuskEVM block that can't be reorged away.
+ * The highest DuskEVM block that can't be reorged away, and the requests it
+ * took.
  *
  * @param {(method: string, params?: unknown[]) => Promise<any>} rpc
+ * @returns {Promise<{ head: number, requests: number }>}
  */
 async function finalHead(rpc) {
   try {
     const block = await rpc("eth_getBlockByNumber", ["finalized", false]);
 
-    if (block?.number) return Number(BigInt(block.number));
+    if (block?.number) return { head: Number(BigInt(block.number)), requests: 1 };
   } catch {
     // An RPC without "finalized": the latest block, less a margin.
   }
 
-  return Math.max(0, Number(BigInt(await rpc("eth_blockNumber", []))) - UNFINALIZED_MARGIN);
+  return { head: Math.max(0, Number(BigInt(await rpc("eth_blockNumber", []))) - UNFINALIZED_MARGIN), requests: 2 };
+}
+
+/**
+ * A sync's passes over the blocks it covered, as a state gave them, the one
+ * under way no higher than `target`; null for anything else.
+ *
+ * @param {unknown} value
+ * @param {number} target
+ * @returns {NonNullable<WithdrawalScan["recheck"]> | null}
+ */
+function recheckOf(value, target) {
+  const kept = /** @type {any} */ (value);
+  /** @param {unknown} ms */
+  const time = (ms) => typeof ms === "number" && Number.isFinite(ms);
+
+  if (!kept || !time(kept.at) || !time(kept.since)) return null;
+  if (kept.from !== null && !(Number.isSafeInteger(kept.from) && kept.from >= 0)) return null;
+
+  return { at: kept.at, from: kept.from === null ? null : Math.min(kept.from, target), since: kept.since };
 }
 
 /**
@@ -2114,7 +2141,8 @@ export async function syncWithdrawals({ address, bridge, duskEvm, maxRequests =
   if (!contract) throw new TypeError("No DuskEVM bridge to read");
 
   const rpc = evmRpc(duskEvm, net);
-  const target = await finalHead(rpc);
+  const { head: target, requests: asked } = await finalHead(rpc);
+  const time = now();
   // Coverage kept above the final head, as an older run read the latest, is
   // read again, and what was found there is dropped with it: it may have
   // been reorged away, or into another block.
@@ -2124,45 +2152,95 @@ export async function syncWithdrawals({ address, bridge, duskEvm, maxRequests =
   const entries = Object.fromEntries(Object.entries(state?.entries ?? {}).filter(([, entry]) => Number(entry?.blockNumber) <= target));
   /** @type {FoundWithdrawal[]} */
   const found = [];
-  let requests = 1;
+  let requests = asked;
+  let recheck = recheckOf(state?.recheck, target);
   const topics = [ETH_BRIDGE_INITIATED_TOPIC, pad(sender)];
 
-  while (requests < maxRequests) {
-    const gap = topGap(covered, target);
-
-    if (!gap) break;
-
-    const [a, b] = gap;
-    const lo = Math.max(a, b - window + 1);
+  /**
+   * The address's withdrawals in blocks `lo` to `hi`, those blocks covered;
+   * false where the RPC asks for a narrower window, which it then is.
+   *
+   * @param {number} lo
+   * @param {number} hi
+   */
+  const read = async (lo, hi) => {
     /** @type {any[]} */
     let logs;
 
     requests += 1;
 
     try {
-      logs = await rpc("eth_getLogs", [{ address: contract, fromBlock: toHex(lo), toBlock: toHex(b), topics }]);
+      logs = await rpc("eth_getLogs", [{ address: contract, fromBlock: toHex(lo), toBlock: toHex(hi), topics }]);
     } catch (error) {
       if (window > 1_000 && tooWide(error)) {
         window = Math.floor(window / 4);
-        continue;
+        return false;
       }
 
       throw error;
     }
 
     for (const log of logs ?? []) {
+      // One found before, read again, is kept as it was.
+      if (entries[String(log?.transactionHash).toLowerCase()]) continue;
+
       const entry = withdrawalFromLog(log);
 
-      if (!entry || entries[entry.hash]) continue;
+      if (!entry) continue;
 
       entries[entry.hash] = entry;
       found.push(entry);
     }
 
-    covered = addCoverage(covered, lo, b);
+    covered = addCoverage(covered, lo, hi);
+
+    return true;
+  };
+
+  // The newest final blocks, read again on each run, from the newest down,
+  // so the new ones first: in as many windows as they take, though no more
+  // than half the requests the run has left (one at least, where nothing
+  // else is to be read). While older blocks are still to be read, a run
+  // under the same final head as the last leaves them all to those.
+  const lowest = Math.max(0, target - L2_OVERLAP + 1);
+  const due = recheck !== null && (recheck.from !== null || time >= recheck.at);
+  const older = topGap(addCoverage(covered, lowest, target), target) !== null;
+  const half = Math.floor((maxRequests - requests) / 2);
+  const share = older && state?.target === target ? 0 : older || due ? half : Math.max(1, half);
+  let top = target;
+
+  for (let windows = 0; windows < share && top >= lowest && requests < maxRequests; windows += 1) {
+    const lo = Math.max(0, top - window + 1);
+
+    if (await read(lo, top)) top = lo - 1;
   }
 
-  return { found, requests, state: { covered, entries, target, updatedAt: now(), window } };
+  while (requests < maxRequests) {
+    const gap = topGap(covered, target);
+
+    if (gap) {
+      const [a, b] = gap;
+
+      await read(Math.max(a, b - window + 1), b);
+      continue;
+    }
+
+    // All the final blocks covered: they're read again, a pass at a time,
+    // from the newest down, with what a run has left.
+    if (recheck === null) recheck = { at: time + L2_RECHECK_MS, from: null, since: time };
+    if (recheck.from === null && time < recheck.at) break;
+
+    const from = recheck.from ?? target;
+    const lo = Math.max(0, from - window + 1);
+
+    if (!(await read(lo, from))) continue;
+
+    recheck = lo > 0 ? { ...recheck, from: lo - 1 } : { at: time + Math.max(L2_RECHECK_MS, time - recheck.since), from: null, since: recheck.since };
+  }
+
+  if (recheck === null && !topGap(covered, target)) recheck = { at: time + L2_RECHECK_MS, from: null, since: time };
+
+  return { found, requests, state: { covered, entries, ...(recheck ? { recheck } : {}), target, updatedAt: now(), window } };
 }
 
 /**

package/src/history.d.ts

@@ -119,6 +119,12 @@ export type HistoryEntry = {
   refund?: bigint;
   /** The memo as text, or as 0x hex when it isn't text; "" for none. */
   memo: string;
+  /**
+   * True when `memo` is bytes that aren't text, given as 0x hex, so a memo
+   * typed as 0x text isn't taken for them; left out otherwise. An entry an
+   * earlier version kept has none.
+   */
+  memoBinary?: boolean;
   /** Whether the account paid the fee. */
   payer: boolean;
 };
@@ -212,6 +218,13 @@ export type HistoryState = {
   [field: string]: unknown;
 };
 
+/**
+ * A store's check before a write, called with the state kept now, in the
+ * write's own transaction: when it throws, the store writes nothing, and
+ * may reject.
+ */
+export type StoreCheck = (kept: HistoryState | null) => void;
+
 /**
  * Where one account's history is kept on one network: the app's own
  * storage, which the app scopes and keys. Each method may return a value or
@@ -234,12 +247,21 @@ export type HistoryStore = {
     entries: KeptEntry[],
     update: (kept: HistoryState | null) => HistoryState
   ): Promise<HistoryState | void> | HistoryState | void;
-  /** Add details to kept entries, by hash; entries that aren't kept are left alone. */
-  patch(details: Map<string, EntryDetails & Record<string, unknown>>): Promise<void> | void;
+  /**
+   * Add details to kept entries, by hash; entries that aren't kept are left
+   * alone. A store that calls `check` first, in the same transaction, keeps
+   * no details on a history another run started over since.
+   */
+  patch(details: Map<string, EntryDetails & Record<string, unknown>>, check?: StoreCheck): Promise<void> | void;
   /** Kept entries, newest first: at most `limit`, below the height `before` when it's given. */
   list(options: { limit: number; before?: number }): Promise<KeptEntry[]> | KeptEntry[];
-  /** Forget what is kept of the old chain: the node is on another one now. */
-  clear(): Promise<void> | void;
+  /**
+   * Forget what is kept of the old chain: the node is on another one now. A
+   * store that calls `check` first, in the same transaction, forgets
+   * nothing once another run started the history over on the new chain,
+   * and keeps what that run read.
+   */
+  clear(check?: StoreCheck): Promise<void> | void;
 };
 
 /** When nodes last answered without the archive: the app's storage, by URL. */
@@ -472,13 +494,15 @@ export type History = {
    * One sync run: the newest heights first, then older ones, within its
    * budgets. Each step is kept as it lands, so a run that stops anywhere is
    * carried on by the next. A run whose head names another chain than the
-   * history kept starts it over, on what is left of its budget. Rejects as
-   * the store does, as after a wallet reset; with a NoArchiveError for a
-   * node without the archive; with a StaleHistoryError once another run
-   * overtook it; and with a RangeError, before any request, for fewer than
-   * 2 history requests.
+   * history kept starts it over, on what is left of its budget. A details
+   * request that fails ends the run's details, not the run: it resolves
+   * with `detailsError`, and the next run asks for them again. Rejects as
+   * the store does, as after a wallet reset; as a history request does;
+   * with a NoArchiveError for a node without the archive; with a
+   * StaleHistoryError once another run overtook it; and with a RangeError,
+   * before any request, for fewer than 2 history requests.
    */
-  sync(options?: SyncOptions): Promise<{ state: HistoryState; requests: number; added: number }>;
+  sync(options?: SyncOptions): Promise<{ state: HistoryState; requests: number; added: number; detailsError?: Error }>;
   /** Details for kept entries that need them, as those shown before a sync gave them any: one request per 50. */
   fillDetails(entries: ReadonlyArray<KeptEntry | ReadEntry>, options?: { onStep?: () => void }): Promise<void>;
   /**

package/src/history.js

@@ -395,14 +395,16 @@ function readHead(data) {
 const utf8 = new TextDecoder("utf-8", { fatal: true });
 
 /**
- * A memo as text when it is UTF-8 without control characters, else as 0x hex.
+ * A memo as text when it is UTF-8 without control characters, else as 0x hex
+ * (`binary`).
  *
  * @param {unknown} hex
+ * @returns {{ memo: string, binary: boolean }}
  */
-function memoText(hex) {
+function memoOf(hex) {
   const h = String(hex ?? "");
 
-  if (!h || h.length % 2 || !/^[0-9a-f]+$/i.test(h)) return "";
+  if (!h || h.length % 2 || !/^[0-9a-f]+$/i.test(h)) return { binary: false, memo: "" };
 
   const bytes = Uint8Array.from(h.match(/../g) ?? [], (b) => parseInt(b, 16));
 
@@ -410,12 +412,12 @@ function memoText(hex) {
     const text = utf8.decode(bytes);
 
     // eslint-disable-next-line no-control-regex
-    if (!/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(text)) return text;
+    if (!/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(text)) return { binary: false, memo: text };
   } catch {
     // not text
   }
 
-  return `0x${h.toLowerCase()}`;
+  return { binary: true, memo: `0x${h.toLowerCase()}` };
 }
 
 /** @param {any} receiver */
@@ -443,6 +445,7 @@ export function parseHistoryEntry(account, raw) {
   let selfValue = null;
   let payer = false;
   let memo = "";
+  let memoBinary = false;
   /** @type {EntryKind | null} */
   let kind = null;
   /** @type {string | null} */
@@ -476,7 +479,7 @@ export function parseHistoryEntry(account, raw) {
       const ours = d.sender === account || receiver === account;
 
       if (d.sender === account) payer = true;
-      if (ours) memo = memoText(d.memo);
+      if (ours) ({ binary: memoBinary, memo } = memoOf(d.memo));
 
       if (d.sender === account && receiver === account) {
         selfValue = (selfValue ?? 0n) + value;
@@ -574,7 +577,18 @@ export function parseHistoryEntry(account, raw) {
     else counterparty = direction === "out" ? sentTo : receivedFrom;
   }
 
-  return { blockHeight, counterparty: counterparty || null, direction, hash, kind, memo, payer, value, ...(refunded ? { refund: refunded } : {}) };
+  return {
+    blockHeight,
+    counterparty: counterparty || null,
+    direction,
+    hash,
+    kind,
+    memo,
+    ...(memoBinary ? { memoBinary: true } : {}),
+    payer,
+    value,
+    ...(refunded ? { refund: refunded } : {}),
+  };
 }
 
 /**
@@ -1071,14 +1085,16 @@ export function createHistory({ archive, account, store, network = null, callFac
   /**
    * Details for the newest kept entries that need them, or for `entries`,
    * a batch per request, each kept only while the history is still the one
-   * `chain` reads.
+   * `chain` reads. A request that fails rejects, or with `lenient`, as in a
+   * sync run, ends the details there and gives its error.
    *
-   * @param {{ chain: { genesis: string, kept: boolean }, entries?: ReadonlyArray<KeptEntry | ReadEntry>, maxRequests: number, onStep?: () => void }} options
+   * @param {{ chain: { genesis: string, kept: boolean }, entries?: ReadonlyArray<KeptEntry | ReadEntry>, maxRequests: number, onStep?: () => void, lenient?: boolean }} options
+   * @returns {Promise<Error | null>}
    */
-  async function fillDetails({ chain, entries, maxRequests, onStep }) {
+  async function fillDetails({ chain, entries, maxRequests, onStep, lenient = false }) {
     let requests = Math.max(0, Math.floor(maxRequests));
 
-    if (!requests) return;
+    if (!requests) return null;
 
     const due = entries ? entries.filter(needsDetails) : await pendingRows(DETAILS_BATCH * requests);
     // Arguments are asked only of the calls they're read for.
@@ -1087,7 +1103,16 @@ export function createHistory({ archive, account, store, network = null, callFac
 
     for (let i = 0; i < due.length && requests > 0; i += DETAILS_BATCH, requests -= 1) {
       const batch = due.slice(i, i + DETAILS_BATCH).map((e) => e.hash);
-      const details = await fetchTxDetails(archive, batch, { account, args, callFacts, network: net, now });
+      /** @type {Map<string, EntryDetails & Record<string, unknown>>} */
+      let details;
+
+      try {
+        details = await fetchTxDetails(archive, batch, { account, args, callFacts, network: net, now });
+      } catch (error) {
+        if (!lenient) throw error;
+
+        return error instanceof Error ? error : new Error(messageOf(error));
+      }
 
       // Asked for the bridge, and its call isn't the bridge's: read, and kept so.
       for (const [hash, found] of details) if (forBridge.has(hash) && found.bridge === undefined) found.bridge = null;
@@ -1096,11 +1121,25 @@ export function createHistory({ archive, account, store, network = null, callFac
       if (details.size) {
         if (staleFor(await store.getState(), chain)) throw new StaleHistoryError();
 
-        await store.patch(details);
+        /** @type {StaleHistoryError | null} */
+        let refused = null;
+
+        try {
+          // Checked again in the write's own transaction, by a store that can.
+          await store.patch(details, (kept) => {
+            if (staleFor(kept, chain)) throw (refused = new StaleHistoryError());
+          });
+        } catch (error) {
+          throw refused ?? error;
+        }
+
+        if (refused) throw refused;
       }
 
       onStep?.();
     }
+
+    return null;
   }
 
   /**
@@ -1146,7 +1185,7 @@ export function createHistory({ archive, account, store, network = null, callFac
    * @param {number} maxDetailRequests - 0 or more
    * @param {((state: HistoryState) => void) | undefined} onStep
    * @param {{ budget: { left: number }, head: { finalized: number, genesis: string } } | null} [restart]
-   * @returns {Promise<{ state: HistoryState, requests: number, added: number }>}
+   * @returns {Promise<{ state: HistoryState, requests: number, added: number, detailsError?: Error }>}
    */
   async function run(maxRequests, maxDetailRequests, onStep, restart = null) {
     /** @type {HistoryState | null} */
@@ -1154,6 +1193,9 @@ export function createHistory({ archive, account, store, network = null, callFac
     const budget = restart?.budget ?? { left: maxRequests };
     let detailsLeft = maxDetailRequests;
     let added = 0;
+    // A details request that failed: the run reads no more details, and goes on.
+    /** @type {Error | null} */
+    let detailsError = null;
     // The chain this run reads, once its head names it, and whether it has
     // seen the history kept: one cleared since isn't this run's to write.
     const chain = { genesis: "", kept: state !== null };
@@ -1207,7 +1249,8 @@ export function createHistory({ archive, account, store, network = null, callFac
 
       if (entries.length && added === entries.length && detailsLeft > 0) {
         detailsLeft -= 1;
-        await fillDetails({ chain, maxRequests: 1, onStep: report });
+        detailsError = await fillDetails({ chain, lenient: true, maxRequests: 1, onStep: report });
+        if (detailsError) detailsLeft = 0;
       }
     };
 
@@ -1237,7 +1280,21 @@ export function createHistory({ archive, account, store, network = null, callFac
     if (state?.genesis && genesis && state.genesis !== genesis) {
       if (restart) throw new Error("The store still keeps another chain's history");
 
-      await store.clear();
+      let started = false;
+
+      try {
+        // Only while it is still another chain's, checked in the clear's own
+        // transaction by a store that can: another run may have started it
+        // over on this chain since this one read it, and kept what it read.
+        await store.clear((kept) => {
+          if (kept === null || kept === undefined || kept.genesis === genesis) {
+            started = true;
+            throw new StaleHistoryError();
+          }
+        });
+      } catch (error) {
+        if (!started) throw error;
+      }
 
       return run(maxRequests, maxDetailRequests, onStep, { budget, head: /** @type {{ finalized: number, genesis: string }} */ (first.head) });
     }
@@ -1309,9 +1366,14 @@ export function createHistory({ archive, account, store, network = null, callFac
       await commit(range.entries, { set: { crowded: { height: lo, page: 2 }, window } });
     }
 
-    await fillDetails({ chain, maxRequests: detailsLeft, onStep: report });
+    detailsError ??= await fillDetails({ chain, lenient: true, maxRequests: detailsLeft, onStep: report });
 
-    return { added, requests: maxRequests - budget.left, state: /** @type {HistoryState} */ (state) };
+    return {
+      added,
+      requests: maxRequests - budget.left,
+      state: /** @type {HistoryState} */ (state),
+      ...(detailsError ? { detailsError } : {}),
+    };
   }
 
   return Object.freeze({

package/src/shielded.d.ts

@@ -8,10 +8,10 @@
  *   and the block at that height says which transaction made it: a
  *   `phoenix` event lists the nullifiers its transaction spent and the
  *   notes it made, byte for byte as the leaves hold them, and an event
- *   paying a shielded receiver from a public account or a contract names
- *   part of the note. A transaction that spends notes always makes change
- *   for its sender, so the notes' heights hold every transaction that spent
- *   the profile's notes too.
+ *   paying a shielded receiver from a public account or a contract makes
+ *   one note, and names its address and value. A transaction that spends
+ *   notes always makes change for its sender, so the notes' heights hold
+ *   every transaction that spent the profile's notes too.
  * - Asking for the blocks at those heights alone would tell the node where
  *   the profile's transactions are, so each one is asked for among four
  *   other heights that hold notes, drawn from the note tree's heights. The
@@ -133,7 +133,12 @@ export type ShieldedEntry = {
   /** The transaction's hash: 64 lower-case hex digits. */
   hash: string;
   blockHeight: number;
-  /** What it did: a transfer, a shield or an unshield, a contract call, or a stake operation. */
+  /**
+   * What it did: a transfer, a shield or an unshield, a contract call, or a
+   * stake operation. A conversion is a shield or an unshield by where its
+   * value went, as its event says, also for a profile that only got its
+   * change.
+   */
   kind: EntryKind;
   /**
    * Which way the value went for the profile's notes: "in" for what they
@@ -182,6 +187,15 @@ export type ShieldedStore = {
    * the same decoys.
    */
   put(record: ShieldedRecord): Promise<void> | void;
+  /**
+   * Optional: keep `change(kept)` in place of the record kept now, in one
+   * transaction, and give what was kept. With it, the decoys are drawn from
+   * those kept and kept at once, so two reads of the profile at once (two
+   * pages, a popup and a page) ask among the same decoys; without it, they
+   * are put, then read back, and two such reads can still draw their own.
+   * It rejects and passes over a history as `put` does.
+   */
+  update?(change: (kept: ShieldedRecord | null) => ShieldedRecord): Promise<ShieldedRecord | void> | ShieldedRecord | void;
 };
 
 /** The note tree's distinct heights, from its first `notes` leaves. */
@@ -333,7 +347,8 @@ export declare function fetchShieldedBlocks(
  * first. A transaction that spent the profile's notes sent what they held,
  * less what came back to them (change, the fee's refund) and the fee; one
  * that only made notes is what they received. A note no event of its block
- * names is left out, and reverted events don't count.
+ * names is left out, and reverted events don't count; two notes a block
+ * made for one address go to a transaction each, by their values.
  */
 export declare function shieldedEntries(input: {
   notes: ReadonlyArray<ValuedNote>;