@forestrie/forestrie-cli 0.8.0

package/src/lib/checkpoint-provider.ts

@@ -0,0 +1,297 @@
+/**
+ * Checkpoint-chain providers (FOR-418 Phase 2, plan-2607-32).
+ *
+ * A *provider* yields the log's ordered chain of consistency-proof links folded
+ * into the accumulator at each seal, so consumers (verify, `resolve-receipt`)
+ * work over `CheckpointLink[]` regardless of source. Two tile-free sources:
+ *
+ * - **chain calldata** (`calldataCheckpointChain`) — the `publishCheckpoint`
+ *   transactions carry the whole `ConsistencyReceipt` (Phase 1).
+ * - **retained `.sth`** (`sthCheckpointChain`) — the store's signed checkpoints.
+ *
+ * TRUST (important — see the Phase 2 review, plan-2607-32 R1): a `CheckpointLink`
+ * accumulator here is **RPC-/store-asserted, NOT authenticated by this module**.
+ * The provider folds and does a cheap self-consistency cross-check (calldata
+ * fold vs the `CheckpointPublished` event accumulator), but it does **not**
+ * verify the COSE signature. So that the consumer *can*, each directly-signed
+ * link carries its `seal` (the signature + delegation for calldata; the
+ * checkpoint bytes for `.sth`); the consumer applies the genesis/known-key trust
+ * root against those seals (Phase 4). Both sources decode to the identical
+ * `CheckpointConsistencyProof` shape and fold identically — the parity that
+ * makes them interchangeable (see the tests).
+ *
+ * Each link also carries the raw `proof` it was folded from (its per-peak
+ * `paths`), so a consumer that needs the climb material rather than just the
+ * folded accumulator — `resolve-receipt`'s freshen (Phase 3c), which extends an
+ * old inclusion path to the latest peak — gets it in one pass with no re-decode
+ * and no second RPC sweep: `links.map((l) => l.proof)` is exactly freshen's
+ * ordered `consistencyProofs`. Consumers that only fold (verify) ignore it.
+ */
+import {
+  computeCheckpointAccumulator,
+  checkpointConsistencyProof,
+  type CheckpointConsistencyProof,
+} from "@forestrie/receipt-verify";
+import { fetchPublishedCheckpoints } from "./verify-eventscan.js";
+import {
+  fetchPublishCheckpointCalldata,
+  type CalldataDelegation,
+} from "./decode-checkpoint-calldata.js";
+import { bytesEqual } from "./bytes.js";
+
+/**
+ * The signed checkpoint a link's accumulator is sealed by, retained so the
+ * consumer can verify it (this module does not). Present on the directly-signed
+ * link of each unit — every `.sth`; the FINAL link of each calldata tx's proof
+ * segment (intermediate links in a multi-proof tx are unsigned, transitively
+ * trusted via the fold to the next sealed link).
+ */
+export type CheckpointSeal =
+  | {
+      kind: "calldata";
+      protectedHeader: Uint8Array;
+      signature: Uint8Array;
+      delegation: CalldataDelegation;
+    }
+  | { kind: "sth"; checkpointBytes: Uint8Array };
+
+/** One folded link: the accumulator committed at `treeSize2`. */
+export type CheckpointLink = {
+  treeSize1: bigint;
+  treeSize2: bigint;
+  /** Folded accumulator at `treeSize2` (descending-height / contract order). */
+  accumulator: Uint8Array[];
+  /** The raw consistency proof this link was folded from — its per-peak `paths`
+   * (the tile-free climb material). Retained so a path-extending consumer
+   * (freshen) gets it without a re-decode; folding consumers ignore it. */
+  proof: CheckpointConsistencyProof;
+  /** The signature material to verify this link's accumulator, when directly
+   * signed (see {@link CheckpointSeal}). The provider does NOT verify it. */
+  seal?: CheckpointSeal;
+  /** Where this link came from (a tx hash, an `.sth` name) — for narration. */
+  sourceRef?: string;
+};
+
+/**
+ * Fold an ordered consistency-proof chain into per-link accumulators. Each
+ * link's base must equal the previous link's sealed size — a mismatch is the
+ * legacy / non-contiguous signature and throws. For a suffix chain, pass BOTH
+ * `accumulatorFrom` and `accumulatorFromSize` (the size that seed is for); the
+ * first link's `treeSize1` is bound to it (R2). Without a seed the chain must
+ * start at base 0.
+ */
+export async function foldProofChain(
+  proofs: readonly CheckpointConsistencyProof[],
+  opts: {
+    accumulatorFrom?: Uint8Array[];
+    accumulatorFromSize?: bigint;
+    seals?: readonly (CheckpointSeal | undefined)[];
+    sourceRefs?: readonly string[];
+  } = {},
+): Promise<CheckpointLink[]> {
+  let accumulator = opts.accumulatorFrom ?? [];
+  let expectedBase: bigint;
+  if (opts.accumulatorFrom !== undefined) {
+    if (opts.accumulatorFromSize === undefined) {
+      throw new Error(
+        "foldProofChain: accumulatorFrom requires accumulatorFromSize (the size the seed accumulator is for)",
+      );
+    }
+    expectedBase = opts.accumulatorFromSize;
+  } else {
+    expectedBase = 0n;
+  }
+  const links: CheckpointLink[] = [];
+  for (let i = 0; i < proofs.length; i++) {
+    const p = proofs[i]!;
+    if (p.treeSize1 !== expectedBase) {
+      throw new Error(
+        `checkpoint chain is not contiguous at link ${i}: base ${p.treeSize1} != expected ${expectedBase}`,
+      );
+    }
+    accumulator = await computeCheckpointAccumulator(p, accumulator);
+    const link: CheckpointLink = {
+      treeSize1: p.treeSize1,
+      treeSize2: p.treeSize2,
+      accumulator,
+      proof: p,
+    };
+    const seal = opts.seals?.[i];
+    if (seal !== undefined) link.seal = seal;
+    const ref = opts.sourceRefs?.[i];
+    if (ref !== undefined) link.sourceRef = ref;
+    links.push(link);
+    expectedBase = p.treeSize2;
+  }
+  return links;
+}
+
+/**
+ * Retained-`.sth` provider: decode each checkpoint's embedded consistency proof
+ * and fold. `checkpoints` are the raw `.sth` bytes in ascending massif order;
+ * `sourceRefs` optionally names them (e.g. filenames) for narration.
+ */
+export async function sthCheckpointChain(
+  checkpoints: readonly Uint8Array[],
+  opts: {
+    accumulatorFrom?: Uint8Array[];
+    accumulatorFromSize?: bigint;
+    sourceRefs?: readonly string[];
+  } = {},
+): Promise<CheckpointLink[]> {
+  const proofs = checkpoints.map((bytes) => checkpointConsistencyProof(bytes));
+  const seals: CheckpointSeal[] = checkpoints.map((bytes) => ({
+    kind: "sth",
+    checkpointBytes: bytes,
+  }));
+  return foldProofChain(proofs, {
+    seals,
+    ...(opts.accumulatorFrom !== undefined
+      ? { accumulatorFrom: opts.accumulatorFrom }
+      : {}),
+    ...(opts.accumulatorFromSize !== undefined
+      ? { accumulatorFromSize: opts.accumulatorFromSize }
+      : {}),
+    ...(opts.sourceRefs !== undefined ? { sourceRefs: opts.sourceRefs } : {}),
+  });
+}
+
+/** Run `fn` over `items` with bounded concurrency, preserving order (R5). */
+async function mapBounded<T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const out: R[] = new Array(items.length);
+  let next = 0;
+  async function worker(): Promise<void> {
+    for (;;) {
+      const i = next++;
+      if (i >= items.length) return;
+      out[i] = await fn(items[i]!, i);
+    }
+  }
+  const n = Math.min(Math.max(1, limit), items.length || 1);
+  await Promise.all(Array.from({ length: n }, () => worker()));
+  return out;
+}
+
+function accumulatorsEqual(a: Uint8Array[], b: Uint8Array[]): boolean {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) if (!bytesEqual(a[i]!, b[i]!)) return false;
+  return true;
+}
+
+/**
+ * Chain-calldata provider: find the log's `CheckpointPublished` transactions
+ * (ascending), read each `publishCheckpoint`'s calldata, concatenate the
+ * embedded consistency-proof chains, and fold. Cross-checks each tx's folded
+ * accumulator against the `CheckpointPublished` event's accumulator (R3), and
+ * retains each tx's seal for the consumer to verify (R1); it does NOT verify
+ * signatures here (Phase 4). A bounded `--from-block` scan is only valid with a
+ * trusted seed at that block (R2): pass `accumulatorFrom` + `accumulatorFromSize`.
+ */
+export async function calldataCheckpointChain(opts: {
+  univocity: string;
+  logId: string;
+  rpcUrl: string;
+  fromBlock?: bigint | undefined;
+  accumulatorFrom?: Uint8Array[] | undefined;
+  accumulatorFromSize?: bigint | undefined;
+  fetchImpl?: typeof fetch;
+}): Promise<CheckpointLink[]> {
+  if (opts.fromBlock !== undefined && opts.accumulatorFrom === undefined) {
+    throw new Error(
+      "calldataCheckpointChain: --from-block must be paired with a trusted seed (accumulatorFrom + accumulatorFromSize) — a bounded scan starts mid-chain and cannot fold from base 0",
+    );
+  }
+  const published = await fetchPublishedCheckpoints({
+    univocity: opts.univocity,
+    logId: opts.logId,
+    rpcUrl: opts.rpcUrl,
+    fromBlock: opts.fromBlock,
+    ...(opts.fetchImpl !== undefined ? { fetchImpl: opts.fetchImpl } : {}),
+  });
+  const decoded = await mapBounded(published, 8, (cp) =>
+    fetchPublishCheckpointCalldata({
+      rpcUrl: opts.rpcUrl,
+      txHash: cp.txHash,
+      ...(opts.fetchImpl !== undefined ? { fetchImpl: opts.fetchImpl } : {}),
+    }),
+  );
+
+  const proofs: CheckpointConsistencyProof[] = [];
+  const seals: (CheckpointSeal | undefined)[] = [];
+  const sourceRefs: string[] = [];
+  const finalLinkIndex: number[] = []; // link index of each tx's last proof
+  for (let t = 0; t < published.length; t++) {
+    const d = decoded[t]!;
+    const n = d.consistencyProofs.length;
+    d.consistencyProofs.forEach((p, k) => {
+      proofs.push(p);
+      sourceRefs.push(published[t]!.txHash);
+      seals.push(
+        k === n - 1
+          ? {
+              kind: "calldata",
+              protectedHeader: d.protectedHeader,
+              signature: d.signature,
+              delegation: d.delegation,
+            }
+          : undefined,
+      );
+    });
+    finalLinkIndex.push(proofs.length - 1);
+  }
+
+  const links = await foldProofChain(proofs, {
+    seals,
+    sourceRefs,
+    ...(opts.accumulatorFrom !== undefined
+      ? { accumulatorFrom: opts.accumulatorFrom }
+      : {}),
+    ...(opts.accumulatorFromSize !== undefined
+      ? { accumulatorFromSize: opts.accumulatorFromSize }
+      : {}),
+  });
+
+  // R3: the folded accumulator at each tx's seal MUST equal the accumulator the
+  // CheckpointPublished event reported (both from the RPC — this catches a fold
+  // bug or an RPC serving inconsistent event/calldata, not a fully malicious
+  // RPC; the signature seal is the real anchor, applied by the consumer).
+  for (let t = 0; t < published.length; t++) {
+    const link = links[finalLinkIndex[t]!]!;
+    const cp = published[t]!;
+    if (link.treeSize2 !== cp.size) {
+      throw new Error(
+        `calldata size ${link.treeSize2} disagrees with CheckpointPublished size ${cp.size} (tx ${cp.txHash})`,
+      );
+    }
+    if (!accumulatorsEqual(link.accumulator, cp.accumulator)) {
+      throw new Error(
+        `folded calldata accumulator disagrees with the CheckpointPublished event at size ${cp.size} (tx ${cp.txHash}) — inconsistent RPC data`,
+      );
+    }
+  }
+  return links;
+}
+
+/**
+ * Find the newest link whose folded accumulator contains `peak` (a receipt's
+ * recomputed peak). Newest-first: a match at any authenticated link proves the
+ * receipt — later links' proofs commit it forward. Null when no link covers it.
+ */
+export function findPeakInChain(
+  links: readonly CheckpointLink[],
+  peak: Uint8Array,
+): { link: CheckpointLink; matchedPeak: number } | null {
+  for (let i = links.length - 1; i >= 0; i--) {
+    const link = links[i]!;
+    for (let j = 0; j < link.accumulator.length; j++) {
+      if (bytesEqual(peak, link.accumulator[j]!)) {
+        return { link, matchedPeak: j };
+      }
+    }
+  }
+  return null;
+}

package/src/lib/complete-grant-derive.ts

@@ -0,0 +1,237 @@
+/**
+ * FOR-344 offline grant completion. `register-grant` gets the receipt +
+ * idtimestamp from the operator (register → 303 → poll → receipt);
+ * `complete-grant` instead DERIVES them locally from the checkpoint + massif,
+ * then runs the exact same merge (`completeGrantBase64`). Proving grants are
+ * *derivable from log data*, not operator-issued.
+ *
+ * Pure over bytes: decode the grant, locate its leaf in the massif by content
+ * hash (`findGrantLeafInMassif` — recovers mmrIndex + the sequenced
+ * idtimestamp), rebuild the leaf's inclusion receipt against the checkpoint's
+ * pre-signed peak (`deriveCheckpointReceipt`), and attach receipt (396) +
+ * idtimestamp (−65537) into the grant's unprotected headers with NO re-signing.
+ * No network, no key.
+ */
+import {
+  decodeForestrieGrantCose,
+  findGrantLeafInMassif,
+  MissingIndexError,
+} from "@forestrie/receipt-verify";
+import { base64ToBytes } from "@forestrie/grant-builder";
+import { completeGrantBase64 } from "./register-grant-complete.js";
+import {
+  CreateReceiptFailure,
+  deriveCheckpointReceipt,
+} from "./create-receipt-derive.js";
+
+/** Where offline completion broke. */
+export type CompleteGrantStage = "decode" | "locate" | "parse" | "derive";
+
+/** Stable snake_case reasons — the `--json` error shape is a contract. */
+export type CompleteGrantReason =
+  | "grant_decode_failed"
+  | "grant_leaf_not_found"
+  | "massif_missing_index"
+  | "idtimestamp_unresolved"
+  // Passed through from the receipt derivation (create-receipt taxonomy).
+  | "massif_parse_failed"
+  | "checkpoint_parse_failed"
+  | "checkpoint_missing_peak_receipts"
+  | "checkpoint_missing_sealed_size"
+  | "checkpoint_does_not_cover_leaf"
+  | "leaf_not_in_massif"
+  | "derive_failed";
+
+export class CompleteGrantFailure extends Error {
+  readonly stage: CompleteGrantStage;
+  readonly reason: CompleteGrantReason;
+
+  constructor(
+    stage: CompleteGrantStage,
+    reason: CompleteGrantReason,
+    message: string,
+  ) {
+    super(message);
+    this.name = "CompleteGrantFailure";
+    this.stage = stage;
+    this.reason = reason;
+  }
+}
+
+/** Which input supplied the entry-id idtimestamp half. */
+export type IdtimestampSource = "massif" | "override" | "grant";
+
+export type DeriveCompletedGrantInput = {
+  /** Registered (uncompleted) grant, Forestrie-Grant header base64 (trimmed). */
+  grantBase64: string;
+  massifBytes: Uint8Array;
+  checkpointBytes: Uint8Array;
+  /** 8-byte idtimestamp override (`--idtimestamp`), used only as a fallback. */
+  idtimestampOverride?: Uint8Array | undefined;
+};
+
+export type CompletedGrant = {
+  /** Completed grant, Forestrie-Grant header base64. */
+  completedBase64: string;
+  /** Permanent entry id (`idtimestamp_be8 || mmrIndex_be8`), 32 hex chars. */
+  entryIdHex: string;
+  mmrIndex: bigint;
+  idtimestampBe8: Uint8Array;
+  idtimestampSource: IdtimestampSource;
+  receiptBytes: number;
+  proofLength: number;
+  peakIndex: number;
+  peakCount: number;
+  sealedSize: bigint;
+  certCopied: boolean;
+};
+
+function bytesToHex(bytes: Uint8Array): string {
+  let s = "";
+  for (const b of bytes) s += b.toString(16).padStart(2, "0");
+  return s;
+}
+
+function mmrIndexToBe8Hex(mmrIndex: bigint): string {
+  const out = new Uint8Array(8);
+  new DataView(out.buffer).setBigUint64(0, mmrIndex & 0xffffffffffffffffn, false);
+  return bytesToHex(out);
+}
+
+function isAllZero(bytes: Uint8Array): boolean {
+  for (const b of bytes) if (b !== 0) return false;
+  return true;
+}
+
+/**
+ * Resolve the idtimestamp that goes into the entry id. The massif leaf key is
+ * ground truth — it is exactly what the sequencer committed, so verification
+ * (`leafHash = H(idtimestamp || contentHash)`) only reproduces the anchored
+ * leaf when this value is used. `--idtimestamp` / an already-embedded grant
+ * idtimestamp are fallbacks for the (unusual) case where the massif index
+ * region did not record the key.
+ */
+function resolveIdtimestamp(
+  fromMassif: Uint8Array,
+  override: Uint8Array | undefined,
+  fromGrant: Uint8Array,
+): { idtimestampBe8: Uint8Array; source: IdtimestampSource } {
+  if (!isAllZero(fromMassif)) {
+    return { idtimestampBe8: fromMassif, source: "massif" };
+  }
+  if (override !== undefined && !isAllZero(override)) {
+    return { idtimestampBe8: override, source: "override" };
+  }
+  if (!isAllZero(fromGrant)) {
+    return { idtimestampBe8: fromGrant, source: "grant" };
+  }
+  throw new CompleteGrantFailure(
+    "parse",
+    "idtimestamp_unresolved",
+    "could not resolve the leaf idtimestamp from the massif, --idtimestamp, " +
+      "or the grant — supply --idtimestamp",
+  );
+}
+
+/**
+ * Derive the completed grant from local artefacts. Throws
+ * {@link CompleteGrantFailure} with a stable stage + reason on any failure.
+ */
+export async function deriveCompletedGrant(
+  input: DeriveCompletedGrantInput,
+): Promise<CompletedGrant> {
+  // 1. Decode the signed grant statement into its Grant payload (+ any
+  //    already-embedded idtimestamp).
+  let grant: ReturnType<typeof decodeForestrieGrantCose>["grant"];
+  let grantIdtimestampBe8: Uint8Array;
+  try {
+    ({ grant, idtimestampBe8: grantIdtimestampBe8 } = decodeForestrieGrantCose(
+      base64ToBytes(input.grantBase64),
+    ));
+  } catch (err) {
+    throw new CompleteGrantFailure(
+      "decode",
+      "grant_decode_failed",
+      `not a Forestrie-Grant COSE Sign1: ${errorMessage(err)}`,
+    );
+  }
+
+  // 2. Locate the grant's leaf in the massif by its commitment hash — recovers
+  //    both the mmrIndex and the sequenced idtimestamp, offline.
+  let located: Awaited<ReturnType<typeof findGrantLeafInMassif>>;
+  try {
+    located = await findGrantLeafInMassif(input.massifBytes, grant);
+  } catch (err) {
+    if (err instanceof MissingIndexError) {
+      throw new CompleteGrantFailure(
+        "locate",
+        "massif_missing_index",
+        `massif has no index region to search: ${err.message}`,
+      );
+    }
+    throw new CompleteGrantFailure("locate", "grant_leaf_not_found", errorMessage(err));
+  }
+  if (located === null) {
+    throw new CompleteGrantFailure(
+      "locate",
+      "grant_leaf_not_found",
+      "this grant's commitment hash is not a leaf in the supplied massif — " +
+        "wrong massif, or the grant is not sequenced yet",
+    );
+  }
+
+  const { idtimestampBe8, source } = resolveIdtimestamp(
+    located.idtimestampBe8,
+    input.idtimestampOverride,
+    grantIdtimestampBe8,
+  );
+
+  // 3. Rebuild the leaf's inclusion receipt against the checkpoint's pre-signed
+  //    peak (identical derivation to `create-receipt`).
+  let derived: Awaited<ReturnType<typeof deriveCheckpointReceipt>>;
+  try {
+    derived = await deriveCheckpointReceipt({
+      massifBytes: input.massifBytes,
+      checkpointBytes: input.checkpointBytes,
+      mmrIndex: located.mmrIndex,
+    });
+  } catch (err) {
+    if (err instanceof CreateReceiptFailure) {
+      // Reuse the create-receipt reason taxonomy verbatim (it is a subset of
+      // CompleteGrantReason).
+      throw new CompleteGrantFailure(
+        err.stage === "parse" ? "parse" : "derive",
+        err.reason,
+        err.message,
+      );
+    }
+    throw new CompleteGrantFailure("derive", "derive_failed", errorMessage(err));
+  }
+
+  // 4. Merge receipt (396) + idtimestamp (−65537) into the grant's unprotected
+  //    headers — no re-signing — and emit the completed grant base64.
+  const entryIdHex = bytesToHex(idtimestampBe8) + mmrIndexToBe8Hex(located.mmrIndex);
+  const completedBase64 = completeGrantBase64(
+    input.grantBase64,
+    derived.receiptCbor,
+    entryIdHex,
+  );
+
+  return {
+    completedBase64,
+    entryIdHex,
+    mmrIndex: located.mmrIndex,
+    idtimestampBe8,
+    idtimestampSource: source,
+    receiptBytes: derived.receiptCbor.length,
+    proofLength: derived.details.proofLength,
+    peakIndex: derived.details.peakIndex,
+    peakCount: derived.details.peakCount,
+    sealedSize: derived.details.sealedSize,
+    certCopied: derived.details.certCopied,
+  };
+}
+
+function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}