@secondlayer/verify 0.1.0 → 0.3.0

package/dist/index.js

@@ -100,7 +100,34 @@ function parseNakamotoHeader(raw) {
   };
 }
 var signerSignatureHash = (h) => sha512_2562(h.signerSighashPreimage);
-var blockId = (h) => hashAll([signerSignatureHash(h), h.consensusHash]);
+var EPOCH2_HEADER_LENGTH = 247;
+function parseEpoch2Header(raw, consensusHash) {
+  if (consensusHash.length !== 20)
+    throw new Error(`consensus hash is ${consensusHash.length} bytes, not 20`);
+  const r = new Reader(raw);
+  const header = {
+    version: r.u8(),
+    burnSpent: r.u64(),
+    chainLength: r.u64(),
+    vrfProof: r.bytes(80),
+    parentBlockHash: r.bytes(32),
+    parentMicroblock: r.bytes(32),
+    parentMicroblockSequence: r.u16(),
+    txMerkleRoot: r.bytes(32),
+    stateIndexRoot: r.bytes(32),
+    microblockPubkeyHash: r.bytes(20)
+  };
+  return {
+    ...header,
+    blockHash: sha512_2562(raw.subarray(0, EPOCH2_HEADER_LENGTH)),
+    consensusHash
+  };
+}
+var isEpoch2Header = (h) => ("parentBlockHash" in h);
+var blockId = (h) => hashAll([
+  isEpoch2Header(h) ? h.blockHash : signerSignatureHash(h),
+  h.consensusHash
+]);
 // src/marf.ts
 import { sha512_256 as sha512_2563 } from "@noble/hashes/sha2.js";
 var MARF_VALUE_SIZE = 40;
@@ -120,6 +147,7 @@ var NODE_PTR_COUNT = {
 };
 var isBackptr = (id) => (id & 128) !== 0;
 var marfPath = (key) => sha512_2563(new TextEncoder().encode(key));
+var heightToHashKey = (height) => `__MARF_BLOCK_HEIGHT_TO_HASH::${height}`;
 function marfValue(value) {
   const v = new Uint8Array(MARF_VALUE_SIZE);
   v.set(sha512_2563(new TextEncoder().encode(value)));
@@ -352,6 +380,14 @@ function marfProofAncestors(proof) {
   }
   return [...ids];
 }
+function marfProofValue(proof) {
+  try {
+    const leaf = decodeProof(proof)[0];
+    return leaf?.kind === "Leaf" ? leaf.value : null;
+  } catch {
+    return null;
+  }
+}
 function verifyMarfProof(input) {
   const rootToBlock = new Map;
   for (const h of input.headers)
@@ -982,6 +1018,15 @@ var required = (url, v) => {
   return v;
 };
 var trimSlash = (s) => s.replace(/\/+$/, "");
+var toEpoch2 = (url, j) => {
+  const tip = required(url, j)[0];
+  if (typeof tip?.header !== "string" || typeof tip.consensus_hash !== "string")
+    throw new SourceError(url, 200, "answer needs header and consensus_hash");
+  return {
+    header: unhex(tip.header),
+    consensusHash: unhex(tip.consensus_hash)
+  };
+};
 var toMarf = (url, j) => {
   if (j === null)
     return null;
@@ -1018,6 +1063,7 @@ async function pages(field, url, http) {
 
 class SecondlayerProofSource {
   getBlock;
+  getEpoch2Header;
   getMarfProof;
   getWitness;
   getBurnPreimage;
@@ -1034,6 +1080,10 @@ class SecondlayerProofSource {
       const url = typeof ref === "number" ? `${base}/v1/proofs/block/height/${ref}` : `${base}/v1/proofs/block/${ref}`;
       return required(url, await getBytes(url, http));
     };
+    this.getEpoch2Header = async (blockId) => {
+      const url = `${base}/v1/proofs/epoch2-header/${blockId}`;
+      return toEpoch2(url, await getJson(url, http));
+    };
     this.getMarfProof = async (pathHex, tipId) => {
       const url = `${base}/v1/proofs/marf/${pathHex}?tip=${tipId}`;
       return toMarf(url, await getJson(url, http));
@@ -1054,7 +1104,15 @@ class SecondlayerProofSource {
         throw new SourceError(url, 200, `headers start at ${j.from}`);
       return j.headers;
     };
-    this.getStateWrites = (height) => pages("state_writes", `${base}/v1/index/state-writes?block_height=${height}&limit=${PAGE}`, http);
+    this.getStateWrites = async (height) => {
+      const url = `${base}/v1/proofs/writes/${height}`;
+      const j = await getJson(url, http);
+      if (j === null)
+        return null;
+      if (!Array.isArray(j.state_writes))
+        throw new SourceError(url, 200, "answer has no state_writes array");
+      return j.state_writes;
+    };
     this.getVmEvents = async (height) => {
       const out = [];
       for (const type of VM_WRITE_TYPES) {
@@ -1068,6 +1126,7 @@ class SecondlayerProofSource {
 
 class NodeRpcProofSource {
   getBlock;
+  getEpoch2Header;
   getMarfProof;
   constructor(opts) {
     const base = trimSlash(opts.nodeUrl);
@@ -1076,6 +1135,10 @@ class NodeRpcProofSource {
       const url = typeof ref === "number" ? `${base}/v3/blocks/height/${ref}` : `${base}/v3/blocks/${ref}`;
       return required(url, await getBytes(url, http));
     };
+    this.getEpoch2Header = async (blockId) => {
+      const url = `${base}/v2/headers/1?tip=${blockId}`;
+      return toEpoch2(url, await getJson(url, http));
+    };
     this.getMarfProof = async (pathHex, tipId) => {
       const url = `${base}/v2/clarity/marf/${pathHex}?tip=${tipId}&proof=1`;
       return toMarf(url, await getJson(url, http));
@@ -1085,7 +1148,6 @@ class NodeRpcProofSource {
 // src/state.ts
 var INTERNAL = {
   self: "__MARF_BLOCK_HEIGHT_SELF",
-  heightToHash: "__MARF_BLOCK_HEIGHT_TO_HASH",
   hashToHeight: "__MARF_BLOCK_HASH_TO_HEIGHT"
 };
 var heightValue = (h) => {
@@ -1096,7 +1158,7 @@ var heightValue = (h) => {
 var padded = (b32) => `${hex(b32)}${"00".repeat(8)}`;
 function internalEntries(height, parentBlockId, leaves) {
   const out = [[INTERNAL.self, heightValue(height)]];
-  const ownKey = `${INTERNAL.heightToHash}::${height}`;
+  const ownKey = heightToHashKey(height);
   const ownPath = marfPath(ownKey);
   const own = leaves.find((l) => bytesEqual(l.path, ownPath));
   if (own) {
@@ -1107,10 +1169,7 @@ function internalEntries(height, parentBlockId, leaves) {
     ]);
   }
   if (height > 0) {
-    out.push([
-      `${INTERNAL.heightToHash}::${height - 1}`,
-      padded(parentBlockId)
-    ]);
+    out.push([heightToHashKey(height - 1), padded(parentBlockId)]);
     out.push([
       `${INTERNAL.hashToHeight}::${hex(parentBlockId)}`,
       heightValue(height - 1)
@@ -1250,6 +1309,7 @@ async function verifyBlockState(input) {
       ...leafHex(l),
       key: l.key,
       txIndex: last.get(l.key)?.txIndex ?? null,
+      value: last.get(l.key)?.value,
       named: true
     })) : d.hidden.map((l) => ({ ...leafHex(l), named: false })),
     carried: d.carried.map(leafHex),
@@ -1278,6 +1338,58 @@ async function verifyBlockState(input) {
     out.rowsChecked = checkRows(input.rows, leaves, out.failures);
   return out;
 }
+// src/txs.ts
+import { sha512_256 as sha512_2564 } from "@noble/hashes/sha2.js";
+import { splitTransactions } from "@secondlayer/stacks/transactions";
+var LEAF_TAG = new Uint8Array([0]);
+var NODE_TAG = new Uint8Array([1]);
+function txMerkleRoot(txids) {
+  if (txids.length === 0)
+    throw new Error("no transactions");
+  let level = txids.map((t) => sha512_2564(concat([LEAF_TAG, t])));
+  do {
+    if (level.length % 2 === 1)
+      level.push(level[level.length - 1]);
+    const next = [];
+    for (let i = 0;i < level.length; i += 2) {
+      next.push(sha512_2564(concat([NODE_TAG, level[i], level[i + 1]])));
+    }
+    level = next;
+  } while (level.length > 1);
+  return level[0];
+}
+
+class TransactionsError extends Error {
+  code;
+  constructor(code, message) {
+    super(message);
+    this.code = code;
+  }
+}
+function blockTransactions(raw, header) {
+  const body = raw.subarray(header.byteLength);
+  if (body.length === 0)
+    return null;
+  if (body.length < 4)
+    throw new TransactionsError("malformed", "block body has no tx count");
+  const count = new DataView(body.buffer, body.byteOffset, body.byteLength).getUint32(0);
+  if (count === 0)
+    throw new TransactionsError("malformed", "block has no transactions");
+  let txs;
+  try {
+    txs = splitTransactions(body.subarray(4), count);
+  } catch (err) {
+    throw new TransactionsError("malformed", `transactions do not parse: ${err.message}`);
+  }
+  const used = txs.reduce((n, t) => n + t.length, 4);
+  if (used !== body.length)
+    throw new TransactionsError("malformed", `${body.length - used} bytes after the block's ${count} transactions`);
+  const ids = txs.map((t) => sha512_2564(t));
+  const root = txMerkleRoot(ids);
+  if (!bytesEqual(root, header.txMerkleRoot))
+    throw new TransactionsError("tx-root-mismatch", `transactions hash to ${hex(root)}, header tx_merkle_root is ${hex(header.txMerkleRoot)}`);
+  return txs.map((t, i) => ({ txid: hex(ids[i]), raw: t }));
+}
 // src/verify-block.ts
 import { uintCV } from "@secondlayer/stacks/clarity";
 class Broken extends Error {
@@ -1305,6 +1417,7 @@ async function fetchFor(step, what, call, extra = {}) {
   }
 }
 var BITCOIN_BATCH = 2016;
+var MAX_WALK = 16;
 async function findAnchor(low, high, window, probe, maxProbes = 32) {
   let [lo, hi] = [low, high];
   const target = (window[0] + window[1]) / 2;
@@ -1339,8 +1452,10 @@ class BlockVerifier {
   sets = new Map;
   anchors = new Map;
   headers = new Map;
+  below = new Map;
   burns = new Map;
   preimages = new Map;
+  raws = new Map;
   start;
   constructor(opts) {
     this.source = opts.source;
@@ -1366,26 +1481,42 @@ class BlockVerifier {
     return out;
   }
   async run(ref, out) {
-    const src = this.source;
-    const raw = await fetchFor("block", `block ${ref}`, () => src.getBlock(ref));
-    const header = parseHeader2(raw, "block");
+    const start = Number(this.start.header.chainLength);
+    if (typeof ref === "number" && ref < start)
+      return this.runBelow(ref, undefined, out);
+    let header;
+    if (typeof ref === "string") {
+      const id = ref.replace(/^0x/, "").toLowerCase();
+      header = await this.header(id, "block");
+      const height = Number(header.chainLength);
+      if (height < start)
+        return this.runBelow(height, id, out);
+      if (isEpoch2Header(header))
+        fail({
+          step: "block",
+          code: "invalid-header",
+          message: `epoch 2.x block ${id} claims height ${height}, at or above the checkpoint`
+        });
+    } else {
+      const raw = await fetchFor("block", `block ${ref}`, () => this.source.getBlock(ref));
+      header = parseHeader2(raw, "block");
+      this.rememberRaw(hex(blockId(header)), raw);
+    }
     const id = hex(blockId(header));
     const height = Number(header.chainLength);
-    if (typeof ref === "string" && id !== ref.replace(/^0x/, "").toLowerCase())
-      fail({
-        step: "block",
-        code: "id-mismatch",
-        message: `source returned block ${id} for ${ref}`
-      });
     if (typeof ref === "number" && height !== ref)
       fail({
         step: "block",
         code: "height-mismatch",
         message: `source returned height ${height} for ${ref}`
       });
+    await this.runForward(header, id, out);
+  }
+  async runForward(header, id, out) {
+    const height = Number(header.chainLength);
     out.height = height;
     out.blockId = id;
-    out.stateRoot = hex(header.stateIndexRoot);
+    describeHeader(header, out);
     const burn = await this.bindBurn(header, "burn");
     const cycle = rewardCycle(burn.burnHeight);
     out.burnHeight = burn.burnHeight;
@@ -1401,20 +1532,174 @@ class BlockVerifier {
     out.threshold = sig.threshold;
     if (!sig.valid)
       fail(signatureFailure(sig, "signatures", cycle));
+    await this.checkTransactions(header, id, out);
+    await this.verifyState(header, id, out);
+  }
+  async runBelow(height, claimed, out) {
+    out.height = height;
+    const { id, ancestry } = await this.ancestor(height);
+    out.ancestry = ancestry;
+    if (claimed !== undefined && claimed !== id)
+      fail({
+        step: "ancestry",
+        code: "not-ancestor",
+        message: `block ${claimed} is not on the checkpoint's chain: its block at height ${height} is ${id}`,
+        blockId: claimed
+      });
+    const header = await this.header(id, "block");
+    if (Number(header.chainLength) !== height)
+      fail({
+        step: "block",
+        code: "height-mismatch",
+        message: `block ${id} claims height ${header.chainLength}, proven at ${height}`
+      });
+    out.blockId = id;
+    describeHeader(header, out);
+    await this.bindBurnBelow(header, out);
+    await this.checkTransactions(header, id, out);
+    await this.verifyState(header, id, out);
+  }
+  rememberRaw(id, raw) {
+    this.raws.delete(id);
+    this.raws.set(id, raw);
+    while (this.raws.size > 4) {
+      const oldest = this.raws.keys().next().value;
+      this.raws.delete(oldest);
+    }
+  }
+  async checkTransactions(header, id, out) {
+    if (isEpoch2Header(header)) {
+      out.notes.push("epoch 2.x: transactions not checked");
+      return;
+    }
+    const raw = this.raws.get(id) ?? await fetchFor("txs", `block ${id}`, () => this.source.getBlock(id));
+    let txs;
+    try {
+      txs = blockTransactions(raw, header);
+    } catch (err) {
+      if (!(err instanceof TransactionsError))
+        throw err;
+      return fail({ step: "txs", code: err.code, message: err.message });
+    }
+    if (txs === null) {
+      out.notes.push(`transactions not checked: the source served block ${id}'s header only`);
+      return;
+    }
+    out.transactions = txs;
+  }
+  async ancestor(height) {
+    const known = this.below.get(height);
+    if (known)
+      return known;
+    const from = this.nearestAbove(height);
+    if (from.height - height <= MAX_WALK) {
+      const walked = await this.walk(from, height);
+      if (walked)
+        return walked;
+    }
+    return this.jump(this.nearestAbove(height), height);
+  }
+  nearestAbove(height) {
+    let best = {
+      height: Number(this.start.header.chainLength),
+      id: this.start.id
+    };
+    for (const [h, { id }] of this.below)
+      if (h > height && h < best.height)
+        best = { height: h, id };
+    return best;
+  }
+  async walk(from, height) {
+    const ancestry = {
+      fromHeight: from.height,
+      fromBlockId: from.id,
+      via: "parents"
+    };
+    let child = await this.header(from.id, "ancestry");
+    for (let h = from.height - 1;h >= height; h--) {
+      if (isEpoch2Header(child))
+        return null;
+      const id = hex(child.parentBlockId);
+      const parent = await this.header(id, "ancestry", { blockId: id });
+      if (Number(parent.chainLength) !== h)
+        fail({
+          step: "ancestry",
+          code: "height-mismatch",
+          message: `parent ${id} claims height ${parent.chainLength}, expected ${h}`,
+          blockId: id
+        });
+      this.below.set(h, { id, ancestry });
+      child = parent;
+    }
+    return this.below.get(height) ?? null;
+  }
+  async jump(from, height) {
+    const at = { blockId: from.id };
+    const tip = await this.header(from.id, "ancestry", at);
+    const key = heightToHashKey(height);
+    const path = marfPath(key);
+    const answer = await fetchFor("ancestry", `${key} at ${from.id}`, () => this.source.getMarfProof(hex(path), from.id), at);
+    if (!answer)
+      return fail({
+        step: "ancestry",
+        code: "unavailable",
+        message: `source has no MARF proof of ${key} at block ${from.height}, ${from.height - height} blocks up; parent links reach ${MAX_WALK}`,
+        ...at
+      });
+    const value = marfProofValue(answer.proof) ?? new Uint8Array;
+    const proven = value.length === MARF_VALUE_SIZE && value.subarray(32).every((b) => b === 0) && await this.proveAt(tip, path, value, answer.proof, "ancestry", at);
+    if (!proven)
+      return fail({
+        step: "ancestry",
+        code: "ancestry-proof-invalid",
+        message: `${key} is not proven in block ${from.height}'s state`,
+        ...at
+      });
+    const ancestry = {
+      fromHeight: from.height,
+      fromBlockId: from.id,
+      via: "marf"
+    };
+    const found = { id: hex(value.subarray(0, 32)), ancestry };
+    this.below.set(height, found);
+    return found;
+  }
+  async bindBurnBelow(header, out) {
+    let hint;
+    try {
+      hint = await this.burnHint(header);
+    } catch (err) {
+      out.notes.push(`burn block not checked: ${err.message}`);
+      return;
+    }
+    if (hint < this.chain.checkpointHeight) {
+      out.notes.push(`burn block not checked: its height ${hint} is below the Bitcoin checkpoint ${this.chain.checkpointHeight}`);
+      return;
+    }
+    const burn = await this.bindBurn(header, "burn");
+    out.burnHeight = burn.burnHeight;
+    out.bitcoinBlockHash = burn.hash;
+  }
+  async verifyState(header, id, out) {
+    const src = this.source;
+    const height = Number(header.chainLength);
     const witness = await fetchFor("witness", `witness ${id}`, () => src.getWitness(id));
-    const { getStateWrites, getVmEvents } = this.rows ? src : {};
-    const writes = getStateWrites ? await fetchFor("names", `state_writes ${height}`, () => getStateWrites.call(src, height)) : null;
+    const writes = await this.stateWrites(height, out.notes);
+    if (writes)
+      out.writes = writes;
+    const { getVmEvents } = this.rows ? src : {};
     const rows = getVmEvents ? await fetchFor("rows", `vm_events ${height}`, () => getVmEvents.call(src, height)) : undefined;
-    const parentId = hex(header.parentBlockId);
+    const parent = isEpoch2Header(header) ? epoch2ParentId(witness, header) : header.parentBlockId;
+    const parentId = parent && hex(parent);
     const state = await verifyBlockState({
       height,
-      parentBlockId: header.parentBlockId,
+      parentBlockId: parent ?? new Uint8Array(32),
       stateRoot: header.stateIndexRoot,
       witness,
       writes,
       rows,
-      parentLeaves: await this.parentLeaves(parentId, out.notes),
-      parentHolds: (leaf) => this.parentHolds(parentId, leaf)
+      parentLeaves: parentId ? await this.parentLeaves(parentId, out.notes) : undefined,
+      parentHolds: async (leaf) => parentId ? this.parentHolds(parentId, leaf) : false
     });
     out.diff = state.diff;
     if (rows)
@@ -1422,12 +1707,23 @@ class BlockVerifier {
     out.notes.push(...state.notes);
     out.failures.push(...state.failures);
   }
+  async stateWrites(height, notes) {
+    const { getStateWrites } = this.source;
+    if (!getStateWrites)
+      return null;
+    try {
+      const writes = await getStateWrites.call(this.source, height);
+      return writes?.length ? writes : null;
+    } catch (err) {
+      notes.push(`state writes for block ${height} unavailable: ${err.message}`);
+      return null;
+    }
+  }
   async header(id, step, extra = {}) {
     const known = this.headers.get(id);
     if (known)
       return known;
-    const raw = await fetchFor(step, `block ${id}`, () => this.source.getBlock(id), extra);
-    const header = parseHeader2(raw, step);
+    const header = await this.fetchHeader(id, step, extra);
     if (hex(blockId(header)) !== id)
       fail({
         step,
@@ -1439,6 +1735,31 @@ class BlockVerifier {
     this.headers.set(id, header);
     return header;
   }
+  async fetchHeader(id, step, extra) {
+    const src = this.source;
+    let raw;
+    try {
+      raw = await src.getBlock(id);
+      this.rememberRaw(id, raw);
+    } catch (err) {
+      const reason = `block ${id}: ${err.message}`;
+      const { getEpoch2Header } = src;
+      if (!getEpoch2Header)
+        return fail({ step, code: "unavailable", message: reason, ...extra });
+      const e2 = await fetchFor(step, `${reason}; epoch 2.x header`, () => getEpoch2Header.call(src, id), extra);
+      try {
+        return parseEpoch2Header(e2.header, e2.consensusHash);
+      } catch (parseErr) {
+        return fail({
+          step,
+          code: "invalid-header",
+          message: `epoch 2.x header ${id} does not parse: ${parseErr.message}`,
+          ...extra
+        });
+      }
+    }
+    return parseHeader2(raw, step);
+  }
   async proveAt(tip, path, value, proof, step, extra = {}) {
     const headers = [tip];
     for (const id of marfProofAncestors(proof))
@@ -1639,6 +1960,14 @@ class BlockVerifier {
     return set;
   }
 }
+function describeHeader(header, out) {
+  out.stateRoot = hex(header.stateIndexRoot);
+  out.consensusHash = hex(header.consensusHash);
+  if (!isEpoch2Header(header)) {
+    out.blockHash = hex(signerSignatureHash(header));
+    out.timestamp = Number(header.timestamp);
+  }
+}
 function parseHeader2(raw, step) {
   try {
     return parseNakamotoHeader(raw);
@@ -1650,6 +1979,20 @@ function parseHeader2(raw, step) {
     });
   }
 }
+function epoch2ParentId(witness, header) {
+  const height = Number(header.chainLength);
+  if (height === 0)
+    return null;
+  try {
+    const w = parseWitness(witness);
+    if (!bytesEqual(w.root, header.stateIndexRoot))
+      return null;
+    const path = marfPath(heightToHashKey(height - 1));
+    return w.leaves.find((l) => bytesEqual(l.path, path))?.valueHash ?? null;
+  } catch {
+    return null;
+  }
+}
 function signatureFailure(sig, step, cycle) {
   const weight = `${sig.signedWeight}/${sig.totalWeight} signed, threshold ${sig.threshold}`;
   if (sig.unknown.length)
@@ -1693,21 +2036,26 @@ export {
   SIGNERS_CONTRACT,
   SecondlayerProofSource,
   SourceError,
+  TransactionsError,
   blockId,
+  blockTransactions,
   classifyBlockDiff,
   cycleStart,
   dataVarKey,
   decodeSignerSet,
   ftBalanceKey,
   hex,
+  isEpoch2Header,
   mapEntryKey,
   marfPath,
   marfProofAncestors,
   marfValue,
+  parseEpoch2Header,
   parseNakamotoHeader,
   parseWitness,
   rewardCycle,
   signerSignatureHash,
+  txMerkleRoot,
   unhex,
   verifyBlock,
   verifyBlockState,
@@ -1716,5 +2064,5 @@ export {
   verifySignerSignatures
 };
 
-//# debugId=67FB81E2AAB4645564756E2164756E21
+//# debugId=AFAA56E81913DBC664756E2164756E21
 //# sourceMappingURL=index.js.map