keychat-save 1.4.1 → 1.4.3

package/index.mjs

@@ -75,8 +75,8 @@ async function getUnspent (address) {
 // The bridge answers in its own shape when our node can serve the tx:
 //     { outputs: [ { satoshis, script, index } ] }
 // When it CANNOT — the node is pruned with no -txindex, so getrawtransaction fails for
-// any confirmed tx in a discarded block — the web proxy falls back to WhatsOnChain and
-// returns WoC's body untouched:
+// any confirmed tx in a discarded block — the bridge falls through to GorillaPool and
+// the body comes back in the other shape:
 //     { vout: [ { value, n, scriptPubKey: { hex } } ] }
 //
 // Callers looked only at `tx.outputs`, so on the WoC shape they iterated an empty array

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keychat-save",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "Save sessions, files and whole git repositories to the BSV blockchain via KeyChat",
   "type": "module",
   "bin": {

package/repo.mjs

@@ -48,6 +48,14 @@ import { createHash } from 'node:crypto'
 import fs from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
+import { installFileLedger } from './spent-ledger-fs.js'
+import { markSpentOutpoints, withoutSpent, pruneConfirmed, rebuildFromNode } from '../src/wallet/spent-ledger.js'
+
+// Share one spent-ledger with keychat-save and every other Node writer on this
+// machine. A repo snapshot and a session save draw from the SAME wallet; a
+// snapshot can spend many seconds building a multi-MB tx, and without this a
+// save started in that window picks the same inputs and dies as a double-spend.
+installFileLedger()
 
 const CONFIG_DIR = path.join(os.homedir(), '.keychat')
 const WIF_PATH = path.join(CONFIG_DIR, 'identity.wif')
@@ -75,14 +83,36 @@ function key () {
 }
 
 // ── bridge ───────────────────────────────────────────────────────
-async function getUnspent (address) {
+async function getUnspent (address, { skipLedger = false } = {}) {
   const res = await fetch(`${BRIDGE}/api/address/${address}/unspent`)
   if (!res.ok) throw new Error(`unspent ${res.status}`)
-  return (await res.json()).map(u => ({
+  const mapped = (await res.json()).map(u => ({
     txid: u.tx_hash || u.txid,
     vout: u.tx_pos ?? u.vout,
     satoshis: u.value ?? u.satoshis
   }))
+  if (skipLedger) return mapped
+  // The node says what exists; the ledger only subtracts coins another writer
+  // has already claimed. Prune each load so it stays the size of what is in
+  // flight rather than growing forever.
+  pruneConfirmed(mapped)
+  return withoutSpent(mapped)
+}
+
+/**
+ * A snapshot that never reached the network still holds its coins. Re-derive
+ * from a fresh node scan rather than un-marking by hand — the node is
+ * authoritative at this moment, and a hand-rolled release can be wrong in the
+ * direction that double-spends. Failure path only.
+ */
+async function releaseLedgerAfterFailure (address) {
+  try {
+    const fresh = await getUnspent(address, { skipLedger: true })
+    const r = rebuildFromNode(fresh)
+    console.warn(`  ledger rebuilt from node: ${r.released} released, ${r.dropped} dropped`)
+  } catch (err) {
+    console.warn(`  could not rebuild ledger: ${err.message}`)
+  }
 }
 
 async function broadcast (rawHex) {
@@ -103,24 +133,36 @@ async function broadcast (rawHex) {
 // 68 MB of hex in the live web server to satisfy a restore is not worth it. A query
 // string makes its isTxBody regex miss, so the request passes straight through.
 //
-// But that same regex miss also skips its WhatsOnChain fallback, which is the only
-// second source we have once a tx is mined. So fall back to the plain path on
-// failure — retrievability beats keeping the proxy cache tidy.
+// The plain path is still tried on failure — retrievability beats keeping the
+// proxy cache tidy. (It used to also carry the proxy's WhatsOnChain fallback;
+// that was removed 2026-08-28, so the third source below is now the only
+// second opinion once a tx is mined.)
 async function getRawHex (txid) {
   // Three sources, in order of sovereignty:
   //   1. bridge ?raw=1  — our node, bypassing the web proxy's forever-cache.
-  //   2. bridge plain   — same node, but with the proxy's WoC fallback behind it.
-  //   3. WoC direct     — last resort, and it earns its place: our node is PRUNED
-  //      with no -txindex, so getrawtransaction CANNOT return a confirmed tx at
-  //      all. Every mined snapshot is therefore unfetchable from us, and with
-  //      only the first two sources a rate-limited proxy made the whole delta
-  //      chain unreplayable — which breaks restore, not just new deltas.
-  // 429 gets a backoff rather than being treated as absence; hammering a closed
-  // limiter is what turned a transient failure into a hard one.
+  //   2. bridge plain   — same node; the bridge itself falls through to GP.
+  //   3. GorillaPool raw — last resort, and it earns its place: our node is
+  //      PRUNED with no -txindex, so getrawtransaction CANNOT return a
+  //      confirmed tx at all. Every mined snapshot is unfetchable from us
+  //      directly, and without a third source a rate-limited proxy made the
+  //      whole delta chain unreplayable — which breaks restore, not just new
+  //      deltas.
+  //
+  // WAS WhatsOnChain until 2026-08-28. WoC TRUNCATES LARGE SCRIPTS, and this
+  // function fetches repo snapshots — the multi-MB transactions most likely to
+  // be cut short. A truncated body does not error; it decrypts to garbage or
+  // fails a hash check, so the failure surfaces as a corrupt restore rather
+  // than a fetch error. It also throttled at ~3 req/s, and the responses that
+  // lost were the biggest ones. GP serves the same transactions whole and
+  // faster — measured: a 4.8 MB snapshot in 3.4s.
+  //
+  // GP returns raw BYTES, so convert rather than reading text as hex.
+  // 429 still gets a backoff rather than being treated as absence; hammering a
+  // closed limiter is what turned a transient failure into a hard one.
   const urls = [
     `${BRIDGE}/api/tx/${txid}/hex?raw=1`,
     `${BRIDGE}/api/tx/${txid}/hex`,
-    `https://api.whatsonchain.com/v1/bsv/main/tx/${txid}/hex`
+    `https://ordinals.gorillapool.io/api/tx/${txid}/raw`
   ]
   let wait = 500
   for (let attempt = 0; attempt < 3; attempt++) {
@@ -129,6 +171,15 @@ async function getRawHex (txid) {
       try { res = await fetch(url) } catch { continue }
       if (res.status === 429) { await new Promise(r => setTimeout(r, wait)); continue }
       if (!res.ok) continue
+      // GP's /raw returns BINARY tx bytes, not hex text. Reading it with
+      // .text() yields mojibake that fails the hex test below, so the source
+      // would silently never work — a fallback that is present but dead is
+      // worse than no fallback, because nothing tells you.
+      if (url.endsWith('/raw')) {
+        const h = Buffer.from(await res.arrayBuffer()).toString('hex')
+        if (h && /^[0-9a-f]+$/i.test(h)) return h
+        continue
+      }
       const ct = res.headers.get('content-type') || ''
       const data = ct.includes('json') ? await res.json() : await res.text()
       const h = (typeof data === 'string' ? data.replace(/"/g, '').trim() : data.hex)
@@ -137,7 +188,7 @@ async function getRawHex (txid) {
     wait = Math.min(wait * 4, 8000)
     await new Promise(r => setTimeout(r, wait))
   }
-  throw new Error(`cannot fetch raw tx ${txid} from node, bridge or WoC`)
+  throw new Error(`cannot fetch raw tx ${txid} from node, bridge or GorillaPool`)
 }
 
 // ── OP_RETURN ────────────────────────────────────────────────────
@@ -286,6 +337,39 @@ function addP2PKHInput (tx, k, lockingScript, utxo) {
 const INPUT_BYTES = 148 // 36 outpoint + 1 len + ~107 script + 4 sequence
 const OUTPUT_P2PKH_BYTES = 34
 
+// Every KeyChat tx leaves a 3-sat output at the sender's own address, so a wallet
+// rebuilds its history by scanning its own dust rather than trusting a server's
+// watch list. RepoChain was the worst offender: the funding path built a payload
+// tx whose ONLY output was the 0-sat OP_RETURN — verified on chain, b8f8b337 has
+// exactly one output — so every full snapshot and every large delta was invisible
+// to output-keyed indexing. That is the same defect that hid 301 of this wallet's
+// transactions, sitting in the tool that writes our backups.
+const SELF_MARKER_SATS = 3
+
+const varIntSize = n => n < 0xfd ? 1 : n <= 0xffff ? 3 : n <= 0xffffffff ? 5 : 9
+
+// An output is 8 satoshi bytes + the script's length prefix + the script. That prefix
+// is 3 bytes past 252 and 5 past 65,535; this was estimated at a flat 9 (i.e. a 1-byte
+// prefix), so EVERY repo tx was sized 2-4 bytes short of what it actually serialises to.
+const outputBytes = scriptLen => 8 + varIntSize(scriptLen) + scriptLen
+
+// The charge is always rounded up to the 0.25-Compute sub-unit — that is deliberate and
+// unchanged. What matters is that the ROUNDING INPUT be the real byte count: when the
+// estimate ran short and the raw fee happened to land just above a 125-sat boundary, the
+// round-up added ~nothing and the tx went out below 100 sats/KB. Under that floor a tx is
+// accepted, given a txid, never mined, and gone hours later (pt31 killed saves pt27/pt28;
+// RepoChain link 7294a683 died the same way and took the keychat-save chain with it).
+function assertFeeRate (tx, fee, label) {
+  const bytes = tx.toHex().length / 2
+  const rate = fee / (bytes / 1000)
+  if (rate < FEE_PER_KB) {
+    throw new Error(`${label} fee too low: ${fee} sats for ${bytes} bytes = ` +
+      `${rate.toFixed(2)} sats/KB, floor is ${FEE_PER_KB}. Refusing to broadcast — ` +
+      `under the floor a tx is accepted, never mined, and silently lost.`)
+  }
+  return { bytes, rate }
+}
+
 /**
  * Consolidate small Compute UTXOs into one output worth exactly `target` sats.
  * Its own miner fee grows with each input added, so the pick converges rather
@@ -295,7 +379,7 @@ function planFunding (utxos, target) {
   let n = 0
   let sum = 0
   for (;;) {
-    const bytes = 10 + n * INPUT_BYTES + OUTPUT_P2PKH_BYTES * 2
+    const bytes = 10 + varIntSize(n) - 1 + n * INPUT_BYTES + OUTPUT_P2PKH_BYTES * 2
     const need = target + roundToSubUnit(feeFor(bytes))
     if (sum >= need && n > 0) return { picked: utxos.slice(0, n), sum, need }
     if (n >= utxos.length) {
@@ -327,7 +411,13 @@ async function publish (scriptHex, k, address, lockingScript, { dry = false } =
   let sum = 0
   let fee = 0
   while (n < DIRECT_MAX_INPUTS) {
-    fee = roundToSubUnit(feeFor(10 + n * INPUT_BYTES + 9 + scriptBytes + OUTPUT_P2PKH_BYTES))
+    // Two P2PKH outputs: change and the self-marker. `spend` is everything that
+    // leaves as fee-or-marker; the marker's 3 sats are INSIDE the 125-sat
+    // round-up, not added on top of it — adding on top would leave change at
+    // -3 mod 125, which is not a Compute denomination and so unspendable.
+    // Miner still clears the rate: round-up >= feeFor + 3, so spend - 3 >= feeFor.
+    fee = roundToSubUnit(feeFor(10 + varIntSize(n) - 1 + n * INPUT_BYTES +
+      outputBytes(scriptBytes) + OUTPUT_P2PKH_BYTES * 2) + SELF_MARKER_SATS)
     if (sum >= fee && n > 0) break
     if (n >= utxos.length) throw new Error('insufficient Compute')
     sum += utxos[n].satoshis
@@ -335,13 +425,34 @@ async function publish (scriptHex, k, address, lockingScript, { dry = false } =
   }
 
   if (sum >= fee && n > 0 && n < DIRECT_MAX_INPUTS) {
-    const tx = new Transaction()
-    for (const u of utxos.slice(0, n)) addP2PKHInput(tx, k, lockingScript, u)
-    tx.addOutput({ lockingScript: opReturn, satoshis: 0 })
-    if (sum - fee > 0) tx.addOutput({ lockingScript, satoshis: sum - fee })
-    await tx.sign()
-    console.log(`payload tx    ${(tx.toHex().length / 2 / 1024).toFixed(1)} KB, ${n} inputs, ` +
-      `fee ${fee} sats = ${fee / SATS_PER_COMPUTE} Compute (single tx)`)
+    const build = async () => {
+      const t = new Transaction()
+      for (const u of utxos.slice(0, n)) addP2PKHInput(t, k, lockingScript, u)
+      t.addOutput({ lockingScript: opReturn, satoshis: 0 })
+      // SELF-MARKER, UNCONDITIONAL — the change output below is conditional and
+      // a snapshot that lands exactly would otherwise leave nothing at our address.
+      t.addOutput({ lockingScript, satoshis: SELF_MARKER_SATS })
+      if (sum - fee > 0) t.addOutput({ lockingScript, satoshis: sum - fee })
+      await t.sign()
+      return t
+    }
+    // Price off the SIGNED bytes, not the estimate, then round up to 0.25 Compute as
+    // usual. Converges in one pass in practice; the loop is there because raising the
+    // fee shrinks change, which can drop the change output and change the size again.
+    let tx = await build()
+    for (let i = 0; i < 4; i++) {
+      const want = roundToSubUnit(feeFor(tx.toHex().length / 2) + SELF_MARKER_SATS)
+      if (want <= fee) break
+      while (sum < want && n < utxos.length && n < DIRECT_MAX_INPUTS) sum += utxos[n++].satoshis
+      if (sum < want) throw new Error(`insufficient Compute for corrected fee ${want} sats`)
+      fee = want
+      tx = await build()
+    }
+    // The marker is an OUTPUT, not fee — the miner receives fee minus the marker,
+    // and that is the number the floor must be checked against.
+    const { bytes, rate } = assertFeeRate(tx, fee - SELF_MARKER_SATS, 'payload tx')
+    console.log(`payload tx    ${(bytes / 1024).toFixed(1)} KB, ${n} inputs, ` +
+      `fee ${fee} sats = ${fee / SATS_PER_COMPUTE} Compute (single tx, ${rate.toFixed(2)} sats/KB)`)
     if (dry) return { txid: tx.id('hex'), funding: null, cost: fee / SATS_PER_COMPUTE, dry: true }
     process.stdout.write('broadcasting ... ')
     const txid = await broadcast(tx.toHex())
@@ -358,11 +469,25 @@ async function publish (scriptHex, k, address, lockingScript, { dry = false } =
   // bought nothing and it ate two repo snapshots anyway. What actually protects this output
   // is the splitter being deposits-only (src/messages/split.js) — it reads the parent tx,
   // sees we signed it, and leaves our change alone.
-  const payloadFee = roundToSubUnit(feeFor(10 + INPUT_BYTES + 9 + scriptBytes))
-  const fundingValue = payloadFee
+  // The payload tx is single-input with no change, so its fee IS this funding output —
+  // it cannot be topped up after the fact without rebuilding both txs. So the estimate
+  // has to be right the first time: real output framing, and the input sized at its
+  // maximum (a low-S DER signature caps the unlocking script at 107 bytes).
+  const payloadBytes = 10 + INPUT_BYTES + outputBytes(scriptBytes) + OUTPUT_P2PKH_BYTES
+  const payloadFee = roundToSubUnit(feeFor(payloadBytes))
+  // The funding output is the payload tx's ENTIRE input, so it must cover the
+  // miner fee AND the self-marker. Without the marker the payload tx had exactly
+  // one output — a 0-sat OP_RETURN — and nothing at our address, which made every
+  // large snapshot invisible to output-keyed indexing.
+  const fundingValue = payloadFee + SELF_MARKER_SATS
   const { picked, sum: fsum, need } = planFunding(utxos, fundingValue)
   const fundFee = need - fundingValue
-  console.log(`payload tx    ${((10 + INPUT_BYTES + 9 + scriptBytes) / 1048576).toFixed(2)} MB, ` +
+  // Claim the inputs before signing. A multi-MB payload can spend tens of
+  // seconds in sign(), and any save started in that window would otherwise
+  // pick the very same coins. Released below on every path that does not
+  // broadcast the funding tx.
+  markSpentOutpoints(picked)
+  console.log(`payload tx    ${(payloadBytes / 1048576).toFixed(2)} MB, ` +
     `fee ${payloadFee} sats = ${payloadFee / SATS_PER_COMPUTE} Compute`)
   console.log(`funding tx    ${picked.length} inputs, fee ${fundFee} sats`)
   console.log(`total cost    ${(fundingValue + fundFee) / SATS_PER_COMPUTE} Compute\n`)
@@ -381,18 +506,43 @@ async function publish (scriptHex, k, address, lockingScript, { dry = false } =
   const payTx = new Transaction()
   addP2PKHInput(payTx, k, lockingScript, { txid: fundTx.id('hex'), vout: 0, satoshis: fundingValue })
   payTx.addOutput({ lockingScript: opReturn, satoshis: 0 })
+  payTx.addOutput({ lockingScript, satoshis: SELF_MARKER_SATS })
   process.stdout.write('signing payload tx ... ')
   t = Date.now()
   await payTx.sign()
   console.log(`${((Date.now() - t) / 1000).toFixed(1)}s`)
 
+  // Both must clear the floor BEFORE either goes out. Broadcasting the funding tx and
+  // then discovering the payload is underpaid strands the fee in a spent output.
+  try {
+    assertFeeRate(fundTx, fundFee, 'funding tx')
+    // Miner receives the funding value MINUS the marker — check the floor against
+    // what the miner actually gets, not against what the tx consumes.
+    assertFeeRate(payTx, fundingValue - SELF_MARKER_SATS, 'payload tx')
+  } catch (err) {
+    await releaseLedgerAfterFailure(address)
+    throw err
+  }
+
   if (dry) {
+    // Nothing goes to the network on a dry run, so the coins are still ours.
+    await releaseLedgerAfterFailure(address)
     return { txid: payTx.id('hex'), funding: fundTx.id('hex'), cost: (fundingValue + fundFee) / SATS_PER_COMPUTE, dry: true }
   }
   process.stdout.write('broadcasting funding tx ... ')
-  const ftx = await broadcast(fundTx.toHex())
+  let ftx
+  try {
+    ftx = await broadcast(fundTx.toHex())
+  } catch (err) {
+    // Funding never left, so `picked` is still ours.
+    await releaseLedgerAfterFailure(address)
+    throw err
+  }
   console.log(ftx)
   process.stdout.write('broadcasting payload tx ... ')
+  // Deliberately NO release if the payload fails: the funding tx is already on
+  // the network, so `picked` is genuinely spent. Releasing here would hand the
+  // picker coins that are gone — the one direction that double-spends.
   const ptx = await broadcast(payTx.toHex())
   console.log(ptx)
   return { txid: ptx, funding: ftx, cost: (fundingValue + fundFee) / SATS_PER_COMPUTE }
@@ -414,7 +564,7 @@ function measure (repoPath) {
   // ECIES adds an ephemeral pubkey, IV, PKCS#7 padding and an HMAC.
   const cipherBytes = tar.length + 200
   const scriptBytes = cipherBytes + 60
-  const repoTxBytes = 10 + INPUT_BYTES + 9 + scriptBytes
+  const repoTxBytes = 10 + INPUT_BYTES + outputBytes(scriptBytes)
   const repoFee = roundToSubUnit(feeFor(repoTxBytes))
   return { name, abs, tar, repoTxBytes, repoFee }
 }