@secondlayer/cli 12.1.0 → 12.2.0

package/README.md

@@ -51,7 +51,44 @@ No account. Writes `.env.local`, restores history, prints the Stacks observer st
 | `secondlayer bootstrap --against <manifest> [--to-block <n>] [--public-key <pem>] [-y] [--json]` | Restore chain history from a verified archive into an empty database. Exit `0` restored, `1` diverged, `2` refused |
 | `secondlayer observer [--mode indexer\|signer-shared] [--endpoint host:port] [--recovery journal\|archive] [--network …]` | Print the `[[events_observer]]` stanza. Signer-shared requires `--recovery` |
 | `secondlayer verify [all\|raw\|decode:<name>\|subgraph:<name>] --against <manifest> [--quick\|--deep\|--anchor]` | Compare local data to a signed archive. Default target `raw`. Exit `0` clean, `1` diverged, `2` unanchored |
-| `secondlayer repair --against <archive> [--apply]` | Plan (default) or apply an archive repair |
+| `secondlayer repair --against <archive> [--apply] [-y]` | Plan (default) or apply an archive repair |
+
+Bootstrap and repair against the official hosted archive (`archive.secondlayer.tools`)
+are metered per partition; against any other manifest (a mirror, a teammate's
+box, a local file) they are free. See [Metered fetches](#metered-fetches).
+
+### Metered fetches
+
+`secondlayer bootstrap` and `secondlayer repair` pull partitions from the
+signed archive instead of replaying the chain, and that pull costs money only
+when it targets the official hosted archive. Point `--against` at a mirror or
+a local manifest and nothing is charged, nothing is even contacted beyond
+that manifest, because self-hosting the archive is a supported way to use
+these commands, not a workaround.
+
+Against the official host, both commands quote before they charge:
+
+1. The manifest's partition list is priced with a free, no-write call
+   (`POST /api/archive/quote`).
+2. The quote prints into the existing plan output, for example
+   `metered: 528 partitions ≈ $44.00 · balance $50.00`, or for `repair`
+   inside its monthly allowance, `metered: free (4 of 6 monthly repair
+   fetches remaining)`.
+3. You confirm, or pass `-y` to skip the prompt. `-y` never skips the quote
+   or the balance check: if the balance is short, the command exits before
+   any partition is fetched and prints the shortfall and
+   `secondlayer credits buy`.
+4. Only then does the command fetch, and only the partitions it actually
+   reads are charged.
+
+A partition already charged in the last 24 hours re-presigns for free, so a
+resumed or retried bootstrap never pays twice for the same bytes. `repair`
+gets 6 free range-bundles a month; `bootstrap` does not, since it is the
+whole-chain operation the free tier exists to not subsidize.
+
+`secondlayer verify` is unaffected: it reads manifests and digests, never
+partition bytes, and stays free and anonymous no matter which archive it
+points at.
 
 ### Subgraphs
 

package/dist/cli.js

@@ -34320,7 +34320,7 @@ init_esm();
 // package.json
 var package_default = {
   name: "@secondlayer/cli",
-  version: "12.1.0",
+  version: "12.2.0",
   description: "Run a Stacks index on your own hardware — set it up, restore verified history, and keep it current.",
   repository: {
     type: "git",
@@ -34364,7 +34364,7 @@ var package_default = {
     "@secondlayer/bundler": "^0.5.1",
     "@secondlayer/scaffold": "^1.5.4",
     "@secondlayer/sdk": "^8.0.1",
-    "@secondlayer/shared": "^10.0.0",
+    "@secondlayer/shared": "^10.0.1",
     "@secondlayer/stacks": "^4.0.0",
     "@secondlayer/subgraphs": "^4.0.2",
     esbuild: "^0.27.4",
@@ -38217,9 +38217,22 @@ async function resolveArchivePublicKey(input4) {
   }
   return;
 }
-async function fetchVerifiedPartition(reference, partition) {
+async function fetchViaGate(gate, partition, retried) {
+  const url = await gate.getUrl(partition.path, retried ? { forceRefresh: true } : undefined);
+  const response2 = await fetch(url, { signal: AbortSignal.timeout(300000) });
+  if (response2.status === 403 && !retried) {
+    return fetchViaGate(gate, partition, true);
+  }
+  if (!response2.ok) {
+    throw new Error(`could not fetch ${partition.path} (${response2.status} ${response2.statusText})`);
+  }
+  return Buffer.from(await response2.arrayBuffer());
+}
+async function fetchVerifiedPartition(reference, partition, gate) {
   let bytes;
-  if (reference.isRemote) {
+  if (gate) {
+    bytes = await fetchViaGate(gate, partition, false);
+  } else if (reference.isRemote) {
     const url = `${reference.root.replace(/\/$/, "")}/${partition.path}`;
     const response2 = await fetch(url, { signal: AbortSignal.timeout(300000) });
     if (!response2.ok) {
@@ -38448,11 +38461,123 @@ import { unlink as unlink2, writeFile as writeFile5 } from "node:fs/promises";
 import { tmpdir as tmpdir2 } from "node:os";
 import { join as join15 } from "node:path";
 import { ParquetReader } from "@dsnp/parquetjs";
+import { confirm as confirm7 } from "@inquirer/prompts";
 import {
   compareRangeDigests as compareRangeDigests2,
   computeRangeDigest as computeRangeDigest2
 } from "@secondlayer/shared/archive/range-digest";
 import { getDb as getDb3, sql as sql2 } from "@secondlayer/shared/db";
+
+// src/lib/archive-gate.ts
+init_http();
+var OFFICIAL_ARCHIVE_HOST = "archive.secondlayer.tools";
+function isOfficialArchive(reference) {
+  if (!reference.isRemote)
+    return false;
+  try {
+    return new URL(reference.origin).hostname === OFFICIAL_ARCHIVE_HOST;
+  } catch {
+    return false;
+  }
+}
+var defaultDeps = { httpArchiveOps };
+function quoteFromBody(body) {
+  return {
+    partitions: body.partitions,
+    bundles: body.bundles,
+    usdMicros: body.usd_micros,
+    usd: body.usd,
+    freeAllowanceAppliedMicros: body.free_allowance_applied_micros,
+    allowanceRemainingBundles: body.allowance_remaining_bundles,
+    balanceUsdMicros: body.balance_usd_micros,
+    sufficient: body.sufficient
+  };
+}
+async function quoteArchiveFetch(paths, flow, deps = defaultDeps) {
+  try {
+    const body = await deps.httpArchiveOps("/api/archive/quote", { method: "POST", body: { paths, flow } });
+    return { ok: true, quote: quoteFromBody(body) };
+  } catch (err) {
+    if (err instanceof CliHttpError) {
+      if (err.status === 503)
+        return { ok: false, kind: "not_configured" };
+      if (err.status === 401) {
+        return { ok: false, kind: "not_authed", message: err.message };
+      }
+      return { ok: false, kind: "error", message: err.message };
+    }
+    throw err;
+  }
+}
+var BUY_CREDITS_COMMAND = "secondlayer credits buy";
+function formatInsufficientMessage(quote) {
+  const shortfallUsd = ((quote.usdMicros - quote.balanceUsdMicros) / 1e6).toFixed(2);
+  const balanceUsd = (quote.balanceUsdMicros / 1e6).toFixed(2);
+  return `Insufficient archive credits: quote $${quote.usd}, balance $${balanceUsd}, short $${shortfallUsd}. Buy more with \`${BUY_CREDITS_COMMAND}\`.`;
+}
+var ARCHIVE_GATE_NOT_CONFIGURED_MESSAGE = "the archive gate is not configured on the server; contact the operator";
+var ALLOWANCE_MONTHLY_BUNDLES = 6;
+function formatQuoteValue(quote, flow) {
+  if (flow === "repair" && quote.usdMicros === 0 && quote.freeAllowanceAppliedMicros > 0) {
+    return `free (${quote.allowanceRemainingBundles} of ${ALLOWANCE_MONTHLY_BUNDLES} monthly repair fetches remaining)`;
+  }
+  const balance = `$${(quote.balanceUsdMicros / 1e6).toFixed(2)}`;
+  return `${quote.partitions} partitions ≈ $${quote.usd} · balance ${balance}`;
+}
+function shouldPromptForGatedFetch(opts) {
+  return !opts.yes && !opts.json;
+}
+var FETCH_BATCH_SIZE = 16;
+function createGatedFetcher(paths, flow, deps = defaultDeps) {
+  const cache = new Map;
+  const batchOf = new Map;
+  for (const [index, path11] of paths.entries()) {
+    batchOf.set(path11, Math.floor(index / FETCH_BATCH_SIZE));
+  }
+  const fetchedBatches = new Set;
+  const inFlight = new Map;
+  async function fetchAndCache(batchPaths) {
+    const body = await deps.httpArchiveOps("/api/archive/fetch", { method: "POST", body: { paths: batchPaths, flow } });
+    for (const u3 of body.urls)
+      cache.set(u3.path, u3.url);
+  }
+  return {
+    async getUrl(path11, opts) {
+      if (opts?.forceRefresh) {
+        if (process.env.DEBUG) {
+          console.error(`[archive-gate] re-issuing expired URL for ${path11}`);
+        }
+        await fetchAndCache([path11]);
+        const fresh = cache.get(path11);
+        if (!fresh) {
+          throw new Error(`archive gate: no URL returned for ${path11}`);
+        }
+        return fresh;
+      }
+      const batchIndex = batchOf.get(path11);
+      if (batchIndex === undefined) {
+        throw new Error(`archive gate: ${path11} was not in the quoted batch`);
+      }
+      if (!fetchedBatches.has(batchIndex)) {
+        let inflight = inFlight.get(batchIndex);
+        if (!inflight) {
+          const batchPaths = paths.filter((p2) => batchOf.get(p2) === batchIndex);
+          inflight = fetchAndCache(batchPaths).then(() => {
+            fetchedBatches.add(batchIndex);
+          });
+          inFlight.set(batchIndex, inflight);
+        }
+        await inflight;
+      }
+      const url = cache.get(path11);
+      if (!url)
+        throw new Error(`archive gate: no URL returned for ${path11}`);
+      return url;
+    }
+  };
+}
+
+// src/commands/repair.ts
 init_output();
 var REPAIR_EXIT = {
   OK: 0,
@@ -38502,8 +38627,8 @@ function parseHeight3(value, flag) {
   }
   return parsed;
 }
-async function planRange(db, reference, partition) {
-  const bytes = await fetchVerifiedPartition(reference, partition);
+async function planRange(db, reference, partition, gate) {
+  const bytes = await fetchVerifiedPartition(reference, partition, gate);
   const archiveBlocks = await readBlocksPartition(bytes, `${partition.from_block}-${partition.to_block}`);
   const localRows = await db.selectFrom("blocks").select(["height", "hash"]).where("canonical", "=", true).where("height", ">=", partition.from_block).where("height", "<=", partition.to_block).execute();
   const localByHeight = new Map(localRows.map((r2) => [Number(r2.height), r2.hash]));
@@ -38530,10 +38655,10 @@ async function planRange(db, reference, partition) {
   const extraHeights = [...localByHeight.keys()].sort((a4, b3) => a4 - b3);
   return { fixes, extraHeights };
 }
-async function applyFixes(db, reference, partition, fixes) {
+async function applyFixes(db, reference, partition, fixes, gate) {
   if (fixes.length === 0)
     return 0;
-  const bytes = await fetchVerifiedPartition(reference, partition);
+  const bytes = await fetchVerifiedPartition(reference, partition, gate);
   const archiveBlocks = await readBlocksPartition(bytes, `apply-${partition.from_block}`);
   const byHeight = new Map(archiveBlocks.map((b3) => [b3.height, b3]));
   const targets = fixes.map((fix) => byHeight.get(fix.height)).filter((b3) => b3 !== undefined);
@@ -38562,7 +38687,7 @@ async function applyFixes(db, reference, partition, fixes) {
   return targets.length;
 }
 function registerRepairCommand(program2) {
-  program2.command("repair").description("Replace local chain data that diverges from a signed archive (dry-run by default)").requiredOption("--against <manifest>", "archive manifest: an https URL or a local file path").option("--from-block <n>", "first height to consider").option("--to-block <n>", "last height to consider").option("--apply", "write the repair (default is a dry-run plan)").option("--public-key <pem>", "pin the signing key instead of fetching it").option("--json", "Output as JSON").addHelpText("after", `
+  program2.command("repair").description("Replace local chain data that diverges from a signed archive (dry-run by default)").requiredOption("--against <manifest>", "archive manifest: an https URL or a local file path").option("--from-block <n>", "first height to consider").option("--to-block <n>", "last height to consider").option("--apply", "write the repair (default is a dry-run plan)").option("--public-key <pem>", "pin the signing key instead of fetching it").option("-y, --yes", "skip the confirmation prompt for a metered fetch").option("--json", "Output as JSON").addHelpText("after", `
 Examples:
   $ secondlayer repair --against ./snapshot.json                      # plan only
   $ secondlayer repair --against ./snapshot.json --apply              # write the fix
@@ -38608,12 +38733,37 @@ Exit codes:
         process.exit(REPAIR_EXIT.OK);
       }
       const partitions = (reference.manifest.partitions ?? []).filter((p2) => p2.dataset === "blocks" && divergent.some((d2) => d2.from_block === p2.from_block && d2.to_block === p2.to_block));
+      let gate;
+      if (isOfficialArchive(reference)) {
+        const paths = partitions.map((p2) => p2.path);
+        const result = await quoteArchiveFetch(paths, "repair");
+        if (!result.ok) {
+          printError(result.kind === "not_configured" ? ARCHIVE_GATE_NOT_CONFIGURED_MESSAGE : result.message);
+          process.exit(REPAIR_EXIT.UNANCHORED);
+        }
+        if (!result.quote.sufficient) {
+          printError(formatInsufficientMessage(result.quote));
+          process.exit(REPAIR_EXIT.UNANCHORED);
+        }
+        note(`  metered: ${formatQuoteValue(result.quote, "repair")}`);
+        if (shouldPromptForGatedFetch(opts)) {
+          const proceed = await confirm7({
+            message: `Fetch ${partitions.length} partition(s) from the archive?`,
+            default: true
+          });
+          if (!proceed) {
+            note("Nothing was fetched.");
+            process.exit(REPAIR_EXIT.UNANCHORED);
+          }
+        }
+        gate = createGatedFetcher(paths, "repair");
+      }
       const allFixes = [];
       const allExtra = [];
       const planned = [];
       for (const partition of partitions) {
         note(`  reading ${partition.dataset} ${partition.from_block}-${partition.to_block}`);
-        const { fixes, extraHeights } = await planRange(db, reference, partition);
+        const { fixes, extraHeights } = await planRange(db, reference, partition, gate);
         planned.push({ partition, fixes });
         allFixes.push(...fixes);
         allExtra.push(...extraHeights);
@@ -38621,7 +38771,7 @@ Exit codes:
       let applied = 0;
       if (opts.apply) {
         for (const { partition, fixes } of planned) {
-          applied += await applyFixes(db, reference, partition, fixes);
+          applied += await applyFixes(db, reference, partition, fixes, gate);
         }
       }
       let remaining = divergent.length;
@@ -38698,7 +38848,7 @@ import { unlink as unlink3, writeFile as writeFile6 } from "node:fs/promises";
 import { tmpdir as tmpdir3 } from "node:os";
 import { join as join16 } from "node:path";
 import { ParquetReader as ParquetReader2 } from "@dsnp/parquetjs";
-import { confirm as confirm7 } from "@inquirer/prompts";
+import { confirm as confirm8 } from "@inquirer/prompts";
 import {
   copyStatement,
   writeRowsToCopyStream
@@ -38794,8 +38944,8 @@ async function* readPartitionRows(bytes, label) {
     await unlink3(path11).catch(() => {});
   }
 }
-async function loadPartition(reference, partition, rawClient) {
-  const bytes = await fetchVerifiedPartition(reference, partition);
+async function loadPartition(reference, partition, rawClient, gate) {
+  const bytes = await fetchVerifiedPartition(reference, partition, gate);
   const writable2 = await rawClient.unsafe(copyStatement(partition.dataset)).writable();
   const written = await writeRowsToCopyStream({
     writable: writable2,
@@ -38872,6 +39022,23 @@ Exit codes:
       const tipHeight = Math.max(...partitions.map((p2) => p2.to_block));
       const restoreSeconds = totalRows / RESTORE_ROWS_PER_SECOND;
       const genesisSeconds = tipHeight / BLOCKS_PER_DAY_ESTIMATE * 86400;
+      const gated = isOfficialArchive(reference);
+      let quoteLine;
+      let gate;
+      if (gated) {
+        const paths = partitions.map((p2) => p2.path);
+        const result = await quoteArchiveFetch(paths, "bootstrap");
+        if (!result.ok) {
+          printError(result.kind === "not_configured" ? ARCHIVE_GATE_NOT_CONFIGURED_MESSAGE : result.message);
+          process.exit(BOOTSTRAP_EXIT.REFUSED);
+        }
+        if (!result.quote.sufficient) {
+          printError(formatInsufficientMessage(result.quote));
+          process.exit(BOOTSTRAP_EXIT.REFUSED);
+        }
+        quoteLine = formatQuoteValue(result.quote, "bootstrap");
+        gate = createGatedFetcher(paths, "bootstrap");
+      }
       if (!opts.json) {
         console.error("");
         console.error(formatKeyValue([
@@ -38882,15 +39049,16 @@ Exit codes:
           ],
           ["rows", totalRows.toLocaleString()],
           ["download", `${(totalBytes / 1e9).toFixed(1)} GB`],
-          ["signature", "verified"]
+          ["signature", "verified"],
+          ...quoteLine ? [["metered", quoteLine]] : []
         ]));
         console.error("");
         console.error(`  sync from genesis   ${bold(`~${formatDuration2(genesisSeconds)}`)} ${dim("(estimate)")}`);
         console.error(`  restore from archive ${bold(`~${formatDuration2(restoreSeconds)}`)}`);
         console.error("");
       }
-      if (!opts.yes && !opts.json) {
-        const proceed = await confirm7({
+      if (shouldPromptForGatedFetch(opts)) {
+        const proceed = await confirm8({
           message: "Restore from the verified archive?",
           default: true
         });
@@ -38907,7 +39075,7 @@ Exit codes:
       for (const dataset of order) {
         const datasetPartitions = partitions.filter((p2) => p2.dataset === dataset).sort((a4, b3) => a4.from_block - b3.from_block);
         for (const [index, partition] of datasetPartitions.entries()) {
-          loaded[dataset] += await loadPartition(reference, partition, rawClient);
+          loaded[dataset] += await loadPartition(reference, partition, rawClient, gate);
           const done = index + 1;
           if (done % 10 === 0 || done === datasetPartitions.length) {
             const elapsed2 = (Date.now() - startedAt) / 1000;
@@ -38964,7 +39132,8 @@ Exit codes:
         elapsed_seconds: Math.round(elapsed),
         resume_from: tipHeight + 1,
         node_tip_at_start: nodeTipAtStart,
-        catch_up_blocks: seam?.gap ?? null
+        catch_up_blocks: seam?.gap ?? null,
+        metered: quoteLine ?? null
       };
       output({
         json: opts.json,
@@ -39639,5 +39808,5 @@ program.commandsGroup("Archive:");
 registerCreditsCommand(program);
 program.parse();
 
-//# debugId=135579D3DFB23FDC64756E2164756E21
+//# debugId=5BF91F30BB4466C964756E2164756E21
 //# sourceMappingURL=cli.js.map