@cogcoin/client 1.1.16 → 1.2.0

package/dist/wallet/mining/mempool-index.js

@@ -0,0 +1,451 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { COG_PREFIX } from "../cogop/constants.js";
+import { writeJsonFileAtomic } from "../fs/atomic.js";
+import { extractOpReturnPayloadFromScriptHex } from "../tx/register.js";
+const MINING_MEMPOOL_INDEX_SCHEMA_VERSION = 1;
+const MINING_MEMPOOL_INDEX_RAW_TX_FETCH_CONCURRENCY = 8;
+const MINING_MEMPOOL_INDEX_PROGRESS_REPORT_EVERY = 25;
+const indexStates = new Map();
+const rawTxSubscribers = new Map();
+export function resolveMiningMempoolIndexCachePath(paths) {
+    return join(paths.miningRoot, "mempool-index.json");
+}
+export function resolveMiningMempoolServiceIdentity(options) {
+    return [
+        options.dataDir,
+        options.pid ?? "unknown-pid",
+        options.zmqEndpoint,
+        options.rawTxTopic ?? "no-rawtx",
+    ].join("|");
+}
+function cacheKey(walletRootId, serviceIdentity, cachePath) {
+    return `${walletRootId}\n${serviceIdentity}\n${cachePath}`;
+}
+function isCogPayload(payload) {
+    return payload !== null
+        && payload.length >= 3
+        && payload[0] === COG_PREFIX[0]
+        && payload[1] === COG_PREFIX[1]
+        && payload[2] === COG_PREFIX[2];
+}
+function serializeContext(context) {
+    return {
+        txid: context.txid,
+        senderScriptHex: context.senderScriptHex,
+        inputTxids: [...context.inputTxids],
+        payloadHex: context.payload === null ? null : Buffer.from(context.payload).toString("hex"),
+    };
+}
+function deserializeContext(context) {
+    if (typeof context.txid !== "string"
+        || (context.senderScriptHex !== null && typeof context.senderScriptHex !== "string")
+        || !Array.isArray(context.inputTxids)
+        || (context.payloadHex !== null && typeof context.payloadHex !== "string")) {
+        return null;
+    }
+    return {
+        txid: context.txid,
+        senderScriptHex: context.senderScriptHex,
+        inputTxids: context.inputTxids.filter((txid) => typeof txid === "string"),
+        payload: context.payloadHex === null ? null : Buffer.from(context.payloadHex, "hex"),
+    };
+}
+async function loadStateFromDisk(state) {
+    if (state.loaded) {
+        return;
+    }
+    state.loaded = true;
+    const raw = await readFile(state.cachePath, "utf8").catch(() => null);
+    if (raw === null) {
+        return;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return;
+    }
+    if (parsed.schemaVersion !== MINING_MEMPOOL_INDEX_SCHEMA_VERSION
+        || parsed.walletRootId !== state.walletRootId
+        || parsed.serviceIdentity !== state.serviceIdentity
+        || !Array.isArray(parsed.contexts)
+        || !Array.isArray(parsed.negativeTxids)) {
+        return;
+    }
+    for (const persistedContext of parsed.contexts) {
+        const context = deserializeContext(persistedContext);
+        if (context !== null && isCogPayload(context.payload)) {
+            state.contexts.set(context.txid, context);
+        }
+    }
+    for (const txid of parsed.negativeTxids) {
+        if (typeof txid === "string") {
+            state.negativeTxids.add(txid);
+        }
+    }
+}
+function getOrCreateState(options) {
+    const key = cacheKey(options.walletRootId, options.serviceIdentity, options.cachePath);
+    const existing = indexStates.get(key);
+    if (existing !== undefined) {
+        return existing;
+    }
+    const created = {
+        walletRootId: options.walletRootId,
+        serviceIdentity: options.serviceIdentity,
+        cachePath: options.cachePath,
+        contexts: new Map(),
+        negativeTxids: new Set(),
+        loaded: false,
+        savePromise: Promise.resolve(),
+    };
+    indexStates.set(key, created);
+    return created;
+}
+async function saveState(state) {
+    const payload = {
+        schemaVersion: MINING_MEMPOOL_INDEX_SCHEMA_VERSION,
+        walletRootId: state.walletRootId,
+        serviceIdentity: state.serviceIdentity,
+        contexts: [...state.contexts.values()]
+            .sort((left, right) => left.txid.localeCompare(right.txid))
+            .map(serializeContext),
+        negativeTxids: [...state.negativeTxids].sort(),
+    };
+    state.savePromise = state.savePromise.catch(() => undefined).then(async () => {
+        await mkdir(dirname(state.cachePath), { recursive: true });
+        await writeJsonFileAtomic(state.cachePath, payload, { mode: 0o600 });
+    });
+    await state.savePromise;
+}
+function pruneStateToVisibleTxids(state, visibleTxids) {
+    const visibleSet = new Set(visibleTxids);
+    let changed = false;
+    for (const txid of [...state.contexts.keys()]) {
+        if (!visibleSet.has(txid)) {
+            state.contexts.delete(txid);
+            changed = true;
+        }
+    }
+    for (const txid of [...state.negativeTxids]) {
+        if (!visibleSet.has(txid)) {
+            state.negativeTxids.delete(txid);
+            changed = true;
+        }
+    }
+    return changed;
+}
+function createContextFromRpcTransaction(txid, tx) {
+    const payloadHex = tx.vout.find((entry) => entry.scriptPubKey?.hex?.startsWith("6a") === true)?.scriptPubKey?.hex;
+    const payload = payloadHex === undefined ? null : extractOpReturnPayloadFromScriptHex(payloadHex);
+    if (!isCogPayload(payload)) {
+        return null;
+    }
+    return {
+        txid,
+        senderScriptHex: tx.vin[0]?.prevout?.scriptPubKey?.hex ?? null,
+        inputTxids: tx.vin.map((input) => input.txid).filter((inputTxid) => inputTxid !== undefined),
+        payload,
+    };
+}
+async function maybeYieldDuringIndexHydration(options) {
+    const yieldEvery = options.cooperativeYieldEvery ?? MINING_MEMPOOL_INDEX_PROGRESS_REPORT_EVERY;
+    if (yieldEvery <= 0 || options.iteration === 0 || (options.iteration % yieldEvery) !== 0) {
+        return;
+    }
+    await (options.cooperativeYield ?? (() => new Promise((resolve) => {
+        setImmediate(resolve);
+    })))();
+}
+async function hydrateUnknownTxids(options) {
+    if (options.unknownTxids.length === 0) {
+        return;
+    }
+    let completed = 0;
+    let nextIndex = 0;
+    let lastReportedProcessed = -1;
+    let reportPromise = Promise.resolve();
+    const reportProgress = async (processed, force = false) => {
+        if (options.onWarmupProgress === undefined) {
+            return;
+        }
+        if (!force && processed !== options.unknownTxids.length && (processed % MINING_MEMPOOL_INDEX_PROGRESS_REPORT_EVERY) !== 0) {
+            return;
+        }
+        if (processed === lastReportedProcessed) {
+            return;
+        }
+        lastReportedProcessed = processed;
+        reportPromise = reportPromise.then(async () => {
+            await options.onWarmupProgress?.({
+                processed,
+                total: options.unknownTxids.length,
+            });
+        });
+        await reportPromise;
+    };
+    await reportProgress(0, true);
+    let failed = false;
+    const workerCount = Math.min(MINING_MEMPOOL_INDEX_RAW_TX_FETCH_CONCURRENCY, options.unknownTxids.length);
+    const workers = Array.from({ length: workerCount }, async () => {
+        while (true) {
+            const iteration = nextIndex;
+            if (iteration >= options.unknownTxids.length) {
+                return;
+            }
+            nextIndex += 1;
+            await maybeYieldDuringIndexHydration({
+                iteration,
+                cooperativeYield: options.cooperativeYield,
+                cooperativeYieldEvery: options.cooperativeYieldEvery,
+            });
+            options.throwIfStopping?.();
+            const txid = options.unknownTxids[iteration];
+            const tx = await options.rpc.getRawTransaction(txid, true).catch(() => null);
+            options.throwIfStopping?.();
+            if (tx === null) {
+                failed = true;
+            }
+            else {
+                const context = createContextFromRpcTransaction(txid, tx);
+                if (context === null) {
+                    options.state.negativeTxids.add(txid);
+                }
+                else {
+                    options.state.contexts.set(txid, context);
+                    options.state.negativeTxids.delete(txid);
+                }
+            }
+            completed += 1;
+            await reportProgress(completed, completed === options.unknownTxids.length);
+        }
+    });
+    await Promise.all(workers);
+    if (failed) {
+        throw new Error("mining_mempool_index_hydration_incomplete");
+    }
+}
+export async function hydrateMiningMempoolIndex(options) {
+    const state = getOrCreateState({
+        walletRootId: options.walletRootId,
+        serviceIdentity: options.serviceIdentity,
+        cachePath: options.cachePath,
+    });
+    await loadStateFromDisk(state);
+    let changed = pruneStateToVisibleTxids(state, options.visibleTxids);
+    const unknownTxids = options.visibleTxids.filter((txid) => !state.contexts.has(txid) && !state.negativeTxids.has(txid));
+    if (unknownTxids.length > 0) {
+        await hydrateUnknownTxids({
+            state,
+            rpc: options.rpc,
+            unknownTxids,
+            cooperativeYield: options.cooperativeYield,
+            cooperativeYieldEvery: options.cooperativeYieldEvery,
+            throwIfStopping: options.throwIfStopping,
+            onWarmupProgress: options.onWarmupProgress,
+        });
+        changed = true;
+    }
+    if (changed) {
+        await saveState(state);
+    }
+    return {
+        contexts: new Map(state.contexts),
+        cacheStatus: unknownTxids.length === 0 ? "indexed" : "index-warming",
+        hydratedCount: unknownTxids.length,
+    };
+}
+function readVarInt(bytes, offset) {
+    const first = bytes[offset];
+    if (first === undefined) {
+        return null;
+    }
+    if (first < 0xfd) {
+        return { value: first, nextOffset: offset + 1, raw: bytes.subarray(offset, offset + 1) };
+    }
+    if (first === 0xfd && (offset + 3) <= bytes.length) {
+        return {
+            value: bytes.readUInt16LE(offset + 1),
+            nextOffset: offset + 3,
+            raw: bytes.subarray(offset, offset + 3),
+        };
+    }
+    if (first === 0xfe && (offset + 5) <= bytes.length) {
+        return {
+            value: bytes.readUInt32LE(offset + 1),
+            nextOffset: offset + 5,
+            raw: bytes.subarray(offset, offset + 5),
+        };
+    }
+    if (first === 0xff && (offset + 9) <= bytes.length) {
+        const value = bytes.readBigUInt64LE(offset + 1);
+        if (value > BigInt(Number.MAX_SAFE_INTEGER)) {
+            return null;
+        }
+        return {
+            value: Number(value),
+            nextOffset: offset + 9,
+            raw: bytes.subarray(offset, offset + 9),
+        };
+    }
+    return null;
+}
+function doubleSha256(bytes) {
+    return createHash("sha256").update(createHash("sha256").update(bytes).digest()).digest();
+}
+function parseRawTransactionForIndex(rawHex) {
+    const bytes = Buffer.from(rawHex, "hex");
+    if (bytes.length < 10) {
+        return null;
+    }
+    let offset = 4;
+    const txidParts = [bytes.subarray(0, 4)];
+    const segwit = bytes[offset] === 0x00 && bytes[offset + 1] !== undefined && bytes[offset + 1] !== 0x00;
+    if (segwit) {
+        offset += 2;
+    }
+    const inputCount = readVarInt(bytes, offset);
+    if (inputCount === null) {
+        return null;
+    }
+    txidParts.push(inputCount.raw);
+    offset = inputCount.nextOffset;
+    const inputTxids = [];
+    for (let index = 0; index < inputCount.value; index += 1) {
+        if ((offset + 36) > bytes.length) {
+            return null;
+        }
+        const prevoutHash = bytes.subarray(offset, offset + 32);
+        inputTxids.push(Buffer.from(prevoutHash).reverse().toString("hex"));
+        const inputFixed = bytes.subarray(offset, offset + 36);
+        txidParts.push(inputFixed);
+        offset += 36;
+        const scriptLength = readVarInt(bytes, offset);
+        if (scriptLength === null || (scriptLength.nextOffset + scriptLength.value + 4) > bytes.length) {
+            return null;
+        }
+        txidParts.push(scriptLength.raw);
+        offset = scriptLength.nextOffset;
+        txidParts.push(bytes.subarray(offset, offset + scriptLength.value + 4));
+        offset += scriptLength.value + 4;
+    }
+    const outputCount = readVarInt(bytes, offset);
+    if (outputCount === null) {
+        return null;
+    }
+    txidParts.push(outputCount.raw);
+    offset = outputCount.nextOffset;
+    let payload = null;
+    for (let index = 0; index < outputCount.value; index += 1) {
+        if ((offset + 8) > bytes.length) {
+            return null;
+        }
+        txidParts.push(bytes.subarray(offset, offset + 8));
+        offset += 8;
+        const scriptLength = readVarInt(bytes, offset);
+        if (scriptLength === null || (scriptLength.nextOffset + scriptLength.value) > bytes.length) {
+            return null;
+        }
+        txidParts.push(scriptLength.raw);
+        offset = scriptLength.nextOffset;
+        const script = bytes.subarray(offset, offset + scriptLength.value);
+        txidParts.push(script);
+        payload ??= extractOpReturnPayloadFromScriptHex(script.toString("hex"));
+        offset += scriptLength.value;
+    }
+    if (segwit) {
+        for (let index = 0; index < inputCount.value; index += 1) {
+            const witnessCount = readVarInt(bytes, offset);
+            if (witnessCount === null) {
+                return null;
+            }
+            offset = witnessCount.nextOffset;
+            for (let witnessIndex = 0; witnessIndex < witnessCount.value; witnessIndex += 1) {
+                const witnessLength = readVarInt(bytes, offset);
+                if (witnessLength === null || (witnessLength.nextOffset + witnessLength.value) > bytes.length) {
+                    return null;
+                }
+                offset = witnessLength.nextOffset + witnessLength.value;
+            }
+        }
+    }
+    if ((offset + 4) !== bytes.length) {
+        return null;
+    }
+    txidParts.push(bytes.subarray(offset, offset + 4));
+    return {
+        txid: Buffer.from(doubleSha256(Buffer.concat(txidParts))).reverse().toString("hex"),
+        inputTxids,
+        payload,
+    };
+}
+function normalizeZmqFrames(frames) {
+    if (Array.isArray(frames)) {
+        return frames.filter(Buffer.isBuffer);
+    }
+    return Buffer.isBuffer(frames) ? [frames] : [];
+}
+async function loadZeroMqModule() {
+    return await import("zeromq");
+}
+export async function ensureMiningMempoolRawTxSubscriber(options) {
+    if (options.rawTxTopic !== "rawtx") {
+        return false;
+    }
+    const key = cacheKey(options.walletRootId, options.serviceIdentity, options.cachePath);
+    if (rawTxSubscribers.has(key)) {
+        return true;
+    }
+    const state = getOrCreateState({
+        walletRootId: options.walletRootId,
+        serviceIdentity: options.serviceIdentity,
+        cachePath: options.cachePath,
+    });
+    await loadStateFromDisk(state);
+    let subscriber;
+    try {
+        const zeroMq = options.loadZeroMq === undefined ? await loadZeroMqModule() : await options.loadZeroMq();
+        subscriber = new zeroMq.Subscriber();
+        subscriber.connect(options.zmqEndpoint);
+        subscriber.subscribe(options.rawTxTopic);
+    }
+    catch {
+        return false;
+    }
+    const loop = (async () => {
+        try {
+            for await (const frames of subscriber) {
+                const normalized = normalizeZmqFrames(frames);
+                if (normalized.length < 2 || normalized[0]?.toString() !== options.rawTxTopic) {
+                    continue;
+                }
+                const parsed = parseRawTransactionForIndex(normalized[1].toString("hex"));
+                if (parsed === null || isCogPayload(parsed.payload)) {
+                    continue;
+                }
+                state.negativeTxids.add(parsed.txid);
+                void saveState(state).catch(() => undefined);
+            }
+        }
+        catch {
+            // The index remains conservative; the gate hydrates unknown txids by RPC.
+        }
+    })();
+    rawTxSubscribers.set(key, { key, subscriber, loop });
+    return true;
+}
+export async function closeMiningMempoolIndexSubscribersForTesting() {
+    const subscribers = [...rawTxSubscribers.values()];
+    rawTxSubscribers.clear();
+    for (const state of subscribers) {
+        state.subscriber.close();
+        await state.loop.catch(() => undefined);
+    }
+}
+export function clearMiningMempoolIndexCacheForTesting() {
+    indexStates.clear();
+}
+export const parseRawTransactionForMiningMempoolIndexTesting = parseRawTransactionForIndex;

package/dist/wallet/mining/runner.js

@@ -36,6 +36,7 @@ import { createMiningSentenceRequestLimits } from "./sentence-protocol.js";
 import { generateMiningSentences, MiningProviderRequestError } from "./sentences.js";
 import { MiningFollowVisualizer, } from "./visualizer.js";
 import { createIndexedMiningFollowVisualizerState, findRecentMiningWin, loadMiningVisibleFollowBlockTimes, resolveFundingDisplaySats, resolveSettledBoard, syncMiningUiForCurrentTip, syncMiningVisualizerBalances, syncMiningVisualizerBlockTimes, } from "./visualizer-sync.js";
+import { ensureMiningMempoolRawTxSubscriber, resolveMiningMempoolIndexCachePath, resolveMiningMempoolServiceIdentity, } from "./mempool-index.js";
 const BEST_BLOCK_POLL_INTERVAL_MS = 500;
 const MINING_SUSPEND_HEARTBEAT_INTERVAL_MS = 1_000;
 class MiningSuspendDetectedError extends Error {
@@ -237,6 +238,24 @@ async function performMiningCycle(options) {
         throwIfStopping();
         throwIfMiningSuspendDetected(options.suspendDetector);
         const rpc = options.rpcFactory(service.rpc);
+        const serviceZmq = service.zmq;
+        const mempoolIndexCachePath = resolveMiningMempoolIndexCachePath(options.paths);
+        const mempoolIndexServiceIdentity = resolveMiningMempoolServiceIdentity({
+            dataDir: service.dataDir ?? options.dataDir,
+            pid: service.pid,
+            zmqEndpoint: serviceZmq?.endpoint ?? "unknown-zmq-endpoint",
+            rawTxTopic: serviceZmq?.rawTxTopic,
+        });
+        const mempoolIndexRawTxSupported = serviceZmq?.rawTxTopic === "rawtx";
+        if (mempoolIndexRawTxSupported && serviceZmq?.endpoint !== undefined) {
+            await ensureMiningMempoolRawTxSubscriber({
+                walletRootId: readContext.localState.state.walletRootId,
+                serviceIdentity: mempoolIndexServiceIdentity,
+                cachePath: mempoolIndexCachePath,
+                zmqEndpoint: serviceZmq.endpoint,
+                rawTxTopic: serviceZmq.rawTxTopic,
+            }).catch(() => false);
+        }
         const reconciliation = await reconcileLiveMiningState({
             state: readContext.localState.state,
             rpc,
@@ -437,6 +456,11 @@ async function performMiningCycle(options) {
             assaySentencesImpl: options.assaySentencesImpl,
             cooperativeYieldImpl: options.cooperativeYieldImpl,
             cooperativeYieldEvery: options.cooperativeYieldEvery,
+            mempoolIndex: {
+                rawTxSupported: mempoolIndexRawTxSupported,
+                cachePath: mempoolIndexCachePath,
+                serviceIdentity: mempoolIndexServiceIdentity,
+            },
             nowImpl: now,
             saveCycleStatus: async (context, overrides) => await saveCycleStatus(context, overrides),
             appendEvent: async (event) => await appendEvent(options.paths, event),

package/dist/wallet/mining/types.d.ts

@@ -109,7 +109,7 @@ export interface MiningRuntimeStatusV1 {
     higherRankedCompetitorDomainCount: number | null;
     dedupedCompetitorDomainCount: number | null;
     competitivenessGateIndeterminate: boolean | null;
-    mempoolSequenceCacheStatus: "reused" | "refreshed" | null;
+    mempoolSequenceCacheStatus: "reused" | "refreshed" | "indexed" | "index-warming" | "fallback-scan" | null;
     currentPublishDecision: string | null;
     lastMempoolSequence: string | null;
     lastCompetitivenessGateAtUnixMs: number | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogcoin/client",
-  "version": "1.1.16",
+  "version": "1.2.0",
   "description": "Store-backed Cogcoin client with wallet flows, SQLite persistence, and managed Bitcoin Core integration.",
   "license": "MIT",
   "type": "module",
@@ -65,7 +65,7 @@
   "dependencies": {
     "@cogcoin/bitcoin": "30.2.0",
     "@cogcoin/genesis": "1.0.0",
-    "@cogcoin/indexer": "1.0.1",
+    "@cogcoin/indexer": "1.0.2",
     "@cogcoin/scoring": "1.0.0",
     "@noble/hashes": "2.0.1",
     "@scure/base": "^2.0.0",
@@ -75,7 +75,7 @@
     "zeromq": "6.5.0"
   },
   "devDependencies": {
-    "@cogcoin/vectors": "1.0.0",
+    "@cogcoin/vectors": "1.0.1",
     "@types/node": "^25.5.0",
     "typescript": "^6.0.2"
   }