@bsv/overlay-express 2.4.0 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +122 -3
  2. package/dist/cjs/mod.js +8 -1
  3. package/dist/cjs/mod.js.map +1 -1
  4. package/dist/cjs/package.json +9 -9
  5. package/dist/cjs/src/ArcadeProvider.js +196 -0
  6. package/dist/cjs/src/ArcadeProvider.js.map +1 -0
  7. package/dist/cjs/src/ChaintracksProvider.js +65 -0
  8. package/dist/cjs/src/ChaintracksProvider.js.map +1 -0
  9. package/dist/cjs/src/OverlayExpress.js +235 -11
  10. package/dist/cjs/src/OverlayExpress.js.map +1 -1
  11. package/dist/cjs/src/OverlayMonitor.js +117 -3
  12. package/dist/cjs/src/OverlayMonitor.js.map +1 -1
  13. package/dist/cjs/src/ProviderChainBroadcaster.js +91 -0
  14. package/dist/cjs/src/ProviderChainBroadcaster.js.map +1 -0
  15. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  16. package/dist/esm/mod.js +3 -0
  17. package/dist/esm/mod.js.map +1 -1
  18. package/dist/esm/src/ArcadeProvider.js +189 -0
  19. package/dist/esm/src/ArcadeProvider.js.map +1 -0
  20. package/dist/esm/src/ChaintracksProvider.js +62 -0
  21. package/dist/esm/src/ChaintracksProvider.js.map +1 -0
  22. package/dist/esm/src/OverlayExpress.js +231 -11
  23. package/dist/esm/src/OverlayExpress.js.map +1 -1
  24. package/dist/esm/src/OverlayMonitor.js +117 -3
  25. package/dist/esm/src/OverlayMonitor.js.map +1 -1
  26. package/dist/esm/src/ProviderChainBroadcaster.js +88 -0
  27. package/dist/esm/src/ProviderChainBroadcaster.js.map +1 -0
  28. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  29. package/dist/types/mod.d.ts +3 -0
  30. package/dist/types/mod.d.ts.map +1 -1
  31. package/dist/types/src/ArcadeProvider.d.ts +37 -0
  32. package/dist/types/src/ArcadeProvider.d.ts.map +1 -0
  33. package/dist/types/src/ChaintracksProvider.d.ts +29 -0
  34. package/dist/types/src/ChaintracksProvider.d.ts.map +1 -0
  35. package/dist/types/src/OverlayExpress.d.ts +38 -1
  36. package/dist/types/src/OverlayExpress.d.ts.map +1 -1
  37. package/dist/types/src/OverlayMonitor.d.ts +28 -0
  38. package/dist/types/src/OverlayMonitor.d.ts.map +1 -1
  39. package/dist/types/src/ProviderChainBroadcaster.d.ts +18 -0
  40. package/dist/types/src/ProviderChainBroadcaster.d.ts.map +1 -0
  41. package/dist/types/tsconfig.types.tsbuildinfo +1 -1
  42. package/mod.ts +15 -0
  43. package/package.json +9 -9
  44. package/src/ArcadeProvider.ts +232 -0
  45. package/src/ChaintracksProvider.ts +79 -0
  46. package/src/OverlayExpress.ts +259 -11
  47. package/src/OverlayMonitor.ts +163 -3
  48. package/src/ProviderChainBroadcaster.ts +108 -0
  49. package/src/__tests__/OverlayExpress.test.ts +37 -3
  50. package/src/__tests__/OverlayMonitor.test.ts +74 -0
  51. package/src/__tests__/OverlayProviders.live.test.ts +39 -0
  52. package/src/__tests__/OverlayProviders.test.ts +123 -0
@@ -55,6 +55,9 @@ const BanAwareDiscoveryStorage_js_1 = require("./BanAwareDiscoveryStorage.js");
55
55
  const ReorgStream_js_1 = require("./ReorgStream.js");
56
56
  const wallet_toolbox_client_1 = require("@bsv/wallet-toolbox-client");
57
57
  const auth_express_middleware_1 = require("@bsv/auth-express-middleware");
58
+ const ArcadeProvider_js_1 = require("./ArcadeProvider.js");
59
+ const ProviderChainBroadcaster_js_1 = require("./ProviderChainBroadcaster.js");
60
+ const ChaintracksProvider_js_1 = require("./ChaintracksProvider.js");
58
61
  /**
59
62
  * In-memory migration source for Knex migrations.
60
63
  * Allows running migrations defined in code rather than files.
@@ -133,12 +136,21 @@ class OverlayExpress {
133
136
  // chain with empty anchors, so the cumulative TAC advances "after each new
134
137
  // block" per BRC-136. Set to 0 to disable polling (startup extension still runs).
135
138
  this.basmBlockPollIntervalMs = 10 * 60 * 1000;
139
+ // How often (ms) to refresh proofs for old unproven rows and then evict rows
140
+ // that still have no proof. Set to 0 to disable background maintenance.
141
+ this.unprovenMaintenanceIntervalMs = 0;
136
142
  // Depth (in blocks from the tip) for the reorg revalidation sweep.
137
143
  this.reorgScanDepth = 3;
138
144
  // ARC API Key
139
145
  this.arcApiKey = undefined;
140
146
  // Optional ARC callback token for /arc-ingest notifications
141
147
  this.arcCallbackToken = undefined;
148
+ // Optional Arcade URL/API key used for propagation, proof refresh, and
149
+ // go-chaintracks header/reorg access when available.
150
+ this.arcadeUrl = undefined;
151
+ this.arcadeApiKey = undefined;
152
+ this.arcadeDeploymentId = undefined;
153
+ this.arcadeChaintracksApiPrefix = '/chaintracks/v2';
142
154
  // Verbose request logging
143
155
  this.verboseRequestLogging = false;
144
156
  // Web UI configuration
@@ -277,6 +289,42 @@ class OverlayExpress {
277
289
  this.arcCallbackToken = token;
278
290
  this.logger.log(chalk_1.default.blue('ARC callback token has been configured.'));
279
291
  }
292
+ /**
293
+ * Configures Arcade for first-choice transaction propagation and proof lookup.
294
+ */
295
+ configureArcade(url, config = {}) {
296
+ this.arcadeUrl = url;
297
+ this.arcadeApiKey = config.apiKey;
298
+ this.arcadeDeploymentId = config.deploymentId;
299
+ if (config.chaintracksApiPrefix !== undefined) {
300
+ this.arcadeChaintracksApiPrefix = config.chaintracksApiPrefix;
301
+ }
302
+ this.logger.log(chalk_1.default.blue('Arcade provider has been configured.'));
303
+ }
304
+ /**
305
+ * Configures a go-chaintracks compatible service for header validation and
306
+ * BASM reorg streaming. Arcade exposes this at `/chaintracks/v2`.
307
+ */
308
+ configureChaintracks(url, config = {}) {
309
+ var _a;
310
+ const apiPrefix = (_a = config.apiPrefix) !== null && _a !== void 0 ? _a : '/chaintracks/v2';
311
+ const client = new ChaintracksProvider_js_1.ChaintracksProvider(url, { apiPrefix });
312
+ this.configureChainTracker(client);
313
+ this.configureTopicAnchorHeaderResolver(async (blockHeight) => {
314
+ const header = await client.findHeaderForHeight(blockHeight);
315
+ if (header === undefined)
316
+ return undefined;
317
+ return {
318
+ blockHeight,
319
+ blockHash: header.hash,
320
+ merkleRoot: header.merkleRoot
321
+ };
322
+ });
323
+ if (config.reorgStream !== false) {
324
+ this.configureReorgStream(client.reorgStreamUrl(), config.scanDepth);
325
+ }
326
+ this.logger.log(chalk_1.default.blue('go-chaintracks provider has been configured.'));
327
+ }
280
328
  /**
281
329
  * Enables or disables GASP synchronization (high-level setting).
282
330
  * This is a broad toggle that can be overridden or customized through syncConfiguration.
@@ -323,6 +371,19 @@ class OverlayExpress {
323
371
  }
324
372
  this.logger.log(chalk_1.default.blue('Unproven transaction eviction has been configured.'));
325
373
  }
374
+ /**
375
+ * Configures periodic unproven maintenance. Each run first tries configured
376
+ * proof providers, then evicts rows that are still unproven past the threshold.
377
+ */
378
+ configureUnprovenMaintenance(config) {
379
+ if (config.intervalMs !== undefined) {
380
+ this.unprovenMaintenanceIntervalMs = config.intervalMs;
381
+ }
382
+ if (config.thresholdBlocks !== undefined) {
383
+ this.unprovenEvictionBlocks = config.thresholdBlocks;
384
+ }
385
+ this.logger.log(chalk_1.default.blue('Unproven transaction maintenance has been configured.'));
386
+ }
326
387
  /**
327
388
  * Configures how often the BASM anchor chain is extended with empty anchors to
328
389
  * follow the chain tip. Set to 0 to disable periodic polling.
@@ -471,7 +532,7 @@ class OverlayExpress {
471
532
  const EngineWithBASM = overlay_1.Engine;
472
533
  this.engine = new EngineWithBASM(this.managers, this.services, storage, (_a = this.engineConfig.chainTracker) !== null && _a !== void 0 ? _a : this.chainTracker, `https://${this.advertisableFQDN}`, this.network === 'test'
473
534
  ? ((_b = this.engineConfig.shipTrackers) !== null && _b !== void 0 ? _b : sdk_1.DEFAULT_TESTNET_SLAP_TRACKERS)
474
- : this.engineConfig.shipTrackers, this.resolveSlapTrackers(), broadcaster !== null && broadcaster !== void 0 ? broadcaster : this.engineConfig.broadcaster, advertiser, syncConfig, (_c = this.engineConfig.logTime) !== null && _c !== void 0 ? _c : false, (_d = this.engineConfig.logPrefix) !== null && _d !== void 0 ? _d : '[OVERLAY_ENGINE] ', (_e = this.engineConfig.throwOnBroadcastFailure) !== null && _e !== void 0 ? _e : false, (_f = this.engineConfig.overlayBroadcastFacilitator) !== null && _f !== void 0 ? _f : new sdk_1.HTTPSOverlayBroadcastFacilitator(), this.logger, (_g = this.engineConfig.suppressDefaultSyncAdvertisements) !== null && _g !== void 0 ? _g : true, this.buildTopicAnchorHeaderResolver(), (_h = this.engineConfig.enableBASMSync) !== null && _h !== void 0 ? _h : this.enableBASMSync, (_j = this.engineConfig.unprovenEvictionBlocks) !== null && _j !== void 0 ? _j : this.unprovenEvictionBlocks);
535
+ : this.engineConfig.shipTrackers, this.resolveSlapTrackers(), broadcaster !== null && broadcaster !== void 0 ? broadcaster : this.engineConfig.broadcaster, advertiser, syncConfig, (_c = this.engineConfig.logTime) !== null && _c !== void 0 ? _c : false, (_d = this.engineConfig.logPrefix) !== null && _d !== void 0 ? _d : '[OVERLAY_ENGINE] ', (_e = this.engineConfig.throwOnBroadcastFailure) !== null && _e !== void 0 ? _e : true, (_f = this.engineConfig.overlayBroadcastFacilitator) !== null && _f !== void 0 ? _f : new sdk_1.HTTPSOverlayBroadcastFacilitator(), this.logger, (_g = this.engineConfig.suppressDefaultSyncAdvertisements) !== null && _g !== void 0 ? _g : true, this.buildTopicAnchorHeaderResolver(), (_h = this.engineConfig.enableBASMSync) !== null && _h !== void 0 ? _h : this.enableBASMSync, (_j = this.engineConfig.unprovenEvictionBlocks) !== null && _j !== void 0 ? _j : this.unprovenEvictionBlocks);
475
536
  this.initServerWallet();
476
537
  this.logger.log(chalk_1.default.green('Engine has been configured.'));
477
538
  }
@@ -506,16 +567,88 @@ class OverlayExpress {
506
567
  }
507
568
  return syncConfig;
508
569
  }
509
- /** Build the ARC broadcaster if an API key is configured. */
570
+ /** Build the configured transaction propagation provider chain. */
510
571
  buildBroadcaster() {
511
- if (typeof this.arcApiKey !== 'string')
572
+ const providers = [];
573
+ const callbackUrl = `https://${this.advertisableFQDN}/arc-ingest`;
574
+ if (typeof this.arcadeUrl === 'string' && this.arcadeUrl.length > 0) {
575
+ this.arcadeProvider = new ArcadeProvider_js_1.ArcadeProvider(this.arcadeUrl, {
576
+ apiKey: this.arcadeApiKey,
577
+ callbackUrl,
578
+ callbackToken: this.arcCallbackToken,
579
+ deploymentId: this.arcadeDeploymentId
580
+ });
581
+ providers.push({
582
+ name: 'Arcade',
583
+ broadcaster: this.arcadeProvider
584
+ });
585
+ }
586
+ else {
587
+ this.arcadeProvider = undefined;
588
+ }
589
+ if (typeof this.arcApiKey === 'string' && this.arcApiKey.length > 0) {
590
+ const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com';
591
+ providers.push({
592
+ name: 'ARC',
593
+ broadcaster: new sdk_1.ARC(arcUrl, {
594
+ apiKey: this.arcApiKey,
595
+ callbackUrl,
596
+ callbackToken: this.arcCallbackToken
597
+ })
598
+ });
599
+ }
600
+ if (providers.length === 0)
601
+ return undefined;
602
+ if (providers.length === 1)
603
+ return providers[0].broadcaster;
604
+ return new ProviderChainBroadcaster_js_1.ProviderChainBroadcaster(providers);
605
+ }
606
+ ensureArcadeProvider() {
607
+ if (this.arcadeProvider !== undefined)
608
+ return this.arcadeProvider;
609
+ if (typeof this.arcadeUrl !== 'string' || this.arcadeUrl.length === 0)
512
610
  return undefined;
513
- const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com';
514
- return new sdk_1.ARC(arcUrl, {
515
- apiKey: this.arcApiKey,
611
+ this.arcadeProvider = new ArcadeProvider_js_1.ArcadeProvider(this.arcadeUrl, {
612
+ apiKey: this.arcadeApiKey,
516
613
  callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
517
- callbackToken: this.arcCallbackToken
614
+ callbackToken: this.arcCallbackToken,
615
+ deploymentId: this.arcadeDeploymentId
518
616
  });
617
+ return this.arcadeProvider;
618
+ }
619
+ async fetchArcadeProof(txid) {
620
+ var _a, _b;
621
+ const provider = this.ensureArcadeProvider();
622
+ if (provider === undefined)
623
+ return undefined;
624
+ const proof = await provider.fetchMerkleProof(txid);
625
+ if (proof === undefined)
626
+ return undefined;
627
+ const chainTracker = (_a = this.engineConfig.chainTracker) !== null && _a !== void 0 ? _a : this.chainTracker;
628
+ if (chainTracker === 'scripts only') {
629
+ throw new Error('Cannot validate Arcade proof with scripts-only chain tracker');
630
+ }
631
+ const blockHeight = (_b = proof.blockHeight) !== null && _b !== void 0 ? _b : proof.merklePath.blockHeight;
632
+ if (blockHeight === undefined) {
633
+ throw new Error(`Arcade proof for ${txid} did not include a block height`);
634
+ }
635
+ const valid = await chainTracker.isValidRootForHeight(proof.merkleRoot, blockHeight);
636
+ if (!valid) {
637
+ throw new Error(`Arcade proof for ${txid} did not match the chain tracker at height ${blockHeight}`);
638
+ }
639
+ return {
640
+ ...proof,
641
+ blockHeight
642
+ };
643
+ }
644
+ async fetchConfiguredMerkleProof(txid) {
645
+ const proof = await this.fetchArcadeProof(txid);
646
+ if (proof === undefined)
647
+ return undefined;
648
+ return {
649
+ merklePath: proof.merklePath,
650
+ blockHeight: proof.blockHeight
651
+ };
519
652
  }
520
653
  /** Build the BASM block header resolver. */
521
654
  buildTopicAnchorHeaderResolver() {
@@ -1160,8 +1293,9 @@ class OverlayExpress {
1160
1293
  });
1161
1294
  });
1162
1295
  });
1163
- // ARC ingest route (only if we have an ARC API key)
1164
- if (typeof this.arcApiKey === 'string' && this.arcApiKey.length > 0) {
1296
+ // ARC/Arcade ingest route (only if a provider is configured)
1297
+ if ((typeof this.arcApiKey === 'string' && this.arcApiKey.length > 0) ||
1298
+ (typeof this.arcadeUrl === 'string' && this.arcadeUrl.length > 0)) {
1165
1299
  this.app.post('/arc-ingest', (req, res) => {
1166
1300
  ;
1167
1301
  (async () => {
@@ -1178,9 +1312,44 @@ class OverlayExpress {
1178
1312
  return res.status(401).json({ status: 'error', message: 'Unauthorized callback' });
1179
1313
  }
1180
1314
  }
1181
- const { txid, merklePath: merklePathHex, blockHeight } = req.body;
1315
+ const { txid, merklePath: merklePathHex, blockHeight, txStatus, extraInfo, competingTxs, topic } = req.body;
1316
+ if (typeof txid !== 'string' || txid === '') {
1317
+ throw new TypeError('Provider callback is missing txid');
1318
+ }
1319
+ if ((0, ArcadeProvider_js_1.isTerminalArcStatus)(txStatus, extraInfo)) {
1320
+ const report = await engine.evictAppliedTransaction(txid, {
1321
+ topic: typeof topic === 'string' ? topic : undefined,
1322
+ reason: `${txStatus !== null && txStatus !== void 0 ? txStatus : ''} ${extraInfo !== null && extraInfo !== void 0 ? extraInfo : ''}`.trim()
1323
+ });
1324
+ this.logger.warn({
1325
+ operation: 'overlay.provider_callback',
1326
+ outcome: 'terminal_evicted',
1327
+ txid,
1328
+ txStatus,
1329
+ competingTxs,
1330
+ report
1331
+ });
1332
+ return res.status(200).json({
1333
+ status: 'success',
1334
+ message: 'Terminal transaction status processed',
1335
+ data: {
1336
+ ...report,
1337
+ txStatus,
1338
+ competingTxs
1339
+ }
1340
+ });
1341
+ }
1342
+ if (typeof merklePathHex !== 'string' || merklePathHex === '') {
1343
+ return res.status(202).json({ status: 'success', message: 'Transaction status received without proof' });
1344
+ }
1182
1345
  const merklePath = sdk_1.MerklePath.fromHex(merklePathHex);
1183
1346
  await engine.handleNewMerkleProof(txid, merklePath, blockHeight);
1347
+ this.logger.log({
1348
+ operation: 'overlay.provider_callback',
1349
+ outcome: 'proof_ingested',
1350
+ txid,
1351
+ blockHeight
1352
+ });
1184
1353
  return res.status(200).json({ status: 'success', message: 'Transaction status updated' });
1185
1354
  }
1186
1355
  catch (error) {
@@ -1199,7 +1368,7 @@ class OverlayExpress {
1199
1368
  });
1200
1369
  }
1201
1370
  else {
1202
- this.logger.warn(chalk_1.default.yellow('Disabling ARC because no ARC API key was provided.'));
1371
+ this.logger.warn(chalk_1.default.yellow('Disabling ARC/Arcade ingest because no provider was configured.'));
1203
1372
  }
1204
1373
  // GASP sync routes if enabled
1205
1374
  if (this.enableGASPSync) {
@@ -1718,8 +1887,38 @@ class OverlayExpress {
1718
1887
  topic: typeof topic === 'string' ? topic : undefined,
1719
1888
  thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined
1720
1889
  });
1890
+ this.logger.log({ operation: 'overlay.unproven_eviction', outcome: 'ok', report });
1721
1891
  return { status: 'success', message: 'Unproven eviction completed', data: report };
1722
1892
  }, checkAdminAuth);
1893
+ /**
1894
+ * Admin route to refresh proofs for expired unproven topic transactions
1895
+ * using the configured proof providers.
1896
+ */
1897
+ registerJsonRoute('/admin/refreshUnprovenProofs', async (req) => {
1898
+ var _a;
1899
+ const { topic, thresholdBlocks } = (_a = req.body) !== null && _a !== void 0 ? _a : {};
1900
+ const report = await basmEngine.refreshUnprovenTransactionProofs({
1901
+ topic: typeof topic === 'string' ? topic : undefined,
1902
+ thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined,
1903
+ proofProvider: async (txid) => await this.fetchConfiguredMerkleProof(txid)
1904
+ });
1905
+ this.logger.log({ operation: 'overlay.unproven_proof_refresh', outcome: 'ok', report });
1906
+ return { status: 'success', message: 'Unproven proof refresh completed', data: report };
1907
+ }, checkAdminAuth);
1908
+ /**
1909
+ * Admin route to refresh proofs first, then evict rows that remain unproven.
1910
+ */
1911
+ registerJsonRoute('/admin/maintainUnproven', async (req) => {
1912
+ var _a;
1913
+ const { topic, thresholdBlocks } = (_a = req.body) !== null && _a !== void 0 ? _a : {};
1914
+ const report = await basmEngine.maintainUnprovenTransactions({
1915
+ topic: typeof topic === 'string' ? topic : undefined,
1916
+ thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined,
1917
+ proofProvider: async (txid) => await this.fetchConfiguredMerkleProof(txid)
1918
+ });
1919
+ this.logger.log({ operation: 'overlay.unproven_maintenance', outcome: 'ok', report });
1920
+ return { status: 'success', message: 'Unproven maintenance completed', data: report };
1921
+ }, checkAdminAuth);
1723
1922
  /**
1724
1923
  * Admin route to evict an outpoint, either from all services or a specific one.
1725
1924
  */
@@ -1830,6 +2029,7 @@ class OverlayExpress {
1830
2029
  }
1831
2030
  await this.runGaspStartupSync();
1832
2031
  await this.runBasmStartupSync();
2032
+ this.startUnprovenMaintenance();
1833
2033
  }
1834
2034
  /** Attempt a GASP sync at startup when enabled. */
1835
2035
  async runGaspStartupSync() {
@@ -1920,6 +2120,30 @@ class OverlayExpress {
1920
2120
  console.error(chalk_1.default.red('Failed to revalidate BASM anchors'), e);
1921
2121
  }
1922
2122
  }
2123
+ startUnprovenMaintenance() {
2124
+ var _a, _b, _c;
2125
+ const intervalMs = (_a = this.engineConfig.unprovenMaintenanceIntervalMs) !== null && _a !== void 0 ? _a : this.unprovenMaintenanceIntervalMs;
2126
+ if (intervalMs <= 0)
2127
+ return;
2128
+ const run = () => {
2129
+ void (async () => {
2130
+ var _a, _b;
2131
+ try {
2132
+ const report = await ((_a = this.engine) === null || _a === void 0 ? void 0 : _a.maintainUnprovenTransactions({
2133
+ thresholdBlocks: (_b = this.engineConfig.unprovenEvictionBlocks) !== null && _b !== void 0 ? _b : this.unprovenEvictionBlocks,
2134
+ proofProvider: async (txid) => await this.fetchConfiguredMerkleProof(txid)
2135
+ }));
2136
+ this.logger.log(chalk_1.default.green('Unproven transaction maintenance complete'), report);
2137
+ }
2138
+ catch (e) {
2139
+ console.error(chalk_1.default.red('Failed to maintain unproven transactions'), e);
2140
+ }
2141
+ })();
2142
+ };
2143
+ run();
2144
+ this.unprovenMaintenanceTimer = setInterval(run, intervalMs);
2145
+ (_c = (_b = this.unprovenMaintenanceTimer).unref) === null || _c === void 0 ? void 0 : _c.call(_b);
2146
+ }
1923
2147
  }
1924
2148
  exports.default = OverlayExpress;
1925
2149
  //# sourceMappingURL=OverlayExpress.js.map