@bsv/overlay-express 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/cjs/mod.js +10 -1
- package/dist/cjs/mod.js.map +1 -1
- package/dist/cjs/package.json +12 -11
- package/dist/cjs/src/BanAwareDiscoveryStorage.js +78 -0
- package/dist/cjs/src/BanAwareDiscoveryStorage.js.map +1 -0
- package/dist/cjs/src/BanAwareTopicManager.js +71 -0
- package/dist/cjs/src/BanAwareTopicManager.js.map +1 -0
- package/dist/cjs/src/OverlayExpress.js +375 -96
- package/dist/cjs/src/OverlayExpress.js.map +1 -1
- package/dist/cjs/src/OverlayMonitor.js +464 -0
- package/dist/cjs/src/OverlayMonitor.js.map +1 -0
- package/dist/cjs/src/ReorgStream.js +193 -0
- package/dist/cjs/src/ReorgStream.js.map +1 -0
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/esm/mod.js +3 -0
- package/dist/esm/mod.js.map +1 -1
- package/dist/esm/src/BanAwareDiscoveryStorage.js +79 -0
- package/dist/esm/src/BanAwareDiscoveryStorage.js.map +1 -0
- package/dist/esm/src/BanAwareTopicManager.js +71 -0
- package/dist/esm/src/BanAwareTopicManager.js.map +1 -0
- package/dist/esm/src/OverlayExpress.js +375 -94
- package/dist/esm/src/OverlayExpress.js.map +1 -1
- package/dist/esm/src/OverlayMonitor.js +458 -0
- package/dist/esm/src/OverlayMonitor.js.map +1 -0
- package/dist/esm/src/ReorgStream.js +190 -0
- package/dist/esm/src/ReorgStream.js.map +1 -0
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/types/mod.d.ts +4 -1
- package/dist/types/mod.d.ts.map +1 -1
- package/dist/types/src/BanAwareDiscoveryStorage.d.ts +36 -0
- package/dist/types/src/BanAwareDiscoveryStorage.d.ts.map +1 -0
- package/dist/types/src/BanAwareTopicManager.d.ts +28 -0
- package/dist/types/src/BanAwareTopicManager.d.ts.map +1 -0
- package/dist/types/src/OverlayExpress.d.ts +82 -4
- package/dist/types/src/OverlayExpress.d.ts.map +1 -1
- package/dist/types/src/OverlayMonitor.d.ts +169 -0
- package/dist/types/src/OverlayMonitor.d.ts.map +1 -0
- package/dist/types/src/ReorgStream.d.ts +74 -0
- package/dist/types/src/ReorgStream.d.ts.map +1 -0
- package/dist/types/tsconfig.types.tsbuildinfo +1 -1
- package/mod.ts +20 -1
- package/package.json +12 -11
- package/src/BanAwareDiscoveryStorage.ts +91 -0
- package/src/BanAwareTopicManager.ts +87 -0
- package/src/OverlayExpress.ts +461 -102
- package/src/OverlayMonitor.ts +657 -0
- package/src/ReorgStream.ts +221 -0
- package/src/__tests__/BanAwareDiscoveryStorage.test.ts +131 -0
- package/src/__tests__/BanAwareTopicManager.test.ts +132 -0
- package/src/__tests__/OverlayExpress.test.ts +86 -1
- package/src/__tests__/OverlayMonitor.test.ts +181 -0
- package/src/__tests__/ReorgStream.test.ts +66 -0
package/src/OverlayExpress.ts
CHANGED
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import express from 'express'
|
|
2
2
|
import bodyParser from 'body-parser'
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
Engine,
|
|
5
|
+
KnexStorage,
|
|
6
|
+
LookupService,
|
|
7
|
+
TopicManager,
|
|
8
|
+
KnexStorageMigrations,
|
|
9
|
+
Advertiser
|
|
10
|
+
} from '@bsv/overlay'
|
|
4
11
|
import {
|
|
5
12
|
ARC,
|
|
6
13
|
ChainTracker,
|
|
@@ -30,6 +37,9 @@ import { v4 as uuidv4 } from 'uuid'
|
|
|
30
37
|
import { JanitorService, type JanitorReport } from './JanitorService.js'
|
|
31
38
|
import { BanService } from './BanService.js'
|
|
32
39
|
import { BanAwareLookupWrapper } from './BanAwareLookupWrapper.js'
|
|
40
|
+
import { BanAwareTopicManager } from './BanAwareTopicManager.js'
|
|
41
|
+
import { BanAwareSHIPStorage, BanAwareSLAPStorage } from './BanAwareDiscoveryStorage.js'
|
|
42
|
+
import { ReorgSseAdapter, type ReorgHandlerInput } from './ReorgStream.js'
|
|
33
43
|
import { Wallet, WalletSigner, WalletStorageManager, Services } from '@bsv/wallet-toolbox-client'
|
|
34
44
|
import { createAuthMiddleware, type AuthRequest } from '@bsv/auth-express-middleware'
|
|
35
45
|
|
|
@@ -92,6 +102,11 @@ export interface EngineConfig {
|
|
|
92
102
|
throwOnBroadcastFailure?: boolean
|
|
93
103
|
overlayBroadcastFacilitator?: OverlayBroadcastFacilitator
|
|
94
104
|
suppressDefaultSyncAdvertisements?: boolean
|
|
105
|
+
topicAnchorHeaderResolver?: TopicAnchorHeaderResolver
|
|
106
|
+
enableBASMSync?: boolean
|
|
107
|
+
unprovenEvictionBlocks?: number
|
|
108
|
+
reorgStreamUrl?: string
|
|
109
|
+
reorgScanDepth?: number
|
|
95
110
|
}
|
|
96
111
|
|
|
97
112
|
export type HealthStatus = 'ok' | 'degraded' | 'error'
|
|
@@ -139,6 +154,25 @@ export interface HealthReport {
|
|
|
139
154
|
context?: Record<string, any>
|
|
140
155
|
}
|
|
141
156
|
|
|
157
|
+
export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise<{
|
|
158
|
+
blockHeight: number
|
|
159
|
+
blockHash: string
|
|
160
|
+
merkleRoot?: string
|
|
161
|
+
} | undefined>
|
|
162
|
+
|
|
163
|
+
interface BASMCapableEngine extends Engine {
|
|
164
|
+
provideTopicAnchorTip: (topic: string) => Promise<any>
|
|
165
|
+
provideTopicAnchorRange: (topic: string, fromHeight: number, toHeight: number) => Promise<any>
|
|
166
|
+
provideAdmittedList: (topic: string, blockHeight: number, blockHash?: string) => Promise<any>
|
|
167
|
+
provideCompoundMerklePath: (topic: string, blockHeight: number, txids: string[]) => Promise<any>
|
|
168
|
+
provideRawTransactions: (txids: string[]) => Promise<any>
|
|
169
|
+
startBASMSync: () => Promise<any>
|
|
170
|
+
advanceTopicAnchorChains: (toHeight?: number) => Promise<void>
|
|
171
|
+
evictUnprovenTransactions: (options?: { topic?: string, thresholdBlocks?: number }) => Promise<any>
|
|
172
|
+
handleReorg: (input: ReorgHandlerInput) => Promise<any>
|
|
173
|
+
revalidateRecentAnchors: (depth?: number) => Promise<any>
|
|
174
|
+
}
|
|
175
|
+
|
|
142
176
|
/**
|
|
143
177
|
* OverlayExpress class provides an Express-based server for hosting Overlay Services.
|
|
144
178
|
* It allows configuration of various components like databases, topic managers, and lookup services.
|
|
@@ -186,9 +220,41 @@ export default class OverlayExpress {
|
|
|
186
220
|
// (We allow an on/off toggle, but also can do advanced custom sync config below)
|
|
187
221
|
enableGASPSync: boolean = true
|
|
188
222
|
|
|
223
|
+
// Enable BRC-136 BASM sync. Off by default; endpoints remain available when
|
|
224
|
+
// storage supports anchors.
|
|
225
|
+
enableBASMSync: boolean = false
|
|
226
|
+
|
|
227
|
+
// Opt-in unproven eviction default threshold in blocks.
|
|
228
|
+
unprovenEvictionBlocks: number = 144
|
|
229
|
+
|
|
230
|
+
// How often (ms) to poll the chain tip and extend each topic's BASM anchor
|
|
231
|
+
// chain with empty anchors, so the cumulative TAC advances "after each new
|
|
232
|
+
// block" per BRC-136. Set to 0 to disable polling (startup extension still runs).
|
|
233
|
+
basmBlockPollIntervalMs: number = 10 * 60 * 1000
|
|
234
|
+
|
|
235
|
+
// Handle for the BASM block-poll timer so it can be stopped.
|
|
236
|
+
private basmBlockPollTimer?: ReturnType<typeof setInterval>
|
|
237
|
+
|
|
238
|
+
// Optional go-chaintracks (Arcade) reorg SSE URL (e.g. `<base>/v2/reorg/stream`).
|
|
239
|
+
// When set, reorgs are reconciled in real time; the block poll also runs a
|
|
240
|
+
// revalidation sweep as a fallback / reconnect catch-up.
|
|
241
|
+
reorgStreamUrl?: string
|
|
242
|
+
|
|
243
|
+
// Depth (in blocks from the tip) for the reorg revalidation sweep.
|
|
244
|
+
reorgScanDepth: number = 3
|
|
245
|
+
|
|
246
|
+
// Handle for the reorg SSE adapter.
|
|
247
|
+
private reorgAdapter?: ReorgSseAdapter
|
|
248
|
+
|
|
249
|
+
// Optional resolver for block hashes and header merkle roots used by BASM.
|
|
250
|
+
topicAnchorHeaderResolver?: TopicAnchorHeaderResolver
|
|
251
|
+
|
|
189
252
|
// ARC API Key
|
|
190
253
|
arcApiKey: string | undefined = undefined
|
|
191
254
|
|
|
255
|
+
// Optional ARC callback token for /arc-ingest notifications
|
|
256
|
+
arcCallbackToken: string | undefined = undefined
|
|
257
|
+
|
|
192
258
|
// Verbose request logging
|
|
193
259
|
verboseRequestLogging: boolean = false
|
|
194
260
|
|
|
@@ -372,6 +438,15 @@ export default class OverlayExpress {
|
|
|
372
438
|
this.logger.log(chalk.blue('ARC API key has been configured.'))
|
|
373
439
|
}
|
|
374
440
|
|
|
441
|
+
/**
|
|
442
|
+
* Configures the ARC callback token expected by /arc-ingest.
|
|
443
|
+
* @param token - The token ARC should present when posting callback notifications.
|
|
444
|
+
*/
|
|
445
|
+
configureArcCallbackToken (token: string): void {
|
|
446
|
+
this.arcCallbackToken = token
|
|
447
|
+
this.logger.log(chalk.blue('ARC callback token has been configured.'))
|
|
448
|
+
}
|
|
449
|
+
|
|
375
450
|
/**
|
|
376
451
|
* Enables or disables GASP synchronization (high-level setting).
|
|
377
452
|
* This is a broad toggle that can be overridden or customized through syncConfiguration.
|
|
@@ -382,6 +457,56 @@ export default class OverlayExpress {
|
|
|
382
457
|
this.logger.log(chalk.blue(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`))
|
|
383
458
|
}
|
|
384
459
|
|
|
460
|
+
/**
|
|
461
|
+
* Enables or disables BRC-136 BASM synchronization.
|
|
462
|
+
* BASM is opt-in because it requires direct proofs and block hash resolution.
|
|
463
|
+
*/
|
|
464
|
+
configureEnableBASMSync (enable: boolean): void {
|
|
465
|
+
this.enableBASMSync = enable
|
|
466
|
+
this.logger.log(chalk.blue(`BASM synchronization ${enable ? 'enabled' : 'disabled'}.`))
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Configures the block header resolver used to derive BASM block hashes.
|
|
471
|
+
*/
|
|
472
|
+
configureTopicAnchorHeaderResolver (resolver: TopicAnchorHeaderResolver): void {
|
|
473
|
+
this.topicAnchorHeaderResolver = resolver
|
|
474
|
+
this.logger.log(chalk.blue('BASM topic anchor header resolver has been configured.'))
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/**
|
|
478
|
+
* Configures the go-chaintracks (Arcade) reorg SSE stream used to reconcile
|
|
479
|
+
* BASM anchors with blockchain reorganizations in real time.
|
|
480
|
+
* @param url - The reorg stream URL, e.g. `https://arcade.example/v2/reorg/stream`.
|
|
481
|
+
* @param scanDepth - Optional revalidation-sweep depth in blocks (default 3).
|
|
482
|
+
*/
|
|
483
|
+
configureReorgStream (url: string, scanDepth?: number): void {
|
|
484
|
+
this.reorgStreamUrl = url
|
|
485
|
+
if (scanDepth !== undefined) {
|
|
486
|
+
this.reorgScanDepth = scanDepth
|
|
487
|
+
}
|
|
488
|
+
this.logger.log(chalk.blue('BASM reorg stream has been configured.'))
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
/**
|
|
492
|
+
* Configures the opt-in unproven state eviction threshold.
|
|
493
|
+
*/
|
|
494
|
+
configureUnprovenEviction (config: { thresholdBlocks?: number }): void {
|
|
495
|
+
if (config.thresholdBlocks !== undefined) {
|
|
496
|
+
this.unprovenEvictionBlocks = config.thresholdBlocks
|
|
497
|
+
}
|
|
498
|
+
this.logger.log(chalk.blue('Unproven transaction eviction has been configured.'))
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Configures how often the BASM anchor chain is extended with empty anchors to
|
|
503
|
+
* follow the chain tip. Set to 0 to disable periodic polling.
|
|
504
|
+
*/
|
|
505
|
+
configureBASMBlockPollInterval (intervalMs: number): void {
|
|
506
|
+
this.basmBlockPollIntervalMs = intervalMs
|
|
507
|
+
this.logger.log(chalk.blue(`BASM block poll interval set to ${intervalMs}ms.`))
|
|
508
|
+
}
|
|
509
|
+
|
|
385
510
|
/**
|
|
386
511
|
* Enables or disables verbose request logging.
|
|
387
512
|
* @param enable - true to enable, false to disable
|
|
@@ -499,9 +624,9 @@ export default class OverlayExpress {
|
|
|
499
624
|
* By default, auto-configures SHIP and SLAP unless autoConfigureShipSlap = false
|
|
500
625
|
* Then it merges in any advanced engine config from `this.engineConfig`.
|
|
501
626
|
*
|
|
502
|
-
* When a BanService is available (from configureMongo), SHIP
|
|
503
|
-
*
|
|
504
|
-
*
|
|
627
|
+
* When a BanService is available (from configureMongo), auto-configured SHIP
|
|
628
|
+
* and SLAP managers, discovery storage, and lookup services are wrapped so
|
|
629
|
+
* banned outputs are not admitted or indexed.
|
|
505
630
|
*
|
|
506
631
|
* @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true)
|
|
507
632
|
*/
|
|
@@ -511,12 +636,20 @@ export default class OverlayExpress {
|
|
|
511
636
|
if (autoConfigureShipSlap) {
|
|
512
637
|
this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager())
|
|
513
638
|
this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager())
|
|
514
|
-
this.configureLookupServiceWithMongo('ls_ship', (db) =>
|
|
515
|
-
new DiscoveryServices.SHIPStorage(db)
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
639
|
+
this.configureLookupServiceWithMongo('ls_ship', (db) => {
|
|
640
|
+
const storage = new DiscoveryServices.SHIPStorage(db)
|
|
641
|
+
const storageForLookup = this.banService === undefined
|
|
642
|
+
? storage
|
|
643
|
+
: new BanAwareSHIPStorage(storage, this.banService, this.logger)
|
|
644
|
+
return new DiscoveryServices.SHIPLookupService(storageForLookup as any)
|
|
645
|
+
})
|
|
646
|
+
this.configureLookupServiceWithMongo('ls_slap', (db) => {
|
|
647
|
+
const storage = new DiscoveryServices.SLAPStorage(db)
|
|
648
|
+
const storageForLookup = this.banService === undefined
|
|
649
|
+
? storage
|
|
650
|
+
: new BanAwareSLAPStorage(storage, this.banService, this.logger)
|
|
651
|
+
return new DiscoveryServices.SLAPLookupService(storageForLookup as any)
|
|
652
|
+
})
|
|
520
653
|
}
|
|
521
654
|
|
|
522
655
|
this.wrapBanAwareServices()
|
|
@@ -528,7 +661,8 @@ export default class OverlayExpress {
|
|
|
528
661
|
const broadcaster = this.buildBroadcaster()
|
|
529
662
|
const advertiser = await this.buildAdvertiser()
|
|
530
663
|
|
|
531
|
-
|
|
664
|
+
const EngineWithBASM = Engine as unknown as new (...args: any[]) => Engine
|
|
665
|
+
this.engine = new EngineWithBASM(
|
|
532
666
|
this.managers,
|
|
533
667
|
this.services,
|
|
534
668
|
storage,
|
|
@@ -546,16 +680,26 @@ export default class OverlayExpress {
|
|
|
546
680
|
this.engineConfig.throwOnBroadcastFailure ?? false,
|
|
547
681
|
this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(),
|
|
548
682
|
this.logger,
|
|
549
|
-
this.engineConfig.suppressDefaultSyncAdvertisements ?? true
|
|
683
|
+
this.engineConfig.suppressDefaultSyncAdvertisements ?? true,
|
|
684
|
+
this.buildTopicAnchorHeaderResolver(),
|
|
685
|
+
this.engineConfig.enableBASMSync ?? this.enableBASMSync,
|
|
686
|
+
this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks
|
|
550
687
|
)
|
|
551
688
|
|
|
552
689
|
this.initServerWallet()
|
|
553
690
|
this.logger.log(chalk.green('Engine has been configured.'))
|
|
554
691
|
}
|
|
555
692
|
|
|
556
|
-
/** Wrap SHIP/SLAP with ban-aware
|
|
693
|
+
/** Wrap SHIP/SLAP managers and services with ban-aware filters if BanService is configured. */
|
|
557
694
|
private wrapBanAwareServices (): void {
|
|
558
695
|
if (this.banService === undefined) return
|
|
696
|
+
for (const key of ['tm_ship', 'tm_slap'] as const) {
|
|
697
|
+
if (this.managers[key] !== undefined) {
|
|
698
|
+
const label = key === 'tm_ship' ? 'SHIP' : 'SLAP'
|
|
699
|
+
this.managers[key] = new BanAwareTopicManager(this.managers[key], this.banService, label, this.logger)
|
|
700
|
+
this.logger.log(chalk.blue(`${label} topic manager wrapped with ban-aware filter.`))
|
|
701
|
+
}
|
|
702
|
+
}
|
|
559
703
|
for (const key of ['ls_ship', 'ls_slap'] as const) {
|
|
560
704
|
if (this.services[key] !== undefined) {
|
|
561
705
|
const label = key === 'ls_ship' ? 'SHIP' : 'SLAP'
|
|
@@ -581,7 +725,38 @@ export default class OverlayExpress {
|
|
|
581
725
|
private buildBroadcaster (): Broadcaster | undefined {
|
|
582
726
|
if (typeof this.arcApiKey !== 'string') return undefined
|
|
583
727
|
const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com'
|
|
584
|
-
return new ARC(arcUrl, {
|
|
728
|
+
return new ARC(arcUrl, {
|
|
729
|
+
apiKey: this.arcApiKey,
|
|
730
|
+
callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
|
|
731
|
+
callbackToken: this.arcCallbackToken
|
|
732
|
+
})
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/** Build the BASM block header resolver. */
|
|
736
|
+
private buildTopicAnchorHeaderResolver (): TopicAnchorHeaderResolver | undefined {
|
|
737
|
+
const configured = this.engineConfig.topicAnchorHeaderResolver ?? this.topicAnchorHeaderResolver
|
|
738
|
+
if (configured !== undefined) {
|
|
739
|
+
return configured
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
return async (blockHeight: number) => {
|
|
743
|
+
const response = await fetch(`https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, {
|
|
744
|
+
method: 'GET',
|
|
745
|
+
headers: { Accept: 'application/json' }
|
|
746
|
+
})
|
|
747
|
+
if (!response.ok) {
|
|
748
|
+
throw new Error(`WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}`)
|
|
749
|
+
}
|
|
750
|
+
const header = await response.json() as { hash?: string, merkleroot?: string }
|
|
751
|
+
if (typeof header.hash !== 'string') {
|
|
752
|
+
throw new TypeError(`WhatsOnChain did not return a block hash for height ${blockHeight}`)
|
|
753
|
+
}
|
|
754
|
+
return {
|
|
755
|
+
blockHeight,
|
|
756
|
+
blockHash: header.hash,
|
|
757
|
+
merkleRoot: header.merkleroot
|
|
758
|
+
}
|
|
759
|
+
}
|
|
585
760
|
}
|
|
586
761
|
|
|
587
762
|
/** Resolve the SLAP trackers from config or network defaults. */
|
|
@@ -901,6 +1076,71 @@ export default class OverlayExpress {
|
|
|
901
1076
|
return report
|
|
902
1077
|
}
|
|
903
1078
|
|
|
1079
|
+
/**
|
|
1080
|
+
* Renders a request or response body for verbose logging, truncating overly long payloads.
|
|
1081
|
+
*/
|
|
1082
|
+
private formatBodyForLog (body: any, tooLongLabel: string, okPrefix: string): string {
|
|
1083
|
+
let bodyString: string
|
|
1084
|
+
if (typeof body === 'object') {
|
|
1085
|
+
bodyString = JSON.stringify(body, null, 2)
|
|
1086
|
+
} else if (Buffer.isBuffer(body)) {
|
|
1087
|
+
bodyString = body.toString('utf8')
|
|
1088
|
+
} else {
|
|
1089
|
+
bodyString = String(body)
|
|
1090
|
+
}
|
|
1091
|
+
return bodyString.length > 280
|
|
1092
|
+
? chalk.yellow(`(${tooLongLabel}, length: ${String(bodyString.length)} characters)`)
|
|
1093
|
+
: chalk.green(`${okPrefix}\n${bodyString}`)
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* Installs middleware that verbosely logs incoming requests and outgoing responses.
|
|
1098
|
+
*/
|
|
1099
|
+
private setupVerboseRequestLogging (): void {
|
|
1100
|
+
this.app.use((req, res, next) => {
|
|
1101
|
+
const startTime = Date.now()
|
|
1102
|
+
|
|
1103
|
+
// Log incoming request details
|
|
1104
|
+
this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`))
|
|
1105
|
+
// Pretty-print headers
|
|
1106
|
+
this.logger.log(chalk.cyan('Headers:'))
|
|
1107
|
+
this.logger.log(util.inspect(req.headers, { colors: true, depth: null }))
|
|
1108
|
+
|
|
1109
|
+
// Handle request body
|
|
1110
|
+
if (req.body != null && Object.keys(req.body).length > 0) {
|
|
1111
|
+
this.logger.log(this.formatBodyForLog(req.body, 'Body too long to display', 'Request Body:'))
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
// Intercept the res.send method to log responses
|
|
1115
|
+
const originalSend = res.send
|
|
1116
|
+
let responseBody: any
|
|
1117
|
+
|
|
1118
|
+
res.send = function (body?: any): any {
|
|
1119
|
+
responseBody = body
|
|
1120
|
+
return originalSend.call(this, body)
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
// Log outgoing response details after the response is finished
|
|
1124
|
+
res.on('finish', () => {
|
|
1125
|
+
const duration = Date.now() - startTime
|
|
1126
|
+
this.logger.log(
|
|
1127
|
+
chalk.magenta.bold(
|
|
1128
|
+
`Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`
|
|
1129
|
+
)
|
|
1130
|
+
)
|
|
1131
|
+
this.logger.log(chalk.cyan('Response Headers:'))
|
|
1132
|
+
this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }))
|
|
1133
|
+
|
|
1134
|
+
// Handle response body
|
|
1135
|
+
if (responseBody != null) {
|
|
1136
|
+
this.logger.log(this.formatBodyForLog(responseBody, 'Response body too long to display', 'Response Body:'))
|
|
1137
|
+
}
|
|
1138
|
+
})
|
|
1139
|
+
|
|
1140
|
+
next()
|
|
1141
|
+
})
|
|
1142
|
+
}
|
|
1143
|
+
|
|
904
1144
|
/**
|
|
905
1145
|
* Starts the Express server.
|
|
906
1146
|
* Sets up routes and begins listening on the configured port.
|
|
@@ -914,78 +1154,7 @@ export default class OverlayExpress {
|
|
|
914
1154
|
this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }))
|
|
915
1155
|
|
|
916
1156
|
if (this.verboseRequestLogging) {
|
|
917
|
-
this.
|
|
918
|
-
const startTime = Date.now()
|
|
919
|
-
|
|
920
|
-
// Log incoming request details
|
|
921
|
-
this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`))
|
|
922
|
-
// Pretty-print headers
|
|
923
|
-
this.logger.log(chalk.cyan('Headers:'))
|
|
924
|
-
this.logger.log(util.inspect(req.headers, { colors: true, depth: null }))
|
|
925
|
-
|
|
926
|
-
// Handle request body
|
|
927
|
-
if (req.body != null && Object.keys(req.body).length > 0) {
|
|
928
|
-
let bodyContent
|
|
929
|
-
let bodyString
|
|
930
|
-
if (typeof req.body === 'object') {
|
|
931
|
-
bodyString = JSON.stringify(req.body, null, 2)
|
|
932
|
-
} else if (Buffer.isBuffer(req.body)) {
|
|
933
|
-
bodyString = req.body.toString('utf8')
|
|
934
|
-
} else {
|
|
935
|
-
bodyString = String(req.body)
|
|
936
|
-
}
|
|
937
|
-
if (bodyString.length > 280) {
|
|
938
|
-
bodyContent = chalk.yellow(`(Body too long to display, length: ${String(bodyString.length)} characters)`)
|
|
939
|
-
} else {
|
|
940
|
-
bodyContent = chalk.green(`Request Body:\n${String(bodyString)}`)
|
|
941
|
-
}
|
|
942
|
-
this.logger.log(bodyContent)
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
// Intercept the res.send method to log responses
|
|
946
|
-
const originalSend = res.send
|
|
947
|
-
let responseBody: any
|
|
948
|
-
|
|
949
|
-
res.send = function (body?: any): any {
|
|
950
|
-
responseBody = body
|
|
951
|
-
return originalSend.call(this, body)
|
|
952
|
-
}
|
|
953
|
-
|
|
954
|
-
// Log outgoing response details after the response is finished
|
|
955
|
-
res.on('finish', () => {
|
|
956
|
-
const duration = Date.now() - startTime
|
|
957
|
-
this.logger.log(
|
|
958
|
-
chalk.magenta.bold(
|
|
959
|
-
`Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`
|
|
960
|
-
)
|
|
961
|
-
)
|
|
962
|
-
this.logger.log(chalk.cyan('Response Headers:'))
|
|
963
|
-
this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }))
|
|
964
|
-
|
|
965
|
-
// Handle response body
|
|
966
|
-
if (responseBody != null) {
|
|
967
|
-
let bodyContent
|
|
968
|
-
let bodyString
|
|
969
|
-
if (typeof responseBody === 'object') {
|
|
970
|
-
bodyString = JSON.stringify(responseBody, null, 2)
|
|
971
|
-
} else if (Buffer.isBuffer(responseBody)) {
|
|
972
|
-
bodyString = responseBody.toString('utf8')
|
|
973
|
-
} else if (typeof responseBody === 'string') {
|
|
974
|
-
bodyString = responseBody
|
|
975
|
-
} else {
|
|
976
|
-
bodyString = String(responseBody)
|
|
977
|
-
}
|
|
978
|
-
if (bodyString.length > 280) {
|
|
979
|
-
bodyContent = chalk.yellow(`(Response body too long to display, length: ${String(bodyString.length)} characters)`)
|
|
980
|
-
} else {
|
|
981
|
-
bodyContent = chalk.green(`Response Body:\n${String(bodyString)}`)
|
|
982
|
-
}
|
|
983
|
-
this.logger.log(bodyContent)
|
|
984
|
-
}
|
|
985
|
-
})
|
|
986
|
-
|
|
987
|
-
next()
|
|
988
|
-
})
|
|
1157
|
+
this.setupVerboseRequestLogging()
|
|
989
1158
|
}
|
|
990
1159
|
|
|
991
1160
|
// Enable CORS
|
|
@@ -1253,6 +1422,18 @@ export default class OverlayExpress {
|
|
|
1253
1422
|
this.app.post('/arc-ingest', (req, res) => {
|
|
1254
1423
|
; (async () => {
|
|
1255
1424
|
try {
|
|
1425
|
+
if (typeof this.arcCallbackToken === 'string' && this.arcCallbackToken.length > 0) {
|
|
1426
|
+
const authorization = req.headers.authorization
|
|
1427
|
+
const headerToken = Array.isArray(authorization) ? authorization[0] : authorization
|
|
1428
|
+
const xCallbackToken = req.headers['x-callback-token']
|
|
1429
|
+
const callbackToken = Array.isArray(xCallbackToken) ? xCallbackToken[0] : xCallbackToken
|
|
1430
|
+
const bearerToken = typeof headerToken === 'string' && headerToken.startsWith('Bearer ')
|
|
1431
|
+
? headerToken.slice('Bearer '.length)
|
|
1432
|
+
: headerToken
|
|
1433
|
+
if (bearerToken !== this.arcCallbackToken && callbackToken !== this.arcCallbackToken) {
|
|
1434
|
+
return res.status(401).json({ status: 'error', message: 'Unauthorized callback' })
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1256
1437
|
const { txid, merklePath: merklePathHex, blockHeight } = req.body
|
|
1257
1438
|
const merklePath = MerklePath.fromHex(merklePathHex)
|
|
1258
1439
|
await engine.handleNewMerkleProof(txid, merklePath, blockHeight)
|
|
@@ -1322,6 +1503,78 @@ export default class OverlayExpress {
|
|
|
1322
1503
|
this.logger.warn(chalk.yellow('GASP sync is disabled.'))
|
|
1323
1504
|
}
|
|
1324
1505
|
|
|
1506
|
+
// BRC-136 BASM anchor and raw transaction endpoints.
|
|
1507
|
+
const basmEngine = engine as BASMCapableEngine
|
|
1508
|
+
const readBasmTopic = (req: express.Request): string => {
|
|
1509
|
+
const header = req.headers['x-bsv-topic']
|
|
1510
|
+
if (typeof header !== 'string' || header.length === 0) {
|
|
1511
|
+
throw new TypeError('Missing x-bsv-topic header')
|
|
1512
|
+
}
|
|
1513
|
+
return header
|
|
1514
|
+
}
|
|
1515
|
+
|
|
1516
|
+
/**
|
|
1517
|
+
* Registers a POST route whose handler resolves to the JSON payload returned
|
|
1518
|
+
* with HTTP 200. Any thrown error is logged and returned as HTTP 400, while
|
|
1519
|
+
* unexpected rejections fall back to HTTP 500. This consolidates the shared
|
|
1520
|
+
* async/try-catch boilerplate used by the BRC-136 BASM endpoints (and the
|
|
1521
|
+
* BASM-related admin endpoints, which additionally pass `checkAdminAuth`).
|
|
1522
|
+
*/
|
|
1523
|
+
const registerJsonRoute = (
|
|
1524
|
+
path: string,
|
|
1525
|
+
handler: (req: express.Request) => Promise<unknown>,
|
|
1526
|
+
...middleware: express.RequestHandler[]
|
|
1527
|
+
): void => {
|
|
1528
|
+
this.app.post(path, ...(middleware as any[]), (req: express.Request, res: express.Response) => {
|
|
1529
|
+
; (async () => {
|
|
1530
|
+
try {
|
|
1531
|
+
return res.status(200).json(await handler(req))
|
|
1532
|
+
} catch (error) {
|
|
1533
|
+
console.error(chalk.red(`Error in ${path}:`), error)
|
|
1534
|
+
return res.status(400).json({
|
|
1535
|
+
status: 'error',
|
|
1536
|
+
message: error instanceof Error ? error.message : 'An unknown error occurred'
|
|
1537
|
+
})
|
|
1538
|
+
}
|
|
1539
|
+
})().catch(() => {
|
|
1540
|
+
res.status(500).json({ status: 'error', message: 'Unexpected error' })
|
|
1541
|
+
})
|
|
1542
|
+
})
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
const requireTxids = (value: unknown): string[] => {
|
|
1546
|
+
if (!Array.isArray(value) || !value.every(txid => typeof txid === 'string')) {
|
|
1547
|
+
throw new TypeError('txids must be an array of strings')
|
|
1548
|
+
}
|
|
1549
|
+
return value
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
registerJsonRoute('/requestTopicAnchorTip', async req =>
|
|
1553
|
+
await basmEngine.provideTopicAnchorTip(readBasmTopic(req)))
|
|
1554
|
+
|
|
1555
|
+
registerJsonRoute('/requestTopicAnchorRange', async req => {
|
|
1556
|
+
const { fromHeight, toHeight } = req.body
|
|
1557
|
+
return await basmEngine.provideTopicAnchorRange(readBasmTopic(req), Number(fromHeight), Number(toHeight))
|
|
1558
|
+
})
|
|
1559
|
+
|
|
1560
|
+
registerJsonRoute('/requestAdmittedList', async req => {
|
|
1561
|
+
const { blockHeight, blockHash } = req.body
|
|
1562
|
+
return await basmEngine.provideAdmittedList(
|
|
1563
|
+
readBasmTopic(req),
|
|
1564
|
+
Number(blockHeight),
|
|
1565
|
+
typeof blockHash === 'string' ? blockHash : undefined
|
|
1566
|
+
)
|
|
1567
|
+
})
|
|
1568
|
+
|
|
1569
|
+
registerJsonRoute('/requestCompoundMerklePath', async req => {
|
|
1570
|
+
const topic = readBasmTopic(req)
|
|
1571
|
+
const { blockHeight, txids } = req.body
|
|
1572
|
+
return await basmEngine.provideCompoundMerklePath(topic, Number(blockHeight), requireTxids(txids))
|
|
1573
|
+
})
|
|
1574
|
+
|
|
1575
|
+
registerJsonRoute('/requestRawTransactions', async req =>
|
|
1576
|
+
await basmEngine.provideRawTransactions(requireTxids(req.body.txids)))
|
|
1577
|
+
|
|
1325
1578
|
/**
|
|
1326
1579
|
* ============== ADMIN ROUTES ==============
|
|
1327
1580
|
* These routes expose advanced engine operations.
|
|
@@ -1351,7 +1604,7 @@ export default class OverlayExpress {
|
|
|
1351
1604
|
*/
|
|
1352
1605
|
const checkAdminAuth = (req: express.Request, res: express.Response, next: express.NextFunction): void => {
|
|
1353
1606
|
// Method 1: BSV mutual authentication (identity key match)
|
|
1354
|
-
const authReq = req as AuthRequest
|
|
1607
|
+
const authReq = req as unknown as AuthRequest
|
|
1355
1608
|
if (
|
|
1356
1609
|
typeof this.adminIdentityKey === 'string' &&
|
|
1357
1610
|
authReq.auth !== undefined &&
|
|
@@ -1419,7 +1672,9 @@ export default class OverlayExpress {
|
|
|
1419
1672
|
totalBans: banStats.totalBans,
|
|
1420
1673
|
topicManagers: Object.keys(this.managers),
|
|
1421
1674
|
lookupServices: Object.keys(this.services),
|
|
1422
|
-
gaspSyncEnabled: this.enableGASPSync
|
|
1675
|
+
gaspSyncEnabled: this.enableGASPSync,
|
|
1676
|
+
basmSyncEnabled: this.enableBASMSync,
|
|
1677
|
+
unprovenEvictionBlocks: this.unprovenEvictionBlocks
|
|
1423
1678
|
}
|
|
1424
1679
|
})
|
|
1425
1680
|
} catch (error) {
|
|
@@ -1728,6 +1983,26 @@ export default class OverlayExpress {
|
|
|
1728
1983
|
})
|
|
1729
1984
|
})
|
|
1730
1985
|
|
|
1986
|
+
/**
|
|
1987
|
+
* Admin route to manually start BASM sync, calling `engine.startBASMSync()`.
|
|
1988
|
+
*/
|
|
1989
|
+
registerJsonRoute('/admin/startBASMSync', async () => {
|
|
1990
|
+
const report = await basmEngine.startBASMSync()
|
|
1991
|
+
return { status: 'success', message: 'BASM sync started and completed', data: report }
|
|
1992
|
+
}, checkAdminAuth as any)
|
|
1993
|
+
|
|
1994
|
+
/**
|
|
1995
|
+
* Admin route to evict expired unproven topic transactions.
|
|
1996
|
+
*/
|
|
1997
|
+
registerJsonRoute('/admin/evictUnproven', async req => {
|
|
1998
|
+
const { topic, thresholdBlocks } = req.body ?? {}
|
|
1999
|
+
const report = await basmEngine.evictUnprovenTransactions({
|
|
2000
|
+
topic: typeof topic === 'string' ? topic : undefined,
|
|
2001
|
+
thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined
|
|
2002
|
+
})
|
|
2003
|
+
return { status: 'success', message: 'Unproven eviction completed', data: report }
|
|
2004
|
+
}, checkAdminAuth as any)
|
|
2005
|
+
|
|
1731
2006
|
/**
|
|
1732
2007
|
* Admin route to evict an outpoint, either from all services or a specific one.
|
|
1733
2008
|
*/
|
|
@@ -1804,6 +2079,20 @@ export default class OverlayExpress {
|
|
|
1804
2079
|
})
|
|
1805
2080
|
})
|
|
1806
2081
|
|
|
2082
|
+
await this.runStartupSync()
|
|
2083
|
+
|
|
2084
|
+
// Start listening on the configured port
|
|
2085
|
+
this.app.listen(this.port, () => {
|
|
2086
|
+
this.isListening = true
|
|
2087
|
+
this.logger.log(chalk.green.bold(`${this.name} is ready and listening on local port ${this.port}`))
|
|
2088
|
+
})
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
/**
|
|
2092
|
+
* Runs the post-listen startup work: advertiser init, advertisement sync,
|
|
2093
|
+
* and the optional GASP/BASM background syncs.
|
|
2094
|
+
*/
|
|
2095
|
+
private async runStartupSync (): Promise<void> {
|
|
1807
2096
|
// The legacy Ninja advertiser has a setLookupEngine method.
|
|
1808
2097
|
if (this.engine?.advertiser instanceof DiscoveryServices.WalletAdvertiser) {
|
|
1809
2098
|
this.logger.log(
|
|
@@ -1827,23 +2116,93 @@ export default class OverlayExpress {
|
|
|
1827
2116
|
this.logger.log(chalk.red('Error syncing advertisements:'), e)
|
|
1828
2117
|
}
|
|
1829
2118
|
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
console.error(chalk.red('Failed to GASP sync'), e)
|
|
1838
|
-
}
|
|
1839
|
-
} else {
|
|
2119
|
+
await this.runGaspStartupSync()
|
|
2120
|
+
await this.runBasmStartupSync()
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
/** Attempt a GASP sync at startup when enabled. */
|
|
2124
|
+
private async runGaspStartupSync (): Promise<void> {
|
|
2125
|
+
if (!this.enableGASPSync) {
|
|
1840
2126
|
this.logger.log(chalk.yellow(`${this.name} will not sync because GASP has been disabled.`))
|
|
2127
|
+
return
|
|
2128
|
+
}
|
|
2129
|
+
try {
|
|
2130
|
+
this.logger.log(chalk.green('Starting GASP sync...'))
|
|
2131
|
+
await this.engine?.startGASPSync()
|
|
2132
|
+
this.logger.log(chalk.green('GASP sync complete!'))
|
|
2133
|
+
} catch (e) {
|
|
2134
|
+
console.error(chalk.red('Failed to GASP sync'), e)
|
|
1841
2135
|
}
|
|
2136
|
+
}
|
|
1842
2137
|
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
2138
|
+
/** Attempt a BASM sync at startup when enabled, then begin tip-following. */
|
|
2139
|
+
private async runBasmStartupSync (): Promise<void> {
|
|
2140
|
+
if (!(this.enableBASMSync || this.engineConfig.enableBASMSync === true)) {
|
|
2141
|
+
return
|
|
2142
|
+
}
|
|
2143
|
+
try {
|
|
2144
|
+
this.logger.log(chalk.green('Starting BASM sync...'))
|
|
2145
|
+
const report = await (this.engine as BASMCapableEngine | undefined)?.startBASMSync()
|
|
2146
|
+
this.logger.log(chalk.green('BASM sync complete!'), report)
|
|
2147
|
+
} catch (e) {
|
|
2148
|
+
console.error(chalk.red('Failed to BASM sync'), e)
|
|
2149
|
+
}
|
|
2150
|
+
|
|
2151
|
+
// Extend each topic's anchor chain to the current tip on startup, then keep
|
|
2152
|
+
// it following the tip so the cumulative TAC advances after each new block.
|
|
2153
|
+
await this.advanceBASMAnchorChains()
|
|
2154
|
+
this.startBASMBlockPolling()
|
|
2155
|
+
this.startBASMReorgStream()
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
/** Poll for new blocks to advance anchor chains and detect reorgs. */
|
|
2159
|
+
private startBASMBlockPolling (): void {
|
|
2160
|
+
if (this.basmBlockPollIntervalMs <= 0) {
|
|
2161
|
+
return
|
|
2162
|
+
}
|
|
2163
|
+
this.basmBlockPollTimer = setInterval(() => {
|
|
2164
|
+
void this.advanceBASMAnchorChains()
|
|
2165
|
+
// Fallback reorg detection for chain trackers without a reorg stream,
|
|
2166
|
+
// and a safety net even when the SSE adapter is active.
|
|
2167
|
+
void this.revalidateBASMAnchors()
|
|
2168
|
+
}, this.basmBlockPollIntervalMs)
|
|
2169
|
+
this.basmBlockPollTimer.unref?.()
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
/** Real-time reorg reconciliation via the go-chaintracks (Arcade) reorg SSE. */
|
|
2173
|
+
private startBASMReorgStream (): void {
|
|
2174
|
+
const reorgStreamUrl = this.engineConfig.reorgStreamUrl ?? this.reorgStreamUrl
|
|
2175
|
+
const reorgScanDepth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth
|
|
2176
|
+
if (reorgStreamUrl === undefined || reorgStreamUrl === '') {
|
|
2177
|
+
return
|
|
2178
|
+
}
|
|
2179
|
+
const basmEngine = this.engine as BASMCapableEngine | undefined
|
|
2180
|
+
this.reorgAdapter = new ReorgSseAdapter({
|
|
2181
|
+
url: reorgStreamUrl,
|
|
2182
|
+
onReorg: async input => { await basmEngine?.handleReorg(input) },
|
|
2183
|
+
onConnect: async () => { await basmEngine?.revalidateRecentAnchors(reorgScanDepth) },
|
|
2184
|
+
logger: this.logger
|
|
1847
2185
|
})
|
|
2186
|
+
this.reorgAdapter.start()
|
|
2187
|
+
this.logger.log(chalk.green(`BASM reorg stream listening at ${reorgStreamUrl}`))
|
|
2188
|
+
}
|
|
2189
|
+
|
|
2190
|
+
/** Extend every topic's BASM anchor chain to the current chain tip. */
|
|
2191
|
+
private async advanceBASMAnchorChains (): Promise<void> {
|
|
2192
|
+
try {
|
|
2193
|
+
await (this.engine as BASMCapableEngine | undefined)?.advanceTopicAnchorChains()
|
|
2194
|
+
} catch (e) {
|
|
2195
|
+
console.error(chalk.red('Failed to advance BASM anchor chains'), e)
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
|
|
2199
|
+
/** Revalidate recent BASM anchors against the chain tracker, reconciling any reorg. */
|
|
2200
|
+
private async revalidateBASMAnchors (): Promise<void> {
|
|
2201
|
+
try {
|
|
2202
|
+
const depth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth
|
|
2203
|
+
await (this.engine as BASMCapableEngine | undefined)?.revalidateRecentAnchors(depth)
|
|
2204
|
+
} catch (e) {
|
|
2205
|
+
console.error(chalk.red('Failed to revalidate BASM anchors'), e)
|
|
2206
|
+
}
|
|
1848
2207
|
}
|
|
1849
2208
|
}
|