@bsv/overlay-express 2.2.1 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +33 -0
  2. package/dist/cjs/mod.js +10 -1
  3. package/dist/cjs/mod.js.map +1 -1
  4. package/dist/cjs/package.json +12 -11
  5. package/dist/cjs/src/BanAwareDiscoveryStorage.js +78 -0
  6. package/dist/cjs/src/BanAwareDiscoveryStorage.js.map +1 -0
  7. package/dist/cjs/src/BanAwareTopicManager.js +71 -0
  8. package/dist/cjs/src/BanAwareTopicManager.js.map +1 -0
  9. package/dist/cjs/src/OverlayExpress.js +375 -96
  10. package/dist/cjs/src/OverlayExpress.js.map +1 -1
  11. package/dist/cjs/src/OverlayMonitor.js +464 -0
  12. package/dist/cjs/src/OverlayMonitor.js.map +1 -0
  13. package/dist/cjs/src/ReorgStream.js +193 -0
  14. package/dist/cjs/src/ReorgStream.js.map +1 -0
  15. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  16. package/dist/esm/mod.js +3 -0
  17. package/dist/esm/mod.js.map +1 -1
  18. package/dist/esm/src/BanAwareDiscoveryStorage.js +79 -0
  19. package/dist/esm/src/BanAwareDiscoveryStorage.js.map +1 -0
  20. package/dist/esm/src/BanAwareTopicManager.js +71 -0
  21. package/dist/esm/src/BanAwareTopicManager.js.map +1 -0
  22. package/dist/esm/src/OverlayExpress.js +375 -94
  23. package/dist/esm/src/OverlayExpress.js.map +1 -1
  24. package/dist/esm/src/OverlayMonitor.js +458 -0
  25. package/dist/esm/src/OverlayMonitor.js.map +1 -0
  26. package/dist/esm/src/ReorgStream.js +190 -0
  27. package/dist/esm/src/ReorgStream.js.map +1 -0
  28. package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
  29. package/dist/types/mod.d.ts +4 -1
  30. package/dist/types/mod.d.ts.map +1 -1
  31. package/dist/types/src/BanAwareDiscoveryStorage.d.ts +36 -0
  32. package/dist/types/src/BanAwareDiscoveryStorage.d.ts.map +1 -0
  33. package/dist/types/src/BanAwareTopicManager.d.ts +28 -0
  34. package/dist/types/src/BanAwareTopicManager.d.ts.map +1 -0
  35. package/dist/types/src/OverlayExpress.d.ts +82 -4
  36. package/dist/types/src/OverlayExpress.d.ts.map +1 -1
  37. package/dist/types/src/OverlayMonitor.d.ts +169 -0
  38. package/dist/types/src/OverlayMonitor.d.ts.map +1 -0
  39. package/dist/types/src/ReorgStream.d.ts +74 -0
  40. package/dist/types/src/ReorgStream.d.ts.map +1 -0
  41. package/dist/types/tsconfig.types.tsbuildinfo +1 -1
  42. package/mod.ts +20 -1
  43. package/package.json +12 -11
  44. package/src/BanAwareDiscoveryStorage.ts +91 -0
  45. package/src/BanAwareTopicManager.ts +87 -0
  46. package/src/OverlayExpress.ts +461 -102
  47. package/src/OverlayMonitor.ts +657 -0
  48. package/src/ReorgStream.ts +221 -0
  49. package/src/__tests__/BanAwareDiscoveryStorage.test.ts +131 -0
  50. package/src/__tests__/BanAwareTopicManager.test.ts +132 -0
  51. package/src/__tests__/OverlayExpress.test.ts +86 -1
  52. package/src/__tests__/OverlayMonitor.test.ts +181 -0
  53. package/src/__tests__/ReorgStream.test.ts +66 -0
@@ -1,6 +1,13 @@
1
1
  import express from 'express'
2
2
  import bodyParser from 'body-parser'
3
- import { Engine, KnexStorage, LookupService, TopicManager, KnexStorageMigrations, Advertiser } from '@bsv/overlay'
3
+ import {
4
+ Engine,
5
+ KnexStorage,
6
+ LookupService,
7
+ TopicManager,
8
+ KnexStorageMigrations,
9
+ Advertiser
10
+ } from '@bsv/overlay'
4
11
  import {
5
12
  ARC,
6
13
  ChainTracker,
@@ -30,6 +37,9 @@ import { v4 as uuidv4 } from 'uuid'
30
37
  import { JanitorService, type JanitorReport } from './JanitorService.js'
31
38
  import { BanService } from './BanService.js'
32
39
  import { BanAwareLookupWrapper } from './BanAwareLookupWrapper.js'
40
+ import { BanAwareTopicManager } from './BanAwareTopicManager.js'
41
+ import { BanAwareSHIPStorage, BanAwareSLAPStorage } from './BanAwareDiscoveryStorage.js'
42
+ import { ReorgSseAdapter, type ReorgHandlerInput } from './ReorgStream.js'
33
43
  import { Wallet, WalletSigner, WalletStorageManager, Services } from '@bsv/wallet-toolbox-client'
34
44
  import { createAuthMiddleware, type AuthRequest } from '@bsv/auth-express-middleware'
35
45
 
@@ -92,6 +102,11 @@ export interface EngineConfig {
92
102
  throwOnBroadcastFailure?: boolean
93
103
  overlayBroadcastFacilitator?: OverlayBroadcastFacilitator
94
104
  suppressDefaultSyncAdvertisements?: boolean
105
+ topicAnchorHeaderResolver?: TopicAnchorHeaderResolver
106
+ enableBASMSync?: boolean
107
+ unprovenEvictionBlocks?: number
108
+ reorgStreamUrl?: string
109
+ reorgScanDepth?: number
95
110
  }
96
111
 
97
112
  export type HealthStatus = 'ok' | 'degraded' | 'error'
@@ -139,6 +154,25 @@ export interface HealthReport {
139
154
  context?: Record<string, any>
140
155
  }
141
156
 
157
+ export type TopicAnchorHeaderResolver = (blockHeight: number) => Promise<{
158
+ blockHeight: number
159
+ blockHash: string
160
+ merkleRoot?: string
161
+ } | undefined>
162
+
163
+ interface BASMCapableEngine extends Engine {
164
+ provideTopicAnchorTip: (topic: string) => Promise<any>
165
+ provideTopicAnchorRange: (topic: string, fromHeight: number, toHeight: number) => Promise<any>
166
+ provideAdmittedList: (topic: string, blockHeight: number, blockHash?: string) => Promise<any>
167
+ provideCompoundMerklePath: (topic: string, blockHeight: number, txids: string[]) => Promise<any>
168
+ provideRawTransactions: (txids: string[]) => Promise<any>
169
+ startBASMSync: () => Promise<any>
170
+ advanceTopicAnchorChains: (toHeight?: number) => Promise<void>
171
+ evictUnprovenTransactions: (options?: { topic?: string, thresholdBlocks?: number }) => Promise<any>
172
+ handleReorg: (input: ReorgHandlerInput) => Promise<any>
173
+ revalidateRecentAnchors: (depth?: number) => Promise<any>
174
+ }
175
+
142
176
  /**
143
177
  * OverlayExpress class provides an Express-based server for hosting Overlay Services.
144
178
  * It allows configuration of various components like databases, topic managers, and lookup services.
@@ -186,9 +220,41 @@ export default class OverlayExpress {
186
220
  // (We allow an on/off toggle, but also can do advanced custom sync config below)
187
221
  enableGASPSync: boolean = true
188
222
 
223
+ // Enable BRC-136 BASM sync. Off by default; endpoints remain available when
224
+ // storage supports anchors.
225
+ enableBASMSync: boolean = false
226
+
227
+ // Opt-in unproven eviction default threshold in blocks.
228
+ unprovenEvictionBlocks: number = 144
229
+
230
+ // How often (ms) to poll the chain tip and extend each topic's BASM anchor
231
+ // chain with empty anchors, so the cumulative TAC advances "after each new
232
+ // block" per BRC-136. Set to 0 to disable polling (startup extension still runs).
233
+ basmBlockPollIntervalMs: number = 10 * 60 * 1000
234
+
235
+ // Handle for the BASM block-poll timer so it can be stopped.
236
+ private basmBlockPollTimer?: ReturnType<typeof setInterval>
237
+
238
+ // Optional go-chaintracks (Arcade) reorg SSE URL (e.g. `<base>/v2/reorg/stream`).
239
+ // When set, reorgs are reconciled in real time; the block poll also runs a
240
+ // revalidation sweep as a fallback / reconnect catch-up.
241
+ reorgStreamUrl?: string
242
+
243
+ // Depth (in blocks from the tip) for the reorg revalidation sweep.
244
+ reorgScanDepth: number = 3
245
+
246
+ // Handle for the reorg SSE adapter.
247
+ private reorgAdapter?: ReorgSseAdapter
248
+
249
+ // Optional resolver for block hashes and header merkle roots used by BASM.
250
+ topicAnchorHeaderResolver?: TopicAnchorHeaderResolver
251
+
189
252
  // ARC API Key
190
253
  arcApiKey: string | undefined = undefined
191
254
 
255
+ // Optional ARC callback token for /arc-ingest notifications
256
+ arcCallbackToken: string | undefined = undefined
257
+
192
258
  // Verbose request logging
193
259
  verboseRequestLogging: boolean = false
194
260
 
@@ -372,6 +438,15 @@ export default class OverlayExpress {
372
438
  this.logger.log(chalk.blue('ARC API key has been configured.'))
373
439
  }
374
440
 
441
+ /**
442
+ * Configures the ARC callback token expected by /arc-ingest.
443
+ * @param token - The token ARC should present when posting callback notifications.
444
+ */
445
+ configureArcCallbackToken (token: string): void {
446
+ this.arcCallbackToken = token
447
+ this.logger.log(chalk.blue('ARC callback token has been configured.'))
448
+ }
449
+
375
450
  /**
376
451
  * Enables or disables GASP synchronization (high-level setting).
377
452
  * This is a broad toggle that can be overridden or customized through syncConfiguration.
@@ -382,6 +457,56 @@ export default class OverlayExpress {
382
457
  this.logger.log(chalk.blue(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`))
383
458
  }
384
459
 
460
+ /**
461
+ * Enables or disables BRC-136 BASM synchronization.
462
+ * BASM is opt-in because it requires direct proofs and block hash resolution.
463
+ */
464
+ configureEnableBASMSync (enable: boolean): void {
465
+ this.enableBASMSync = enable
466
+ this.logger.log(chalk.blue(`BASM synchronization ${enable ? 'enabled' : 'disabled'}.`))
467
+ }
468
+
469
+ /**
470
+ * Configures the block header resolver used to derive BASM block hashes.
471
+ */
472
+ configureTopicAnchorHeaderResolver (resolver: TopicAnchorHeaderResolver): void {
473
+ this.topicAnchorHeaderResolver = resolver
474
+ this.logger.log(chalk.blue('BASM topic anchor header resolver has been configured.'))
475
+ }
476
+
477
+ /**
478
+ * Configures the go-chaintracks (Arcade) reorg SSE stream used to reconcile
479
+ * BASM anchors with blockchain reorganizations in real time.
480
+ * @param url - The reorg stream URL, e.g. `https://arcade.example/v2/reorg/stream`.
481
+ * @param scanDepth - Optional revalidation-sweep depth in blocks (default 3).
482
+ */
483
+ configureReorgStream (url: string, scanDepth?: number): void {
484
+ this.reorgStreamUrl = url
485
+ if (scanDepth !== undefined) {
486
+ this.reorgScanDepth = scanDepth
487
+ }
488
+ this.logger.log(chalk.blue('BASM reorg stream has been configured.'))
489
+ }
490
+
491
+ /**
492
+ * Configures the opt-in unproven state eviction threshold.
493
+ */
494
+ configureUnprovenEviction (config: { thresholdBlocks?: number }): void {
495
+ if (config.thresholdBlocks !== undefined) {
496
+ this.unprovenEvictionBlocks = config.thresholdBlocks
497
+ }
498
+ this.logger.log(chalk.blue('Unproven transaction eviction has been configured.'))
499
+ }
500
+
501
+ /**
502
+ * Configures how often the BASM anchor chain is extended with empty anchors to
503
+ * follow the chain tip. Set to 0 to disable periodic polling.
504
+ */
505
+ configureBASMBlockPollInterval (intervalMs: number): void {
506
+ this.basmBlockPollIntervalMs = intervalMs
507
+ this.logger.log(chalk.blue(`BASM block poll interval set to ${intervalMs}ms.`))
508
+ }
509
+
385
510
  /**
386
511
  * Enables or disables verbose request logging.
387
512
  * @param enable - true to enable, false to disable
@@ -499,9 +624,9 @@ export default class OverlayExpress {
499
624
  * By default, auto-configures SHIP and SLAP unless autoConfigureShipSlap = false
500
625
  * Then it merges in any advanced engine config from `this.engineConfig`.
501
626
  *
502
- * When a BanService is available (from configureMongo), SHIP and SLAP lookup
503
- * services are automatically wrapped with BanAwareLookupWrapper to prevent
504
- * GASP from re-syncing banned tokens.
627
+ * When a BanService is available (from configureMongo), auto-configured SHIP
628
+ * and SLAP managers, discovery storage, and lookup services are wrapped so
629
+ * banned outputs are not admitted or indexed.
505
630
  *
506
631
  * @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true)
507
632
  */
@@ -511,12 +636,20 @@ export default class OverlayExpress {
511
636
  if (autoConfigureShipSlap) {
512
637
  this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager())
513
638
  this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager())
514
- this.configureLookupServiceWithMongo('ls_ship', (db) => new DiscoveryServices.SHIPLookupService(
515
- new DiscoveryServices.SHIPStorage(db)
516
- ))
517
- this.configureLookupServiceWithMongo('ls_slap', (db) => new DiscoveryServices.SLAPLookupService(
518
- new DiscoveryServices.SLAPStorage(db)
519
- ))
639
+ this.configureLookupServiceWithMongo('ls_ship', (db) => {
640
+ const storage = new DiscoveryServices.SHIPStorage(db)
641
+ const storageForLookup = this.banService === undefined
642
+ ? storage
643
+ : new BanAwareSHIPStorage(storage, this.banService, this.logger)
644
+ return new DiscoveryServices.SHIPLookupService(storageForLookup as any)
645
+ })
646
+ this.configureLookupServiceWithMongo('ls_slap', (db) => {
647
+ const storage = new DiscoveryServices.SLAPStorage(db)
648
+ const storageForLookup = this.banService === undefined
649
+ ? storage
650
+ : new BanAwareSLAPStorage(storage, this.banService, this.logger)
651
+ return new DiscoveryServices.SLAPLookupService(storageForLookup as any)
652
+ })
520
653
  }
521
654
 
522
655
  this.wrapBanAwareServices()
@@ -528,7 +661,8 @@ export default class OverlayExpress {
528
661
  const broadcaster = this.buildBroadcaster()
529
662
  const advertiser = await this.buildAdvertiser()
530
663
 
531
- this.engine = new Engine(
664
+ const EngineWithBASM = Engine as unknown as new (...args: any[]) => Engine
665
+ this.engine = new EngineWithBASM(
532
666
  this.managers,
533
667
  this.services,
534
668
  storage,
@@ -546,16 +680,26 @@ export default class OverlayExpress {
546
680
  this.engineConfig.throwOnBroadcastFailure ?? false,
547
681
  this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(),
548
682
  this.logger,
549
- this.engineConfig.suppressDefaultSyncAdvertisements ?? true
683
+ this.engineConfig.suppressDefaultSyncAdvertisements ?? true,
684
+ this.buildTopicAnchorHeaderResolver(),
685
+ this.engineConfig.enableBASMSync ?? this.enableBASMSync,
686
+ this.engineConfig.unprovenEvictionBlocks ?? this.unprovenEvictionBlocks
550
687
  )
551
688
 
552
689
  this.initServerWallet()
553
690
  this.logger.log(chalk.green('Engine has been configured.'))
554
691
  }
555
692
 
556
- /** Wrap SHIP/SLAP with ban-aware filter if BanService is configured. */
693
+ /** Wrap SHIP/SLAP managers and services with ban-aware filters if BanService is configured. */
557
694
  private wrapBanAwareServices (): void {
558
695
  if (this.banService === undefined) return
696
+ for (const key of ['tm_ship', 'tm_slap'] as const) {
697
+ if (this.managers[key] !== undefined) {
698
+ const label = key === 'tm_ship' ? 'SHIP' : 'SLAP'
699
+ this.managers[key] = new BanAwareTopicManager(this.managers[key], this.banService, label, this.logger)
700
+ this.logger.log(chalk.blue(`${label} topic manager wrapped with ban-aware filter.`))
701
+ }
702
+ }
559
703
  for (const key of ['ls_ship', 'ls_slap'] as const) {
560
704
  if (this.services[key] !== undefined) {
561
705
  const label = key === 'ls_ship' ? 'SHIP' : 'SLAP'
@@ -581,7 +725,38 @@ export default class OverlayExpress {
581
725
  private buildBroadcaster (): Broadcaster | undefined {
582
726
  if (typeof this.arcApiKey !== 'string') return undefined
583
727
  const arcUrl = this.network === 'test' ? 'https://arc-test.taal.com' : 'https://arc.taal.com'
584
- return new ARC(arcUrl, { apiKey: this.arcApiKey })
728
+ return new ARC(arcUrl, {
729
+ apiKey: this.arcApiKey,
730
+ callbackUrl: `https://${this.advertisableFQDN}/arc-ingest`,
731
+ callbackToken: this.arcCallbackToken
732
+ })
733
+ }
734
+
735
+ /** Build the BASM block header resolver. */
736
+ private buildTopicAnchorHeaderResolver (): TopicAnchorHeaderResolver | undefined {
737
+ const configured = this.engineConfig.topicAnchorHeaderResolver ?? this.topicAnchorHeaderResolver
738
+ if (configured !== undefined) {
739
+ return configured
740
+ }
741
+
742
+ return async (blockHeight: number) => {
743
+ const response = await fetch(`https://api.whatsonchain.com/v1/bsv/${this.network}/block/${blockHeight}/header`, {
744
+ method: 'GET',
745
+ headers: { Accept: 'application/json' }
746
+ })
747
+ if (!response.ok) {
748
+ throw new Error(`WhatsOnChain header lookup failed for height ${blockHeight}: ${response.status}`)
749
+ }
750
+ const header = await response.json() as { hash?: string, merkleroot?: string }
751
+ if (typeof header.hash !== 'string') {
752
+ throw new TypeError(`WhatsOnChain did not return a block hash for height ${blockHeight}`)
753
+ }
754
+ return {
755
+ blockHeight,
756
+ blockHash: header.hash,
757
+ merkleRoot: header.merkleroot
758
+ }
759
+ }
585
760
  }
586
761
 
587
762
  /** Resolve the SLAP trackers from config or network defaults. */
@@ -901,6 +1076,71 @@ export default class OverlayExpress {
901
1076
  return report
902
1077
  }
903
1078
 
1079
+ /**
1080
+ * Renders a request or response body for verbose logging, truncating overly long payloads.
1081
+ */
1082
+ private formatBodyForLog (body: any, tooLongLabel: string, okPrefix: string): string {
1083
+ let bodyString: string
1084
+ if (typeof body === 'object') {
1085
+ bodyString = JSON.stringify(body, null, 2)
1086
+ } else if (Buffer.isBuffer(body)) {
1087
+ bodyString = body.toString('utf8')
1088
+ } else {
1089
+ bodyString = String(body)
1090
+ }
1091
+ return bodyString.length > 280
1092
+ ? chalk.yellow(`(${tooLongLabel}, length: ${String(bodyString.length)} characters)`)
1093
+ : chalk.green(`${okPrefix}\n${bodyString}`)
1094
+ }
1095
+
1096
+ /**
1097
+ * Installs middleware that verbosely logs incoming requests and outgoing responses.
1098
+ */
1099
+ private setupVerboseRequestLogging (): void {
1100
+ this.app.use((req, res, next) => {
1101
+ const startTime = Date.now()
1102
+
1103
+ // Log incoming request details
1104
+ this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`))
1105
+ // Pretty-print headers
1106
+ this.logger.log(chalk.cyan('Headers:'))
1107
+ this.logger.log(util.inspect(req.headers, { colors: true, depth: null }))
1108
+
1109
+ // Handle request body
1110
+ if (req.body != null && Object.keys(req.body).length > 0) {
1111
+ this.logger.log(this.formatBodyForLog(req.body, 'Body too long to display', 'Request Body:'))
1112
+ }
1113
+
1114
+ // Intercept the res.send method to log responses
1115
+ const originalSend = res.send
1116
+ let responseBody: any
1117
+
1118
+ res.send = function (body?: any): any {
1119
+ responseBody = body
1120
+ return originalSend.call(this, body)
1121
+ }
1122
+
1123
+ // Log outgoing response details after the response is finished
1124
+ res.on('finish', () => {
1125
+ const duration = Date.now() - startTime
1126
+ this.logger.log(
1127
+ chalk.magenta.bold(
1128
+ `Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`
1129
+ )
1130
+ )
1131
+ this.logger.log(chalk.cyan('Response Headers:'))
1132
+ this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }))
1133
+
1134
+ // Handle response body
1135
+ if (responseBody != null) {
1136
+ this.logger.log(this.formatBodyForLog(responseBody, 'Response body too long to display', 'Response Body:'))
1137
+ }
1138
+ })
1139
+
1140
+ next()
1141
+ })
1142
+ }
1143
+
904
1144
  /**
905
1145
  * Starts the Express server.
906
1146
  * Sets up routes and begins listening on the configured port.
@@ -914,78 +1154,7 @@ export default class OverlayExpress {
914
1154
  this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }))
915
1155
 
916
1156
  if (this.verboseRequestLogging) {
917
- this.app.use((req, res, next) => {
918
- const startTime = Date.now()
919
-
920
- // Log incoming request details
921
- this.logger.log(chalk.magenta.bold(`Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`))
922
- // Pretty-print headers
923
- this.logger.log(chalk.cyan('Headers:'))
924
- this.logger.log(util.inspect(req.headers, { colors: true, depth: null }))
925
-
926
- // Handle request body
927
- if (req.body != null && Object.keys(req.body).length > 0) {
928
- let bodyContent
929
- let bodyString
930
- if (typeof req.body === 'object') {
931
- bodyString = JSON.stringify(req.body, null, 2)
932
- } else if (Buffer.isBuffer(req.body)) {
933
- bodyString = req.body.toString('utf8')
934
- } else {
935
- bodyString = String(req.body)
936
- }
937
- if (bodyString.length > 280) {
938
- bodyContent = chalk.yellow(`(Body too long to display, length: ${String(bodyString.length)} characters)`)
939
- } else {
940
- bodyContent = chalk.green(`Request Body:\n${String(bodyString)}`)
941
- }
942
- this.logger.log(bodyContent)
943
- }
944
-
945
- // Intercept the res.send method to log responses
946
- const originalSend = res.send
947
- let responseBody: any
948
-
949
- res.send = function (body?: any): any {
950
- responseBody = body
951
- return originalSend.call(this, body)
952
- }
953
-
954
- // Log outgoing response details after the response is finished
955
- res.on('finish', () => {
956
- const duration = Date.now() - startTime
957
- this.logger.log(
958
- chalk.magenta.bold(
959
- `Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`
960
- )
961
- )
962
- this.logger.log(chalk.cyan('Response Headers:'))
963
- this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }))
964
-
965
- // Handle response body
966
- if (responseBody != null) {
967
- let bodyContent
968
- let bodyString
969
- if (typeof responseBody === 'object') {
970
- bodyString = JSON.stringify(responseBody, null, 2)
971
- } else if (Buffer.isBuffer(responseBody)) {
972
- bodyString = responseBody.toString('utf8')
973
- } else if (typeof responseBody === 'string') {
974
- bodyString = responseBody
975
- } else {
976
- bodyString = String(responseBody)
977
- }
978
- if (bodyString.length > 280) {
979
- bodyContent = chalk.yellow(`(Response body too long to display, length: ${String(bodyString.length)} characters)`)
980
- } else {
981
- bodyContent = chalk.green(`Response Body:\n${String(bodyString)}`)
982
- }
983
- this.logger.log(bodyContent)
984
- }
985
- })
986
-
987
- next()
988
- })
1157
+ this.setupVerboseRequestLogging()
989
1158
  }
990
1159
 
991
1160
  // Enable CORS
@@ -1253,6 +1422,18 @@ export default class OverlayExpress {
1253
1422
  this.app.post('/arc-ingest', (req, res) => {
1254
1423
  ; (async () => {
1255
1424
  try {
1425
+ if (typeof this.arcCallbackToken === 'string' && this.arcCallbackToken.length > 0) {
1426
+ const authorization = req.headers.authorization
1427
+ const headerToken = Array.isArray(authorization) ? authorization[0] : authorization
1428
+ const xCallbackToken = req.headers['x-callback-token']
1429
+ const callbackToken = Array.isArray(xCallbackToken) ? xCallbackToken[0] : xCallbackToken
1430
+ const bearerToken = typeof headerToken === 'string' && headerToken.startsWith('Bearer ')
1431
+ ? headerToken.slice('Bearer '.length)
1432
+ : headerToken
1433
+ if (bearerToken !== this.arcCallbackToken && callbackToken !== this.arcCallbackToken) {
1434
+ return res.status(401).json({ status: 'error', message: 'Unauthorized callback' })
1435
+ }
1436
+ }
1256
1437
  const { txid, merklePath: merklePathHex, blockHeight } = req.body
1257
1438
  const merklePath = MerklePath.fromHex(merklePathHex)
1258
1439
  await engine.handleNewMerkleProof(txid, merklePath, blockHeight)
@@ -1322,6 +1503,78 @@ export default class OverlayExpress {
1322
1503
  this.logger.warn(chalk.yellow('GASP sync is disabled.'))
1323
1504
  }
1324
1505
 
1506
+ // BRC-136 BASM anchor and raw transaction endpoints.
1507
+ const basmEngine = engine as BASMCapableEngine
1508
+ const readBasmTopic = (req: express.Request): string => {
1509
+ const header = req.headers['x-bsv-topic']
1510
+ if (typeof header !== 'string' || header.length === 0) {
1511
+ throw new TypeError('Missing x-bsv-topic header')
1512
+ }
1513
+ return header
1514
+ }
1515
+
1516
+ /**
1517
+ * Registers a POST route whose handler resolves to the JSON payload returned
1518
+ * with HTTP 200. Any thrown error is logged and returned as HTTP 400, while
1519
+ * unexpected rejections fall back to HTTP 500. This consolidates the shared
1520
+ * async/try-catch boilerplate used by the BRC-136 BASM endpoints (and the
1521
+ * BASM-related admin endpoints, which additionally pass `checkAdminAuth`).
1522
+ */
1523
+ const registerJsonRoute = (
1524
+ path: string,
1525
+ handler: (req: express.Request) => Promise<unknown>,
1526
+ ...middleware: express.RequestHandler[]
1527
+ ): void => {
1528
+ this.app.post(path, ...(middleware as any[]), (req: express.Request, res: express.Response) => {
1529
+ ; (async () => {
1530
+ try {
1531
+ return res.status(200).json(await handler(req))
1532
+ } catch (error) {
1533
+ console.error(chalk.red(`Error in ${path}:`), error)
1534
+ return res.status(400).json({
1535
+ status: 'error',
1536
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
1537
+ })
1538
+ }
1539
+ })().catch(() => {
1540
+ res.status(500).json({ status: 'error', message: 'Unexpected error' })
1541
+ })
1542
+ })
1543
+ }
1544
+
1545
+ const requireTxids = (value: unknown): string[] => {
1546
+ if (!Array.isArray(value) || !value.every(txid => typeof txid === 'string')) {
1547
+ throw new TypeError('txids must be an array of strings')
1548
+ }
1549
+ return value
1550
+ }
1551
+
1552
+ registerJsonRoute('/requestTopicAnchorTip', async req =>
1553
+ await basmEngine.provideTopicAnchorTip(readBasmTopic(req)))
1554
+
1555
+ registerJsonRoute('/requestTopicAnchorRange', async req => {
1556
+ const { fromHeight, toHeight } = req.body
1557
+ return await basmEngine.provideTopicAnchorRange(readBasmTopic(req), Number(fromHeight), Number(toHeight))
1558
+ })
1559
+
1560
+ registerJsonRoute('/requestAdmittedList', async req => {
1561
+ const { blockHeight, blockHash } = req.body
1562
+ return await basmEngine.provideAdmittedList(
1563
+ readBasmTopic(req),
1564
+ Number(blockHeight),
1565
+ typeof blockHash === 'string' ? blockHash : undefined
1566
+ )
1567
+ })
1568
+
1569
+ registerJsonRoute('/requestCompoundMerklePath', async req => {
1570
+ const topic = readBasmTopic(req)
1571
+ const { blockHeight, txids } = req.body
1572
+ return await basmEngine.provideCompoundMerklePath(topic, Number(blockHeight), requireTxids(txids))
1573
+ })
1574
+
1575
+ registerJsonRoute('/requestRawTransactions', async req =>
1576
+ await basmEngine.provideRawTransactions(requireTxids(req.body.txids)))
1577
+
1325
1578
  /**
1326
1579
  * ============== ADMIN ROUTES ==============
1327
1580
  * These routes expose advanced engine operations.
@@ -1351,7 +1604,7 @@ export default class OverlayExpress {
1351
1604
  */
1352
1605
  const checkAdminAuth = (req: express.Request, res: express.Response, next: express.NextFunction): void => {
1353
1606
  // Method 1: BSV mutual authentication (identity key match)
1354
- const authReq = req as AuthRequest
1607
+ const authReq = req as unknown as AuthRequest
1355
1608
  if (
1356
1609
  typeof this.adminIdentityKey === 'string' &&
1357
1610
  authReq.auth !== undefined &&
@@ -1419,7 +1672,9 @@ export default class OverlayExpress {
1419
1672
  totalBans: banStats.totalBans,
1420
1673
  topicManagers: Object.keys(this.managers),
1421
1674
  lookupServices: Object.keys(this.services),
1422
- gaspSyncEnabled: this.enableGASPSync
1675
+ gaspSyncEnabled: this.enableGASPSync,
1676
+ basmSyncEnabled: this.enableBASMSync,
1677
+ unprovenEvictionBlocks: this.unprovenEvictionBlocks
1423
1678
  }
1424
1679
  })
1425
1680
  } catch (error) {
@@ -1728,6 +1983,26 @@ export default class OverlayExpress {
1728
1983
  })
1729
1984
  })
1730
1985
 
1986
+ /**
1987
+ * Admin route to manually start BASM sync, calling `engine.startBASMSync()`.
1988
+ */
1989
+ registerJsonRoute('/admin/startBASMSync', async () => {
1990
+ const report = await basmEngine.startBASMSync()
1991
+ return { status: 'success', message: 'BASM sync started and completed', data: report }
1992
+ }, checkAdminAuth as any)
1993
+
1994
+ /**
1995
+ * Admin route to evict expired unproven topic transactions.
1996
+ */
1997
+ registerJsonRoute('/admin/evictUnproven', async req => {
1998
+ const { topic, thresholdBlocks } = req.body ?? {}
1999
+ const report = await basmEngine.evictUnprovenTransactions({
2000
+ topic: typeof topic === 'string' ? topic : undefined,
2001
+ thresholdBlocks: typeof thresholdBlocks === 'number' ? thresholdBlocks : undefined
2002
+ })
2003
+ return { status: 'success', message: 'Unproven eviction completed', data: report }
2004
+ }, checkAdminAuth as any)
2005
+
1731
2006
  /**
1732
2007
  * Admin route to evict an outpoint, either from all services or a specific one.
1733
2008
  */
@@ -1804,6 +2079,20 @@ export default class OverlayExpress {
1804
2079
  })
1805
2080
  })
1806
2081
 
2082
+ await this.runStartupSync()
2083
+
2084
+ // Start listening on the configured port
2085
+ this.app.listen(this.port, () => {
2086
+ this.isListening = true
2087
+ this.logger.log(chalk.green.bold(`${this.name} is ready and listening on local port ${this.port}`))
2088
+ })
2089
+ }
2090
+
2091
+ /**
2092
+ * Runs the post-listen startup work: advertiser init, advertisement sync,
2093
+ * and the optional GASP/BASM background syncs.
2094
+ */
2095
+ private async runStartupSync (): Promise<void> {
1807
2096
  // The legacy Ninja advertiser has a setLookupEngine method.
1808
2097
  if (this.engine?.advertiser instanceof DiscoveryServices.WalletAdvertiser) {
1809
2098
  this.logger.log(
@@ -1827,23 +2116,93 @@ export default class OverlayExpress {
1827
2116
  this.logger.log(chalk.red('Error syncing advertisements:'), e)
1828
2117
  }
1829
2118
 
1830
- // Attempt to do GASP sync if enabled
1831
- if (this.enableGASPSync) {
1832
- try {
1833
- this.logger.log(chalk.green('Starting GASP sync...'))
1834
- await this.engine?.startGASPSync()
1835
- this.logger.log(chalk.green('GASP sync complete!'))
1836
- } catch (e) {
1837
- console.error(chalk.red('Failed to GASP sync'), e)
1838
- }
1839
- } else {
2119
+ await this.runGaspStartupSync()
2120
+ await this.runBasmStartupSync()
2121
+ }
2122
+
2123
+ /** Attempt a GASP sync at startup when enabled. */
2124
+ private async runGaspStartupSync (): Promise<void> {
2125
+ if (!this.enableGASPSync) {
1840
2126
  this.logger.log(chalk.yellow(`${this.name} will not sync because GASP has been disabled.`))
2127
+ return
2128
+ }
2129
+ try {
2130
+ this.logger.log(chalk.green('Starting GASP sync...'))
2131
+ await this.engine?.startGASPSync()
2132
+ this.logger.log(chalk.green('GASP sync complete!'))
2133
+ } catch (e) {
2134
+ console.error(chalk.red('Failed to GASP sync'), e)
1841
2135
  }
2136
+ }
1842
2137
 
1843
- // Start listening on the configured port
1844
- this.app.listen(this.port, () => {
1845
- this.isListening = true
1846
- this.logger.log(chalk.green.bold(`${this.name} is ready and listening on local port ${this.port}`))
2138
+ /** Attempt a BASM sync at startup when enabled, then begin tip-following. */
2139
+ private async runBasmStartupSync (): Promise<void> {
2140
+ if (!(this.enableBASMSync || this.engineConfig.enableBASMSync === true)) {
2141
+ return
2142
+ }
2143
+ try {
2144
+ this.logger.log(chalk.green('Starting BASM sync...'))
2145
+ const report = await (this.engine as BASMCapableEngine | undefined)?.startBASMSync()
2146
+ this.logger.log(chalk.green('BASM sync complete!'), report)
2147
+ } catch (e) {
2148
+ console.error(chalk.red('Failed to BASM sync'), e)
2149
+ }
2150
+
2151
+ // Extend each topic's anchor chain to the current tip on startup, then keep
2152
+ // it following the tip so the cumulative TAC advances after each new block.
2153
+ await this.advanceBASMAnchorChains()
2154
+ this.startBASMBlockPolling()
2155
+ this.startBASMReorgStream()
2156
+ }
2157
+
2158
+ /** Poll for new blocks to advance anchor chains and detect reorgs. */
2159
+ private startBASMBlockPolling (): void {
2160
+ if (this.basmBlockPollIntervalMs <= 0) {
2161
+ return
2162
+ }
2163
+ this.basmBlockPollTimer = setInterval(() => {
2164
+ void this.advanceBASMAnchorChains()
2165
+ // Fallback reorg detection for chain trackers without a reorg stream,
2166
+ // and a safety net even when the SSE adapter is active.
2167
+ void this.revalidateBASMAnchors()
2168
+ }, this.basmBlockPollIntervalMs)
2169
+ this.basmBlockPollTimer.unref?.()
2170
+ }
2171
+
2172
+ /** Real-time reorg reconciliation via the go-chaintracks (Arcade) reorg SSE. */
2173
+ private startBASMReorgStream (): void {
2174
+ const reorgStreamUrl = this.engineConfig.reorgStreamUrl ?? this.reorgStreamUrl
2175
+ const reorgScanDepth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth
2176
+ if (reorgStreamUrl === undefined || reorgStreamUrl === '') {
2177
+ return
2178
+ }
2179
+ const basmEngine = this.engine as BASMCapableEngine | undefined
2180
+ this.reorgAdapter = new ReorgSseAdapter({
2181
+ url: reorgStreamUrl,
2182
+ onReorg: async input => { await basmEngine?.handleReorg(input) },
2183
+ onConnect: async () => { await basmEngine?.revalidateRecentAnchors(reorgScanDepth) },
2184
+ logger: this.logger
1847
2185
  })
2186
+ this.reorgAdapter.start()
2187
+ this.logger.log(chalk.green(`BASM reorg stream listening at ${reorgStreamUrl}`))
2188
+ }
2189
+
2190
+ /** Extend every topic's BASM anchor chain to the current chain tip. */
2191
+ private async advanceBASMAnchorChains (): Promise<void> {
2192
+ try {
2193
+ await (this.engine as BASMCapableEngine | undefined)?.advanceTopicAnchorChains()
2194
+ } catch (e) {
2195
+ console.error(chalk.red('Failed to advance BASM anchor chains'), e)
2196
+ }
2197
+ }
2198
+
2199
+ /** Revalidate recent BASM anchors against the chain tracker, reconciling any reorg. */
2200
+ private async revalidateBASMAnchors (): Promise<void> {
2201
+ try {
2202
+ const depth = this.engineConfig.reorgScanDepth ?? this.reorgScanDepth
2203
+ await (this.engine as BASMCapableEngine | undefined)?.revalidateRecentAnchors(depth)
2204
+ } catch (e) {
2205
+ console.error(chalk.red('Failed to revalidate BASM anchors'), e)
2206
+ }
1848
2207
  }
1849
2208
  }