@bsv/overlay-express 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,586 @@
1
+ import express from 'express'
2
+ import bodyParser from 'body-parser'
3
+ import { Engine, KnexStorage, LookupService, TopicManager, KnexStorageMigrations } from '@bsv/overlay'
4
+ import { ARC, ChainTracker, MerklePath, STEAK, TaggedBEEF, WhatsOnChain } from '@bsv/sdk'
5
+ import Knex from 'knex'
6
+ import { MongoClient, Db } from 'mongodb'
7
+ import makeUserInterface from './makeUserInterface.js'
8
+ import * as DiscoveryServices from '@bsv/overlay-discovery-services'
9
+
10
+ /**
11
+ * Knex database migration.
12
+ */
13
+ type Migration = {
14
+ name?: string
15
+ up: (knex: Knex.Knex) => Promise<void>
16
+ down?: (knex: Knex.Knex) => Promise<void>
17
+ }
18
+
19
+ /**
20
+ * In-memory migration source for Knex migrations.
21
+ * Allows running migrations defined in code rather than files.
22
+ */
23
+ class InMemoryMigrationSource implements Knex.Knex.MigrationSource<Migration> {
24
+ constructor(private migrations: Migration[]) { }
25
+
26
+ /**
27
+ * Gets the list of migrations.
28
+ * @param loadExtensions - Array of file extensions to filter by (not used here)
29
+ * @returns Promise resolving to the array of migrations
30
+ */
31
+ async getMigrations(loadExtensions: readonly string[]): Promise<Migration[]> {
32
+ return this.migrations
33
+ }
34
+
35
+ /**
36
+ * Gets the name of a migration.
37
+ * @param migration - The migration object
38
+ * @returns The name of the migration
39
+ */
40
+ getMigrationName(migration: Migration): string {
41
+ return migration.name || `Migration at index ${this.migrations.indexOf(migration)}`
42
+ }
43
+
44
+ /**
45
+ * Gets the migration object.
46
+ * @param migration - The migration object
47
+ * @returns Promise resolving to the migration object
48
+ */
49
+ getMigration(migration: Migration): Promise<Knex.Knex.Migration> {
50
+ return Promise.resolve(migration)
51
+ }
52
+ }
53
+
54
+ /**
55
+ * OverlayExpress class provides an Express-based server for hosting Overlay Services.
56
+ * It allows configuration of various components like databases, topic managers, and lookup services.
57
+ * It encapsulates an Express application and provides methods to start the server.
58
+ */
59
+ export default class OverlayExpress {
60
+ // Express application
61
+ app: express.Application
62
+
63
+ // Server port
64
+ port: number = 3000
65
+
66
+ // Logger (defaults to console)
67
+ logger: typeof console = console
68
+
69
+ // Knex (SQL) database
70
+ knex: Knex.Knex | undefined = undefined
71
+
72
+ // Knex migrations to run
73
+ migrationsToRun: Array<Migration> = []
74
+
75
+ // MongoDB database
76
+ mongoDb: Db | undefined = undefined
77
+
78
+ // Network ('main' or 'test')
79
+ network: 'main' | 'test' = 'main'
80
+
81
+ // Chain tracker
82
+ chainTracker: ChainTracker = new WhatsOnChain(this.network)
83
+
84
+ // The Overlay Engine
85
+ engine: Engine | undefined = undefined
86
+
87
+ // Configured Topic Managers
88
+ managers: Record<string, TopicManager> = {}
89
+
90
+ // Configured Lookup Services
91
+ services: Record<string, LookupService> = {}
92
+
93
+ // Enable GASP Sync
94
+ enableGASPSync: boolean = true
95
+
96
+ // ARC API Key
97
+ arcApiKey: string | undefined = undefined
98
+
99
+ /**
100
+ * Constructs an instance of OverlayExpress.
101
+ * @param name - The name of the service
102
+ * @param privateKey - Private key used for signing advertisements
103
+ * @param hostingURL - The public URL where this service is hosted
104
+ */
105
+ constructor(
106
+ public name: string,
107
+ public privateKey: string,
108
+ public hostingURL: string
109
+ ) {
110
+ this.app = express()
111
+ this.logger.log(`${name} constructed`)
112
+ }
113
+
114
+ /**
115
+ * Configures the port on which the server will listen.
116
+ * @param port - The port number
117
+ */
118
+ configurePort(port: number) {
119
+ this.port = port
120
+ this.logger.log(`Server port set to ${port}`)
121
+ }
122
+
123
+ /**
124
+ * Configures the logger to be used by the server.
125
+ * @param logger - A logger object (e.g., console)
126
+ */
127
+ configureLogger(logger: typeof console) {
128
+ this.logger = logger
129
+ this.logger.log('Logger has been configured.')
130
+ }
131
+
132
+ /**
133
+ * Configures the BSV Blockchain network to be used ('main' or 'test').
134
+ * @param network - The network ('main' or 'test')
135
+ */
136
+ configureNetwork(network: 'main' | 'test') {
137
+ this.network = network
138
+ this.chainTracker = new WhatsOnChain(this.network)
139
+ this.logger.log(`Network set to ${network}`)
140
+ }
141
+
142
+ /**
143
+ * Configures the ChainTracker to be used.
144
+ * @param chainTracker - An instance of ChainTracker
145
+ */
146
+ configureChainTracker(chainTracker: ChainTracker = new WhatsOnChain(this.network)) {
147
+ this.chainTracker = chainTracker
148
+ this.logger.log('ChainTracker has been configured.')
149
+ }
150
+
151
+ /**
152
+ * Configures the ARC API key.
153
+ * @param apiKey - The ARC API key
154
+ */
155
+ configureArcApiKey(apiKey: string) {
156
+ this.arcApiKey = apiKey
157
+ this.logger.log('ARC API key has been configured.')
158
+ }
159
+
160
+ /**
161
+ * Enables or disables GASP synchronization.
162
+ * @param enable - true to enable, false to disable
163
+ */
164
+ configureEnableGASPSync(enable: boolean) {
165
+ this.enableGASPSync = enable
166
+ this.logger.log(`GASP synchronization ${enable ? 'enabled' : 'disabled'}.`)
167
+ }
168
+
169
+ /**
170
+ * Configures the Knex (SQL) database connection.
171
+ * @param config - Knex configuration object, or MySQL connection string (e.g. mysql://overlayAdmin:overlay123@mysql:3306/overlay).
172
+ */
173
+ async configureKnex(config: Knex.Knex.Config | string) {
174
+ if (typeof config === 'string') {
175
+ config = {
176
+ client: 'mysql2',
177
+ connection: config
178
+ }
179
+ }
180
+ this.knex = Knex(config)
181
+ this.logger.log('Knex successfully configured.')
182
+ }
183
+
184
+ /**
185
+ * Configures the MongoDB database connection.
186
+ * @param connectionString - MongoDB connection string
187
+ */
188
+ async configureMongo(connectionString: string) {
189
+ const mongoClient = new MongoClient(connectionString)
190
+ await mongoClient.connect()
191
+ const db = mongoClient.db(`${this.name}_lookup_services`)
192
+ this.mongoDb = db
193
+ this.logger.log('MongoDB successfully configured and connected.')
194
+ }
195
+
196
+ /**
197
+ * Configures a Topic Manager.
198
+ * @param name - The name of the Topic Manager
199
+ * @param manager - An instance of TopicManager
200
+ */
201
+ configureTopicManager(name: string, manager: TopicManager) {
202
+ this.managers[name] = manager
203
+ this.logger.log(`Configured topic manager ${name}`)
204
+ }
205
+
206
+ /**
207
+ * Configures a Lookup Service.
208
+ * @param name - The name of the Lookup Service
209
+ * @param service - An instance of LookupService
210
+ */
211
+ configureLookupService(name: string, service: LookupService) {
212
+ this.services[name] = service
213
+ this.logger.log(`Configured lookup service ${name}`)
214
+ }
215
+
216
+ /**
217
+ * Configures a Lookup Service using Knex (SQL) database.
218
+ * @param name - The name of the Lookup Service
219
+ * @param serviceFactory - A factory function that creates a LookupService instance using Knex
220
+ */
221
+ configureLookupServiceWithKnex(
222
+ name: string,
223
+ serviceFactory: (knex: Knex.Knex) => { service: LookupService; migrations: Array<Migration> }
224
+ ) {
225
+ this.ensureKnex()
226
+ const factoryResult = serviceFactory(this.knex as Knex.Knex)
227
+ this.services[name] = factoryResult.service
228
+ this.migrationsToRun.push(...factoryResult.migrations)
229
+ this.logger.log(`Configured lookup service ${name} with Knex`)
230
+ }
231
+
232
+ /**
233
+ * Configures a Lookup Service using MongoDB.
234
+ * @param name - The name of the Lookup Service
235
+ * @param serviceFactory - A factory function that creates a LookupService instance using MongoDB
236
+ */
237
+ configureLookupServiceWithMongo(name: string, serviceFactory: (mongoDb: Db) => LookupService) {
238
+ this.ensureMongo()
239
+ this.services[name] = serviceFactory(this.mongoDb as Db)
240
+ this.logger.log(`Configured lookup service ${name} with MongoDB`)
241
+ }
242
+
243
+ /**
244
+ * Configures the Overlay Engine.
245
+ * @param autoConfigureShipSlap - Whether to auto-configure SHIP and SLAP services (default: true)
246
+ */
247
+ async configureEngine(autoConfigureShipSlap = true) {
248
+ this.ensureKnex()
249
+
250
+ if (autoConfigureShipSlap) {
251
+ // Auto-configure SHIP and SLAP services
252
+ this.configureTopicManager('tm_ship', new DiscoveryServices.SHIPTopicManager())
253
+ this.configureTopicManager('tm_slap', new DiscoveryServices.SLAPTopicManager())
254
+ this.configureLookupServiceWithMongo('ls_ship', (db) => new DiscoveryServices.SHIPLookupService(
255
+ new DiscoveryServices.SHIPStorage(db)
256
+ ))
257
+ this.configureLookupServiceWithMongo('ls_slap', (db) => new DiscoveryServices.SLAPLookupService(
258
+ new DiscoveryServices.SLAPStorage(db)
259
+ ))
260
+ }
261
+
262
+ // Sync configuration
263
+ let syncConfig: Record<string, false> = {}
264
+ if (!this.enableGASPSync) {
265
+ for (const manager of Object.keys(this.managers)) {
266
+ syncConfig[manager] = false
267
+ }
268
+ }
269
+
270
+ const storage = new KnexStorage(this.knex as Knex.Knex)
271
+ // Include the KnexStorage migrations
272
+ this.migrationsToRun = [...KnexStorageMigrations.default, ...this.migrationsToRun]
273
+
274
+ this.engine = new Engine(
275
+ this.managers,
276
+ this.services,
277
+ storage,
278
+ this.chainTracker,
279
+ this.hostingURL,
280
+ undefined,
281
+ undefined,
282
+ this.arcApiKey ? new ARC('https://arc.taal.com', {
283
+ apiKey: this.arcApiKey
284
+ }) : undefined,
285
+ new DiscoveryServices.LegacyNinjaAdvertiser(this.privateKey, 'https://dojo.babbage.systems', this.hostingURL),
286
+ syncConfig
287
+ )
288
+ this.logger.log('Engine has been configured.')
289
+ }
290
+
291
+ /**
292
+ * Ensures that Knex is configured.
293
+ * @throws Error if Knex is not configured
294
+ */
295
+ private ensureKnex() {
296
+ if (typeof this.knex === 'undefined') {
297
+ throw new Error('You\'ll need to configure your SQL database with the .configureKnex() method first!')
298
+ }
299
+ }
300
+
301
+ /**
302
+ * Ensures that MongoDB is configured.
303
+ * @throws Error if MongoDB is not configured
304
+ */
305
+ private ensureMongo() {
306
+ if (typeof this.mongoDb === 'undefined') {
307
+ throw new Error('You\'ll need to configure your MongoDB connection with the .configureMongo() method first!')
308
+ }
309
+ }
310
+
311
+ /**
312
+ * Ensures that the Overlay Engine is configured.
313
+ * @throws Error if the Engine is not configured
314
+ */
315
+ private ensureEngine() {
316
+ if (typeof this.engine === 'undefined') {
317
+ throw new Error('You\'ll need to configure your Overlay Services engine with the .configureEngine() method first!')
318
+ }
319
+ }
320
+
321
+ /**
322
+ * Starts the Express server.
323
+ * Sets up routes and begins listening on the configured port.
324
+ */
325
+ async start() {
326
+ this.ensureEngine()
327
+ this.ensureKnex()
328
+ const engine = this.engine as Engine
329
+ const knex = this.knex as Knex.Knex
330
+
331
+ this.app.use(bodyParser.json({ limit: '1gb', type: 'application/json' }))
332
+ this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }))
333
+
334
+ // Enable CORS
335
+ this.app.use((req, res, next) => {
336
+ res.header('Access-Control-Allow-Origin', '*')
337
+ res.header('Access-Control-Allow-Headers', '*')
338
+ res.header('Access-Control-Allow-Methods', '*')
339
+ res.header('Access-Control-Expose-Headers', '*')
340
+ res.header('Access-Control-Allow-Private-Network', 'true')
341
+ if (req.method === 'OPTIONS') {
342
+ res.sendStatus(200)
343
+ } else {
344
+ next()
345
+ }
346
+ })
347
+
348
+ // Serve a static documentation site or user interface
349
+ this.app.get('/', (req, res) => {
350
+ res.set('content-type', 'text/html')
351
+ res.send(makeUserInterface())
352
+ })
353
+
354
+ // List hosted topic managers and lookup services
355
+ this.app.get('/listTopicManagers', (_, res) => {
356
+ (async () => {
357
+ try {
358
+ const result = await engine.listTopicManagers()
359
+ return res.status(200).json(result)
360
+ } catch (error) {
361
+ return res.status(400).json({
362
+ status: 'error',
363
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
364
+ })
365
+ }
366
+ })().catch(() => {
367
+ // This catch is for any unforeseen errors in the async IIFE itself
368
+ res.status(500).json({
369
+ status: 'error',
370
+ message: 'Unexpected error'
371
+ })
372
+ })
373
+ })
374
+
375
+ this.app.get('/listLookupServiceProviders', (_, res) => {
376
+ (async () => {
377
+ try {
378
+ const result = await engine.listLookupServiceProviders()
379
+ return res.status(200).json(result)
380
+ } catch (error) {
381
+ return res.status(400).json({
382
+ status: 'error',
383
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
384
+ })
385
+ }
386
+ })().catch(() => {
387
+ res.status(500).json({
388
+ status: 'error',
389
+ message: 'Unexpected error'
390
+ })
391
+ })
392
+ })
393
+
394
+ // Host documentation for the services
395
+ this.app.get('/getDocumentationForTopicManager', (req, res) => {
396
+ (async () => {
397
+ try {
398
+ const manager = req.query.manager as string
399
+ const result = await engine.getDocumentationForTopicManager(manager)
400
+ res.setHeader('Content-Type', 'text/markdown')
401
+ return res.status(200).send(result)
402
+ } catch (error) {
403
+ return res.status(400).json({
404
+ status: 'error',
405
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
406
+ })
407
+ }
408
+ })().catch(() => {
409
+ res.status(500).json({
410
+ status: 'error',
411
+ message: 'Unexpected error'
412
+ })
413
+ })
414
+ })
415
+
416
+ this.app.get('/getDocumentationForLookupServiceProvider', (req, res) => {
417
+ (async () => {
418
+ try {
419
+ const lookupService = req.query.lookupService as string
420
+ const result = await engine.getDocumentationForLookupServiceProvider(lookupService)
421
+ return res.status(200).json(result)
422
+ } catch (error) {
423
+ return res.status(400).json({
424
+ status: 'error',
425
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
426
+ })
427
+ }
428
+ })().catch(() => {
429
+ res.status(500).json({
430
+ status: 'error',
431
+ message: 'Unexpected error'
432
+ })
433
+ })
434
+ })
435
+
436
+ // Submit transactions and facilitate lookup requests
437
+ this.app.post('/submit', (req, res) => {
438
+ (async () => {
439
+ try {
440
+ // Parse out the topics and construct the tagged BEEF
441
+ const topicsHeader = req.headers['x-topics']
442
+ if (!topicsHeader) {
443
+ throw new Error('Missing x-topics header')
444
+ }
445
+ const topics = JSON.parse(topicsHeader as string)
446
+ const taggedBEEF: TaggedBEEF = {
447
+ beef: Array.from(req.body as number[]),
448
+ topics
449
+ }
450
+
451
+ // Using a callback function, we can just return once our steak is ready
452
+ // instead of having to wait for all the broadcasts to occur.
453
+ await engine.submit(taggedBEEF, (steak: STEAK) => {
454
+ return res.status(200).json(steak)
455
+ })
456
+ } catch (error) {
457
+ console.error(error)
458
+ return res.status(400).json({
459
+ status: 'error',
460
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
461
+ })
462
+ }
463
+ })().catch(() => {
464
+ res.status(500).json({
465
+ status: 'error',
466
+ message: 'Unexpected error'
467
+ })
468
+ })
469
+ })
470
+
471
+ this.app.post('/lookup', (req, res) => {
472
+ (async () => {
473
+ try {
474
+ const result = await engine.lookup(req.body)
475
+ return res.status(200).json(result)
476
+ } catch (error) {
477
+ console.error(error)
478
+ return res.status(400).json({
479
+ status: 'error',
480
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
481
+ })
482
+ }
483
+ })().catch(() => {
484
+ res.status(500).json({
485
+ status: 'error',
486
+ message: 'Unexpected error'
487
+ })
488
+ })
489
+ })
490
+
491
+ if (this.arcApiKey) {
492
+ // Route for ARC ingest
493
+ this.app.post('/arc-ingest', (req, res) => {
494
+ (async () => {
495
+ try {
496
+ const { txid, merklePath: merklePathHex, blockHeight } = req.body
497
+ const merklePath = MerklePath.fromHex(merklePathHex)
498
+ await engine.handleNewMerkleProof(txid, merklePath, blockHeight)
499
+ return res.status(200).json({ status: 'success', message: 'Transaction status updated' })
500
+ } catch (error) {
501
+ console.error(error)
502
+ return res.status(400).json({
503
+ status: 'error',
504
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
505
+ })
506
+ }
507
+ })().catch(() => {
508
+ res.status(500).json({
509
+ status: 'error',
510
+ message: 'Unexpected error'
511
+ })
512
+ })
513
+ })
514
+ } else {
515
+ this.logger.warn('Disabling ARC because no ARC API key was provided.')
516
+ }
517
+
518
+ if (this.enableGASPSync) {
519
+ // Routes for GASP synchronization
520
+ this.app.post('/requestSyncResponse', (req, res) => {
521
+ (async () => {
522
+ try {
523
+ const topic = req.headers['x-bsv-topic'] as string
524
+ const response = await engine.provideForeignSyncResponse(req.body, topic)
525
+ return res.status(200).json(response)
526
+ } catch (error) {
527
+ console.error(error)
528
+ return res.status(400).json({
529
+ status: 'error',
530
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
531
+ })
532
+ }
533
+ })().catch(() => {
534
+ res.status(500).json({
535
+ status: 'error',
536
+ message: 'Unexpected error'
537
+ })
538
+ })
539
+ })
540
+ this.app.post('/requestForeignGASPNode', (req, res) => {
541
+ (async () => {
542
+ try {
543
+ const { graphID, txid, outputIndex } = req.body
544
+ const response = await engine.provideForeignGASPNode(graphID, txid, outputIndex)
545
+ return res.status(200).json(response)
546
+ } catch (error) {
547
+ console.error(error)
548
+ return res.status(400).json({
549
+ status: 'error',
550
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
551
+ })
552
+ }
553
+ })().catch(() => {
554
+ res.status(500).json({
555
+ status: 'error',
556
+ message: 'Unexpected error'
557
+ })
558
+ })
559
+ })
560
+ } else {
561
+ this.logger.warn('GASP sync is disabled.')
562
+ }
563
+
564
+ // Automatically handle migrations
565
+ const migrationSource = new InMemoryMigrationSource(this.migrationsToRun)
566
+ const result = await knex.migrate.latest({
567
+ migrationSource
568
+ })
569
+ this.logger.log('Knex migrations run', result)
570
+
571
+ // 404 handler for all other routes
572
+ this.app.use((req, res) => {
573
+ this.logger.log('404', req.url)
574
+ res.status(404).json({
575
+ status: 'error',
576
+ code: 'ERR_ROUTE_NOT_FOUND',
577
+ description: 'Route not found.'
578
+ })
579
+ })
580
+
581
+ // Start listening on the configured port
582
+ this.app.listen(this.port, () => {
583
+ this.logger.log(`${this.name} listening on local port ${this.port}`)
584
+ })
585
+ }
586
+ }
File without changes