@bsv/overlay-express 0.9.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -103,13 +103,13 @@ export default class OverlayExpress {
103
103
  logger: typeof console = console
104
104
 
105
105
  // Knex (SQL) database
106
- knex: Knex.Knex = {} as unknown as Knex.Knex
106
+ knex?: Knex.Knex
107
107
 
108
108
  // Knex migrations to run
109
109
  migrationsToRun: Migration[] = []
110
110
 
111
111
  // MongoDB database
112
- mongoDb: Db = {} as unknown as Db
112
+ mongoDb?: Db
113
113
 
114
114
  // Network ('main' or 'test')
115
115
  network: 'main' | 'test' = 'main'
@@ -119,7 +119,7 @@ export default class OverlayExpress {
119
119
  chainTracker: ChainTracker | 'scripts only' = new WhatsOnChain(this.network)
120
120
 
121
121
  // The Overlay Engine
122
- engine: Engine = {} as unknown as Engine
122
+ engine?: Engine
123
123
 
124
124
  // Configured Topic Managers
125
125
  managers: Record<string, TopicManager> = {}
@@ -330,7 +330,7 @@ export default class OverlayExpress {
330
330
  serviceFactory: (knex: Knex.Knex) => { service: LookupService, migrations: Migration[] }
331
331
  ): void {
332
332
  this.ensureKnex()
333
- const factoryResult = serviceFactory(this.knex)
333
+ const factoryResult = serviceFactory(this.knex!)
334
334
  this.services[name] = factoryResult.service
335
335
  this.migrationsToRun.push(...factoryResult.migrations)
336
336
  this.logger.log(chalk.blue(`🔍 Configured lookup service ${name} with Knex`))
@@ -343,7 +343,7 @@ export default class OverlayExpress {
343
343
  */
344
344
  configureLookupServiceWithMongo (name: string, serviceFactory: (mongoDb: Db) => LookupService): void {
345
345
  this.ensureMongo()
346
- this.services[name] = serviceFactory(this.mongoDb)
346
+ this.services[name] = serviceFactory(this.mongoDb!)
347
347
  this.logger.log(chalk.blue(`🔍 Configured lookup service ${name} with MongoDB`))
348
348
  }
349
349
 
@@ -404,7 +404,7 @@ export default class OverlayExpress {
404
404
  }
405
405
 
406
406
  // Build the actual Storage
407
- const storage = new KnexStorage(this.knex)
407
+ const storage = new KnexStorage(this.knex!)
408
408
  // Include the KnexStorage migrations
409
409
  this.migrationsToRun = [...KnexStorageMigrations.default, ...this.migrationsToRun]
410
410
 
@@ -518,8 +518,8 @@ export default class OverlayExpress {
518
518
  async start (): Promise<void> {
519
519
  this.ensureEngine()
520
520
  this.ensureKnex()
521
- const engine = this.engine
522
- const knex = this.knex
521
+ const engine = this.engine!
522
+ const knex = this.knex!
523
523
 
524
524
  this.app.use(bodyParser.json({ limit: '1gb', type: 'application/json' }))
525
525
  this.app.use(bodyParser.raw({ limit: '1gb', type: 'application/octet-stream' }))
@@ -1011,7 +1011,7 @@ export default class OverlayExpress {
1011
1011
  try {
1012
1012
  this.ensureMongo()
1013
1013
  const janitor = new JanitorService({
1014
- mongoDb: this.mongoDb,
1014
+ mongoDb: this.mongoDb!,
1015
1015
  logger: this.logger,
1016
1016
  requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
1017
1017
  hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
@@ -0,0 +1,11 @@
1
+ import { describe, it, expect } from '@jest/globals'
2
+
3
+ // Since InMemoryMigrationSource is not exported, we'll test it through OverlayExpress
4
+ // This is a separate test file for clarity
5
+ describe('InMemoryMigrationSource', () => {
6
+ it('should be tested through OverlayExpress integration tests', () => {
7
+ // InMemoryMigrationSource is a private class used internally
8
+ // Its functionality is tested through the OverlayExpress.start() method
9
+ expect(true).toBe(true)
10
+ })
11
+ })
@@ -0,0 +1,604 @@
1
+ import { describe, it, expect, jest, beforeEach } from '@jest/globals'
2
+ import { JanitorService } from '../JanitorService.js'
3
+ import { Db } from 'mongodb'
4
+
5
+ // Mock fetch globally
6
+ global.fetch = jest.fn() as any
7
+
8
+ describe('JanitorService', () => {
9
+ let mockDb: Db
10
+ let mockCollection: any
11
+ let mockLogger: any
12
+
13
+ beforeEach(() => {
14
+ jest.clearAllMocks()
15
+
16
+ mockCollection = {
17
+ find: jest.fn().mockReturnValue({
18
+ toArray: jest.fn<any>().mockResolvedValue([] as any)
19
+ }),
20
+ updateOne: jest.fn<any>().mockResolvedValue({} as any),
21
+ deleteOne: jest.fn<any>().mockResolvedValue({} as any)
22
+ }
23
+
24
+ mockDb = {
25
+ collection: jest.fn().mockReturnValue(mockCollection)
26
+ } as unknown as Db
27
+
28
+ mockLogger = {
29
+ log: jest.fn(),
30
+ error: jest.fn()
31
+ }
32
+ })
33
+
34
+ describe('constructor', () => {
35
+ it('should create instance with required config', () => {
36
+ const janitor = new JanitorService({
37
+ mongoDb: mockDb
38
+ })
39
+
40
+ expect(janitor).toBeInstanceOf(JanitorService)
41
+ })
42
+
43
+ it('should use default values when not provided', () => {
44
+ const janitor = new JanitorService({
45
+ mongoDb: mockDb
46
+ })
47
+
48
+ expect(janitor).toBeDefined()
49
+ })
50
+
51
+ it('should use custom logger when provided', () => {
52
+ const janitor = new JanitorService({
53
+ mongoDb: mockDb,
54
+ logger: mockLogger
55
+ })
56
+
57
+ expect(janitor).toBeDefined()
58
+ })
59
+
60
+ it('should use custom timeout and revoke score', () => {
61
+ const janitor = new JanitorService({
62
+ mongoDb: mockDb,
63
+ requestTimeoutMs: 5000,
64
+ hostDownRevokeScore: 5
65
+ })
66
+
67
+ expect(janitor).toBeDefined()
68
+ })
69
+ })
70
+
71
+ describe('run', () => {
72
+ it('should check both SHIP and SLAP collections', async () => {
73
+ const janitor = new JanitorService({
74
+ mongoDb: mockDb,
75
+ logger: mockLogger
76
+ })
77
+
78
+ await janitor.run()
79
+
80
+ expect(mockDb.collection).toHaveBeenCalledWith('shipRecords')
81
+ expect(mockDb.collection).toHaveBeenCalledWith('slapRecords')
82
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('Running janitor'))
83
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('completed'))
84
+ })
85
+
86
+ it('should handle errors during health checks', async () => {
87
+ (mockDb as any).collection = jest.fn().mockImplementation(() => {
88
+ throw new Error('Database error')
89
+ })
90
+
91
+ const janitor = new JanitorService({
92
+ mongoDb: mockDb,
93
+ logger: mockLogger
94
+ })
95
+
96
+ await janitor.run()
97
+ expect(mockLogger.error).toHaveBeenCalled()
98
+ })
99
+
100
+ it('should process empty collections without errors', async () => {
101
+ const janitor = new JanitorService({
102
+ mongoDb: mockDb,
103
+ logger: mockLogger
104
+ })
105
+
106
+ await janitor.run()
107
+
108
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('completed'))
109
+ })
110
+ })
111
+
112
+ describe('output processing', () => {
113
+ it('should process output with domain field', async () => {
114
+ const mockOutput = {
115
+ _id: '123',
116
+ txid: 'abc123',
117
+ outputIndex: 0,
118
+ domain: 'https://example.com',
119
+ down: 0
120
+ }
121
+
122
+ mockCollection.find.mockReturnValue({
123
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
124
+ })
125
+
126
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
127
+ ok: true,
128
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
129
+ })
130
+
131
+ const janitor = new JanitorService({
132
+ mongoDb: mockDb,
133
+ logger: mockLogger
134
+ })
135
+
136
+ await janitor.run()
137
+
138
+ expect(global.fetch).toHaveBeenCalled()
139
+ })
140
+
141
+ it('should process output with url field', async () => {
142
+ const mockOutput = {
143
+ _id: '123',
144
+ txid: 'abc123',
145
+ outputIndex: 0,
146
+ url: 'https://example.com',
147
+ down: 0
148
+ }
149
+
150
+ mockCollection.find.mockReturnValue({
151
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
152
+ })
153
+
154
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
155
+ ok: true,
156
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
157
+ })
158
+
159
+ const janitor = new JanitorService({
160
+ mongoDb: mockDb,
161
+ logger: mockLogger
162
+ })
163
+
164
+ await janitor.run()
165
+
166
+ expect(global.fetch).toHaveBeenCalled()
167
+ })
168
+
169
+ it('should process output with serviceURL field', async () => {
170
+ const mockOutput = {
171
+ _id: '123',
172
+ txid: 'abc123',
173
+ outputIndex: 0,
174
+ serviceURL: 'https://example.com',
175
+ down: 0
176
+ }
177
+
178
+ mockCollection.find.mockReturnValue({
179
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
180
+ })
181
+
182
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
183
+ ok: true,
184
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
185
+ })
186
+
187
+ const janitor = new JanitorService({
188
+ mongoDb: mockDb,
189
+ logger: mockLogger
190
+ })
191
+
192
+ await janitor.run()
193
+
194
+ expect(global.fetch).toHaveBeenCalled()
195
+ })
196
+
197
+ it('should process output with protocols array', async () => {
198
+ const mockOutput = {
199
+ _id: '123',
200
+ txid: 'abc123',
201
+ outputIndex: 0,
202
+ protocols: ['https://example.com', 'http://other.com'],
203
+ down: 0
204
+ }
205
+
206
+ mockCollection.find.mockReturnValue({
207
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
208
+ })
209
+
210
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
211
+ ok: true,
212
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
213
+ })
214
+
215
+ const janitor = new JanitorService({
216
+ mongoDb: mockDb,
217
+ logger: mockLogger
218
+ })
219
+
220
+ await janitor.run()
221
+
222
+ expect(global.fetch).toHaveBeenCalled()
223
+ })
224
+
225
+ it('should skip output without valid URL', async () => {
226
+ const mockOutput = {
227
+ _id: '123',
228
+ txid: 'abc123',
229
+ outputIndex: 0,
230
+ down: 0
231
+ }
232
+
233
+ mockCollection.find.mockReturnValue({
234
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
235
+ })
236
+
237
+ const janitor = new JanitorService({
238
+ mongoDb: mockDb,
239
+ logger: mockLogger
240
+ })
241
+
242
+ await janitor.run()
243
+
244
+ expect(global.fetch).not.toHaveBeenCalled()
245
+ })
246
+ })
247
+
248
+ describe('domain validation', () => {
249
+ it('should accept valid domain', async () => {
250
+ const mockOutput = {
251
+ _id: '123',
252
+ txid: 'abc123',
253
+ outputIndex: 0,
254
+ domain: 'https://valid-domain.com',
255
+ down: 0
256
+ }
257
+
258
+ mockCollection.find.mockReturnValue({
259
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
260
+ })
261
+
262
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
263
+ ok: true,
264
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
265
+ })
266
+
267
+ const janitor = new JanitorService({
268
+ mongoDb: mockDb,
269
+ logger: mockLogger
270
+ })
271
+
272
+ await janitor.run()
273
+
274
+ expect(global.fetch).toHaveBeenCalled()
275
+ })
276
+
277
+ it('should reject invalid domain and increment down counter', async () => {
278
+ const mockOutput = {
279
+ _id: '123',
280
+ txid: 'abc123',
281
+ outputIndex: 0,
282
+ domain: 'invalid domain!@#',
283
+ down: 0
284
+ }
285
+
286
+ mockCollection.find.mockReturnValue({
287
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
288
+ })
289
+
290
+ const janitor = new JanitorService({
291
+ mongoDb: mockDb,
292
+ logger: mockLogger
293
+ })
294
+
295
+ await janitor.run()
296
+
297
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
298
+ { _id: '123' },
299
+ { $inc: { down: 1 } }
300
+ )
301
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('Invalid domain'))
302
+ })
303
+
304
+ it('should accept localhost', async () => {
305
+ const mockOutput = {
306
+ _id: '123',
307
+ txid: 'abc123',
308
+ outputIndex: 0,
309
+ domain: 'http://localhost:3000',
310
+ down: 0
311
+ }
312
+
313
+ mockCollection.find.mockReturnValue({
314
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
315
+ })
316
+
317
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
318
+ ok: true,
319
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
320
+ })
321
+
322
+ const janitor = new JanitorService({
323
+ mongoDb: mockDb,
324
+ logger: mockLogger
325
+ })
326
+
327
+ await janitor.run()
328
+
329
+ expect(global.fetch).toHaveBeenCalled()
330
+ })
331
+
332
+ it('should accept IP address', async () => {
333
+ const mockOutput = {
334
+ _id: '123',
335
+ txid: 'abc123',
336
+ outputIndex: 0,
337
+ domain: 'http://192.168.1.1',
338
+ down: 0
339
+ }
340
+
341
+ mockCollection.find.mockReturnValue({
342
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
343
+ })
344
+
345
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
346
+ ok: true,
347
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
348
+ })
349
+
350
+ const janitor = new JanitorService({
351
+ mongoDb: mockDb,
352
+ logger: mockLogger
353
+ })
354
+
355
+ await janitor.run()
356
+
357
+ expect(global.fetch).toHaveBeenCalled()
358
+ })
359
+ })
360
+
361
+ describe('health check', () => {
362
+ it('should mark output as healthy when health check passes', async () => {
363
+ const mockOutput = {
364
+ _id: '123',
365
+ txid: 'abc123',
366
+ outputIndex: 0,
367
+ domain: 'https://example.com',
368
+ down: 2
369
+ }
370
+
371
+ mockCollection.find.mockReturnValue({
372
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
373
+ })
374
+
375
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
376
+ ok: true,
377
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
378
+ })
379
+
380
+ const janitor = new JanitorService({
381
+ mongoDb: mockDb,
382
+ logger: mockLogger
383
+ })
384
+
385
+ await janitor.run()
386
+
387
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
388
+ { _id: '123' },
389
+ { $inc: { down: -1 } }
390
+ )
391
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('is healthy'))
392
+ })
393
+
394
+ it('should not decrement when already at 0', async () => {
395
+ const mockOutput = {
396
+ _id: '123',
397
+ txid: 'abc123',
398
+ outputIndex: 0,
399
+ domain: 'https://example.com',
400
+ down: 0
401
+ }
402
+
403
+ mockCollection.find.mockReturnValue({
404
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
405
+ })
406
+
407
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
408
+ ok: true,
409
+ json: jest.fn<any>().mockResolvedValue({ status: 'ok' })
410
+ })
411
+
412
+ const janitor = new JanitorService({
413
+ mongoDb: mockDb,
414
+ logger: mockLogger
415
+ })
416
+
417
+ await janitor.run()
418
+
419
+ expect(mockCollection.updateOne).not.toHaveBeenCalled()
420
+ })
421
+
422
+ it('should increment down counter when health check fails', async () => {
423
+ const mockOutput = {
424
+ _id: '123',
425
+ txid: 'abc123',
426
+ outputIndex: 0,
427
+ domain: 'https://example.com',
428
+ down: 0
429
+ }
430
+
431
+ mockCollection.find.mockReturnValue({
432
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
433
+ })
434
+
435
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
436
+ ok: false
437
+ })
438
+
439
+ const janitor = new JanitorService({
440
+ mongoDb: mockDb,
441
+ logger: mockLogger
442
+ })
443
+
444
+ await janitor.run()
445
+
446
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
447
+ { _id: '123' },
448
+ { $inc: { down: 1 } }
449
+ )
450
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('is unhealthy'))
451
+ })
452
+
453
+ it('should delete output when down count reaches threshold', async () => {
454
+ const mockOutput = {
455
+ _id: '123',
456
+ txid: 'abc123',
457
+ outputIndex: 0,
458
+ domain: 'https://example.com',
459
+ down: 2
460
+ }
461
+
462
+ mockCollection.find.mockReturnValue({
463
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
464
+ })
465
+
466
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
467
+ ok: false
468
+ })
469
+
470
+ const janitor = new JanitorService({
471
+ mongoDb: mockDb,
472
+ logger: mockLogger,
473
+ hostDownRevokeScore: 3
474
+ })
475
+
476
+ await janitor.run()
477
+
478
+ expect(mockCollection.deleteOne).toHaveBeenCalledWith({ _id: '123' })
479
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('Deleting output'))
480
+ })
481
+
482
+ it('should handle fetch timeout', async () => {
483
+ const mockOutput = {
484
+ _id: '123',
485
+ txid: 'abc123',
486
+ outputIndex: 0,
487
+ domain: 'https://example.com',
488
+ down: 0
489
+ }
490
+
491
+ mockCollection.find.mockReturnValue({
492
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
493
+ })
494
+
495
+ const abortError = new Error('Aborted')
496
+ abortError.name = 'AbortError'
497
+ ;(global.fetch as jest.Mock<any>).mockRejectedValue(abortError)
498
+
499
+ const janitor = new JanitorService({
500
+ mongoDb: mockDb,
501
+ logger: mockLogger,
502
+ requestTimeoutMs: 1000
503
+ })
504
+
505
+ await janitor.run()
506
+
507
+ expect(mockLogger.log).toHaveBeenCalledWith(expect.stringContaining('timeout'))
508
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
509
+ { _id: '123' },
510
+ { $inc: { down: 1 } }
511
+ )
512
+ })
513
+
514
+ it('should handle fetch errors', async () => {
515
+ const mockOutput = {
516
+ _id: '123',
517
+ txid: 'abc123',
518
+ outputIndex: 0,
519
+ domain: 'https://example.com',
520
+ down: 0
521
+ }
522
+
523
+ mockCollection.find.mockReturnValue({
524
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
525
+ })
526
+
527
+ ;(global.fetch as jest.Mock<any>).mockRejectedValue(new Error('Network error'))
528
+
529
+ const janitor = new JanitorService({
530
+ mongoDb: mockDb,
531
+ logger: mockLogger
532
+ })
533
+
534
+ await janitor.run()
535
+
536
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
537
+ { _id: '123' },
538
+ { $inc: { down: 1 } }
539
+ )
540
+ })
541
+
542
+ it('should handle invalid JSON response', async () => {
543
+ const mockOutput = {
544
+ _id: '123',
545
+ txid: 'abc123',
546
+ outputIndex: 0,
547
+ domain: 'https://example.com',
548
+ down: 0
549
+ }
550
+
551
+ mockCollection.find.mockReturnValue({
552
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
553
+ })
554
+
555
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
556
+ ok: true,
557
+ json: jest.fn<any>().mockRejectedValue(new Error('Invalid JSON'))
558
+ })
559
+
560
+ const janitor = new JanitorService({
561
+ mongoDb: mockDb,
562
+ logger: mockLogger
563
+ })
564
+
565
+ await janitor.run()
566
+
567
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
568
+ { _id: '123' },
569
+ { $inc: { down: 1 } }
570
+ )
571
+ })
572
+
573
+ it('should verify health endpoint returns status: ok', async () => {
574
+ const mockOutput = {
575
+ _id: '123',
576
+ txid: 'abc123',
577
+ outputIndex: 0,
578
+ domain: 'https://example.com',
579
+ down: 1
580
+ }
581
+
582
+ mockCollection.find.mockReturnValue({
583
+ toArray: jest.fn<any>().mockResolvedValue([mockOutput])
584
+ })
585
+
586
+ ;(global.fetch as jest.Mock<any>).mockResolvedValue({
587
+ ok: true,
588
+ json: jest.fn<any>().mockResolvedValue({ status: 'error' })
589
+ })
590
+
591
+ const janitor = new JanitorService({
592
+ mongoDb: mockDb,
593
+ logger: mockLogger
594
+ })
595
+
596
+ await janitor.run()
597
+
598
+ expect(mockCollection.updateOne).toHaveBeenCalledWith(
599
+ { _id: '123' },
600
+ { $inc: { down: 1 } }
601
+ )
602
+ })
603
+ })
604
+ })