@bsv/overlay-express 0.7.13 โ 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/package.json +5 -5
- package/dist/cjs/src/JanitorService.js +215 -0
- package/dist/cjs/src/JanitorService.js.map +1 -0
- package/dist/cjs/src/OverlayExpress.js +50 -0
- package/dist/cjs/src/OverlayExpress.js.map +1 -1
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/esm/src/JanitorService.js +211 -0
- package/dist/esm/src/JanitorService.js.map +1 -0
- package/dist/esm/src/OverlayExpress.js +50 -0
- package/dist/esm/src/OverlayExpress.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/types/src/JanitorService.d.ts +59 -0
- package/dist/types/src/JanitorService.d.ts.map +1 -0
- package/dist/types/src/OverlayExpress.d.ts +11 -0
- package/dist/types/src/OverlayExpress.d.ts.map +1 -1
- package/dist/types/tsconfig.types.tsbuildinfo +1 -1
- package/package.json +5 -5
- package/src/JanitorService.ts +247 -0
- package/src/OverlayExpress.ts +54 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bsv/overlay-express",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "BSV Blockchain Overlay Express",
|
|
6
6
|
"main": "dist/cjs/mod.js",
|
|
@@ -64,13 +64,13 @@
|
|
|
64
64
|
},
|
|
65
65
|
"dependencies": {
|
|
66
66
|
"@bsv/overlay": "^0.4.8",
|
|
67
|
-
"@bsv/overlay-discovery-services": "^1.
|
|
68
|
-
"@bsv/sdk": "^1.
|
|
67
|
+
"@bsv/overlay-discovery-services": "^1.5.1",
|
|
68
|
+
"@bsv/sdk": "^1.8.2",
|
|
69
69
|
"body-parser": "^1.20.3",
|
|
70
70
|
"chalk": "^5.3.0",
|
|
71
|
-
"express": "^4.21.
|
|
71
|
+
"express": "^4.21.2",
|
|
72
72
|
"knex": "^3.1.0",
|
|
73
|
-
"mongodb": "^6.
|
|
73
|
+
"mongodb": "^6.20.0",
|
|
74
74
|
"uuid": "^11.0.5"
|
|
75
75
|
}
|
|
76
76
|
}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { Db } from 'mongodb'
|
|
2
|
+
import chalk from 'chalk'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Configuration for the Janitor Service
|
|
6
|
+
*/
|
|
7
|
+
export interface JanitorConfig {
|
|
8
|
+
mongoDb: Db
|
|
9
|
+
logger?: typeof console
|
|
10
|
+
requestTimeoutMs?: number
|
|
11
|
+
hostDownRevokeScore?: number
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
|
|
16
|
+
* It validates domain names and checks /health endpoints to ensure services are operational.
|
|
17
|
+
*
|
|
18
|
+
* When a service is down, it increments a "down" counter. When healthy, it decrements.
|
|
19
|
+
* If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
|
|
20
|
+
*
|
|
21
|
+
* This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
|
|
22
|
+
*/
|
|
23
|
+
export class JanitorService {
|
|
24
|
+
private readonly mongoDb: Db
|
|
25
|
+
private readonly logger: typeof console
|
|
26
|
+
private readonly requestTimeoutMs: number
|
|
27
|
+
private readonly hostDownRevokeScore: number
|
|
28
|
+
|
|
29
|
+
constructor (config: JanitorConfig) {
|
|
30
|
+
this.mongoDb = config.mongoDb
|
|
31
|
+
this.logger = config.logger ?? console
|
|
32
|
+
this.requestTimeoutMs = config.requestTimeoutMs ?? 10000 // Default: 10 seconds
|
|
33
|
+
this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Runs a single pass of health checks on all SHIP and SLAP outputs
|
|
38
|
+
*/
|
|
39
|
+
async run (): Promise<void> {
|
|
40
|
+
this.logger.log(chalk.blue('๐งน Running janitor health checks...'))
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
// Check SHIP outputs
|
|
44
|
+
await this.checkTopicOutputs('shipRecords')
|
|
45
|
+
|
|
46
|
+
// Check SLAP outputs
|
|
47
|
+
await this.checkTopicOutputs('slapRecords')
|
|
48
|
+
|
|
49
|
+
this.logger.log(chalk.green('๐งน Janitor health checks completed'))
|
|
50
|
+
} catch (error) {
|
|
51
|
+
this.logger.error(chalk.red('โ Error during health checks:'), error)
|
|
52
|
+
throw error
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Checks all outputs for a specific topic
|
|
58
|
+
*/
|
|
59
|
+
private async checkTopicOutputs (collectionName: string): Promise<void> {
|
|
60
|
+
try {
|
|
61
|
+
const collection = this.mongoDb.collection(collectionName)
|
|
62
|
+
const outputs = await collection.find({}).toArray()
|
|
63
|
+
|
|
64
|
+
this.logger.log(chalk.cyan(`๐ Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`))
|
|
65
|
+
|
|
66
|
+
for (const output of outputs) {
|
|
67
|
+
await this.checkOutput(output, collection)
|
|
68
|
+
}
|
|
69
|
+
} catch (error) {
|
|
70
|
+
this.logger.error(chalk.red(`โ Error checking ${collectionName.toUpperCase()} outputs:`), error)
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Checks a single output for health
|
|
76
|
+
*/
|
|
77
|
+
private async checkOutput (output: Record<string, any>, collection: any): Promise<void> {
|
|
78
|
+
try {
|
|
79
|
+
// Extract the URL from the output
|
|
80
|
+
const url = this.extractURLFromOutput(output)
|
|
81
|
+
if (url === null) {
|
|
82
|
+
return // Skip if no valid URL
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Validate domain
|
|
86
|
+
if (!this.isValidDomain(url)) {
|
|
87
|
+
this.logger.log(chalk.yellow(`โ ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`))
|
|
88
|
+
await this.handleUnhealthyOutput(output, collection)
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Check health endpoint
|
|
93
|
+
const isHealthy = await this.checkHealthEndpoint(url)
|
|
94
|
+
|
|
95
|
+
if (isHealthy) {
|
|
96
|
+
await this.handleHealthyOutput(output, collection)
|
|
97
|
+
} else {
|
|
98
|
+
await this.handleUnhealthyOutput(output, collection)
|
|
99
|
+
}
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.logger.error(chalk.red(`โ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
|
|
102
|
+
await this.handleUnhealthyOutput(output, collection).catch(() => {})
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Extracts URL from output record
|
|
108
|
+
*/
|
|
109
|
+
private extractURLFromOutput (output: Record<string, any>): string | null {
|
|
110
|
+
try {
|
|
111
|
+
// SHIP and SLAP outputs typically have a domain field or URL field
|
|
112
|
+
// Check for common field names
|
|
113
|
+
if (typeof output.domain === 'string') {
|
|
114
|
+
return output.domain
|
|
115
|
+
}
|
|
116
|
+
if (typeof output.url === 'string') {
|
|
117
|
+
return output.url
|
|
118
|
+
}
|
|
119
|
+
if (typeof output.serviceURL === 'string') {
|
|
120
|
+
return output.serviceURL
|
|
121
|
+
}
|
|
122
|
+
// SLAP outputs may have a protocols array with URLs
|
|
123
|
+
if (Array.isArray(output.protocols) && output.protocols.length > 0) {
|
|
124
|
+
const httpsProtocol = output.protocols.find((p: any) =>
|
|
125
|
+
typeof p === 'string' && p.startsWith('https://')
|
|
126
|
+
)
|
|
127
|
+
if (httpsProtocol !== undefined) {
|
|
128
|
+
return httpsProtocol
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return null
|
|
132
|
+
} catch {
|
|
133
|
+
return null
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Validates if a string is a valid domain name
|
|
139
|
+
*/
|
|
140
|
+
private isValidDomain (url: string): boolean {
|
|
141
|
+
try {
|
|
142
|
+
// Parse the URL
|
|
143
|
+
const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`)
|
|
144
|
+
|
|
145
|
+
// Check if hostname is valid
|
|
146
|
+
const hostname = parsedURL.hostname
|
|
147
|
+
|
|
148
|
+
// Basic domain validation
|
|
149
|
+
const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i
|
|
150
|
+
const localhostRegex = /^localhost$/i
|
|
151
|
+
const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/
|
|
152
|
+
|
|
153
|
+
return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname)
|
|
154
|
+
} catch {
|
|
155
|
+
return false
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Checks the /health endpoint of a service
|
|
161
|
+
*/
|
|
162
|
+
private async checkHealthEndpoint (url: string): Promise<boolean> {
|
|
163
|
+
try {
|
|
164
|
+
// Ensure URL has protocol
|
|
165
|
+
const fullURL = url.startsWith('http') ? url : `https://${url}`
|
|
166
|
+
const healthURL = new URL('/health', fullURL).toString()
|
|
167
|
+
|
|
168
|
+
// Create AbortController for timeout
|
|
169
|
+
const controller = new AbortController()
|
|
170
|
+
const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs)
|
|
171
|
+
|
|
172
|
+
try {
|
|
173
|
+
const response = await fetch(healthURL, {
|
|
174
|
+
method: 'GET',
|
|
175
|
+
signal: controller.signal,
|
|
176
|
+
headers: {
|
|
177
|
+
Accept: 'application/json'
|
|
178
|
+
}
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
clearTimeout(timeout)
|
|
182
|
+
|
|
183
|
+
if (!response.ok) {
|
|
184
|
+
return false
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const data = await response.json()
|
|
188
|
+
return data?.status === 'ok'
|
|
189
|
+
} catch (error: any) {
|
|
190
|
+
clearTimeout(timeout)
|
|
191
|
+
if (error.name === 'AbortError') {
|
|
192
|
+
this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`))
|
|
193
|
+
}
|
|
194
|
+
return false
|
|
195
|
+
}
|
|
196
|
+
} catch (error) {
|
|
197
|
+
return false
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Handles a healthy output by decrementing its down counter
|
|
203
|
+
*/
|
|
204
|
+
private async handleHealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
|
|
205
|
+
try {
|
|
206
|
+
const currentDown = typeof output.down === 'number' ? output.down : 0
|
|
207
|
+
|
|
208
|
+
if (currentDown > 0) {
|
|
209
|
+
// Decrement the down counter
|
|
210
|
+
await collection.updateOne(
|
|
211
|
+
{ _id: output._id },
|
|
212
|
+
{ $inc: { down: -1 } }
|
|
213
|
+
)
|
|
214
|
+
this.logger.log(chalk.green(`โ
Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`))
|
|
215
|
+
}
|
|
216
|
+
} catch (error) {
|
|
217
|
+
this.logger.error(chalk.red(`โ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
|
|
223
|
+
*/
|
|
224
|
+
private async handleUnhealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
|
|
225
|
+
try {
|
|
226
|
+
const currentDown = typeof output.down === 'number' ? output.down : 0
|
|
227
|
+
const newDown = currentDown + 1
|
|
228
|
+
|
|
229
|
+
this.logger.log(chalk.yellow(`โ ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`))
|
|
230
|
+
|
|
231
|
+
// Check if we should delete the record
|
|
232
|
+
if (newDown >= this.hostDownRevokeScore) {
|
|
233
|
+
this.logger.log(chalk.red(`๐ซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`))
|
|
234
|
+
await collection.deleteOne({ _id: output._id })
|
|
235
|
+
this.logger.log(chalk.green(`โ
Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`))
|
|
236
|
+
} else {
|
|
237
|
+
// Increment the down counter
|
|
238
|
+
await collection.updateOne(
|
|
239
|
+
{ _id: output._id },
|
|
240
|
+
{ $inc: { down: 1 } }
|
|
241
|
+
)
|
|
242
|
+
}
|
|
243
|
+
} catch (error) {
|
|
244
|
+
this.logger.error(chalk.red(`โ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
package/src/OverlayExpress.ts
CHANGED
|
@@ -22,6 +22,7 @@ import * as DiscoveryServices from '@bsv/overlay-discovery-services'
|
|
|
22
22
|
import chalk from 'chalk'
|
|
23
23
|
import util from 'util'
|
|
24
24
|
import { v4 as uuidv4 } from 'uuid'
|
|
25
|
+
import { JanitorService } from './JanitorService.js'
|
|
25
26
|
|
|
26
27
|
/**
|
|
27
28
|
* Knex database migration.
|
|
@@ -145,6 +146,15 @@ export default class OverlayExpress {
|
|
|
145
146
|
// If not passed in, we'll generate a random one.
|
|
146
147
|
private readonly adminToken: string
|
|
147
148
|
|
|
149
|
+
// Configuration for the janitor service
|
|
150
|
+
janitorConfig: {
|
|
151
|
+
requestTimeoutMs: number
|
|
152
|
+
hostDownRevokeScore: number
|
|
153
|
+
} = {
|
|
154
|
+
requestTimeoutMs: 10000, // 10 seconds
|
|
155
|
+
hostDownRevokeScore: 3
|
|
156
|
+
}
|
|
157
|
+
|
|
148
158
|
/**
|
|
149
159
|
* Constructs an instance of OverlayExpress.
|
|
150
160
|
* @param name - The name of the service
|
|
@@ -189,6 +199,20 @@ export default class OverlayExpress {
|
|
|
189
199
|
this.logger.log(chalk.blue('๐ฅ๏ธ Web UI has been configured.'))
|
|
190
200
|
}
|
|
191
201
|
|
|
202
|
+
/**
|
|
203
|
+
* Configures the janitor service parameters
|
|
204
|
+
* @param config - Janitor configuration options
|
|
205
|
+
* - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
|
|
206
|
+
* - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
|
|
207
|
+
*/
|
|
208
|
+
configureJanitor (config: Partial<typeof this.janitorConfig>): void {
|
|
209
|
+
this.janitorConfig = {
|
|
210
|
+
...this.janitorConfig,
|
|
211
|
+
...config
|
|
212
|
+
}
|
|
213
|
+
this.logger.log(chalk.blue('๐งน Janitor service has been configured.'))
|
|
214
|
+
}
|
|
215
|
+
|
|
192
216
|
/**
|
|
193
217
|
* Configures the logger to be used by the server.
|
|
194
218
|
* @param logger - A logger object (e.g., console)
|
|
@@ -928,6 +952,36 @@ export default class OverlayExpress {
|
|
|
928
952
|
})
|
|
929
953
|
})
|
|
930
954
|
|
|
955
|
+
/**
|
|
956
|
+
* Admin route to run the janitor service.
|
|
957
|
+
*/
|
|
958
|
+
this.app.post('/admin/janitor', checkAdminAuth as any, (req, res) => {
|
|
959
|
+
; (async () => {
|
|
960
|
+
try {
|
|
961
|
+
this.ensureMongo()
|
|
962
|
+
const janitor = new JanitorService({
|
|
963
|
+
mongoDb: this.mongoDb,
|
|
964
|
+
logger: this.logger,
|
|
965
|
+
requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
|
|
966
|
+
hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
|
|
967
|
+
})
|
|
968
|
+
await janitor.run()
|
|
969
|
+
return res.status(200).json({ status: 'success', message: 'Janitor run completed' })
|
|
970
|
+
} catch (error) {
|
|
971
|
+
console.error(chalk.red('โ Error in /admin/janitor:'), error)
|
|
972
|
+
return res.status(400).json({
|
|
973
|
+
status: 'error',
|
|
974
|
+
message: error instanceof Error ? error.message : 'An unknown error occurred'
|
|
975
|
+
})
|
|
976
|
+
}
|
|
977
|
+
})().catch(() => {
|
|
978
|
+
res.status(500).json({
|
|
979
|
+
status: 'error',
|
|
980
|
+
message: 'Unexpected error'
|
|
981
|
+
})
|
|
982
|
+
})
|
|
983
|
+
})
|
|
984
|
+
|
|
931
985
|
// Automatically handle migrations
|
|
932
986
|
const migrationSource = new InMemoryMigrationSource(this.migrationsToRun)
|
|
933
987
|
const result = await knex.migrate.latest({
|