@bsv/overlay-express 0.7.14 โ†’ 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,247 @@
1
+ import { Db } from 'mongodb'
2
+ import chalk from 'chalk'
3
+
4
+ /**
5
+ * Configuration for the Janitor Service
6
+ */
7
+ export interface JanitorConfig {
8
+ mongoDb: Db
9
+ logger?: typeof console
10
+ requestTimeoutMs?: number
11
+ hostDownRevokeScore?: number
12
+ }
13
+
14
+ /**
15
+ * JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
16
+ * It validates domain names and checks /health endpoints to ensure services are operational.
17
+ *
18
+ * When a service is down, it increments a "down" counter. When healthy, it decrements.
19
+ * If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
20
+ *
21
+ * This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
22
+ */
23
+ export class JanitorService {
24
+ private readonly mongoDb: Db
25
+ private readonly logger: typeof console
26
+ private readonly requestTimeoutMs: number
27
+ private readonly hostDownRevokeScore: number
28
+
29
+ constructor (config: JanitorConfig) {
30
+ this.mongoDb = config.mongoDb
31
+ this.logger = config.logger ?? console
32
+ this.requestTimeoutMs = config.requestTimeoutMs ?? 10000 // Default: 10 seconds
33
+ this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3
34
+ }
35
+
36
+ /**
37
+ * Runs a single pass of health checks on all SHIP and SLAP outputs
38
+ */
39
+ async run (): Promise<void> {
40
+ this.logger.log(chalk.blue('๐Ÿงน Running janitor health checks...'))
41
+
42
+ try {
43
+ // Check SHIP outputs
44
+ await this.checkTopicOutputs('shipRecords')
45
+
46
+ // Check SLAP outputs
47
+ await this.checkTopicOutputs('slapRecords')
48
+
49
+ this.logger.log(chalk.green('๐Ÿงน Janitor health checks completed'))
50
+ } catch (error) {
51
+ this.logger.error(chalk.red('โŒ Error during health checks:'), error)
52
+ throw error
53
+ }
54
+ }
55
+
56
+ /**
57
+ * Checks all outputs for a specific topic
58
+ */
59
+ private async checkTopicOutputs (collectionName: string): Promise<void> {
60
+ try {
61
+ const collection = this.mongoDb.collection(collectionName)
62
+ const outputs = await collection.find({}).toArray()
63
+
64
+ this.logger.log(chalk.cyan(`๐Ÿ” Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`))
65
+
66
+ for (const output of outputs) {
67
+ await this.checkOutput(output, collection)
68
+ }
69
+ } catch (error) {
70
+ this.logger.error(chalk.red(`โŒ Error checking ${collectionName.toUpperCase()} outputs:`), error)
71
+ }
72
+ }
73
+
74
+ /**
75
+ * Checks a single output for health
76
+ */
77
+ private async checkOutput (output: Record<string, any>, collection: any): Promise<void> {
78
+ try {
79
+ // Extract the URL from the output
80
+ const url = this.extractURLFromOutput(output)
81
+ if (url === null) {
82
+ return // Skip if no valid URL
83
+ }
84
+
85
+ // Validate domain
86
+ if (!this.isValidDomain(url)) {
87
+ this.logger.log(chalk.yellow(`โš ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`))
88
+ await this.handleUnhealthyOutput(output, collection)
89
+ return
90
+ }
91
+
92
+ // Check health endpoint
93
+ const isHealthy = await this.checkHealthEndpoint(url)
94
+
95
+ if (isHealthy) {
96
+ await this.handleHealthyOutput(output, collection)
97
+ } else {
98
+ await this.handleUnhealthyOutput(output, collection)
99
+ }
100
+ } catch (error) {
101
+ this.logger.error(chalk.red(`โŒ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
102
+ await this.handleUnhealthyOutput(output, collection).catch(() => {})
103
+ }
104
+ }
105
+
106
+ /**
107
+ * Extracts URL from output record
108
+ */
109
+ private extractURLFromOutput (output: Record<string, any>): string | null {
110
+ try {
111
+ // SHIP and SLAP outputs typically have a domain field or URL field
112
+ // Check for common field names
113
+ if (typeof output.domain === 'string') {
114
+ return output.domain
115
+ }
116
+ if (typeof output.url === 'string') {
117
+ return output.url
118
+ }
119
+ if (typeof output.serviceURL === 'string') {
120
+ return output.serviceURL
121
+ }
122
+ // SLAP outputs may have a protocols array with URLs
123
+ if (Array.isArray(output.protocols) && output.protocols.length > 0) {
124
+ const httpsProtocol = output.protocols.find((p: any) =>
125
+ typeof p === 'string' && p.startsWith('https://')
126
+ )
127
+ if (httpsProtocol !== undefined) {
128
+ return httpsProtocol
129
+ }
130
+ }
131
+ return null
132
+ } catch {
133
+ return null
134
+ }
135
+ }
136
+
137
+ /**
138
+ * Validates if a string is a valid domain name
139
+ */
140
+ private isValidDomain (url: string): boolean {
141
+ try {
142
+ // Parse the URL
143
+ const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`)
144
+
145
+ // Check if hostname is valid
146
+ const hostname = parsedURL.hostname
147
+
148
+ // Basic domain validation
149
+ const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i
150
+ const localhostRegex = /^localhost$/i
151
+ const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/
152
+
153
+ return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname)
154
+ } catch {
155
+ return false
156
+ }
157
+ }
158
+
159
+ /**
160
+ * Checks the /health endpoint of a service
161
+ */
162
+ private async checkHealthEndpoint (url: string): Promise<boolean> {
163
+ try {
164
+ // Ensure URL has protocol
165
+ const fullURL = url.startsWith('http') ? url : `https://${url}`
166
+ const healthURL = new URL('/health', fullURL).toString()
167
+
168
+ // Create AbortController for timeout
169
+ const controller = new AbortController()
170
+ const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs)
171
+
172
+ try {
173
+ const response = await fetch(healthURL, {
174
+ method: 'GET',
175
+ signal: controller.signal,
176
+ headers: {
177
+ Accept: 'application/json'
178
+ }
179
+ })
180
+
181
+ clearTimeout(timeout)
182
+
183
+ if (!response.ok) {
184
+ return false
185
+ }
186
+
187
+ const data = await response.json()
188
+ return data?.status === 'ok'
189
+ } catch (error: any) {
190
+ clearTimeout(timeout)
191
+ if (error.name === 'AbortError') {
192
+ this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`))
193
+ }
194
+ return false
195
+ }
196
+ } catch (error) {
197
+ return false
198
+ }
199
+ }
200
+
201
+ /**
202
+ * Handles a healthy output by decrementing its down counter
203
+ */
204
+ private async handleHealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
205
+ try {
206
+ const currentDown = typeof output.down === 'number' ? output.down : 0
207
+
208
+ if (currentDown > 0) {
209
+ // Decrement the down counter
210
+ await collection.updateOne(
211
+ { _id: output._id },
212
+ { $inc: { down: -1 } }
213
+ )
214
+ this.logger.log(chalk.green(`โœ… Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`))
215
+ }
216
+ } catch (error) {
217
+ this.logger.error(chalk.red(`โŒ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
218
+ }
219
+ }
220
+
221
+ /**
222
+ * Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
223
+ */
224
+ private async handleUnhealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
225
+ try {
226
+ const currentDown = typeof output.down === 'number' ? output.down : 0
227
+ const newDown = currentDown + 1
228
+
229
+ this.logger.log(chalk.yellow(`โš ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`))
230
+
231
+ // Check if we should delete the record
232
+ if (newDown >= this.hostDownRevokeScore) {
233
+ this.logger.log(chalk.red(`๐Ÿšซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`))
234
+ await collection.deleteOne({ _id: output._id })
235
+ this.logger.log(chalk.green(`โœ… Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`))
236
+ } else {
237
+ // Increment the down counter
238
+ await collection.updateOne(
239
+ { _id: output._id },
240
+ { $inc: { down: 1 } }
241
+ )
242
+ }
243
+ } catch (error) {
244
+ this.logger.error(chalk.red(`โŒ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
245
+ }
246
+ }
247
+ }
@@ -22,6 +22,7 @@ import * as DiscoveryServices from '@bsv/overlay-discovery-services'
22
22
  import chalk from 'chalk'
23
23
  import util from 'util'
24
24
  import { v4 as uuidv4 } from 'uuid'
25
+ import { JanitorService } from './JanitorService.js'
25
26
 
26
27
  /**
27
28
  * Knex database migration.
@@ -145,6 +146,15 @@ export default class OverlayExpress {
145
146
  // If not passed in, we'll generate a random one.
146
147
  private readonly adminToken: string
147
148
 
149
+ // Configuration for the janitor service
150
+ janitorConfig: {
151
+ requestTimeoutMs: number
152
+ hostDownRevokeScore: number
153
+ } = {
154
+ requestTimeoutMs: 10000, // 10 seconds
155
+ hostDownRevokeScore: 3
156
+ }
157
+
148
158
  /**
149
159
  * Constructs an instance of OverlayExpress.
150
160
  * @param name - The name of the service
@@ -189,6 +199,20 @@ export default class OverlayExpress {
189
199
  this.logger.log(chalk.blue('๐Ÿ–ฅ๏ธ Web UI has been configured.'))
190
200
  }
191
201
 
202
+ /**
203
+ * Configures the janitor service parameters
204
+ * @param config - Janitor configuration options
205
+ * - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
206
+ * - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
207
+ */
208
+ configureJanitor (config: Partial<typeof this.janitorConfig>): void {
209
+ this.janitorConfig = {
210
+ ...this.janitorConfig,
211
+ ...config
212
+ }
213
+ this.logger.log(chalk.blue('๐Ÿงน Janitor service has been configured.'))
214
+ }
215
+
192
216
  /**
193
217
  * Configures the logger to be used by the server.
194
218
  * @param logger - A logger object (e.g., console)
@@ -928,6 +952,36 @@ export default class OverlayExpress {
928
952
  })
929
953
  })
930
954
 
955
+ /**
956
+ * Admin route to run the janitor service.
957
+ */
958
+ this.app.post('/admin/janitor', checkAdminAuth as any, (req, res) => {
959
+ ; (async () => {
960
+ try {
961
+ this.ensureMongo()
962
+ const janitor = new JanitorService({
963
+ mongoDb: this.mongoDb,
964
+ logger: this.logger,
965
+ requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
966
+ hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
967
+ })
968
+ await janitor.run()
969
+ return res.status(200).json({ status: 'success', message: 'Janitor run completed' })
970
+ } catch (error) {
971
+ console.error(chalk.red('โŒ Error in /admin/janitor:'), error)
972
+ return res.status(400).json({
973
+ status: 'error',
974
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
975
+ })
976
+ }
977
+ })().catch(() => {
978
+ res.status(500).json({
979
+ status: 'error',
980
+ message: 'Unexpected error'
981
+ })
982
+ })
983
+ })
984
+
931
985
  // Automatically handle migrations
932
986
  const migrationSource = new InMemoryMigrationSource(this.migrationsToRun)
933
987
  const result = await knex.migrate.latest({