@bsv/overlay-express 0.7.14 โ†’ 0.8.2-rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,247 @@
1
+ import { Db } from 'mongodb'
2
+ import chalk from 'chalk'
3
+
4
+ /**
5
+ * Configuration for the Janitor Service
6
+ */
7
+ export interface JanitorConfig {
8
+ mongoDb: Db
9
+ logger?: typeof console
10
+ requestTimeoutMs?: number
11
+ hostDownRevokeScore?: number
12
+ }
13
+
14
+ /**
15
+ * JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
16
+ * It validates domain names and checks /health endpoints to ensure services are operational.
17
+ *
18
+ * When a service is down, it increments a "down" counter. When healthy, it decrements.
19
+ * If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
20
+ *
21
+ * This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
22
+ */
23
+ export class JanitorService {
24
+ private readonly mongoDb: Db
25
+ private readonly logger: typeof console
26
+ private readonly requestTimeoutMs: number
27
+ private readonly hostDownRevokeScore: number
28
+
29
+ constructor (config: JanitorConfig) {
30
+ this.mongoDb = config.mongoDb
31
+ this.logger = config.logger ?? console
32
+ this.requestTimeoutMs = config.requestTimeoutMs ?? 10000 // Default: 10 seconds
33
+ this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3
34
+ }
35
+
36
+ /**
37
+ * Runs a single pass of health checks on all SHIP and SLAP outputs
38
+ */
39
+ async run (): Promise<void> {
40
+ this.logger.log(chalk.blue('๐Ÿงน Running janitor health checks...'))
41
+
42
+ try {
43
+ // Check SHIP outputs
44
+ await this.checkTopicOutputs('shipRecords')
45
+
46
+ // Check SLAP outputs
47
+ await this.checkTopicOutputs('slapRecords')
48
+
49
+ this.logger.log(chalk.green('๐Ÿงน Janitor health checks completed'))
50
+ } catch (error) {
51
+ this.logger.error(chalk.red('โŒ Error during health checks:'), error)
52
+ throw error
53
+ }
54
+ }
55
+
56
+ /**
57
+ * Checks all outputs for a specific topic
58
+ */
59
+ private async checkTopicOutputs (collectionName: string): Promise<void> {
60
+ try {
61
+ const collection = this.mongoDb.collection(collectionName)
62
+ const outputs = await collection.find({}).toArray()
63
+
64
+ this.logger.log(chalk.cyan(`๐Ÿ” Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`))
65
+
66
+ for (const output of outputs) {
67
+ await this.checkOutput(output, collection)
68
+ }
69
+ } catch (error) {
70
+ this.logger.error(chalk.red(`โŒ Error checking ${collectionName.toUpperCase()} outputs:`), error)
71
+ }
72
+ }
73
+
74
+ /**
75
+ * Checks a single output for health
76
+ */
77
+ private async checkOutput (output: Record<string, any>, collection: any): Promise<void> {
78
+ try {
79
+ // Extract the URL from the output
80
+ const url = this.extractURLFromOutput(output)
81
+ if (url === null) {
82
+ return // Skip if no valid URL
83
+ }
84
+
85
+ // Validate domain
86
+ if (!this.isValidDomain(url)) {
87
+ this.logger.log(chalk.yellow(`โš ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`))
88
+ await this.handleUnhealthyOutput(output, collection)
89
+ return
90
+ }
91
+
92
+ // Check health endpoint
93
+ const isHealthy = await this.checkHealthEndpoint(url)
94
+
95
+ if (isHealthy) {
96
+ await this.handleHealthyOutput(output, collection)
97
+ } else {
98
+ await this.handleUnhealthyOutput(output, collection)
99
+ }
100
+ } catch (error) {
101
+ this.logger.error(chalk.red(`โŒ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
102
+ await this.handleUnhealthyOutput(output, collection).catch(() => {})
103
+ }
104
+ }
105
+
106
+ /**
107
+ * Extracts URL from output record
108
+ */
109
+ private extractURLFromOutput (output: Record<string, any>): string | null {
110
+ try {
111
+ // SHIP and SLAP outputs typically have a domain field or URL field
112
+ // Check for common field names
113
+ if (typeof output.domain === 'string') {
114
+ return output.domain
115
+ }
116
+ if (typeof output.url === 'string') {
117
+ return output.url
118
+ }
119
+ if (typeof output.serviceURL === 'string') {
120
+ return output.serviceURL
121
+ }
122
+ // SLAP outputs may have a protocols array with URLs
123
+ if (Array.isArray(output.protocols) && output.protocols.length > 0) {
124
+ const httpsProtocol = output.protocols.find((p: any) =>
125
+ typeof p === 'string' && p.startsWith('https://')
126
+ )
127
+ if (httpsProtocol !== undefined) {
128
+ return httpsProtocol
129
+ }
130
+ }
131
+ return null
132
+ } catch {
133
+ return null
134
+ }
135
+ }
136
+
137
+ /**
138
+ * Validates if a string is a valid domain name
139
+ */
140
+ private isValidDomain (url: string): boolean {
141
+ try {
142
+ // Parse the URL
143
+ const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`)
144
+
145
+ // Check if hostname is valid
146
+ const hostname = parsedURL.hostname
147
+
148
+ // Basic domain validation
149
+ const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i
150
+ const localhostRegex = /^localhost$/i
151
+ const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/
152
+
153
+ return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname)
154
+ } catch {
155
+ return false
156
+ }
157
+ }
158
+
159
+ /**
160
+ * Checks the /health endpoint of a service
161
+ */
162
+ private async checkHealthEndpoint (url: string): Promise<boolean> {
163
+ try {
164
+ // Ensure URL has protocol
165
+ const fullURL = url.startsWith('http') ? url : `https://${url}`
166
+ const healthURL = new URL('/health', fullURL).toString()
167
+
168
+ // Create AbortController for timeout
169
+ const controller = new AbortController()
170
+ const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs)
171
+
172
+ try {
173
+ const response = await fetch(healthURL, {
174
+ method: 'GET',
175
+ signal: controller.signal,
176
+ headers: {
177
+ Accept: 'application/json'
178
+ }
179
+ })
180
+
181
+ clearTimeout(timeout)
182
+
183
+ if (!response.ok) {
184
+ return false
185
+ }
186
+
187
+ const data = await response.json()
188
+ return data?.status === 'ok'
189
+ } catch (error: any) {
190
+ clearTimeout(timeout)
191
+ if (error.name === 'AbortError') {
192
+ this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`))
193
+ }
194
+ return false
195
+ }
196
+ } catch (error) {
197
+ return false
198
+ }
199
+ }
200
+
201
+ /**
202
+ * Handles a healthy output by decrementing its down counter
203
+ */
204
+ private async handleHealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
205
+ try {
206
+ const currentDown = typeof output.down === 'number' ? output.down : 0
207
+
208
+ if (currentDown > 0) {
209
+ // Decrement the down counter
210
+ await collection.updateOne(
211
+ { _id: output._id },
212
+ { $inc: { down: -1 } }
213
+ )
214
+ this.logger.log(chalk.green(`โœ… Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`))
215
+ }
216
+ } catch (error) {
217
+ this.logger.error(chalk.red(`โŒ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
218
+ }
219
+ }
220
+
221
+ /**
222
+ * Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
223
+ */
224
+ private async handleUnhealthyOutput (output: Record<string, any>, collection: any): Promise<void> {
225
+ try {
226
+ const currentDown = typeof output.down === 'number' ? output.down : 0
227
+ const newDown = currentDown + 1
228
+
229
+ this.logger.log(chalk.yellow(`โš ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`))
230
+
231
+ // Check if we should delete the record
232
+ if (newDown >= this.hostDownRevokeScore) {
233
+ this.logger.log(chalk.red(`๐Ÿšซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`))
234
+ await collection.deleteOne({ _id: output._id })
235
+ this.logger.log(chalk.green(`โœ… Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`))
236
+ } else {
237
+ // Increment the down counter
238
+ await collection.updateOne(
239
+ { _id: output._id },
240
+ { $inc: { down: 1 } }
241
+ )
242
+ }
243
+ } catch (error) {
244
+ this.logger.error(chalk.red(`โŒ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error)
245
+ }
246
+ }
247
+ }
@@ -22,6 +22,7 @@ import * as DiscoveryServices from '@bsv/overlay-discovery-services'
22
22
  import chalk from 'chalk'
23
23
  import util from 'util'
24
24
  import { v4 as uuidv4 } from 'uuid'
25
+ import { JanitorService } from './JanitorService.js'
25
26
 
26
27
  /**
27
28
  * Knex database migration.
@@ -145,6 +146,15 @@ export default class OverlayExpress {
145
146
  // If not passed in, we'll generate a random one.
146
147
  private readonly adminToken: string
147
148
 
149
+ // Configuration for the janitor service
150
+ janitorConfig: {
151
+ requestTimeoutMs: number
152
+ hostDownRevokeScore: number
153
+ } = {
154
+ requestTimeoutMs: 10000, // 10 seconds
155
+ hostDownRevokeScore: 3
156
+ }
157
+
148
158
  /**
149
159
  * Constructs an instance of OverlayExpress.
150
160
  * @param name - The name of the service
@@ -189,6 +199,20 @@ export default class OverlayExpress {
189
199
  this.logger.log(chalk.blue('๐Ÿ–ฅ๏ธ Web UI has been configured.'))
190
200
  }
191
201
 
202
+ /**
203
+ * Configures the janitor service parameters
204
+ * @param config - Janitor configuration options
205
+ * - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
206
+ * - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
207
+ */
208
+ configureJanitor (config: Partial<typeof this.janitorConfig>): void {
209
+ this.janitorConfig = {
210
+ ...this.janitorConfig,
211
+ ...config
212
+ }
213
+ this.logger.log(chalk.blue('๐Ÿงน Janitor service has been configured.'))
214
+ }
215
+
192
216
  /**
193
217
  * Configures the logger to be used by the server.
194
218
  * @param logger - A logger object (e.g., console)
@@ -928,6 +952,36 @@ export default class OverlayExpress {
928
952
  })
929
953
  })
930
954
 
955
+ /**
956
+ * Admin route to run the janitor service.
957
+ */
958
+ this.app.post('/admin/janitor', checkAdminAuth as any, (req, res) => {
959
+ ; (async () => {
960
+ try {
961
+ this.ensureMongo()
962
+ const janitor = new JanitorService({
963
+ mongoDb: this.mongoDb,
964
+ logger: this.logger,
965
+ requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
966
+ hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
967
+ })
968
+ await janitor.run()
969
+ return res.status(200).json({ status: 'success', message: 'Janitor run completed' })
970
+ } catch (error) {
971
+ console.error(chalk.red('โŒ Error in /admin/janitor:'), error)
972
+ return res.status(400).json({
973
+ status: 'error',
974
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
975
+ })
976
+ }
977
+ })().catch(() => {
978
+ res.status(500).json({
979
+ status: 'error',
980
+ message: 'Unexpected error'
981
+ })
982
+ })
983
+ })
984
+
931
985
  // Automatically handle migrations
932
986
  const migrationSource = new InMemoryMigrationSource(this.migrationsToRun)
933
987
  const result = await knex.migrate.latest({
@@ -1,77 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.default = `\`
4
- # General Guide
5
-
6
- ## Topic Managers
7
-
8
- A Topic Manager controls addition and removal of transaction outputs to an Overlay Topic. It processes transaction by exposing a /submit endpoint, request should looks something like:
9
-
10
- ### [POST] /submit
11
-
12
- #### Request Headers
13
- \\\`\\\`\\\`json
14
- {
15
- "x-topics": "["tm_messagebox"]", // array of topic identifiers as stringified JSON
16
- "x-includes-off-chain-values": "false" // string "true" or "false"
17
- }
18
- \\\`\\\`\\\`
19
-
20
- #### Request Body
21
- \\\`\\\`\\\`json
22
- [1,1,1,1... ], // transaction in atomic BEEF format as Uint8Array
23
- \\\`\\\`\\\`
24
-
25
- The engine processes this TaggedBEEF and generates a STEAK (Submitted Transaction Execution AcKnowledgment), which is returned to the client.
26
-
27
- #### Response Body
28
- \\\`\\\`\\\`json
29
- {
30
- "tm_messagebox": {
31
- "outputsToAdmit": [0], // which of the outputs in the BEEF to admit
32
- "coinsToRetain": [], // which of the inputs in the BEEF to mark spent but keep within the Topic
33
- "coinsRemoved": [], // which of the inputs in the BEEF to remove from the Topic
34
- },
35
- ... // per topic
36
- }
37
- \\\`\\\`\\\`
38
-
39
- <br />
40
-
41
- ## Lookup Services
42
-
43
- A Lookup Service is a component that enables data retrieval from the Overlay Services ecosystem. It allows clients to search and retrieve transactions that were previously submitted to Topic Managers, using specific criteria or identifiers.
44
-
45
- ### [POST] /lookup
46
-
47
- #### Request Body
48
- \\\`\\\`\\\`json
49
- {
50
- "service": "ls_messagebox", // Lookup Service identifier as string,
51
- "query": "findAll" // query must be a string, JSON queries should be stringified
52
- }
53
- \\\`\\\`\\\`
54
-
55
- #### Response Body
56
- \\\`\\\`\\\`json
57
- {
58
- "type": "output-list",
59
- "outputs": [{
60
- "beef": [1, 1, 1...], // number[]
61
- "outputIndex": 0, // number
62
- "context"?: [1, 1, 1...], // number[]
63
- }>]
64
- }
65
- \\\`\\\`\\\`
66
-
67
- <br />
68
-
69
- ## Summary
70
-
71
- Usually Topic Managers and Lookup Services are paired, but they can be separated for greater flexibility. Together, they form a complete system for data submission, organization, and retrieval within the Overlay Services framework.
72
-
73
- <br />
74
- <br />
75
- \`
76
- `;
77
- //# sourceMappingURL=generalGuide.md.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"generalGuide.md.js","sourceRoot":"","sources":["../../../src/generalGuide.md.ts"],"names":[],"mappings":";;AAAA,kBAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyEd,CAAA"}
@@ -1,75 +0,0 @@
1
- export default `\`
2
- # General Guide
3
-
4
- ## Topic Managers
5
-
6
- A Topic Manager controls addition and removal of transaction outputs to an Overlay Topic. It processes transaction by exposing a /submit endpoint, request should looks something like:
7
-
8
- ### [POST] /submit
9
-
10
- #### Request Headers
11
- \\\`\\\`\\\`json
12
- {
13
- "x-topics": "["tm_messagebox"]", // array of topic identifiers as stringified JSON
14
- "x-includes-off-chain-values": "false" // string "true" or "false"
15
- }
16
- \\\`\\\`\\\`
17
-
18
- #### Request Body
19
- \\\`\\\`\\\`json
20
- [1,1,1,1... ], // transaction in atomic BEEF format as Uint8Array
21
- \\\`\\\`\\\`
22
-
23
- The engine processes this TaggedBEEF and generates a STEAK (Submitted Transaction Execution AcKnowledgment), which is returned to the client.
24
-
25
- #### Response Body
26
- \\\`\\\`\\\`json
27
- {
28
- "tm_messagebox": {
29
- "outputsToAdmit": [0], // which of the outputs in the BEEF to admit
30
- "coinsToRetain": [], // which of the inputs in the BEEF to mark spent but keep within the Topic
31
- "coinsRemoved": [], // which of the inputs in the BEEF to remove from the Topic
32
- },
33
- ... // per topic
34
- }
35
- \\\`\\\`\\\`
36
-
37
- <br />
38
-
39
- ## Lookup Services
40
-
41
- A Lookup Service is a component that enables data retrieval from the Overlay Services ecosystem. It allows clients to search and retrieve transactions that were previously submitted to Topic Managers, using specific criteria or identifiers.
42
-
43
- ### [POST] /lookup
44
-
45
- #### Request Body
46
- \\\`\\\`\\\`json
47
- {
48
- "service": "ls_messagebox", // Lookup Service identifier as string,
49
- "query": "findAll" // query must be a string, JSON queries should be stringified
50
- }
51
- \\\`\\\`\\\`
52
-
53
- #### Response Body
54
- \\\`\\\`\\\`json
55
- {
56
- "type": "output-list",
57
- "outputs": [{
58
- "beef": [1, 1, 1...], // number[]
59
- "outputIndex": 0, // number
60
- "context"?: [1, 1, 1...], // number[]
61
- }>]
62
- }
63
- \\\`\\\`\\\`
64
-
65
- <br />
66
-
67
- ## Summary
68
-
69
- Usually Topic Managers and Lookup Services are paired, but they can be separated for greater flexibility. Together, they form a complete system for data submission, organization, and retrieval within the Overlay Services framework.
70
-
71
- <br />
72
- <br />
73
- \`
74
- `;
75
- //# sourceMappingURL=generalGuide.md.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"generalGuide.md.js","sourceRoot":"","sources":["../../../src/generalGuide.md.ts"],"names":[],"mappings":"AAAA,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyEd,CAAA"}
@@ -1,3 +0,0 @@
1
- declare const _default: "`\n# General Guide\n\n## Topic Managers \n\nA Topic Manager controls addition and removal of transaction outputs to an Overlay Topic. It processes transaction by exposing a /submit endpoint, request should looks something like:\n\n### [POST] /submit\n\n#### Request Headers\n\\`\\`\\`json\n{\n \"x-topics\": \"[\"tm_messagebox\"]\", // array of topic identifiers as stringified JSON\n \"x-includes-off-chain-values\": \"false\" // string \"true\" or \"false\"\n}\n\\`\\`\\`\n\n#### Request Body\n\\`\\`\\`json\n[1,1,1,1... ], // transaction in atomic BEEF format as Uint8Array\n\\`\\`\\`\n\nThe engine processes this TaggedBEEF and generates a STEAK (Submitted Transaction Execution AcKnowledgment), which is returned to the client.\n\n#### Response Body\n\\`\\`\\`json\n{\n \"tm_messagebox\": {\n \"outputsToAdmit\": [0], // which of the outputs in the BEEF to admit\n \"coinsToRetain\": [], // which of the inputs in the BEEF to mark spent but keep within the Topic\n \"coinsRemoved\": [], // which of the inputs in the BEEF to remove from the Topic\n },\n ... // per topic\n}\n\\`\\`\\`\n\n<br />\n\n## Lookup Services\n\nA Lookup Service is a component that enables data retrieval from the Overlay Services ecosystem. It allows clients to search and retrieve transactions that were previously submitted to Topic Managers, using specific criteria or identifiers.\n\n### [POST] /lookup\n\n#### Request Body\n\\`\\`\\`json\n{\n \"service\": \"ls_messagebox\", // Lookup Service identifier as string,\n \"query\": \"findAll\" // query must be a string, JSON queries should be stringified\n}\n\\`\\`\\`\n\n#### Response Body\n\\`\\`\\`json\n{\n \"type\": \"output-list\",\n \"outputs\": [{\n \"beef\": [1, 1, 1...], // number[]\n \"outputIndex\": 0, // number\n \"context\"?: [1, 1, 1...], // number[]\n }>]\n}\n\\`\\`\\`\n\n<br />\n\n## Summary\n\nUsually Topic Managers and Lookup Services are paired, but they can be separated for greater flexibility. Together, they form a complete system for data submission, organization, and retrieval within the Overlay Services framework.\n\n<br />\n<br />\n`\n";
2
- export default _default;
3
- //# sourceMappingURL=generalGuide.md.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"generalGuide.md.d.ts","sourceRoot":"","sources":["../../../src/generalGuide.md.ts"],"names":[],"mappings":";AAAA,wBAyEC"}