@bsv/overlay-express 0.8.2-rc1 โ†’ 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,211 @@
1
+ import chalk from 'chalk';
2
+ /**
3
+ * JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
4
+ * It validates domain names and checks /health endpoints to ensure services are operational.
5
+ *
6
+ * When a service is down, it increments a "down" counter. When healthy, it decrements.
7
+ * If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
8
+ *
9
+ * This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
10
+ */
11
+ export class JanitorService {
12
+ mongoDb;
13
+ logger;
14
+ requestTimeoutMs;
15
+ hostDownRevokeScore;
16
+ constructor(config) {
17
+ this.mongoDb = config.mongoDb;
18
+ this.logger = config.logger ?? console;
19
+ this.requestTimeoutMs = config.requestTimeoutMs ?? 10000; // Default: 10 seconds
20
+ this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3;
21
+ }
22
+ /**
23
+ * Runs a single pass of health checks on all SHIP and SLAP outputs
24
+ */
25
+ async run() {
26
+ this.logger.log(chalk.blue('๐Ÿงน Running janitor health checks...'));
27
+ try {
28
+ // Check SHIP outputs
29
+ await this.checkTopicOutputs('shipRecords');
30
+ // Check SLAP outputs
31
+ await this.checkTopicOutputs('slapRecords');
32
+ this.logger.log(chalk.green('๐Ÿงน Janitor health checks completed'));
33
+ }
34
+ catch (error) {
35
+ this.logger.error(chalk.red('โŒ Error during health checks:'), error);
36
+ throw error;
37
+ }
38
+ }
39
+ /**
40
+ * Checks all outputs for a specific topic
41
+ */
42
+ async checkTopicOutputs(collectionName) {
43
+ try {
44
+ const collection = this.mongoDb.collection(collectionName);
45
+ const outputs = await collection.find({}).toArray();
46
+ this.logger.log(chalk.cyan(`๐Ÿ” Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`));
47
+ for (const output of outputs) {
48
+ await this.checkOutput(output, collection);
49
+ }
50
+ }
51
+ catch (error) {
52
+ this.logger.error(chalk.red(`โŒ Error checking ${collectionName.toUpperCase()} outputs:`), error);
53
+ }
54
+ }
55
+ /**
56
+ * Checks a single output for health
57
+ */
58
+ async checkOutput(output, collection) {
59
+ try {
60
+ // Extract the URL from the output
61
+ const url = this.extractURLFromOutput(output);
62
+ if (url === null) {
63
+ return; // Skip if no valid URL
64
+ }
65
+ // Validate domain
66
+ if (!this.isValidDomain(url)) {
67
+ this.logger.log(chalk.yellow(`โš ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`));
68
+ await this.handleUnhealthyOutput(output, collection);
69
+ return;
70
+ }
71
+ // Check health endpoint
72
+ const isHealthy = await this.checkHealthEndpoint(url);
73
+ if (isHealthy) {
74
+ await this.handleHealthyOutput(output, collection);
75
+ }
76
+ else {
77
+ await this.handleUnhealthyOutput(output, collection);
78
+ }
79
+ }
80
+ catch (error) {
81
+ this.logger.error(chalk.red(`โŒ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
82
+ await this.handleUnhealthyOutput(output, collection).catch(() => { });
83
+ }
84
+ }
85
+ /**
86
+ * Extracts URL from output record
87
+ */
88
+ extractURLFromOutput(output) {
89
+ try {
90
+ // SHIP and SLAP outputs typically have a domain field or URL field
91
+ // Check for common field names
92
+ if (typeof output.domain === 'string') {
93
+ return output.domain;
94
+ }
95
+ if (typeof output.url === 'string') {
96
+ return output.url;
97
+ }
98
+ if (typeof output.serviceURL === 'string') {
99
+ return output.serviceURL;
100
+ }
101
+ // SLAP outputs may have a protocols array with URLs
102
+ if (Array.isArray(output.protocols) && output.protocols.length > 0) {
103
+ const httpsProtocol = output.protocols.find((p) => typeof p === 'string' && p.startsWith('https://'));
104
+ if (httpsProtocol !== undefined) {
105
+ return httpsProtocol;
106
+ }
107
+ }
108
+ return null;
109
+ }
110
+ catch {
111
+ return null;
112
+ }
113
+ }
114
+ /**
115
+ * Validates if a string is a valid domain name
116
+ */
117
+ isValidDomain(url) {
118
+ try {
119
+ // Parse the URL
120
+ const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`);
121
+ // Check if hostname is valid
122
+ const hostname = parsedURL.hostname;
123
+ // Basic domain validation
124
+ const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i;
125
+ const localhostRegex = /^localhost$/i;
126
+ const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/;
127
+ return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname);
128
+ }
129
+ catch {
130
+ return false;
131
+ }
132
+ }
133
+ /**
134
+ * Checks the /health endpoint of a service
135
+ */
136
+ async checkHealthEndpoint(url) {
137
+ try {
138
+ // Ensure URL has protocol
139
+ const fullURL = url.startsWith('http') ? url : `https://${url}`;
140
+ const healthURL = new URL('/health', fullURL).toString();
141
+ // Create AbortController for timeout
142
+ const controller = new AbortController();
143
+ const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs);
144
+ try {
145
+ const response = await fetch(healthURL, {
146
+ method: 'GET',
147
+ signal: controller.signal,
148
+ headers: {
149
+ Accept: 'application/json'
150
+ }
151
+ });
152
+ clearTimeout(timeout);
153
+ if (!response.ok) {
154
+ return false;
155
+ }
156
+ const data = await response.json();
157
+ return data?.status === 'ok';
158
+ }
159
+ catch (error) {
160
+ clearTimeout(timeout);
161
+ if (error.name === 'AbortError') {
162
+ this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`));
163
+ }
164
+ return false;
165
+ }
166
+ }
167
+ catch (error) {
168
+ return false;
169
+ }
170
+ }
171
+ /**
172
+ * Handles a healthy output by decrementing its down counter
173
+ */
174
+ async handleHealthyOutput(output, collection) {
175
+ try {
176
+ const currentDown = typeof output.down === 'number' ? output.down : 0;
177
+ if (currentDown > 0) {
178
+ // Decrement the down counter
179
+ await collection.updateOne({ _id: output._id }, { $inc: { down: -1 } });
180
+ this.logger.log(chalk.green(`โœ… Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`));
181
+ }
182
+ }
183
+ catch (error) {
184
+ this.logger.error(chalk.red(`โŒ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
185
+ }
186
+ }
187
+ /**
188
+ * Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
189
+ */
190
+ async handleUnhealthyOutput(output, collection) {
191
+ try {
192
+ const currentDown = typeof output.down === 'number' ? output.down : 0;
193
+ const newDown = currentDown + 1;
194
+ this.logger.log(chalk.yellow(`โš ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`));
195
+ // Check if we should delete the record
196
+ if (newDown >= this.hostDownRevokeScore) {
197
+ this.logger.log(chalk.red(`๐Ÿšซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`));
198
+ await collection.deleteOne({ _id: output._id });
199
+ this.logger.log(chalk.green(`โœ… Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`));
200
+ }
201
+ else {
202
+ // Increment the down counter
203
+ await collection.updateOne({ _id: output._id }, { $inc: { down: 1 } });
204
+ }
205
+ }
206
+ catch (error) {
207
+ this.logger.error(chalk.red(`โŒ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
208
+ }
209
+ }
210
+ }
211
+ //# sourceMappingURL=JanitorService.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"JanitorService.js","sourceRoot":"","sources":["../../../src/JanitorService.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAA;AAYzB;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IACR,OAAO,CAAI;IACX,MAAM,CAAgB;IACtB,gBAAgB,CAAQ;IACxB,mBAAmB,CAAQ;IAE5C,YAAa,MAAqB;QAChC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAA;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,KAAK,CAAA,CAAC,sBAAsB;QAC/E,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAElE,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,EAAE,KAAK,CAAC,CAAA;YACpE,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAE,cAAsB;QACrD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;YAEnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAA;YAEvG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,CAAA;QAClG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAE,MAA2B,EAAE,UAAe;QACrE,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;YAC7C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,OAAM,CAAC,uBAAuB;YAChC,CAAC;YAED,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;gBAC1H,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;gBACpD,OAAM;YACR,CAAC;YAED,wBAAwB;YACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAErD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;YACpH,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAE,MAA2B;QACvD,IAAI,CAAC;YACH,mEAAmE;YACnE,+BAA+B;YAC/B,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACtC,OAAO,MAAM,CAAC,MAAM,CAAA;YACtB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACnC,OAAO,MAAM,CAAC,GAAG,CAAA;YACnB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC,UAAU,CAAA;YAC1B,CAAC;YACD,oDAAoD;YACpD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnE,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CACrD,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAClD,CAAA;gBACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;oBAChC,OAAO,aAAa,CAAA;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAE,GAAW;QAChC,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAA;YAE1E,6BAA6B;YAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;YAEnC,0BAA0B;YAC1B,MAAM,WAAW,GAAG,+EAA+E,CAAA;YACnG,MAAM,cAAc,GAAG,cAAc,CAAA;YACrC,MAAM,SAAS,GAAG,yBAAyB,CAAA;YAE3C,OAAO,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAChG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,GAAW;QAC5C,IAAI,CAAC;YACH,0BAA0B;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;YAExD,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAE3E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBACtC,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,MAAM,EAAE,kBAAkB;qBAC3B;iBACF,CAAC,CAAA;gBAEF,YAAY,CAAC,OAAO,CAAC,CAAA;gBAErB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,KAAK,CAAA;gBACd,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAClC,OAAO,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;YAC9B,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,CAAA;gBACrB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC,CAAA;gBAC3E,CAAC;gBACD,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,MAA2B,EAAE,UAAe;QAC7E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAErE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBACpB,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACvB,CAAA;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,sBAAsB,WAAW,OAAO,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;YACvJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9H,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAE,MAA2B,EAAE,UAAe;QAC/E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YACrE,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAA;YAE/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,WAAW,OAAO,OAAO,GAAG,CAAC,CAAC,CAAA;YAEjJ,uCAAuC;YACvC,IAAI,OAAO,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,iBAAiB,OAAO,OAAO,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAA;gBAC7J,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iCAAiC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;YACpH,CAAC;iBAAM,CAAC;gBACN,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CACtB,CAAA;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAChI,CAAC;IACH,CAAC;CACF"}
@@ -1,7 +1,7 @@
1
1
  import express from 'express';
2
2
  import bodyParser from 'body-parser';
3
3
  import { Engine, KnexStorage, KnexStorageMigrations } from '@bsv/overlay';
4
- import { ARC, MerklePath, WhatsOnChain, HTTPSOverlayBroadcastFacilitator, DEFAULT_TESTNET_SLAP_TRACKERS, DEFAULT_SLAP_TRACKERS, Utils } from '@bsv/sdk';
4
+ import { ARC, MerklePath, WhatsOnChain, HTTPSOverlayBroadcastFacilitator, DEFAULT_TESTNET_SLAP_TRACKERS, DEFAULT_SLAP_TRACKERS, Utils, Beef, Transaction } from '@bsv/sdk';
5
5
  import Knex from 'knex';
6
6
  import { MongoClient } from 'mongodb';
7
7
  import makeUserInterface from './makeUserInterface.js';
@@ -9,6 +9,7 @@ import * as DiscoveryServices from '@bsv/overlay-discovery-services';
9
9
  import chalk from 'chalk';
10
10
  import util from 'util';
11
11
  import { v4 as uuidv4 } from 'uuid';
12
+ import { JanitorService } from './JanitorService.js';
12
13
  /**
13
14
  * In-memory migration source for Knex migrations.
14
15
  * Allows running migrations defined in code rather than files.
@@ -59,18 +60,18 @@ export default class OverlayExpress {
59
60
  // Logger (defaults to console)
60
61
  logger = console;
61
62
  // Knex (SQL) database
62
- knex = undefined;
63
+ knex = {};
63
64
  // Knex migrations to run
64
65
  migrationsToRun = [];
65
66
  // MongoDB database
66
- mongoDb = undefined;
67
+ mongoDb = {};
67
68
  // Network ('main' or 'test')
68
69
  network = 'main';
69
70
  // If no custom ChainTracker is configured, default is a WhatsOnChain instance
70
71
  // (We keep a property for it, so we can pass it to Engine)
71
72
  chainTracker = new WhatsOnChain(this.network);
72
73
  // The Overlay Engine
73
- engine = undefined;
74
+ engine = {};
74
75
  // Configured Topic Managers
75
76
  managers = {};
76
77
  // Configured Lookup Services
@@ -90,6 +91,11 @@ export default class OverlayExpress {
90
91
  // The administrative Bearer token used for the admin routes.
91
92
  // If not passed in, we'll generate a random one.
92
93
  adminToken;
94
+ // Configuration for the janitor service
95
+ janitorConfig = {
96
+ requestTimeoutMs: 10000, // 10 seconds
97
+ hostDownRevokeScore: 3
98
+ };
93
99
  /**
94
100
  * Constructs an instance of OverlayExpress.
95
101
  * @param name - The name of the service
@@ -104,7 +110,7 @@ export default class OverlayExpress {
104
110
  this.advertisableFQDN = advertisableFQDN;
105
111
  this.app = express();
106
112
  this.logger.log(chalk.green.bold(`${name} constructed ๐ŸŽ‰`));
107
- this.adminToken = adminToken || uuidv4(); // generate random if not provided
113
+ this.adminToken = adminToken ?? uuidv4(); // generate random if not provided
108
114
  }
109
115
  /**
110
116
  * Returns the current admin token in case you need to programmatically retrieve or display it.
@@ -128,6 +134,19 @@ export default class OverlayExpress {
128
134
  this.webUIConfig = config;
129
135
  this.logger.log(chalk.blue('๐Ÿ–ฅ๏ธ Web UI has been configured.'));
130
136
  }
137
+ /**
138
+ * Configures the janitor service parameters
139
+ * @param config - Janitor configuration options
140
+ * - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
141
+ * - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
142
+ */
143
+ configureJanitor(config) {
144
+ this.janitorConfig = {
145
+ ...this.janitorConfig,
146
+ ...config
147
+ };
148
+ this.logger.log(chalk.blue('๐Ÿงน Janitor service has been configured.'));
149
+ }
131
150
  /**
132
151
  * Configures the logger to be used by the server.
133
152
  * @param logger - A logger object (e.g., console)
@@ -292,7 +311,7 @@ export default class OverlayExpress {
292
311
  }
293
312
  else {
294
313
  // If the user provided a syncConfiguration, use that. Otherwise default to an empty object.
295
- syncConfig = this.engineConfig.syncConfiguration || {};
314
+ syncConfig = this.engineConfig.syncConfiguration ?? {};
296
315
  }
297
316
  // Build the actual Storage
298
317
  const storage = new KnexStorage(this.knex);
@@ -354,7 +373,9 @@ export default class OverlayExpress {
354
373
  // overlayBroadcastFacilitator
355
374
  this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(),
356
375
  // logger
357
- this.logger);
376
+ this.logger,
377
+ // suppressDefaultSyncAdvertisements
378
+ this.engineConfig.suppressDefaultSyncAdvertisements ?? true);
358
379
  this.logger.log(chalk.green('๐Ÿš€ Engine has been configured.'));
359
380
  }
360
381
  /**
@@ -399,12 +420,12 @@ export default class OverlayExpress {
399
420
  this.app.use((req, res, next) => {
400
421
  const startTime = Date.now();
401
422
  // Log incoming request details
402
- this.logger.log(chalk.magenta.bold(`๐Ÿ“ฅ Incoming Request: ${req.method} ${req.originalUrl}`));
423
+ this.logger.log(chalk.magenta.bold(`๐Ÿ“ฅ Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`));
403
424
  // Pretty-print headers
404
- this.logger.log(chalk.cyan(`Headers:`));
425
+ this.logger.log(chalk.cyan('Headers:'));
405
426
  this.logger.log(util.inspect(req.headers, { colors: true, depth: null }));
406
427
  // Handle request body
407
- if (req.body && Object.keys(req.body).length > 0) {
428
+ if (req.body != null && Object.keys(req.body).length > 0) {
408
429
  let bodyContent;
409
430
  let bodyString;
410
431
  if (typeof req.body === 'object') {
@@ -414,13 +435,13 @@ export default class OverlayExpress {
414
435
  bodyString = req.body.toString('utf8');
415
436
  }
416
437
  else {
417
- bodyString = req.body.toString();
438
+ bodyString = String(req.body);
418
439
  }
419
440
  if (bodyString.length > 280) {
420
- bodyContent = chalk.yellow(`(Body too long to display, length: ${bodyString.length} characters)`);
441
+ bodyContent = chalk.yellow(`(Body too long to display, length: ${String(bodyString.length)} characters)`);
421
442
  }
422
443
  else {
423
- bodyContent = chalk.green(`Request Body:\n${bodyString}`);
444
+ bodyContent = chalk.green(`Request Body:\n${String(bodyString)}`);
424
445
  }
425
446
  this.logger.log(bodyContent);
426
447
  }
@@ -434,11 +455,11 @@ export default class OverlayExpress {
434
455
  // Log outgoing response details after the response is finished
435
456
  res.on('finish', () => {
436
457
  const duration = Date.now() - startTime;
437
- this.logger.log(chalk.magenta.bold(`๐Ÿ“ค Outgoing Response: ${req.method} ${req.originalUrl} - Status: ${res.statusCode} - Duration: ${duration}ms`));
438
- this.logger.log(chalk.cyan(`Response Headers:`));
458
+ this.logger.log(chalk.magenta.bold(`๐Ÿ“ค Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`));
459
+ this.logger.log(chalk.cyan('Response Headers:'));
439
460
  this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }));
440
461
  // Handle response body
441
- if (responseBody) {
462
+ if (responseBody != null) {
442
463
  let bodyContent;
443
464
  let bodyString;
444
465
  if (typeof responseBody === 'object') {
@@ -447,14 +468,17 @@ export default class OverlayExpress {
447
468
  else if (Buffer.isBuffer(responseBody)) {
448
469
  bodyString = responseBody.toString('utf8');
449
470
  }
471
+ else if (typeof responseBody === 'string') {
472
+ bodyString = responseBody;
473
+ }
450
474
  else {
451
- bodyString = responseBody.toString();
475
+ bodyString = String(responseBody);
452
476
  }
453
477
  if (bodyString.length > 280) {
454
- bodyContent = chalk.yellow(`(Response body too long to display, length: ${bodyString.length} characters)`);
478
+ bodyContent = chalk.yellow(`(Response body too long to display, length: ${String(bodyString.length)} characters)`);
455
479
  }
456
480
  else {
457
- bodyContent = chalk.green(`Response Body:\n${bodyString}`);
481
+ bodyContent = chalk.green(`Response Body:\n${String(bodyString)}`);
458
482
  }
459
483
  this.logger.log(bodyContent);
460
484
  }
@@ -481,6 +505,10 @@ export default class OverlayExpress {
481
505
  res.set('content-type', 'text/html');
482
506
  res.send(makeUserInterface(this.webUIConfig));
483
507
  });
508
+ // Serve a health check endpoint
509
+ this.app.get('/health', (_, res) => {
510
+ res.status(200).json({ status: 'ok' });
511
+ });
484
512
  // List hosted topic managers and lookup services
485
513
  this.app.get('/listTopicManagers', (_, res) => {
486
514
  ;
@@ -598,7 +626,7 @@ export default class OverlayExpress {
598
626
  const steak = await engine.submit(taggedBEEF, (steak) => {
599
627
  responseSent = true;
600
628
  return res.status(200).json(steak);
601
- });
629
+ }, 'current-tx', offChainValues);
602
630
  if (!responseSent) {
603
631
  res.status(200).json(steak);
604
632
  }
@@ -621,11 +649,52 @@ export default class OverlayExpress {
621
649
  ;
622
650
  (async () => {
623
651
  try {
624
- const result = await engine.lookup(req.body);
625
- return res.status(200).json(result);
652
+ // Check for aggregation header to determine response format
653
+ const aggregationHeader = req.headers['x-aggregation'];
654
+ const shouldReturnBinary = aggregationHeader === 'yes';
655
+ // Validate request body structure
656
+ const lookupRequest = req.body;
657
+ if (typeof lookupRequest.service !== 'string' || lookupRequest.query === undefined) {
658
+ return res.status(400).json({
659
+ status: 'error',
660
+ message: 'Invalid request: body must contain "service" (string) and "query" fields'
661
+ });
662
+ }
663
+ const result = await engine.lookup(lookupRequest);
664
+ if (!shouldReturnBinary) {
665
+ // Return JSON response (default behavior)
666
+ return res.status(200).json(result);
667
+ }
668
+ const beef = new Beef();
669
+ const outputs = result.outputs;
670
+ // Serialize in the format expected by LookupResolver
671
+ const writer = new Utils.Writer();
672
+ // Write number of outpoints
673
+ writer.writeVarIntNum(outputs.length);
674
+ // Write each outpoint data
675
+ for (const output of outputs) {
676
+ const tx = Transaction.fromBEEF(output.beef);
677
+ // Write txid (32 bytes)
678
+ writer.write(tx.id());
679
+ // Write outputIndex
680
+ writer.writeVarIntNum(output.outputIndex);
681
+ // Write context length and data
682
+ if ((output.context != null) && output.context.length > 0) {
683
+ writer.writeVarIntNum(output.context.length);
684
+ writer.write(output.context);
685
+ }
686
+ else {
687
+ writer.writeVarIntNum(0);
688
+ }
689
+ beef.mergeTransaction(tx);
690
+ }
691
+ // Write the beef data
692
+ writer.write(beef.toBinary());
693
+ res.setHeader('Content-Type', 'application/octet-stream');
694
+ return res.status(200).send(Buffer.from(writer.toArray()));
626
695
  }
627
696
  catch (error) {
628
- console.error(chalk.red('โŒ Error in /lookup:'), error);
697
+ console.error(chalk.red('Error in /lookup:'), error);
629
698
  return res.status(400).json({
630
699
  status: 'error',
631
700
  message: error instanceof Error ? error.message : 'An unknown error occurred'
@@ -726,7 +795,7 @@ export default class OverlayExpress {
726
795
  * Middleware for checking the admin bearer token.
727
796
  */
728
797
  const checkAdminAuth = (req, res, next) => {
729
- const authHeader = req.headers['authorization'];
798
+ const authHeader = req.headers.authorization;
730
799
  if (typeof authHeader !== 'string' || !authHeader.startsWith('Bearer ')) {
731
800
  res.status(401).json({ status: 'error', message: 'Unauthorized: Missing Bearer token' });
732
801
  return;
@@ -824,6 +893,37 @@ export default class OverlayExpress {
824
893
  });
825
894
  });
826
895
  });
896
+ /**
897
+ * Admin route to run the janitor service.
898
+ */
899
+ this.app.post('/admin/janitor', checkAdminAuth, (req, res) => {
900
+ ;
901
+ (async () => {
902
+ try {
903
+ this.ensureMongo();
904
+ const janitor = new JanitorService({
905
+ mongoDb: this.mongoDb,
906
+ logger: this.logger,
907
+ requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
908
+ hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
909
+ });
910
+ await janitor.run();
911
+ return res.status(200).json({ status: 'success', message: 'Janitor run completed' });
912
+ }
913
+ catch (error) {
914
+ console.error(chalk.red('โŒ Error in /admin/janitor:'), error);
915
+ return res.status(400).json({
916
+ status: 'error',
917
+ message: error instanceof Error ? error.message : 'An unknown error occurred'
918
+ });
919
+ }
920
+ })().catch(() => {
921
+ res.status(500).json({
922
+ status: 'error',
923
+ message: 'Unexpected error'
924
+ });
925
+ });
926
+ });
827
927
  // Automatically handle migrations
828
928
  const migrationSource = new InMemoryMigrationSource(this.migrationsToRun);
829
929
  const result = await knex.migrate.latest({
@@ -871,7 +971,7 @@ export default class OverlayExpress {
871
971
  this.logger.log(chalk.yellow(`${this.name} will not sync because GASP has been disabled.`));
872
972
  }
873
973
  // Start listening on the configured port
874
- this.app.listen(this.port, async () => {
974
+ this.app.listen(this.port, () => {
875
975
  this.logger.log(chalk.green.bold(`๐ŸŽง ${this.name} is ready and listening on local port ${this.port}`));
876
976
  });
877
977
  }