@bsv/overlay-express 0.8.2-rc1 โ 0.8.2-rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/package.json +13 -13
- package/dist/cjs/src/JanitorService.js +215 -0
- package/dist/cjs/src/JanitorService.js.map +1 -0
- package/dist/cjs/src/OverlayExpress.js +130 -30
- package/dist/cjs/src/OverlayExpress.js.map +1 -1
- package/dist/cjs/src/generalGuide.md.js +77 -0
- package/dist/cjs/src/generalGuide.md.js.map +1 -0
- package/dist/cjs/src/makeUserInterface.js +290 -93
- package/dist/cjs/src/makeUserInterface.js.map +1 -1
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/esm/src/JanitorService.js +211 -0
- package/dist/esm/src/JanitorService.js.map +1 -0
- package/dist/esm/src/OverlayExpress.js +125 -25
- package/dist/esm/src/OverlayExpress.js.map +1 -1
- package/dist/esm/src/generalGuide.md.js +75 -0
- package/dist/esm/src/generalGuide.md.js.map +1 -0
- package/dist/esm/src/makeUserInterface.js +287 -93
- package/dist/esm/src/makeUserInterface.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/types/src/JanitorService.d.ts +59 -0
- package/dist/types/src/JanitorService.d.ts.map +1 -0
- package/dist/types/src/OverlayExpress.d.ts +15 -3
- package/dist/types/src/OverlayExpress.d.ts.map +1 -1
- package/dist/types/src/generalGuide.md.d.ts +3 -0
- package/dist/types/src/generalGuide.md.d.ts.map +1 -0
- package/dist/types/src/makeUserInterface.d.ts +7 -2
- package/dist/types/src/makeUserInterface.d.ts.map +1 -1
- package/dist/types/tsconfig.types.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/OverlayExpress.ts +55 -4
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
/**
|
|
3
|
+
* JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
|
|
4
|
+
* It validates domain names and checks /health endpoints to ensure services are operational.
|
|
5
|
+
*
|
|
6
|
+
* When a service is down, it increments a "down" counter. When healthy, it decrements.
|
|
7
|
+
* If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
|
|
8
|
+
*
|
|
9
|
+
* This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
|
|
10
|
+
*/
|
|
11
|
+
export class JanitorService {
|
|
12
|
+
mongoDb;
|
|
13
|
+
logger;
|
|
14
|
+
requestTimeoutMs;
|
|
15
|
+
hostDownRevokeScore;
|
|
16
|
+
constructor(config) {
|
|
17
|
+
this.mongoDb = config.mongoDb;
|
|
18
|
+
this.logger = config.logger ?? console;
|
|
19
|
+
this.requestTimeoutMs = config.requestTimeoutMs ?? 10000; // Default: 10 seconds
|
|
20
|
+
this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Runs a single pass of health checks on all SHIP and SLAP outputs
|
|
24
|
+
*/
|
|
25
|
+
async run() {
|
|
26
|
+
this.logger.log(chalk.blue('๐งน Running janitor health checks...'));
|
|
27
|
+
try {
|
|
28
|
+
// Check SHIP outputs
|
|
29
|
+
await this.checkTopicOutputs('shipRecords');
|
|
30
|
+
// Check SLAP outputs
|
|
31
|
+
await this.checkTopicOutputs('slapRecords');
|
|
32
|
+
this.logger.log(chalk.green('๐งน Janitor health checks completed'));
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
this.logger.error(chalk.red('โ Error during health checks:'), error);
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Checks all outputs for a specific topic
|
|
41
|
+
*/
|
|
42
|
+
async checkTopicOutputs(collectionName) {
|
|
43
|
+
try {
|
|
44
|
+
const collection = this.mongoDb.collection(collectionName);
|
|
45
|
+
const outputs = await collection.find({}).toArray();
|
|
46
|
+
this.logger.log(chalk.cyan(`๐ Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`));
|
|
47
|
+
for (const output of outputs) {
|
|
48
|
+
await this.checkOutput(output, collection);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
this.logger.error(chalk.red(`โ Error checking ${collectionName.toUpperCase()} outputs:`), error);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Checks a single output for health
|
|
57
|
+
*/
|
|
58
|
+
async checkOutput(output, collection) {
|
|
59
|
+
try {
|
|
60
|
+
// Extract the URL from the output
|
|
61
|
+
const url = this.extractURLFromOutput(output);
|
|
62
|
+
if (url === null) {
|
|
63
|
+
return; // Skip if no valid URL
|
|
64
|
+
}
|
|
65
|
+
// Validate domain
|
|
66
|
+
if (!this.isValidDomain(url)) {
|
|
67
|
+
this.logger.log(chalk.yellow(`โ ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`));
|
|
68
|
+
await this.handleUnhealthyOutput(output, collection);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
// Check health endpoint
|
|
72
|
+
const isHealthy = await this.checkHealthEndpoint(url);
|
|
73
|
+
if (isHealthy) {
|
|
74
|
+
await this.handleHealthyOutput(output, collection);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
await this.handleUnhealthyOutput(output, collection);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
this.logger.error(chalk.red(`โ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
82
|
+
await this.handleUnhealthyOutput(output, collection).catch(() => { });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Extracts URL from output record
|
|
87
|
+
*/
|
|
88
|
+
extractURLFromOutput(output) {
|
|
89
|
+
try {
|
|
90
|
+
// SHIP and SLAP outputs typically have a domain field or URL field
|
|
91
|
+
// Check for common field names
|
|
92
|
+
if (typeof output.domain === 'string') {
|
|
93
|
+
return output.domain;
|
|
94
|
+
}
|
|
95
|
+
if (typeof output.url === 'string') {
|
|
96
|
+
return output.url;
|
|
97
|
+
}
|
|
98
|
+
if (typeof output.serviceURL === 'string') {
|
|
99
|
+
return output.serviceURL;
|
|
100
|
+
}
|
|
101
|
+
// SLAP outputs may have a protocols array with URLs
|
|
102
|
+
if (Array.isArray(output.protocols) && output.protocols.length > 0) {
|
|
103
|
+
const httpsProtocol = output.protocols.find((p) => typeof p === 'string' && p.startsWith('https://'));
|
|
104
|
+
if (httpsProtocol !== undefined) {
|
|
105
|
+
return httpsProtocol;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Validates if a string is a valid domain name
|
|
116
|
+
*/
|
|
117
|
+
isValidDomain(url) {
|
|
118
|
+
try {
|
|
119
|
+
// Parse the URL
|
|
120
|
+
const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`);
|
|
121
|
+
// Check if hostname is valid
|
|
122
|
+
const hostname = parsedURL.hostname;
|
|
123
|
+
// Basic domain validation
|
|
124
|
+
const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i;
|
|
125
|
+
const localhostRegex = /^localhost$/i;
|
|
126
|
+
const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/;
|
|
127
|
+
return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Checks the /health endpoint of a service
|
|
135
|
+
*/
|
|
136
|
+
async checkHealthEndpoint(url) {
|
|
137
|
+
try {
|
|
138
|
+
// Ensure URL has protocol
|
|
139
|
+
const fullURL = url.startsWith('http') ? url : `https://${url}`;
|
|
140
|
+
const healthURL = new URL('/health', fullURL).toString();
|
|
141
|
+
// Create AbortController for timeout
|
|
142
|
+
const controller = new AbortController();
|
|
143
|
+
const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs);
|
|
144
|
+
try {
|
|
145
|
+
const response = await fetch(healthURL, {
|
|
146
|
+
method: 'GET',
|
|
147
|
+
signal: controller.signal,
|
|
148
|
+
headers: {
|
|
149
|
+
Accept: 'application/json'
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
clearTimeout(timeout);
|
|
153
|
+
if (!response.ok) {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
const data = await response.json();
|
|
157
|
+
return data?.status === 'ok';
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
clearTimeout(timeout);
|
|
161
|
+
if (error.name === 'AbortError') {
|
|
162
|
+
this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`));
|
|
163
|
+
}
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
return false;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Handles a healthy output by decrementing its down counter
|
|
173
|
+
*/
|
|
174
|
+
async handleHealthyOutput(output, collection) {
|
|
175
|
+
try {
|
|
176
|
+
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
177
|
+
if (currentDown > 0) {
|
|
178
|
+
// Decrement the down counter
|
|
179
|
+
await collection.updateOne({ _id: output._id }, { $inc: { down: -1 } });
|
|
180
|
+
this.logger.log(chalk.green(`โ
Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`));
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
catch (error) {
|
|
184
|
+
this.logger.error(chalk.red(`โ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
|
|
189
|
+
*/
|
|
190
|
+
async handleUnhealthyOutput(output, collection) {
|
|
191
|
+
try {
|
|
192
|
+
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
193
|
+
const newDown = currentDown + 1;
|
|
194
|
+
this.logger.log(chalk.yellow(`โ ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`));
|
|
195
|
+
// Check if we should delete the record
|
|
196
|
+
if (newDown >= this.hostDownRevokeScore) {
|
|
197
|
+
this.logger.log(chalk.red(`๐ซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`));
|
|
198
|
+
await collection.deleteOne({ _id: output._id });
|
|
199
|
+
this.logger.log(chalk.green(`โ
Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`));
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
// Increment the down counter
|
|
203
|
+
await collection.updateOne({ _id: output._id }, { $inc: { down: 1 } });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
this.logger.error(chalk.red(`โ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
//# sourceMappingURL=JanitorService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JanitorService.js","sourceRoot":"","sources":["../../../src/JanitorService.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAA;AAYzB;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IACR,OAAO,CAAI;IACX,MAAM,CAAgB;IACtB,gBAAgB,CAAQ;IACxB,mBAAmB,CAAQ;IAE5C,YAAa,MAAqB;QAChC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAA;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,KAAK,CAAA,CAAC,sBAAsB;QAC/E,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAElE,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,EAAE,KAAK,CAAC,CAAA;YACpE,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAE,cAAsB;QACrD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;YAEnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAA;YAEvG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,CAAA;QAClG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAE,MAA2B,EAAE,UAAe;QACrE,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;YAC7C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,OAAM,CAAC,uBAAuB;YAChC,CAAC;YAED,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;gBAC1H,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;gBACpD,OAAM;YACR,CAAC;YAED,wBAAwB;YACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAErD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;YACpH,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAE,MAA2B;QACvD,IAAI,CAAC;YACH,mEAAmE;YACnE,+BAA+B;YAC/B,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACtC,OAAO,MAAM,CAAC,MAAM,CAAA;YACtB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACnC,OAAO,MAAM,CAAC,GAAG,CAAA;YACnB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC,UAAU,CAAA;YAC1B,CAAC;YACD,oDAAoD;YACpD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnE,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CACrD,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAClD,CAAA;gBACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;oBAChC,OAAO,aAAa,CAAA;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAE,GAAW;QAChC,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAA;YAE1E,6BAA6B;YAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;YAEnC,0BAA0B;YAC1B,MAAM,WAAW,GAAG,+EAA+E,CAAA;YACnG,MAAM,cAAc,GAAG,cAAc,CAAA;YACrC,MAAM,SAAS,GAAG,yBAAyB,CAAA;YAE3C,OAAO,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAChG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,GAAW;QAC5C,IAAI,CAAC;YACH,0BAA0B;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;YAExD,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAE3E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBACtC,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,MAAM,EAAE,kBAAkB;qBAC3B;iBACF,CAAC,CAAA;gBAEF,YAAY,CAAC,OAAO,CAAC,CAAA;gBAErB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,KAAK,CAAA;gBACd,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAClC,OAAO,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;YAC9B,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,CAAA;gBACrB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC,CAAA;gBAC3E,CAAC;gBACD,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,MAA2B,EAAE,UAAe;QAC7E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAErE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBACpB,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACvB,CAAA;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,sBAAsB,WAAW,OAAO,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;YACvJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9H,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAE,MAA2B,EAAE,UAAe;QAC/E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YACrE,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAA;YAE/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,WAAW,OAAO,OAAO,GAAG,CAAC,CAAC,CAAA;YAEjJ,uCAAuC;YACvC,IAAI,OAAO,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,iBAAiB,OAAO,OAAO,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAA;gBAC7J,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iCAAiC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;YACpH,CAAC;iBAAM,CAAC;gBACN,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CACtB,CAAA;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAChI,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import express from 'express';
|
|
2
2
|
import bodyParser from 'body-parser';
|
|
3
3
|
import { Engine, KnexStorage, KnexStorageMigrations } from '@bsv/overlay';
|
|
4
|
-
import { ARC, MerklePath, WhatsOnChain, HTTPSOverlayBroadcastFacilitator, DEFAULT_TESTNET_SLAP_TRACKERS, DEFAULT_SLAP_TRACKERS, Utils } from '@bsv/sdk';
|
|
4
|
+
import { ARC, MerklePath, WhatsOnChain, HTTPSOverlayBroadcastFacilitator, DEFAULT_TESTNET_SLAP_TRACKERS, DEFAULT_SLAP_TRACKERS, Utils, Beef, Transaction } from '@bsv/sdk';
|
|
5
5
|
import Knex from 'knex';
|
|
6
6
|
import { MongoClient } from 'mongodb';
|
|
7
7
|
import makeUserInterface from './makeUserInterface.js';
|
|
@@ -9,6 +9,7 @@ import * as DiscoveryServices from '@bsv/overlay-discovery-services';
|
|
|
9
9
|
import chalk from 'chalk';
|
|
10
10
|
import util from 'util';
|
|
11
11
|
import { v4 as uuidv4 } from 'uuid';
|
|
12
|
+
import { JanitorService } from './JanitorService.js';
|
|
12
13
|
/**
|
|
13
14
|
* In-memory migration source for Knex migrations.
|
|
14
15
|
* Allows running migrations defined in code rather than files.
|
|
@@ -59,18 +60,18 @@ export default class OverlayExpress {
|
|
|
59
60
|
// Logger (defaults to console)
|
|
60
61
|
logger = console;
|
|
61
62
|
// Knex (SQL) database
|
|
62
|
-
knex =
|
|
63
|
+
knex = {};
|
|
63
64
|
// Knex migrations to run
|
|
64
65
|
migrationsToRun = [];
|
|
65
66
|
// MongoDB database
|
|
66
|
-
mongoDb =
|
|
67
|
+
mongoDb = {};
|
|
67
68
|
// Network ('main' or 'test')
|
|
68
69
|
network = 'main';
|
|
69
70
|
// If no custom ChainTracker is configured, default is a WhatsOnChain instance
|
|
70
71
|
// (We keep a property for it, so we can pass it to Engine)
|
|
71
72
|
chainTracker = new WhatsOnChain(this.network);
|
|
72
73
|
// The Overlay Engine
|
|
73
|
-
engine =
|
|
74
|
+
engine = {};
|
|
74
75
|
// Configured Topic Managers
|
|
75
76
|
managers = {};
|
|
76
77
|
// Configured Lookup Services
|
|
@@ -90,6 +91,11 @@ export default class OverlayExpress {
|
|
|
90
91
|
// The administrative Bearer token used for the admin routes.
|
|
91
92
|
// If not passed in, we'll generate a random one.
|
|
92
93
|
adminToken;
|
|
94
|
+
// Configuration for the janitor service
|
|
95
|
+
janitorConfig = {
|
|
96
|
+
requestTimeoutMs: 10000, // 10 seconds
|
|
97
|
+
hostDownRevokeScore: 3
|
|
98
|
+
};
|
|
93
99
|
/**
|
|
94
100
|
* Constructs an instance of OverlayExpress.
|
|
95
101
|
* @param name - The name of the service
|
|
@@ -104,7 +110,7 @@ export default class OverlayExpress {
|
|
|
104
110
|
this.advertisableFQDN = advertisableFQDN;
|
|
105
111
|
this.app = express();
|
|
106
112
|
this.logger.log(chalk.green.bold(`${name} constructed ๐`));
|
|
107
|
-
this.adminToken = adminToken
|
|
113
|
+
this.adminToken = adminToken ?? uuidv4(); // generate random if not provided
|
|
108
114
|
}
|
|
109
115
|
/**
|
|
110
116
|
* Returns the current admin token in case you need to programmatically retrieve or display it.
|
|
@@ -128,6 +134,19 @@ export default class OverlayExpress {
|
|
|
128
134
|
this.webUIConfig = config;
|
|
129
135
|
this.logger.log(chalk.blue('๐ฅ๏ธ Web UI has been configured.'));
|
|
130
136
|
}
|
|
137
|
+
/**
|
|
138
|
+
* Configures the janitor service parameters
|
|
139
|
+
* @param config - Janitor configuration options
|
|
140
|
+
* - requestTimeoutMs: Timeout for health check requests (default: 10000ms)
|
|
141
|
+
* - hostDownRevokeScore: Number of consecutive failures before deleting output (default: 3)
|
|
142
|
+
*/
|
|
143
|
+
configureJanitor(config) {
|
|
144
|
+
this.janitorConfig = {
|
|
145
|
+
...this.janitorConfig,
|
|
146
|
+
...config
|
|
147
|
+
};
|
|
148
|
+
this.logger.log(chalk.blue('๐งน Janitor service has been configured.'));
|
|
149
|
+
}
|
|
131
150
|
/**
|
|
132
151
|
* Configures the logger to be used by the server.
|
|
133
152
|
* @param logger - A logger object (e.g., console)
|
|
@@ -292,7 +311,7 @@ export default class OverlayExpress {
|
|
|
292
311
|
}
|
|
293
312
|
else {
|
|
294
313
|
// If the user provided a syncConfiguration, use that. Otherwise default to an empty object.
|
|
295
|
-
syncConfig = this.engineConfig.syncConfiguration
|
|
314
|
+
syncConfig = this.engineConfig.syncConfiguration ?? {};
|
|
296
315
|
}
|
|
297
316
|
// Build the actual Storage
|
|
298
317
|
const storage = new KnexStorage(this.knex);
|
|
@@ -354,7 +373,9 @@ export default class OverlayExpress {
|
|
|
354
373
|
// overlayBroadcastFacilitator
|
|
355
374
|
this.engineConfig.overlayBroadcastFacilitator ?? new HTTPSOverlayBroadcastFacilitator(),
|
|
356
375
|
// logger
|
|
357
|
-
this.logger
|
|
376
|
+
this.logger,
|
|
377
|
+
// suppressDefaultSyncAdvertisements
|
|
378
|
+
this.engineConfig.suppressDefaultSyncAdvertisements ?? true);
|
|
358
379
|
this.logger.log(chalk.green('๐ Engine has been configured.'));
|
|
359
380
|
}
|
|
360
381
|
/**
|
|
@@ -399,12 +420,12 @@ export default class OverlayExpress {
|
|
|
399
420
|
this.app.use((req, res, next) => {
|
|
400
421
|
const startTime = Date.now();
|
|
401
422
|
// Log incoming request details
|
|
402
|
-
this.logger.log(chalk.magenta.bold(`๐ฅ Incoming Request: ${req.method} ${req.originalUrl}`));
|
|
423
|
+
this.logger.log(chalk.magenta.bold(`๐ฅ Incoming Request: ${String(req.method)} ${String(req.originalUrl)}`));
|
|
403
424
|
// Pretty-print headers
|
|
404
|
-
this.logger.log(chalk.cyan(
|
|
425
|
+
this.logger.log(chalk.cyan('Headers:'));
|
|
405
426
|
this.logger.log(util.inspect(req.headers, { colors: true, depth: null }));
|
|
406
427
|
// Handle request body
|
|
407
|
-
if (req.body && Object.keys(req.body).length > 0) {
|
|
428
|
+
if (req.body != null && Object.keys(req.body).length > 0) {
|
|
408
429
|
let bodyContent;
|
|
409
430
|
let bodyString;
|
|
410
431
|
if (typeof req.body === 'object') {
|
|
@@ -414,13 +435,13 @@ export default class OverlayExpress {
|
|
|
414
435
|
bodyString = req.body.toString('utf8');
|
|
415
436
|
}
|
|
416
437
|
else {
|
|
417
|
-
bodyString = req.body
|
|
438
|
+
bodyString = String(req.body);
|
|
418
439
|
}
|
|
419
440
|
if (bodyString.length > 280) {
|
|
420
|
-
bodyContent = chalk.yellow(`(Body too long to display, length: ${bodyString.length} characters)`);
|
|
441
|
+
bodyContent = chalk.yellow(`(Body too long to display, length: ${String(bodyString.length)} characters)`);
|
|
421
442
|
}
|
|
422
443
|
else {
|
|
423
|
-
bodyContent = chalk.green(`Request Body:\n${bodyString}`);
|
|
444
|
+
bodyContent = chalk.green(`Request Body:\n${String(bodyString)}`);
|
|
424
445
|
}
|
|
425
446
|
this.logger.log(bodyContent);
|
|
426
447
|
}
|
|
@@ -434,11 +455,11 @@ export default class OverlayExpress {
|
|
|
434
455
|
// Log outgoing response details after the response is finished
|
|
435
456
|
res.on('finish', () => {
|
|
436
457
|
const duration = Date.now() - startTime;
|
|
437
|
-
this.logger.log(chalk.magenta.bold(`๐ค Outgoing Response: ${req.method} ${req.originalUrl} - Status: ${res.statusCode} - Duration: ${duration}ms`));
|
|
438
|
-
this.logger.log(chalk.cyan(
|
|
458
|
+
this.logger.log(chalk.magenta.bold(`๐ค Outgoing Response: ${String(req.method)} ${String(req.originalUrl)} - Status: ${String(res.statusCode)} - Duration: ${String(duration)}ms`));
|
|
459
|
+
this.logger.log(chalk.cyan('Response Headers:'));
|
|
439
460
|
this.logger.log(util.inspect(res.getHeaders(), { colors: true, depth: null }));
|
|
440
461
|
// Handle response body
|
|
441
|
-
if (responseBody) {
|
|
462
|
+
if (responseBody != null) {
|
|
442
463
|
let bodyContent;
|
|
443
464
|
let bodyString;
|
|
444
465
|
if (typeof responseBody === 'object') {
|
|
@@ -447,14 +468,17 @@ export default class OverlayExpress {
|
|
|
447
468
|
else if (Buffer.isBuffer(responseBody)) {
|
|
448
469
|
bodyString = responseBody.toString('utf8');
|
|
449
470
|
}
|
|
471
|
+
else if (typeof responseBody === 'string') {
|
|
472
|
+
bodyString = responseBody;
|
|
473
|
+
}
|
|
450
474
|
else {
|
|
451
|
-
bodyString = responseBody
|
|
475
|
+
bodyString = String(responseBody);
|
|
452
476
|
}
|
|
453
477
|
if (bodyString.length > 280) {
|
|
454
|
-
bodyContent = chalk.yellow(`(Response body too long to display, length: ${bodyString.length} characters)`);
|
|
478
|
+
bodyContent = chalk.yellow(`(Response body too long to display, length: ${String(bodyString.length)} characters)`);
|
|
455
479
|
}
|
|
456
480
|
else {
|
|
457
|
-
bodyContent = chalk.green(`Response Body:\n${bodyString}`);
|
|
481
|
+
bodyContent = chalk.green(`Response Body:\n${String(bodyString)}`);
|
|
458
482
|
}
|
|
459
483
|
this.logger.log(bodyContent);
|
|
460
484
|
}
|
|
@@ -481,6 +505,10 @@ export default class OverlayExpress {
|
|
|
481
505
|
res.set('content-type', 'text/html');
|
|
482
506
|
res.send(makeUserInterface(this.webUIConfig));
|
|
483
507
|
});
|
|
508
|
+
// Serve a health check endpoint
|
|
509
|
+
this.app.get('/health', (_, res) => {
|
|
510
|
+
res.status(200).json({ status: 'ok' });
|
|
511
|
+
});
|
|
484
512
|
// List hosted topic managers and lookup services
|
|
485
513
|
this.app.get('/listTopicManagers', (_, res) => {
|
|
486
514
|
;
|
|
@@ -598,7 +626,7 @@ export default class OverlayExpress {
|
|
|
598
626
|
const steak = await engine.submit(taggedBEEF, (steak) => {
|
|
599
627
|
responseSent = true;
|
|
600
628
|
return res.status(200).json(steak);
|
|
601
|
-
});
|
|
629
|
+
}, 'current-tx', offChainValues);
|
|
602
630
|
if (!responseSent) {
|
|
603
631
|
res.status(200).json(steak);
|
|
604
632
|
}
|
|
@@ -621,11 +649,52 @@ export default class OverlayExpress {
|
|
|
621
649
|
;
|
|
622
650
|
(async () => {
|
|
623
651
|
try {
|
|
624
|
-
|
|
625
|
-
|
|
652
|
+
// Check for aggregation header to determine response format
|
|
653
|
+
const aggregationHeader = req.headers['x-aggregation'];
|
|
654
|
+
const shouldReturnBinary = aggregationHeader === 'yes';
|
|
655
|
+
// Validate request body structure
|
|
656
|
+
const lookupRequest = req.body;
|
|
657
|
+
if (typeof lookupRequest.service !== 'string' || lookupRequest.query === undefined) {
|
|
658
|
+
return res.status(400).json({
|
|
659
|
+
status: 'error',
|
|
660
|
+
message: 'Invalid request: body must contain "service" (string) and "query" fields'
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
const result = await engine.lookup(lookupRequest);
|
|
664
|
+
if (!shouldReturnBinary) {
|
|
665
|
+
// Return JSON response (default behavior)
|
|
666
|
+
return res.status(200).json(result);
|
|
667
|
+
}
|
|
668
|
+
const beef = new Beef();
|
|
669
|
+
const outputs = result.outputs;
|
|
670
|
+
// Serialize in the format expected by LookupResolver
|
|
671
|
+
const writer = new Utils.Writer();
|
|
672
|
+
// Write number of outpoints
|
|
673
|
+
writer.writeVarIntNum(outputs.length);
|
|
674
|
+
// Write each outpoint data
|
|
675
|
+
for (const output of outputs) {
|
|
676
|
+
const tx = Transaction.fromBEEF(output.beef);
|
|
677
|
+
// Write txid (32 bytes)
|
|
678
|
+
writer.write(tx.id());
|
|
679
|
+
// Write outputIndex
|
|
680
|
+
writer.writeVarIntNum(output.outputIndex);
|
|
681
|
+
// Write context length and data
|
|
682
|
+
if ((output.context != null) && output.context.length > 0) {
|
|
683
|
+
writer.writeVarIntNum(output.context.length);
|
|
684
|
+
writer.write(output.context);
|
|
685
|
+
}
|
|
686
|
+
else {
|
|
687
|
+
writer.writeVarIntNum(0);
|
|
688
|
+
}
|
|
689
|
+
beef.mergeTransaction(tx);
|
|
690
|
+
}
|
|
691
|
+
// Write the beef data
|
|
692
|
+
writer.write(beef.toBinary());
|
|
693
|
+
res.setHeader('Content-Type', 'application/octet-stream');
|
|
694
|
+
return res.status(200).send(Buffer.from(writer.toArray()));
|
|
626
695
|
}
|
|
627
696
|
catch (error) {
|
|
628
|
-
console.error(chalk.red('
|
|
697
|
+
console.error(chalk.red('Error in /lookup:'), error);
|
|
629
698
|
return res.status(400).json({
|
|
630
699
|
status: 'error',
|
|
631
700
|
message: error instanceof Error ? error.message : 'An unknown error occurred'
|
|
@@ -726,7 +795,7 @@ export default class OverlayExpress {
|
|
|
726
795
|
* Middleware for checking the admin bearer token.
|
|
727
796
|
*/
|
|
728
797
|
const checkAdminAuth = (req, res, next) => {
|
|
729
|
-
const authHeader = req.headers
|
|
798
|
+
const authHeader = req.headers.authorization;
|
|
730
799
|
if (typeof authHeader !== 'string' || !authHeader.startsWith('Bearer ')) {
|
|
731
800
|
res.status(401).json({ status: 'error', message: 'Unauthorized: Missing Bearer token' });
|
|
732
801
|
return;
|
|
@@ -824,6 +893,37 @@ export default class OverlayExpress {
|
|
|
824
893
|
});
|
|
825
894
|
});
|
|
826
895
|
});
|
|
896
|
+
/**
|
|
897
|
+
* Admin route to run the janitor service.
|
|
898
|
+
*/
|
|
899
|
+
this.app.post('/admin/janitor', checkAdminAuth, (req, res) => {
|
|
900
|
+
;
|
|
901
|
+
(async () => {
|
|
902
|
+
try {
|
|
903
|
+
this.ensureMongo();
|
|
904
|
+
const janitor = new JanitorService({
|
|
905
|
+
mongoDb: this.mongoDb,
|
|
906
|
+
logger: this.logger,
|
|
907
|
+
requestTimeoutMs: this.janitorConfig.requestTimeoutMs,
|
|
908
|
+
hostDownRevokeScore: this.janitorConfig.hostDownRevokeScore
|
|
909
|
+
});
|
|
910
|
+
await janitor.run();
|
|
911
|
+
return res.status(200).json({ status: 'success', message: 'Janitor run completed' });
|
|
912
|
+
}
|
|
913
|
+
catch (error) {
|
|
914
|
+
console.error(chalk.red('โ Error in /admin/janitor:'), error);
|
|
915
|
+
return res.status(400).json({
|
|
916
|
+
status: 'error',
|
|
917
|
+
message: error instanceof Error ? error.message : 'An unknown error occurred'
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
})().catch(() => {
|
|
921
|
+
res.status(500).json({
|
|
922
|
+
status: 'error',
|
|
923
|
+
message: 'Unexpected error'
|
|
924
|
+
});
|
|
925
|
+
});
|
|
926
|
+
});
|
|
827
927
|
// Automatically handle migrations
|
|
828
928
|
const migrationSource = new InMemoryMigrationSource(this.migrationsToRun);
|
|
829
929
|
const result = await knex.migrate.latest({
|
|
@@ -871,7 +971,7 @@ export default class OverlayExpress {
|
|
|
871
971
|
this.logger.log(chalk.yellow(`${this.name} will not sync because GASP has been disabled.`));
|
|
872
972
|
}
|
|
873
973
|
// Start listening on the configured port
|
|
874
|
-
this.app.listen(this.port,
|
|
974
|
+
this.app.listen(this.port, () => {
|
|
875
975
|
this.logger.log(chalk.green.bold(`๐ง ${this.name} is ready and listening on local port ${this.port}`));
|
|
876
976
|
});
|
|
877
977
|
}
|