@bsv/overlay-express 0.8.1 โ 0.8.2-rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/package.json +13 -13
- package/dist/cjs/src/OverlayExpress.js +27 -86
- package/dist/cjs/src/OverlayExpress.js.map +1 -1
- package/dist/cjs/src/makeUserInterface.js +93 -290
- package/dist/cjs/src/makeUserInterface.js.map +1 -1
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/esm/src/OverlayExpress.js +21 -80
- package/dist/esm/src/OverlayExpress.js.map +1 -1
- package/dist/esm/src/makeUserInterface.js +93 -287
- package/dist/esm/src/makeUserInterface.js.map +1 -1
- package/dist/esm/tsconfig.esm.tsbuildinfo +1 -1
- package/dist/types/src/OverlayExpress.d.ts +3 -15
- package/dist/types/src/OverlayExpress.d.ts.map +1 -1
- package/dist/types/src/makeUserInterface.d.ts +2 -7
- package/dist/types/src/makeUserInterface.d.ts.map +1 -1
- package/dist/types/tsconfig.types.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/dist/cjs/src/JanitorService.js +0 -215
- package/dist/cjs/src/JanitorService.js.map +0 -1
- package/dist/cjs/src/generalGuide.md.js +0 -77
- package/dist/cjs/src/generalGuide.md.js.map +0 -1
- package/dist/esm/src/JanitorService.js +0 -211
- package/dist/esm/src/JanitorService.js.map +0 -1
- package/dist/esm/src/generalGuide.md.js +0 -75
- package/dist/esm/src/generalGuide.md.js.map +0 -1
- package/dist/types/src/JanitorService.d.ts +0 -59
- package/dist/types/src/JanitorService.d.ts.map +0 -1
- package/dist/types/src/generalGuide.md.d.ts +0 -3
- package/dist/types/src/generalGuide.md.d.ts.map +0 -1
|
@@ -1,215 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.JanitorService = void 0;
|
|
7
|
-
const chalk_1 = __importDefault(require("chalk"));
|
|
8
|
-
/**
|
|
9
|
-
* JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
|
|
10
|
-
* It validates domain names and checks /health endpoints to ensure services are operational.
|
|
11
|
-
*
|
|
12
|
-
* When a service is down, it increments a "down" counter. When healthy, it decrements.
|
|
13
|
-
* If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
|
|
14
|
-
*
|
|
15
|
-
* This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
|
|
16
|
-
*/
|
|
17
|
-
class JanitorService {
|
|
18
|
-
constructor(config) {
|
|
19
|
-
var _a, _b, _c;
|
|
20
|
-
this.mongoDb = config.mongoDb;
|
|
21
|
-
this.logger = (_a = config.logger) !== null && _a !== void 0 ? _a : console;
|
|
22
|
-
this.requestTimeoutMs = (_b = config.requestTimeoutMs) !== null && _b !== void 0 ? _b : 10000; // Default: 10 seconds
|
|
23
|
-
this.hostDownRevokeScore = (_c = config.hostDownRevokeScore) !== null && _c !== void 0 ? _c : 3;
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Runs a single pass of health checks on all SHIP and SLAP outputs
|
|
27
|
-
*/
|
|
28
|
-
async run() {
|
|
29
|
-
this.logger.log(chalk_1.default.blue('๐งน Running janitor health checks...'));
|
|
30
|
-
try {
|
|
31
|
-
// Check SHIP outputs
|
|
32
|
-
await this.checkTopicOutputs('shipRecords');
|
|
33
|
-
// Check SLAP outputs
|
|
34
|
-
await this.checkTopicOutputs('slapRecords');
|
|
35
|
-
this.logger.log(chalk_1.default.green('๐งน Janitor health checks completed'));
|
|
36
|
-
}
|
|
37
|
-
catch (error) {
|
|
38
|
-
this.logger.error(chalk_1.default.red('โ Error during health checks:'), error);
|
|
39
|
-
throw error;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* Checks all outputs for a specific topic
|
|
44
|
-
*/
|
|
45
|
-
async checkTopicOutputs(collectionName) {
|
|
46
|
-
try {
|
|
47
|
-
const collection = this.mongoDb.collection(collectionName);
|
|
48
|
-
const outputs = await collection.find({}).toArray();
|
|
49
|
-
this.logger.log(chalk_1.default.cyan(`๐ Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`));
|
|
50
|
-
for (const output of outputs) {
|
|
51
|
-
await this.checkOutput(output, collection);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
catch (error) {
|
|
55
|
-
this.logger.error(chalk_1.default.red(`โ Error checking ${collectionName.toUpperCase()} outputs:`), error);
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
/**
|
|
59
|
-
* Checks a single output for health
|
|
60
|
-
*/
|
|
61
|
-
async checkOutput(output, collection) {
|
|
62
|
-
try {
|
|
63
|
-
// Extract the URL from the output
|
|
64
|
-
const url = this.extractURLFromOutput(output);
|
|
65
|
-
if (url === null) {
|
|
66
|
-
return; // Skip if no valid URL
|
|
67
|
-
}
|
|
68
|
-
// Validate domain
|
|
69
|
-
if (!this.isValidDomain(url)) {
|
|
70
|
-
this.logger.log(chalk_1.default.yellow(`โ ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`));
|
|
71
|
-
await this.handleUnhealthyOutput(output, collection);
|
|
72
|
-
return;
|
|
73
|
-
}
|
|
74
|
-
// Check health endpoint
|
|
75
|
-
const isHealthy = await this.checkHealthEndpoint(url);
|
|
76
|
-
if (isHealthy) {
|
|
77
|
-
await this.handleHealthyOutput(output, collection);
|
|
78
|
-
}
|
|
79
|
-
else {
|
|
80
|
-
await this.handleUnhealthyOutput(output, collection);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
catch (error) {
|
|
84
|
-
this.logger.error(chalk_1.default.red(`โ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
85
|
-
await this.handleUnhealthyOutput(output, collection).catch(() => { });
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Extracts URL from output record
|
|
90
|
-
*/
|
|
91
|
-
extractURLFromOutput(output) {
|
|
92
|
-
try {
|
|
93
|
-
// SHIP and SLAP outputs typically have a domain field or URL field
|
|
94
|
-
// Check for common field names
|
|
95
|
-
if (typeof output.domain === 'string') {
|
|
96
|
-
return output.domain;
|
|
97
|
-
}
|
|
98
|
-
if (typeof output.url === 'string') {
|
|
99
|
-
return output.url;
|
|
100
|
-
}
|
|
101
|
-
if (typeof output.serviceURL === 'string') {
|
|
102
|
-
return output.serviceURL;
|
|
103
|
-
}
|
|
104
|
-
// SLAP outputs may have a protocols array with URLs
|
|
105
|
-
if (Array.isArray(output.protocols) && output.protocols.length > 0) {
|
|
106
|
-
const httpsProtocol = output.protocols.find((p) => typeof p === 'string' && p.startsWith('https://'));
|
|
107
|
-
if (httpsProtocol !== undefined) {
|
|
108
|
-
return httpsProtocol;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
113
|
-
catch {
|
|
114
|
-
return null;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Validates if a string is a valid domain name
|
|
119
|
-
*/
|
|
120
|
-
isValidDomain(url) {
|
|
121
|
-
try {
|
|
122
|
-
// Parse the URL
|
|
123
|
-
const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`);
|
|
124
|
-
// Check if hostname is valid
|
|
125
|
-
const hostname = parsedURL.hostname;
|
|
126
|
-
// Basic domain validation
|
|
127
|
-
const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i;
|
|
128
|
-
const localhostRegex = /^localhost$/i;
|
|
129
|
-
const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/;
|
|
130
|
-
return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname);
|
|
131
|
-
}
|
|
132
|
-
catch {
|
|
133
|
-
return false;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* Checks the /health endpoint of a service
|
|
138
|
-
*/
|
|
139
|
-
async checkHealthEndpoint(url) {
|
|
140
|
-
try {
|
|
141
|
-
// Ensure URL has protocol
|
|
142
|
-
const fullURL = url.startsWith('http') ? url : `https://${url}`;
|
|
143
|
-
const healthURL = new URL('/health', fullURL).toString();
|
|
144
|
-
// Create AbortController for timeout
|
|
145
|
-
const controller = new AbortController();
|
|
146
|
-
const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs);
|
|
147
|
-
try {
|
|
148
|
-
const response = await fetch(healthURL, {
|
|
149
|
-
method: 'GET',
|
|
150
|
-
signal: controller.signal,
|
|
151
|
-
headers: {
|
|
152
|
-
Accept: 'application/json'
|
|
153
|
-
}
|
|
154
|
-
});
|
|
155
|
-
clearTimeout(timeout);
|
|
156
|
-
if (!response.ok) {
|
|
157
|
-
return false;
|
|
158
|
-
}
|
|
159
|
-
const data = await response.json();
|
|
160
|
-
return (data === null || data === void 0 ? void 0 : data.status) === 'ok';
|
|
161
|
-
}
|
|
162
|
-
catch (error) {
|
|
163
|
-
clearTimeout(timeout);
|
|
164
|
-
if (error.name === 'AbortError') {
|
|
165
|
-
this.logger.log(chalk_1.default.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`));
|
|
166
|
-
}
|
|
167
|
-
return false;
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
catch (error) {
|
|
171
|
-
return false;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
/**
|
|
175
|
-
* Handles a healthy output by decrementing its down counter
|
|
176
|
-
*/
|
|
177
|
-
async handleHealthyOutput(output, collection) {
|
|
178
|
-
try {
|
|
179
|
-
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
180
|
-
if (currentDown > 0) {
|
|
181
|
-
// Decrement the down counter
|
|
182
|
-
await collection.updateOne({ _id: output._id }, { $inc: { down: -1 } });
|
|
183
|
-
this.logger.log(chalk_1.default.green(`โ
Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`));
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
catch (error) {
|
|
187
|
-
this.logger.error(chalk_1.default.red(`โ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
|
|
192
|
-
*/
|
|
193
|
-
async handleUnhealthyOutput(output, collection) {
|
|
194
|
-
try {
|
|
195
|
-
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
196
|
-
const newDown = currentDown + 1;
|
|
197
|
-
this.logger.log(chalk_1.default.yellow(`โ ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`));
|
|
198
|
-
// Check if we should delete the record
|
|
199
|
-
if (newDown >= this.hostDownRevokeScore) {
|
|
200
|
-
this.logger.log(chalk_1.default.red(`๐ซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`));
|
|
201
|
-
await collection.deleteOne({ _id: output._id });
|
|
202
|
-
this.logger.log(chalk_1.default.green(`โ
Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`));
|
|
203
|
-
}
|
|
204
|
-
else {
|
|
205
|
-
// Increment the down counter
|
|
206
|
-
await collection.updateOne({ _id: output._id }, { $inc: { down: 1 } });
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
catch (error) {
|
|
210
|
-
this.logger.error(chalk_1.default.red(`โ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
exports.JanitorService = JanitorService;
|
|
215
|
-
//# sourceMappingURL=JanitorService.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"JanitorService.js","sourceRoot":"","sources":["../../../src/JanitorService.ts"],"names":[],"mappings":";;;;;;AACA,kDAAyB;AAYzB;;;;;;;;GAQG;AACH,MAAa,cAAc;IAMzB,YAAa,MAAqB;;QAChC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAA,MAAM,CAAC,MAAM,mCAAI,OAAO,CAAA;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAA,MAAM,CAAC,gBAAgB,mCAAI,KAAK,CAAA,CAAC,sBAAsB;QAC/E,IAAI,CAAC,mBAAmB,GAAG,MAAA,MAAM,CAAC,mBAAmB,mCAAI,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAElE,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,EAAE,KAAK,CAAC,CAAA;YACpE,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAE,cAAsB;QACrD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;YAEnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAA;YAEvG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,oBAAoB,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,CAAA;QAClG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAE,MAA2B,EAAE,UAAe;QACrE,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;YAC7C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,OAAM,CAAC,uBAAuB;YAChC,CAAC;YAED,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;gBAC1H,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;gBACpD,OAAM;YACR,CAAC;YAED,wBAAwB;YACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAErD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;YACpH,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAE,MAA2B;QACvD,IAAI,CAAC;YACH,mEAAmE;YACnE,+BAA+B;YAC/B,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACtC,OAAO,MAAM,CAAC,MAAM,CAAA;YACtB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACnC,OAAO,MAAM,CAAC,GAAG,CAAA;YACnB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC,UAAU,CAAA;YAC1B,CAAC;YACD,oDAAoD;YACpD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnE,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CACrD,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAClD,CAAA;gBACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;oBAChC,OAAO,aAAa,CAAA;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAE,GAAW;QAChC,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAA;YAE1E,6BAA6B;YAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;YAEnC,0BAA0B;YAC1B,MAAM,WAAW,GAAG,+EAA+E,CAAA;YACnG,MAAM,cAAc,GAAG,cAAc,CAAA;YACrC,MAAM,SAAS,GAAG,yBAAyB,CAAA;YAE3C,OAAO,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAChG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,GAAW;QAC5C,IAAI,CAAC;YACH,0BAA0B;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;YAExD,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAE3E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBACtC,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,MAAM,EAAE,kBAAkB;qBAC3B;iBACF,CAAC,CAAA;gBAEF,YAAY,CAAC,OAAO,CAAC,CAAA;gBAErB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,KAAK,CAAA;gBACd,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAClC,OAAO,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,MAAM,MAAK,IAAI,CAAA;YAC9B,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,CAAA;gBACrB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC,CAAA;gBAC3E,CAAC;gBACD,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,MAA2B,EAAE,UAAe;QAC7E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAErE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBACpB,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACvB,CAAA;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,sBAAsB,WAAW,OAAO,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;YACvJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9H,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAE,MAA2B,EAAE,UAAe;QAC/E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YACrE,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAA;YAE/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,WAAW,OAAO,OAAO,GAAG,CAAC,CAAC,CAAA;YAEjJ,uCAAuC;YACvC,IAAI,OAAO,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,iBAAiB,OAAO,OAAO,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAA;gBAC7J,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAK,CAAC,KAAK,CAAC,iCAAiC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;YACpH,CAAC;iBAAM,CAAC;gBACN,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CACtB,CAAA;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAChI,CAAC;IACH,CAAC;CACF;AAhOD,wCAgOC"}
|
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.default = `\`
|
|
4
|
-
# General Guide
|
|
5
|
-
|
|
6
|
-
## Topic Managers
|
|
7
|
-
|
|
8
|
-
A Topic Manager controls addition and removal of transaction outputs to an Overlay Topic. It processes transaction by exposing a /submit endpoint, request should looks something like:
|
|
9
|
-
|
|
10
|
-
### [POST] /submit
|
|
11
|
-
|
|
12
|
-
#### Request Headers
|
|
13
|
-
\\\`\\\`\\\`json
|
|
14
|
-
{
|
|
15
|
-
"x-topics": "["tm_messagebox"]", // array of topic identifiers as stringified JSON
|
|
16
|
-
"x-includes-off-chain-values": "false" // string "true" or "false"
|
|
17
|
-
}
|
|
18
|
-
\\\`\\\`\\\`
|
|
19
|
-
|
|
20
|
-
#### Request Body
|
|
21
|
-
\\\`\\\`\\\`json
|
|
22
|
-
[1,1,1,1... ], // transaction in atomic BEEF format as Uint8Array
|
|
23
|
-
\\\`\\\`\\\`
|
|
24
|
-
|
|
25
|
-
The engine processes this TaggedBEEF and generates a STEAK (Submitted Transaction Execution AcKnowledgment), which is returned to the client.
|
|
26
|
-
|
|
27
|
-
#### Response Body
|
|
28
|
-
\\\`\\\`\\\`json
|
|
29
|
-
{
|
|
30
|
-
"tm_messagebox": {
|
|
31
|
-
"outputsToAdmit": [0], // which of the outputs in the BEEF to admit
|
|
32
|
-
"coinsToRetain": [], // which of the inputs in the BEEF to mark spent but keep within the Topic
|
|
33
|
-
"coinsRemoved": [], // which of the inputs in the BEEF to remove from the Topic
|
|
34
|
-
},
|
|
35
|
-
... // per topic
|
|
36
|
-
}
|
|
37
|
-
\\\`\\\`\\\`
|
|
38
|
-
|
|
39
|
-
<br />
|
|
40
|
-
|
|
41
|
-
## Lookup Services
|
|
42
|
-
|
|
43
|
-
A Lookup Service is a component that enables data retrieval from the Overlay Services ecosystem. It allows clients to search and retrieve transactions that were previously submitted to Topic Managers, using specific criteria or identifiers.
|
|
44
|
-
|
|
45
|
-
### [POST] /lookup
|
|
46
|
-
|
|
47
|
-
#### Request Body
|
|
48
|
-
\\\`\\\`\\\`json
|
|
49
|
-
{
|
|
50
|
-
"service": "ls_messagebox", // Lookup Service identifier as string,
|
|
51
|
-
"query": "findAll" // query must be a string, JSON queries should be stringified
|
|
52
|
-
}
|
|
53
|
-
\\\`\\\`\\\`
|
|
54
|
-
|
|
55
|
-
#### Response Body
|
|
56
|
-
\\\`\\\`\\\`json
|
|
57
|
-
{
|
|
58
|
-
"type": "output-list",
|
|
59
|
-
"outputs": [{
|
|
60
|
-
"beef": [1, 1, 1...], // number[]
|
|
61
|
-
"outputIndex": 0, // number
|
|
62
|
-
"context"?: [1, 1, 1...], // number[]
|
|
63
|
-
}>]
|
|
64
|
-
}
|
|
65
|
-
\\\`\\\`\\\`
|
|
66
|
-
|
|
67
|
-
<br />
|
|
68
|
-
|
|
69
|
-
## Summary
|
|
70
|
-
|
|
71
|
-
Usually Topic Managers and Lookup Services are paired, but they can be separated for greater flexibility. Together, they form a complete system for data submission, organization, and retrieval within the Overlay Services framework.
|
|
72
|
-
|
|
73
|
-
<br />
|
|
74
|
-
<br />
|
|
75
|
-
\`
|
|
76
|
-
`;
|
|
77
|
-
//# sourceMappingURL=generalGuide.md.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"generalGuide.md.js","sourceRoot":"","sources":["../../../src/generalGuide.md.ts"],"names":[],"mappings":";;AAAA,kBAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyEd,CAAA"}
|
|
@@ -1,211 +0,0 @@
|
|
|
1
|
-
import chalk from 'chalk';
|
|
2
|
-
/**
|
|
3
|
-
* JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
|
|
4
|
-
* It validates domain names and checks /health endpoints to ensure services are operational.
|
|
5
|
-
*
|
|
6
|
-
* When a service is down, it increments a "down" counter. When healthy, it decrements.
|
|
7
|
-
* If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
|
|
8
|
-
*
|
|
9
|
-
* This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
|
|
10
|
-
*/
|
|
11
|
-
export class JanitorService {
|
|
12
|
-
mongoDb;
|
|
13
|
-
logger;
|
|
14
|
-
requestTimeoutMs;
|
|
15
|
-
hostDownRevokeScore;
|
|
16
|
-
constructor(config) {
|
|
17
|
-
this.mongoDb = config.mongoDb;
|
|
18
|
-
this.logger = config.logger ?? console;
|
|
19
|
-
this.requestTimeoutMs = config.requestTimeoutMs ?? 10000; // Default: 10 seconds
|
|
20
|
-
this.hostDownRevokeScore = config.hostDownRevokeScore ?? 3;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Runs a single pass of health checks on all SHIP and SLAP outputs
|
|
24
|
-
*/
|
|
25
|
-
async run() {
|
|
26
|
-
this.logger.log(chalk.blue('๐งน Running janitor health checks...'));
|
|
27
|
-
try {
|
|
28
|
-
// Check SHIP outputs
|
|
29
|
-
await this.checkTopicOutputs('shipRecords');
|
|
30
|
-
// Check SLAP outputs
|
|
31
|
-
await this.checkTopicOutputs('slapRecords');
|
|
32
|
-
this.logger.log(chalk.green('๐งน Janitor health checks completed'));
|
|
33
|
-
}
|
|
34
|
-
catch (error) {
|
|
35
|
-
this.logger.error(chalk.red('โ Error during health checks:'), error);
|
|
36
|
-
throw error;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Checks all outputs for a specific topic
|
|
41
|
-
*/
|
|
42
|
-
async checkTopicOutputs(collectionName) {
|
|
43
|
-
try {
|
|
44
|
-
const collection = this.mongoDb.collection(collectionName);
|
|
45
|
-
const outputs = await collection.find({}).toArray();
|
|
46
|
-
this.logger.log(chalk.cyan(`๐ Checking ${outputs.length} ${collectionName.toUpperCase()} outputs...`));
|
|
47
|
-
for (const output of outputs) {
|
|
48
|
-
await this.checkOutput(output, collection);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
catch (error) {
|
|
52
|
-
this.logger.error(chalk.red(`โ Error checking ${collectionName.toUpperCase()} outputs:`), error);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Checks a single output for health
|
|
57
|
-
*/
|
|
58
|
-
async checkOutput(output, collection) {
|
|
59
|
-
try {
|
|
60
|
-
// Extract the URL from the output
|
|
61
|
-
const url = this.extractURLFromOutput(output);
|
|
62
|
-
if (url === null) {
|
|
63
|
-
return; // Skip if no valid URL
|
|
64
|
-
}
|
|
65
|
-
// Validate domain
|
|
66
|
-
if (!this.isValidDomain(url)) {
|
|
67
|
-
this.logger.log(chalk.yellow(`โ ๏ธ Invalid domain for output ${String(output.txid)}:${String(output.outputIndex)}: ${url}`));
|
|
68
|
-
await this.handleUnhealthyOutput(output, collection);
|
|
69
|
-
return;
|
|
70
|
-
}
|
|
71
|
-
// Check health endpoint
|
|
72
|
-
const isHealthy = await this.checkHealthEndpoint(url);
|
|
73
|
-
if (isHealthy) {
|
|
74
|
-
await this.handleHealthyOutput(output, collection);
|
|
75
|
-
}
|
|
76
|
-
else {
|
|
77
|
-
await this.handleUnhealthyOutput(output, collection);
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
this.logger.error(chalk.red(`โ Error checking output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
82
|
-
await this.handleUnhealthyOutput(output, collection).catch(() => { });
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Extracts URL from output record
|
|
87
|
-
*/
|
|
88
|
-
extractURLFromOutput(output) {
|
|
89
|
-
try {
|
|
90
|
-
// SHIP and SLAP outputs typically have a domain field or URL field
|
|
91
|
-
// Check for common field names
|
|
92
|
-
if (typeof output.domain === 'string') {
|
|
93
|
-
return output.domain;
|
|
94
|
-
}
|
|
95
|
-
if (typeof output.url === 'string') {
|
|
96
|
-
return output.url;
|
|
97
|
-
}
|
|
98
|
-
if (typeof output.serviceURL === 'string') {
|
|
99
|
-
return output.serviceURL;
|
|
100
|
-
}
|
|
101
|
-
// SLAP outputs may have a protocols array with URLs
|
|
102
|
-
if (Array.isArray(output.protocols) && output.protocols.length > 0) {
|
|
103
|
-
const httpsProtocol = output.protocols.find((p) => typeof p === 'string' && p.startsWith('https://'));
|
|
104
|
-
if (httpsProtocol !== undefined) {
|
|
105
|
-
return httpsProtocol;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return null;
|
|
109
|
-
}
|
|
110
|
-
catch {
|
|
111
|
-
return null;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Validates if a string is a valid domain name
|
|
116
|
-
*/
|
|
117
|
-
isValidDomain(url) {
|
|
118
|
-
try {
|
|
119
|
-
// Parse the URL
|
|
120
|
-
const parsedURL = new URL(url.startsWith('http') ? url : `https://${url}`);
|
|
121
|
-
// Check if hostname is valid
|
|
122
|
-
const hostname = parsedURL.hostname;
|
|
123
|
-
// Basic domain validation
|
|
124
|
-
const domainRegex = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/i;
|
|
125
|
-
const localhostRegex = /^localhost$/i;
|
|
126
|
-
const ipv4Regex = /^(\d{1,3}\.){3}\d{1,3}$/;
|
|
127
|
-
return domainRegex.test(hostname) || localhostRegex.test(hostname) || ipv4Regex.test(hostname);
|
|
128
|
-
}
|
|
129
|
-
catch {
|
|
130
|
-
return false;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* Checks the /health endpoint of a service
|
|
135
|
-
*/
|
|
136
|
-
async checkHealthEndpoint(url) {
|
|
137
|
-
try {
|
|
138
|
-
// Ensure URL has protocol
|
|
139
|
-
const fullURL = url.startsWith('http') ? url : `https://${url}`;
|
|
140
|
-
const healthURL = new URL('/health', fullURL).toString();
|
|
141
|
-
// Create AbortController for timeout
|
|
142
|
-
const controller = new AbortController();
|
|
143
|
-
const timeout = setTimeout(() => controller.abort(), this.requestTimeoutMs);
|
|
144
|
-
try {
|
|
145
|
-
const response = await fetch(healthURL, {
|
|
146
|
-
method: 'GET',
|
|
147
|
-
signal: controller.signal,
|
|
148
|
-
headers: {
|
|
149
|
-
Accept: 'application/json'
|
|
150
|
-
}
|
|
151
|
-
});
|
|
152
|
-
clearTimeout(timeout);
|
|
153
|
-
if (!response.ok) {
|
|
154
|
-
return false;
|
|
155
|
-
}
|
|
156
|
-
const data = await response.json();
|
|
157
|
-
return data?.status === 'ok';
|
|
158
|
-
}
|
|
159
|
-
catch (error) {
|
|
160
|
-
clearTimeout(timeout);
|
|
161
|
-
if (error.name === 'AbortError') {
|
|
162
|
-
this.logger.log(chalk.yellow(`โฑ๏ธ Health check timeout for ${healthURL}`));
|
|
163
|
-
}
|
|
164
|
-
return false;
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
catch (error) {
|
|
168
|
-
return false;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* Handles a healthy output by decrementing its down counter
|
|
173
|
-
*/
|
|
174
|
-
async handleHealthyOutput(output, collection) {
|
|
175
|
-
try {
|
|
176
|
-
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
177
|
-
if (currentDown > 0) {
|
|
178
|
-
// Decrement the down counter
|
|
179
|
-
await collection.updateOne({ _id: output._id }, { $inc: { down: -1 } });
|
|
180
|
-
this.logger.log(chalk.green(`โ
Output ${String(output.txid)}:${String(output.outputIndex)} is healthy (down: ${currentDown} -> ${currentDown - 1})`));
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
this.logger.error(chalk.red(`โ Error handling healthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
/**
|
|
188
|
-
* Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
|
|
189
|
-
*/
|
|
190
|
-
async handleUnhealthyOutput(output, collection) {
|
|
191
|
-
try {
|
|
192
|
-
const currentDown = typeof output.down === 'number' ? output.down : 0;
|
|
193
|
-
const newDown = currentDown + 1;
|
|
194
|
-
this.logger.log(chalk.yellow(`โ ๏ธ Output ${String(output.txid)}:${String(output.outputIndex)} is unhealthy (down: ${currentDown} -> ${newDown})`));
|
|
195
|
-
// Check if we should delete the record
|
|
196
|
-
if (newDown >= this.hostDownRevokeScore) {
|
|
197
|
-
this.logger.log(chalk.red(`๐ซ Deleting output ${String(output.txid)}:${String(output.outputIndex)} (down count: ${newDown} >= ${this.hostDownRevokeScore})`));
|
|
198
|
-
await collection.deleteOne({ _id: output._id });
|
|
199
|
-
this.logger.log(chalk.green(`โ
Successfully deleted output ${String(output.txid)}:${String(output.outputIndex)}`));
|
|
200
|
-
}
|
|
201
|
-
else {
|
|
202
|
-
// Increment the down counter
|
|
203
|
-
await collection.updateOne({ _id: output._id }, { $inc: { down: 1 } });
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
catch (error) {
|
|
207
|
-
this.logger.error(chalk.red(`โ Error handling unhealthy output ${String(output.txid)}:${String(output.outputIndex)}:`), error);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
//# sourceMappingURL=JanitorService.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"JanitorService.js","sourceRoot":"","sources":["../../../src/JanitorService.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAA;AAYzB;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IACR,OAAO,CAAI;IACX,MAAM,CAAgB;IACtB,gBAAgB,CAAQ;IACxB,mBAAmB,CAAQ;IAE5C,YAAa,MAAqB;QAChC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAA;QACtC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,KAAK,CAAA,CAAC,sBAAsB;QAC/E,IAAI,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC,CAAA;QAElE,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,qBAAqB;YACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAA;YAE3C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAA;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,+BAA+B,CAAC,EAAE,KAAK,CAAC,CAAA;YACpE,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAE,cAAsB;QACrD,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAA;YAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;YAEnD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,MAAM,IAAI,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAA;YAEvG,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YAC5C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,KAAK,CAAC,CAAA;QAClG,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAE,MAA2B,EAAE,UAAe;QACrE,IAAI,CAAC;YACH,kCAAkC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA;YAC7C,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;gBACjB,OAAM,CAAC,uBAAuB;YAChC,CAAC;YAED,kBAAkB;YAClB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,gCAAgC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;gBAC1H,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;gBACpD,OAAM;YACR,CAAC;YAED,wBAAwB;YACxB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAErD,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACpD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;YACpH,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAE,MAA2B;QACvD,IAAI,CAAC;YACH,mEAAmE;YACnE,+BAA+B;YAC/B,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACtC,OAAO,MAAM,CAAC,MAAM,CAAA;YACtB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBACnC,OAAO,MAAM,CAAC,GAAG,CAAA;YACnB,CAAC;YACD,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC,UAAU,CAAA;YAC1B,CAAC;YACD,oDAAoD;YACpD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnE,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CACrD,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAClD,CAAA;gBACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;oBAChC,OAAO,aAAa,CAAA;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAE,GAAW;QAChC,IAAI,CAAC;YACH,gBAAgB;YAChB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC,CAAA;YAE1E,6BAA6B;YAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAA;YAEnC,0BAA0B;YAC1B,MAAM,WAAW,GAAG,+EAA+E,CAAA;YACnG,MAAM,cAAc,GAAG,cAAc,CAAA;YACrC,MAAM,SAAS,GAAG,yBAAyB,CAAA;YAE3C,OAAO,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAChG,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,GAAW;QAC5C,IAAI,CAAC;YACH,0BAA0B;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;YAExD,qCAAqC;YACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;YACxC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAE3E,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;oBACtC,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,OAAO,EAAE;wBACP,MAAM,EAAE,kBAAkB;qBAC3B;iBACF,CAAC,CAAA;gBAEF,YAAY,CAAC,OAAO,CAAC,CAAA;gBAErB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,OAAO,KAAK,CAAA;gBACd,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAClC,OAAO,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;YAC9B,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,YAAY,CAAC,OAAO,CAAC,CAAA;gBACrB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,+BAA+B,SAAS,EAAE,CAAC,CAAC,CAAA;gBAC3E,CAAC;gBACD,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAE,MAA2B,EAAE,UAAe;QAC7E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAErE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;gBACpB,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CACvB,CAAA;gBACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,sBAAsB,WAAW,OAAO,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;YACvJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9H,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAAE,MAA2B,EAAE,UAAe;QAC/E,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YACrE,MAAM,OAAO,GAAG,WAAW,GAAG,CAAC,CAAA;YAE/B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,wBAAwB,WAAW,OAAO,OAAO,GAAG,CAAC,CAAC,CAAA;YAEjJ,uCAAuC;YACvC,IAAI,OAAO,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,iBAAiB,OAAO,OAAO,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAA;gBAC7J,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAA;gBAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,iCAAiC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;YACpH,CAAC;iBAAM,CAAC;gBACN,6BAA6B;gBAC7B,MAAM,UAAU,CAAC,SAAS,CACxB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACnB,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,CACtB,CAAA;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAChI,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
export default `\`
|
|
2
|
-
# General Guide
|
|
3
|
-
|
|
4
|
-
## Topic Managers
|
|
5
|
-
|
|
6
|
-
A Topic Manager controls addition and removal of transaction outputs to an Overlay Topic. It processes transaction by exposing a /submit endpoint, request should looks something like:
|
|
7
|
-
|
|
8
|
-
### [POST] /submit
|
|
9
|
-
|
|
10
|
-
#### Request Headers
|
|
11
|
-
\\\`\\\`\\\`json
|
|
12
|
-
{
|
|
13
|
-
"x-topics": "["tm_messagebox"]", // array of topic identifiers as stringified JSON
|
|
14
|
-
"x-includes-off-chain-values": "false" // string "true" or "false"
|
|
15
|
-
}
|
|
16
|
-
\\\`\\\`\\\`
|
|
17
|
-
|
|
18
|
-
#### Request Body
|
|
19
|
-
\\\`\\\`\\\`json
|
|
20
|
-
[1,1,1,1... ], // transaction in atomic BEEF format as Uint8Array
|
|
21
|
-
\\\`\\\`\\\`
|
|
22
|
-
|
|
23
|
-
The engine processes this TaggedBEEF and generates a STEAK (Submitted Transaction Execution AcKnowledgment), which is returned to the client.
|
|
24
|
-
|
|
25
|
-
#### Response Body
|
|
26
|
-
\\\`\\\`\\\`json
|
|
27
|
-
{
|
|
28
|
-
"tm_messagebox": {
|
|
29
|
-
"outputsToAdmit": [0], // which of the outputs in the BEEF to admit
|
|
30
|
-
"coinsToRetain": [], // which of the inputs in the BEEF to mark spent but keep within the Topic
|
|
31
|
-
"coinsRemoved": [], // which of the inputs in the BEEF to remove from the Topic
|
|
32
|
-
},
|
|
33
|
-
... // per topic
|
|
34
|
-
}
|
|
35
|
-
\\\`\\\`\\\`
|
|
36
|
-
|
|
37
|
-
<br />
|
|
38
|
-
|
|
39
|
-
## Lookup Services
|
|
40
|
-
|
|
41
|
-
A Lookup Service is a component that enables data retrieval from the Overlay Services ecosystem. It allows clients to search and retrieve transactions that were previously submitted to Topic Managers, using specific criteria or identifiers.
|
|
42
|
-
|
|
43
|
-
### [POST] /lookup
|
|
44
|
-
|
|
45
|
-
#### Request Body
|
|
46
|
-
\\\`\\\`\\\`json
|
|
47
|
-
{
|
|
48
|
-
"service": "ls_messagebox", // Lookup Service identifier as string,
|
|
49
|
-
"query": "findAll" // query must be a string, JSON queries should be stringified
|
|
50
|
-
}
|
|
51
|
-
\\\`\\\`\\\`
|
|
52
|
-
|
|
53
|
-
#### Response Body
|
|
54
|
-
\\\`\\\`\\\`json
|
|
55
|
-
{
|
|
56
|
-
"type": "output-list",
|
|
57
|
-
"outputs": [{
|
|
58
|
-
"beef": [1, 1, 1...], // number[]
|
|
59
|
-
"outputIndex": 0, // number
|
|
60
|
-
"context"?: [1, 1, 1...], // number[]
|
|
61
|
-
}>]
|
|
62
|
-
}
|
|
63
|
-
\\\`\\\`\\\`
|
|
64
|
-
|
|
65
|
-
<br />
|
|
66
|
-
|
|
67
|
-
## Summary
|
|
68
|
-
|
|
69
|
-
Usually Topic Managers and Lookup Services are paired, but they can be separated for greater flexibility. Together, they form a complete system for data submission, organization, and retrieval within the Overlay Services framework.
|
|
70
|
-
|
|
71
|
-
<br />
|
|
72
|
-
<br />
|
|
73
|
-
\`
|
|
74
|
-
`;
|
|
75
|
-
//# sourceMappingURL=generalGuide.md.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"generalGuide.md.js","sourceRoot":"","sources":["../../../src/generalGuide.md.ts"],"names":[],"mappings":"AAAA,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyEd,CAAA"}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
import { Db } from 'mongodb';
|
|
2
|
-
/**
|
|
3
|
-
* Configuration for the Janitor Service
|
|
4
|
-
*/
|
|
5
|
-
export interface JanitorConfig {
|
|
6
|
-
mongoDb: Db;
|
|
7
|
-
logger?: typeof console;
|
|
8
|
-
requestTimeoutMs?: number;
|
|
9
|
-
hostDownRevokeScore?: number;
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* JanitorService runs a single pass of health checks on SHIP and SLAP outputs.
|
|
13
|
-
* It validates domain names and checks /health endpoints to ensure services are operational.
|
|
14
|
-
*
|
|
15
|
-
* When a service is down, it increments a "down" counter. When healthy, it decrements.
|
|
16
|
-
* If the down counter reaches HOST_DOWN_REVOKE_SCORE, it deletes the output from the database.
|
|
17
|
-
*
|
|
18
|
-
* This service is designed to be run periodically via external schedulers (e.g., cron, docker-compose).
|
|
19
|
-
*/
|
|
20
|
-
export declare class JanitorService {
|
|
21
|
-
private readonly mongoDb;
|
|
22
|
-
private readonly logger;
|
|
23
|
-
private readonly requestTimeoutMs;
|
|
24
|
-
private readonly hostDownRevokeScore;
|
|
25
|
-
constructor(config: JanitorConfig);
|
|
26
|
-
/**
|
|
27
|
-
* Runs a single pass of health checks on all SHIP and SLAP outputs
|
|
28
|
-
*/
|
|
29
|
-
run(): Promise<void>;
|
|
30
|
-
/**
|
|
31
|
-
* Checks all outputs for a specific topic
|
|
32
|
-
*/
|
|
33
|
-
private checkTopicOutputs;
|
|
34
|
-
/**
|
|
35
|
-
* Checks a single output for health
|
|
36
|
-
*/
|
|
37
|
-
private checkOutput;
|
|
38
|
-
/**
|
|
39
|
-
* Extracts URL from output record
|
|
40
|
-
*/
|
|
41
|
-
private extractURLFromOutput;
|
|
42
|
-
/**
|
|
43
|
-
* Validates if a string is a valid domain name
|
|
44
|
-
*/
|
|
45
|
-
private isValidDomain;
|
|
46
|
-
/**
|
|
47
|
-
* Checks the /health endpoint of a service
|
|
48
|
-
*/
|
|
49
|
-
private checkHealthEndpoint;
|
|
50
|
-
/**
|
|
51
|
-
* Handles a healthy output by decrementing its down counter
|
|
52
|
-
*/
|
|
53
|
-
private handleHealthyOutput;
|
|
54
|
-
/**
|
|
55
|
-
* Handles an unhealthy output by incrementing its down counter and deleting if threshold is reached
|
|
56
|
-
*/
|
|
57
|
-
private handleUnhealthyOutput;
|
|
58
|
-
}
|
|
59
|
-
//# sourceMappingURL=JanitorService.d.ts.map
|