@tradejs/cli 1.0.9 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +29497 -13537
- package/dist/lib/aiPocketSearch.js +1316 -0
- package/dist/lib/aiTrainCharts.js +711 -0
- package/dist/lib/aiTrainDataset.js +40 -0
- package/dist/lib/aiTrainEvaluationDump.js +76 -0
- package/dist/lib/aiTrainMetrics.js +148 -8
- package/dist/lib/aiTrainOptions.js +130 -0
- package/dist/lib/aiTrainQuarantine.js +199 -0
- package/dist/lib/aiTrainResearch.js +467 -0
- package/dist/lib/binanceMarketContextBackfill.js +841 -0
- package/dist/lib/binanceMarketData.js +392 -0
- package/dist/lib/coinMarketCapContextBackfill.js +1088 -0
- package/dist/lib/continuity.js +60 -0
- package/dist/lib/derivativesContextBackfill.js +490 -77
- package/dist/lib/executionCalibration.js +705 -0
- package/dist/lib/marketContextPrepare.js +4403 -0
- package/dist/lib/paritySummary.js +101 -0
- package/dist/lib/runBot.js +1 -0
- package/dist/lib/runEnvironment.js +2254 -0
- package/dist/lib/runFormatting.js +1657 -0
- package/dist/lib/runtimeDebugEvidence.js +526 -0
- package/dist/lib/runtimeModeConfig.js +53 -0
- package/dist/lib/runtimeParity.js +88 -11
- package/dist/lib/runtimeParityDetails.js +751 -0
- package/dist/lib/runtimeRedis.js +157 -0
- package/dist/lib/runtimeSignalsLoader.js +40 -6
- package/dist/lib/runtimeSignalsStorage.js +45 -22
- package/dist/lib/runtimeStrategyBacktest.js +134 -0
- package/dist/lib/runtimeTradeSync.js +263 -0
- package/dist/lib/telegramReports.js +9 -0
- package/dist/lib/tickerUniverseCache.js +116 -0
- package/dist/lib/timeWindow.js +3 -3
- package/dist/scripts/agentRun.js +14 -8
- package/dist/scripts/aiExport.js +1721 -7
- package/dist/scripts/aiExportSelect.js +16 -4
- package/dist/scripts/aiPocketSearch.js +4190 -0
- package/dist/scripts/aiTrain.js +1866 -87
- package/dist/scripts/backtest.js +4509 -818
- package/dist/scripts/binanceMarketIngest.js +565 -0
- package/dist/scripts/bot.js +16 -2
- package/dist/scripts/candlesMigrateProvider.js +32 -22
- package/dist/scripts/cleanDir.js +15 -2
- package/dist/scripts/cleanRedis.js +15 -2
- package/dist/scripts/cleanTests.js +38 -2
- package/dist/scripts/cleanupMarketContext.js +69 -0
- package/dist/scripts/continuity.js +148 -67
- package/dist/scripts/derivativesIngest.js +14 -4
- package/dist/scripts/derivativesIngestCoinalyzeAll.js +14 -4
- package/dist/scripts/doctor.js +14 -4
- package/dist/scripts/executionCalibration.js +792 -0
- package/dist/scripts/infraCommon.js +10 -0
- package/dist/scripts/infraDown.js +14 -4
- package/dist/scripts/infraInit.js +24 -4
- package/dist/scripts/infraUp.js +14 -4
- package/dist/scripts/marketWs.js +222 -0
- package/dist/scripts/migration.js +15 -2
- package/dist/scripts/mlExport.js +1698 -10
- package/dist/scripts/mlExportSelect.js +16 -4
- package/dist/scripts/mlInspect.js +13 -3
- package/dist/scripts/mlTrainLatestSelect.js +14 -4
- package/dist/scripts/replay.js +8824 -0
- package/dist/scripts/replayRunner.js +8827 -0
- package/dist/scripts/replayRuntimeEvidence.js +868 -0
- package/dist/scripts/researchAuto.js +286 -59
- package/dist/scripts/results.js +3 -3
- package/dist/scripts/runtimeEvidence.js +605 -0
- package/dist/scripts/runtimeParity.js +6362 -1043
- package/dist/scripts/serverHealth.js +588 -0
- package/dist/scripts/signals.js +5590 -511
- package/dist/scripts/signalsDaemon.js +6167 -0
- package/dist/scripts/signalsSummary.js +833 -248
- package/dist/scripts/test-ml.js +25 -3
- package/dist/scripts/test.js +26 -1
- package/dist/scripts/user-add.js +15 -2
- package/dist/workers/testerWorker.js +130 -21
- package/package.json +18 -12
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/scripts/serverHealth.ts
|
|
31
|
+
var serverHealth_exports = {};
|
|
32
|
+
__export(serverHealth_exports, {
|
|
33
|
+
buildAlertMessage: () => buildAlertMessage,
|
|
34
|
+
buildRecoveryMessage: () => buildRecoveryMessage,
|
|
35
|
+
buildServerHealthDiagnostics: () => buildServerHealthDiagnostics,
|
|
36
|
+
collectServerHealthSnapshot: () => collectServerHealthSnapshot,
|
|
37
|
+
evaluateServerHealth: () => evaluateServerHealth,
|
|
38
|
+
main: () => main,
|
|
39
|
+
resolveServerHealthThresholds: () => resolveServerHealthThresholds
|
|
40
|
+
});
|
|
41
|
+
module.exports = __toCommonJS(serverHealth_exports);
|
|
42
|
+
var import_config = require("dotenv/config");
|
|
43
|
+
var import_node_child_process = require("child_process");
|
|
44
|
+
var import_node_os = __toESM(require("os"));
|
|
45
|
+
var import_node_fs = require("fs");
|
|
46
|
+
var import_args = __toESM(require("args"));
|
|
47
|
+
var import_redis = require("@tradejs/infra/redis");
|
|
48
|
+
|
|
49
|
+
// src/lib/telegramReports.ts
|
|
50
|
+
var import_cli = require("@tradejs/node/cli");
|
|
51
|
+
var TELEGRAM_REPORT_CHUNK_LIMIT = 3900;
|
|
52
|
+
var buildPartHeader = (index, total) => `\u{1F4E9} <b>Part ${index}/${total}</b>`;
|
|
53
|
+
var splitLongLine = (line, limit) => {
|
|
54
|
+
const chunks = [];
|
|
55
|
+
for (let index = 0; index < line.length; index += limit) {
|
|
56
|
+
chunks.push(line.slice(index, index + limit));
|
|
57
|
+
}
|
|
58
|
+
return chunks;
|
|
59
|
+
};
|
|
60
|
+
var splitTelegramReport = (message, limit = TELEGRAM_REPORT_CHUNK_LIMIT) => {
|
|
61
|
+
const normalizedLimit = Math.max(100, Math.floor(limit));
|
|
62
|
+
if (message.length <= normalizedLimit) {
|
|
63
|
+
return [message];
|
|
64
|
+
}
|
|
65
|
+
const chunks = [];
|
|
66
|
+
let current = "";
|
|
67
|
+
const flush = () => {
|
|
68
|
+
if (!current) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
chunks.push(current);
|
|
72
|
+
current = "";
|
|
73
|
+
};
|
|
74
|
+
for (const line of message.split("\n")) {
|
|
75
|
+
const next = current ? `${current}
|
|
76
|
+
${line}` : line;
|
|
77
|
+
if (next.length <= normalizedLimit) {
|
|
78
|
+
current = next;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
flush();
|
|
82
|
+
if (line.length <= normalizedLimit) {
|
|
83
|
+
current = line;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
chunks.push(...splitLongLine(line, normalizedLimit));
|
|
87
|
+
}
|
|
88
|
+
flush();
|
|
89
|
+
return chunks;
|
|
90
|
+
};
|
|
91
|
+
var sendTelegramReport = async (message, options = {}) => {
|
|
92
|
+
const chunks = splitTelegramReport(message);
|
|
93
|
+
const sendAttachments = async () => {
|
|
94
|
+
for (const attachment of options.attachments || []) {
|
|
95
|
+
await (0, import_cli.sendDocumentToTG)(attachment, {
|
|
96
|
+
userName: options.userName
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
if (chunks.length === 1) {
|
|
101
|
+
await (0, import_cli.sendTextToTG)(chunks[0], options);
|
|
102
|
+
await sendAttachments();
|
|
103
|
+
return chunks.length;
|
|
104
|
+
}
|
|
105
|
+
for (const [index, chunk] of chunks.entries()) {
|
|
106
|
+
const header = buildPartHeader(index + 1, chunks.length);
|
|
107
|
+
await (0, import_cli.sendTextToTG)(`${header}
|
|
108
|
+
|
|
109
|
+
${chunk}`, {
|
|
110
|
+
userName: options.userName,
|
|
111
|
+
markup: index === chunks.length - 1 ? options.markup : void 0
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
await sendAttachments();
|
|
115
|
+
return chunks.length;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// src/scripts/serverHealth.ts
|
|
119
|
+
var DEFAULT_STATE = {
|
|
120
|
+
status: "healthy",
|
|
121
|
+
lastAlertAt: null,
|
|
122
|
+
lastOkAt: null,
|
|
123
|
+
activeIssueCodes: []
|
|
124
|
+
};
|
|
125
|
+
var DEFAULT_THRESHOLDS = {
|
|
126
|
+
loadPerCpuWarn: 0.9,
|
|
127
|
+
loadPerCpuRecover: 0.7,
|
|
128
|
+
memoryWarnPct: 92,
|
|
129
|
+
memoryRecoverPct: 88,
|
|
130
|
+
diskWarnPct: 95,
|
|
131
|
+
diskRecoverPct: 92
|
|
132
|
+
};
|
|
133
|
+
var escapeHtml = (value) => value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
134
|
+
var formatPctValue = (value) => `${value.toFixed(1)}%`;
|
|
135
|
+
var formatLoadPerCore = (value) => value.toFixed(2);
|
|
136
|
+
var formatGiB = (bytes) => `${(bytes / 1024 ** 3).toFixed(1)} GB`;
|
|
137
|
+
var formatDuration = (uptimeSec) => {
|
|
138
|
+
const total = Math.max(0, Math.floor(uptimeSec));
|
|
139
|
+
const days = Math.floor(total / 86400);
|
|
140
|
+
const hours = Math.floor(total % 86400 / 3600);
|
|
141
|
+
const minutes = Math.floor(total % 3600 / 60);
|
|
142
|
+
if (days > 0) {
|
|
143
|
+
return `${days}d ${hours}h ${minutes}m`;
|
|
144
|
+
}
|
|
145
|
+
if (hours > 0) {
|
|
146
|
+
return `${hours}h ${minutes}m`;
|
|
147
|
+
}
|
|
148
|
+
return `${minutes}m`;
|
|
149
|
+
};
|
|
150
|
+
var readDiskUsage = (path) => {
|
|
151
|
+
try {
|
|
152
|
+
const stats = (0, import_node_fs.statfsSync)(path);
|
|
153
|
+
const totalBlocks = Number(stats.blocks ?? 0);
|
|
154
|
+
const availableBlocks = Number(stats.bavail ?? 0);
|
|
155
|
+
if (totalBlocks <= 0) {
|
|
156
|
+
return { path, usedPct: null };
|
|
157
|
+
}
|
|
158
|
+
const usedPct = (totalBlocks - Math.max(0, availableBlocks)) / totalBlocks * 100;
|
|
159
|
+
return {
|
|
160
|
+
path,
|
|
161
|
+
usedPct: Number.isFinite(usedPct) ? usedPct : null
|
|
162
|
+
};
|
|
163
|
+
} catch {
|
|
164
|
+
return { path, usedPct: null };
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
var parseMeminfoBytes = (value) => {
|
|
168
|
+
const trimmed = value.trim();
|
|
169
|
+
if (!trimmed) {
|
|
170
|
+
return null;
|
|
171
|
+
}
|
|
172
|
+
const [amountRaw, unitRaw = "kB"] = trimmed.split(/\s+/, 2);
|
|
173
|
+
const amount = Number(amountRaw);
|
|
174
|
+
if (!Number.isFinite(amount)) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
const unit = unitRaw.toLowerCase();
|
|
178
|
+
if (unit === "kb") {
|
|
179
|
+
return Math.round(amount * 1024);
|
|
180
|
+
}
|
|
181
|
+
if (unit === "mb") {
|
|
182
|
+
return Math.round(amount * 1024 ** 2);
|
|
183
|
+
}
|
|
184
|
+
if (unit === "gb") {
|
|
185
|
+
return Math.round(amount * 1024 ** 3);
|
|
186
|
+
}
|
|
187
|
+
return Math.round(amount);
|
|
188
|
+
};
|
|
189
|
+
var readMemorySnapshot = () => {
|
|
190
|
+
const totalMem = import_node_os.default.totalmem();
|
|
191
|
+
const freeMem = import_node_os.default.freemem();
|
|
192
|
+
try {
|
|
193
|
+
const meminfo = (0, import_node_fs.readFileSync)("/proc/meminfo", "utf8");
|
|
194
|
+
const values = /* @__PURE__ */ new Map();
|
|
195
|
+
for (const line of meminfo.split("\n")) {
|
|
196
|
+
const match = line.match(/^([^:]+):\s*(.+)$/);
|
|
197
|
+
if (!match) {
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
const parsed = parseMeminfoBytes(match[2]);
|
|
201
|
+
if (parsed != null) {
|
|
202
|
+
values.set(match[1], parsed);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
totalBytes: values.get("MemTotal") ?? totalMem,
|
|
207
|
+
availableBytes: values.get("MemAvailable") ?? freeMem,
|
|
208
|
+
freeBytes: values.get("MemFree") ?? freeMem,
|
|
209
|
+
buffersBytes: values.get("Buffers") ?? null,
|
|
210
|
+
cachedBytes: values.get("Cached") ?? null,
|
|
211
|
+
shmemBytes: values.get("Shmem") ?? null,
|
|
212
|
+
slabBytes: values.get("Slab") ?? null,
|
|
213
|
+
reclaimableBytes: values.get("SReclaimable") ?? null
|
|
214
|
+
};
|
|
215
|
+
} catch {
|
|
216
|
+
return {
|
|
217
|
+
totalBytes: totalMem,
|
|
218
|
+
availableBytes: freeMem,
|
|
219
|
+
freeBytes: freeMem,
|
|
220
|
+
buffersBytes: null,
|
|
221
|
+
cachedBytes: null,
|
|
222
|
+
shmemBytes: null,
|
|
223
|
+
slabBytes: null,
|
|
224
|
+
reclaimableBytes: null
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
var runDiagnosticCommand = (command, args2) => {
|
|
229
|
+
const output = (0, import_node_child_process.execFileSync)(command, args2, {
|
|
230
|
+
encoding: "utf8",
|
|
231
|
+
timeout: 5e3,
|
|
232
|
+
maxBuffer: 1024 * 1024,
|
|
233
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
234
|
+
}).trim();
|
|
235
|
+
return output || "no output";
|
|
236
|
+
};
|
|
237
|
+
var runDiagnosticCommands = (commands, fallbackLabel) => {
|
|
238
|
+
const errors = [];
|
|
239
|
+
for (const candidate of commands) {
|
|
240
|
+
try {
|
|
241
|
+
return runDiagnosticCommand(candidate.command, candidate.args);
|
|
242
|
+
} catch (error) {
|
|
243
|
+
errors.push(
|
|
244
|
+
`${candidate.command} ${candidate.args.join(" ")} -> ${String(error)}`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return `${fallbackLabel}: ${errors.join("\n")}`;
|
|
249
|
+
};
|
|
250
|
+
var collectServerHealthSnapshot = (diskPath) => {
|
|
251
|
+
const [load1, load5, load15] = import_node_os.default.loadavg();
|
|
252
|
+
const cpuCount = Math.max(1, import_node_os.default.cpus().length);
|
|
253
|
+
const memory = readMemorySnapshot();
|
|
254
|
+
const usedMemPct = memory.totalBytes > 0 ? (memory.totalBytes - memory.availableBytes) / memory.totalBytes * 100 : 0;
|
|
255
|
+
return {
|
|
256
|
+
hostname: import_node_os.default.hostname(),
|
|
257
|
+
timestamp: Date.now(),
|
|
258
|
+
cpuCount,
|
|
259
|
+
load1,
|
|
260
|
+
load5,
|
|
261
|
+
load15,
|
|
262
|
+
loadPerCpu1: load1 / cpuCount,
|
|
263
|
+
loadPerCpu5: load5 / cpuCount,
|
|
264
|
+
memoryUsedPct: usedMemPct,
|
|
265
|
+
memory,
|
|
266
|
+
uptimeSec: import_node_os.default.uptime(),
|
|
267
|
+
disk: readDiskUsage(diskPath)
|
|
268
|
+
};
|
|
269
|
+
};
|
|
270
|
+
var buildHumanSummaryLines = (snapshot) => {
|
|
271
|
+
const usedMem = Math.max(
|
|
272
|
+
0,
|
|
273
|
+
snapshot.memory.totalBytes - snapshot.memory.availableBytes
|
|
274
|
+
);
|
|
275
|
+
return [
|
|
276
|
+
`\u2022 vCPU: <b>${escapeHtml(formatPctValue(Math.min(100, snapshot.loadPerCpu1 * 100)))}</b>`,
|
|
277
|
+
`\u2022 RAM: <b>${escapeHtml(formatPctValue(snapshot.memoryUsedPct))}</b> (${escapeHtml(
|
|
278
|
+
`${formatGiB(usedMem)} / ${formatGiB(snapshot.memory.totalBytes)}`
|
|
279
|
+
)})`,
|
|
280
|
+
`\u2022 Storage: <b>${escapeHtml(
|
|
281
|
+
snapshot.disk.usedPct == null ? "n/a" : formatPctValue(snapshot.disk.usedPct)
|
|
282
|
+
)}</b>`
|
|
283
|
+
];
|
|
284
|
+
};
|
|
285
|
+
var evaluateServerHealth = (snapshot, thresholds) => {
|
|
286
|
+
const issues = [];
|
|
287
|
+
if (snapshot.loadPerCpu1 >= thresholds.loadPerCpuWarn) {
|
|
288
|
+
issues.push({
|
|
289
|
+
code: "load",
|
|
290
|
+
summary: `CPU load high: 1m/core=${formatLoadPerCore(
|
|
291
|
+
snapshot.loadPerCpu1
|
|
292
|
+
)} (${snapshot.load1.toFixed(2)} on ${snapshot.cpuCount} cores), 5m/core=${formatLoadPerCore(snapshot.loadPerCpu5)}`
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
if (snapshot.memoryUsedPct >= thresholds.memoryWarnPct) {
|
|
296
|
+
issues.push({
|
|
297
|
+
code: "memory",
|
|
298
|
+
summary: `Memory high: used=${formatPctValue(snapshot.memoryUsedPct)}`
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
if (snapshot.disk.usedPct != null && snapshot.disk.usedPct >= thresholds.diskWarnPct) {
|
|
302
|
+
issues.push({
|
|
303
|
+
code: "disk",
|
|
304
|
+
summary: `Disk high: ${snapshot.disk.path} used=${formatPctValue(snapshot.disk.usedPct)}`
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
return {
|
|
308
|
+
snapshot,
|
|
309
|
+
issues,
|
|
310
|
+
isHealthy: issues.length === 0
|
|
311
|
+
};
|
|
312
|
+
};
|
|
313
|
+
var shouldRecover = (snapshot, thresholds) => snapshot.loadPerCpu1 < thresholds.loadPerCpuRecover && snapshot.memoryUsedPct < thresholds.memoryRecoverPct && (snapshot.disk.usedPct == null || snapshot.disk.usedPct < thresholds.diskRecoverPct);
|
|
314
|
+
var buildAlertMessage = ({
|
|
315
|
+
evaluation,
|
|
316
|
+
prevState,
|
|
317
|
+
reminder
|
|
318
|
+
}) => {
|
|
319
|
+
const { snapshot, issues } = evaluation;
|
|
320
|
+
const title = reminder ? "\u{1F6A8} <b>TradeJS server still unhealthy</b>" : "\u{1F6A8} <b>TradeJS server health alert</b>";
|
|
321
|
+
return [
|
|
322
|
+
title,
|
|
323
|
+
"",
|
|
324
|
+
`Host: <b>${escapeHtml(snapshot.hostname)}</b>`,
|
|
325
|
+
`Time: <b>${escapeHtml(new Date(snapshot.timestamp).toISOString())}</b>`,
|
|
326
|
+
`Uptime: <b>${escapeHtml(formatDuration(snapshot.uptimeSec))}</b>`,
|
|
327
|
+
"",
|
|
328
|
+
"Summary:",
|
|
329
|
+
...buildHumanSummaryLines(snapshot),
|
|
330
|
+
"",
|
|
331
|
+
"Issues:",
|
|
332
|
+
...issues.map((issue) => `\u2022 ${escapeHtml(issue.summary)}`),
|
|
333
|
+
"",
|
|
334
|
+
"Pressure:",
|
|
335
|
+
`\u2022 Load avg: <b>${snapshot.load1.toFixed(2)} / ${snapshot.load5.toFixed(2)} / ${snapshot.load15.toFixed(2)}</b>`,
|
|
336
|
+
`\u2022 Load per core: <b>${formatLoadPerCore(
|
|
337
|
+
snapshot.loadPerCpu1
|
|
338
|
+
)}</b> (1m), <b>${formatLoadPerCore(snapshot.loadPerCpu5)}</b> (5m)`,
|
|
339
|
+
prevState.lastAlertAt ? `\u2022 Previous alert: <b>${escapeHtml(new Date(prevState.lastAlertAt).toISOString())}</b>` : "\u2022 Previous alert: <b>none</b>"
|
|
340
|
+
].join("\n");
|
|
341
|
+
};
|
|
342
|
+
var buildRecoveryMessage = ({
|
|
343
|
+
snapshot,
|
|
344
|
+
prevState
|
|
345
|
+
}) => [
|
|
346
|
+
"\u2705 <b>TradeJS server health recovered</b>",
|
|
347
|
+
"",
|
|
348
|
+
`Host: <b>${escapeHtml(snapshot.hostname)}</b>`,
|
|
349
|
+
`Time: <b>${escapeHtml(new Date(snapshot.timestamp).toISOString())}</b>`,
|
|
350
|
+
`Uptime: <b>${escapeHtml(formatDuration(snapshot.uptimeSec))}</b>`,
|
|
351
|
+
"",
|
|
352
|
+
"Summary:",
|
|
353
|
+
...buildHumanSummaryLines(snapshot),
|
|
354
|
+
"",
|
|
355
|
+
"Pressure:",
|
|
356
|
+
`\u2022 Load avg: <b>${snapshot.load1.toFixed(2)} / ${snapshot.load5.toFixed(2)} / ${snapshot.load15.toFixed(2)}</b>`,
|
|
357
|
+
`\u2022 Load per core: <b>${formatLoadPerCore(
|
|
358
|
+
snapshot.loadPerCpu1
|
|
359
|
+
)}</b> (1m), <b>${formatLoadPerCore(snapshot.loadPerCpu5)}</b> (5m)`,
|
|
360
|
+
prevState.lastAlertAt ? `\u2022 Alert started: <b>${escapeHtml(new Date(prevState.lastAlertAt).toISOString())}</b>` : "\u2022 Alert started: <b>unknown</b>"
|
|
361
|
+
].join("\n");
|
|
362
|
+
var buildServerHealthDiagnostics = ({
|
|
363
|
+
snapshot,
|
|
364
|
+
diskPath
|
|
365
|
+
}) => {
|
|
366
|
+
const topCpu = runDiagnosticCommands(
|
|
367
|
+
[
|
|
368
|
+
{
|
|
369
|
+
command: "ps",
|
|
370
|
+
args: [
|
|
371
|
+
"axo",
|
|
372
|
+
"pid,ppid,%cpu,%mem,etime,state,comm,args",
|
|
373
|
+
"--sort=-%cpu"
|
|
374
|
+
]
|
|
375
|
+
},
|
|
376
|
+
{
|
|
377
|
+
command: "sh",
|
|
378
|
+
args: [
|
|
379
|
+
"-lc",
|
|
380
|
+
"ps -o pid,ppid,%cpu,%mem,etime,state,comm,args | sort -k3 -nr | head -20"
|
|
381
|
+
]
|
|
382
|
+
}
|
|
383
|
+
],
|
|
384
|
+
"ps cpu failed"
|
|
385
|
+
);
|
|
386
|
+
const topMemory = runDiagnosticCommands(
|
|
387
|
+
[
|
|
388
|
+
{
|
|
389
|
+
command: "ps",
|
|
390
|
+
args: [
|
|
391
|
+
"axo",
|
|
392
|
+
"pid,ppid,%mem,%cpu,etime,state,comm,args",
|
|
393
|
+
"--sort=-%mem"
|
|
394
|
+
]
|
|
395
|
+
},
|
|
396
|
+
{
|
|
397
|
+
command: "sh",
|
|
398
|
+
args: [
|
|
399
|
+
"-lc",
|
|
400
|
+
"ps -o pid,ppid,%mem,%cpu,etime,state,comm,args | sort -k3 -nr | head -20"
|
|
401
|
+
]
|
|
402
|
+
}
|
|
403
|
+
],
|
|
404
|
+
"ps memory failed"
|
|
405
|
+
);
|
|
406
|
+
const freeMemory = runDiagnosticCommands(
|
|
407
|
+
[{ command: "free", args: ["-m"] }],
|
|
408
|
+
"free failed"
|
|
409
|
+
);
|
|
410
|
+
const diskUsage = runDiagnosticCommands(
|
|
411
|
+
[{ command: "df", args: ["-h", diskPath] }],
|
|
412
|
+
"df failed"
|
|
413
|
+
);
|
|
414
|
+
const uptimeText = runDiagnosticCommands(
|
|
415
|
+
[{ command: "uptime", args: [] }],
|
|
416
|
+
"uptime failed"
|
|
417
|
+
);
|
|
418
|
+
const memoryDetails = [
|
|
419
|
+
`total: ${formatGiB(snapshot.memory.totalBytes)}`,
|
|
420
|
+
`available: ${formatGiB(snapshot.memory.availableBytes)}`,
|
|
421
|
+
`free: ${formatGiB(snapshot.memory.freeBytes)}`,
|
|
422
|
+
snapshot.memory.buffersBytes == null ? null : `buffers: ${formatGiB(snapshot.memory.buffersBytes)}`,
|
|
423
|
+
snapshot.memory.cachedBytes == null ? null : `cached: ${formatGiB(snapshot.memory.cachedBytes)}`,
|
|
424
|
+
snapshot.memory.shmemBytes == null ? null : `shmem: ${formatGiB(snapshot.memory.shmemBytes)}`,
|
|
425
|
+
snapshot.memory.reclaimableBytes == null ? null : `sreclaimable: ${formatGiB(snapshot.memory.reclaimableBytes)}`,
|
|
426
|
+
snapshot.memory.slabBytes == null ? null : `slab: ${formatGiB(snapshot.memory.slabBytes)}`
|
|
427
|
+
].filter(Boolean).join("\n");
|
|
428
|
+
return [
|
|
429
|
+
`TradeJS server health diagnostics`,
|
|
430
|
+
`host: ${snapshot.hostname}`,
|
|
431
|
+
`time: ${new Date(snapshot.timestamp).toISOString()}`,
|
|
432
|
+
`uptime: ${formatDuration(snapshot.uptimeSec)}`,
|
|
433
|
+
`load: ${snapshot.load1.toFixed(2)} / ${snapshot.load5.toFixed(2)} / ${snapshot.load15.toFixed(2)}`,
|
|
434
|
+
`load_per_core: ${formatLoadPerCore(snapshot.loadPerCpu1)} (1m), ${formatLoadPerCore(snapshot.loadPerCpu5)} (5m)`,
|
|
435
|
+
`memory_used: ${formatPctValue(snapshot.memoryUsedPct)}`,
|
|
436
|
+
`disk_used: ${snapshot.disk.usedPct == null ? `${snapshot.disk.path} n/a` : `${snapshot.disk.path} ${formatPctValue(snapshot.disk.usedPct)}`}`,
|
|
437
|
+
"",
|
|
438
|
+
"=== uptime ===",
|
|
439
|
+
uptimeText,
|
|
440
|
+
"",
|
|
441
|
+
"=== free -m ===",
|
|
442
|
+
freeMemory,
|
|
443
|
+
"",
|
|
444
|
+
"=== memory breakdown ===",
|
|
445
|
+
memoryDetails,
|
|
446
|
+
"",
|
|
447
|
+
`=== df -h ${diskPath} ===`,
|
|
448
|
+
diskUsage,
|
|
449
|
+
"",
|
|
450
|
+
"=== top processes by cpu ===",
|
|
451
|
+
topCpu,
|
|
452
|
+
"",
|
|
453
|
+
"=== top processes by memory ===",
|
|
454
|
+
topMemory,
|
|
455
|
+
""
|
|
456
|
+
].join("\n");
|
|
457
|
+
};
|
|
458
|
+
var getStateKey = (userName, hostname) => `ops:server-health:${userName}:${hostname}`;
|
|
459
|
+
var toFiniteNumber = (value) => {
|
|
460
|
+
const parsed = Number(value);
|
|
461
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
462
|
+
};
|
|
463
|
+
var resolveServerHealthThresholds = (raw) => {
|
|
464
|
+
const loadPerCpuWarn = toFiniteNumber(raw.loadPerCpuWarn);
|
|
465
|
+
const loadPerCpuRecover = toFiniteNumber(raw.loadPerCpuRecover);
|
|
466
|
+
const memoryWarnPct = toFiniteNumber(raw.memoryWarnPct);
|
|
467
|
+
const memoryRecoverPct = toFiniteNumber(raw.memoryRecoverPct);
|
|
468
|
+
const diskWarnPct = toFiniteNumber(raw.diskWarnPct);
|
|
469
|
+
const diskRecoverPct = toFiniteNumber(raw.diskRecoverPct);
|
|
470
|
+
return {
|
|
471
|
+
loadPerCpuWarn: loadPerCpuWarn != null && loadPerCpuWarn > 0 ? loadPerCpuWarn : DEFAULT_THRESHOLDS.loadPerCpuWarn,
|
|
472
|
+
loadPerCpuRecover: loadPerCpuRecover != null && loadPerCpuRecover > 0 ? loadPerCpuRecover : DEFAULT_THRESHOLDS.loadPerCpuRecover,
|
|
473
|
+
memoryWarnPct: memoryWarnPct != null && memoryWarnPct > 0 ? memoryWarnPct : DEFAULT_THRESHOLDS.memoryWarnPct,
|
|
474
|
+
memoryRecoverPct: memoryRecoverPct != null && memoryRecoverPct > 0 ? memoryRecoverPct : DEFAULT_THRESHOLDS.memoryRecoverPct,
|
|
475
|
+
diskWarnPct: diskWarnPct != null && diskWarnPct > 0 ? diskWarnPct : DEFAULT_THRESHOLDS.diskWarnPct,
|
|
476
|
+
diskRecoverPct: diskRecoverPct != null && diskRecoverPct > 0 ? diskRecoverPct : DEFAULT_THRESHOLDS.diskRecoverPct
|
|
477
|
+
};
|
|
478
|
+
};
|
|
479
|
+
import_args.default.option(["u", "user"], "Use user config", "root");
|
|
480
|
+
import_args.default.option(["P", "printOnly"], "Print report instead of Telegram", false);
|
|
481
|
+
import_args.default.option(
|
|
482
|
+
["d", "diskPath"],
|
|
483
|
+
"Filesystem path to inspect for disk pressure",
|
|
484
|
+
"/"
|
|
485
|
+
);
|
|
486
|
+
import_args.default.option(
|
|
487
|
+
["c", "cooldownSec"],
|
|
488
|
+
"Minimum seconds between repeated unhealthy reminders",
|
|
489
|
+
900
|
|
490
|
+
);
|
|
491
|
+
import_args.default.option(
|
|
492
|
+
["l", "loadPerCpuWarn"],
|
|
493
|
+
"Alert when 1m load per CPU reaches this value",
|
|
494
|
+
String(DEFAULT_THRESHOLDS.loadPerCpuWarn)
|
|
495
|
+
);
|
|
496
|
+
import_args.default.option(
|
|
497
|
+
["L", "loadPerCpuRecover"],
|
|
498
|
+
"Recovery threshold for 1m load per CPU",
|
|
499
|
+
String(DEFAULT_THRESHOLDS.loadPerCpuRecover)
|
|
500
|
+
);
|
|
501
|
+
import_args.default.option(
|
|
502
|
+
["m", "memoryWarnPct"],
|
|
503
|
+
"Alert when memory used percent reaches this value",
|
|
504
|
+
String(DEFAULT_THRESHOLDS.memoryWarnPct)
|
|
505
|
+
);
|
|
506
|
+
import_args.default.option(
|
|
507
|
+
["M", "memoryRecoverPct"],
|
|
508
|
+
"Recovery threshold for memory used percent",
|
|
509
|
+
String(DEFAULT_THRESHOLDS.memoryRecoverPct)
|
|
510
|
+
);
|
|
511
|
+
import_args.default.option(
|
|
512
|
+
["D", "diskWarnPct"],
|
|
513
|
+
"Alert when disk used percent reaches this value",
|
|
514
|
+
String(DEFAULT_THRESHOLDS.diskWarnPct)
|
|
515
|
+
);
|
|
516
|
+
import_args.default.option(
|
|
517
|
+
["R", "diskRecoverPct"],
|
|
518
|
+
"Recovery threshold for disk used percent",
|
|
519
|
+
String(DEFAULT_THRESHOLDS.diskRecoverPct)
|
|
520
|
+
);
|
|
521
|
+
var main = async () => {
|
|
522
|
+
const flags = import_args.default.parse(process.argv);
|
|
523
|
+
const thresholds = resolveServerHealthThresholds(flags);
|
|
524
|
+
const diskPath = String(flags.diskPath || "/");
|
|
525
|
+
const snapshot = collectServerHealthSnapshot(diskPath);
|
|
526
|
+
const stateKey = getStateKey(String(flags.user), snapshot.hostname);
|
|
527
|
+
const prevState = await (0, import_redis.getData)(stateKey, DEFAULT_STATE) ?? DEFAULT_STATE;
|
|
528
|
+
const evaluation = evaluateServerHealth(snapshot, thresholds);
|
|
529
|
+
const cooldownMs = Math.max(60, Number(flags.cooldownSec)) * 1e3;
|
|
530
|
+
const now = snapshot.timestamp;
|
|
531
|
+
if (!evaluation.isHealthy) {
|
|
532
|
+
const reminder = prevState.status === "alerting" && typeof prevState.lastAlertAt === "number" && now - prevState.lastAlertAt < cooldownMs ? false : prevState.status === "alerting";
|
|
533
|
+
const shouldSend = prevState.status !== "alerting" || !prevState.lastAlertAt || now - prevState.lastAlertAt >= cooldownMs;
|
|
534
|
+
const nextState2 = {
|
|
535
|
+
status: "alerting",
|
|
536
|
+
lastAlertAt: shouldSend ? now : prevState.lastAlertAt,
|
|
537
|
+
lastOkAt: prevState.lastOkAt,
|
|
538
|
+
activeIssueCodes: evaluation.issues.map((issue) => issue.code)
|
|
539
|
+
};
|
|
540
|
+
await (0, import_redis.setData)(stateKey, nextState2, { expire: 0 });
|
|
541
|
+
if (shouldSend) {
|
|
542
|
+
const message = buildAlertMessage({
|
|
543
|
+
evaluation,
|
|
544
|
+
prevState,
|
|
545
|
+
reminder
|
|
546
|
+
});
|
|
547
|
+
if (flags.printOnly) {
|
|
548
|
+
console.log(message);
|
|
549
|
+
console.log("");
|
|
550
|
+
console.log(
|
|
551
|
+
buildServerHealthDiagnostics({
|
|
552
|
+
snapshot,
|
|
553
|
+
diskPath
|
|
554
|
+
})
|
|
555
|
+
);
|
|
556
|
+
} else {
|
|
557
|
+
await sendTelegramReport(message, { userName: flags.user });
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return;
|
|
561
|
+
}
|
|
562
|
+
const recovered = prevState.status === "alerting" && shouldRecover(snapshot, thresholds);
|
|
563
|
+
const nextState = {
|
|
564
|
+
status: "healthy",
|
|
565
|
+
lastAlertAt: prevState.lastAlertAt,
|
|
566
|
+
lastOkAt: now,
|
|
567
|
+
activeIssueCodes: []
|
|
568
|
+
};
|
|
569
|
+
await (0, import_redis.setData)(stateKey, nextState, { expire: 0 });
|
|
570
|
+
if (recovered) {
|
|
571
|
+
const message = buildRecoveryMessage({ snapshot, prevState });
|
|
572
|
+
if (flags.printOnly) {
|
|
573
|
+
console.log(message);
|
|
574
|
+
} else {
|
|
575
|
+
await sendTelegramReport(message, { userName: flags.user });
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
580
|
+
0 && (module.exports = {
|
|
581
|
+
buildAlertMessage,
|
|
582
|
+
buildRecoveryMessage,
|
|
583
|
+
buildServerHealthDiagnostics,
|
|
584
|
+
collectServerHealthSnapshot,
|
|
585
|
+
evaluateServerHealth,
|
|
586
|
+
main,
|
|
587
|
+
resolveServerHealthThresholds
|
|
588
|
+
});
|