midline-agent 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +288 -252
- package/dist/agent.d.ts +105 -6
- package/dist/agent.js +663 -102
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +63 -0
- package/dist/config.d.ts +63 -0
- package/dist/config.js +230 -0
- package/dist/context.d.ts +13 -0
- package/dist/context.js +38 -0
- package/dist/errorHandler.d.ts +11 -2
- package/dist/errorHandler.js +32 -12
- package/dist/index.d.ts +9 -2
- package/dist/index.js +9 -1
- package/dist/middleware.d.ts +19 -2
- package/dist/middleware.js +92 -41
- package/dist/proxy.d.ts +70 -0
- package/dist/proxy.js +383 -0
- package/dist/redact.d.ts +35 -0
- package/dist/redact.js +223 -0
- package/dist/tap.d.ts +18 -0
- package/dist/tap.js +62 -0
- package/dist/transport.d.ts +35 -0
- package/dist/transport.js +133 -0
- package/dist/types.d.ts +107 -5
- package/package.json +13 -8
- package/src/agent.ts +734 -102
- package/src/cli.ts +69 -0
- package/src/config.ts +232 -0
- package/src/context.ts +48 -0
- package/src/errorHandler.ts +36 -13
- package/src/index.ts +19 -2
- package/src/middleware.ts +118 -43
- package/src/proxy.ts +435 -0
- package/src/redact.ts +231 -0
- package/src/tap.ts +55 -0
- package/src/transport.ts +125 -0
- package/src/types.ts +144 -31
- package/test/agent.test.js +353 -0
- package/test/helpers.js +151 -0
- package/test/middleware.test.js +178 -0
- package/test/proxy.test.js +274 -0
- package/test/redact.test.js +105 -0
package/dist/agent.js
CHANGED
|
@@ -1,123 +1,684 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.MidlineAgent = void 0;
|
|
7
|
-
const
|
|
3
|
+
exports.MidlineAgent = exports.SDK_VERSION = void 0;
|
|
4
|
+
const config_1 = require("./config");
|
|
5
|
+
const redact_1 = require("./redact");
|
|
6
|
+
const transport_1 = require("./transport");
|
|
7
|
+
exports.SDK_VERSION = (() => {
|
|
8
|
+
try {
|
|
9
|
+
return require("../package.json").version;
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return "unknown";
|
|
13
|
+
}
|
|
14
|
+
})();
|
|
15
|
+
const MAX_REQUEST_BYTES = 512 * 1024;
|
|
16
|
+
const MIN_REQUEST_BYTES = 16 * 1024;
|
|
17
|
+
const MAX_QUEUE_BYTES = 16 * 1024 * 1024;
|
|
18
|
+
/** Verification failures. Retrying is still right: they clear once the server is fixed. */
|
|
19
|
+
const TLS_ERROR_REASONS = {
|
|
20
|
+
DEPTH_ZERO_SELF_SIGNED_CERT: "presented a self-signed certificate",
|
|
21
|
+
SELF_SIGNED_CERT_IN_CHAIN: "presented a chain that ends in an untrusted self-signed CA",
|
|
22
|
+
UNABLE_TO_VERIFY_LEAF_SIGNATURE: "presented a certificate whose issuer could not be verified (missing intermediate, or a private CA)",
|
|
23
|
+
UNABLE_TO_GET_ISSUER_CERT: "presented a certificate whose issuer could not be found",
|
|
24
|
+
UNABLE_TO_GET_ISSUER_CERT_LOCALLY: "presented a certificate whose issuer is not in the trust store (missing intermediate, or a private CA)",
|
|
25
|
+
CERT_HAS_EXPIRED: "presented an expired certificate",
|
|
26
|
+
CERT_NOT_YET_VALID: "presented a certificate that is not valid yet (check this machine's clock)",
|
|
27
|
+
CERT_REVOKED: "presented a revoked certificate",
|
|
28
|
+
CERT_UNTRUSTED: "presented an untrusted certificate",
|
|
29
|
+
CERT_REJECTED: "presented a rejected certificate",
|
|
30
|
+
ERR_TLS_CERT_ALTNAME_INVALID: "presented a certificate issued for a different hostname",
|
|
31
|
+
HOSTNAME_MISMATCH: "presented a certificate issued for a different hostname",
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Ships request/error events to the Midline server.
|
|
35
|
+
*
|
|
36
|
+
* The contract with the host application is that the agent is never allowed to
|
|
37
|
+
* affect it: an unreachable, untrusted, slow or misconfigured Midline server costs a
|
|
38
|
+
* single explanatory log line and bounded buffered memory — never a crash, never a
|
|
39
|
+
* stalled request, never a disabled certificate check.
|
|
40
|
+
*/
|
|
8
41
|
class MidlineAgent {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
};
|
|
18
|
-
this.startQueue();
|
|
42
|
+
/** Creates the process-wide agent used by `midlineMiddleware()` and the static helpers. */
|
|
43
|
+
static init(config = {}) {
|
|
44
|
+
MidlineAgent.defaultAgent?.close();
|
|
45
|
+
MidlineAgent.defaultAgent = new MidlineAgent(config);
|
|
46
|
+
return MidlineAgent.defaultAgent;
|
|
47
|
+
}
|
|
48
|
+
static get current() {
|
|
49
|
+
return MidlineAgent.defaultAgent;
|
|
19
50
|
}
|
|
20
51
|
static addEvent(event) {
|
|
21
|
-
|
|
52
|
+
MidlineAgent.defaultAgent?.addEvent(event);
|
|
53
|
+
}
|
|
54
|
+
static flush() {
|
|
55
|
+
return MidlineAgent.defaultAgent?.flush() ?? Promise.resolve();
|
|
56
|
+
}
|
|
57
|
+
static shutdown(timeoutMs) {
|
|
58
|
+
return MidlineAgent.defaultAgent?.shutdown(timeoutMs) ?? Promise.resolve();
|
|
59
|
+
}
|
|
60
|
+
constructor(config = {}) {
|
|
61
|
+
this.config = null;
|
|
62
|
+
this.transport = null;
|
|
63
|
+
this.queue = [];
|
|
64
|
+
this.queueBytes = 0;
|
|
65
|
+
this.timer = null;
|
|
66
|
+
this.drainPromise = null;
|
|
67
|
+
this.failures = 0;
|
|
68
|
+
this.retryAfter = 0;
|
|
69
|
+
this.stopped = false;
|
|
70
|
+
this.lastNotice = "";
|
|
71
|
+
this.dropped = 0;
|
|
72
|
+
this.maxRequestBytes = MAX_REQUEST_BYTES;
|
|
73
|
+
this.batchLimit = Number.MAX_SAFE_INTEGER;
|
|
74
|
+
this.onErrorHook = config.onError;
|
|
75
|
+
this.debugLogs = config.debug ?? (0, config_1.envFlag)("MIDLINE_DEBUG") ?? false;
|
|
76
|
+
this.redactor = new redact_1.Redactor([...(config.redactFields ?? []), ...(config.maskFields ?? [])], config.redactHeaders);
|
|
77
|
+
if ((config.enabled ?? (0, config_1.envFlag)("MIDLINE_ENABLED")) === false) {
|
|
78
|
+
this.stopped = true;
|
|
22
79
|
return;
|
|
23
80
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
81
|
+
let resolved;
|
|
82
|
+
try {
|
|
83
|
+
resolved = (0, config_1.resolveConfig)(config);
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
this.stopped = true;
|
|
87
|
+
const detail = err instanceof config_1.ConfigError ? err.message : String(err?.message ?? err);
|
|
88
|
+
this.log("error", `midline: ${detail} — monitoring is off.`);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (!resolved.apiKey) {
|
|
92
|
+
this.stopped = true;
|
|
93
|
+
this.log("warn", "midline: no apiKey (or MIDLINE_API_KEY) — monitoring is off.");
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
this.config = resolved;
|
|
97
|
+
this.transport = new transport_1.Transport(resolved.ingestUrl, {
|
|
98
|
+
ca: resolved.ca,
|
|
99
|
+
connectTimeoutMs: resolved.connectTimeoutMs,
|
|
100
|
+
timeoutMs: resolved.timeoutMs,
|
|
101
|
+
userAgent: `midline-agent/${exports.SDK_VERSION} node/${process.version}`,
|
|
102
|
+
});
|
|
103
|
+
this.timer = setInterval(() => {
|
|
104
|
+
void this.drain(false);
|
|
105
|
+
}, resolved.flushIntervalMs);
|
|
106
|
+
// Telemetry must never be the reason a process refuses to exit.
|
|
107
|
+
this.timer.unref?.();
|
|
108
|
+
}
|
|
109
|
+
/** False when the agent is off: no key, bad config, disabled, rejected key, or closed. */
|
|
110
|
+
get active() {
|
|
111
|
+
return !this.stopped && this.config !== null;
|
|
112
|
+
}
|
|
113
|
+
get capture() {
|
|
114
|
+
return this.config?.capture ?? null;
|
|
115
|
+
}
|
|
116
|
+
get queued() {
|
|
117
|
+
return this.queue.length;
|
|
118
|
+
}
|
|
119
|
+
addEvent(event) {
|
|
120
|
+
if (!this.active)
|
|
121
|
+
return;
|
|
122
|
+
try {
|
|
123
|
+
this.enqueue(this.toWire(event, false));
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
this.report("event-build", `midline: could not build an event (${err?.message}); skipped it.`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/** Records one HTTP exchange. Captured parts are redacted here, before queueing. */
|
|
130
|
+
recordHttp(exchange) {
|
|
131
|
+
if (!this.active)
|
|
132
|
+
return;
|
|
133
|
+
try {
|
|
134
|
+
const capture = exchange.capture ?? this.config.capture;
|
|
135
|
+
const queryIndex = exchange.url.indexOf("?");
|
|
136
|
+
const path = queryIndex === -1 ? exchange.url : exchange.url.slice(0, queryIndex);
|
|
137
|
+
const search = queryIndex === -1 ? "" : exchange.url.slice(queryIndex + 1);
|
|
138
|
+
const failed = Boolean(exchange.errorCode);
|
|
139
|
+
const event = {
|
|
140
|
+
type: failed ? "error" : "request",
|
|
141
|
+
path: path || "/",
|
|
142
|
+
method: exchange.method,
|
|
143
|
+
statusCode: exchange.statusCode,
|
|
144
|
+
duration: exchange.durationMs,
|
|
145
|
+
ip: exchange.ip,
|
|
146
|
+
userAgent: exchange.userAgent,
|
|
147
|
+
requestId: exchange.context.requestId,
|
|
148
|
+
correlationId: exchange.context.correlationId,
|
|
149
|
+
traceId: exchange.context.traceId,
|
|
150
|
+
spanId: exchange.context.spanId,
|
|
151
|
+
routeTemplate: exchange.routeTemplate,
|
|
152
|
+
integration: exchange.integration,
|
|
153
|
+
destination: exchange.destination,
|
|
154
|
+
errorCode: exchange.errorCode,
|
|
155
|
+
message: exchange.errorMessage,
|
|
156
|
+
aborted: exchange.aborted || undefined,
|
|
157
|
+
severity: failed ? "high" : undefined,
|
|
158
|
+
category: failed && exchange.integration === "proxy" ? "infrastructure" : undefined,
|
|
159
|
+
request: this.captureMessage(capture, exchange.request, search),
|
|
160
|
+
response: this.captureMessage(capture, exchange.response),
|
|
71
161
|
};
|
|
162
|
+
this.enqueue(this.toWire(event, true));
|
|
72
163
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
164
|
+
catch (err) {
|
|
165
|
+
this.report("event-build", `midline: could not record a request (${err?.message}); skipped it.`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/** Sends whatever is buffered now, ignoring backoff, and keeps the process alive until done. */
|
|
169
|
+
async flush() {
|
|
170
|
+
if (!this.active)
|
|
171
|
+
return;
|
|
172
|
+
this.retryAfter = 0;
|
|
173
|
+
if (this.drainPromise) {
|
|
174
|
+
await this.drainPromise;
|
|
175
|
+
}
|
|
176
|
+
await this.drain(true);
|
|
177
|
+
}
|
|
178
|
+
/** Flushes with a deadline, then closes. For graceful shutdown. */
|
|
179
|
+
async shutdown(timeoutMs = 5000) {
|
|
180
|
+
if (this.active) {
|
|
181
|
+
let timeout;
|
|
182
|
+
await Promise.race([
|
|
183
|
+
this.flush().catch(() => undefined),
|
|
184
|
+
new Promise((resolve) => {
|
|
185
|
+
timeout = setTimeout(resolve, timeoutMs);
|
|
186
|
+
}),
|
|
187
|
+
]);
|
|
188
|
+
if (timeout)
|
|
189
|
+
clearTimeout(timeout);
|
|
190
|
+
}
|
|
191
|
+
this.close();
|
|
192
|
+
}
|
|
193
|
+
/** Stops the timer, drops the buffer and releases sockets. Safe to call more than once. */
|
|
194
|
+
close() {
|
|
195
|
+
this.stopped = true;
|
|
196
|
+
if (this.timer) {
|
|
197
|
+
clearInterval(this.timer);
|
|
198
|
+
this.timer = null;
|
|
199
|
+
}
|
|
200
|
+
this.queue = [];
|
|
201
|
+
this.queueBytes = 0;
|
|
202
|
+
this.transport?.destroy();
|
|
203
|
+
}
|
|
204
|
+
captureMessage(capture, message, search) {
|
|
205
|
+
if (!message && !search)
|
|
206
|
+
return undefined;
|
|
207
|
+
const out = {};
|
|
208
|
+
if (capture.headers && message?.headers) {
|
|
209
|
+
out.headers = this.redactor.headers(message.headers);
|
|
210
|
+
}
|
|
211
|
+
if (capture.query && search) {
|
|
212
|
+
out.query = this.redactor.query(search);
|
|
213
|
+
}
|
|
214
|
+
if (message && message.body !== undefined) {
|
|
215
|
+
Object.assign(out, this.redactor.body(message.body, message.contentType, capture.maxBodyBytes));
|
|
216
|
+
if (message.truncated)
|
|
217
|
+
out.truncated = true;
|
|
218
|
+
}
|
|
219
|
+
if (message?.bodyBytes !== undefined && (out.body !== undefined || out.omitted)) {
|
|
220
|
+
out.bodyBytes = message.bodyBytes;
|
|
221
|
+
}
|
|
222
|
+
return Object.keys(out).length ? out : undefined;
|
|
223
|
+
}
|
|
224
|
+
redactManualMessage(message) {
|
|
225
|
+
if (!message)
|
|
226
|
+
return undefined;
|
|
227
|
+
const maxBodyBytes = this.config.capture.maxBodyBytes || 4096;
|
|
228
|
+
return {
|
|
229
|
+
...message,
|
|
230
|
+
headers: this.redactor.headers(message.headers),
|
|
231
|
+
query: message.query ? this.redactor.value(message.query) : undefined,
|
|
232
|
+
...(message.body !== undefined ? this.redactor.body(message.body, undefined, maxBodyBytes) : {}),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Maps an event onto the ingest schema. Everything beyond the long-standing
|
|
237
|
+
* top-level fields travels in `metadata` and `payload`, which every Midline
|
|
238
|
+
* server version accepts — so upgrading the agent never gets events rejected by a
|
|
239
|
+
* server that hasn't been upgraded yet.
|
|
240
|
+
*/
|
|
241
|
+
toWire(event, preRedacted) {
|
|
242
|
+
const config = this.config;
|
|
243
|
+
const type = event.type;
|
|
244
|
+
const statusCode = Number.isInteger(event.statusCode) && event.statusCode >= 100 && event.statusCode <= 599
|
|
245
|
+
? event.statusCode
|
|
246
|
+
: type === "error" ? 500 : undefined;
|
|
247
|
+
const { severity, category } = classify(event, statusCode);
|
|
248
|
+
const route = this.redactor.string(stripQuery(event.path) || "/", 2048);
|
|
249
|
+
const payload = {};
|
|
250
|
+
if (type === "error" || event.message) {
|
|
251
|
+
payload.error = event.message ? this.redactor.string(event.message, 1024) : type === "error" ? "Unknown error" : undefined;
|
|
252
|
+
}
|
|
253
|
+
if (event.stack)
|
|
254
|
+
payload.stack = this.redactor.string(event.stack, 16 * 1024);
|
|
255
|
+
if (event.breadcrumbs?.length) {
|
|
256
|
+
payload.breadcrumbs = event.breadcrumbs.slice(-20).map((crumb) => ({
|
|
257
|
+
type: String(crumb.type).slice(0, 32),
|
|
258
|
+
message: this.redactor.string(String(crumb.message), 256),
|
|
259
|
+
timestamp: crumb.timestamp,
|
|
260
|
+
}));
|
|
261
|
+
}
|
|
262
|
+
if (type === "error") {
|
|
263
|
+
payload.context = { route, method: event.method };
|
|
264
|
+
}
|
|
265
|
+
const request = preRedacted ? event.request : this.redactManualMessage(event.request);
|
|
266
|
+
const response = preRedacted ? event.response : this.redactManualMessage(event.response);
|
|
267
|
+
if (request)
|
|
268
|
+
payload.request = request;
|
|
269
|
+
if (response)
|
|
270
|
+
payload.response = response;
|
|
271
|
+
if (event.destination)
|
|
272
|
+
payload.destination = { url: this.redactor.string(event.destination.url, 2048) };
|
|
273
|
+
if (event.errorCode)
|
|
274
|
+
payload.code = String(event.errorCode).slice(0, 64);
|
|
275
|
+
const metadata = {
|
|
276
|
+
source: "sdk",
|
|
277
|
+
sdk: "midline-agent",
|
|
278
|
+
version: exports.SDK_VERSION,
|
|
279
|
+
integrationType: event.integration ?? "manual",
|
|
280
|
+
};
|
|
281
|
+
if (event.requestId)
|
|
282
|
+
metadata.requestId = event.requestId;
|
|
283
|
+
if (event.correlationId)
|
|
284
|
+
metadata.correlationId = event.correlationId;
|
|
285
|
+
if (event.routeTemplate)
|
|
286
|
+
metadata.routeTemplate = event.routeTemplate.slice(0, 512);
|
|
287
|
+
if (event.aborted)
|
|
288
|
+
metadata.aborted = true;
|
|
289
|
+
const wire = {
|
|
290
|
+
apiKey: config.apiKey,
|
|
291
|
+
eventType: type,
|
|
292
|
+
route,
|
|
293
|
+
method: (event.method || "GET").toUpperCase().slice(0, 16),
|
|
294
|
+
statusCode,
|
|
295
|
+
responseTime: Math.min(Math.max(0, Math.round(Number(event.duration) || 0)), 86400000),
|
|
296
|
+
timestamp: validTimestamp(event.timestamp),
|
|
297
|
+
// Clamped to the server's validation limits: one over-long field would
|
|
298
|
+
// otherwise get every event rejected.
|
|
299
|
+
ip: clamp(event.ip, 64),
|
|
300
|
+
userAgent: clamp(event.userAgent, 512),
|
|
301
|
+
service: clamp(config.serviceName, 128),
|
|
302
|
+
environment: clamp(config.environment, 128),
|
|
303
|
+
host: clamp(config.host, 256),
|
|
304
|
+
region: clamp(config.region, 64),
|
|
305
|
+
release: clamp(config.release, 128),
|
|
306
|
+
severity,
|
|
307
|
+
category,
|
|
308
|
+
traceId: clamp(event.traceId, 128),
|
|
309
|
+
spanId: clamp(event.spanId, 128),
|
|
310
|
+
ruleId: clamp(event.ruleId, 128),
|
|
311
|
+
threatDetected: typeof event.threatDetected === "boolean" ? event.threatDetected : undefined,
|
|
312
|
+
metadata,
|
|
313
|
+
payload: Object.keys(payload).length ? payload : undefined,
|
|
314
|
+
};
|
|
315
|
+
for (const key of Object.keys(wire)) {
|
|
316
|
+
if (wire[key] === undefined)
|
|
317
|
+
delete wire[key];
|
|
318
|
+
}
|
|
319
|
+
return wire;
|
|
320
|
+
}
|
|
321
|
+
enqueue(wire) {
|
|
322
|
+
const config = this.config;
|
|
323
|
+
let bytes = Buffer.byteLength(JSON.stringify(wire));
|
|
324
|
+
if (bytes > config.maxEventBytes) {
|
|
325
|
+
bytes = shrink(wire, config.maxEventBytes);
|
|
326
|
+
if (bytes > config.maxEventBytes) {
|
|
327
|
+
this.dropped += 1;
|
|
328
|
+
this.report("event-too-large", `midline: an event was larger than maxEventBytes (${config.maxEventBytes}) even without bodies; dropped it.`);
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
this.queue.push({ wire, bytes });
|
|
333
|
+
this.queueBytes += bytes;
|
|
334
|
+
this.trimQueue();
|
|
335
|
+
}
|
|
336
|
+
/** Oldest events go first — during an outage the most recent minute is worth more than the first. */
|
|
337
|
+
trimQueue() {
|
|
338
|
+
const config = this.config;
|
|
339
|
+
while (this.queue.length > config.maxQueueSize || (this.queueBytes > MAX_QUEUE_BYTES && this.queue.length > 1)) {
|
|
340
|
+
const removed = this.queue.shift();
|
|
341
|
+
this.queueBytes -= removed.bytes;
|
|
342
|
+
this.dropped += 1;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
requeue(batch) {
|
|
346
|
+
this.queue.unshift(...batch);
|
|
347
|
+
this.queueBytes += batch.reduce((sum, item) => sum + item.bytes, 0);
|
|
348
|
+
this.trimQueue();
|
|
349
|
+
}
|
|
350
|
+
takeBatch() {
|
|
351
|
+
const config = this.config;
|
|
352
|
+
const batch = [];
|
|
353
|
+
let bytes = 0;
|
|
354
|
+
const limit = Math.min(config.maxBatchSize, this.batchLimit);
|
|
355
|
+
while (this.queue.length && batch.length < limit) {
|
|
356
|
+
const next = this.queue[0];
|
|
357
|
+
if (batch.length > 0 && bytes + next.bytes > this.maxRequestBytes)
|
|
358
|
+
break;
|
|
359
|
+
batch.push(this.queue.shift());
|
|
360
|
+
bytes += next.bytes;
|
|
361
|
+
this.queueBytes -= next.bytes;
|
|
362
|
+
}
|
|
363
|
+
return batch;
|
|
364
|
+
}
|
|
365
|
+
drain(keepProcessAlive) {
|
|
366
|
+
if (this.drainPromise)
|
|
367
|
+
return this.drainPromise;
|
|
368
|
+
if (!this.active || !this.queue.length || Date.now() < this.retryAfter) {
|
|
369
|
+
return Promise.resolve();
|
|
370
|
+
}
|
|
371
|
+
this.drainPromise = this.runDrain(keepProcessAlive).finally(() => {
|
|
372
|
+
this.drainPromise = null;
|
|
373
|
+
});
|
|
374
|
+
return this.drainPromise;
|
|
375
|
+
}
|
|
376
|
+
async runDrain(keepProcessAlive) {
|
|
377
|
+
while (this.queue.length && this.active) {
|
|
378
|
+
// Taken off the queue while in flight, so overflow trimming can't remove
|
|
379
|
+
// events that are mid-send and then be confused about what was accepted.
|
|
380
|
+
const batch = this.takeBatch();
|
|
381
|
+
const outcome = await this.send(batch, keepProcessAlive);
|
|
382
|
+
if (outcome.kind === "ok") {
|
|
383
|
+
this.onSuccess();
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (outcome.kind === "stop") {
|
|
387
|
+
this.disable();
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
if (outcome.kind === "tooLarge") {
|
|
391
|
+
if (batch.length > 1) {
|
|
392
|
+
// Halve by count as well as bytes: a byte floor alone would resend the
|
|
393
|
+
// same batch of small events forever. The server's limit doesn't move,
|
|
394
|
+
// so the smaller size sticks.
|
|
395
|
+
const batchBytes = batch.reduce((sum, item) => sum + item.bytes, 0);
|
|
396
|
+
this.batchLimit = Math.max(1, Math.floor(batch.length / 2));
|
|
397
|
+
this.maxRequestBytes = Math.max(MIN_REQUEST_BYTES, Math.min(this.maxRequestBytes, Math.floor(batchBytes / 2)));
|
|
398
|
+
this.requeue(batch);
|
|
85
399
|
}
|
|
86
400
|
else {
|
|
87
|
-
|
|
401
|
+
this.dropped += 1;
|
|
402
|
+
this.report("http-413", "midline: the Midline server rejected an event as too large (HTTP 413); dropped it.");
|
|
403
|
+
}
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (outcome.kind === "rejected") {
|
|
407
|
+
if (batch.length === 1) {
|
|
408
|
+
this.dropped += 1;
|
|
409
|
+
this.report(`rejected:${outcome.detail}`, `midline: the Midline server rejected an event (${outcome.detail}); dropped it.`);
|
|
410
|
+
continue;
|
|
88
411
|
}
|
|
412
|
+
// One malformed event fails validation for the whole batch. Send them one
|
|
413
|
+
// at a time so it only costs that event.
|
|
414
|
+
const isolated = await this.sendIndividually(batch, keepProcessAlive);
|
|
415
|
+
if (!isolated)
|
|
416
|
+
return;
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
this.requeue(batch);
|
|
420
|
+
this.onFailure(outcome.retryAfterMs);
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
/** Returns false if delivery should stop for this drain. */
|
|
425
|
+
async sendIndividually(batch, keepProcessAlive) {
|
|
426
|
+
for (let index = 0; index < batch.length; index++) {
|
|
427
|
+
const outcome = await this.send([batch[index]], keepProcessAlive);
|
|
428
|
+
if (outcome.kind === "ok") {
|
|
429
|
+
this.onSuccess();
|
|
430
|
+
}
|
|
431
|
+
else if (outcome.kind === "rejected" || outcome.kind === "tooLarge") {
|
|
432
|
+
this.dropped += 1;
|
|
433
|
+
const detail = outcome.kind === "rejected" ? outcome.detail : "HTTP 413";
|
|
434
|
+
this.report(`rejected:${detail}`, `midline: the Midline server rejected an event (${detail}); dropped it.`);
|
|
435
|
+
}
|
|
436
|
+
else if (outcome.kind === "stop") {
|
|
437
|
+
this.disable();
|
|
438
|
+
return false;
|
|
89
439
|
}
|
|
90
440
|
else {
|
|
91
|
-
|
|
441
|
+
this.requeue(batch.slice(index));
|
|
442
|
+
this.onFailure(outcome.retryAfterMs);
|
|
443
|
+
return false;
|
|
92
444
|
}
|
|
93
445
|
}
|
|
94
|
-
return
|
|
446
|
+
return true;
|
|
95
447
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
this.
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
448
|
+
async send(batch, keepProcessAlive) {
|
|
449
|
+
const config = this.config;
|
|
450
|
+
const body = JSON.stringify({ events: batch.map((item) => item.wire) });
|
|
451
|
+
let result;
|
|
452
|
+
try {
|
|
453
|
+
result = await this.transport.post(config.batchUrl, body, { "x-api-key": config.apiKey }, keepProcessAlive);
|
|
454
|
+
}
|
|
455
|
+
catch (err) {
|
|
456
|
+
this.report(`transport:${errorCode(err)}`, this.describe(err, batch.length));
|
|
457
|
+
return { kind: "retry" };
|
|
458
|
+
}
|
|
459
|
+
const { status } = result;
|
|
460
|
+
if (status >= 200 && status < 300) {
|
|
461
|
+
// Servers that predate 401-on-bad-key answer 201 and count the rejects.
|
|
462
|
+
const summary = parseJson(result.body);
|
|
463
|
+
const failed = typeof summary?.failed === "number" ? summary.failed : 0;
|
|
464
|
+
if (failed >= batch.length && batch.length > 0) {
|
|
465
|
+
this.log("error", "midline: the Midline server did not accept this API key. Monitoring is now off — retrying wouldn't help. Check apiKey / MIDLINE_API_KEY.");
|
|
466
|
+
return { kind: "stop" };
|
|
467
|
+
}
|
|
468
|
+
if (failed > 0) {
|
|
469
|
+
this.dropped += failed;
|
|
470
|
+
this.report("partial", `midline: the Midline server rejected ${failed} of ${batch.length} events.`);
|
|
118
471
|
}
|
|
119
|
-
|
|
472
|
+
return { kind: "ok" };
|
|
473
|
+
}
|
|
474
|
+
if (status === 401 || status === 403) {
|
|
475
|
+
this.log("error", `midline: the Midline server rejected the API key (HTTP ${status}). ` +
|
|
476
|
+
"Monitoring is now off — retrying wouldn't help. Check apiKey / MIDLINE_API_KEY.");
|
|
477
|
+
return { kind: "stop" };
|
|
478
|
+
}
|
|
479
|
+
if (status === 413) {
|
|
480
|
+
return { kind: "tooLarge" };
|
|
481
|
+
}
|
|
482
|
+
if (status === 408 || status === 429 || status >= 500) {
|
|
483
|
+
const retryAfterMs = parseRetryAfter(result.headers["retry-after"]);
|
|
484
|
+
this.report(`http-${status}`, `midline: the Midline server returned HTTP ${status}; events are buffered and will be retried.`);
|
|
485
|
+
return { kind: "retry", retryAfterMs };
|
|
486
|
+
}
|
|
487
|
+
if (status >= 300 && status < 400) {
|
|
488
|
+
// Never followed: that would hand the API key to wherever the redirect points.
|
|
489
|
+
const location = String(result.headers.location ?? "").slice(0, 200);
|
|
490
|
+
this.report(`http-${status}`, `midline: the Midline endpoint redirected (HTTP ${status}${location ? ` to ${location}` : ""}). ` +
|
|
491
|
+
"Redirects are not followed; set MIDLINE_ENDPOINT to the final URL.");
|
|
492
|
+
return { kind: "retry" };
|
|
493
|
+
}
|
|
494
|
+
return { kind: "rejected", detail: `HTTP ${status}${serverMessage(result.body)}` };
|
|
495
|
+
}
|
|
496
|
+
/** One line per distinct fault, not one per failed event. */
|
|
497
|
+
report(signature, message) {
|
|
498
|
+
if (this.lastNotice === signature && !this.debugLogs) {
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
this.lastNotice = signature;
|
|
502
|
+
this.log("warn", message);
|
|
503
|
+
}
|
|
504
|
+
describe(err, inFlight) {
|
|
505
|
+
const config = this.config;
|
|
506
|
+
const origin = config.ingestUrl.origin;
|
|
507
|
+
const code = errorCode(err);
|
|
508
|
+
const buffered = ` ${this.queue.length + inFlight} event(s) buffered; your application is unaffected.`;
|
|
509
|
+
if (TLS_ERROR_REASONS[code] || code.startsWith("ERR_SSL") || code === "EPROTO") {
|
|
510
|
+
const reason = TLS_ERROR_REASONS[code] ?? "failed the TLS handshake";
|
|
511
|
+
return (`midline: ${origin} ${reason} (${code}), so the connection was refused. Certificate verification stays on. ` +
|
|
512
|
+
(config.hasCustomCa
|
|
513
|
+
? "The configured ca / MIDLINE_CUSTOM_CA did not validate it either. "
|
|
514
|
+
: "If this is the Midline server, its HTTPS certificate needs fixing on the server; for a self-hosted server behind a private CA, set ca / MIDLINE_CUSTOM_CA. ") +
|
|
515
|
+
`Delivery resumes on its own once a trusted certificate is served.${buffered}`);
|
|
516
|
+
}
|
|
517
|
+
if (code === "ETIMEDOUT") {
|
|
518
|
+
return `midline: ${origin} did not answer within ${config.timeoutMs}ms; retrying with backoff.${buffered}`;
|
|
519
|
+
}
|
|
520
|
+
if (code === "ECONNECT_TIMEOUT") {
|
|
521
|
+
return `midline: could not connect to ${origin} within ${config.connectTimeoutMs}ms; retrying with backoff.${buffered}`;
|
|
522
|
+
}
|
|
523
|
+
if (code === "ENOTFOUND" || code === "EAI_AGAIN") {
|
|
524
|
+
return `midline: cannot resolve ${config.ingestUrl.hostname} (${code}) — check MIDLINE_ENDPOINT and DNS; retrying with backoff.${buffered}`;
|
|
525
|
+
}
|
|
526
|
+
if (code === "ECONNREFUSED") {
|
|
527
|
+
return `midline: ${origin} refused the connection; retrying with backoff.${buffered}`;
|
|
528
|
+
}
|
|
529
|
+
if (code === "ECONNRESET" || code === "EPIPE") {
|
|
530
|
+
return `midline: the connection to ${origin} was reset; retrying with backoff.${buffered}`;
|
|
531
|
+
}
|
|
532
|
+
return `midline: could not reach ${origin} (${code || err?.message || "unknown error"}); retrying with backoff.${buffered}`;
|
|
533
|
+
}
|
|
534
|
+
onSuccess() {
|
|
535
|
+
if (this.failures > 0) {
|
|
536
|
+
const lost = this.dropped;
|
|
537
|
+
this.log("info", `midline: delivery to the Midline server recovered${lost ? `; ${lost} event(s) were dropped meanwhile` : ""}.`);
|
|
538
|
+
this.dropped = 0;
|
|
539
|
+
// Only a recovered outage resets de-duplication; a stream of individually
|
|
540
|
+
// rejected events between successes should still log once.
|
|
541
|
+
this.lastNotice = "";
|
|
542
|
+
}
|
|
543
|
+
this.failures = 0;
|
|
544
|
+
this.retryAfter = 0;
|
|
545
|
+
}
|
|
546
|
+
onFailure(retryAfterMs) {
|
|
547
|
+
const config = this.config;
|
|
548
|
+
this.failures += 1;
|
|
549
|
+
// Exponential backoff, capped, with jitter so a fleet of instances doesn't
|
|
550
|
+
// retry in lockstep against a server that is coming back up.
|
|
551
|
+
const ceiling = Math.min(config.flushIntervalMs * 2 ** Math.min(this.failures, 12), config.maxRetryDelayMs);
|
|
552
|
+
const backoff = Math.round(ceiling * (0.5 + Math.random() * 0.5));
|
|
553
|
+
const wait = retryAfterMs !== undefined ? Math.min(Math.max(retryAfterMs, backoff), config.maxRetryDelayMs) : backoff;
|
|
554
|
+
this.retryAfter = Date.now() + wait;
|
|
555
|
+
}
|
|
556
|
+
/** Unrecoverable configuration problem: go quiet instead of looping forever. */
|
|
557
|
+
disable() {
|
|
558
|
+
this.close();
|
|
559
|
+
}
|
|
560
|
+
log(level, message) {
|
|
561
|
+
if (this.onErrorHook) {
|
|
562
|
+
try {
|
|
563
|
+
this.onErrorHook(message);
|
|
564
|
+
}
|
|
565
|
+
catch {
|
|
566
|
+
// A broken logging hook must not become the host application's problem.
|
|
567
|
+
}
|
|
568
|
+
if (!this.debugLogs)
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
if (level === "error")
|
|
572
|
+
console.error(message);
|
|
573
|
+
else if (level === "warn")
|
|
574
|
+
console.warn(message);
|
|
575
|
+
else
|
|
576
|
+
console.info(message);
|
|
120
577
|
}
|
|
121
578
|
}
|
|
122
579
|
exports.MidlineAgent = MidlineAgent;
|
|
123
|
-
MidlineAgent.
|
|
580
|
+
MidlineAgent.defaultAgent = null;
|
|
581
|
+
function classify(event, statusCode) {
|
|
582
|
+
if (event.severity && event.category) {
|
|
583
|
+
return { severity: event.severity, category: event.category };
|
|
584
|
+
}
|
|
585
|
+
let severity = "low";
|
|
586
|
+
let category = "performance";
|
|
587
|
+
if (event.type === "error") {
|
|
588
|
+
severity = "critical";
|
|
589
|
+
category = "application";
|
|
590
|
+
}
|
|
591
|
+
else if (event.type === "security") {
|
|
592
|
+
severity = "high";
|
|
593
|
+
category = "security";
|
|
594
|
+
}
|
|
595
|
+
else if (event.type === "custom") {
|
|
596
|
+
category = "business";
|
|
597
|
+
}
|
|
598
|
+
else if (statusCode && statusCode >= 500) {
|
|
599
|
+
severity = "high";
|
|
600
|
+
category = "application";
|
|
601
|
+
}
|
|
602
|
+
else if (statusCode && statusCode >= 400) {
|
|
603
|
+
severity = "medium";
|
|
604
|
+
category = "application";
|
|
605
|
+
}
|
|
606
|
+
return { severity: event.severity ?? severity, category: event.category ?? category };
|
|
607
|
+
}
|
|
608
|
+
/** Drops the heaviest optional parts until the event fits. Returns the new size. */
|
|
609
|
+
function shrink(wire, maxBytes) {
|
|
610
|
+
const payload = wire.payload;
|
|
611
|
+
const size = () => Buffer.byteLength(JSON.stringify(wire));
|
|
612
|
+
if (!payload)
|
|
613
|
+
return size();
|
|
614
|
+
const steps = [
|
|
615
|
+
() => markOmitted(payload.response, "body"),
|
|
616
|
+
() => markOmitted(payload.request, "body"),
|
|
617
|
+
() => {
|
|
618
|
+
if (typeof payload.stack === "string")
|
|
619
|
+
payload.stack = payload.stack.slice(0, 2048);
|
|
620
|
+
},
|
|
621
|
+
() => delete payload.breadcrumbs,
|
|
622
|
+
() => markOmitted(payload.response, "headers"),
|
|
623
|
+
() => markOmitted(payload.request, "headers"),
|
|
624
|
+
() => markOmitted(payload.request, "query"),
|
|
625
|
+
];
|
|
626
|
+
let bytes = size();
|
|
627
|
+
for (const step of steps) {
|
|
628
|
+
if (bytes <= maxBytes)
|
|
629
|
+
break;
|
|
630
|
+
step();
|
|
631
|
+
bytes = size();
|
|
632
|
+
}
|
|
633
|
+
return bytes;
|
|
634
|
+
}
|
|
635
|
+
function markOmitted(message, field) {
|
|
636
|
+
if (message && message[field] !== undefined) {
|
|
637
|
+
delete message[field];
|
|
638
|
+
message.omitted = "event size limit";
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
function clamp(value, max) {
|
|
642
|
+
if (value === undefined || value === null || value === "")
|
|
643
|
+
return undefined;
|
|
644
|
+
return String(value).slice(0, max);
|
|
645
|
+
}
|
|
646
|
+
function stripQuery(path) {
|
|
647
|
+
const index = path.search(/[?#]/);
|
|
648
|
+
return index === -1 ? path : path.slice(0, index);
|
|
649
|
+
}
|
|
650
|
+
function validTimestamp(value) {
|
|
651
|
+
if (value) {
|
|
652
|
+
const parsed = Date.parse(value);
|
|
653
|
+
if (Number.isFinite(parsed))
|
|
654
|
+
return new Date(parsed).toISOString();
|
|
655
|
+
}
|
|
656
|
+
return new Date().toISOString();
|
|
657
|
+
}
|
|
658
|
+
function errorCode(err) {
|
|
659
|
+
const e = err;
|
|
660
|
+
return String(e?.code ?? e?.cause?.code ?? e?.errno ?? e?.name ?? "");
|
|
661
|
+
}
|
|
662
|
+
function parseJson(text) {
|
|
663
|
+
try {
|
|
664
|
+
return JSON.parse(text);
|
|
665
|
+
}
|
|
666
|
+
catch {
|
|
667
|
+
return undefined;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
function serverMessage(body) {
|
|
671
|
+
const message = parseJson(body)?.message;
|
|
672
|
+
const text = Array.isArray(message) ? message.join("; ") : typeof message === "string" ? message : "";
|
|
673
|
+
return text ? `: ${text.slice(0, 300)}` : "";
|
|
674
|
+
}
|
|
675
|
+
function parseRetryAfter(header) {
|
|
676
|
+
const value = Array.isArray(header) ? header[0] : header;
|
|
677
|
+
if (!value)
|
|
678
|
+
return undefined;
|
|
679
|
+
const seconds = Number(value);
|
|
680
|
+
if (Number.isFinite(seconds))
|
|
681
|
+
return Math.max(0, seconds * 1000);
|
|
682
|
+
const date = Date.parse(value);
|
|
683
|
+
return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
|
|
684
|
+
}
|