midline-agent 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +115 -2
- package/browser/package.json +8 -0
- package/dist/agent.d.ts +14 -1
- package/dist/agent.js +106 -20
- package/dist/browser/client.d.ts +59 -0
- package/dist/browser/client.js +608 -0
- package/dist/browser/index.d.ts +34 -0
- package/dist/browser/index.js +65 -0
- package/dist/browser/instrument.d.ts +39 -0
- package/dist/browser/instrument.js +217 -0
- package/dist/browser/transport.d.ts +43 -0
- package/dist/browser/transport.js +168 -0
- package/dist/browser/types.d.ts +94 -0
- package/dist/browser/types.js +2 -0
- package/dist/browser/version.d.ts +2 -0
- package/dist/browser/version.js +5 -0
- package/dist/browser/vitals.d.ts +16 -0
- package/dist/browser/vitals.js +135 -0
- package/dist/cli.js +0 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.js +1 -0
- package/dist/console.d.ts +46 -0
- package/dist/console.js +167 -0
- package/dist/esm/browser/client.js +601 -0
- package/dist/esm/browser/index.js +52 -0
- package/dist/esm/browser/instrument.js +210 -0
- package/dist/esm/browser/transport.js +164 -0
- package/dist/esm/browser/types.js +1 -0
- package/dist/esm/browser/version.js +2 -0
- package/dist/esm/browser/vitals.js +132 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/redact.js +224 -0
- package/dist/esm/types.js +1 -0
- package/dist/redact.d.ts +3 -0
- package/dist/redact.js +12 -6
- package/dist/types.d.ts +9 -2
- package/package.json +27 -4
- package/scripts/mark-esm.js +6 -0
- package/src/agent.ts +111 -15
- package/src/browser/client.ts +686 -0
- package/src/browser/index.ts +74 -0
- package/src/browser/instrument.ts +275 -0
- package/src/browser/transport.ts +184 -0
- package/src/browser/types.ts +105 -0
- package/src/browser/version.ts +2 -0
- package/src/browser/vitals.ts +149 -0
- package/src/config.ts +2 -0
- package/src/console.ts +182 -0
- package/src/redact.ts +12 -6
- package/src/types.ts +9 -2
- package/test/browser.test.js +328 -0
- package/test/console.test.js +182 -0
- package/tsconfig.esm.json +14 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared by the Node agent and the browser SDK, so nothing here may touch a
|
|
3
|
+
* Node-only global (Buffer, process) — TextEncoder and URLSearchParams exist in both.
|
|
4
|
+
*
|
|
5
|
+
* Redaction happens in the host process, before an event is queued. Whatever is
|
|
6
|
+
* removed here never reaches a socket, a log line or the Midline server.
|
|
7
|
+
*
|
|
8
|
+
* Matching is on a normalised key — lower-cased with punctuation stripped — so
|
|
9
|
+
* `X-API-Key`, `api_key` and `apiKey` are all the same key.
|
|
10
|
+
*/
|
|
11
|
+
export const REDACTED = "[REDACTED]";
|
|
12
|
+
/** Substrings: any key containing one of these is sensitive. */
|
|
13
|
+
const SENSITIVE_KEY_PARTS = [
|
|
14
|
+
"password",
|
|
15
|
+
"passwd",
|
|
16
|
+
"passphrase",
|
|
17
|
+
"secret",
|
|
18
|
+
"token",
|
|
19
|
+
"apikey",
|
|
20
|
+
"accesskey",
|
|
21
|
+
"privatekey",
|
|
22
|
+
"authorization",
|
|
23
|
+
"cookie",
|
|
24
|
+
"session",
|
|
25
|
+
"credential",
|
|
26
|
+
"csrf",
|
|
27
|
+
"xsrf",
|
|
28
|
+
"signature",
|
|
29
|
+
"creditcard",
|
|
30
|
+
"cardnumber",
|
|
31
|
+
"cvv",
|
|
32
|
+
"cvc",
|
|
33
|
+
"ssn",
|
|
34
|
+
"socialsecurity",
|
|
35
|
+
];
|
|
36
|
+
/** Whole keys only — as substrings these would hit words like "author" or "spinner". */
|
|
37
|
+
const SENSITIVE_KEYS_EXACT = new Set(["auth", "pwd", "pin", "otp", "sid", "jwt", "bearer"]);
|
|
38
|
+
/** Always redacted by name, even if a user-supplied list somehow unmatched them. */
|
|
39
|
+
export const DEFAULT_SENSITIVE_HEADERS = [
|
|
40
|
+
"authorization",
|
|
41
|
+
"proxy-authorization",
|
|
42
|
+
"cookie",
|
|
43
|
+
"set-cookie",
|
|
44
|
+
"x-api-key",
|
|
45
|
+
"api-key",
|
|
46
|
+
"x-auth-token",
|
|
47
|
+
"x-access-token",
|
|
48
|
+
"x-refresh-token",
|
|
49
|
+
"x-csrf-token",
|
|
50
|
+
"x-xsrf-token",
|
|
51
|
+
"x-amz-security-token",
|
|
52
|
+
];
|
|
53
|
+
const VALUE_PATTERNS = [
|
|
54
|
+
[/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, REDACTED],
|
|
55
|
+
[/\b(Bearer|Basic|Digest|Token)\s+[A-Za-z0-9._~+\/=-]{8,}/gi, `$1 ${REDACTED}`],
|
|
56
|
+
[/\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g, REDACTED],
|
|
57
|
+
[/\bak_[A-Fa-f0-9]{16,}\b/g, REDACTED],
|
|
58
|
+
[/\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}\b/g, REDACTED],
|
|
59
|
+
[/\bAKIA[0-9A-Z]{16}\b/g, REDACTED],
|
|
60
|
+
[/(\b[a-z][a-z0-9+.-]*:\/\/)[^\s\/@:]+:[^\s\/@]+@/gi, `$1${REDACTED}@`],
|
|
61
|
+
];
|
|
62
|
+
/** `key=value` / `key: value` in query strings, log lines and error messages. */
|
|
63
|
+
const KEY_VALUE_PAIR = /(^|[?&;,\s(\[{])([A-Za-z0-9_.%-]{1,64})(\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s&#,;)\]}]+)/g;
|
|
64
|
+
/** `"key": value` in JSON text, including JSON that was cut off mid-value. */
|
|
65
|
+
const JSON_PAIR = /"([^"\\]{1,100})"\s*:\s*("(?:[^"\\]|\\.)*"?|-?\d+(?:\.\d+)?|true|false|null)/g;
|
|
66
|
+
export function normalizeKey(key) {
|
|
67
|
+
return key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
68
|
+
}
|
|
69
|
+
export class Redactor {
|
|
70
|
+
constructor(extraFields = [], extraHeaders = []) {
|
|
71
|
+
this.extraKeys = extraFields.map(normalizeKey).filter(Boolean);
|
|
72
|
+
this.headerNames = new Set([...DEFAULT_SENSITIVE_HEADERS, ...extraHeaders].map((name) => name.toLowerCase()));
|
|
73
|
+
}
|
|
74
|
+
isSensitiveKey(key) {
|
|
75
|
+
const normalized = normalizeKey(key);
|
|
76
|
+
if (!normalized)
|
|
77
|
+
return false;
|
|
78
|
+
if (SENSITIVE_KEYS_EXACT.has(normalized))
|
|
79
|
+
return true;
|
|
80
|
+
if (SENSITIVE_KEY_PARTS.some((part) => normalized.includes(part)))
|
|
81
|
+
return true;
|
|
82
|
+
return this.extraKeys.some((part) => normalized.includes(part));
|
|
83
|
+
}
|
|
84
|
+
/** Masks credentials embedded in free text: bearer tokens, JWTs, key formats, URL userinfo and query params. */
|
|
85
|
+
string(value, maxLength = 2048) {
|
|
86
|
+
let out = value.length > maxLength * 4 ? value.slice(0, maxLength * 4) : value;
|
|
87
|
+
for (const [pattern, replacement] of VALUE_PATTERNS) {
|
|
88
|
+
out = out.replace(pattern, replacement);
|
|
89
|
+
}
|
|
90
|
+
if (out.includes("=") || out.includes(":")) {
|
|
91
|
+
out = out.replace(KEY_VALUE_PAIR, (match, sep, key, delimiter) => {
|
|
92
|
+
let decoded = key;
|
|
93
|
+
try {
|
|
94
|
+
decoded = decodeURIComponent(key);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// keep the raw key
|
|
98
|
+
}
|
|
99
|
+
return this.isSensitiveKey(decoded) ? `${sep}${key}${delimiter}${REDACTED}` : match;
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
if (out.includes('"')) {
|
|
103
|
+
out = out.replace(JSON_PAIR, (match, key) => this.isSensitiveKey(key) ? `"${key}":"${REDACTED}"` : match);
|
|
104
|
+
}
|
|
105
|
+
// The ellipsis counts toward the limit: servers validate these lengths exactly.
|
|
106
|
+
return out.length > maxLength ? `${out.slice(0, Math.max(0, maxLength - 1))}…` : out;
|
|
107
|
+
}
|
|
108
|
+
/** Deep copy with sensitive keys and values masked. Bounded in depth, breadth and string length. */
|
|
109
|
+
value(input, depth = 0, seen = new WeakSet()) {
|
|
110
|
+
if (input === null || input === undefined)
|
|
111
|
+
return input;
|
|
112
|
+
if (typeof input === "string")
|
|
113
|
+
return this.string(input);
|
|
114
|
+
if (typeof input === "number" || typeof input === "boolean")
|
|
115
|
+
return input;
|
|
116
|
+
if (typeof input === "bigint")
|
|
117
|
+
return input.toString();
|
|
118
|
+
if (typeof input === "function" || typeof input === "symbol")
|
|
119
|
+
return undefined;
|
|
120
|
+
if (input instanceof Date)
|
|
121
|
+
return Number.isNaN(input.getTime()) ? null : input.toISOString();
|
|
122
|
+
// Buffer is a Uint8Array, so this also covers it without naming a Node-only global.
|
|
123
|
+
if (ArrayBuffer.isView(input)) {
|
|
124
|
+
return `[Binary ${input.byteLength} bytes]`;
|
|
125
|
+
}
|
|
126
|
+
if (typeof input !== "object")
|
|
127
|
+
return undefined;
|
|
128
|
+
if (seen.has(input))
|
|
129
|
+
return "[Circular]";
|
|
130
|
+
if (depth >= 8)
|
|
131
|
+
return "[Truncated]";
|
|
132
|
+
seen.add(input);
|
|
133
|
+
if (Array.isArray(input)) {
|
|
134
|
+
const items = input.slice(0, 100).map((item) => this.value(item, depth + 1, seen));
|
|
135
|
+
if (input.length > 100)
|
|
136
|
+
items.push(`[${input.length - 100} more]`);
|
|
137
|
+
return items;
|
|
138
|
+
}
|
|
139
|
+
const out = {};
|
|
140
|
+
let count = 0;
|
|
141
|
+
for (const [key, nested] of Object.entries(input)) {
|
|
142
|
+
if (count++ >= 200) {
|
|
143
|
+
out["[truncated]"] = "too many keys";
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
out[key] = this.isSensitiveKey(key) ? REDACTED : this.value(nested, depth + 1, seen);
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
}
|
|
150
|
+
headers(headers) {
|
|
151
|
+
if (!headers)
|
|
152
|
+
return undefined;
|
|
153
|
+
const out = {};
|
|
154
|
+
for (const [rawName, rawValue] of Object.entries(headers)) {
|
|
155
|
+
if (rawValue === undefined)
|
|
156
|
+
continue;
|
|
157
|
+
const name = rawName.toLowerCase();
|
|
158
|
+
if (this.headerNames.has(name) || this.isSensitiveKey(name)) {
|
|
159
|
+
out[name] = REDACTED;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const joined = Array.isArray(rawValue) ? rawValue.join(", ") : String(rawValue);
|
|
163
|
+
out[name] = this.string(joined, 1024);
|
|
164
|
+
}
|
|
165
|
+
return out;
|
|
166
|
+
}
|
|
167
|
+
query(search) {
|
|
168
|
+
if (!search)
|
|
169
|
+
return undefined;
|
|
170
|
+
const out = {};
|
|
171
|
+
const params = new URLSearchParams(search.startsWith("?") ? search.slice(1) : search);
|
|
172
|
+
for (const key of new Set(params.keys())) {
|
|
173
|
+
const values = params.getAll(key).map((value) => this.isSensitiveKey(key) ? REDACTED : this.string(value, 512));
|
|
174
|
+
out[key] = values.length === 1 ? values[0] : values;
|
|
175
|
+
}
|
|
176
|
+
return out;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Redacts a captured body. Structured content is parsed and redacted by key;
|
|
180
|
+
* text that cannot be parsed (usually because it was truncated) gets key/value
|
|
181
|
+
* pattern masking instead, so a cut-off JSON body still loses its passwords.
|
|
182
|
+
*/
|
|
183
|
+
body(raw, contentType, maxBytes) {
|
|
184
|
+
if (raw === undefined || raw === null || maxBytes <= 0)
|
|
185
|
+
return {};
|
|
186
|
+
const type = (contentType || "").toLowerCase();
|
|
187
|
+
if (typeof raw === "object" && !ArrayBuffer.isView(raw)) {
|
|
188
|
+
return this.fit(this.value(raw), maxBytes);
|
|
189
|
+
}
|
|
190
|
+
const text = ArrayBuffer.isView(raw)
|
|
191
|
+
? new TextDecoder().decode(raw)
|
|
192
|
+
: String(raw);
|
|
193
|
+
if (type.includes("json")) {
|
|
194
|
+
try {
|
|
195
|
+
return this.fit(this.value(JSON.parse(text)), maxBytes);
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
return this.cut(this.string(text, text.length), maxBytes);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
if (type.includes("application/x-www-form-urlencoded")) {
|
|
202
|
+
return this.fit(this.query(text) ?? {}, maxBytes);
|
|
203
|
+
}
|
|
204
|
+
if (!type || type.startsWith("text/") || type.includes("xml") || type.includes("graphql")) {
|
|
205
|
+
return this.cut(this.string(text, maxBytes * 2), maxBytes);
|
|
206
|
+
}
|
|
207
|
+
return { omitted: `content-type ${type.split(";")[0]}` };
|
|
208
|
+
}
|
|
209
|
+
fit(value, maxBytes) {
|
|
210
|
+
const serialized = JSON.stringify(value) ?? "";
|
|
211
|
+
if (new TextEncoder().encode(serialized).length <= maxBytes) {
|
|
212
|
+
return { body: value };
|
|
213
|
+
}
|
|
214
|
+
return this.cut(serialized, maxBytes);
|
|
215
|
+
}
|
|
216
|
+
cut(text, maxBytes) {
|
|
217
|
+
const bytes = new TextEncoder().encode(text);
|
|
218
|
+
if (bytes.length <= maxBytes) {
|
|
219
|
+
return { body: text };
|
|
220
|
+
}
|
|
221
|
+
// Slicing bytes can split a multi-byte character; the replacement char is harmless here.
|
|
222
|
+
return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)), truncated: true };
|
|
223
|
+
}
|
|
224
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/redact.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
+
* Shared by the Node agent and the browser SDK, so nothing here may touch a
|
|
3
|
+
* Node-only global (Buffer, process) — TextEncoder and URLSearchParams exist in both.
|
|
4
|
+
*
|
|
2
5
|
* Redaction happens in the host process, before an event is queued. Whatever is
|
|
3
6
|
* removed here never reaches a socket, a log line or the Midline server.
|
|
4
7
|
*
|
package/dist/redact.js
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
+
* Shared by the Node agent and the browser SDK, so nothing here may touch a
|
|
4
|
+
* Node-only global (Buffer, process) — TextEncoder and URLSearchParams exist in both.
|
|
5
|
+
*
|
|
3
6
|
* Redaction happens in the host process, before an event is queued. Whatever is
|
|
4
7
|
* removed here never reaches a socket, a log line or the Midline server.
|
|
5
8
|
*
|
|
@@ -120,7 +123,8 @@ class Redactor {
|
|
|
120
123
|
return undefined;
|
|
121
124
|
if (input instanceof Date)
|
|
122
125
|
return Number.isNaN(input.getTime()) ? null : input.toISOString();
|
|
123
|
-
|
|
126
|
+
// Buffer is a Uint8Array, so this also covers it without naming a Node-only global.
|
|
127
|
+
if (ArrayBuffer.isView(input)) {
|
|
124
128
|
return `[Binary ${input.byteLength} bytes]`;
|
|
125
129
|
}
|
|
126
130
|
if (typeof input !== "object")
|
|
@@ -184,10 +188,12 @@ class Redactor {
|
|
|
184
188
|
if (raw === undefined || raw === null || maxBytes <= 0)
|
|
185
189
|
return {};
|
|
186
190
|
const type = (contentType || "").toLowerCase();
|
|
187
|
-
if (typeof raw === "object" && !
|
|
191
|
+
if (typeof raw === "object" && !ArrayBuffer.isView(raw)) {
|
|
188
192
|
return this.fit(this.value(raw), maxBytes);
|
|
189
193
|
}
|
|
190
|
-
const text =
|
|
194
|
+
const text = ArrayBuffer.isView(raw)
|
|
195
|
+
? new TextDecoder().decode(raw)
|
|
196
|
+
: String(raw);
|
|
191
197
|
if (type.includes("json")) {
|
|
192
198
|
try {
|
|
193
199
|
return this.fit(this.value(JSON.parse(text)), maxBytes);
|
|
@@ -206,18 +212,18 @@ class Redactor {
|
|
|
206
212
|
}
|
|
207
213
|
fit(value, maxBytes) {
|
|
208
214
|
const serialized = JSON.stringify(value) ?? "";
|
|
209
|
-
if (
|
|
215
|
+
if (new TextEncoder().encode(serialized).length <= maxBytes) {
|
|
210
216
|
return { body: value };
|
|
211
217
|
}
|
|
212
218
|
return this.cut(serialized, maxBytes);
|
|
213
219
|
}
|
|
214
220
|
cut(text, maxBytes) {
|
|
215
|
-
const bytes =
|
|
221
|
+
const bytes = new TextEncoder().encode(text);
|
|
216
222
|
if (bytes.length <= maxBytes) {
|
|
217
223
|
return { body: text };
|
|
218
224
|
}
|
|
219
225
|
// Slicing bytes can split a multi-byte character; the replacement char is harmless here.
|
|
220
|
-
return { body: bytes.subarray(0, maxBytes)
|
|
226
|
+
return { body: new TextDecoder().decode(bytes.subarray(0, maxBytes)), truncated: true };
|
|
221
227
|
}
|
|
222
228
|
}
|
|
223
229
|
exports.Redactor = Redactor;
|
package/dist/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** A PEM string, PEM bytes, a path to a PEM file, or several of those. */
|
|
2
2
|
export type CaInput = string | Buffer | Array<string | Buffer>;
|
|
3
|
-
export type EventType = "request" | "error" | "security" | "performance" | "custom";
|
|
3
|
+
export type EventType = "request" | "error" | "security" | "performance" | "custom" | "console";
|
|
4
4
|
export type EventSeverity = "low" | "medium" | "high" | "critical";
|
|
5
5
|
export type EventCategory = "application" | "infrastructure" | "security" | "performance" | "business";
|
|
6
6
|
/**
|
|
@@ -55,6 +55,13 @@ export interface MidlineConfig {
|
|
|
55
55
|
redactHeaders?: string[];
|
|
56
56
|
/** What to capture beyond method/path/status/timing. Defaults to nothing. */
|
|
57
57
|
capture?: CaptureOptions;
|
|
58
|
+
/**
|
|
59
|
+
* Also send what the process prints — `console.log`, Nest's logger, anything
|
|
60
|
+
* written to stdout or stderr — as `console` events, one per line, redacted like
|
|
61
|
+
* any other text. Initialise the agent before creating the app to include its
|
|
62
|
+
* startup lines. Off by default. Env fallback: `MIDLINE_CAPTURE_CONSOLE`.
|
|
63
|
+
*/
|
|
64
|
+
captureConsole?: boolean;
|
|
58
65
|
/** Set to false to keep the agent inert. Env fallback: `MIDLINE_ENABLED`. */
|
|
59
66
|
enabled?: boolean;
|
|
60
67
|
/**
|
|
@@ -128,5 +135,5 @@ export interface MidlineEvent {
|
|
|
128
135
|
errorCode?: string;
|
|
129
136
|
/** Set when the client disconnected before the response finished. */
|
|
130
137
|
aborted?: boolean;
|
|
131
|
-
integration?: "express" | "node-http" | "proxy" | "manual";
|
|
138
|
+
integration?: "express" | "node-http" | "proxy" | "console" | "manual";
|
|
132
139
|
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,29 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "midline-agent",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Midline — request
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Midline — request, error and security monitoring for Node, and error, network and Web Vitals monitoring for browsers",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/index.d.ts",
|
|
10
|
+
"default": "./dist/index.js"
|
|
11
|
+
},
|
|
12
|
+
"./browser": {
|
|
13
|
+
"types": "./dist/browser/index.d.ts",
|
|
14
|
+
"import": "./dist/esm/browser/index.js",
|
|
15
|
+
"default": "./dist/browser/index.js"
|
|
16
|
+
},
|
|
17
|
+
"./package.json": "./package.json",
|
|
18
|
+
"./dist/*": "./dist/*"
|
|
19
|
+
},
|
|
20
|
+
"typesVersions": {
|
|
21
|
+
"*": {
|
|
22
|
+
"browser": [
|
|
23
|
+
"dist/browser/index.d.ts"
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
},
|
|
7
27
|
"bin": {
|
|
8
28
|
"midline-agent": "dist/cli.js"
|
|
9
29
|
},
|
|
@@ -11,7 +31,7 @@
|
|
|
11
31
|
"node": ">=18"
|
|
12
32
|
},
|
|
13
33
|
"scripts": {
|
|
14
|
-
"build": "tsc",
|
|
34
|
+
"build": "tsc && tsc -p tsconfig.esm.json && node scripts/mark-esm.js",
|
|
15
35
|
"prepare": "npm run build",
|
|
16
36
|
"clean": "rm -rf dist",
|
|
17
37
|
"test": "npm run build && node --test test/*.test.js"
|
|
@@ -24,7 +44,10 @@
|
|
|
24
44
|
"security",
|
|
25
45
|
"request",
|
|
26
46
|
"errors",
|
|
27
|
-
"proxy"
|
|
47
|
+
"proxy",
|
|
48
|
+
"browser",
|
|
49
|
+
"web-vitals",
|
|
50
|
+
"frontend"
|
|
28
51
|
],
|
|
29
52
|
"author": "Your Name",
|
|
30
53
|
"license": "MIT",
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// dist/esm holds the browser build as ES modules. Marking the folder as a module
|
|
2
|
+
// scope lets bundlers and Node treat those .js files as ESM without renaming them.
|
|
3
|
+
const fs = require("fs");
|
|
4
|
+
const path = require("path");
|
|
5
|
+
|
|
6
|
+
fs.writeFileSync(path.join(__dirname, "..", "dist", "esm", "package.json"), '{ "type": "module" }\n');
|
package/src/agent.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ConfigError, ResolvedCapture, ResolvedConfig, envFlag, resolveConfig } from "./config";
|
|
2
|
+
import { ConsoleCapture, ConsoleLevel, ConsoleLine, MAX_LINE_CHARS, withoutConsoleCapture } from "./console";
|
|
2
3
|
import type { RequestContext } from "./context";
|
|
3
4
|
import { Redactor } from "./redact";
|
|
4
5
|
import { Transport } from "./transport";
|
|
@@ -16,6 +17,11 @@ const MAX_REQUEST_BYTES = 512 * 1024;
|
|
|
16
17
|
const MIN_REQUEST_BYTES = 16 * 1024;
|
|
17
18
|
const MAX_QUEUE_BYTES = 16 * 1024 * 1024;
|
|
18
19
|
|
|
20
|
+
/** Console lines: a startup burst (Nest mapping every route) fits, a log storm can't crowd out requests. */
|
|
21
|
+
const CONSOLE_BURST_LINES = 1000;
|
|
22
|
+
const CONSOLE_LINES_PER_SECOND = 100;
|
|
23
|
+
const CONSOLE_SEVERITY: Record<ConsoleLevel, EventSeverity> = { info: "low", warn: "medium", error: "high" };
|
|
24
|
+
|
|
19
25
|
/** Verification failures. Retrying is still right: they clear once the server is fixed. */
|
|
20
26
|
const TLS_ERROR_REASONS: Record<string, string> = {
|
|
21
27
|
DEPTH_ZERO_SELF_SIGNED_CERT: "presented a self-signed certificate",
|
|
@@ -76,7 +82,7 @@ export interface ExchangeMessage {
|
|
|
76
82
|
}
|
|
77
83
|
|
|
78
84
|
/**
|
|
79
|
-
* Ships request
|
|
85
|
+
* Ships request, error and console events to the Midline server.
|
|
80
86
|
*
|
|
81
87
|
* The contract with the host application is that the agent is never allowed to
|
|
82
88
|
* affect it: an unreachable, untrusted, slow or misconfigured Midline server costs a
|
|
@@ -126,6 +132,11 @@ export class MidlineAgent {
|
|
|
126
132
|
private dropped = 0;
|
|
127
133
|
private maxRequestBytes = MAX_REQUEST_BYTES;
|
|
128
134
|
private batchLimit = Number.MAX_SAFE_INTEGER;
|
|
135
|
+
private consoleCapture: ConsoleCapture | null = null;
|
|
136
|
+
private consoleTokens = CONSOLE_BURST_LINES;
|
|
137
|
+
private consoleRefilledAt = Date.now();
|
|
138
|
+
/** The server refused a console event: it predates them, so no more are sent. */
|
|
139
|
+
private consoleUnsupported = false;
|
|
129
140
|
|
|
130
141
|
constructor(config: MidlineConfig = {}) {
|
|
131
142
|
this.onErrorHook = config.onError;
|
|
@@ -162,10 +173,16 @@ export class MidlineAgent {
|
|
|
162
173
|
});
|
|
163
174
|
|
|
164
175
|
this.timer = setInterval(() => {
|
|
176
|
+
this.consoleCapture?.flushPending();
|
|
165
177
|
void this.drain(false);
|
|
166
178
|
}, resolved.flushIntervalMs);
|
|
167
179
|
// Telemetry must never be the reason a process refuses to exit.
|
|
168
180
|
this.timer.unref?.();
|
|
181
|
+
|
|
182
|
+
if (resolved.captureConsole) {
|
|
183
|
+
this.consoleCapture = new ConsoleCapture((line) => this.recordConsole(line));
|
|
184
|
+
this.consoleCapture.install();
|
|
185
|
+
}
|
|
169
186
|
}
|
|
170
187
|
|
|
171
188
|
/** False when the agent is off: no key, bad config, disabled, rejected key, or closed. */
|
|
@@ -229,9 +246,52 @@ export class MidlineAgent {
|
|
|
229
246
|
}
|
|
230
247
|
}
|
|
231
248
|
|
|
249
|
+
private recordConsole(line: ConsoleLine): void {
|
|
250
|
+
if (!this.active || this.consoleUnsupported) return;
|
|
251
|
+
if (!this.takeConsoleToken()) {
|
|
252
|
+
this.dropped += 1;
|
|
253
|
+
this.report(
|
|
254
|
+
"console-rate",
|
|
255
|
+
`midline: console output is arriving faster than ${CONSOLE_LINES_PER_SECOND} lines/s; the extra lines still print but are not sent.`,
|
|
256
|
+
);
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
try {
|
|
260
|
+
this.enqueue(
|
|
261
|
+
this.toWire(
|
|
262
|
+
{
|
|
263
|
+
type: "console",
|
|
264
|
+
path: line.stream,
|
|
265
|
+
message: line.text,
|
|
266
|
+
timestamp: line.timestamp,
|
|
267
|
+
severity: CONSOLE_SEVERITY[line.level],
|
|
268
|
+
category: "application",
|
|
269
|
+
integration: "console",
|
|
270
|
+
},
|
|
271
|
+
false,
|
|
272
|
+
),
|
|
273
|
+
);
|
|
274
|
+
} catch (err) {
|
|
275
|
+
this.report("event-build", `midline: could not record a console line (${(err as Error)?.message}); skipped it.`);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
private takeConsoleToken(): boolean {
|
|
280
|
+
const now = Date.now();
|
|
281
|
+
this.consoleTokens = Math.min(
|
|
282
|
+
CONSOLE_BURST_LINES,
|
|
283
|
+
this.consoleTokens + ((now - this.consoleRefilledAt) / 1000) * CONSOLE_LINES_PER_SECOND,
|
|
284
|
+
);
|
|
285
|
+
this.consoleRefilledAt = now;
|
|
286
|
+
if (this.consoleTokens < 1) return false;
|
|
287
|
+
this.consoleTokens -= 1;
|
|
288
|
+
return true;
|
|
289
|
+
}
|
|
290
|
+
|
|
232
291
|
/** Sends whatever is buffered now, ignoring backoff, and keeps the process alive until done. */
|
|
233
292
|
async flush(): Promise<void> {
|
|
234
293
|
if (!this.active) return;
|
|
294
|
+
this.consoleCapture?.flushPending(true);
|
|
235
295
|
this.retryAfter = 0;
|
|
236
296
|
if (this.drainPromise) {
|
|
237
297
|
await this.drainPromise;
|
|
@@ -261,6 +321,8 @@ export class MidlineAgent {
|
|
|
261
321
|
clearInterval(this.timer);
|
|
262
322
|
this.timer = null;
|
|
263
323
|
}
|
|
324
|
+
this.consoleCapture?.uninstall();
|
|
325
|
+
this.consoleCapture = null;
|
|
264
326
|
this.queue = [];
|
|
265
327
|
this.queueBytes = 0;
|
|
266
328
|
this.transport?.destroy();
|
|
@@ -306,6 +368,7 @@ export class MidlineAgent {
|
|
|
306
368
|
private toWire(event: MidlineEvent, preRedacted: boolean): Record<string, unknown> {
|
|
307
369
|
const config = this.config!;
|
|
308
370
|
const type = event.type;
|
|
371
|
+
const isConsole = type === "console";
|
|
309
372
|
const statusCode = Number.isInteger(event.statusCode) && event.statusCode! >= 100 && event.statusCode! <= 599
|
|
310
373
|
? event.statusCode
|
|
311
374
|
: type === "error" ? 500 : undefined;
|
|
@@ -313,7 +376,9 @@ export class MidlineAgent {
|
|
|
313
376
|
const route = this.redactor.string(stripQuery(event.path) || "/", 2048);
|
|
314
377
|
|
|
315
378
|
const payload: Record<string, unknown> = {};
|
|
316
|
-
if (
|
|
379
|
+
if (isConsole) {
|
|
380
|
+
payload.message = this.redactor.string(event.message ?? "", MAX_LINE_CHARS);
|
|
381
|
+
} else if (type === "error" || event.message) {
|
|
317
382
|
payload.error = event.message ? this.redactor.string(event.message, 1024) : type === "error" ? "Unknown error" : undefined;
|
|
318
383
|
}
|
|
319
384
|
if (event.stack) payload.stack = this.redactor.string(event.stack, 16 * 1024);
|
|
@@ -349,9 +414,10 @@ export class MidlineAgent {
|
|
|
349
414
|
apiKey: config.apiKey,
|
|
350
415
|
eventType: type,
|
|
351
416
|
route,
|
|
352
|
-
|
|
417
|
+
// A printed line has no method or timing; defaults here would make it look like a request.
|
|
418
|
+
method: isConsole ? undefined : (event.method || "GET").toUpperCase().slice(0, 16),
|
|
353
419
|
statusCode,
|
|
354
|
-
responseTime: Math.min(Math.max(0, Math.round(Number(event.duration) || 0)), 86_400_000),
|
|
420
|
+
responseTime: isConsole ? undefined : Math.min(Math.max(0, Math.round(Number(event.duration) || 0)), 86_400_000),
|
|
355
421
|
timestamp: validTimestamp(event.timestamp),
|
|
356
422
|
// Clamped to the server's validation limits: one over-long field would
|
|
357
423
|
// otherwise get every event rejected.
|
|
@@ -470,6 +536,7 @@ export class MidlineAgent {
|
|
|
470
536
|
}
|
|
471
537
|
if (outcome.kind === "rejected") {
|
|
472
538
|
if (batch.length === 1) {
|
|
539
|
+
if (this.refusedConsole(batch[0], outcome.detail)) continue;
|
|
473
540
|
this.dropped += 1;
|
|
474
541
|
this.report(`rejected:${outcome.detail}`, `midline: the Midline server rejected an event (${outcome.detail}); dropped it.`);
|
|
475
542
|
continue;
|
|
@@ -490,12 +557,14 @@ export class MidlineAgent {
|
|
|
490
557
|
/** Returns false if delivery should stop for this drain. */
|
|
491
558
|
private async sendIndividually(batch: QueuedEvent[], keepProcessAlive: boolean): Promise<boolean> {
|
|
492
559
|
for (let index = 0; index < batch.length; index++) {
|
|
560
|
+
if (this.consoleUnsupported && batch[index].wire.eventType === "console") continue;
|
|
493
561
|
const outcome = await this.send([batch[index]], keepProcessAlive);
|
|
494
562
|
if (outcome.kind === "ok") {
|
|
495
563
|
this.onSuccess();
|
|
496
564
|
} else if (outcome.kind === "rejected" || outcome.kind === "tooLarge") {
|
|
497
|
-
this.dropped += 1;
|
|
498
565
|
const detail = outcome.kind === "rejected" ? outcome.detail : "HTTP 413";
|
|
566
|
+
if (outcome.kind === "rejected" && this.refusedConsole(batch[index], detail)) continue;
|
|
567
|
+
this.dropped += 1;
|
|
499
568
|
this.report(`rejected:${detail}`, `midline: the Midline server rejected an event (${detail}); dropped it.`);
|
|
500
569
|
} else if (outcome.kind === "stop") {
|
|
501
570
|
this.disable();
|
|
@@ -565,6 +634,27 @@ export class MidlineAgent {
|
|
|
565
634
|
return { kind: "rejected", detail: `HTTP ${status}${serverMessage(result.body)}` };
|
|
566
635
|
}
|
|
567
636
|
|
|
637
|
+
/**
|
|
638
|
+
* True when a rejected event is a console line the server has no event type for,
|
|
639
|
+
* i.e. the server predates console capture. Capture switches itself off rather
|
|
640
|
+
* than paying a refused request for every line printed from then on.
|
|
641
|
+
*/
|
|
642
|
+
private refusedConsole(item: QueuedEvent, detail: string): boolean {
|
|
643
|
+
if (item.wire.eventType !== "console" || !detail.includes("eventType")) return false;
|
|
644
|
+
if (!this.consoleUnsupported) {
|
|
645
|
+
this.consoleUnsupported = true;
|
|
646
|
+
this.consoleCapture?.uninstall();
|
|
647
|
+
this.consoleCapture = null;
|
|
648
|
+
this.queue = this.queue.filter((queued) => queued.wire.eventType !== "console");
|
|
649
|
+
this.queueBytes = this.queue.reduce((sum, queued) => sum + queued.bytes, 0);
|
|
650
|
+
this.log(
|
|
651
|
+
"warn",
|
|
652
|
+
"midline: this Midline server does not accept console events yet (it needs updating), so console capture is off. Request and error monitoring carry on.",
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
return true;
|
|
656
|
+
}
|
|
657
|
+
|
|
568
658
|
/** One line per distinct fault, not one per failed event. */
|
|
569
659
|
private report(signature: string, message: string): void {
|
|
570
660
|
if (this.lastNotice === signature && !this.debugLogs) {
|
|
@@ -638,17 +728,21 @@ export class MidlineAgent {
|
|
|
638
728
|
}
|
|
639
729
|
|
|
640
730
|
private log(level: "info" | "warn" | "error", message: string): void {
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
731
|
+
// The agent's own diagnostics are never the application's console output,
|
|
732
|
+
// including when a hook hands them to a logger that prints them.
|
|
733
|
+
withoutConsoleCapture(() => {
|
|
734
|
+
if (this.onErrorHook) {
|
|
735
|
+
try {
|
|
736
|
+
this.onErrorHook(message);
|
|
737
|
+
} catch {
|
|
738
|
+
// A broken logging hook must not become the host application's problem.
|
|
739
|
+
}
|
|
740
|
+
if (!this.debugLogs) return;
|
|
646
741
|
}
|
|
647
|
-
if (
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
else console.info(message);
|
|
742
|
+
if (level === "error") console.error(message);
|
|
743
|
+
else if (level === "warn") console.warn(message);
|
|
744
|
+
else console.info(message);
|
|
745
|
+
});
|
|
652
746
|
}
|
|
653
747
|
}
|
|
654
748
|
|
|
@@ -667,6 +761,8 @@ function classify(event: MidlineEvent, statusCode: number | undefined): { severi
|
|
|
667
761
|
category = "security";
|
|
668
762
|
} else if (event.type === "custom") {
|
|
669
763
|
category = "business";
|
|
764
|
+
} else if (event.type === "console") {
|
|
765
|
+
category = "application";
|
|
670
766
|
} else if (statusCode && statusCode >= 500) {
|
|
671
767
|
severity = "high";
|
|
672
768
|
category = "application";
|