midline-agent 0.1.9 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +288 -252
- package/dist/agent.d.ts +105 -6
- package/dist/agent.js +663 -102
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +63 -0
- package/dist/config.d.ts +63 -0
- package/dist/config.js +230 -0
- package/dist/context.d.ts +13 -0
- package/dist/context.js +38 -0
- package/dist/errorHandler.d.ts +11 -2
- package/dist/errorHandler.js +32 -12
- package/dist/index.d.ts +9 -2
- package/dist/index.js +9 -1
- package/dist/middleware.d.ts +19 -2
- package/dist/middleware.js +92 -41
- package/dist/proxy.d.ts +70 -0
- package/dist/proxy.js +383 -0
- package/dist/redact.d.ts +35 -0
- package/dist/redact.js +223 -0
- package/dist/tap.d.ts +18 -0
- package/dist/tap.js +62 -0
- package/dist/transport.d.ts +35 -0
- package/dist/transport.js +133 -0
- package/dist/types.d.ts +107 -5
- package/package.json +13 -8
- package/src/agent.ts +734 -102
- package/src/cli.ts +69 -0
- package/src/config.ts +232 -0
- package/src/context.ts +48 -0
- package/src/errorHandler.ts +36 -13
- package/src/index.ts +19 -2
- package/src/middleware.ts +118 -43
- package/src/proxy.ts +435 -0
- package/src/redact.ts +231 -0
- package/src/tap.ts +55 -0
- package/src/transport.ts +125 -0
- package/src/types.ts +144 -31
- package/test/agent.test.js +353 -0
- package/test/helpers.js +151 -0
- package/test/middleware.test.js +178 -0
- package/test/proxy.test.js +274 -0
- package/test/redact.test.js +105 -0
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# midline-agent
|
|
2
2
|
|
|
3
|
-
**The Node.js SDK for Midline** —
|
|
3
|
+
**The Node.js SDK for Midline** — request and error monitoring with security and threat detection.
|
|
4
4
|
|
|
5
5
|
Midline itself is not tied to Node. Every event lands on the same stream through a plain JSON
|
|
6
6
|
endpoint, so a Django, Rails, Laravel, Spring, Go or .NET service reports exactly what an Express
|
|
@@ -9,328 +9,381 @@ to [Any other backend](#any-other-backend).
|
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
12
|
-
##
|
|
12
|
+
## How it fits together
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
- Capture all errors and exceptions
|
|
16
|
-
- Queue and batch events to Midline Cloud
|
|
17
|
-
- Mask sensitive fields (passwords, tokens, etc.)
|
|
18
|
-
- Non-blocking, minimal boilerplate
|
|
19
|
-
- Works with Express, and with anything that accepts Express-style middleware (NestJS, Fastify via `middie`)
|
|
14
|
+
There are two separate things, and the agent never confuses them:
|
|
20
15
|
|
|
21
|
-
|
|
16
|
+
| | What it is | Configured with |
|
|
17
|
+
| --- | --- | --- |
|
|
18
|
+
| **Midline server** (control plane) | Where events, API keys, logs, analytics and security data live. `https://api.usemidline.com` | `endpoint` / `MIDLINE_ENDPOINT`, `apiKey` / `MIDLINE_API_KEY`, `ca` / `MIDLINE_CUSTOM_CA` |
|
|
19
|
+
| **Destination API** (data plane) | The API being monitored — local, staging or production | `target` / `TARGET_API_URL`, `targetCa` / `TARGET_API_CA` (proxy mode only) |
|
|
22
20
|
|
|
23
|
-
|
|
21
|
+
Your traffic never goes through the Midline server. The agent runs next to your API, observes each
|
|
22
|
+
request, and ships a redacted event to Midline in the background:
|
|
24
23
|
|
|
25
|
-
|
|
24
|
+
```
|
|
25
|
+
┌──────────────── your infrastructure ────────────────┐
|
|
26
|
+
client ────────► │ midline-agent ──────────────► destination API │
|
|
27
|
+
│ (middleware in your app, or proxy in front of it) │
|
|
28
|
+
└───────┬─────────────────────────────────────────────┘
|
|
29
|
+
│ events, batched, redacted, async (HTTPS, verified)
|
|
30
|
+
▼
|
|
31
|
+
https://api.usemidline.com
|
|
32
|
+
```
|
|
26
33
|
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
That is deliberate. If Midline is down, slow or misconfigured, your API keeps serving and only
|
|
35
|
+
telemetry is buffered. A hosted gateway that sits in the request path can't do that, and it can't
|
|
36
|
+
reach `http://localhost:4000` on your laptop at all.
|
|
37
|
+
|
|
38
|
+
Two ways to run the agent:
|
|
39
|
+
|
|
40
|
+
- **Middleware** — inside an Express/NestJS/Node app. No extra hop.
|
|
41
|
+
- **Proxy** — a small reverse proxy in front of any HTTP API, in any language. `TARGET_API_URL`
|
|
42
|
+
decides where requests go.
|
|
29
43
|
|
|
30
44
|
---
|
|
31
45
|
|
|
32
|
-
|
|
46
|
+
## Install
|
|
33
47
|
|
|
34
48
|
```bash
|
|
35
49
|
npm install midline-agent
|
|
36
50
|
```
|
|
37
51
|
|
|
38
|
-
|
|
52
|
+
Get a project API key from the Midline dashboard (**Project → API Keys**).
|
|
39
53
|
|
|
40
54
|
---
|
|
41
55
|
|
|
42
|
-
##
|
|
56
|
+
## Middleware mode
|
|
43
57
|
|
|
44
|
-
### Express
|
|
45
|
-
|
|
46
|
-
```bash
|
|
47
|
-
npm install midline-agent express
|
|
48
|
-
```
|
|
58
|
+
### Express
|
|
49
59
|
|
|
50
60
|
```ts
|
|
51
61
|
import express from "express";
|
|
52
62
|
import { MidlineAgent, midlineMiddleware, midlineErrorHandler } from "midline-agent";
|
|
53
63
|
|
|
54
|
-
const app = express();
|
|
55
|
-
|
|
56
|
-
// Initialize Midline Agent (do this once, early in your app startup)
|
|
57
64
|
MidlineAgent.init({
|
|
58
|
-
apiKey: process.env.
|
|
65
|
+
apiKey: process.env.MIDLINE_API_KEY,
|
|
59
66
|
serviceName: "checkout-api",
|
|
60
|
-
environment:
|
|
61
|
-
maskFields: ["password", "token", "card_number", "cvv"]
|
|
67
|
+
environment: process.env.NODE_ENV,
|
|
62
68
|
});
|
|
63
69
|
|
|
64
|
-
|
|
70
|
+
const app = express();
|
|
71
|
+
app.use(midlineMiddleware()); // before your routes
|
|
65
72
|
app.use(express.json());
|
|
66
73
|
|
|
67
|
-
|
|
68
|
-
app.use(midlineMiddleware());
|
|
74
|
+
app.post("/v1/charges", (req, res) => res.json({ ok: true }));
|
|
69
75
|
|
|
70
|
-
//
|
|
71
|
-
app.
|
|
72
|
-
app.post("/v1/charges", async (req, res) => {
|
|
73
|
-
// Handle payment processing
|
|
74
|
-
res.json({ success: true });
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
// Add error handler middleware AFTER your routes
|
|
78
|
-
// This must be the last middleware registered
|
|
79
|
-
app.use(midlineErrorHandler());
|
|
80
|
-
|
|
81
|
-
app.listen(3000, () => console.log("Server running on port 3000"));
|
|
76
|
+
app.use(midlineErrorHandler()); // after your routes; passes the error on untouched
|
|
77
|
+
app.listen(3000);
|
|
82
78
|
```
|
|
83
79
|
|
|
84
|
-
**Key points:**
|
|
85
|
-
- `MidlineAgent.init()` must be called once during app startup
|
|
86
|
-
- `midlineMiddleware()` captures all requests and responses
|
|
87
|
-
- `midlineErrorHandler()` must be registered **after** all other middleware and routes
|
|
88
|
-
- Express only routes errors to handlers declared after the code that threw, so order matters
|
|
89
|
-
|
|
90
|
-
---
|
|
91
|
-
|
|
92
80
|
### NestJS
|
|
93
81
|
|
|
94
|
-
```bash
|
|
95
|
-
npm install midline-agent
|
|
96
|
-
```
|
|
97
|
-
|
|
98
|
-
**Using with NestJS requires wrapping the Express middleware. Here's the setup:**
|
|
99
|
-
|
|
100
82
|
```ts
|
|
101
|
-
import { NestFactory } from
|
|
102
|
-
import {
|
|
103
|
-
import {
|
|
83
|
+
import { NestFactory } from "@nestjs/core";
|
|
84
|
+
import { MidlineAgent, midlineMiddleware } from "midline-agent";
|
|
85
|
+
import { AppModule } from "./app.module";
|
|
104
86
|
|
|
105
87
|
async function bootstrap() {
|
|
106
|
-
const app = await NestFactory.create(AppModule);
|
|
107
|
-
|
|
108
|
-
// Initialize Midline Agent early in bootstrap
|
|
109
88
|
MidlineAgent.init({
|
|
110
|
-
apiKey: process.env.
|
|
111
|
-
serviceName:
|
|
112
|
-
environment: process.env.NODE_ENV
|
|
113
|
-
maskFields: ['password', 'token', 'apiKey', 'secret']
|
|
89
|
+
apiKey: process.env.MIDLINE_API_KEY,
|
|
90
|
+
serviceName: "nest-api",
|
|
91
|
+
environment: process.env.NODE_ENV,
|
|
114
92
|
});
|
|
115
93
|
|
|
116
|
-
|
|
94
|
+
const app = await NestFactory.create(AppModule);
|
|
117
95
|
app.use(midlineMiddleware());
|
|
118
|
-
|
|
119
|
-
// Apply error handler after NestJS is configured
|
|
120
|
-
app.use(midlineErrorHandler());
|
|
121
|
-
|
|
96
|
+
app.enableShutdownHooks();
|
|
122
97
|
await app.listen(3000);
|
|
123
98
|
}
|
|
124
|
-
|
|
125
99
|
bootstrap();
|
|
100
|
+
|
|
101
|
+
// On shutdown (e.g. in onApplicationShutdown): await MidlineAgent.shutdown();
|
|
126
102
|
```
|
|
127
103
|
|
|
128
|
-
|
|
104
|
+
Nest handles exceptions in its own filters, so they rarely reach Express error middleware. The
|
|
105
|
+
request middleware still records every 4xx/5xx response.
|
|
106
|
+
|
|
107
|
+
### Plain Node `http`
|
|
129
108
|
|
|
130
109
|
```ts
|
|
131
|
-
|
|
132
|
-
import {
|
|
133
|
-
import { midlineMiddleware, midlineErrorHandler } from 'midline-agent';
|
|
134
|
-
import { Request, Response, NextFunction } from 'express';
|
|
135
|
-
|
|
136
|
-
@Injectable()
|
|
137
|
-
export class MidlineMiddleware implements NestMiddleware {
|
|
138
|
-
use(req: Request, res: Response, next: NextFunction) {
|
|
139
|
-
midlineMiddleware()(req, res, next);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
```
|
|
110
|
+
import http from "http";
|
|
111
|
+
import { MidlineAgent, midlineMiddleware } from "midline-agent";
|
|
143
112
|
|
|
144
|
-
|
|
113
|
+
MidlineAgent.init({ apiKey: process.env.MIDLINE_API_KEY, serviceName: "raw-node" });
|
|
114
|
+
const midline = midlineMiddleware();
|
|
145
115
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
controllers: [],
|
|
153
|
-
providers: [],
|
|
154
|
-
})
|
|
155
|
-
export class AppModule {
|
|
156
|
-
configure(consumer: MiddlewareConsumer) {
|
|
157
|
-
consumer
|
|
158
|
-
.apply(MidlineMiddleware)
|
|
159
|
-
.forRoutes('*');
|
|
160
|
-
}
|
|
161
|
-
}
|
|
116
|
+
http.createServer((req, res) => {
|
|
117
|
+
midline(req, res, () => {
|
|
118
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
119
|
+
res.end('{"ok":true}');
|
|
120
|
+
});
|
|
121
|
+
}).listen(3000);
|
|
162
122
|
```
|
|
163
123
|
|
|
164
124
|
---
|
|
165
125
|
|
|
166
|
-
|
|
126
|
+
## Proxy mode
|
|
167
127
|
|
|
168
|
-
|
|
128
|
+
Put the agent in front of any API — Node or not — without touching its code:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
MIDLINE_API_KEY=... TARGET_API_URL=http://localhost:4000 npx midline-agent proxy --port 8080
|
|
132
|
+
# clients now call http://localhost:8080 instead of :4000
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
The same binary, pointed elsewhere:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
TARGET_API_URL=https://staging.example.com npx midline-agent proxy
|
|
139
|
+
TARGET_API_URL=https://api.example.com npx midline-agent proxy --host 0.0.0.0
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Or from code:
|
|
169
143
|
|
|
170
144
|
```ts
|
|
171
|
-
import
|
|
172
|
-
import { MidlineAgent, midlineMiddleware, midlineErrorHandler } from 'midline-agent';
|
|
145
|
+
import { startMidlineProxy } from "midline-agent";
|
|
173
146
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
maskFields: ['password', 'token']
|
|
147
|
+
await startMidlineProxy({
|
|
148
|
+
target: process.env.TARGET_API_URL, // required; never defaulted
|
|
149
|
+
port: 8080,
|
|
150
|
+
midline: { apiKey: process.env.MIDLINE_API_KEY, serviceName: "orders-gateway" },
|
|
179
151
|
});
|
|
152
|
+
```
|
|
180
153
|
|
|
181
|
-
|
|
182
|
-
// Create Express-like req/res wrapper for middleware
|
|
183
|
-
const next = (err?: any) => {
|
|
184
|
-
if (err) throw err;
|
|
185
|
-
// Handle your route
|
|
186
|
-
if (req.url === '/health') {
|
|
187
|
-
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
188
|
-
res.end(JSON.stringify({ status: 'ok' }));
|
|
189
|
-
} else {
|
|
190
|
-
res.writeHead(404);
|
|
191
|
-
res.end('Not Found');
|
|
192
|
-
}
|
|
193
|
-
};
|
|
194
|
-
|
|
195
|
-
midlineMiddleware()(req, res, next);
|
|
196
|
-
});
|
|
154
|
+
`createMidlineProxy(options)` returns a plain `(req, res)` handler if you want your own server.
|
|
197
155
|
|
|
198
|
-
|
|
199
|
-
|
|
156
|
+
What the proxy does per request:
|
|
157
|
+
|
|
158
|
+
- Forwards method, path, query, headers and body to the target, streaming both ways
|
|
159
|
+
- Strips hop-by-hop headers; sets `Host` to the target (unless `preserveHost`), adds `X-Forwarded-*`,
|
|
160
|
+
`X-Request-Id` and `X-Correlation-Id`
|
|
161
|
+
- Only accepts origin-form request lines (`GET /path`). It can't be used as an open forward proxy.
|
|
162
|
+
- Answers `502` if the destination is unreachable, `504` if it's too slow (`timeoutMs`, default
|
|
163
|
+
30s), `413` if the body exceeds `maxRequestBodyBytes` (default 10 MiB). Error bodies carry a
|
|
164
|
+
`requestId` and no internal detail.
|
|
165
|
+
- Retries (`retries`, default 0) only `GET`/`HEAD`/`OPTIONS` without a body, and only when the
|
|
166
|
+
request never reached the destination. A write is never replayed.
|
|
167
|
+
- Records the exchange — including destination failures as `infrastructure` errors — to Midline
|
|
168
|
+
|
|
169
|
+
WebSocket upgrades are not proxied.
|
|
200
170
|
|
|
201
171
|
---
|
|
202
172
|
|
|
203
|
-
##
|
|
173
|
+
## TLS
|
|
174
|
+
|
|
175
|
+
**Certificate verification is always on**, for both the Midline server and the destination.
|
|
176
|
+
There is no option to turn it off, and the agent never sets `rejectUnauthorized: false`.
|
|
177
|
+
|
|
178
|
+
### The Midline server
|
|
179
|
+
|
|
180
|
+
`https://api.usemidline.com` has a publicly trusted certificate, so nothing needs configuring. If
|
|
181
|
+
verification ever fails, you'll see one line like:
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
midline: https://api.usemidline.com presented a self-signed certificate (DEPTH_ZERO_SELF_SIGNED_CERT),
|
|
185
|
+
so the connection was refused. Certificate verification stays on. ...
|
|
186
|
+
Delivery resumes on its own once a trusted certificate is served. 12 event(s) buffered; your application is unaffected.
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
That is a server-side problem, not something to work around in your app. Events stay buffered and
|
|
190
|
+
delivery resumes on its own once the server is fixed — no restart needed.
|
|
191
|
+
|
|
192
|
+
For a **self-hosted** Midline server behind a private CA, trust that CA for the Midline connection
|
|
193
|
+
only:
|
|
204
194
|
|
|
205
195
|
```ts
|
|
206
|
-
MidlineAgent.init({
|
|
207
|
-
// Required
|
|
208
|
-
apiKey: string; // Get from https://midline.com/dashboard/api-keys
|
|
209
|
-
serviceName: string; // Name of your service (e.g., "checkout-api", "auth-service")
|
|
210
|
-
|
|
211
|
-
// Optional
|
|
212
|
-
maskFields?: string[]; // Field names to mask before sending (passwords, tokens, etc)
|
|
213
|
-
environment?: string; // e.g., "production", "staging", "development"
|
|
214
|
-
endpoint?: string; // Override default ingest URL (for self-hosted)
|
|
215
|
-
host?: string; // Machine/pod name for identifying instances
|
|
216
|
-
region?: string; // Deployment region (e.g., "us-east-1", "eu-west-1")
|
|
217
|
-
});
|
|
196
|
+
MidlineAgent.init({ apiKey, endpoint: "https://midline.internal", ca: fs.readFileSync("/etc/ssl/internal-ca.pem") });
|
|
218
197
|
```
|
|
219
198
|
|
|
220
|
-
|
|
199
|
+
```bash
|
|
200
|
+
MIDLINE_CUSTOM_CA=/etc/ssl/internal-ca.pem # PEM text also works
|
|
201
|
+
```
|
|
221
202
|
|
|
222
|
-
|
|
203
|
+
The CA is **added** to Node's default trust store, not substituted for it. Hostname checks still
|
|
204
|
+
apply.
|
|
223
205
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
- `token`
|
|
227
|
-
- `apiKey`
|
|
228
|
-
- `secret`
|
|
229
|
-
- `authorization`
|
|
206
|
+
`endpoint` must be `https://`. Plain `http://` is only accepted for `localhost`/`127.0.0.1` (running
|
|
207
|
+
a Midline server on your own machine), so the API key never crosses a network in cleartext.
|
|
230
208
|
|
|
231
|
-
|
|
209
|
+
### The destination (proxy mode)
|
|
210
|
+
|
|
211
|
+
A local destination is fine over plain HTTP: `TARGET_API_URL=http://localhost:4000`. That doesn't
|
|
212
|
+
relax anything about the Midline connection.
|
|
213
|
+
|
|
214
|
+
For an HTTPS destination signed by a private/internal CA:
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
TARGET_API_URL=https://orders.internal TARGET_API_CA=/etc/ssl/internal-ca.pem npx midline-agent proxy
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
`TARGET_API_CA` and `MIDLINE_CUSTOM_CA` are independent. Trusting a CA for one never trusts it for
|
|
221
|
+
the other.
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Configuration
|
|
232
226
|
|
|
233
227
|
```ts
|
|
234
228
|
MidlineAgent.init({
|
|
235
|
-
apiKey:
|
|
236
|
-
serviceName
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
229
|
+
apiKey: string, // MIDLINE_API_KEY
|
|
230
|
+
serviceName?: string, // MIDLINE_SERVICE_NAME
|
|
231
|
+
endpoint?: string, // MIDLINE_ENDPOINT — default https://api.usemidline.com (base or full ingest URL)
|
|
232
|
+
ca?: string | Buffer | Array, // MIDLINE_CUSTOM_CA — extra CA for the Midline server only
|
|
233
|
+
environment?: string, // MIDLINE_ENVIRONMENT
|
|
234
|
+
release?: string, // MIDLINE_RELEASE
|
|
235
|
+
host?: string,
|
|
236
|
+
region?: string,
|
|
237
|
+
enabled?: boolean, // MIDLINE_ENABLED=false keeps the agent inert
|
|
238
|
+
|
|
239
|
+
// What to capture beyond method, path, status and timing. All off by default.
|
|
240
|
+
capture?: {
|
|
241
|
+
headers?: boolean,
|
|
242
|
+
query?: boolean,
|
|
243
|
+
requestBody?: boolean, // whatever your body parser produced (req.body)
|
|
244
|
+
responseBody?: boolean,
|
|
245
|
+
maxBodyBytes?: number, // default 4096
|
|
246
|
+
},
|
|
247
|
+
redactFields?: string[], // added to the built-in list (maskFields still works)
|
|
248
|
+
redactHeaders?: string[],
|
|
249
|
+
|
|
250
|
+
// Delivery
|
|
251
|
+
flushIntervalMs?: number, // default 1500
|
|
252
|
+
timeoutMs?: number, // whole request deadline, default 10000
|
|
253
|
+
connectTimeoutMs?: number, // TCP + TLS handshake, default 5000
|
|
254
|
+
maxBatchSize?: number, // default 100
|
|
255
|
+
maxQueueSize?: number, // default 1000; oldest dropped past this
|
|
256
|
+
maxEventBytes?: number, // default 65536; bodies dropped first
|
|
257
|
+
maxRetryDelayMs?: number, // backoff cap, default 300000
|
|
258
|
+
|
|
259
|
+
onError?: (message: string) => void, // route diagnostics to your logger
|
|
260
|
+
debug?: boolean, // MIDLINE_DEBUG — log every diagnostic, not one per fault
|
|
245
261
|
});
|
|
246
262
|
```
|
|
247
263
|
|
|
248
|
-
|
|
264
|
+
### Environment
|
|
265
|
+
|
|
266
|
+
```env
|
|
267
|
+
MIDLINE_API_KEY=...
|
|
268
|
+
MIDLINE_ENDPOINT=https://api.usemidline.com
|
|
269
|
+
|
|
270
|
+
# proxy mode
|
|
271
|
+
TARGET_API_URL=http://localhost:4000
|
|
272
|
+
# TARGET_API_CA=/etc/ssl/internal-ca.pem
|
|
273
|
+
# TARGET_API_TIMEOUT_MS=30000
|
|
274
|
+
# TARGET_API_RETRIES=0
|
|
275
|
+
# MIDLINE_PROXY_PORT=8080
|
|
276
|
+
# MIDLINE_PROXY_HOST=127.0.0.1
|
|
277
|
+
```
|
|
249
278
|
|
|
250
279
|
---
|
|
251
280
|
|
|
252
|
-
##
|
|
281
|
+
## Sensitive data
|
|
282
|
+
|
|
283
|
+
Redaction happens **in your process, before an event is queued**. What's masked never reaches a
|
|
284
|
+
socket, a log line or the Midline server. The server redacts again on ingest as a second line of
|
|
285
|
+
defence.
|
|
286
|
+
|
|
287
|
+
- **Headers, always masked when captured:** `Authorization`, `Proxy-Authorization`, `Cookie`,
|
|
288
|
+
`Set-Cookie`, `X-API-Key`, `API-Key`, `X-Auth-Token`, `X-Access-Token`, `X-Refresh-Token`,
|
|
289
|
+
`X-CSRF-Token`, `X-XSRF-Token`, `X-Amz-Security-Token`, plus anything matching the field rules below.
|
|
290
|
+
- **Fields, at any depth, in bodies, queries and payloads.** Matched case- and
|
|
291
|
+
punctuation-insensitively, so `api_key`, `apiKey` and `X-API-Key` are the same key. Anything
|
|
292
|
+
containing `password`, `passwd`, `passphrase`, `secret`, `token`, `apikey`, `accesskey`,
|
|
293
|
+
`privatekey`, `authorization`, `cookie`, `session`, `credential`, `csrf`, `xsrf`, `signature`,
|
|
294
|
+
`creditcard`, `cardnumber`, `cvv`, `cvc`, `ssn`, `socialsecurity`; and the exact keys `auth`,
|
|
295
|
+
`pwd`, `pin`, `otp`, `sid`, `jwt`, `bearer`.
|
|
296
|
+
- **Values inside any text** — error messages, stack traces, routes: bearer/basic credentials,
|
|
297
|
+
JWTs, Midline keys (`ak_…`), Stripe and AWS key formats, PEM private keys, `user:pass@` in URLs,
|
|
298
|
+
and `key=value` / `"key": value` pairs whose key is sensitive.
|
|
299
|
+
- Query strings are never part of `route`.
|
|
300
|
+
- Bodies are only captured if you turn them on. They're capped at `maxBodyBytes`, and compressed or
|
|
301
|
+
binary bodies are skipped.
|
|
302
|
+
|
|
303
|
+
The lists err on the side of masking (`tokenCount` is masked too). Extend them with `redactFields`
|
|
304
|
+
and `redactHeaders`.
|
|
253
305
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
## Delivery and failure behaviour
|
|
309
|
+
|
|
310
|
+
The agent's contract is that the Midline server can never break your application.
|
|
258
311
|
|
|
259
|
-
|
|
312
|
+
- Recording happens after the response is handed to the socket; nothing waits on the network
|
|
313
|
+
- Events are batched (`POST /api/api-monitor/events/batch`) with the key in the `X-API-Key` header
|
|
314
|
+
- **Unreachable, DNS failure, timeout, TLS failure, 5xx, 408, 429:** events stay buffered.
|
|
315
|
+
Retries use exponential backoff with jitter, capped by `maxRetryDelayMs`, and honour `Retry-After`.
|
|
316
|
+
- **401/403:** the key is wrong. The agent logs once and turns itself off — retrying can't fix it.
|
|
317
|
+
- **413:** the batch is split and retried; a single event that's still too large is dropped
|
|
318
|
+
- **Other 4xx:** the batch is re-sent one event at a time, so a malformed event only costs itself
|
|
319
|
+
- **Redirects** are never followed, because that would hand the API key to wherever they point
|
|
320
|
+
- One log line per distinct fault, and one when delivery recovers
|
|
321
|
+
- The buffer is bounded (`maxQueueSize`, 16 MB total); past that, the oldest events go first
|
|
322
|
+
- The flush timer and background sockets are `unref`'d, so telemetry never keeps your process alive
|
|
323
|
+
|
|
324
|
+
Events still in memory are lost if the process crashes. That's fine for operational telemetry, but
|
|
325
|
+
not for billing or audit. On graceful shutdown:
|
|
260
326
|
|
|
261
327
|
```ts
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
throw new Error('Amount is required');
|
|
266
|
-
}
|
|
267
|
-
// Process payment...
|
|
268
|
-
} catch (error) {
|
|
269
|
-
next(error); // Pass to Midline error handler
|
|
270
|
-
}
|
|
328
|
+
process.on("SIGTERM", async () => {
|
|
329
|
+
await MidlineAgent.shutdown(5000); // flush with a deadline, then close
|
|
330
|
+
process.exit(0);
|
|
271
331
|
});
|
|
272
|
-
|
|
273
|
-
// The error handler will capture and report this
|
|
274
|
-
app.use(midlineErrorHandler());
|
|
275
332
|
```
|
|
276
333
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
## How Delivery Works
|
|
334
|
+
### Request and correlation IDs
|
|
280
335
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
336
|
+
Every request gets a `requestId`: an incoming `X-Request-Id` if it's sane, otherwise a UUID. It also
|
|
337
|
+
gets a `correlationId`: `X-Correlation-Id` if present, otherwise the request ID. A W3C
|
|
338
|
+
`traceparent` header becomes `traceId`/`spanId`. Read them in your own code to stamp logs:
|
|
284
339
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
340
|
+
```ts
|
|
341
|
+
import { getRequestContext } from "midline-agent";
|
|
342
|
+
app.get("/x", (req, res) => { logger.info({ requestId: getRequestContext(req)?.requestId }); });
|
|
343
|
+
```
|
|
289
344
|
|
|
290
345
|
---
|
|
291
346
|
|
|
292
|
-
##
|
|
347
|
+
## Any other backend
|
|
293
348
|
|
|
294
|
-
|
|
295
|
-
the Midline ingest API.
|
|
296
|
-
|
|
349
|
+
The agent does two things: it turns a request or an error into an event, and it POSTs that event to
|
|
350
|
+
the Midline ingest API. Your own service can do both directly, in whatever language it's written in
|
|
351
|
+
— or run `midline-agent proxy` in front of it.
|
|
297
352
|
|
|
298
353
|
**Endpoints**
|
|
299
354
|
|
|
300
355
|
| Endpoint | Use |
|
|
301
356
|
| --- | --- |
|
|
302
357
|
| `POST https://api.usemidline.com/api/api-monitor/events` | One event |
|
|
303
|
-
| `POST https://api.usemidline.com/api/api-monitor/events/batch` | Many events — `{ "events": [ … ] }
|
|
358
|
+
| `POST https://api.usemidline.com/api/api-monitor/events/batch` | Many events — `{ "events": [ … ] }`, up to 500 |
|
|
304
359
|
|
|
305
|
-
|
|
360
|
+
Authenticate with the `X-API-Key` header. An `apiKey` field in the body is still accepted.
|
|
306
361
|
|
|
307
362
|
**Event shape**
|
|
308
363
|
|
|
309
364
|
| Field | Required | Notes |
|
|
310
365
|
| --- | --- | --- |
|
|
311
|
-
| `apiKey` | yes | From Dashboard → API Keys |
|
|
312
366
|
| `eventType` | yes | `request` · `error` · `security` · `performance` · `custom` |
|
|
313
|
-
| `route` | yes |
|
|
367
|
+
| `route` | yes | Path only, e.g. `/v1/charges` |
|
|
314
368
|
| `method` | no | HTTP method |
|
|
315
369
|
| `statusCode` | no | 100–599 |
|
|
316
370
|
| `responseTime` | no | Milliseconds |
|
|
317
371
|
| `severity` | no | `low` · `medium` · `high` · `critical` |
|
|
318
372
|
| `category` | no | `application` · `infrastructure` · `security` · `performance` · `business` |
|
|
319
|
-
| `service` | no |
|
|
320
|
-
| `
|
|
321
|
-
| `
|
|
322
|
-
| `payload` | no | Free-form context — error message, stack,
|
|
373
|
+
| `service` / `environment` / `release` | no | Where it came from |
|
|
374
|
+
| `timestamp` | no | ISO 8601; clamped to server time if implausibly far off |
|
|
375
|
+
| `requestId` / `correlationId` / `traceId` | no | For joining events across services |
|
|
376
|
+
| `payload` | no | Free-form context — error message, stack, captured request/response |
|
|
377
|
+
| `metadata` | no | Free-form, e.g. SDK name and version |
|
|
323
378
|
|
|
324
|
-
Mask sensitive fields in your own process
|
|
379
|
+
Mask sensitive fields in your own process before you build the payload. Nothing you don't send can
|
|
325
380
|
be stored.
|
|
326
381
|
|
|
327
|
-
**curl**
|
|
328
|
-
|
|
329
382
|
```bash
|
|
330
383
|
curl -X POST https://api.usemidline.com/api/api-monitor/events \
|
|
331
384
|
-H "Content-Type: application/json" \
|
|
385
|
+
-H "X-API-Key: $MIDLINE_API_KEY" \
|
|
332
386
|
-d '{
|
|
333
|
-
"apiKey": "YOUR_API_KEY",
|
|
334
387
|
"eventType": "error",
|
|
335
388
|
"service": "checkout-api",
|
|
336
389
|
"route": "/v1/charges",
|
|
@@ -343,67 +396,50 @@ curl -X POST https://api.usemidline.com/api/api-monitor/events \
|
|
|
343
396
|
}'
|
|
344
397
|
```
|
|
345
398
|
|
|
346
|
-
**Python (Django / Flask / FastAPI)**
|
|
347
|
-
|
|
348
399
|
```python
|
|
349
|
-
import
|
|
400
|
+
import os, threading, requests
|
|
350
401
|
|
|
351
402
|
INGEST = "https://api.usemidline.com/api/api-monitor/events"
|
|
352
403
|
|
|
353
404
|
def report(event: dict) -> None:
|
|
354
|
-
|
|
355
|
-
event.setdefault("service", "checkout-api")
|
|
356
|
-
# off the request path - never make your user wait on telemetry
|
|
405
|
+
# off the request path - never make a user wait on telemetry
|
|
357
406
|
threading.Thread(
|
|
358
|
-
target=lambda: requests.post(INGEST, json=event, timeout=3),
|
|
407
|
+
target=lambda: requests.post(INGEST, json=event, headers={"X-API-Key": os.environ["MIDLINE_API_KEY"]}, timeout=3),
|
|
359
408
|
daemon=True,
|
|
360
409
|
).start()
|
|
361
|
-
|
|
362
|
-
report({
|
|
363
|
-
"eventType": "error",
|
|
364
|
-
"route": "/v1/charges",
|
|
365
|
-
"method": "POST",
|
|
366
|
-
"statusCode": 500,
|
|
367
|
-
"severity": "critical",
|
|
368
|
-
"category": "application",
|
|
369
|
-
"payload": {"error": str(exc)},
|
|
370
|
-
})
|
|
371
410
|
```
|
|
372
411
|
|
|
373
|
-
**Go**
|
|
374
|
-
|
|
375
412
|
```go
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
func Report(e Event) {
|
|
387
|
-
e.APIKey = os.Getenv("MIDLINE_KEY")
|
|
388
|
-
body, _ := json.Marshal(e)
|
|
389
|
-
go http.Post(
|
|
390
|
-
"https://api.usemidline.com/api/api-monitor/events",
|
|
391
|
-
"application/json",
|
|
392
|
-
bytes.NewReader(body),
|
|
393
|
-
) // fire and forget
|
|
413
|
+
func Report(body []byte) {
|
|
414
|
+
go func() {
|
|
415
|
+
req, _ := http.NewRequest("POST", "https://api.usemidline.com/api/api-monitor/events", bytes.NewReader(body))
|
|
416
|
+
req.Header.Set("Content-Type", "application/json")
|
|
417
|
+
req.Header.Set("X-API-Key", os.Getenv("MIDLINE_API_KEY"))
|
|
418
|
+
client := &http.Client{Timeout: 3 * time.Second}
|
|
419
|
+
if resp, err := client.Do(req); err == nil {
|
|
420
|
+
resp.Body.Close()
|
|
421
|
+
}
|
|
422
|
+
}()
|
|
394
423
|
}
|
|
395
424
|
```
|
|
396
425
|
|
|
397
|
-
|
|
398
|
-
|
|
426
|
+
Same rule everywhere: send events from a goroutine, a background thread or a queue worker. Keep TLS
|
|
427
|
+
verification on. Telemetry should never be able to slow down or fail a response.
|
|
428
|
+
|
|
429
|
+
---
|
|
430
|
+
|
|
431
|
+
## Upgrading from 0.1.x
|
|
399
432
|
|
|
400
|
-
|
|
433
|
+
- `insecureTLS` / `MIDLINE_INSECURE_TLS` are gone (they were never published). Fix the server's
|
|
434
|
+
certificate, or pass the private CA with `ca` / `MIDLINE_CUSTOM_CA`.
|
|
435
|
+
- `endpoint` can now be just `https://api.usemidline.com`; full ingest URLs still work.
|
|
436
|
+
- `maskFields` used to be accepted and ignored. It's now honoured, as an alias for `redactFields`.
|
|
437
|
+
- `serviceName` is optional; `MIDLINE_API_KEY` / `MIDLINE_ENDPOINT` are read if not passed.
|
|
438
|
+
- `cross-fetch` is no longer a dependency; Node 18+ is required.
|
|
439
|
+
- New: `MidlineAgent.shutdown(timeoutMs)` flushes with a deadline and stops the agent.
|
|
440
|
+
- The agent sends the key both in the `X-API-Key` header and in each event body. That keeps it
|
|
441
|
+
compatible with Midline servers that predate header auth.
|
|
401
442
|
|
|
402
|
-
|
|
403
|
-
at your fence URL and every request is authenticated, scanned, logged and forwarded upstream
|
|
404
|
-
unchanged. That path is language-agnostic by construction — the gateway never knows or cares what
|
|
405
|
-
your backend is written in.
|
|
443
|
+
## License
|
|
406
444
|
|
|
407
|
-
### License
|
|
408
445
|
MIT
|
|
409
|
-
# Midline-agent
|