haechi 1.1.2 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,29 +1,207 @@
1
1
  import { createServer } from "node:http";
2
+ import { createServer as createHttpsServer } from "node:https";
2
3
  import { createHash, randomUUID } from "node:crypto";
3
4
  import { isUtf8 } from "node:buffer";
5
+ import { readFileSync } from "node:fs";
6
+ import { fileURLToPath } from "node:url";
4
7
  import { inspectResponseStream } from "../stream-filter/index.mjs";
5
8
 
6
9
  export const DEFAULT_PROXY_PORT = 11016;
7
10
 
8
- export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "127.0.0.1", allowRemoteBind = false }) {
9
- assertSafeProxyBind({ host, allowRemoteBind });
11
+ // The published package version, read once from the package's own manifest.
12
+ // package.json IS in the published tarball, and packages/proxy/index.mjs sits
13
+ // two levels below the repo root, so this URL resolves in both the dev tree and
14
+ // the packed tarball. Falls back to "unknown" rather than throwing — a version
15
+ // read must never break proxy startup.
16
+ export const HAECHI_VERSION = readPackageVersion();
17
+
18
+ function readPackageVersion() {
19
+ try {
20
+ const pkgUrl = new URL("../../package.json", import.meta.url);
21
+ const pkg = JSON.parse(readFileSync(fileURLToPath(pkgUrl), "utf8"));
22
+ return typeof pkg.version === "string" ? pkg.version : "unknown";
23
+ } catch {
24
+ return "unknown";
25
+ }
26
+ }
27
+
28
+ // A tlsContext is usable iff it can actually terminate TLS: (key && cert) or pfx.
29
+ // This is the SINGLE source of truth for both the bind guard and server
30
+ // selection — the SAME shape the haechi-dashboard satellite uses, so the proxy
31
+ // and the dashboard share one TLS-material predicate. A non-null tlsContext that
32
+ // fails this check must fail closed (never green-light a remote bind that then
33
+ // builds a plaintext http server).
34
+ export function hasUsableTlsMaterial(ctx) {
35
+ if (!ctx || typeof ctx !== "object" || Array.isArray(ctx)) {
36
+ return false;
37
+ }
38
+ const hasKeyCert = Boolean(ctx.key) && Boolean(ctx.cert);
39
+ const hasPfx = Boolean(ctx.pfx);
40
+ return hasKeyCert || hasPfx;
41
+ }
42
+
43
+ // Structured logger honoring config.logging.format. In "json" mode it emits a
44
+ // single JSON line carrying a correlationId and an error NAME/class — NEVER a
45
+ // request/response payload, headers, token, or any PII. In "text" mode it
46
+ // preserves the prior human-readable console output.
47
+ function createLogger(format = "text") {
48
+ const json = format === "json";
49
+ return {
50
+ error(event, fields = {}) {
51
+ if (json) {
52
+ process.stderr.write(`${JSON.stringify({ level: "error", event, ...fields })}\n`);
53
+ } else {
54
+ const parts = Object.entries(fields)
55
+ .filter(([, value]) => value !== undefined && value !== null)
56
+ .map(([key, value]) => `${key}=${value}`);
57
+ process.stderr.write(`haechi ${event}${parts.length ? `: ${parts.join(" ")}` : ""}\n`);
58
+ }
59
+ }
60
+ };
61
+ }
62
+
63
+ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "127.0.0.1", allowRemoteBind = false, tlsContext, trustForwardedProto }) {
10
64
  const { haechi, config, protocolAdapter } = runtime;
11
- const rateLimiter = createRateLimiter();
12
65
 
13
- const server = createServer(async (request, response) => {
66
+ // WS6 TLS hardening. The tlsContext / trustForwardedProto source of truth is
67
+ // the normalized config (proxy.tls is loaded into a tlsContext at startup;
68
+ // proxy.trustForwardedProto is a boolean), but an explicit argument overrides
69
+ // it (so a hand-built runtime / a test can drive these directly). hasUsableTls
70
+ // is the same predicate the dashboard satellite uses.
71
+ const resolvedTlsContext = tlsContext !== undefined ? tlsContext : (config.proxy?.tls ?? null);
72
+ const resolvedTrustForwardedProto = trustForwardedProto !== undefined
73
+ ? trustForwardedProto
74
+ : Boolean(config.proxy?.trustForwardedProto);
75
+ const usableTls = hasUsableTlsMaterial(resolvedTlsContext);
76
+
77
+ // Bind guard, two layers. (1) the loopback/remote-bind gate (shared with the
78
+ // dashboard). (2) WS6: a remote bind ADDITIONALLY requires a usable tlsContext
79
+ // OR an explicit trustForwardedProto acknowledgement (a trusted reverse proxy
80
+ // terminates TLS in front of Haechi). Otherwise it THROWS at startup — the
81
+ // proxy must NEVER serve bearer tokens + payloads in plaintext on a remote
82
+ // bind. Loopback dev is unaffected (plain http, no TLS).
83
+ assertSafeProxyBind({ host, allowRemoteBind });
84
+ assertSafeProxyTransport({
85
+ host,
86
+ allowRemoteBind,
87
+ hasUsableTls: usableTls,
88
+ trustForwardedProto: resolvedTrustForwardedProto
89
+ });
90
+
91
+ // When the remote bind rests on trustForwardedProto (plain http behind a
92
+ // trusted TLS hop) we REJECT any protected-route request whose
93
+ // X-Forwarded-Proto is not https — a plaintext request that bypassed the hop.
94
+ // This is only meaningful for a non-loopback, plain-http, trust-forwarded
95
+ // listener; a loopback dev server or an https-terminating server never gates.
96
+ const enforceForwardedProto = !isLoopbackHost(host)
97
+ && allowRemoteBind
98
+ && !usableTls
99
+ && resolvedTrustForwardedProto;
100
+ // The runtime owns the rate limiter (an injectable collaborator). Fall back to
101
+ // a local per-process default so a hand-built runtime object without a
102
+ // rateLimiter still works (backward-compatible). The default and the runtime's
103
+ // default share the same allow(key, limit) -> boolean fixed-window contract.
104
+ const rateLimiter = runtime.rateLimiter ?? createRateLimiter();
105
+ // The metrics collector is owned by the runtime (injectable). Fall back to a
106
+ // no-op so a hand-built runtime object without metrics still works.
107
+ const metrics = runtime.metrics ?? noopMetrics();
108
+ const logger = createLogger(config.logging?.format ?? "text");
109
+
110
+ // WS4-B backpressure: a configurable global max-in-flight ceiling. 0 (default)
111
+ // disables it, preserving 1.1 behavior. When > 0 and the live count is at the
112
+ // ceiling, a NEW non-exempt request is rejected 503 + Retry-After BEFORE auth
113
+ // and body-read. The /__haechi/* observability routes are EXEMPT so metrics +
114
+ // liveness can be scraped under saturation.
115
+ const maxInFlight = config.limits.maxInFlight ?? 0;
116
+ const retryAfterSeconds = Math.max(1, Math.ceil((config.limits.shutdownGraceMs ?? 10000) / 1000));
117
+ // Live in-flight request count for the drain-tracking AND the ceiling. A
118
+ // bounded integer — never identity/value bearing.
119
+ let inFlight = 0;
120
+ // Resolves once in-flight drains to zero during a graceful close().
121
+ let drained = null;
122
+ let resolveDrained = null;
123
+
124
+ const requestHandler = async (request, response) => {
125
+ // Per-REQUEST correlation id: generated here, threaded into every protect
126
+ // context (so the audit events of one request share it) AND into the error
127
+ // log. A UUID — never a payload/identity/PII value.
128
+ const correlationId = randomUUID();
129
+ const startedAt = process.hrtime.bigint();
130
+ let routeId = "unknown";
131
+
132
+ // Observability routes are exempt from the in-flight ceiling and are NOT
133
+ // counted toward it: liveness/readiness/metrics must answer under saturation.
134
+ const exemptRoute = isObservabilityRoute(request);
135
+
136
+ // Backpressure: reject at the ceiling BEFORE doing any work. Counted in
137
+ // metrics by a bounded enum decision; the body is never read.
138
+ if (!exemptRoute && maxInFlight > 0 && inFlight >= maxInFlight) {
139
+ metrics.increment("haechi_overloaded_total");
140
+ response.writeHead(503, {
141
+ "content-type": "application/json",
142
+ "retry-after": String(retryAfterSeconds)
143
+ });
144
+ response.end(`${JSON.stringify({ error: "haechi_overloaded", message: "Server at max in-flight capacity; retry later" }, null, 2)}\n`);
145
+ return;
146
+ }
147
+
148
+ // Track in-flight for graceful drain + the ceiling. Decrement in finally so a
149
+ // throw/early-return can never leak the count (which would wedge close()).
150
+ let counted = false;
151
+ if (!exemptRoute) {
152
+ inFlight += 1;
153
+ counted = true;
154
+ }
14
155
  try {
156
+ // WS6 forwarded-proto enforcement. When this is a non-loopback plain-http
157
+ // listener resting on trustForwardedProto (a trusted reverse proxy
158
+ // terminates TLS in front of Haechi), a request whose X-Forwarded-Proto is
159
+ // not "https" arrived over plaintext that BYPASSED the TLS hop — reject it
160
+ // fail-closed BEFORE auth and body-read, so a protected route never serves
161
+ // tokens/payloads over an unverified-plaintext hop. The /__haechi/* liveness
162
+ // routes are EXEMPT (they leak nothing) so a health check / metrics scrape
163
+ // from the loopback sidecar still answers.
164
+ if (enforceForwardedProto && !exemptRoute && !isForwardedHttps(request)) {
165
+ writeJson(response, 403, {
166
+ error: "haechi_forwarded_proto_required",
167
+ message: "This proxy runs behind a trusted TLS-terminating hop (proxy.trustForwardedProto). A request without X-Forwarded-Proto: https bypassed the hop and is refused."
168
+ });
169
+ return;
170
+ }
171
+ // Health + telemetry endpoints are unauthenticated and checked BEFORE auth
172
+ // and body-read. They live under the reserved /__haechi/* prefix.
173
+ if (request.method === "GET" && request.url === "/__haechi/live") {
174
+ // Cheap process liveness. Always 200 while the event loop is serving.
175
+ writeJson(response, 200, { ok: true, version: HAECHI_VERSION });
176
+ return;
177
+ }
178
+ if (request.method === "GET" && request.url === "/__haechi/ready") {
179
+ await handleReady({ runtime, response });
180
+ return;
181
+ }
15
182
  if (request.method === "GET" && request.url === "/__haechi/health") {
16
- // Intentionally unauthenticated; exposes only the mode.
17
- writeJson(response, 200, { ok: true, mode: config.mode });
183
+ // Back-compat: keep the original shape (ok + mode) and add version.
184
+ writeJson(response, 200, { ok: true, mode: config.mode, version: HAECHI_VERSION });
185
+ return;
186
+ }
187
+ if (request.method === "GET" && request.url === "/__haechi/metrics") {
188
+ if (!config.metrics?.enabled) {
189
+ writeJson(response, 404, { error: "haechi_metrics_disabled", message: "Metrics endpoint is disabled (metrics.enabled: false)" });
190
+ return;
191
+ }
192
+ response.writeHead(200, { "content-type": "text/plain; version=0.0.4; charset=utf-8" });
193
+ response.end(metrics.render());
18
194
  return;
19
195
  }
20
196
 
21
197
  assertRelativeProxyTarget(request.url);
22
198
  const routeContext = protocolAdapter.classifyRequest(request);
199
+ routeId = routeContext?.routeId ?? "unknown";
200
+ const mode = config.policy.mode ?? config.mode;
23
201
 
24
202
  // Authenticate, resolve the policy profile, and rate-limit BEFORE reading
25
203
  // the body, so a denied/throttled request cannot stream a large body.
26
- const gate = await authorizeRequest({ runtime, request, routeContext, rateLimiter });
204
+ const gate = await authorizeRequest({ runtime, request, routeContext, rateLimiter, metrics, correlationId });
27
205
  if (gate.denied) {
28
206
  writeJson(response, gate.denied.status, {
29
207
  error: gate.denied.error,
@@ -32,7 +210,7 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
32
210
  return;
33
211
  }
34
212
  const { identity, profile, policyEngine, modelAllowlist } = gate;
35
- const authContext = { identity, profile, policyEngine };
213
+ const authContext = { identity, profile, policyEngine, correlationId };
36
214
 
37
215
  const body = await readBody(request, {
38
216
  maxBytes: config.limits.maxRequestBytes
@@ -42,12 +220,14 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
42
220
  // Model allowlist runs after body read (the model field is in the body).
43
221
  if (modelAllowlist && typeof json?.model === "string" && !modelAllowlist.includes(json.model)) {
44
222
  await recordProxyDecision({
45
- runtime, routeContext, identity, profile,
223
+ runtime, routeContext, identity, profile, correlationId,
46
224
  decision: "model_not_allowed",
47
225
  reason: `model:${json.model}`,
48
226
  enforced: true,
49
227
  blocked: true
50
228
  });
229
+ countDecision(metrics, { routeContext, mode, decision: "model_not_allowed" });
230
+ metrics.increment("haechi_blocks_total");
51
231
  writeJson(response, 403, {
52
232
  error: "haechi_model_not_allowed",
53
233
  message: `Model not allowed: ${json.model}`
@@ -57,7 +237,7 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
57
237
 
58
238
  if (isStreamingRequest(json, routeContext)) {
59
239
  if (config.streaming.requestMode === "inspect") {
60
- await handleInspectedStream({ runtime, request, response, routeContext, json, authContext });
240
+ await handleInspectedStream({ runtime, request, response, routeContext, json, authContext, metrics });
61
241
  return;
62
242
  }
63
243
 
@@ -67,16 +247,19 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
67
247
  routeContext,
68
248
  identity,
69
249
  profile,
250
+ correlationId,
70
251
  decision: "streaming_request_pass_through",
71
252
  reason: "streaming_request_pass_through",
72
253
  enforced: false,
73
254
  blocked: false
74
255
  });
256
+ countDecision(metrics, { routeContext, mode, decision: "forwarded" });
75
257
  const upstreamResponse = await forward({
76
258
  upstream: config.target.upstream,
77
259
  request,
78
260
  body,
79
- timeoutMs: config.limits.upstreamTimeoutMs
261
+ timeoutMs: config.limits.upstreamTimeoutMs,
262
+ metrics
80
263
  });
81
264
  const { body: rawBody } = await readUpstreamBody(upstreamResponse);
82
265
  response.writeHead(upstreamResponse.status, Object.fromEntries(upstreamResponse.headers.entries()));
@@ -84,6 +267,7 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
84
267
  return;
85
268
  }
86
269
 
270
+ countDecision(metrics, { routeContext, mode, decision: "streaming_blocked" });
87
271
  writeJson(response, 501, {
88
272
  error: "haechi_streaming_unsupported",
89
273
  message: "Streaming requests are blocked unless streaming.requestMode is set to pass-through or inspect"
@@ -97,11 +281,13 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
97
281
  ...authContext,
98
282
  operation: `request:${routeContext.operation}`,
99
283
  direction: "request",
100
- mode: config.policy.mode ?? config.mode
284
+ mode
101
285
  })
102
286
  : { payload: json, blocked: false };
103
287
 
104
288
  if (result.blocked) {
289
+ countDecision(metrics, { routeContext, mode, decision: "blocked" });
290
+ metrics.increment("haechi_blocks_total");
105
291
  writeJson(response, 403, {
106
292
  error: "haechi_policy_block",
107
293
  summary: result.summary,
@@ -114,7 +300,8 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
114
300
  upstream: config.target.upstream,
115
301
  request,
116
302
  body: JSON.stringify(result.payload),
117
- timeoutMs: config.limits.upstreamTimeoutMs
303
+ timeoutMs: config.limits.upstreamTimeoutMs,
304
+ metrics
118
305
  });
119
306
 
120
307
  const forwarded = await maybeProtectResponse({
@@ -122,58 +309,234 @@ export function createHaechiProxy({ runtime, port = DEFAULT_PROXY_PORT, host = "
122
309
  routeContext,
123
310
  runtime,
124
311
  authContext,
125
- issuedTokens: result.issuedTokens ?? []
312
+ issuedTokens: result.issuedTokens ?? [],
313
+ metrics
126
314
  });
127
315
 
316
+ countDecision(metrics, {
317
+ routeContext,
318
+ mode,
319
+ decision: forwarded.decision ?? "forwarded"
320
+ });
128
321
  response.writeHead(forwarded.status, forwarded.headers);
129
322
  response.end(forwarded.body);
130
323
  } catch (error) {
131
324
  const expected = typeof error?.statusCode === "number";
132
325
  if (!expected) {
133
- console.error(`haechi proxy internal error: ${error?.stack ?? error?.message ?? error}`);
326
+ // Carry the error NAME/class + correlationId only — NEVER the payload,
327
+ // headers, token, or any PII.
328
+ logger.error("proxy_internal_error", {
329
+ correlationId,
330
+ errorName: error?.name ?? "Error",
331
+ statusCode: error?.statusCode ?? 500
332
+ });
333
+ metrics.increment("haechi_internal_error_total");
134
334
  }
135
335
  writeJson(response, error.statusCode ?? 500, {
136
336
  error: error.errorCode ?? "haechi_proxy_error",
137
337
  message: expected ? error.message : "Internal proxy error"
138
338
  });
339
+ } finally {
340
+ const elapsedSeconds = Number(process.hrtime.bigint() - startedAt) / 1e9;
341
+ // route label is a bounded route id (or "unknown") — never an identity/value.
342
+ metrics.observe("haechi_request_duration_seconds", elapsedSeconds, { route: routeId });
343
+ if (counted) {
344
+ inFlight -= 1;
345
+ // If a graceful close() is awaiting drain and we just hit zero, resolve it.
346
+ if (resolveDrained && inFlight <= 0) {
347
+ resolveDrained();
348
+ }
349
+ }
139
350
  }
140
- });
351
+ };
352
+
353
+ // Server selection: a usable tlsContext → an https listener terminating TLS in
354
+ // this process; otherwise plain http (unchanged for loopback/dev). The bind
355
+ // guard above already guarantees a non-loopback bind without usable TLS carries
356
+ // an explicit trustForwardedProto acknowledgement, so a plain-http server is
357
+ // only ever exposed remotely behind a trusted TLS hop (and gated below).
358
+ const server = usableTls
359
+ ? createHttpsServer(resolvedTlsContext, requestHandler)
360
+ : createServer(requestHandler);
361
+ const servesHttps = usableTls;
362
+
363
+ // WS4-B tuned timeouts. Only override Node's server defaults when a value is
364
+ // configured (null = leave Node's default untouched, so behavior is unchanged
365
+ // unless an operator opts in). requestTimeout caps the whole request; a value
366
+ // of 0 disables the timeout (Node semantics) — validated upstream.
367
+ if (config.limits.requestTimeoutMs !== null && config.limits.requestTimeoutMs !== undefined) {
368
+ server.requestTimeout = config.limits.requestTimeoutMs;
369
+ }
370
+ if (config.limits.headersTimeoutMs !== null && config.limits.headersTimeoutMs !== undefined) {
371
+ server.headersTimeout = config.limits.headersTimeoutMs;
372
+ }
141
373
 
142
374
  return {
143
375
  server,
376
+ // Whether THIS listener terminates TLS (https) — the CLI/log line reflects
377
+ // the right scheme, and a caller can assert the selected transport.
378
+ servesHttps,
144
379
  listen() {
145
380
  return new Promise((resolve) => {
146
381
  server.listen(port, host, () => {
147
382
  const address = server.address();
148
- resolve({ host: address.address, port: address.port });
383
+ resolve({ host: address.address, port: address.port, tls: servesHttps });
149
384
  });
150
385
  });
151
386
  },
387
+ // WS4-B graceful drain. Stop accepting new connections, immediately close
388
+ // idle keep-alive sockets, and wait for in-flight requests to drain. After a
389
+ // configurable grace period (limits.shutdownGraceMs) force-close any lingering
390
+ // socket so a stuck keep-alive cannot hold shutdown open forever. The grace
391
+ // timer is .unref()-ed and cleared on a clean drain so `node --test` never
392
+ // hangs on a leaked timer.
152
393
  close() {
394
+ const graceMs = config.limits.shutdownGraceMs ?? 10000;
153
395
  return new Promise((resolve, reject) => {
154
- server.close((error) => error ? reject(error) : resolve());
396
+ let settled = false;
397
+ let graceTimer = null;
398
+
399
+ const finish = (error) => {
400
+ if (settled) {
401
+ return;
402
+ }
403
+ settled = true;
404
+ if (graceTimer) {
405
+ clearTimeout(graceTimer);
406
+ graceTimer = null;
407
+ }
408
+ resolveDrained = null;
409
+ drained = null;
410
+ if (error) {
411
+ reject(error);
412
+ } else {
413
+ resolve();
414
+ }
415
+ };
416
+
417
+ // Stop accepting new connections; the callback fires once all
418
+ // connections are closed (idle ones we close now, in-flight ones once
419
+ // they drain or the grace timer force-closes them).
420
+ server.close((error) => finish(error));
421
+
422
+ // Close idle keep-alive sockets immediately so they don't keep the
423
+ // server open waiting for a request that will never come.
424
+ if (typeof server.closeIdleConnections === "function") {
425
+ server.closeIdleConnections();
426
+ }
427
+
428
+ // If nothing is in flight, the close callback will fire promptly; still
429
+ // arm a drain resolver in case requests are mid-flight.
430
+ if (inFlight <= 0) {
431
+ // No in-flight work; closeIdleConnections handled keep-alive, so
432
+ // server.close() resolves on its own. Nothing more to wait for.
433
+ return;
434
+ }
435
+
436
+ // Wait for in-flight requests to drain, then force-close stragglers
437
+ // (the force close covers a request whose socket lingers after we stop).
438
+ drained = new Promise((res) => { resolveDrained = res; });
439
+ drained.then(() => {
440
+ if (typeof server.closeAllConnections === "function") {
441
+ server.closeAllConnections();
442
+ }
443
+ });
444
+
445
+ // Grace cap: after graceMs force every remaining connection closed so a
446
+ // lingering keep-alive socket cannot hold shutdown open forever. unref()
447
+ // so this timer alone never keeps the event loop (and `node --test`) alive.
448
+ graceTimer = setTimeout(() => {
449
+ if (typeof server.closeAllConnections === "function") {
450
+ server.closeAllConnections();
451
+ }
452
+ }, graceMs);
453
+ if (typeof graceTimer.unref === "function") {
454
+ graceTimer.unref();
455
+ }
155
456
  });
156
457
  }
157
458
  };
158
459
  }
159
460
 
461
+ // True for the reserved /__haechi/* observability routes (live/ready/health/
462
+ // metrics). These are EXEMPT from the in-flight ceiling so liveness + metrics
463
+ // stay scrapable under saturation, and they do not count toward drain tracking.
464
+ function isObservabilityRoute(request) {
465
+ return request.method === "GET" && typeof request.url === "string" && request.url.startsWith("/__haechi/");
466
+ }
467
+
468
+ // Readiness probe (WS4-A). FAIL-CLOSED: a gateway that cannot write its audit
469
+ // log is NOT ready (503). Runs the audit sink's optional ready()/healthCheck()
470
+ // — if the sink lacks one, audit is treated as ready. The checks object carries
471
+ // only booleans/enums; never a path, payload, or PII value.
472
+ async function handleReady({ runtime, response }) {
473
+ const checks = {};
474
+ let ready = true;
475
+
476
+ const probe = runtime.auditSink?.ready ?? runtime.auditSink?.healthCheck;
477
+ if (typeof probe === "function") {
478
+ try {
479
+ const result = await probe.call(runtime.auditSink);
480
+ checks.auditWritable = result === true || result?.ok === true;
481
+ } catch {
482
+ checks.auditWritable = false;
483
+ }
484
+ } else {
485
+ // No probe method on the sink → cannot disprove writability; treat as ready.
486
+ checks.auditWritable = true;
487
+ }
488
+ if (checks.auditWritable !== true) {
489
+ ready = false;
490
+ }
491
+
492
+ writeJson(response, ready ? 200 : 503, { ready, version: HAECHI_VERSION, checks });
493
+ }
494
+
495
+ // Increment the request counter with a bounded enum label set. route is a route
496
+ // id, mode is the policy mode, decision is a fixed decision class — NEVER an
497
+ // identity/token/detected value (no-PII-in-telemetry invariant).
498
+ function countDecision(metrics, { routeContext, mode, decision }) {
499
+ metrics?.increment("haechi_requests_total", {
500
+ route: routeContext?.routeId ?? "unknown",
501
+ mode: mode ?? "unknown",
502
+ decision
503
+ });
504
+ }
505
+
506
+ // Backward-compat fallback for a hand-built runtime object without metrics: a
507
+ // no-op collector with the same increment/observe/render contract.
508
+ function noopMetrics() {
509
+ return {
510
+ increment() {},
511
+ observe() {},
512
+ render() {
513
+ return "";
514
+ }
515
+ };
516
+ }
517
+
160
518
  // Authenticate → resolve policy profile → rate-limit. Returns the request's
161
519
  // identity/profile/policyEngine/modelAllowlist, or a denial. Auth is required
162
520
  // exactly when an authProvider is configured (auth.provider !== "none").
163
- async function authorizeRequest({ runtime, request, routeContext, rateLimiter }) {
164
- const { authProvider, policyProfiles } = runtime;
521
+ async function authorizeRequest({ runtime, request, routeContext, rateLimiter, metrics, correlationId }) {
522
+ const { authProvider, policyProfiles, config } = runtime;
523
+ const mode = config.policy.mode ?? config.mode;
165
524
  let identity = null;
166
525
 
167
526
  if (authProvider) {
168
527
  try {
169
528
  identity = await authProvider.authenticate(request);
170
529
  } catch {
171
- await recordAuthDenied({ runtime, routeContext, reason: "provider_error" });
530
+ await recordAuthDenied({ runtime, routeContext, reason: "provider_error", correlationId });
531
+ countDecision(metrics, { routeContext, mode, decision: "auth_denied" });
532
+ metrics.increment("haechi_auth_denied_total");
172
533
  return { denied: { status: 401, error: "haechi_auth_denied", message: "Authentication failed" } };
173
534
  }
174
535
  if (!identity) {
175
536
  const reason = hasBearerHeader(request) ? "invalid_token" : "no_token";
176
- await recordAuthDenied({ runtime, routeContext, reason });
537
+ await recordAuthDenied({ runtime, routeContext, reason, correlationId });
538
+ countDecision(metrics, { routeContext, mode, decision: "auth_denied" });
539
+ metrics.increment("haechi_auth_denied_total");
177
540
  return { denied: { status: 401, error: "haechi_auth_denied", message: "Authentication required" } };
178
541
  }
179
542
  }
@@ -184,12 +547,14 @@ async function authorizeRequest({ runtime, request, routeContext, rateLimiter })
184
547
  const key = identity?.id ?? "anonymous";
185
548
  if (!rateLimiter.allow(key, resolved.rate.requestsPerMinute)) {
186
549
  await recordProxyDecision({
187
- runtime, routeContext, identity, profile: resolved.profile,
550
+ runtime, routeContext, identity, profile: resolved.profile, correlationId,
188
551
  decision: "rate_limited",
189
552
  reason: `rate:${resolved.rate.requestsPerMinute}`,
190
553
  enforced: true,
191
554
  blocked: true
192
555
  });
556
+ countDecision(metrics, { routeContext, mode, decision: "rate_limited" });
557
+ metrics.increment("haechi_rate_limited_total");
193
558
  return { denied: { status: 429, error: "haechi_rate_limited", message: "Rate limit exceeded" } };
194
559
  }
195
560
  }
@@ -208,13 +573,36 @@ function hasBearerHeader(request) {
208
573
  }
209
574
 
210
575
  function createRateLimiter() {
211
- // In-memory fixed-window counter. Per-process: resets on restart, not shared
212
- // across replicas acceptable for a single-process self-hosted preview.
576
+ // Backward-compat fallback ONLY: the canonical default lives in the runtime
577
+ // (createRuntime owns providers.rateLimiter). This path runs when a hand-built
578
+ // runtime object lacks rateLimiter. In-memory fixed-window counter, per-process
579
+ // (resets on restart, not shared across replicas). The window Map is bounded by
580
+ // a lazy, amortized sweep — NO timer — so aged-out one-shot identities do not
581
+ // accumulate unboundedly (mirrors runtime's createRateLimiter).
213
582
  const windows = new Map();
214
583
  const windowMs = 60000;
584
+ const sweepThreshold = 1024;
585
+ const sweepBudget = 256;
586
+
587
+ function sweepExpired(now) {
588
+ let scanned = 0;
589
+ for (const [key, slot] of windows) {
590
+ if (scanned >= sweepBudget) {
591
+ break;
592
+ }
593
+ scanned += 1;
594
+ if (now - slot.windowStart >= windowMs) {
595
+ windows.delete(key);
596
+ }
597
+ }
598
+ }
599
+
215
600
  return {
216
601
  allow(key, limit) {
217
602
  const now = Date.now();
603
+ if (windows.size >= sweepThreshold) {
604
+ sweepExpired(now);
605
+ }
218
606
  const slot = windows.get(key);
219
607
  if (!slot || now - slot.windowStart >= windowMs) {
220
608
  windows.set(key, { windowStart: now, count: 1 });
@@ -229,9 +617,9 @@ function createRateLimiter() {
229
617
  };
230
618
  }
231
619
 
232
- async function recordAuthDenied({ runtime, routeContext, reason }) {
620
+ async function recordAuthDenied({ runtime, routeContext, reason, correlationId = null }) {
233
621
  await recordProxyDecision({
234
- runtime, routeContext, identity: null, profile: null,
622
+ runtime, routeContext, identity: null, profile: null, correlationId,
235
623
  decision: "auth_denied",
236
624
  reason,
237
625
  enforced: true,
@@ -239,8 +627,9 @@ async function recordAuthDenied({ runtime, routeContext, reason }) {
239
627
  });
240
628
  }
241
629
 
242
- async function handleInspectedStream({ runtime, request, response, routeContext, json, authContext = {} }) {
630
+ async function handleInspectedStream({ runtime, request, response, routeContext, json, authContext = {}, metrics = null }) {
243
631
  const { haechi, config } = runtime;
632
+ const requestMode = config.policy.mode ?? config.mode;
244
633
 
245
634
  // Inspection needs to know the wire format and delta channel for this route.
246
635
  if (!routeContext.streaming) {
@@ -264,6 +653,8 @@ async function handleInspectedStream({ runtime, request, response, routeContext,
264
653
  : { payload: json, blocked: false };
265
654
 
266
655
  if (requestResult.blocked) {
656
+ countDecision(metrics, { routeContext, mode: requestMode, decision: "blocked" });
657
+ metrics?.increment("haechi_blocks_total");
267
658
  writeJson(response, 403, {
268
659
  error: "haechi_policy_block",
269
660
  summary: requestResult.summary,
@@ -276,7 +667,8 @@ async function handleInspectedStream({ runtime, request, response, routeContext,
276
667
  upstream: config.target.upstream,
277
668
  request,
278
669
  body: JSON.stringify(requestResult.payload),
279
- timeoutMs: config.limits.upstreamTimeoutMs
670
+ timeoutMs: config.limits.upstreamTimeoutMs,
671
+ metrics
280
672
  });
281
673
 
282
674
  const streamMode = config.streaming.responseMode ?? config.responseProtection.mode ?? config.policy.mode ?? config.mode;
@@ -300,8 +692,13 @@ async function handleInspectedStream({ runtime, request, response, routeContext,
300
692
 
301
693
  await recordStreamDecision({
302
694
  runtime, routeContext, blocked, summary, mode: streamMode,
303
- identity: authContext.identity ?? null, profile: authContext.profile ?? null
695
+ identity: authContext.identity ?? null, profile: authContext.profile ?? null,
696
+ correlationId: authContext.correlationId ?? null
304
697
  });
698
+ countDecision(metrics, { routeContext, mode: streamMode, decision: blocked ? "stream_blocked" : "stream_inspected" });
699
+ if (blocked) {
700
+ metrics?.increment("haechi_blocks_total");
701
+ }
305
702
  response.end();
306
703
  }
307
704
 
@@ -324,12 +721,13 @@ async function* emptyAsyncIterable() {
324
721
  // No upstream body to inspect.
325
722
  }
326
723
 
327
- async function recordStreamDecision({ runtime, routeContext, blocked, summary, mode, identity = null, profile = null }) {
724
+ async function recordStreamDecision({ runtime, routeContext, blocked, summary, mode, identity = null, profile = null, correlationId = null }) {
328
725
  if (typeof runtime.auditSink?.record !== "function") {
329
726
  return;
330
727
  }
331
728
  await runtime.auditSink.record({
332
729
  id: randomUUID(),
730
+ correlationId,
333
731
  timestamp: new Date().toISOString(),
334
732
  protocol: routeContext?.protocol ?? "proxy",
335
733
  operation: `response-stream:${routeContext?.operation ?? "unknown"}`,
@@ -346,7 +744,7 @@ async function recordStreamDecision({ runtime, routeContext, blocked, summary, m
346
744
  });
347
745
  }
348
746
 
349
- async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, authContext = {}, issuedTokens = [] }) {
747
+ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, authContext = {}, issuedTokens = [], metrics = null }) {
350
748
  const headers = Object.fromEntries(upstreamResponse.headers.entries());
351
749
 
352
750
  if (!runtime.config.responseProtection.enabled || !routeContext.protectResponse) {
@@ -354,7 +752,8 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
354
752
  return {
355
753
  status: upstreamResponse.status,
356
754
  headers,
357
- body: rawBody
755
+ body: rawBody,
756
+ decision: "forwarded"
358
757
  };
359
758
  }
360
759
 
@@ -372,6 +771,8 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
372
771
  responsePolicy,
373
772
  routeContext,
374
773
  runtime,
774
+ correlationId: authContext.correlationId ?? null,
775
+ metrics,
375
776
  hardDeny: true
376
777
  });
377
778
  }
@@ -388,6 +789,8 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
388
789
  responsePolicy,
389
790
  routeContext,
390
791
  runtime,
792
+ correlationId: authContext.correlationId ?? null,
793
+ metrics,
391
794
  hardDeny: true
392
795
  });
393
796
  }
@@ -401,7 +804,9 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
401
804
  rawBody,
402
805
  responsePolicy,
403
806
  routeContext,
404
- runtime
807
+ runtime,
808
+ correlationId: authContext.correlationId ?? null,
809
+ metrics
405
810
  });
406
811
  }
407
812
 
@@ -414,7 +819,9 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
414
819
  rawBody,
415
820
  responsePolicy,
416
821
  routeContext,
417
- runtime
822
+ runtime,
823
+ correlationId: authContext.correlationId ?? null,
824
+ metrics
418
825
  });
419
826
  }
420
827
 
@@ -430,7 +837,9 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
430
837
  rawBody,
431
838
  responsePolicy,
432
839
  routeContext,
433
- runtime
840
+ runtime,
841
+ correlationId: authContext.correlationId ?? null,
842
+ metrics
434
843
  });
435
844
  }
436
845
 
@@ -446,7 +855,9 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
446
855
  });
447
856
 
448
857
  if (result.blocked) {
858
+ metrics?.increment("haechi_blocks_total");
449
859
  return {
860
+ decision: "response_blocked",
450
861
  status: 502,
451
862
  headers: { "content-type": "application/json" },
452
863
  body: Buffer.from(`${JSON.stringify({
@@ -474,6 +885,7 @@ async function maybeProtectResponse({ upstreamResponse, routeContext, runtime, a
474
885
  }
475
886
 
476
887
  return {
888
+ decision: "forwarded",
477
889
  status: upstreamResponse.status,
478
890
  headers: transformedJsonHeaders(headers),
479
891
  body: Buffer.from(`${JSON.stringify(responsePayload)}\n`)
@@ -498,7 +910,7 @@ function restoreTokens(value, tokenValues) {
498
910
  return value;
499
911
  }
500
912
 
501
- async function forward({ upstream, request, body, timeoutMs = null }) {
913
+ async function forward({ upstream, request, body, timeoutMs = null, metrics = null }) {
502
914
  const target = buildUpstreamUrl({ upstream, requestUrl: request.url });
503
915
  try {
504
916
  return await fetch(target, {
@@ -509,12 +921,14 @@ async function forward({ upstream, request, body, timeoutMs = null }) {
509
921
  });
510
922
  } catch (error) {
511
923
  if (error?.name === "TimeoutError" || error?.name === "AbortError") {
924
+ metrics?.increment("haechi_upstream_timeout_total");
512
925
  throw proxyError({
513
926
  statusCode: 504,
514
927
  errorCode: "haechi_upstream_timeout",
515
928
  message: `Upstream did not respond within limits.upstreamTimeoutMs (${timeoutMs})`
516
929
  });
517
930
  }
931
+ metrics?.increment("haechi_upstream_error_total");
518
932
  throw proxyError({
519
933
  statusCode: 502,
520
934
  errorCode: "haechi_upstream_unreachable",
@@ -637,27 +1051,39 @@ async function unprotectedResponseDecision({
637
1051
  responsePolicy,
638
1052
  routeContext,
639
1053
  runtime,
1054
+ metrics = null,
1055
+ correlationId = null,
640
1056
  hardDeny = false
641
1057
  }) {
642
1058
  const allowed = responsePolicy.failureMode === "allow" && !hardDeny;
1059
+ const decision = allowed ? "response_unprotected_allowed" : "response_unprotected_blocked";
643
1060
  await recordProxyDecision({
644
1061
  runtime,
645
1062
  routeContext,
646
- decision: allowed ? "response_unprotected_allowed" : "response_unprotected_blocked",
1063
+ correlationId,
1064
+ decision,
647
1065
  reason,
648
1066
  enforced: !allowed,
649
1067
  blocked: !allowed
650
1068
  });
1069
+ // A forwarded-without-protection (or blocked-because-unprotectable) response is
1070
+ // an operability signal. The label is the bounded reason enum, never a value.
1071
+ metrics?.increment("haechi_response_unprotected_total");
651
1072
 
652
1073
  if (allowed) {
653
1074
  return {
1075
+ decision,
654
1076
  status: upstreamResponse.status,
655
1077
  headers,
656
1078
  body: rawBody
657
1079
  };
658
1080
  }
659
1081
 
1082
+ if (!hardDeny) {
1083
+ metrics?.increment("haechi_blocks_total");
1084
+ }
660
1085
  return {
1086
+ decision,
661
1087
  status: 502,
662
1088
  headers: { "content-type": "application/json" },
663
1089
  body: Buffer.from(`${JSON.stringify({
@@ -738,13 +1164,16 @@ async function cancelReader(reader) {
738
1164
  }
739
1165
  }
740
1166
 
741
- async function recordProxyDecision({ runtime, routeContext, decision, reason, enforced, blocked, identity = null, profile = null }) {
1167
+ async function recordProxyDecision({ runtime, routeContext, decision, reason, enforced, blocked, identity = null, profile = null, correlationId = null }) {
742
1168
  if (typeof runtime.auditSink?.record !== "function") {
743
1169
  return;
744
1170
  }
745
1171
 
746
1172
  await runtime.auditSink.record({
747
1173
  id: randomUUID(),
1174
+ // Per-request correlation id so a proxy-decision event shares the id of the
1175
+ // protect events of the same request. A UUID — never a payload/PII value.
1176
+ correlationId,
748
1177
  timestamp: new Date().toISOString(),
749
1178
  protocol: routeContext?.protocol ?? "proxy",
750
1179
  operation: routeContext ? `proxy:${routeContext.protocol}:${routeContext.routeId ?? "unknown"}` : "proxy",
@@ -804,6 +1233,27 @@ function shortHash(value) {
804
1233
  return createHash("sha256").update(value).digest("hex").slice(0, 12);
805
1234
  }
806
1235
 
1236
+ // X-Forwarded-Proto enforcement helper. Node lowercases header names; a comma-
1237
+ // joined multi-value header (e.g. "https, http" from a chain of proxies) is
1238
+ // trusted only when the FIRST hop — the one closest to the client, set by the
1239
+ // trusted terminator — is https. Any other value (http, missing, malformed)
1240
+ // fails closed.
1241
+ function isForwardedHttps(request) {
1242
+ const raw = request?.headers?.["x-forwarded-proto"];
1243
+ if (typeof raw !== "string" || raw.length === 0) {
1244
+ return false;
1245
+ }
1246
+ const first = raw.split(",")[0].trim().toLowerCase();
1247
+ return first === "https";
1248
+ }
1249
+
1250
+ // Loopback / remote-bind gate. Loopback always binds (plain http for dev). A
1251
+ // non-loopback bind is refused UNLESS allowRemoteBind is set. This is the SHARED
1252
+ // primitive the haechi-dashboard satellite reuses, so its contract is preserved
1253
+ // unchanged: { host, allowRemoteBind } and nothing more. The WS6 TLS fail-closed
1254
+ // rule is layered ON TOP of this in assertSafeProxyTransport (the proxy's own
1255
+ // requirement that a remote bind carry TLS), mirroring how the dashboard layers
1256
+ // its own tlsContext precedence after calling assertSafeProxyBind.
807
1257
  export function assertSafeProxyBind({ host = "127.0.0.1", allowRemoteBind = false } = {}) {
808
1258
  if (allowRemoteBind || isLoopbackHost(host)) {
809
1259
  return;
@@ -812,6 +1262,35 @@ export function assertSafeProxyBind({ host = "127.0.0.1", allowRemoteBind = fals
812
1262
  throw new Error(`Refusing to bind Haechi proxy to non-loopback host ${host}. Use --allow-remote-bind only for explicitly secured environments.`);
813
1263
  }
814
1264
 
1265
+ // WS6 fail-closed TLS requirement for a REMOTE bind. After the loopback/remote
1266
+ // gate above, a non-loopback (allowRemoteBind) listener must ALSO carry usable
1267
+ // TLS material (Haechi terminates TLS) OR an explicit trustForwardedProto
1268
+ // acknowledgement (a trusted reverse proxy terminates TLS in front of Haechi).
1269
+ // Neither → THROW: never serve bearer tokens + payloads in plaintext on a remote
1270
+ // bind. Loopback dev is exempt (plain http, no TLS needed). Separate from
1271
+ // assertSafeProxyBind so the dashboard satellite's reuse of that primitive is
1272
+ // unaffected.
1273
+ export function assertSafeProxyTransport({
1274
+ host = "127.0.0.1",
1275
+ allowRemoteBind = false,
1276
+ hasUsableTls = false,
1277
+ trustForwardedProto = false
1278
+ } = {}) {
1279
+ if (isLoopbackHost(host) || !allowRemoteBind) {
1280
+ // Loopback (or an already-refused remote bind) needs no TLS check here.
1281
+ return;
1282
+ }
1283
+ if (!hasUsableTls && !trustForwardedProto) {
1284
+ throw new Error(
1285
+ `Refusing to bind Haechi proxy to non-loopback host ${host} without TLS. ` +
1286
+ `A remote bind would expose bearer tokens and payloads in plaintext. ` +
1287
+ `Set proxy.tls (a keyFile+certFile or pfxFile so Haechi terminates TLS), ` +
1288
+ `or set proxy.trustForwardedProto: true only when a trusted reverse proxy ` +
1289
+ `terminates TLS in front of Haechi (Haechi will then require X-Forwarded-Proto: https).`
1290
+ );
1291
+ }
1292
+ }
1293
+
815
1294
  function isLoopbackHost(host) {
816
1295
  const normalized = String(host).trim().toLowerCase();
817
1296
  return normalized === "localhost"