haechi 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +97 -97
- package/README.md +2 -2
- package/SECURITY.md +19 -11
- package/docs/README.md +2 -0
- package/docs/current/api-stability.ko.md +26 -26
- package/docs/current/compliance-mapping.ko.md +53 -0
- package/docs/current/compliance-mapping.md +53 -0
- package/docs/current/config-version.ko.md +30 -0
- package/docs/current/config-version.md +51 -0
- package/docs/current/configuration.ko.md +242 -102
- package/docs/current/configuration.md +149 -9
- package/docs/current/operations-runbook.ko.md +121 -0
- package/docs/current/operations-runbook.md +204 -0
- package/docs/current/release-process.ko.md +19 -20
- package/docs/current/release-process.md +1 -2
- package/docs/current/reliability-hardening-track.ko.md +77 -0
- package/docs/current/reliability-hardening-track.md +77 -0
- package/docs/current/risk-register-release-gate.ko.md +26 -27
- package/docs/current/risk-register-release-gate.md +27 -20
- package/docs/current/security-whitepaper.ko.md +102 -0
- package/docs/current/security-whitepaper.md +102 -0
- package/docs/current/shared-responsibility.ko.md +33 -24
- package/docs/current/shared-responsibility.md +12 -3
- package/docs/current/threat-model.ko.md +12 -12
- package/docs/current/threat-model.md +3 -3
- package/haechi.config.example.json +19 -3
- package/package.json +6 -2
- package/packages/audit/index.mjs +26 -2
- package/packages/cli/bin/haechi.mjs +54 -8
- package/packages/cli/runtime.mjs +398 -10
- package/packages/core/index.mjs +189 -15
- package/packages/filter/index.mjs +299 -9
- package/packages/metrics/index.mjs +181 -0
- package/packages/proxy/index.mjs +535 -41
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# Haechi Configuration Reference
|
|
2
2
|
|
|
3
|
-
- Status: Living document
|
|
4
|
-
- Target version: 0.6.0
|
|
3
|
+
- Status: Living document (tracks core 1.2.x)
|
|
5
4
|
|
|
6
5
|
`haechi init` writes `haechi.config.json`; a non-secret template is at `haechi.config.example.json`. Every command reads it with `--config <path>` (default `haechi.config.json`). Configuration is **validated fail-closed**: unknown providers, out-of-range numbers, and malformed values throw at load time rather than degrading silently. `haechi config` prints this reference; `haechi status` prints the *effective* state of a given config.
|
|
7
6
|
|
|
@@ -9,18 +8,21 @@
|
|
|
9
8
|
|
|
10
9
|
```json
|
|
11
10
|
{
|
|
11
|
+
"configVersion": 1,
|
|
12
12
|
"mode": "dry-run",
|
|
13
13
|
"target": { "type": "llm-http", "adapter": "openai-compatible", "upstream": "http://127.0.0.1:9999" },
|
|
14
|
-
"proxy": { "host": "127.0.0.1", "port": 11016 },
|
|
14
|
+
"proxy": { "host": "127.0.0.1", "port": 11016, "tls": null, "trustForwardedProto": false },
|
|
15
15
|
"responseProtection": { "enabled": false, "mode": "enforce", "failureMode": "fail-closed", "allowNonJson": false, "allowCompressed": false, "maxBytes": 1048576 },
|
|
16
16
|
"streaming": { "requestMode": "block" },
|
|
17
|
-
"limits": { "maxRequestBytes": 1048576, "upstreamTimeoutMs": 120000 },
|
|
17
|
+
"limits": { "maxRequestBytes": 1048576, "upstreamTimeoutMs": 120000, "maxNestingDepth": 256, "maxInFlight": 0, "shutdownGraceMs": 10000, "requestTimeoutMs": null, "headersTimeoutMs": null },
|
|
18
18
|
"policy": { "mode": "dry-run", "presets": ["korean-pii", "secrets-only", "llm-redact"], "defaultAction": "redact", "actions": { "card": "block" } },
|
|
19
19
|
"filters": { "customRules": [] },
|
|
20
20
|
"keys": { "provider": "local", "keyFile": ".haechi/dev.keys.json" },
|
|
21
21
|
"audit": { "sink": "jsonl", "path": ".haechi/audit.jsonl" },
|
|
22
22
|
"tokenVault": { "provider": "local", "path": ".haechi/token-vault.json", "revealPolicy": "disabled", "retentionDays": 30, "deterministic": false, "deterministicTypes": null, "detokenizeResponses": false },
|
|
23
23
|
"privacy": { "profile": null },
|
|
24
|
+
"logging": { "format": "text" },
|
|
25
|
+
"metrics": { "enabled": true },
|
|
24
26
|
"mcp": { "allowedMethods": ["initialize", "tools/call", "resources/read", "prompts/get"], "protectParams": true, "protectResults": true, "requireJsonRpc": true }
|
|
25
27
|
}
|
|
26
28
|
```
|
|
@@ -29,6 +31,7 @@
|
|
|
29
31
|
|
|
30
32
|
| Key | Type / values | Default | Notes |
|
|
31
33
|
|---|---|---|---|
|
|
34
|
+
| `configVersion` | positive integer | `1` | Config schema version stamp. Absent = treated as the current version. A value **newer** than this build understands **fails closed** at load; a non-positive/non-integer value throws. See [`config-version.md`](./config-version.md). |
|
|
32
35
|
| `mode` | `dry-run` \| `report-only` \| `enforce` | `dry-run` | Global enforcement mode. `dry-run`/`report-only` detect + audit only; `enforce` transforms/blocks. Overridden by `policy.mode` when set. |
|
|
33
36
|
|
|
34
37
|
## `target`
|
|
@@ -45,6 +48,8 @@
|
|
|
45
48
|
|---|---|---|---|
|
|
46
49
|
| `proxy.host` | non-empty string | `127.0.0.1` | Bind address. Non-loopback hosts require the `--allow-remote-bind` CLI flag — config alone will not start (see [Binding beyond loopback](#binding-beyond-loopback)). |
|
|
47
50
|
| `proxy.port` | integer 0–65535 | `11016` | Listen port (`0` = ephemeral). Override per-run with `--port`. |
|
|
51
|
+
| `proxy.tls` | `null` or `{ keyFile, certFile }` / `{ pfxFile, passphrase? }` | `null` | TLS material loaded from **file paths** at startup into a TLS context. When present, Haechi terminates TLS itself (serves `https`). Required (or `trustForwardedProto`) for a remote bind — see [Binding beyond loopback](#binding-beyond-loopback). Fail-closed: a non-null value that does not resolve to usable material `((key && cert) or pfx)`, mixes `pfxFile` with `keyFile`/`certFile`, or names an unreadable file throws at load. |
|
|
52
|
+
| `proxy.trustForwardedProto` | boolean | `false` | Operator acknowledgement that a **trusted reverse proxy terminates TLS** in front of Haechi. When `true`, a remote bind may stay plain `http`, but Haechi then **refuses any request whose `X-Forwarded-Proto` is not `https`** (checked before auth/body; the `/__haechi/*` liveness routes are exempt). Never a substitute for real TLS when Haechi is itself internet-facing. |
|
|
48
53
|
|
|
49
54
|
## `responseProtection`
|
|
50
55
|
|
|
@@ -74,6 +79,11 @@ Inspects upstream JSON responses (off by default — turn on to protect what com
|
|
|
74
79
|
|---|---|---|---|
|
|
75
80
|
| `limits.maxRequestBytes` | positive integer | `1048576` | Request body cap; over the limit returns `413`. Enforced incrementally (the body is not fully buffered first). |
|
|
76
81
|
| `limits.upstreamTimeoutMs` | positive integer | `120000` | Upstream request timeout; on expiry returns `504 haechi_upstream_timeout`. Connection failure returns `502 haechi_upstream_unreachable`. |
|
|
82
|
+
| `limits.maxNestingDepth` | positive integer | `256` | Max JSON nesting depth walked during detection. A more deeply nested body is rejected `413 haechi_request_too_deeply_nested` (fail-closed, before upstream), guarding the recursive payload walk against a stack overflow. Bounds container descent; leaves at the limit are still inspected. (Separately, a non-UTF-8 request body is rejected fail-closed: `400 haechi_request_body_not_utf8`.) |
|
|
83
|
+
| `limits.maxInFlight` | non-negative integer | `0` | Global max-in-flight backpressure ceiling. `0` disables it (no ceiling — 1.1 behavior). When `> 0` and the live in-flight count is at the ceiling, a **new** request is rejected `503` with a `Retry-After` header and `{ "error": "haechi_overloaded" }`, **before** auth/body-read. The `/__haechi/*` observability routes are **exempt** (liveness + metrics stay scrapable under saturation). Each rejection increments `haechi_overloaded_total`. See the [operations runbook](./operations-runbook.md#5-backpressure-tuning). |
|
|
84
|
+
| `limits.shutdownGraceMs` | non-negative integer (ms) | `10000` | Graceful-shutdown grace period. On `SIGINT`/`SIGTERM` the proxy stops accepting connections, closes idle keep-alive sockets immediately, waits for in-flight requests to drain, then after this grace force-closes any lingering socket so a stuck keep-alive cannot hold shutdown open forever. Also seeds the backpressure `Retry-After` seconds. Set your orchestrator's termination grace **above** this value. |
|
|
85
|
+
| `limits.requestTimeoutMs` | `null` \| non-negative integer (ms) | `null` | Maps to the Node HTTP server `requestTimeout`. `null` leaves Node's default untouched (behavior unchanged). Set a number to cap slow whole-request delivery; `0` disables the timeout (Node semantics). |
|
|
86
|
+
| `limits.headersTimeoutMs` | `null` \| non-negative integer (ms) | `null` | Maps to the Node HTTP server `headersTimeout`. `null` leaves Node's default untouched. Set a number to cap slow header delivery (slow-loris); `0` disables it. |
|
|
77
87
|
|
|
78
88
|
## `policy`
|
|
79
89
|
|
|
@@ -93,6 +103,19 @@ The detect→decide core. See [Detection types & actions](#detection-types--acti
|
|
|
93
103
|
| Key | Type / values | Default | Notes |
|
|
94
104
|
|---|---|---|---|
|
|
95
105
|
| `filters.customRules` | array of rule objects | `[]` | Extra detection rules: `{ id, type, pattern, flags?, confidence? }`. Patterns are ReDoS-screened (≤500 chars, no nested quantifiers, no backreferences) and rejected at load if unsafe. |
|
|
106
|
+
| `filters.minConfidence` | number in `[0, 1]` | `0` | Precision dial. Each rule carries a `confidence` (0.6–0.95); a detection whose confidence is **below** this threshold is dropped before the policy decides. `0` (the default) gates nothing, preserving prior behavior. **Hard-block exemption:** a hard-block type (`secret`, `api_key`, `kr_rrn`, `card`) is **never** dropped on confidence — `minConfidence` trims only the precision-risky soft types (e.g. `phone`, `email`, `injection`), so a low-confidence credential/PII leak is still acted on (fail-closed). |
|
|
107
|
+
| `filters.allowlist` | array of strings and/or `{ value?, path? }` | `[]` | Operator false-positive exceptions. A detection whose matched **value** equals a string/`value` entry, or whose PII-safe JSON **path** (the hashed `pathText`, as shown in the audit) equals a `path` entry, is suppressed before the policy decides (when an entry sets both `value` and `path`, **both** must match). **Hard-block exemption:** an entry that would suppress a hard-block type (`secret`/`api_key`/`kr_rrn`/`card`) is **ignored** and the detection still fires — the allowlist can only clear a benign **soft-type** FP, never silence a credential/PII leak. Every suppression and every `minConfidence` drop is **audited** by count and type (`summary.suppressedByType` / `summary.droppedByType` / `suppressedCount` / `droppedCount`) — never the raw value. Use this to clear one benign FP without deleting a whole rule. |
|
|
108
|
+
|
|
109
|
+
### Detection benchmark
|
|
110
|
+
|
|
111
|
+
Detection precision/recall is measured, not assumed. A labeled corpus of synthetic test fixtures (`tests/fixtures/detection-corpus.json` — positive samples per type plus benign hard-negatives) drives a per-type scorer:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
npm run bench:detection # print the per-type TP/FP/FN + precision/recall table
|
|
115
|
+
npm run scan:detection # CI regression gate: fail if any type regresses below baseline
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`bench:detection` (`scripts/bench-detection.mjs`) runs the default filter engine over each corpus case and reports true/false positives and false negatives per type. `scan:detection` compares the live scores against the pinned baseline (`scripts/detection-baseline.json`) and **fails only on a regression** — a precision or recall drop below the recorded numbers. The baseline deliberately bakes in the current imperfect state (the audit-reproduced false positives on `phone`/`card`/`secret`, and the known coverage-gap misses for AWS/GitHub/Google/Slack keys, JWT, and PEM headers), so the gate passes today and trips only when a change makes detection worse. It runs in `release:preflight` after the doc-freshness gate. Regenerate the baseline after an intentional rule change with `node scripts/bench-detection.mjs --write-baseline` and review the diff. Closing the recorded gaps and false positives is WS2b/WS2c of the reliability-hardening track.
|
|
96
119
|
|
|
97
120
|
## `keys`
|
|
98
121
|
|
|
@@ -126,6 +149,69 @@ The detect→decide core. See [Detection types & actions](#detection-types--acti
|
|
|
126
149
|
|---|---|---|---|
|
|
127
150
|
| `privacy.profile` | `null` \| `kr-pipa` \| `eu-gdpr` \| `us-general` | `null` | Applies a regional baseline action set before enforcement. Profiles may **strengthen** but never weaken your explicit actions. Engineering defaults, not legal advice. |
|
|
128
151
|
|
|
152
|
+
## `logging`
|
|
153
|
+
|
|
154
|
+
| Key | Type / values | Default | Notes |
|
|
155
|
+
|---|---|---|---|
|
|
156
|
+
| `logging.format` | `text` \| `json` | `text` | `text` keeps the human-readable startup/shutdown/error lines (unchanged). `json` emits one single-line JSON object per event. Fail-closed: any other value throws. |
|
|
157
|
+
|
|
158
|
+
In `json` mode the proxy's internal-error log is a single line `{ "level": "error", "event": "proxy_internal_error", "correlationId", "errorName", "statusCode" }`, and startup/shutdown emit `proxy_listening` / `proxy_shutdown` (plus `*_warn` events for remote-bind / non-enforce-mode / response-protection-disabled). **No log field ever carries a request/response payload, header, token, or any PII** — error logs carry the error *class name* and the request `correlationId` only.
|
|
159
|
+
|
|
160
|
+
## `metrics`
|
|
161
|
+
|
|
162
|
+
| Key | Type / values | Default | Notes |
|
|
163
|
+
|---|---|---|---|
|
|
164
|
+
| `metrics.enabled` | boolean | `true` | Gates the `GET /__haechi/metrics` route. When `false`, that route returns `404`. Fail-closed: a non-boolean throws. |
|
|
165
|
+
|
|
166
|
+
The metrics collector is also an **injectable collaborator** (`createRuntime(config, { metrics })`); see [Operability endpoints](#operability-endpoints) for the contract and the no-PII guarantee.
|
|
167
|
+
|
|
168
|
+
## Operability endpoints
|
|
169
|
+
|
|
170
|
+
The proxy serves four unauthenticated endpoints under the reserved `/__haechi/*` prefix, checked **before** auth and body-read. They never proxy upstream.
|
|
171
|
+
|
|
172
|
+
| Endpoint | Status | Body | Purpose |
|
|
173
|
+
|---|---|---|---|
|
|
174
|
+
| `GET /__haechi/live` | `200` | `{ ok: true, version }` | Cheap process liveness. |
|
|
175
|
+
| `GET /__haechi/ready` | `200` / `503` | `{ ready, version, checks }` | Readiness. **Fail-closed**: a gateway that cannot append to its audit log is **not** ready (`503`). The default JSONL sink's `checks.auditWritable` confirms its audit directory/file is writable without writing an event; a sink lacking a `ready()`/`healthCheck()` method is treated as ready. |
|
|
176
|
+
| `GET /__haechi/health` | `200` | `{ ok: true, mode, version }` | Back-compat (the original health endpoint, now with `version`). |
|
|
177
|
+
| `GET /__haechi/metrics` | `200` / `404` | Prometheus text | Telemetry (see below). `404` when `metrics.enabled: false`. |
|
|
178
|
+
|
|
179
|
+
`version` is the running package version (`package.json`).
|
|
180
|
+
|
|
181
|
+
### Telemetry (`/__haechi/metrics`)
|
|
182
|
+
|
|
183
|
+
The endpoint renders the **Prometheus text exposition format** (`# HELP` / `# TYPE` + `name{label="..."} value`), `Content-Type: text/plain`. Counters: `haechi_requests_total{route,mode,decision}` plus `haechi_blocks_total`, `haechi_auth_denied_total`, `haechi_rate_limited_total`, `haechi_upstream_timeout_total`, `haechi_upstream_error_total`, `haechi_response_unprotected_total`, `haechi_internal_error_total`; one histogram `haechi_request_duration_seconds{route}`.
|
|
184
|
+
|
|
185
|
+
**No-PII-in-telemetry invariant.** Every metric name and **every label value** is a bounded enum — a route id, a policy mode, or a fixed decision class (`forwarded` / `blocked` / `auth_denied` / `rate_limited` / `model_not_allowed` / …). A metric label is **never** an identity id/subject, a token, or a detected value: there is no per-identity or per-value label cardinality. This is the no-plaintext-in-audit invariant extended to telemetry; the metrics module additionally length-caps and charset-sanitizes label values as defence in depth.
|
|
186
|
+
|
|
187
|
+
### `providers.metrics` injection seam
|
|
188
|
+
|
|
189
|
+
The metrics collector is supplied programmatically through `createRuntime(config, providers)` — the same seam as `cryptoProvider`/`authProvider`/`rateLimiter`. It is **not** a JSON config key.
|
|
190
|
+
|
|
191
|
+
```js
|
|
192
|
+
const runtime = createRuntime(config, { metrics });
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
An injected `metrics` must implement `increment(name, labels?, amount?)`, `observe(name, value, labels?)`, and `render() -> string`; `createRuntime` fails closed at construction if it does not. The **default** is a zero-dependency in-memory collector that renders the Prometheus text above. A multi-replica operator injects a shared/remote collector satisfying the same contract.
|
|
196
|
+
|
|
197
|
+
### `correlationId` (audit + logs)
|
|
198
|
+
|
|
199
|
+
The proxy generates a per-**request** `correlationId` (a UUID). It is threaded into the protect context, so each request's request- and response-direction audit events carry the same additive top-level `correlationId` field, and into the proxy's internal-error log line — letting an operator join a logged error to its audit trail. It is `null` for non-proxy `protectJson()` calls (preserving prior behavior). The id is a UUID and is **never** a payload/identity/PII value.
|
|
200
|
+
|
|
201
|
+
## Env-var configuration overlay (deploy)
|
|
202
|
+
|
|
203
|
+
For container / 12-factor deploys, a **fixed allowlist of NON-SECRET operational keys** can be overridden from the environment. The env value **wins over the config file** and is validated **fail-closed** — an invalid value makes the process fail to start. Applied in `loadConfig()` after reading the file and before validation.
|
|
204
|
+
|
|
205
|
+
| Env var | Config key | Type / values |
|
|
206
|
+
|---|---|---|
|
|
207
|
+
| `HAECHI_PROXY_PORT` | `proxy.port` | integer 0–65535 |
|
|
208
|
+
| `HAECHI_PROXY_HOST` | `proxy.host` | non-empty string |
|
|
209
|
+
| `HAECHI_UPSTREAM` | `target.upstream` | URL string |
|
|
210
|
+
| `HAECHI_MODE` | `mode` | `dry-run` \| `report-only` \| `enforce` |
|
|
211
|
+
| `HAECHI_LOG_FORMAT` | `logging.format` | `text` \| `json` |
|
|
212
|
+
|
|
213
|
+
**Secrets are NOT overlayable — by design.** There is **no** `HAECHI_*` variable for `keys.*`, the auth token store, or any token/secret. Secrets stay in the config file or are supplied via injected providers (`createRuntime(config, { cryptoProvider, authProvider, … })`). Putting a secret in a process environment risks leaking it through `/proc`, crash dumps, and orchestrator inspect output, so the overlay allowlist excludes them. See the [operations runbook](./operations-runbook.md#2-configuration-via-the-env-var-overlay).
|
|
214
|
+
|
|
129
215
|
## `mcp`
|
|
130
216
|
|
|
131
217
|
Applies to `mcp-stdio` and `mcp-wrap`.
|
|
@@ -173,15 +259,54 @@ Per-client controls layered on top of the base `policy`. See [Named profiles](#n
|
|
|
173
259
|
| `policy.profiles` | `{ <name>: { presets?, actions?, modelAllowlist?, rate? } }` | `{}` | Named profiles; each overrides the base policy. |
|
|
174
260
|
| `policy.profileBinding` | `{ byScope?, byLabel?, default }` | unset | Maps identity scopes/labels (`"k=v"` for labels) to profile names. `default` is **required** when `profiles` is set and should be the strictest profile (fail-closed). |
|
|
175
261
|
| `policy.modelAllowlist` | string array | unset | Allowed `model` values (base level; also settable per profile). A disallowed model → `403`. Empty/absent = allow all. |
|
|
176
|
-
| `policy.rate` | `{ requestsPerMinute }` | unset | Per-identity request rate limit (base level or per profile). Over the limit → `429`. In-memory, per-process. |
|
|
262
|
+
| `policy.rate` | `{ requestsPerMinute }` | unset | Per-identity request rate limit (base level or per profile). Over the limit → `429`. In-memory, per-process; see [Rate limiter injection](#rate-limiter-injection) for the multi-replica seam. |
|
|
177
263
|
|
|
178
264
|
### Named profiles
|
|
179
265
|
|
|
180
266
|
When an identity authenticates, its profile resolves in order **scope → label → `default`**; scope precedes label and the first match wins. Without `profiles`, or under `auth.provider: none`, the base policy applies. The resolved profile's policy engine, `modelAllowlist`, and `rate` govern that request.
|
|
181
267
|
|
|
268
|
+
### Rate limiter injection
|
|
269
|
+
|
|
270
|
+
The rate limiter is an **injectable collaborator**, supplied programmatically through the `providers` argument of `createRuntime(config, providers)` — the same seam as the external `cryptoProvider`/`authProvider`. It is **not** a JSON config key.
|
|
271
|
+
|
|
272
|
+
```js
|
|
273
|
+
const runtime = createRuntime(config, { rateLimiter });
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
An injected `rateLimiter` must implement `allow(key, limit) -> boolean` (where `key` is the per-identity bucket and `limit` is the resolved `requestsPerMinute`); `createRuntime` fails closed at construction if it does not. The proxy consults `runtime.rateLimiter` for every rate-governed request.
|
|
277
|
+
|
|
278
|
+
The **default** is a per-process, in-memory fixed-window counter: it resets on restart and is **not shared across replicas**, so total throughput multiplies by the replica count behind a load balancer. Its window map is self-bounding (a lazy, amortized sweep evicts aged-out one-shot identities — no background timer). For a multi-replica deployment, enforce a per-identity limit at a shared front door **or** inject a shared-store implementation (e.g. Redis-backed) that satisfies the same `allow(key, limit)` contract. See [Shared responsibility §4](./shared-responsibility.md#4-horizontal-scale--multiple-replicas).
|
|
279
|
+
|
|
182
280
|
## Detection types & actions
|
|
183
281
|
|
|
184
|
-
Built-in detection `type` values: `email`, `phone`, `kr_rrn`, `card`, `api_key`, `secret`, and `injection` (response-direction heuristic, report-only by default). Custom rules may introduce new types.
|
|
282
|
+
Built-in detection `type` values: `email`, `phone`, `kr_rrn`, `card`, `api_key`, `secret`, `us_ssn`, `iban`, and `injection` (response-direction heuristic, report-only by default). Custom rules may introduce new types.
|
|
283
|
+
|
|
284
|
+
### Supported credential & PII matrix
|
|
285
|
+
|
|
286
|
+
Detection is regex + optional validator (no ML). Every rule is **anchored tightly** to keep precision high; precision is prioritized over recall, and the corpus (`tests/fixtures/detection-corpus.json`) carries a hard-negative for each rule. The KR phone rule and the US SSN/IBAN validators reject look-alike ids/timestamps.
|
|
287
|
+
|
|
288
|
+
| Type | Detects | Anchor / validator | Notes |
|
|
289
|
+
|---|---|---|---|
|
|
290
|
+
| `email` | RFC-style addresses | local + domain + TLD | — |
|
|
291
|
+
| `phone` | KR mobile (`01[016789]`, `+82`) | bare separator-less runs must be `0`-led | KR landlines out of scope. |
|
|
292
|
+
| `phone` | E.164 international | **leading `+` required** (`+[1-9]` + 6–14 digits) | A bare digit run is never matched (collides with ids/timestamps). |
|
|
293
|
+
| `phone` | US/NANP national | **separators required** (`(NXX) NXX-XXXX` or `NXX-NXX-XXXX`) | A separator-less 10-digit run is not matched. |
|
|
294
|
+
| `kr_rrn` | KR resident registration number | check-digit validator | Shape-valid but checksum-invalid → rejected. |
|
|
295
|
+
| `card` | Payment card (PAN) | Luhn validator, 13–19 digits | — |
|
|
296
|
+
| `us_ssn` | US Social Security Number | `AAA-GG-SSSS` + SSA-range validator (rejects area `000`/`666`/`900-999`, group `00`, serial `0000`) | Separators required; a bare 9-digit id is not an SSN. |
|
|
297
|
+
| `iban` | International Bank Account Number | **mod-97 checksum** validator | The checksum is the precision guard — IBAN-shaped non-97-valid strings are rejected. |
|
|
298
|
+
| `api_key` | OpenAI-style (`sk_`/`rk_`/`pk_`) | prefix + ≥24 chars | — |
|
|
299
|
+
| `api_key` | AWS access key id | `AKIA`/`ASIA` + exactly 16 uppercase-alnum | — |
|
|
300
|
+
| `api_key` | Google API key | `AIza` + 35 URL-safe chars | — |
|
|
301
|
+
| `secret` | `Bearer <token>` | `Bearer` + ≥16 chars | — |
|
|
302
|
+
| `secret` | Assignment `<key> = <value>` | key vocabulary: `api_key`, `api_secret`, `secret`, `secret_key`, `aws_secret_access_key`, `client_secret`, `private_key`, `access_token`, `refresh_token`, `token`, `password` | Catches bare-base64 secrets (e.g. AWS secret access key) via the assignment form. |
|
|
303
|
+
| `secret` | GitHub token | `gh[pousr]_` + ≥36 base64-ish chars | pat/oauth/user/server/refresh variants. |
|
|
304
|
+
| `secret` | Slack token | `xox[baprs]-` + ≥10-char body | bot/user/refresh/legacy variants. |
|
|
305
|
+
| `secret` | JWT | three base64url segments, first starts `eyJ` (the base64 of `{"`) | The `eyJ` anchor rejects arbitrary dotted tokens. |
|
|
306
|
+
| `secret` | PEM private key | `-----BEGIN … PRIVATE KEY-----` armor header | The header presence is the signal; prose mentioning "private key" is not matched. |
|
|
307
|
+
| `injection` | prompt-injection heuristics | response-direction only, `allow` by default | See [Action strength](#action-strength); report-only. |
|
|
308
|
+
|
|
309
|
+
Detection covers string values, JSON number leaves (request direction), and object keys. Each **string leaf is NFKC-normalized before matching**, so Unicode-evasion forms (full-width digits `4242…`, full-width `@`, mathematical/enclosed alphanumerics) are folded to their ASCII compatibility form and still detected. When the fold preserves UTF-16 length the exact evaded span is redacted/blocked; when it changes length (e.g. mathematical digits, ligatures) detection fails closed and the whole leaf is redacted/blocked. Base64/percent-encoded values (after decoding) and URL query strings remain documented exclusions (see `docs/current/threat-model.md`). On the response direction, Haechi's own transform markers and bare JSON number leaves are skipped (request direction is always full-scan).
|
|
185
310
|
|
|
186
311
|
Actions (weakest → strongest):
|
|
187
312
|
|
|
@@ -240,17 +365,32 @@ When a preset and an override (or a privacy profile) disagree, the **stronger**
|
|
|
240
365
|
|
|
241
366
|
## Binding beyond loopback
|
|
242
367
|
|
|
243
|
-
The proxy refuses non-loopback hosts unless the CLI flag is passed explicitly — `proxy.host: "0.0.0.0"` in config alone will not start, by design
|
|
368
|
+
The proxy refuses non-loopback hosts unless the CLI flag is passed explicitly — `proxy.host: "0.0.0.0"` in config alone will not start, by design. A remote bind **additionally requires TLS**: either Haechi terminates TLS itself (`proxy.tls`), or you explicitly acknowledge a fronting TLS terminator (`proxy.trustForwardedProto`). A remote bind with neither **throws at startup** — Haechi will not serve bearer tokens and payloads in plaintext on a non-loopback listener.
|
|
369
|
+
|
|
370
|
+
**Option A — Haechi terminates TLS** (serves `https`):
|
|
244
371
|
|
|
372
|
+
```jsonc
|
|
373
|
+
// haechi.config.json
|
|
374
|
+
"proxy": { "host": "0.0.0.0", "tls": { "keyFile": "/etc/haechi/tls/key.pem", "certFile": "/etc/haechi/tls/cert.pem" } }
|
|
375
|
+
// or PKCS#12: "tls": { "pfxFile": "/etc/haechi/tls/server.pfx", "passphrase": "…" }
|
|
376
|
+
```
|
|
245
377
|
```bash
|
|
246
378
|
haechi proxy --config haechi.config.json --host 0.0.0.0 --allow-remote-bind
|
|
379
|
+
# → Haechi proxy listening on https://0.0.0.0:11016
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
**Option B — a trusted reverse proxy terminates TLS** in front of Haechi (Haechi stays plain `http` on a private network behind the hop):
|
|
383
|
+
|
|
384
|
+
```jsonc
|
|
385
|
+
"proxy": { "host": "0.0.0.0", "trustForwardedProto": true }
|
|
247
386
|
```
|
|
387
|
+
With `trustForwardedProto: true`, Haechi **refuses any request whose `X-Forwarded-Proto` is not `https`** (a plaintext request that bypassed the hop) with a fail-closed `403`, checked before auth and body-read. The `/__haechi/*` liveness/metrics routes are exempt so a loopback sidecar can still scrape them. Only the trusted terminator may set `X-Forwarded-Proto` — do not enable this if untrusted clients can reach the Haechi port directly.
|
|
248
388
|
|
|
249
|
-
**The proxy
|
|
389
|
+
**The proxy ships bearer client authentication** (`auth.provider: bearer`, shipped in 0.6): a hashed token store, per-identity policy profiles, a model allowlist, and a per-identity rate limit (see [`auth`](#auth) and [Named profiles](#named-profiles)). The default `auth.provider: none` leaves the proxy unauthenticated — with `none`, anyone who can reach the port can use your upstream and the token round-trip path. The built-in rate limit is single-process (in-memory, per-process); front multiple replicas with a shared limiter. Use `--allow-remote-bind` only behind explicit network controls regardless — bind `0.0.0.0` inside a container and restrict the host port mapping (`-p 127.0.0.1:11016:11016`), or front it with a firewall/VPN/authenticating reverse proxy.
|
|
250
390
|
|
|
251
391
|
## Validation cheatsheet
|
|
252
392
|
|
|
253
|
-
These throw at load (fail-closed): unknown `keys.provider`; empty `proxy.host`; out-of-range `proxy.port`; non-`jsonl` `audit.sink`; non-`local` `tokenVault.provider`; bad `revealPolicy`; non-positive `retentionDays`; non-boolean `deterministic`/`detokenizeResponses`; empty/non-string `deterministicTypes`; empty/non-string `mcp.allowedMethods`; non-boolean `mcp.*` flags; unknown `privacy.profile`; bad `responseProtection.failureMode`; non-positive `responseProtection.maxBytes`; non-boolean `responseProtection.scanNumbers`; bad `streaming.requestMode`/`streaming.responseMode`; non-positive `streaming.maxMatchBytes`; bad `auth.provider`; empty `auth.store`; non-string `auth.allowedLabelKeys`; non-object `policy.profiles`; `policy.profileBinding` without a valid `default`; non-string `policy.modelAllowlist`; non-positive `policy.rate.requestsPerMinute`; non-positive `limits
|
|
393
|
+
These throw at load (fail-closed): unknown `keys.provider`; empty `proxy.host`; out-of-range `proxy.port`; non-boolean `proxy.trustForwardedProto`; a `proxy.tls` that is non-`null` but not an object, sets `keyFile` without `certFile` (or vice-versa), mixes `pfxFile` with `keyFile`/`certFile`, names an unreadable file, or does not resolve to usable material `((key && cert) or pfx)`; non-`jsonl` `audit.sink`; non-`local` `tokenVault.provider`; bad `revealPolicy`; non-positive `retentionDays`; non-boolean `deterministic`/`detokenizeResponses`; empty/non-string `deterministicTypes`; empty/non-string `mcp.allowedMethods`; non-boolean `mcp.*` flags; unknown `privacy.profile`; bad `responseProtection.failureMode`; non-positive `responseProtection.maxBytes`; non-boolean `responseProtection.scanNumbers`; bad `streaming.requestMode`/`streaming.responseMode`; non-positive `streaming.maxMatchBytes`; bad `auth.provider`; empty `auth.store`; non-string `auth.allowedLabelKeys`; non-object `policy.profiles`; `policy.profileBinding` without a valid `default`; non-string `policy.modelAllowlist`; non-positive `policy.rate.requestsPerMinute`; non-positive `limits.maxRequestBytes`/`limits.upstreamTimeoutMs`/`limits.maxNestingDepth`; negative or non-integer `limits.maxInFlight`/`limits.shutdownGraceMs`; non-`null`/negative/non-integer `limits.requestTimeoutMs`/`limits.headersTimeoutMs`; non-positive-integer or **newer-than-supported** `configVersion`; unknown `target.type`/`adapter`; unsafe custom regex; weakening action without `allowUnsafeOverrides`; non-`text`/`json` `logging.format`; non-boolean `metrics.enabled`; an invalid `HAECHI_*` env overlay value (bad `HAECHI_PROXY_PORT`, unknown `HAECHI_MODE`, malformed `HAECHI_UPSTREAM`, …).
|
|
254
394
|
|
|
255
395
|
# Satellite operator configuration (0.9)
|
|
256
396
|
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# Haechi 운영 런북 (Day-2)
|
|
2
|
+
|
|
3
|
+
- 상태: Living document (코어 1.2.x 추적)
|
|
4
|
+
|
|
5
|
+
Haechi를 프로덕션에서 운영하기 위한 실무 가이드입니다: 배포, 환경변수 오버레이를 통한 설정, health/readiness/metrics 모니터링, 우아한 종료, 백프레셔 튜닝, 그리고 해시 체인을 깨지 않는 audit 로그 회전입니다.
|
|
6
|
+
|
|
7
|
+
이 문서는 운영 가이드이며 컴플라이언스 보증이 아닙니다. 전체 설정 레퍼런스는 [`configuration.ko.md`](./configuration.ko.md)를, 신뢰 경계는 [`threat-model.ko.md`](./threat-model.ko.md)를 참고하십시오.
|
|
8
|
+
|
|
9
|
+
## 1. 배포
|
|
10
|
+
|
|
11
|
+
Haechi는 런타임 의존성이 0인 Node `>=22` 패키지입니다. 리포지토리 루트의 참조용 [`Dockerfile`](../../Dockerfile), [`docker-compose.yml`](../../docker-compose.yml), [`.dockerignore`](../../.dockerignore)가 하드닝된 이미지를 빌드합니다(이 파일들은 npm 타르볼에 포함되지 **않는** 리포지토리 배포 자산입니다). 이미지는:
|
|
12
|
+
|
|
13
|
+
- Node 22 slim 베이스를 핀으로 고정하고(`engines: ">=22"`와 일치),
|
|
14
|
+
- 비루트 `node` 사용자로 실행하며,
|
|
15
|
+
- 런타임 파일만 복사하고(`.haechi` 비밀, 테스트, 문서 소스 제외),
|
|
16
|
+
- audit 체인 / 키 파일 / 토큰 볼트를 위한 쓰기 가능 `/app/.haechi` 볼륨을 선언하고 나머지 트리는 읽기 전용으로 실행하며,
|
|
17
|
+
- `/__haechi/live`에 대한 `HEALTHCHECK`를 제공합니다.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
docker compose up -d # 참조 스택 빌드 + 실행
|
|
21
|
+
docker compose logs -f haechi
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
**TLS + 인증으로 앞단을 보호하십시오.** Haechi는 자체 TLS가 없습니다. 포트는 TLS를 종단하고 인증하는 리버스 프록시(nginx / Caddy / Traefik / API 게이트웨이)에만 공개하고, 원시 Haechi 포트를 공개 인터페이스에 절대 노출하지 마십시오. compose 예제는 바로 이 이유로 호스트 loopback(`127.0.0.1:11016`)에만 공개합니다.
|
|
25
|
+
|
|
26
|
+
**Loopback 너머 바인딩.** 컨테이너 내부에서는 매핑된 포트가 도달 가능하도록 Haechi가 `0.0.0.0`에 바인딩해야 하며, 이는 `--allow-remote-bind`를 요구합니다(참조 `CMD`가 전달합니다). 호스트에서는 기본 loopback 바인딩을 선호하고 리버스 프록시를 통해 Haechi에 접근하십시오. [Loopback 너머 바인딩](./configuration.ko.md)을 참고하십시오.
|
|
27
|
+
|
|
28
|
+
## 2. 환경변수 오버레이를 통한 설정
|
|
29
|
+
|
|
30
|
+
컨테이너 / 12-factor 배포를 위해 **비밀이 아닌 운영 키의 고정 allowlist**를 환경변수로 덮어쓸 수 있습니다. 환경변수 값은 **설정 파일보다 우선**하며 fail-closed로 검증됩니다 — 잘못된 값(잘못된 포트, 알 수 없는 모드)은 프로세스를 조용히 약화시키지 않고 **기동 실패**시킵니다.
|
|
31
|
+
|
|
32
|
+
| 환경변수 | 설정 키 | 타입 / 값 | 예시 |
|
|
33
|
+
|---|---|---|---|
|
|
34
|
+
| `HAECHI_PROXY_PORT` | `proxy.port` | 정수 0–65535 | `11016` |
|
|
35
|
+
| `HAECHI_PROXY_HOST` | `proxy.host` | 비어 있지 않은 문자열 | `0.0.0.0` |
|
|
36
|
+
| `HAECHI_UPSTREAM` | `target.upstream` | URL 문자열 | `http://llm:8000` |
|
|
37
|
+
| `HAECHI_MODE` | `mode` | `dry-run` \| `report-only` \| `enforce` | `enforce` |
|
|
38
|
+
| `HAECHI_LOG_FORMAT` | `logging.format` | `text` \| `json` | `json` |
|
|
39
|
+
|
|
40
|
+
**비밀은 설계상 오버레이 대상이 아닙니다.** `keys.*`(로컬 키 파일이나 외부 키 경로), auth 토큰 저장소, 어떤 토큰/비밀에 대한 `HAECHI_*` 변수도 **없습니다**. 비밀은 마운트된 설정 파일에 두거나 **주입된 provider**(`createRuntime(config, { cryptoProvider, authProvider, … })`)로 공급합니다. 비밀을 프로세스 환경에 두면 `/proc`, 크래시 덤프, 오케스트레이터 inspect 출력, 자식 프로세스를 통해 누출될 위험이 있으므로 오버레이 allowlist에서 완전히 제외합니다.
|
|
41
|
+
|
|
42
|
+
오버레이는 `loadConfig()`에서 파일을 읽은 뒤 `normalizeConfig()` 이전에 적용되므로, 오버레이된 값도 파일에 설정된 값과 동일한 검증을 거칩니다.
|
|
43
|
+
|
|
44
|
+
## 3. Health, readiness, metrics 스크레이핑
|
|
45
|
+
|
|
46
|
+
예약된 `/__haechi/*` 프리픽스 아래 인증이 필요 없는 네 개의 라우트로, 인증/바디 읽기 이전에 검사되며 upstream을 절대 프록시하지 않습니다(전체 레퍼런스: [운영 엔드포인트](./configuration.ko.md#운영-엔드포인트)):
|
|
47
|
+
|
|
48
|
+
| 엔드포인트 | 용도 |
|
|
49
|
+
|---|---|
|
|
50
|
+
| `GET /__haechi/live` | **Liveness** — 재시작 프로브. 가볍고, 이벤트 루프가 서비스하는 동안 200. |
|
|
51
|
+
| `GET /__haechi/ready` | **Readiness** — 트래픽 게이트. **audit sink에 쓸 수 없으면 503**(감사를 못 하는 게이트웨이는 ready가 아님). 로드밸런서/오케스트레이터 readiness 프로브를 여기로 지정하십시오. |
|
|
52
|
+
| `GET /__haechi/health` | 하위 호환(`ok` + `mode` + `version`). |
|
|
53
|
+
| `GET /__haechi/metrics` | Prometheus 텍스트 노출. `metrics.enabled: false`이면 `404`. |
|
|
54
|
+
|
|
55
|
+
Prometheus(또는 OpenMetrics 호환 스크레이퍼)로 **`/metrics`를 스크레이프**하십시오:
|
|
56
|
+
|
|
57
|
+
```yaml
|
|
58
|
+
scrape_configs:
|
|
59
|
+
- job_name: haechi
|
|
60
|
+
metrics_path: /__haechi/metrics
|
|
61
|
+
static_configs:
|
|
62
|
+
- targets: ["haechi:11016"]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
주요 신호: `haechi_requests_total{route,mode,decision}`, `haechi_blocks_total`, `haechi_auth_denied_total`, `haechi_rate_limited_total`, `haechi_overloaded_total`(백프레셔 503), `haechi_upstream_timeout_total`, `haechi_upstream_error_total`, `haechi_response_unprotected_total`, `haechi_internal_error_total`, 그리고 `haechi_request_duration_seconds{route}` 히스토그램.
|
|
66
|
+
|
|
67
|
+
**텔레메트리 no-PII 불변식.** 모든 메트릭 이름과 **모든 라벨 값**은 경계가 있는 enum(route id / mode / decision class)이며, identity·토큰·탐지 값이 절대 아닙니다. 동일한 불변식이 구조화 로그에도 적용됩니다: `logging.format: json`(또는 `HAECHI_LOG_FORMAT=json`)에서 기동/종료/오류 로그는 `correlationId`와 오류 클래스 이름만 담고 페이로드는 절대 담지 않습니다. `correlationId`는 해당 요청의 audit 이벤트에도 나타나므로, 기록된 오류를 그 audit 추적과 연결할 수 있습니다.
|
|
68
|
+
|
|
69
|
+
## 4. 우아한 종료
|
|
70
|
+
|
|
71
|
+
`SIGINT`/`SIGTERM` 시 CLI는 프록시의 `close()`를 호출하고, 이는 **우아하게 드레인**합니다:
|
|
72
|
+
|
|
73
|
+
1. 새 연결 수락을 멈추고(`server.close()`),
|
|
74
|
+
2. idle keep-alive 소켓을 즉시 닫고(`closeIdleConnections()`),
|
|
75
|
+
3. in-flight 요청이 끝날 때까지 기다리고,
|
|
76
|
+
4. 유예 기간(`limits.shutdownGraceMs`, 기본 10000ms) 후 남은 소켓을 강제 종료하여(`closeAllConnections()`) 멈춘 keep-alive가 종료를 무한정 붙잡지 못하게 합니다.
|
|
77
|
+
|
|
78
|
+
`close()`는 in-flight 요청이 빠지거나 유예가 지나면 resolve합니다. 오케스트레이터의 `terminationGracePeriod`(쿠버네티스) / `stop_grace_period`(compose)를 `limits.shutdownGraceMs`보다 **크게** 설정하여 플랫폼이 드레인 도중 SIGKILL하지 않게 하십시오. 가장 긴 허용 in-flight 요청에 맞춰 `limits.shutdownGraceMs`를 튜닝하십시오.
|
|
79
|
+
|
|
80
|
+
## 5. 백프레셔 튜닝
|
|
81
|
+
|
|
82
|
+
`limits.maxInFlight`는 동시에 처리되는 요청 수의 전역 상한입니다.
|
|
83
|
+
|
|
84
|
+
- `0`(기본)은 상한을 비활성화합니다 — 1.1 동작 그대로.
|
|
85
|
+
- `> 0`: 현재 in-flight 수가 상한에 도달하면 **새** 요청은 `Retry-After` 헤더(`limits.shutdownGraceMs`에서 유도한 초)와 `{ "error": "haechi_overloaded" }` 바디와 함께, 인증/바디 읽기 **이전에** `503`으로 거부됩니다. 거부마다 `haechi_overloaded_total`이 증가합니다.
|
|
86
|
+
- `/__haechi/*` 관측 라우트는 상한에서 **예외**이므로, 포화 상태에서도 liveness와 `/metrics`를 스크레이프할 수 있습니다 — 부하를 떨어내는 *이유*를 여전히 볼 수 있습니다.
|
|
87
|
+
|
|
88
|
+
`maxInFlight`를 upstream + 호스트가 감당할 수 있는 동시성 근처로 설정하고(`haechi_request_duration_seconds`와 upstream 포화를 관찰), 게이트웨이가 붕괴 대신 깔끔한 503으로 부하를 떨어내도록 여유를 두십시오. 느린 upstream이 슬롯을 무한정 점유하지 못하도록 튜닝된 `limits.upstreamTimeoutMs`와 함께 사용하십시오.
|
|
89
|
+
|
|
90
|
+
### 튜닝된 타임아웃
|
|
91
|
+
|
|
92
|
+
`limits.requestTimeoutMs`와 `limits.headersTimeoutMs`는 Node HTTP 서버의 `requestTimeout` / `headersTimeout`에 매핑됩니다. 둘 다 기본값 `null` = Node 서버 기본값을 그대로 둠(옵트인하지 않으면 동작 불변)입니다. slow-loris 류의 느린 요청/헤더 전달을 제한하려면 숫자를 설정하고, `0`은 해당 타임아웃을 비활성화합니다(Node 의미).
|
|
93
|
+
|
|
94
|
+
## 6. 체인 인지 audit 로그 회전 & 보존
|
|
95
|
+
|
|
96
|
+
audit 로그는 **SHA-256 해시 체인**입니다(`audit.path`): 각 이벤트의 `auditIntegrity.previousHash`가 이전 이벤트 해시에 연결되므로, 삽입·삭제·수정·재정렬은 `haechi audit-verify` / `verifyAuditChain`로 탐지됩니다. 선택적 **anchor 스트림**(`audit.anchor`)은 체인 헤드를 별도의 append-only 매체에 기록하여 tail truncation(최신 이벤트 삭제)까지 잡아냅니다. [`audit` 개념](./configuration.ko.md#audit)과 위협 모델을 참고하십시오.
|
|
97
|
+
|
|
98
|
+
**체인을 중간에서 잘라내거나 다시 쓰지 마십시오.** `audit.jsonl`을 제자리에서 truncate하거나 이전 줄을 다시 쓰면 **체인이 깨지고** 검증이 실패합니다(더 나쁘게는 변조 증거가 조용히 사라집니다). **새 세그먼트를 시작**하고 이전 세그먼트를 보존하는 방식으로 회전하십시오:
|
|
99
|
+
|
|
100
|
+
1. writer를 **멈추거나 정지**시킵니다(우아한 종료, 또는 점검 시간대에 회전). 기본 JSONL sink는 append 방식이므로, 열려 있는 파일을 회전하는 일을 피하는 것입니다.
|
|
101
|
+
2. 현재 세그먼트를 **그대로 보존한 채 옆으로 옮깁니다**: `mv .haechi/audit.jsonl .haechi/audit-2026-06-12.jsonl`(대응하는 anchor도: `mv .haechi/audit.anchor.jsonl .haechi/audit-2026-06-12.anchor.jsonl`).
|
|
102
|
+
3. Haechi를 재시작하여(또는 `audit.path` / `audit.anchor.path`를 새 파일로 지정하여) **새 세그먼트를 시작**합니다. 새 체인은 `previousHash: null`로 시작합니다 — 독립적으로 검증 가능한 새 체인입니다. 이는 의도된 동작입니다: 각 세그먼트가 자체적으로 검증 가능한 체인이며, 회전 경계를 넘어 체인을 잇지 **않습니다**.
|
|
103
|
+
4. 보존된 각 세그먼트를 자체 anchor로 **독립 검증**합니다: `haechi audit-verify --audit .haechi/audit-2026-06-12.jsonl --anchor .haechi/audit-2026-06-12.anchor.jsonl`.
|
|
104
|
+
5. 전체 이력이 검증 가능하도록 보존 기간 동안 **이전 세그먼트를 보관**합니다. 가능하면 삭제 대신 append-only / WORM 저장소로 아카이브하십시오. anchor의 방어는 anchor가 별도의 append-only 매체에 존재한다는 전제에 기반합니다.
|
|
105
|
+
|
|
106
|
+
**보존:** 회전된 각 세그먼트(및 그 anchor)를 요구되는 audit 보존 기간 동안 유지한 뒤 세그먼트 단위로 만료시키십시오 — 세그먼트 내 일부 줄을 절대 부분 삭제하지 마십시오. 토큰 볼트 보존은 독립적이며(`tokenVault.retentionDays`), audit 회전은 토큰을 정리하지 않습니다.
|
|
107
|
+
|
|
108
|
+
아카이브 파이프라인에 검증 단계를 유지하지 않는 한, 나중에 재검증이 불가능한 방식으로 세그먼트를 압축/암호화하지 **마십시오**. 회전된 세그먼트는 여전히 검증될 때에만 증거로서 유용합니다.
|
|
109
|
+
|
|
110
|
+
## 7. 빠른 참조
|
|
111
|
+
|
|
112
|
+
| 작업 | 커맨드 |
|
|
113
|
+
|---|---|
|
|
114
|
+
| 시작(compose) | `docker compose up -d` |
|
|
115
|
+
| Liveness | `curl localhost:11016/__haechi/live` |
|
|
116
|
+
| Readiness | `curl localhost:11016/__haechi/ready` |
|
|
117
|
+
| Metrics | `curl localhost:11016/__haechi/metrics` |
|
|
118
|
+
| 세그먼트 검증 | `haechi audit-verify --audit <seg>.jsonl --anchor <seg>.anchor.jsonl` |
|
|
119
|
+
| 우아한 정지 | `docker compose stop` (SIGTERM → 드레인) |
|
|
120
|
+
|
|
121
|
+
참고: `configVersion` 스탬프와 업그레이드 노트는 [`config-version.ko.md`](./config-version.ko.md)를 참고하십시오.
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Haechi Operations Runbook (Day-2)
|
|
2
|
+
|
|
3
|
+
- Status: Living document (tracks core 1.2.x)
|
|
4
|
+
|
|
5
|
+
A practical guide to running Haechi in production: deploy, configure via the
|
|
6
|
+
env-var overlay, monitor with health/readiness/metrics, shut down gracefully,
|
|
7
|
+
tune backpressure, and rotate the audit log without breaking its hash chain.
|
|
8
|
+
|
|
9
|
+
This is an operability guide, not a compliance guarantee. See
|
|
10
|
+
[`configuration.md`](./configuration.md) for the full config reference and
|
|
11
|
+
[`threat-model.md`](./threat-model.md) for the trust boundary.
|
|
12
|
+
|
|
13
|
+
## 1. Deploy
|
|
14
|
+
|
|
15
|
+
Haechi is a zero-runtime-dependency Node `>=22` package. The reference
|
|
16
|
+
[`Dockerfile`](../../Dockerfile), [`docker-compose.yml`](../../docker-compose.yml),
|
|
17
|
+
and [`.dockerignore`](../../.dockerignore) at the repo root build a hardened
|
|
18
|
+
image (these files are **not** shipped in the npm tarball — they are repo deploy
|
|
19
|
+
assets). The image:
|
|
20
|
+
|
|
21
|
+
- pins a Node 22 slim base (matches `engines: ">=22"`),
|
|
22
|
+
- runs as the non-root `node` user,
|
|
23
|
+
- copies only the runtime files (no `.haechi` secrets, no tests, no docs sources),
|
|
24
|
+
- declares a writable `/app/.haechi` volume for the audit chain / key file / token
|
|
25
|
+
vault and runs the rest of the tree read-only,
|
|
26
|
+
- ships a `HEALTHCHECK` against `/__haechi/live`.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
docker compose up -d # build + run the reference stack
|
|
30
|
+
docker compose logs -f haechi
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Front it with TLS + auth.** Haechi has no TLS of its own. Publish its port only
|
|
34
|
+
to a TLS-terminating, authenticating reverse proxy (nginx / Caddy / Traefik / an
|
|
35
|
+
API gateway); never expose the raw Haechi port on a public interface. The compose
|
|
36
|
+
example publishes to host loopback (`127.0.0.1:11016`) for exactly this reason.
|
|
37
|
+
|
|
38
|
+
**Binding beyond loopback.** Inside a container Haechi must bind `0.0.0.0` for the
|
|
39
|
+
mapped port to be reachable, which requires `--allow-remote-bind` (the reference
|
|
40
|
+
`CMD` passes it). On a host, prefer the default loopback bind and reach Haechi
|
|
41
|
+
through the reverse proxy. See [Binding beyond loopback](./configuration.md#binding-beyond-loopback).
|
|
42
|
+
|
|
43
|
+
## 2. Configuration via the env-var overlay
|
|
44
|
+
|
|
45
|
+
For container / 12-factor deploys, a **fixed allowlist of NON-SECRET operational
|
|
46
|
+
keys** may be overridden from the environment. The env value **wins over the
|
|
47
|
+
config file** and is validated fail-closed — an invalid value (bad port, unknown
|
|
48
|
+
mode) makes the process **fail to start** rather than degrade silently.
|
|
49
|
+
|
|
50
|
+
| Env var | Config key | Type / values | Example |
|
|
51
|
+
|---|---|---|---|
|
|
52
|
+
| `HAECHI_PROXY_PORT` | `proxy.port` | integer 0–65535 | `11016` |
|
|
53
|
+
| `HAECHI_PROXY_HOST` | `proxy.host` | non-empty string | `0.0.0.0` |
|
|
54
|
+
| `HAECHI_UPSTREAM` | `target.upstream` | URL string | `http://llm:8000` |
|
|
55
|
+
| `HAECHI_MODE` | `mode` | `dry-run` \| `report-only` \| `enforce` | `enforce` |
|
|
56
|
+
| `HAECHI_LOG_FORMAT` | `logging.format` | `text` \| `json` | `json` |
|
|
57
|
+
|
|
58
|
+
**Secrets are NOT overlayable — by design.** There is **no** `HAECHI_*` variable
|
|
59
|
+
for `keys.*` (the local key file or an external key path), the auth token store,
|
|
60
|
+
or any token/secret. Secrets stay in the mounted config file or are supplied via
|
|
61
|
+
**injected providers** (`createRuntime(config, { cryptoProvider, authProvider, … })`).
|
|
62
|
+
Putting a secret in a process environment invites leaking it through `/proc`,
|
|
63
|
+
crash dumps, orchestrator inspect output, and child processes — so the overlay
|
|
64
|
+
allowlist excludes them outright.
|
|
65
|
+
|
|
66
|
+
The overlay is applied in `loadConfig()` after reading the file and before
|
|
67
|
+
`normalizeConfig()`, so an overlaid value passes the same validation as a
|
|
68
|
+
file-set one.
|
|
69
|
+
|
|
70
|
+
## 3. Health, readiness, and metrics scraping
|
|
71
|
+
|
|
72
|
+
Four unauthenticated routes under the reserved `/__haechi/*` prefix, checked
|
|
73
|
+
before auth/body-read, never proxying upstream (full reference:
|
|
74
|
+
[Operability endpoints](./configuration.md#operability-endpoints)):
|
|
75
|
+
|
|
76
|
+
| Endpoint | Use |
|
|
77
|
+
|---|---|
|
|
78
|
+
| `GET /__haechi/live` | **Liveness** — restart probe. Cheap; 200 while the event loop serves. |
|
|
79
|
+
| `GET /__haechi/ready` | **Readiness** — traffic gate. **503 when the audit sink is not writable** (a gateway that cannot audit is not ready). Point your load balancer / orchestrator readiness probe here. |
|
|
80
|
+
| `GET /__haechi/health` | Back-compat (`ok` + `mode` + `version`). |
|
|
81
|
+
| `GET /__haechi/metrics` | Prometheus text exposition. `404` when `metrics.enabled: false`. |
|
|
82
|
+
|
|
83
|
+
**Scrape `/metrics`** with Prometheus (or any OpenMetrics-compatible scraper):
|
|
84
|
+
|
|
85
|
+
```yaml
|
|
86
|
+
scrape_configs:
|
|
87
|
+
- job_name: haechi
|
|
88
|
+
metrics_path: /__haechi/metrics
|
|
89
|
+
static_configs:
|
|
90
|
+
- targets: ["haechi:11016"]
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Key signals: `haechi_requests_total{route,mode,decision}`, `haechi_blocks_total`,
|
|
94
|
+
`haechi_auth_denied_total`, `haechi_rate_limited_total`, `haechi_overloaded_total`
|
|
95
|
+
(backpressure 503s), `haechi_upstream_timeout_total`, `haechi_upstream_error_total`,
|
|
96
|
+
`haechi_response_unprotected_total`, `haechi_internal_error_total`, and the
|
|
97
|
+
`haechi_request_duration_seconds{route}` histogram.
|
|
98
|
+
|
|
99
|
+
**No-PII-in-telemetry invariant.** Every metric name and **every label value** is
|
|
100
|
+
a bounded enum (route id / mode / decision class) — never an identity, token, or
|
|
101
|
+
detected value. The same invariant covers structured logs: with
|
|
102
|
+
`logging.format: json` (or `HAECHI_LOG_FORMAT=json`), startup/shutdown/error logs
|
|
103
|
+
carry a `correlationId` and an error class name only, never a payload. The
|
|
104
|
+
`correlationId` also appears on the request's audit events, so you can join a
|
|
105
|
+
logged error to its audit trail.
|
|
106
|
+
|
|
107
|
+
## 4. Graceful shutdown
|
|
108
|
+
|
|
109
|
+
On `SIGINT`/`SIGTERM` the CLI calls the proxy's `close()`, which **drains
|
|
110
|
+
gracefully**:
|
|
111
|
+
|
|
112
|
+
1. stops accepting new connections (`server.close()`),
|
|
113
|
+
2. immediately closes idle keep-alive sockets (`closeIdleConnections()`),
|
|
114
|
+
3. waits for in-flight requests to finish,
|
|
115
|
+
4. after a grace period (`limits.shutdownGraceMs`, default 10000ms) force-closes
|
|
116
|
+
any lingering socket (`closeAllConnections()`) so a stuck keep-alive cannot
|
|
117
|
+
hold shutdown open forever.
|
|
118
|
+
|
|
119
|
+
`close()` resolves once in-flight requests drain or the grace elapses. Set your
|
|
120
|
+
orchestrator's `terminationGracePeriod` (Kubernetes) / `stop_grace_period`
|
|
121
|
+
(compose) **above** `limits.shutdownGraceMs` so the platform does not SIGKILL
|
|
122
|
+
mid-drain. Tune `limits.shutdownGraceMs` to your longest acceptable in-flight
|
|
123
|
+
request.
|
|
124
|
+
|
|
125
|
+
## 5. Backpressure tuning
|
|
126
|
+
|
|
127
|
+
`limits.maxInFlight` is a global ceiling on concurrently-processing requests.
|
|
128
|
+
|
|
129
|
+
- `0` (default) disables the ceiling — unchanged 1.1 behavior.
|
|
130
|
+
- `> 0`: when the live in-flight count is at the ceiling, a **new** request is
|
|
131
|
+
rejected `503` with a `Retry-After` header (seconds, derived from
|
|
132
|
+
`limits.shutdownGraceMs`) and a `{ "error": "haechi_overloaded" }` body, **before**
|
|
133
|
+
auth and body-read. Each rejection increments `haechi_overloaded_total`.
|
|
134
|
+
- The `/__haechi/*` observability routes are **exempt** from the ceiling, so
|
|
135
|
+
liveness and `/metrics` stay scrapable under saturation — you can still see
|
|
136
|
+
*why* you are shedding load.
|
|
137
|
+
|
|
138
|
+
Set `maxInFlight` near the concurrency your upstream + host can sustain (watch
|
|
139
|
+
`haechi_request_duration_seconds` and upstream saturation), leaving headroom so
|
|
140
|
+
the gateway sheds load with a clean 503 instead of collapsing. Pair it with a
|
|
141
|
+
tuned `limits.upstreamTimeoutMs` so a slow upstream cannot pin slots indefinitely.
|
|
142
|
+
|
|
143
|
+
### Tuned timeouts
|
|
144
|
+
|
|
145
|
+
`limits.requestTimeoutMs` and `limits.headersTimeoutMs` map to the Node HTTP
|
|
146
|
+
server's `requestTimeout` / `headersTimeout`. Both default to `null` = leave
|
|
147
|
+
Node's server defaults untouched (behavior unchanged unless you opt in). Set a
|
|
148
|
+
number to cap slow-loris-style slow request/header delivery; `0` disables that
|
|
149
|
+
specific timeout (Node semantics).
|
|
150
|
+
|
|
151
|
+
## 6. Chain-aware audit log rotation & retention
|
|
152
|
+
|
|
153
|
+
The audit log is a **SHA-256 hash chain** (`audit.path`): each event's
|
|
154
|
+
`auditIntegrity.previousHash` links to the prior event's hash, so any insert,
|
|
155
|
+
delete, edit, or reorder is detectable by `haechi audit-verify` /
|
|
156
|
+
`verifyAuditChain`. An optional **anchor stream** (`audit.anchor`) appends the
|
|
157
|
+
chain head to separate append-only media so even tail truncation (deleting the
|
|
158
|
+
newest events) is caught. See [`audit` concepts](./configuration.md#audit) and the
|
|
159
|
+
threat model.
|
|
160
|
+
|
|
161
|
+
**Never truncate or rewrite a chain mid-stream.** Rotating by truncating
|
|
162
|
+
`audit.jsonl` in place, or rewriting earlier lines, **breaks the chain** and makes
|
|
163
|
+
verification fail (or, worse, silently destroys tamper evidence). Rotate by
|
|
164
|
+
**starting a new segment**, preserving prior segments:
|
|
165
|
+
|
|
166
|
+
1. **Stop or quiesce** the writer (graceful shutdown, or rotate at a maintenance
|
|
167
|
+
window). The default JSONL sink appends; rotating a file it holds open is what
|
|
168
|
+
you are avoiding.
|
|
169
|
+
2. **Move the current segment aside**, keeping it intact:
|
|
170
|
+
`mv .haechi/audit.jsonl .haechi/audit-2026-06-12.jsonl` (and the matching
|
|
171
|
+
anchor: `mv .haechi/audit.anchor.jsonl .haechi/audit-2026-06-12.anchor.jsonl`).
|
|
172
|
+
3. **Start a fresh segment** by restarting Haechi (or pointing `audit.path` /
|
|
173
|
+
`audit.anchor.path` at the new files). The new chain begins with
|
|
174
|
+
`previousHash: null` — a fresh, independently-verifiable chain. This is
|
|
175
|
+
expected: each segment is its own verifiable chain; you do **not** chain across
|
|
176
|
+
the rotation boundary.
|
|
177
|
+
4. **Verify each retained segment independently** with its own anchor:
|
|
178
|
+
`haechi audit-verify --audit .haechi/audit-2026-06-12.jsonl --anchor .haechi/audit-2026-06-12.anchor.jsonl`.
|
|
179
|
+
5. **Retain prior segments** for your retention window so the full history stays
|
|
180
|
+
verifiable. Archive (don't delete) to append-only / WORM storage where you can;
|
|
181
|
+
the anchor's defense assumes the anchor lives on separate, append-only media.
|
|
182
|
+
|
|
183
|
+
**Retention:** keep each rotated segment (and its anchor) for your required
|
|
184
|
+
audit-retention period, then expire whole segments — never partial lines within a
|
|
185
|
+
segment. Token-vault retention is independent (`tokenVault.retentionDays`); audit
|
|
186
|
+
rotation does not purge tokens.
|
|
187
|
+
|
|
188
|
+
**Do not** compress/encrypt a segment in a way that prevents later
|
|
189
|
+
re-verification unless you keep the verification step in your archival pipeline. A
|
|
190
|
+
rotated segment is only useful as evidence if it still verifies.
|
|
191
|
+
|
|
192
|
+
## 7. Quick reference
|
|
193
|
+
|
|
194
|
+
| Task | Command |
|
|
195
|
+
|---|---|
|
|
196
|
+
| Start (compose) | `docker compose up -d` |
|
|
197
|
+
| Liveness | `curl localhost:11016/__haechi/live` |
|
|
198
|
+
| Readiness | `curl localhost:11016/__haechi/ready` |
|
|
199
|
+
| Metrics | `curl localhost:11016/__haechi/metrics` |
|
|
200
|
+
| Verify a segment | `haechi audit-verify --audit <seg>.jsonl --anchor <seg>.anchor.jsonl` |
|
|
201
|
+
| Graceful stop | `docker compose stop` (SIGTERM → drain) |
|
|
202
|
+
|
|
203
|
+
See also: [`config-version.md`](./config-version.md) for the `configVersion`
|
|
204
|
+
stamp and upgrade notes.
|