@bounded-sh/observe 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -0
- package/dist/context.d.ts +18 -0
- package/dist/context.js +105 -0
- package/dist/counters.d.ts +34 -0
- package/dist/counters.js +141 -0
- package/dist/denylist.d.ts +7 -0
- package/dist/denylist.js +81 -0
- package/dist/envelope.d.ts +71 -0
- package/dist/envelope.js +20 -0
- package/dist/escort.d.ts +92 -0
- package/dist/escort.js +269 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.js +274 -0
- package/dist/intercept/fetch.d.ts +2 -0
- package/dist/intercept/fetch.js +534 -0
- package/dist/intercept/http.d.ts +2 -0
- package/dist/intercept/http.js +262 -0
- package/dist/llm-prices.d.ts +136 -0
- package/dist/llm-prices.js +240 -0
- package/dist/manifest.d.ts +168 -0
- package/dist/manifest.js +941 -0
- package/dist/pipeline.d.ts +18 -0
- package/dist/pipeline.js +212 -0
- package/dist/recognize.d.ts +38 -0
- package/dist/recognize.js +185 -0
- package/dist/register.d.ts +9 -0
- package/dist/register.js +138 -0
- package/dist/reporter.d.ts +27 -0
- package/dist/reporter.js +210 -0
- package/dist/shapes.d.ts +15 -0
- package/dist/shapes.js +56 -0
- package/dist/state.d.ts +32 -0
- package/dist/state.js +42 -0
- package/dist/template.d.ts +7 -0
- package/dist/template.js +63 -0
- package/dist/types.d.ts +125 -0
- package/dist/types.js +10 -0
- package/package.json +43 -0
package/README.md
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
# @bounded-sh/observe — Bounded observe shim for Node
|
|
2
|
+
|
|
3
|
+
CP1 origin for the observe/enforce project (`SPEC-OBSERVE-ENFORCE-CUSTODY.md`
|
|
4
|
+
§3.1a). Intercepts your service's **egress** — `globalThis.fetch` (undici) and
|
|
5
|
+
`http`/`https.request` — and reports metadata-only event batches to the
|
|
6
|
+
Bounded observe ingest. Dependency-free at runtime. **Never breaks the app**:
|
|
7
|
+
every shim code path is wrapped; original behavior is always preserved;
|
|
8
|
+
observe is always fail-open (Bounded down ⇒ nothing changes for your app, T9).
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
npm install @bounded-sh/observe
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
```js
|
|
17
|
+
const { init, middleware, runAs } = require("@bounded-sh/observe");
|
|
18
|
+
|
|
19
|
+
init({
|
|
20
|
+
ingestBase: "https://observe-ingest.buildwithtarobase.workers.dev",
|
|
21
|
+
token: process.env.BOUNDED_SENSOR_TOKEN, // obs1.<keyId>.<sig>
|
|
22
|
+
org: "your-org",
|
|
23
|
+
});
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
That's the whole install: call `init()` once, as early as possible (before
|
|
27
|
+
other modules grab references to `fetch`). ESM: `import { init } from
|
|
28
|
+
"@bounded-sh/observe"`.
|
|
29
|
+
|
|
30
|
+
## Actor context — who did it
|
|
31
|
+
|
|
32
|
+
Attribution rides the **report only**; nothing is ever added to your outbound
|
|
33
|
+
requests (any `X-Bounded-*` header is actively **stripped from egress** and
|
|
34
|
+
used purely as attribution fallback).
|
|
35
|
+
|
|
36
|
+
```js
|
|
37
|
+
// Per-request (Express or Hono): maps your session to an actor for the whole
|
|
38
|
+
// request's async chain.
|
|
39
|
+
app.use(middleware()); // default: req.session.user / req.user
|
|
40
|
+
app.use(middleware((req) => ({ actor: req.session.user.id, kind: "human" })));
|
|
41
|
+
|
|
42
|
+
// Explicit scope (bots, jobs, agents):
|
|
43
|
+
await runAs({ actor: "agent:support-1", kind: "agent", onBehalfOf: customerId }, async () => {
|
|
44
|
+
await stripe.refunds.create({ charge, amount }); // reported as agent:support-1
|
|
45
|
+
});
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Actor ids should be **opaque internal ids, never emails** (L9 — the ingest
|
|
49
|
+
scrubber redacts email-shaped values as suspected PII). Unattributed calls are
|
|
50
|
+
still captured (they roll up under the `unattributed` pseudo-actor).
|
|
51
|
+
|
|
52
|
+
## What's captured vs. NOT (PII posture)
|
|
53
|
+
|
|
54
|
+
Captured (metadata only, per event): destination host, **templated** path
|
|
55
|
+
(UUIDs / numeric ids / hashes / vendor ids become `{id}` before anything
|
|
56
|
+
leaves your process), method, status, duration, byte counts, actor context.
|
|
57
|
+
|
|
58
|
+
Manifest-recognized routes (built-in v0 registry: Stripe
|
|
59
|
+
refunds/charges/payment_intents; OpenAI + Anthropic spend) additionally
|
|
60
|
+
recognize **safe fields only**: amounts in cents, opaque ids (`ch_…`,
|
|
61
|
+
`cus_…`), model names, token counts. As of **envelope v2** these ride the wire
|
|
62
|
+
under `rec.fields` (a bounded, one-level object of primitive SAFE values) —
|
|
63
|
+
`rec{rail, action, registryVersion, fields}` — and drive deterministic action
|
|
64
|
+
stories on the dashboard. They also remain available locally via the `onEvent`
|
|
65
|
+
hook. Server-side, ingest re-checks `rec.fields` KEYS against the L2 denylist
|
|
66
|
+
and scrubs the VALUES (L4), so a leaked email/PAN never lands raw.
|
|
67
|
+
|
|
68
|
+
NOT captured, ever:
|
|
69
|
+
- request/response **bodies** (unrecognized routes: top-level field
|
|
70
|
+
**names/types only**, on the first sighting of a shape);
|
|
71
|
+
- query-string **values** (names only, on shape samples);
|
|
72
|
+
- headers (only scanned to strip `X-Bounded-*`);
|
|
73
|
+
- prompts/completions/messages of LLM calls (not in any safe-field list);
|
|
74
|
+
- **PII-named fields — hard denylist (L2)**: `email`, `card*`, `*_name`,
|
|
75
|
+
`password`, `*token` (auth-shaped; token *counts* are fine), `address`,
|
|
76
|
+
`ssn`, `dob`, … are compiled into the shim and re-checked at extraction
|
|
77
|
+
time. Even a malicious/buggy manifest listing `customer_email` cannot
|
|
78
|
+
capture it, and even the *names* of PII-shaped fields are dropped from
|
|
79
|
+
shape samples.
|
|
80
|
+
|
|
81
|
+
Fail-safe: no/invalid manifest ⇒ metadata-only mode. Server side, ingest
|
|
82
|
+
re-filters (L3) and value-scrubs (L4) independently — the shim never sends
|
|
83
|
+
`org`/`sensor` (server-stamped from your sensor key), `scrubbed`, or any
|
|
84
|
+
unknown field.
|
|
85
|
+
|
|
86
|
+
Honest counts: if the shim ever drops events (bounded memory, backpressure),
|
|
87
|
+
the drop count is reported on the next successful event (`dropped`) and
|
|
88
|
+
surfaces downstream as completeness flags.
|
|
89
|
+
|
|
90
|
+
## Event classes (SPEC §3.2)
|
|
91
|
+
|
|
92
|
+
| class | when | notes |
|
|
93
|
+
|---------|------------------------------------------|-------|
|
|
94
|
+
| action | recognized route, 2xx | `rec{rail, action, registryVersion, fields}` (v2 `rec.fields` = SAFE values) |
|
|
95
|
+
| error | recognized route, non-2xx / network fail | `rec{…, fields}`; status 0 = never completed |
|
|
96
|
+
| shape | unrecognized route, first sighting | deduped in-memory, rate-capped, names/types only |
|
|
97
|
+
| counter | manifest `counters` matchers + hot unrecognized GETs (> 60 calls/min, exact from call #1) | one event per (route, minute): exact count, summed dur/bytes, class-representative status |
|
|
98
|
+
|
|
99
|
+
## Kill switch
|
|
100
|
+
|
|
101
|
+
`BOUNDED_OBSERVE_DISABLED=1`
|
|
102
|
+
- at process start: nothing is patched at all;
|
|
103
|
+
- at runtime: observation stops within one flush tick (≤2s). Clearing the env
|
|
104
|
+
resumes it.
|
|
105
|
+
|
|
106
|
+
## Reporting behavior
|
|
107
|
+
|
|
108
|
+
Async batches (≤500 events or 2s), fire-and-forget with a 5s send timeout,
|
|
109
|
+
single in-flight send, one retry for transient failures, bounded queue
|
|
110
|
+
(default 5000 events) with drop-and-count on overflow. `shutdown()` performs
|
|
111
|
+
a final drain (call it on SIGTERM to flush the tail). Hot-path overhead is
|
|
112
|
+
microseconds (measured in `test/perf.test.ts`; U2 target <1ms p99).
|
|
113
|
+
|
|
114
|
+
## Supported environments
|
|
115
|
+
|
|
116
|
+
- **Node 18 / 20 / 22 server apps** (fetch + http/https interception).
|
|
117
|
+
- **Vercel / Next.js Node runtime**: supported (call `init()` in
|
|
118
|
+
`instrumentation.ts` or the server entry).
|
|
119
|
+
- **Edge runtimes** (Vercel Edge, Workers): **fetch-wrap only** — `http/https`
|
|
120
|
+
and `AsyncLocalStorage`-based middleware depend on Node APIs; use `runAs`
|
|
121
|
+
with the fetch path only. (Documented limitation, SPEC §3.1a.)
|
|
122
|
+
|
|
123
|
+
Notes: `http/https` path records duration to response *headers* and takes
|
|
124
|
+
`bytes.o` from `Content-Length` (0 when chunked); response-field recognition
|
|
125
|
+
(LLM token counts) is fetch-path only.
|
|
126
|
+
|
|
127
|
+
## Config quick reference
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
init({
|
|
131
|
+
ingestBase, token, // required
|
|
132
|
+
org, // manifest polling scope
|
|
133
|
+
sessionMapper, // default mapper for middleware()
|
|
134
|
+
aliasHosts, // {"127.0.0.1:4242": "api.stripe.com"} — dev/demo
|
|
135
|
+
flushIntervalMs: 2000, sendTimeoutMs: 5000, maxQueueEvents: 5000,
|
|
136
|
+
counterPromoteThreshold: 60, // unrecognized-GET calls/min -> counter class
|
|
137
|
+
shapesPerMinute: 30, // new-shape rate cap
|
|
138
|
+
manifest, // override (tests); invalid -> built-in (fail-safe)
|
|
139
|
+
disableManifestPoll, manifestPollMs,
|
|
140
|
+
debug, onEvent, // debug hook: (wireEvent, recognizedFields)
|
|
141
|
+
});
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
## Manifest (living capture policy, §3.1g)
|
|
145
|
+
|
|
146
|
+
The shim polls `<ingestBase>/v1/manifest?org=…` every 60s (T1: capture policy
|
|
147
|
+
changes reach every interceptor without re-install). The endpoint doesn't
|
|
148
|
+
exist yet — 404/invalid keeps the built-in manifest. Signed-manifest
|
|
149
|
+
verification is a marked TODO in `src/manifest.ts`; until it lands, the
|
|
150
|
+
built-in registry is the trust anchor.
|
|
151
|
+
|
|
152
|
+
## Development
|
|
153
|
+
|
|
154
|
+
- `npm run build` · `npm test` (76 tests: interception, templating,
|
|
155
|
+
recognizers, counters, ALS context, header stripping, kill switch, L1/L2
|
|
156
|
+
PII behavior, envelope contract vs the real `observe-shared` validator,
|
|
157
|
+
perf, and an integration run of the sample app in
|
|
158
|
+
`~/bounded/observe-sample-app`).
|
|
159
|
+
- `npm run test:live` — LIVE prod verification: mints a dev sensor key
|
|
160
|
+
(needs `/tmp/observe-admin-secret.txt` or `$OBSERVE_ADMIN_SECRET`), runs
|
|
161
|
+
the sample app against prod ingest, asserts exact rollup deltas via the
|
|
162
|
+
consumer's `/internal/rollup/acme-demo`.
|
|
163
|
+
- Wire contract: `src/envelope.ts` mirrors the versioned envelope (v2) from
|
|
164
|
+
`packages/cdk/cloudflare/observe-shared`; `test/envelope-contract.test.ts`
|
|
165
|
+
fails on any drift (types + constants + runtime validation), including that
|
|
166
|
+
every emitted v2 event (with `rec.fields`) passes the real ingest validator
|
|
167
|
+
with ZERO stripped fields.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ActorContext, ActorInput, SessionMapper } from "./types";
|
|
2
|
+
/** Current actor context, if any (interceptors read this at call time). */
|
|
3
|
+
export declare function currentActor(): ActorContext | undefined;
|
|
4
|
+
/**
|
|
5
|
+
* Run `fn` attributed to an actor. The context follows the async chain, so
|
|
6
|
+
* every intercepted egress inside (awaited or fire-and-forget started within
|
|
7
|
+
* `fn`) reports this actor. Returns fn's return value; errors propagate
|
|
8
|
+
* unchanged (app semantics are never altered).
|
|
9
|
+
*/
|
|
10
|
+
export declare function runAs<T>(input: ActorInput, fn: () => T): T;
|
|
11
|
+
/**
|
|
12
|
+
* Request middleware establishing actor context for the whole request.
|
|
13
|
+
* Express: app.use(bounded.middleware()) — (req, res, next).
|
|
14
|
+
* Hono: app.use(bounded.middleware(mapper)) — (c, next), promise returned.
|
|
15
|
+
* The mapper receives the framework-native request object (Express `req` /
|
|
16
|
+
* Hono `Context`); Hono users should pass an explicit mapper.
|
|
17
|
+
*/
|
|
18
|
+
export declare function middleware(mapper?: SessionMapper): (a: unknown, b: unknown, c?: unknown) => unknown;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// context.ts -- actor attribution context (SPEC §3.1f, the "who" pillar).
|
|
4
|
+
//
|
|
5
|
+
// AsyncLocalStorage carries {actor, onBehalfOf, kind} across the async chain.
|
|
6
|
+
// The context rides the REPORT only — it is never added to third-party egress
|
|
7
|
+
// (interceptors additionally STRIP any X-Bounded-* header from real egress).
|
|
8
|
+
//
|
|
9
|
+
// runAs() and middleware() must NEVER break the app: mapper/normalization
|
|
10
|
+
// errors fall back to running the wrapped function without context.
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.currentActor = currentActor;
|
|
14
|
+
exports.runAs = runAs;
|
|
15
|
+
exports.middleware = middleware;
|
|
16
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
17
|
+
const state_1 = require("./state");
|
|
18
|
+
const als = new node_async_hooks_1.AsyncLocalStorage();
|
|
19
|
+
const KINDS = ["human", "agent", "service", "unknown"];
|
|
20
|
+
function normalize(input, defaultKind) {
|
|
21
|
+
if (!input || typeof input.actor !== "string" || input.actor.length === 0)
|
|
22
|
+
return undefined;
|
|
23
|
+
const kind = typeof input.kind === "string" && KINDS.includes(input.kind) ? input.kind : defaultKind;
|
|
24
|
+
const ctx = { id: input.actor.slice(0, 256), kind };
|
|
25
|
+
if (typeof input.onBehalfOf === "string" && input.onBehalfOf.length > 0) {
|
|
26
|
+
ctx.onBehalfOf = input.onBehalfOf.slice(0, 256);
|
|
27
|
+
}
|
|
28
|
+
return ctx;
|
|
29
|
+
}
|
|
30
|
+
/** Current actor context, if any (interceptors read this at call time). */
|
|
31
|
+
function currentActor() {
|
|
32
|
+
try {
|
|
33
|
+
return als.getStore();
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Run `fn` attributed to an actor. The context follows the async chain, so
|
|
41
|
+
* every intercepted egress inside (awaited or fire-and-forget started within
|
|
42
|
+
* `fn`) reports this actor. Returns fn's return value; errors propagate
|
|
43
|
+
* unchanged (app semantics are never altered).
|
|
44
|
+
*/
|
|
45
|
+
function runAs(input, fn) {
|
|
46
|
+
let ctx;
|
|
47
|
+
try {
|
|
48
|
+
ctx = normalize(input, "unknown");
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
ctx = undefined;
|
|
52
|
+
}
|
|
53
|
+
return ctx ? als.run(ctx, fn) : fn();
|
|
54
|
+
}
|
|
55
|
+
/** Default session -> actor mapper: duck-types common Express shapes. */
|
|
56
|
+
function defaultMapper(req) {
|
|
57
|
+
const r = req;
|
|
58
|
+
if (!r || typeof r !== "object")
|
|
59
|
+
return null;
|
|
60
|
+
// Explicit escape hatch: req.boundedActor = {actor, kind?, onBehalfOf?} | string
|
|
61
|
+
const explicit = r.boundedActor;
|
|
62
|
+
if (typeof explicit === "string")
|
|
63
|
+
return { actor: explicit };
|
|
64
|
+
if (explicit && typeof explicit.actor === "string")
|
|
65
|
+
return explicit;
|
|
66
|
+
const sessionUser = r.session?.user;
|
|
67
|
+
const user = sessionUser ?? r.user;
|
|
68
|
+
if (user) {
|
|
69
|
+
const id = user.id ?? user.userId ?? user.email;
|
|
70
|
+
if (typeof id === "string" && id.length > 0)
|
|
71
|
+
return { actor: id, kind: "human" };
|
|
72
|
+
}
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Request middleware establishing actor context for the whole request.
|
|
77
|
+
* Express: app.use(bounded.middleware()) — (req, res, next).
|
|
78
|
+
* Hono: app.use(bounded.middleware(mapper)) — (c, next), promise returned.
|
|
79
|
+
* The mapper receives the framework-native request object (Express `req` /
|
|
80
|
+
* Hono `Context`); Hono users should pass an explicit mapper.
|
|
81
|
+
*/
|
|
82
|
+
function middleware(mapper) {
|
|
83
|
+
return function boundedObserveMiddleware(a, b, c) {
|
|
84
|
+
const isExpress = typeof c === "function";
|
|
85
|
+
const next = isExpress
|
|
86
|
+
? c
|
|
87
|
+
: typeof b === "function"
|
|
88
|
+
? b
|
|
89
|
+
: undefined;
|
|
90
|
+
if (!next)
|
|
91
|
+
return undefined; // unrecognized signature: do nothing, never throw
|
|
92
|
+
let ctx;
|
|
93
|
+
try {
|
|
94
|
+
const m = mapper ?? state_1.state.config?.sessionMapper ?? defaultMapper;
|
|
95
|
+
ctx = normalize(m(a) ?? undefined, "human");
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
(0, state_1.debugLog)("middleware mapper failed", err);
|
|
99
|
+
ctx = undefined;
|
|
100
|
+
}
|
|
101
|
+
// next() runs INSIDE the ALS scope so the entire downstream async chain
|
|
102
|
+
// (route handlers, their egress) carries the actor.
|
|
103
|
+
return ctx ? als.run(ctx, () => next()) : next();
|
|
104
|
+
};
|
|
105
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export interface CounterKeyParts {
|
|
2
|
+
host: string;
|
|
3
|
+
method: string;
|
|
4
|
+
pathTemplate: string;
|
|
5
|
+
}
|
|
6
|
+
export interface CounterEmit extends CounterKeyParts {
|
|
7
|
+
windowStart: number;
|
|
8
|
+
windowSeconds: number;
|
|
9
|
+
/** Event ts = window end (or `now` for forced partial flush). */
|
|
10
|
+
ts: number;
|
|
11
|
+
count: number;
|
|
12
|
+
durMsSum: number;
|
|
13
|
+
bytesInSum: number;
|
|
14
|
+
bytesOutSum: number;
|
|
15
|
+
/** Class-representative status: 200 | 400 | 500 | 0. */
|
|
16
|
+
status: number;
|
|
17
|
+
}
|
|
18
|
+
export declare class CounterAggregator {
|
|
19
|
+
private promoteThreshold;
|
|
20
|
+
private entries;
|
|
21
|
+
private promoted;
|
|
22
|
+
constructor(promoteThreshold: number);
|
|
23
|
+
private routeKey;
|
|
24
|
+
isPromoted(p: CounterKeyParts): boolean;
|
|
25
|
+
/** Aggregate one call. `provisional` = heuristic tally (unrecognized GET). */
|
|
26
|
+
add(p: CounterKeyParts, ts: number, status: number, durMs: number, bytesIn: number, bytesOut: number, provisional: boolean): void;
|
|
27
|
+
/**
|
|
28
|
+
* Emit closed windows (or everything when `force`, e.g. shutdown).
|
|
29
|
+
* Provisional tallies below the promote threshold are discarded.
|
|
30
|
+
*/
|
|
31
|
+
flush(now: number, force?: boolean): CounterEmit[];
|
|
32
|
+
/** Test/diagnostic helper. */
|
|
33
|
+
size(): number;
|
|
34
|
+
}
|
package/dist/counters.js
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// counters.ts -- exact per-minute counter aggregation (SPEC §3.2).
|
|
4
|
+
//
|
|
5
|
+
// One counter event per (route, minute-window): exact count + SUMMED
|
|
6
|
+
// dur_ms/bytes + class-representative status. Two entry modes:
|
|
7
|
+
// * forced — manifest `counters` matchers: always aggregate, always emit.
|
|
8
|
+
// * provisional — the unrecognized-GET heuristic: every unrecognized GET is
|
|
9
|
+
// tallied from call #1; at window close, tallies >= threshold emit (exact
|
|
10
|
+
// — nothing was lost) and the route is promoted (sticky) to forced
|
|
11
|
+
// counter mode; tallies below threshold are discarded (those calls are
|
|
12
|
+
// represented by their deduped shape sample only).
|
|
13
|
+
//
|
|
14
|
+
// Windows emit after (window end + grace) so late-processed raw records from
|
|
15
|
+
// the pipeline's tick cadence still land in their window — guaranteeing the
|
|
16
|
+
// one-event-per-(route,window) contract.
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.CounterAggregator = void 0;
|
|
20
|
+
const WINDOW_MS = 60_000;
|
|
21
|
+
/** Late-record grace before a closed window emits (2x max tick cadence). */
|
|
22
|
+
const FLUSH_GRACE_MS = 5_000;
|
|
23
|
+
const MAX_ENTRIES = 2_000;
|
|
24
|
+
const MAX_PROMOTED_ROUTES = 512;
|
|
25
|
+
function statusClassCounts(entry, status) {
|
|
26
|
+
if (status === 0)
|
|
27
|
+
entry.s0++;
|
|
28
|
+
else if (status >= 500)
|
|
29
|
+
entry.s5xx++;
|
|
30
|
+
else if (status >= 400)
|
|
31
|
+
entry.s4xx++;
|
|
32
|
+
else
|
|
33
|
+
entry.s2xx++;
|
|
34
|
+
}
|
|
35
|
+
/** Majority status class; ties break toward the more severe class. */
|
|
36
|
+
function representativeStatus(e) {
|
|
37
|
+
const ranked = [
|
|
38
|
+
[500, e.s5xx],
|
|
39
|
+
[400, e.s4xx],
|
|
40
|
+
[0, e.s0],
|
|
41
|
+
[200, e.s2xx],
|
|
42
|
+
];
|
|
43
|
+
let best = ranked[0];
|
|
44
|
+
for (const r of ranked) {
|
|
45
|
+
if (r[1] > best[1])
|
|
46
|
+
best = r;
|
|
47
|
+
}
|
|
48
|
+
return best[1] === 0 ? 200 : best[0];
|
|
49
|
+
}
|
|
50
|
+
class CounterAggregator {
|
|
51
|
+
promoteThreshold;
|
|
52
|
+
entries = new Map();
|
|
53
|
+
promoted = new Set();
|
|
54
|
+
constructor(promoteThreshold) {
|
|
55
|
+
this.promoteThreshold = promoteThreshold;
|
|
56
|
+
}
|
|
57
|
+
routeKey(p) {
|
|
58
|
+
return `${p.host}|${p.method}|${p.pathTemplate}`;
|
|
59
|
+
}
|
|
60
|
+
isPromoted(p) {
|
|
61
|
+
return this.promoted.has(this.routeKey(p));
|
|
62
|
+
}
|
|
63
|
+
/** Aggregate one call. `provisional` = heuristic tally (unrecognized GET). */
|
|
64
|
+
add(p, ts, status, durMs, bytesIn, bytesOut, provisional) {
|
|
65
|
+
const windowStart = Math.floor(ts / WINDOW_MS) * WINDOW_MS;
|
|
66
|
+
const key = `${this.routeKey(p)}|${windowStart}`;
|
|
67
|
+
let e = this.entries.get(key);
|
|
68
|
+
if (!e) {
|
|
69
|
+
if (this.entries.size >= MAX_ENTRIES)
|
|
70
|
+
return; // cardinality guard
|
|
71
|
+
e = {
|
|
72
|
+
host: p.host,
|
|
73
|
+
method: p.method,
|
|
74
|
+
pathTemplate: p.pathTemplate,
|
|
75
|
+
windowStart,
|
|
76
|
+
count: 0,
|
|
77
|
+
durMsSum: 0,
|
|
78
|
+
bytesInSum: 0,
|
|
79
|
+
bytesOutSum: 0,
|
|
80
|
+
s2xx: 0,
|
|
81
|
+
s4xx: 0,
|
|
82
|
+
s5xx: 0,
|
|
83
|
+
s0: 0,
|
|
84
|
+
provisional,
|
|
85
|
+
};
|
|
86
|
+
this.entries.set(key, e);
|
|
87
|
+
}
|
|
88
|
+
// A forced add upgrades a provisional entry (manifest matcher wins).
|
|
89
|
+
if (!provisional)
|
|
90
|
+
e.provisional = false;
|
|
91
|
+
e.count++;
|
|
92
|
+
e.durMsSum += durMs;
|
|
93
|
+
e.bytesInSum += bytesIn;
|
|
94
|
+
e.bytesOutSum += bytesOut;
|
|
95
|
+
statusClassCounts(e, status);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Emit closed windows (or everything when `force`, e.g. shutdown).
|
|
99
|
+
* Provisional tallies below the promote threshold are discarded.
|
|
100
|
+
*/
|
|
101
|
+
flush(now, force = false) {
|
|
102
|
+
const out = [];
|
|
103
|
+
for (const [key, e] of this.entries) {
|
|
104
|
+
const closed = e.windowStart + WINDOW_MS + FLUSH_GRACE_MS <= now;
|
|
105
|
+
if (!closed && !force)
|
|
106
|
+
continue;
|
|
107
|
+
this.entries.delete(key);
|
|
108
|
+
if (e.provisional) {
|
|
109
|
+
if (e.count < this.promoteThreshold)
|
|
110
|
+
continue; // discard quiet tallies
|
|
111
|
+
// Promote: subsequent windows aggregate from call #1 as forced.
|
|
112
|
+
if (this.promoted.size < MAX_PROMOTED_ROUTES) {
|
|
113
|
+
this.promoted.add(this.routeKey(e));
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const partial = !closed; // force-flushed mid-window
|
|
117
|
+
const windowSeconds = partial
|
|
118
|
+
? Math.min(60, Math.max(1, Math.ceil((now - e.windowStart) / 1000)))
|
|
119
|
+
: 60;
|
|
120
|
+
out.push({
|
|
121
|
+
host: e.host,
|
|
122
|
+
method: e.method,
|
|
123
|
+
pathTemplate: e.pathTemplate,
|
|
124
|
+
windowStart: e.windowStart,
|
|
125
|
+
windowSeconds,
|
|
126
|
+
ts: partial ? now : e.windowStart + WINDOW_MS,
|
|
127
|
+
count: e.count,
|
|
128
|
+
durMsSum: e.durMsSum,
|
|
129
|
+
bytesInSum: e.bytesInSum,
|
|
130
|
+
bytesOutSum: e.bytesOutSum,
|
|
131
|
+
status: representativeStatus(e),
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return out;
|
|
135
|
+
}
|
|
136
|
+
/** Test/diagnostic helper. */
|
|
137
|
+
size() {
|
|
138
|
+
return this.entries.size;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
exports.CounterAggregator = CounterAggregator;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** True if a single field-name segment is PII-shaped (hard-denied). */
|
|
2
|
+
export declare function isDeniedSegment(segment: string): boolean;
|
|
3
|
+
/**
|
|
4
|
+
* True if ANY segment of a dot-path field reference is PII-shaped.
|
|
5
|
+
* "usage.total_tokens" -> false; "customer.email" -> true; "card.last4" -> true.
|
|
6
|
+
*/
|
|
7
|
+
export declare function isDeniedFieldPath(path: string): boolean;
|
package/dist/denylist.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// denylist.ts -- L2 HARD PII denylist (SPEC §3.7 L2).
|
|
4
|
+
//
|
|
5
|
+
// PII-shaped field names can NEVER enter capture, even if a (bad or
|
|
6
|
+
// malicious) manifest lists them in requestFields/responseFields. This list
|
|
7
|
+
// is compiled into the shim and is not configurable at runtime — the manifest
|
|
8
|
+
// linter (registry CI) is the other half of L2; this is the fail-safe half.
|
|
9
|
+
//
|
|
10
|
+
// Matching is applied to EVERY segment of a field dot-path (e.g.
|
|
11
|
+
// "customer.email" is blocked because the segment "email" matches).
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.isDeniedSegment = isDeniedSegment;
|
|
15
|
+
exports.isDeniedFieldPath = isDeniedFieldPath;
|
|
16
|
+
const DENY_PATTERNS = [
|
|
17
|
+
/e[-_]?mail/i, // email, e_mail, customerEmail
|
|
18
|
+
/phone/i,
|
|
19
|
+
/mobile/i,
|
|
20
|
+
/ssn/i,
|
|
21
|
+
/social[-_]?security/i,
|
|
22
|
+
/passport/i,
|
|
23
|
+
/driver[-_]?licen[cs]e/i,
|
|
24
|
+
/tax[-_]?id/i,
|
|
25
|
+
/(?:^|_)(?:vat)(?:_|$)/i,
|
|
26
|
+
/passw(?:or)?d/i,
|
|
27
|
+
/(?:^|_)pwd(?:_|$)/i,
|
|
28
|
+
/secret/i,
|
|
29
|
+
/credential/i,
|
|
30
|
+
/private[-_]?key/i,
|
|
31
|
+
/api[-_]?key/i,
|
|
32
|
+
/(?:^|_)key$/i, // api_key, session_key — but NOT "monkey" etc. (segment-suffix)
|
|
33
|
+
/token$/i, // access_token, id_token — NOT prompt_tokens / total_tokens
|
|
34
|
+
/authorization/i,
|
|
35
|
+
/(?:^|_)auth(?:_|$)/i,
|
|
36
|
+
/bearer/i,
|
|
37
|
+
/(?:^|_)jwt(?:_|$)/i,
|
|
38
|
+
/cookie/i,
|
|
39
|
+
/session[-_]?id/i,
|
|
40
|
+
/card/i, // card, card_number, cardNumber, card_country (safe-side)
|
|
41
|
+
/cvv|cvc|cvn/i,
|
|
42
|
+
/(?:^|_)pan(?:_|$)/i,
|
|
43
|
+
/pin[-_]?code|(?:^|_)pin(?:_|$)/i,
|
|
44
|
+
/account[-_]?number/i,
|
|
45
|
+
/iban/i,
|
|
46
|
+
/(?:^|_)bic(?:_|$)/i,
|
|
47
|
+
/swift/i,
|
|
48
|
+
/routing/i,
|
|
49
|
+
/(?:^|_)dob(?:_|$)/i,
|
|
50
|
+
/birth/i,
|
|
51
|
+
/address/i, // street/billing/ip_address — safe-side
|
|
52
|
+
/street/i,
|
|
53
|
+
/(?:^|_)zip(?:_|$)|postal/i,
|
|
54
|
+
/(?:first|last|full|middle|maiden|user|customer|holder|given|family)[-_]?name/i,
|
|
55
|
+
/(?:^|_)name$/i, // bare "name" — could be a person (safe-side); "model" etc. unaffected
|
|
56
|
+
/gender/i,
|
|
57
|
+
/nationality/i,
|
|
58
|
+
/biometric/i,
|
|
59
|
+
/fingerprint/i,
|
|
60
|
+
/(?:^|_)ip(?:_|$)/i,
|
|
61
|
+
/latitude|longitude|geo[-_]?loc/i,
|
|
62
|
+
];
|
|
63
|
+
/** True if a single field-name segment is PII-shaped (hard-denied). */
|
|
64
|
+
function isDeniedSegment(segment) {
|
|
65
|
+
for (const re of DENY_PATTERNS) {
|
|
66
|
+
if (re.test(segment))
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* True if ANY segment of a dot-path field reference is PII-shaped.
|
|
73
|
+
* "usage.total_tokens" -> false; "customer.email" -> true; "card.last4" -> true.
|
|
74
|
+
*/
|
|
75
|
+
function isDeniedFieldPath(path) {
|
|
76
|
+
for (const seg of path.split(".")) {
|
|
77
|
+
if (isDeniedSegment(seg))
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/** Current envelope version (== observe-shared EVENT_SCHEMA_VERSION). */
|
|
2
|
+
export declare const EVENT_SCHEMA_VERSION: 2;
|
|
3
|
+
/** Versions the ingest still accepts (== observe-shared SUPPORTED_SCHEMA_VERSIONS). */
|
|
4
|
+
export type EventSchemaVersion = 1 | 2;
|
|
5
|
+
/** Max events per POST /v1/events batch (== observe-shared MAX_BATCH_EVENTS). */
|
|
6
|
+
export declare const MAX_BATCH_EVENTS = 500;
|
|
7
|
+
export type EventClass = "action" | "counter" | "shape" | "error";
|
|
8
|
+
export type ActorKind = "human" | "agent" | "service" | "unknown";
|
|
9
|
+
export type ActorGrade = "asserted" | "attested";
|
|
10
|
+
export interface EventActor {
|
|
11
|
+
id: string;
|
|
12
|
+
kind: ActorKind;
|
|
13
|
+
grade: ActorGrade;
|
|
14
|
+
}
|
|
15
|
+
export interface EventDest {
|
|
16
|
+
host: string;
|
|
17
|
+
/** IDs stripped to templates SHIM-SIDE, e.g. "/v1/charges/{id}/refunds". */
|
|
18
|
+
pathTemplate: string;
|
|
19
|
+
method: string;
|
|
20
|
+
}
|
|
21
|
+
export interface EventBytes {
|
|
22
|
+
i: number;
|
|
23
|
+
o: number;
|
|
24
|
+
}
|
|
25
|
+
/** v2 rec.fields value: primitive leaves only (one level, no nesting). */
|
|
26
|
+
export type RecFieldValue = string | number | boolean;
|
|
27
|
+
export type RecFields = Record<string, RecFieldValue>;
|
|
28
|
+
export interface EventRec {
|
|
29
|
+
rail: string;
|
|
30
|
+
action: string;
|
|
31
|
+
registryVersion: string;
|
|
32
|
+
/** v2: recognizer-extracted SAFE values (amounts in cents, ids, models,
|
|
33
|
+
* token counts). One level, primitives only. Denylist-filtered at extraction
|
|
34
|
+
* (L1/L2); ingest re-checks KEYS (L2) and scrubs VALUES (L4). */
|
|
35
|
+
fields?: RecFields;
|
|
36
|
+
}
|
|
37
|
+
export interface EventWindow {
|
|
38
|
+
start: number;
|
|
39
|
+
seconds: number;
|
|
40
|
+
}
|
|
41
|
+
export interface ShapeField {
|
|
42
|
+
name: string;
|
|
43
|
+
type: string;
|
|
44
|
+
}
|
|
45
|
+
export interface EventShape {
|
|
46
|
+
fields: ShapeField[];
|
|
47
|
+
}
|
|
48
|
+
/** The v1 observe event envelope (FROZEN — see header). */
|
|
49
|
+
export interface ObserveEventV1 {
|
|
50
|
+
v: EventSchemaVersion;
|
|
51
|
+
ts: number;
|
|
52
|
+
/** SERVER-AUTHORITATIVE (stamped from the sensor key); shims omit it. */
|
|
53
|
+
org: string;
|
|
54
|
+
/** SERVER-AUTHORITATIVE (stamped from the sensor key); shims omit it. */
|
|
55
|
+
sensor: string;
|
|
56
|
+
class: EventClass;
|
|
57
|
+
actor?: EventActor;
|
|
58
|
+
onBehalfOf?: string;
|
|
59
|
+
dest: EventDest;
|
|
60
|
+
status: number;
|
|
61
|
+
dur_ms: number;
|
|
62
|
+
bytes: EventBytes;
|
|
63
|
+
rec?: EventRec;
|
|
64
|
+
policyVersion?: string;
|
|
65
|
+
dropped?: number;
|
|
66
|
+
count?: number;
|
|
67
|
+
window?: EventWindow;
|
|
68
|
+
shape?: EventShape;
|
|
69
|
+
/** SERVER-SET by the ingest L4 scrubber; shims NEVER send it. */
|
|
70
|
+
scrubbed?: boolean;
|
|
71
|
+
}
|
package/dist/envelope.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// envelope.ts -- FROZEN observe event envelope v1, mirrored.
|
|
4
|
+
//
|
|
5
|
+
// SOURCE OF TRUTH: packages/cdk/cloudflare/observe-shared/src/index.ts
|
|
6
|
+
// (EVENT_SCHEMA_VERSION = 1). Mirrored here so the shim builds hermetically
|
|
7
|
+
// (dependency-free dist, no cross-package rootDir coupling). Any drift fails
|
|
8
|
+
// CI: test/envelope-contract.test.ts asserts bidirectional type-assignability
|
|
9
|
+
// against the real @bounded-sh/observe-shared source AND validates every
|
|
10
|
+
// pipeline-emitted event with the shared validateAndFilterEvent().
|
|
11
|
+
//
|
|
12
|
+
// If observe-shared bumps EVENT_SCHEMA_VERSION, update this file with it and
|
|
13
|
+
// follow the CHANGELOG discipline in the observe-shared README.
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.MAX_BATCH_EVENTS = exports.EVENT_SCHEMA_VERSION = void 0;
|
|
17
|
+
/** Current envelope version (== observe-shared EVENT_SCHEMA_VERSION). */
|
|
18
|
+
exports.EVENT_SCHEMA_VERSION = 2;
|
|
19
|
+
/** Max events per POST /v1/events batch (== observe-shared MAX_BATCH_EVENTS). */
|
|
20
|
+
exports.MAX_BATCH_EVENTS = 500;
|