@bounded-sh/observe 0.1.0

package/dist/reporter.js

@@ -0,0 +1,210 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// reporter.ts -- async batch reporting (SPEC §3.1a).
+//
+// Fire-and-forget POST {events} to <ingestBase>/v1/events with the sensor
+// token; batches of <= 500 (MAX_BATCH_EVENTS), 2s cadence (driven by the
+// shared tick in index.ts) or immediately at 500 queued. Bounded memory:
+// queue overflow drops the newest event and counts it; drops are reported on
+// the first event of the next successful batch via `dropped` (honest counts).
+// One retry for retryable failures (network/timeout/429/5xx); terminal
+// failures (401/400/413) drop + count. Uses the ORIGINAL pre-patch fetch so
+// reporting is never itself captured.
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Reporter = void 0;
+const state_1 = require("./state");
+const MAX_BATCH = 500; // == observe-shared MAX_BATCH_EVENTS (asserted in tests)
+class Reporter {
+    maxQueue;
+    sendTimeoutMs;
+    queue = [];
+    retry = null;
+    /** In-flight send (single-flight). drain() AWAITS it — never abandons it. */
+    inFlight = null;
+    immediateScheduled = false;
+    /** Drops awaiting report on the next batch's first event. */
+    pendingDrops = 0;
+    totalDropped = 0;
+    sentEvents = 0;
+    sentBatches = 0;
+    sendFailures = 0;
+    constructor(maxQueue, sendTimeoutMs) {
+        this.maxQueue = maxQueue;
+        this.sendTimeoutMs = sendTimeoutMs;
+    }
+    get queueDepth() {
+        return this.queue.length + (this.retry?.events.length ?? 0);
+    }
+    countDrop(n = 1) {
+        this.pendingDrops += n;
+        this.totalDropped += n;
+    }
+    push(ev) {
+        if (this.queue.length >= this.maxQueue) {
+            this.countDrop(1);
+            return;
+        }
+        this.queue.push(ev);
+        try {
+            state_1.state.config?.onEvent?.(ev.wire, ev.fields);
+        }
+        catch {
+            /* debug hook errors never propagate */
+        }
+        if (this.queue.length >= MAX_BATCH && !this.immediateScheduled) {
+            this.immediateScheduled = true;
+            const t = setTimeout(() => {
+                this.immediateScheduled = false;
+                void this.flushOnce();
+            }, 0);
+            t.unref?.();
+        }
+    }
+    /** Send at most one batch. Single-flight; never throws. */
+    async flushOnce() {
+        if (this.inFlight)
+            return;
+        if (!state_1.state.enabled)
+            return;
+        const run = (async () => {
+            try {
+                await this.sendNext();
+            }
+            catch (err) {
+                (0, state_1.noteInternalError)("reporter.flushOnce", err);
+            }
+        })();
+        this.inFlight = run;
+        try {
+            await run;
+        }
+        finally {
+            this.inFlight = null;
+        }
+    }
+    /** Drain everything (graceful shutdown / tests). WAITS for any in-flight
+     * send first — a shutdown never abandons a batch mid-air. Never throws. */
+    async drain() {
+        // Wait out in-flight sends (bounded: each send has its own timeout).
+        let waitGuard = 8;
+        while (this.inFlight && waitGuard-- > 0) {
+            try {
+                await this.inFlight;
+            }
+            catch {
+                /* sendNext never throws, belt-and-suspenders */
+            }
+        }
+        const run = (async () => {
+            try {
+                // Bounded loop: at most queue/MAX_BATCH + retries iterations.
+                let guard = Math.ceil(this.maxQueue / MAX_BATCH) + 4;
+                while ((this.queue.length > 0 || this.retry) && guard-- > 0) {
+                    const before = this.queueDepth;
+                    await this.sendNext();
+                    if (this.queueDepth >= before)
+                        break; // no progress (e.g. offline)
+                }
+            }
+            catch (err) {
+                (0, state_1.noteInternalError)("reporter.drain", err);
+            }
+        })();
+        this.inFlight = run;
+        try {
+            await run;
+        }
+        finally {
+            this.inFlight = null;
+        }
+    }
+    takeBatch() {
+        if (this.retry) {
+            const r = this.retry;
+            this.retry = null;
+            return r;
+        }
+        if (this.queue.length === 0)
+            return null;
+        const items = this.queue.splice(0, MAX_BATCH);
+        const events = items.map((i) => i.wire);
+        // Report accumulated drops on the first event of this batch.
+        let claimedDrops = 0;
+        if (this.pendingDrops > 0) {
+            claimedDrops = this.pendingDrops;
+            this.pendingDrops = 0;
+            events[0] = { ...events[0], dropped: claimedDrops };
+        }
+        return { events, attempts: 0, claimedDrops };
+    }
+    async sendNext() {
+        const batch = this.takeBatch();
+        if (!batch)
+            return;
+        const cfg = state_1.state.config;
+        const doFetch = state_1.state.originalFetch;
+        if (!cfg || !doFetch) {
+            this.countDrop(batch.events.length);
+            this.pendingDrops += batch.claimedDrops;
+            return;
+        }
+        let status = 0;
+        try {
+            const ac = new AbortController();
+            const timer = setTimeout(() => ac.abort(), this.sendTimeoutMs);
+            timer.unref?.();
+            try {
+                const res = await doFetch(`${cfg.ingestBase.replace(/\/$/, "")}/v1/events`, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${cfg.token}`,
+                    },
+                    body: JSON.stringify({ events: batch.events }),
+                    signal: ac.signal,
+                });
+                status = res.status;
+                if (res.ok) {
+                    this.sentEvents += batch.events.length;
+                    this.sentBatches++;
+                    (0, state_1.debugLog)(`reported ${batch.events.length} events (${status})`);
+                    // Drain the body so the socket is reusable; response is small.
+                    try {
+                        await res.text();
+                    }
+                    catch {
+                        /* ignore */
+                    }
+                    return;
+                }
+                try {
+                    await res.text();
+                }
+                catch {
+                    /* ignore */
+                }
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }
+        catch {
+            status = 0; // network error / timeout
+        }
+        this.sendFailures++;
+        const retryable = status === 0 || status === 408 || status === 429 || status >= 500;
+        if (retryable && batch.attempts < 1) {
+            this.retry = { events: batch.events, attempts: batch.attempts + 1, claimedDrops: batch.claimedDrops };
+            (0, state_1.debugLog)(`send failed (${status}); will retry once`);
+            return;
+        }
+        // Terminal: drop the batch and report it honestly on the next success.
+        // claimedDrops rode this batch's first event and never landed — re-claim
+        // them (already counted in totalDropped when first dropped).
+        this.countDrop(batch.events.length);
+        this.pendingDrops += batch.claimedDrops;
+        (0, state_1.debugLog)(`send failed terminally (${status}); dropped ${batch.events.length} events`);
+    }
+}
+exports.Reporter = Reporter;

package/dist/shapes.d.ts

@@ -0,0 +1,15 @@
+export declare class ShapeDeduper {
+    private maxPerMinute;
+    private known;
+    private windowStart;
+    private emittedThisWindow;
+    constructor(maxPerMinute: number);
+    /** True if this route's shape was already sampled (cheap pre-check). */
+    hasSeen(host: string, method: string, pathTemplate: string): boolean;
+    /**
+     * Returns true exactly once per route (and only under the rate cap) —
+     * the caller emits the shape sample on true.
+     */
+    shouldEmit(host: string, method: string, pathTemplate: string, now: number): boolean;
+    size(): number;
+}

package/dist/shapes.js

@@ -0,0 +1,56 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// shapes.ts -- shape-sample dedupe + rate cap (SPEC §3.2).
+//
+// Unrecognized routes emit ONE shape sample per (host, method, pathTemplate)
+// per process (in-memory LRU dedupe), rate-capped per minute. Field NAMES +
+// types only ride the sample; repeat traffic on a known shape emits nothing
+// (high-volume GETs graduate to counter aggregation instead — counters.ts).
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ShapeDeduper = void 0;
+const MAX_KNOWN_SHAPES = 1_024;
+class ShapeDeduper {
+    maxPerMinute;
+    known = new Map(); // Map preserves insert order => LRU-ish eviction
+    windowStart = 0;
+    emittedThisWindow = 0;
+    constructor(maxPerMinute) {
+        this.maxPerMinute = maxPerMinute;
+    }
+    /** True if this route's shape was already sampled (cheap pre-check). */
+    hasSeen(host, method, pathTemplate) {
+        return this.known.has(`${host}|${method}|${pathTemplate}`);
+    }
+    /**
+     * Returns true exactly once per route (and only under the rate cap) —
+     * the caller emits the shape sample on true.
+     */
+    shouldEmit(host, method, pathTemplate, now) {
+        const key = `${host}|${method}|${pathTemplate}`;
+        if (this.known.has(key))
+            return false;
+        // Rate cap: max N NEW shapes per minute; overflow shapes are simply not
+        // recorded as known, so they get another chance in a later window.
+        const win = Math.floor(now / 60_000);
+        if (win !== this.windowStart) {
+            this.windowStart = win;
+            this.emittedThisWindow = 0;
+        }
+        if (this.emittedThisWindow >= this.maxPerMinute)
+            return false;
+        this.emittedThisWindow++;
+        if (this.known.size >= MAX_KNOWN_SHAPES) {
+            // Evict the oldest entry (first key in insertion order).
+            const oldest = this.known.keys().next().value;
+            if (oldest !== undefined)
+                this.known.delete(oldest);
+        }
+        this.known.set(key, true);
+        return true;
+    }
+    size() {
+        return this.known.size;
+    }
+}
+exports.ShapeDeduper = ShapeDeduper;

package/dist/state.d.ts

@@ -0,0 +1,32 @@
+import type { ObserveConfig } from "./types";
+import { CompiledManifest } from "./manifest";
+import type { Pipeline } from "./pipeline";
+import type { Reporter } from "./reporter";
+export declare const KILL_SWITCH_ENV = "BOUNDED_OBSERVE_DISABLED";
+export interface ShimStats {
+    captured: number;
+    internalErrors: number;
+}
+export interface ShimState {
+    initialized: boolean;
+    /** Master gate consulted by every interceptor on every call. */
+    enabled: boolean;
+    patched: boolean;
+    config?: ObserveConfig;
+    /** Host (host[:port]) of the ingest endpoint — self-traffic is never captured. */
+    ingestHost?: string;
+    /** The pre-patch fetch: used for reporting + manifest polling (no self-loop). */
+    originalFetch?: typeof globalThis.fetch;
+    manifest: CompiledManifest;
+    pipeline?: Pipeline;
+    reporter?: Reporter;
+    tickTimer?: NodeJS.Timeout;
+    manifestTimer?: NodeJS.Timeout;
+    stats: ShimStats;
+}
+export declare const state: ShimState;
+export declare function killSwitchActive(): boolean;
+/** Debug logger — never throws, never logs unless debug is on. */
+export declare function debugLog(msg: string, extra?: unknown): void;
+/** Count an internal shim error (never propagates to the app). */
+export declare function noteInternalError(where: string, err: unknown): void;

package/dist/state.js

@@ -0,0 +1,42 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// state.ts -- module singleton state for the shim. One shim per process.
+//
+// Kept as a plain mutable object so the interceptors' hot path reads are a
+// property access + boolean check, nothing more.
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.state = exports.KILL_SWITCH_ENV = void 0;
+exports.killSwitchActive = killSwitchActive;
+exports.debugLog = debugLog;
+exports.noteInternalError = noteInternalError;
+const manifest_1 = require("./manifest");
+exports.KILL_SWITCH_ENV = "BOUNDED_OBSERVE_DISABLED";
+exports.state = {
+    initialized: false,
+    enabled: false,
+    patched: false,
+    manifest: new manifest_1.CompiledManifest(manifest_1.BUILTIN_MANIFEST, "builtin"),
+    stats: { captured: 0, internalErrors: 0 },
+};
+function killSwitchActive() {
+    const v = process.env[exports.KILL_SWITCH_ENV];
+    return v === "1" || v === "true" || v === "yes";
+}
+/** Debug logger — never throws, never logs unless debug is on. */
+function debugLog(msg, extra) {
+    try {
+        if (exports.state.config?.debug) {
+            // eslint-disable-next-line no-console
+            console.error(`[bounded-observe] ${msg}`, extra !== undefined ? extra : "");
+        }
+    }
+    catch {
+        /* never */
+    }
+}
+/** Count an internal shim error (never propagates to the app). */
+function noteInternalError(where, err) {
+    exports.state.stats.internalErrors++;
+    debugLog(`internal error in ${where}`, err);
+}

package/dist/template.d.ts

@@ -0,0 +1,7 @@
+/** True if one path segment looks like an ID and should template to {id}. */
+export declare function isIdSegment(segment: string): boolean;
+/**
+ * Template a raw path: ID-looking segments become {id}. The input must
+ * already be query-less (interceptors split query before calling).
+ */
+export declare function templatePath(path: string): string;

package/dist/template.js

@@ -0,0 +1,63 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// template.ts -- path templating (SPEC §3.2 / L1).
+//
+// IDs are stripped into {id} segments SHIM-SIDE, before anything is reported:
+// ingest never sees raw path IDs. Query strings are dropped entirely by the
+// interceptors (names may ride shape samples; values never leave the process).
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isIdSegment = isIdSegment;
+exports.templatePath = templatePath;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const NUMERIC_RE = /^\d+$/;
+const HEX_RE = /^[0-9a-f]{16,}$/i;
+// Vendor-prefixed opaque ids: ch_3Nq1..., cus_ABC123, pi_..., in_..., acct_...
+// The suffix must contain a digit — API resource words ("payment_intents",
+// "chat_completions") never do, real vendor ids essentially always do.
+const PREFIXED_ID_RE = /^[a-z]{1,8}_(?=[A-Za-z0-9]*\d)[A-Za-z0-9]{6,}$/;
+// Long random tokens (ULIDs, base64url ids, ...): letters+digits, len >= 20.
+const LONG_RANDOM_RE = /^[A-Za-z0-9_-]{20,}$/;
+const HAS_DIGIT_RE = /\d/;
+const HAS_ALPHA_RE = /[A-Za-z]/;
+// Trailing file extension keeps static assets recognizable (/app.3f9c2b.js).
+const MONGO_OBJECTID_RE = /^[0-9a-f]{24}$/i;
+/** True if one path segment looks like an ID and should template to {id}. */
+function isIdSegment(segment) {
+    if (segment.length === 0)
+        return false;
+    if (NUMERIC_RE.test(segment))
+        return true;
+    if (UUID_RE.test(segment))
+        return true;
+    if (MONGO_OBJECTID_RE.test(segment))
+        return true;
+    if (HEX_RE.test(segment))
+        return true;
+    if (PREFIXED_ID_RE.test(segment))
+        return true;
+    if (LONG_RANDOM_RE.test(segment) && HAS_DIGIT_RE.test(segment) && HAS_ALPHA_RE.test(segment)) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Template a raw path: ID-looking segments become {id}. The input must
+ * already be query-less (interceptors split query before calling).
+ */
+function templatePath(path) {
+    if (path === "" || path === "/")
+        return "/";
+    const parts = path.split("/");
+    let changed = false;
+    for (let i = 0; i < parts.length; i++) {
+        const seg = parts[i];
+        if (seg && isIdSegment(seg)) {
+            parts[i] = "{id}";
+            changed = true;
+        }
+    }
+    const out = changed ? parts.join("/") : path;
+    // Envelope caps pathTemplate at 1024 chars.
+    return out.length > 1024 ? out.slice(0, 1024) : out;
+}

package/dist/types.d.ts

@@ -0,0 +1,125 @@
+import type { ObserveEventV1, ActorKind, RecFields } from "./envelope";
+import type { EscortPin } from "./manifest";
+/** What the shim actually puts on the wire: envelope v1 minus org/sensor. */
+export type WireEvent = Omit<ObserveEventV1, "org" | "sensor">;
+/** Actor context input for runAs() / middleware mappers. */
+export interface ActorInput {
+    actor: string;
+    onBehalfOf?: string;
+    kind?: ActorKind;
+}
+/** Normalized actor context carried by AsyncLocalStorage. */
+export interface ActorContext {
+    id: string;
+    kind: ActorKind;
+    onBehalfOf?: string;
+}
+/**
+ * One raw capture from an interceptor. Hot-path cheap: refs + numbers only.
+ * All heavy work (templating, recognition, body parsing, dedupe, counters)
+ * happens later in the pipeline tick, off the request path.
+ */
+export interface RawCapture {
+    /** Request start, epoch ms. */
+    ts: number;
+    /** Effective destination host (after aliasHosts mapping), incl. port. */
+    host: string;
+    /** Raw URL path WITHOUT query string. */
+    path: string;
+    /** Query parameter NAMES only (values are never captured). */
+    queryNames?: string[];
+    /** Uppercase method. */
+    method: string;
+    /** HTTP status; 0 = network error / never completed. */
+    status: number;
+    durMs: number;
+    bytesIn: number;
+    bytesOut: number;
+    /** Actor context at call initiation (ALS wins over header fallback). */
+    actor?: ActorContext;
+    /** Raw request body text (capped) — manifest hosts / shape sampling only. */
+    reqBodyText?: string;
+    reqContentType?: string;
+    /** Parsed response JSON (recognized routes with responseFields only). */
+    respJson?: unknown;
+}
+/**
+ * A processed event awaiting report. `fields` holds manifest-recognized SAFE
+ * values (amounts in cents, opaque ids, model names, token counts). Envelope
+ * v1 has no slot for them, so they are NEVER serialized to the wire — they
+ * are exposed via the onEvent debug hook and reserved for envelope v2.
+ */
+export interface InternalEvent {
+    wire: WireEvent;
+    fields?: RecFields;
+}
+/** Session -> actor mapper for middleware(). Receives the framework-native
+ * request object (Express `req` or Hono `Context`). Return null to leave the
+ * request unattributed. */
+export type SessionMapper = (req: unknown) => ActorInput | null | undefined;
+export interface ObserveConfig {
+    /** Ingest base URL, e.g. https://observe-ingest.buildwithtarobase.workers.dev */
+    ingestBase: string;
+    /** Sensor token: obs1.<keyId>.<sig> */
+    token: string;
+    /** Org slug — used ONLY for manifest polling (org/sensor are server-stamped). */
+    org?: string;
+    /** Informational; the wire sensor identity comes from the token. */
+    sensor?: string;
+    /** Session -> actor mapper used by middleware() when no explicit mapper given. */
+    sessionMapper?: SessionMapper;
+    /**
+     * Map local/mock hosts to their canonical identity for recognition +
+     * reporting (e.g. {"127.0.0.1:4242": "api.stripe.com"}). Dev/demo aid.
+     */
+    aliasHosts?: Record<string, string>;
+    /** Flush cadence, ms (default 2000). Batches also flush at 500 events. */
+    flushIntervalMs?: number;
+    /** Per-send timeout, ms (default 5000). */
+    sendTimeoutMs?: number;
+    /** Bounded report queue (default 5000 events). Overflow -> dropped counter. */
+    maxQueueEvents?: number;
+    /** Unrecognized-GET calls/min above which a route becomes counter-class (default 60). */
+    counterPromoteThreshold?: number;
+    /** Max new shape samples emitted per minute (default 30). */
+    shapesPerMinute?: number;
+    /** Manifest override (tests/dev). Skips the built-in + polling when set. */
+    manifest?: unknown;
+    /**
+     * Env-derived escort pin (set by the register preload from
+     * BOUNDED_VERDICT_URL). Applied over the initial manifest AND re-applied
+     * over every fetched manifest swap, so a served manifest can never silently
+     * drop the pinned enforcement. A fetched escort block on the pinned route
+     * wins over the pin (server-side promotion supersedes the boot-time env
+     * default).
+     */
+    escortPin?: EscortPin;
+    /** Disable the 60s manifest poll (tests). */
+    disableManifestPoll?: boolean;
+    /** Manifest poll interval ms (default 60000). */
+    manifestPollMs?: number;
+    /** Debug logging to stderr (never throws). */
+    debug?: boolean;
+    /**
+     * Debug/test hook: called for every event as it is queued, with the wire
+     * event and the (never-transmitted) recognized fields. Exceptions ignored.
+     */
+    onEvent?: (wire: WireEvent, fields?: RecFields) => void;
+}
+export interface Diagnostics {
+    enabled: boolean;
+    patched: boolean;
+    queueDepth: number;
+    rawQueueDepth: number;
+    /** Drops not yet reported on a wire event. */
+    pendingDrops: number;
+    totalDropped: number;
+    captured: number;
+    sentEvents: number;
+    sentBatches: number;
+    sendFailures: number;
+    /** Internal shim errors swallowed (never propagated to the app). */
+    internalErrors: number;
+    manifestVersion: string;
+    manifestSource: "builtin" | "override" | "fetched";
+}

package/dist/types.js

@@ -0,0 +1,10 @@
+"use strict";
+// ---------------------------------------------------------------------------
+// types.ts -- internal + public types for @bounded-sh/observe.
+//
+// The wire contract is the FROZEN envelope v1 from
+// packages/cdk/cloudflare/observe-shared, mirrored in ./envelope (drift is
+// CI-checked; see envelope.ts header). org/sensor are SERVER-AUTHORITATIVE
+// (stamped from the sensor key), so the shim omits them entirely.
+// ---------------------------------------------------------------------------
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@bounded-sh/observe",
+  "version": "0.1.0",
+  "description": "Bounded observe shim for Node: intercepts fetch + http/https egress, attributes actors, reports metadata-only events to the Bounded observe ingest. Never breaks the app.",
+  "license": "MIT",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js",
+      "import": "./dist/index.js"
+    },
+    "./register": {
+      "types": "./dist/register.d.ts",
+      "require": "./dist/register.js",
+      "import": "./dist/register.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run",
+    "test:live": "node scripts/live-test.mjs"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.8.0",
+    "vitest": "^3.2.0"
+  }
+}