@agenthoney/analytics 0.0.0-stage → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +85 -2
- package/dist/answers.cjs +20034 -0
- package/dist/answers.cjs.map +1 -0
- package/dist/answers.d.ts +148 -0
- package/dist/answers.js +139 -0
- package/dist/answers.js.map +1 -0
- package/dist/chunk-CD4WLJX7.js +48 -0
- package/dist/chunk-CD4WLJX7.js.map +1 -0
- package/dist/chunk-F3PRHEXB.js +20143 -0
- package/dist/chunk-F3PRHEXB.js.map +1 -0
- package/dist/chunk-L22VERBM.js +911 -0
- package/dist/chunk-L22VERBM.js.map +1 -0
- package/dist/chunk-OH4H2B7O.js +150 -0
- package/dist/chunk-OH4H2B7O.js.map +1 -0
- package/dist/chunk-R76CTIBG.js +701 -0
- package/dist/chunk-R76CTIBG.js.map +1 -0
- package/dist/chunk-UG3REZCJ.js +147 -0
- package/dist/chunk-UG3REZCJ.js.map +1 -0
- package/dist/core/breaker.d.ts +33 -0
- package/dist/core/collector.d.ts +51 -0
- package/dist/core/config.d.ts +124 -0
- package/dist/core/encode.d.ts +32 -0
- package/dist/core/queue.d.ts +39 -0
- package/dist/core/record-gate.d.ts +17 -0
- package/dist/core/safe.d.ts +17 -0
- package/dist/core/transport.d.ts +45 -0
- package/dist/express.cjs +21789 -0
- package/dist/express.cjs.map +1 -0
- package/dist/express.d.ts +65 -0
- package/dist/express.js +6 -0
- package/dist/express.js.map +1 -0
- package/dist/index.cjs +22118 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +51 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/next.cjs +21186 -0
- package/dist/next.cjs.map +1 -0
- package/dist/next.d.ts +90 -0
- package/dist/next.js +5 -0
- package/dist/next.js.map +1 -0
- package/dist/observe/client-ip.d.ts +109 -0
- package/dist/observe/next-router.d.ts +22 -0
- package/dist/observe/redact.d.ts +58 -0
- package/dist/observe/request.d.ts +75 -0
- package/dist/observe/response.d.ts +24 -0
- package/dist/runtime.d.ts +27 -0
- package/dist/serve/accept.d.ts +7 -0
- package/dist/serve/discovery.d.ts +56 -0
- package/dist/serve/hosted.d.ts +135 -0
- package/dist/serve/source.d.ts +48 -0
- package/dist/serve/tag-asset.generated.d.ts +14 -0
- package/dist/serve/tag.d.ts +131 -0
- package/dist/serve/twin.d.ts +162 -0
- package/dist/web.cjs +21225 -0
- package/dist/web.cjs.map +1 -0
- package/dist/web.d.ts +52 -0
- package/dist/web.js +6 -0
- package/dist/web.js.map +1 -0
- package/install.md +463 -0
- package/package.json +76 -4
package/dist/next.d.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { type Collector } from "./core/collector.js";
|
|
2
|
+
import { type AgentHoneyConfig } from "./core/config.js";
|
|
3
|
+
import { advertiseHeader, type TwinOptions } from "./serve/twin.js";
|
|
4
|
+
/**
|
|
5
|
+
* The Next.js adapter, for `proxy.ts` (called `middleware.ts` before Next 16).
|
|
6
|
+
*
|
|
7
|
+
* ── ⚠️ Why this exists rather than "just use ./web" ──────────────────────────
|
|
8
|
+
* Because `./web` produces FALSE DATA in a Next proxy, and does it silently.
|
|
9
|
+
*
|
|
10
|
+
* A proxy runs **before the route**. It hands control onward by returning
|
|
11
|
+
* `NextResponse.next()` — a sentinel with status 200, no real content-type and
|
|
12
|
+
* no content-length. `./web` reads that sentinel and records
|
|
13
|
+
* `{ status: 200, observation: "measured" }`, so **every event claims a
|
|
14
|
+
* measured 200**, including requests the route renders as 404 or 500.
|
|
15
|
+
*
|
|
16
|
+
* ⚠️ **Our own schema already said this was wrong and refused it**:
|
|
17
|
+
* `observation: "unknown"` may not carry a status, and the Next case is the one
|
|
18
|
+
* where the `response` object is absent entirely — "we never had a chance to
|
|
19
|
+
* look". Correct behaviour, specified, enforced by a `superRefine`, and until
|
|
20
|
+
* now unreachable because no adapter emitted it. That reads as done, which is
|
|
21
|
+
* worse than reading as missing.
|
|
22
|
+
*
|
|
23
|
+
* ── What this emits ──────────────────────────────────────────────────────────
|
|
24
|
+
* Exactly what a proxy genuinely knows — method, path, host, user agent,
|
|
25
|
+
* referrer origin, campaign — and **no `response` object at all**, unless it
|
|
26
|
+
* served the twin itself, in which case it made the response and can measure it
|
|
27
|
+
* honestly.
|
|
28
|
+
*
|
|
29
|
+
* ── ⚠️ The requestId merge is DESIGNED and NOT BUILT ─────────────────────────
|
|
30
|
+
* `requestId` rides every event so a later post-response observation can be
|
|
31
|
+
* merged onto it. **Nothing performs that merge today** — measured: nothing in
|
|
32
|
+
* the worker or the database reads the field. Next offers no general
|
|
33
|
+
* post-response hook (`onRequestError` fires only on errors; `after()` still
|
|
34
|
+
* runs in the proxy, which cannot see the route's response), so the honest
|
|
35
|
+
* second observation point is a wrapper around each route handler — a second
|
|
36
|
+
* install step, easy to forget, and a half-installed product reports half its
|
|
37
|
+
* traffic.
|
|
38
|
+
*
|
|
39
|
+
* So this ships the single honest observation. That is not a degraded event: a
|
|
40
|
+
* proxy cannot see a response, and saying so precisely is the product's
|
|
41
|
+
* differentiator rather than a shortfall.
|
|
42
|
+
*
|
|
43
|
+
* ── ⚠️ `after` is an OPTION, not an import ───────────────────────────────────
|
|
44
|
+
* There is no `ctx.waitUntil` in a Next proxy; the equivalent is `after()` from
|
|
45
|
+
* `next/server`. Importing it here — even dynamically — would put a bare
|
|
46
|
+
* specifier in a package whose whole discipline is zero runtime dependencies,
|
|
47
|
+
* and a dynamic `import()` slips past `check-sdk-artifact.mjs`'s regex, which
|
|
48
|
+
* would be a gate quietly stopping working. So the customer passes it, exactly
|
|
49
|
+
* as `observe()` already takes `waitUntil`:
|
|
50
|
+
*
|
|
51
|
+
* ```ts
|
|
52
|
+
* import { after } from "next/server";
|
|
53
|
+
* import { proxy } from "@agenthoney/analytics/next";
|
|
54
|
+
*
|
|
55
|
+
* export default proxy({ after });
|
|
56
|
+
* export const config = { matcher: ["/((?!_next/static).*)"] };
|
|
57
|
+
* ```
|
|
58
|
+
*
|
|
59
|
+
* Without it the collector falls back to its own 2s timer, which a serverless
|
|
60
|
+
* deployment can freeze before it fires.
|
|
61
|
+
*/
|
|
62
|
+
export interface ProxyOptions extends AgentHoneyConfig {
|
|
63
|
+
/**
|
|
64
|
+
* `after` from `next/server`. Passed rather than imported — see the docblock.
|
|
65
|
+
* Omitting it is supported and costs reliability on serverless.
|
|
66
|
+
*/
|
|
67
|
+
after?: (task: () => void | Promise<void>) => void;
|
|
68
|
+
/** For tests, and for a host that already has a collector. */
|
|
69
|
+
collector?: Collector;
|
|
70
|
+
/**
|
|
71
|
+
* Serve a markdown twin from the proxy.
|
|
72
|
+
*
|
|
73
|
+
* ⚠️ A proxy is a GOOD place for this, unlike the observation half: it runs
|
|
74
|
+
* before the route, so returning the twin short-circuits rendering entirely
|
|
75
|
+
* and the agent never pays for a React tree it discards. And because the
|
|
76
|
+
* proxy built that response itself, the event can honestly say `measured`.
|
|
77
|
+
*/
|
|
78
|
+
twin?: TwinOptions;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* ⚠️ Returns `undefined` to continue, rather than `NextResponse.next()`.
|
|
82
|
+
*
|
|
83
|
+
* Next treats a void return as "carry on", which means this adapter never has
|
|
84
|
+
* to import `next/server` at all. It also removes the exact sentinel that made
|
|
85
|
+
* `./web` lie here: there is no fake 200 to accidentally observe.
|
|
86
|
+
*/
|
|
87
|
+
export type ProxyResult = Response | undefined;
|
|
88
|
+
export declare function proxy(options?: ProxyOptions): (request: Request) => Promise<ProxyResult>;
|
|
89
|
+
/** Advertise a twin from a route handler, where the response actually exists. */
|
|
90
|
+
export { advertiseHeader };
|
package/dist/next.js
ADDED
package/dist/next.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"names":[],"mappings":"","file":"next.js"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import type { RequestEvent } from "@agenthoney/event-schema";
|
|
2
|
+
/**
|
|
3
|
+
* The end client's address, and the coarse country — the two facts this SDK is
|
|
4
|
+
* the ONLY thing positioned to supply.
|
|
5
|
+
*
|
|
6
|
+
* ── ⚠️ Why the SDK has to send it ────────────────────────────────────────────
|
|
7
|
+
* Ingest's socket peer is the customer's SERVER, not their visitor. So unlike a
|
|
8
|
+
* browser beacon, AgentHoney cannot observe a visitor's address itself, and
|
|
9
|
+
* `network.ip` is the one field whose absence quietly disables a whole tier of
|
|
10
|
+
* the product: **verification is IP work** — forward-confirmed reverse DNS, or
|
|
11
|
+
* a vendor-published range. With no address every crawler stays `CLAIMED`
|
|
12
|
+
* forever and `VERIFIED` is unreachable. Measured on live traffic 2026-09-17,
|
|
13
|
+
* before this existed: 590 stored events, 581 `UNVERIFIABLE`, 9 `CLAIMED`,
|
|
14
|
+
* **0 `VERIFIED`**, 2 country codes, and those two came from a hand-made test
|
|
15
|
+
* payload rather than from any adapter.
|
|
16
|
+
*
|
|
17
|
+
* ── ⚠️ Trust is the whole design, not a detail ───────────────────────────────
|
|
18
|
+
* `x-forwarded-for` is **client-supplied**. Anyone can send one. And a spoofed
|
|
19
|
+
* address is not merely a wrong row: `GPTBot` in the user agent plus a real
|
|
20
|
+
* OpenAI address in `x-forwarded-for` would forward-confirm and mint a
|
|
21
|
+
* **`VERIFIED` GPTBot** row in a customer's dashboard. Verification would then
|
|
22
|
+
* be a badge an attacker can award themselves, which is worse than not having
|
|
23
|
+
* one — the honesty of that pill is the product.
|
|
24
|
+
*
|
|
25
|
+
* So the default source is **platform edge headers plus what the framework
|
|
26
|
+
* already believes**, never a bare `x-forwarded-for`:
|
|
27
|
+
*
|
|
28
|
+
* - Edge headers (`cf-connecting-ip` and friends) are written by the
|
|
29
|
+
* proxy itself and OVERWRITE whatever the client sent -- ON that
|
|
30
|
+
* platform. **Their PRESENCE proves nothing by itself**: a visitor
|
|
31
|
+
* talking directly to a customer's own nginx can send a `cf-connecting-ip`
|
|
32
|
+
* that nothing ever overwrites, because there is no Cloudflare in the
|
|
33
|
+
* path to overwrite it. Reading these in `platform` mode is trusting that
|
|
34
|
+
* the deployment matches the header's name, which is the whole of what
|
|
35
|
+
* the mode name promises the customer is asserting.
|
|
36
|
+
* - Express's `req.ip` is the customer's OWN `trust proxy` decision. We
|
|
37
|
+
* inherit it rather than inventing a trust policy on their behalf; with no
|
|
38
|
+
* `trust proxy` configured it is simply the socket address.
|
|
39
|
+
* - `x-forwarded-for` is read only when a customer sets
|
|
40
|
+
* `clientIp: "forwarded"`, and then only its RIGHTMOST hop -- see
|
|
41
|
+
* `rightmost` below, which is what makes that opt-in safe rather than a
|
|
42
|
+
* promise the customer has to keep. It is a statement that a trusted proxy sets
|
|
43
|
+
* it. The option's name says what it trusts.
|
|
44
|
+
*
|
|
45
|
+
* ⚠️ **`forwarded` mode never reads a platform header.** Choosing `forwarded`
|
|
46
|
+
* is the customer saying "my own reverse proxy is what's in front of me, and
|
|
47
|
+
* it sets `x-forwarded-for` honestly" -- which is precisely the case where a
|
|
48
|
+
* platform edge is NOT terminating the connection, so a `cf-connecting-ip` on
|
|
49
|
+
* that request was written by the visitor, not overwritten by Cloudflare.
|
|
50
|
+
* Reading it anyway, as this did before 2026-09-20, let `GPTBot` plus a
|
|
51
|
+
* forged `cf-connecting-ip` reverse-DNS-confirm and mint a `VERIFIED` row on
|
|
52
|
+
* exactly the self-hosted installs Phase 69 pointed `forwarded` at. See
|
|
53
|
+
* `docs/design/accepted-risks.md` R14.
|
|
54
|
+
*
|
|
55
|
+
* ⚠️ **Country is independent of the address.** A platform that already
|
|
56
|
+
* resolved one hands it over in a header, and taking it costs nothing and
|
|
57
|
+
* discloses nothing: it is coarse by construction. So `clientIp: false` still
|
|
58
|
+
* yields geography, which is the cheapest honest version of this feature.
|
|
59
|
+
*/
|
|
60
|
+
/** Where an address may come from. `false` sends none, ever. */
|
|
61
|
+
export type ClientIpSource = "platform" | "forwarded" | false | ((facts: IpFacts) => string | undefined);
|
|
62
|
+
export interface IpFacts {
|
|
63
|
+
/** Case-insensitive header lookup. Lowercase names in, single value out. */
|
|
64
|
+
header: (name: string) => string | undefined;
|
|
65
|
+
/** The peer on the socket. Express only; a `Request` does not carry one. */
|
|
66
|
+
socketIp?: string | undefined;
|
|
67
|
+
/** Express's `req.ip` — the app's own `trust proxy` verdict. */
|
|
68
|
+
frameworkIp?: string | undefined;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* One address, in one canonical form, or nothing.
|
|
72
|
+
*
|
|
73
|
+
* ⚠️ **`::ffff:1.2.3.4` and `1.2.3.4` are the SAME client**, and a Node socket
|
|
74
|
+
* reports the first form on a dual-stack listener while every proxy header
|
|
75
|
+
* reports the second. Left alone they hash to two different pseudonyms, so one
|
|
76
|
+
* visitor becomes two and one crawler's verification is done twice — which is
|
|
77
|
+
* exactly the kind of quiet wrongness this product cannot afford. They are
|
|
78
|
+
* folded here, once, rather than at three call sites.
|
|
79
|
+
*
|
|
80
|
+
* ⚠️ A port is stripped (`1.2.3.4:51234`, and `[::1]:8080`), because a port is
|
|
81
|
+
* per-connection: keeping it would make every request from one client a
|
|
82
|
+
* different address.
|
|
83
|
+
*/
|
|
84
|
+
export declare function normaliseIp(raw: string | undefined | null): string | undefined;
|
|
85
|
+
export declare function resolveClientIp(facts: IpFacts, source: ClientIpSource): string | undefined;
|
|
86
|
+
export declare function resolveCountry(facts: IpFacts): string | undefined;
|
|
87
|
+
/**
|
|
88
|
+
* What the OPERATOR configured, which is not the same question as where this
|
|
89
|
+
* one address came from.
|
|
90
|
+
*
|
|
91
|
+
* ⚠️ It reports the SETTING, not the header that happened to answer. "platform
|
|
92
|
+
* mode, and nothing is arriving" is an install that was never finished;
|
|
93
|
+
* "off" is an answer somebody gave. Without this field a dashboard cannot tell
|
|
94
|
+
* them apart, and the only honest thing it can say to a customer who chose to
|
|
95
|
+
* send nothing is to accuse them of a broken install -- which is what the first
|
|
96
|
+
* draft of Phase 69 would have shipped.
|
|
97
|
+
*/
|
|
98
|
+
export declare function ipSourceName(source: ClientIpSource): "platform" | "forwarded" | "off" | "custom";
|
|
99
|
+
/**
|
|
100
|
+
* The `network` block.
|
|
101
|
+
*
|
|
102
|
+
* ⚠️ **It now carries `ipSource` even when there is no address**, which is the
|
|
103
|
+
* one case that matters: a block saying "platform, and nothing arrived" is how
|
|
104
|
+
* an install that can never verify a crawler becomes visible. That is not the
|
|
105
|
+
* `{}` the old docblock refused -- an empty object says "we looked and found
|
|
106
|
+
* nothing" with no way to act on it; this says which setting produced the
|
|
107
|
+
* silence. An absent BLOCK stays the fourth state and means an older SDK.
|
|
108
|
+
*/
|
|
109
|
+
export declare function observeNetwork(facts: IpFacts, source: ClientIpSource): RequestEvent["network"] | undefined;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Was this request the page's own background traffic, rather than a reader's?
|
|
3
|
+
* (Phase 130.) Read from the headers Next's client router sets, and nothing
|
|
4
|
+
* else -- the proxy never sees the answer, so these are the only facts there.
|
|
5
|
+
*
|
|
6
|
+
* - **prefetch**: `next-router-prefetch` or `next-router-segment-prefetch`.
|
|
7
|
+
* Every `<Link>` in the viewport sends one; nobody has read anything yet.
|
|
8
|
+
* - **refresh**: an RSC request (`rsc: 1`) for the very page it was sent from.
|
|
9
|
+
* In Next 16 `router.refresh()` goes through the same code as a navigation
|
|
10
|
+
* (`navigateToKnownRoute`) and sends the same headers, so the one fact that
|
|
11
|
+
* tells them apart is the `Referer`: a refresh asks for the URL it is on, a
|
|
12
|
+
* navigation for a different one. The `_rsc` cache-buster is ignored.
|
|
13
|
+
*
|
|
14
|
+
* ⚠️ **Absence keeps the row.** No Referer (a `no-referrer` policy), a
|
|
15
|
+
* cross-origin one, or one that does not parse is `undefined` -- recorded as
|
|
16
|
+
* today. Dropping a navigation we could not tell from a refresh would erase a
|
|
17
|
+
* reader; keeping a refresh we could not tell costs one row.
|
|
18
|
+
*
|
|
19
|
+
* ⚠️ A client-side NAVIGATION is deliberately not background: it is a person
|
|
20
|
+
* moving to a new page, and the assistant card's "pages also read" counts it.
|
|
21
|
+
*/
|
|
22
|
+
export declare function nextBackground(request: Request): "prefetch" | "refresh" | undefined;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decide whether a path segment looks like a secret, and redact it by default.
|
|
3
|
+
*
|
|
4
|
+
* ── Why this is ON by default, when `redactPatterns` already exists ──────────
|
|
5
|
+
* Because opt-in is what failed. `redactPatterns`, `routeTemplate` and
|
|
6
|
+
* `isInternal` are all well designed and all opt-in, and an independent install
|
|
7
|
+
* attempt (`docs/design/install-findings-2026-09-16.md`) found that the install
|
|
8
|
+
* guide mentioned **none of the three**. The site in question puts single-use
|
|
9
|
+
* capability tokens in its PATHS -- `/reveal/<token>`, `/join/<code>` -- so an
|
|
10
|
+
* agent following the guide exactly would have POSTed live credential material
|
|
11
|
+
* to our event store.
|
|
12
|
+
*
|
|
13
|
+
* The guide now has a step about it, and a step an agent can skip is a step an
|
|
14
|
+
* agent will skip. **The failure mode of skipping it must be a redacted path,
|
|
15
|
+
* not a leaked one.**
|
|
16
|
+
*
|
|
17
|
+
* ── ⚠️ The hard part is not the regex ────────────────────────────────────────
|
|
18
|
+
* A high-entropy segment is INDISTINGUISHABLE from a capability token by shape:
|
|
19
|
+
*
|
|
20
|
+
* cmu4psgny0003s397tb2t5gx6 cuid -- our own site ids look exactly like this
|
|
21
|
+
* 7f3a9c2b1d4e6f8a hex -- a build hash, or a reset token
|
|
22
|
+
* a1b2c3d4-...-e5f6 uuid -- a public resource id, or a ticket
|
|
23
|
+
*
|
|
24
|
+
* So entropy cannot decide, and any default is wrong in one direction. The
|
|
25
|
+
* decision, written down rather than left in the code: **be wrong in the
|
|
26
|
+
* direction of redaction.** A redacted path costs an operator route detail,
|
|
27
|
+
* recoverable by setting `redactHighEntropyPaths: false`. A leaked token is
|
|
28
|
+
* recoverable by nothing -- it is already in somebody else's database.
|
|
29
|
+
*
|
|
30
|
+
* ── ⚠️ What it deliberately does NOT touch ───────────────────────────────────
|
|
31
|
+
* - **Anything with a file extension.** `/assets/app.4f3b2a.css` is an asset
|
|
32
|
+
* path, and mangling it would destroy the format analysis that is half this
|
|
33
|
+
* product.
|
|
34
|
+
* - **Template placeholders.** A customer whose `routeTemplate` produced
|
|
35
|
+
* `/users/:id` has told us that segment is a dimension, not a secret.
|
|
36
|
+
* - **Slugs.** `why-markdown-twins-matter-for-agents` is long and mixed and
|
|
37
|
+
* entirely public. Word structure is what separates it from a token, so word
|
|
38
|
+
* structure is what is tested -- not length, which would redact every blog.
|
|
39
|
+
*
|
|
40
|
+
* ⚠️ **It is a backstop, not a policy.** Only the customer knows which of their
|
|
41
|
+
* paths are secret; a short token (`/j/aB3xK9`) is under the length floor and
|
|
42
|
+
* will not be caught. The guide's step remains the real fix, and says so.
|
|
43
|
+
*/
|
|
44
|
+
export declare function looksLikeSecret(segment: string): boolean;
|
|
45
|
+
export declare const REDACTED = "[redacted]";
|
|
46
|
+
export interface RedactionResult {
|
|
47
|
+
path: string;
|
|
48
|
+
/** True when at least one segment was replaced, by any rule. */
|
|
49
|
+
redacted: boolean;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Apply the customer's patterns and, unless disabled, the default detector.
|
|
53
|
+
*
|
|
54
|
+
* ⚠️ Custom patterns run FIRST and independently. A customer who wrote a
|
|
55
|
+
* pattern gets it applied whether or not the default is on, and turning the
|
|
56
|
+
* default off never turns theirs off with it.
|
|
57
|
+
*/
|
|
58
|
+
export declare function redactPath(path: string, patterns: readonly RegExp[], useDefault: boolean): RedactionResult;
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { type RequestEvent } from "@agenthoney/event-schema";
|
|
2
|
+
import type { ResolvedConfig } from "../core/config.js";
|
|
3
|
+
/**
|
|
4
|
+
* Turn a request into the fields the wire event carries -- and, just as
|
|
5
|
+
* importantly, leave everything else behind.
|
|
6
|
+
*
|
|
7
|
+
* ── This module IS the privacy promise ───────────────────────────────────────
|
|
8
|
+
* `docs/design/collected-fields.md` lists what we take. This is the only place
|
|
9
|
+
* that decides. Every value here is read from a named header or a named,
|
|
10
|
+
* allowlisted query parameter; there is no path that copies "the rest" of
|
|
11
|
+
* anything, which is why a cookie or a bearer token cannot arrive by accident.
|
|
12
|
+
*
|
|
13
|
+
* A reviewer should be able to read this file and check the published list
|
|
14
|
+
* against it line by line.
|
|
15
|
+
*/
|
|
16
|
+
/** The only query parameters ever read. Everything else is discarded. */
|
|
17
|
+
export declare const CAMPAIGN_PARAMS: readonly ["utm_source", "utm_medium", "utm_campaign", "utm_content"];
|
|
18
|
+
export interface ObservedRequest {
|
|
19
|
+
method: string;
|
|
20
|
+
path: string;
|
|
21
|
+
route?: string;
|
|
22
|
+
host?: string;
|
|
23
|
+
protocol?: "http" | "https";
|
|
24
|
+
userAgent?: string;
|
|
25
|
+
referrerOrigin?: string;
|
|
26
|
+
campaign?: RequestEvent["campaign"];
|
|
27
|
+
clickIdType?: RequestEvent["clickIdType"];
|
|
28
|
+
internal?: boolean;
|
|
29
|
+
/**
|
|
30
|
+
* ⚠️ Set when any segment of the path or route was replaced, by a customer
|
|
31
|
+
* pattern or by the default detector. **Visible, never silent** -- an
|
|
32
|
+
* operator staring at `/reveal/[redacted]` needs to know it was us, not a
|
|
33
|
+
* bug in their router, and needs to be able to find the setting.
|
|
34
|
+
*/
|
|
35
|
+
pathRedacted?: boolean;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Normalise a path: no query, no fragment, no repeated slashes, bounded length.
|
|
39
|
+
*
|
|
40
|
+
* ⚠️ The query string is dropped HERE, before anything else looks at it.
|
|
41
|
+
* Allowlisted campaign values are read separately from the parsed URL. There is
|
|
42
|
+
* deliberately no code path in which a raw query string is carried forward and
|
|
43
|
+
* filtered later -- a filter can be wrong; an absence cannot.
|
|
44
|
+
*/
|
|
45
|
+
export declare function normalisePath(rawPath: string, redactPatterns?: readonly RegExp[]): string;
|
|
46
|
+
/** Scheme and host only. The page someone came from is never collected. */
|
|
47
|
+
export declare function referrerOrigin(referer: string | null | undefined): string | undefined;
|
|
48
|
+
/**
|
|
49
|
+
* The hostname the CLIENT asked for, which is not always the one the server
|
|
50
|
+
* thinks it is.
|
|
51
|
+
*
|
|
52
|
+
* ⚠️ **Measured 2026-09-17 on a self-hosted Next 16 standalone server**: the
|
|
53
|
+
* proxy receives a `request.url` of `http://0.0.0.0:3013/…`, so reading
|
|
54
|
+
* `url.host` recorded **`0.0.0.0:3013`** as the host of every request to
|
|
55
|
+
* `app.agenthoney.ai`. The site's own dashboard could not tell you which
|
|
56
|
+
* hostname its traffic arrived on. A managed platform rewrites the URL for you,
|
|
57
|
+
* which is why the same code looked correct on Vercel and on Express.
|
|
58
|
+
*
|
|
59
|
+
* ⚠️ It is a CLAIM, like every header — but it is the same claim the customer's
|
|
60
|
+
* own router already dispatched on, so recording it adds no trust that their
|
|
61
|
+
* application had not already extended. `x-forwarded-host` first: a proxy that
|
|
62
|
+
* rewrites `Host` to an internal name sets it precisely to preserve the
|
|
63
|
+
* original.
|
|
64
|
+
*/
|
|
65
|
+
export declare function observedHost(header: (name: string) => string | undefined, fallback?: string | undefined): string | undefined;
|
|
66
|
+
export interface RequestFacts {
|
|
67
|
+
method: string;
|
|
68
|
+
/** Path AND query, as it arrived. Never stored; parsed here and discarded. */
|
|
69
|
+
url: string;
|
|
70
|
+
host?: string | undefined;
|
|
71
|
+
protocol?: "http" | "https" | undefined;
|
|
72
|
+
userAgent?: string | undefined;
|
|
73
|
+
referer?: string | undefined;
|
|
74
|
+
}
|
|
75
|
+
export declare function observeRequest(facts: RequestFacts, config: ResolvedConfig): ObservedRequest;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { RequestEvent } from "@agenthoney/event-schema";
|
|
2
|
+
/**
|
|
3
|
+
* Response facts, read from headers that already exist.
|
|
4
|
+
*
|
|
5
|
+
* ⚠️ **Nothing here ever touches the body.** A response is a single-read
|
|
6
|
+
* stream: to learn what it contained you would have to wrap `write`/`end` and
|
|
7
|
+
* buffer it, which adds latency, adds memory proportional to the page, and can
|
|
8
|
+
* corrupt what the customer sends if the encoding or backpressure handling is
|
|
9
|
+
* wrong. `contentLength` is therefore read from the header and is simply absent
|
|
10
|
+
* for a chunked or streamed response -- the dashboard shows "not measured",
|
|
11
|
+
* which is honest and free.
|
|
12
|
+
*/
|
|
13
|
+
export interface ResponseFacts {
|
|
14
|
+
status?: number | undefined;
|
|
15
|
+
contentType?: string | null | undefined;
|
|
16
|
+
contentLength?: string | number | null | undefined;
|
|
17
|
+
latencyMs?: number | undefined;
|
|
18
|
+
observation: RequestEvent extends {
|
|
19
|
+
response?: infer R;
|
|
20
|
+
} ? R extends {
|
|
21
|
+
observation: infer O;
|
|
22
|
+
} ? O : never : never;
|
|
23
|
+
}
|
|
24
|
+
export declare function observeResponse(facts: ResponseFacts): NonNullable<RequestEvent["response"]>;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ⚠️ The SDK's own name and version, in ONE place.
|
|
3
|
+
*
|
|
4
|
+
* They were declared separately in `web.ts` and `express.ts`, and adding a
|
|
5
|
+
* third adapter would have made three copies of a version string that must
|
|
6
|
+
* match `package.json` -- a list nothing checks, written down three times.
|
|
7
|
+
* `runtime.test.ts` asserts the version against the manifest.
|
|
8
|
+
*/
|
|
9
|
+
export declare const SDK_NAME = "@agenthoney/analytics";
|
|
10
|
+
export declare const SDK_VERSION = "0.14.0";
|
|
11
|
+
/**
|
|
12
|
+
* Identify the runtime, for the `sdk.runtime` field.
|
|
13
|
+
*
|
|
14
|
+
* Kept deliberately dumb and total: it must never throw at import time, on any
|
|
15
|
+
* runtime, because that would take the host application down before it serves
|
|
16
|
+
* a request. Every branch is guarded and there is a final fallback.
|
|
17
|
+
*/
|
|
18
|
+
export declare function runtimeName(): string;
|
|
19
|
+
/**
|
|
20
|
+
* A unique event id.
|
|
21
|
+
*
|
|
22
|
+
* `crypto.randomUUID()` where available -- it is in every supported runtime and
|
|
23
|
+
* costs no dependency. The fallback exists so an exotic runtime degrades to a
|
|
24
|
+
* slightly weaker id rather than to a crash; a collision costs one deduplicated
|
|
25
|
+
* event, a throw costs the customer a 500.
|
|
26
|
+
*/
|
|
27
|
+
export declare function newEventId(): string;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export interface MediaRange {
|
|
2
|
+
type: string;
|
|
3
|
+
q: number;
|
|
4
|
+
}
|
|
5
|
+
export declare function parseAccept(header: string | null | undefined): MediaRange[];
|
|
6
|
+
export declare const MARKDOWN_TYPES: readonly ["text/markdown", "text/x-markdown"];
|
|
7
|
+
export declare function prefersMarkdown(header: string | null | undefined): boolean;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { TwinResolver } from "./twin.js";
|
|
2
|
+
/**
|
|
3
|
+
* The three files that let an agent find a site's content without crawling it
|
|
4
|
+
* blindly: `/llms.txt`, `/llms-full.txt` and `/install.md`.
|
|
5
|
+
*
|
|
6
|
+
* ── Why generate them rather than let customers write them ───────────────────
|
|
7
|
+
* A hand-written index is wrong the first time a page is added, and an index
|
|
8
|
+
* that lists pages which no longer exist is worse than none -- an agent spends
|
|
9
|
+
* its budget on 404s and concludes the site is broken. These are derived from
|
|
10
|
+
* the same manifest that serves the twins, so they cannot disagree with what is
|
|
11
|
+
* actually servable.
|
|
12
|
+
*
|
|
13
|
+
* `llms.txt` is the convention proposed by Answer.AI in 2024: a markdown index
|
|
14
|
+
* of a site's high-value content. `llms-full.txt` is the same set with the
|
|
15
|
+
* bodies inline, for an agent that would rather make one request than a hundred.
|
|
16
|
+
*/
|
|
17
|
+
export interface TwinEntry {
|
|
18
|
+
/** The page path, e.g. `/docs/intro`. */
|
|
19
|
+
path: string;
|
|
20
|
+
/** One line. This is what an agent reads when deciding whether to fetch. */
|
|
21
|
+
summary?: string;
|
|
22
|
+
title?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface DiscoveryOptions {
|
|
25
|
+
siteName: string;
|
|
26
|
+
/** One or two sentences. Appears under the heading in `llms.txt`. */
|
|
27
|
+
description?: string;
|
|
28
|
+
origin?: string;
|
|
29
|
+
entries: readonly TwinEntry[];
|
|
30
|
+
}
|
|
31
|
+
/** The index: every page, one line each. */
|
|
32
|
+
export declare function renderLlmsTxt(options: DiscoveryOptions): string;
|
|
33
|
+
/** The bulk file: one request instead of a hundred. */
|
|
34
|
+
export declare function renderLlmsFullTxt(options: DiscoveryOptions, resolve: TwinResolver): Promise<string>;
|
|
35
|
+
/**
|
|
36
|
+
* ⚠️ **The guide's own version, and it is the PACKAGE's version.**
|
|
37
|
+
*
|
|
38
|
+
* The dashboard's prompt links a versioned URL so an agent is told to read the
|
|
39
|
+
* guide that matches the code it is about to install, rather than whatever is
|
|
40
|
+
* at `/install.md` at the moment it happens to fetch.
|
|
41
|
+
*
|
|
42
|
+
* Read from `package.json` rather than written here, because a version written
|
|
43
|
+
* in two places is a version that disagrees with itself.
|
|
44
|
+
*/
|
|
45
|
+
export declare const INSTALL_GUIDE_VERSION: string;
|
|
46
|
+
/**
|
|
47
|
+
* The install guide, written for an AI coding agent rather than a person.
|
|
48
|
+
*
|
|
49
|
+
* ⚠️ This is the install path for a product whose users install things by
|
|
50
|
+
* asking a model. It is deliberately imperative, names the exact failure modes,
|
|
51
|
+
* and tells the agent to STOP rather than guess at a missing value -- because an
|
|
52
|
+
* agent that guesses a credential writes a broken config that looks finished.
|
|
53
|
+
*/
|
|
54
|
+
export declare function renderInstallMd(options?: {
|
|
55
|
+
packageName?: string;
|
|
56
|
+
}): string;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import type { Twin } from "./twin.js";
|
|
2
|
+
/**
|
|
3
|
+
* Twins the customer never wrote: resolved from what their own visitors agreed
|
|
4
|
+
* a page says.
|
|
5
|
+
*
|
|
6
|
+
* ── ⚠️ Why this REFRESHES rather than fetches ────────────────────────────────
|
|
7
|
+
* `TwinOptions.resolve` runs inside the customer's request path, and this SDK's
|
|
8
|
+
* first rule is that **nothing awaits the network there**. A resolver that
|
|
9
|
+
* fetched per request would add our latency -- and our availability -- to every
|
|
10
|
+
* page of their site, which is precisely the trade this product refuses to make
|
|
11
|
+
* anywhere else.
|
|
12
|
+
*
|
|
13
|
+
* So the whole published set is pulled into memory and `resolve` is a `Map`
|
|
14
|
+
* lookup, with the network happening in the background. ⚠️ **One measured
|
|
15
|
+
* exception**, below: a COLD process awaits its first load, because answering
|
|
16
|
+
* "no twin" on a fresh isolate sent 2 of 8 live `text/markdown` requests an
|
|
17
|
+
* HTML page instead.
|
|
18
|
+
*
|
|
19
|
+
* ── ⚠️ Demand-driven, with no timer and no I/O at construction ───────────────
|
|
20
|
+
* The first version created this with `setInterval` and fired a refresh from
|
|
21
|
+
* the constructor. **Both are forbidden in a Cloudflare Worker**: I/O in the
|
|
22
|
+
* global scope throws, and a timer does not survive an isolate being evicted.
|
|
23
|
+
* It would have failed on the first runtime it was built for -- agenthoney.ai
|
|
24
|
+
* is a Worker -- and it would have failed at deploy, not in a test.
|
|
25
|
+
*
|
|
26
|
+
* So refreshing is something the HOST asks for: `refreshIfStale()` is cheap to
|
|
27
|
+
* call on every request and does nothing until the data is older than
|
|
28
|
+
* `refreshMs`. A Worker passes it to `ctx.waitUntil`; a Node server can call it
|
|
29
|
+
* per request or on its own interval.
|
|
30
|
+
*
|
|
31
|
+
* ── ⚠️ The cold await was DELIBERATE; where it leaked to was not ─────────────
|
|
32
|
+
* A cold `resolve` awaits, and `hosted.test.ts` carries the measurement that
|
|
33
|
+
* made it so. What nobody noticed is which requests reach `resolve`: until
|
|
34
|
+
* Phase 49, `./express`'s ADVERTISE branch called it on **every passing GET**
|
|
35
|
+
* and deferred `next()` until it settled, and `./web` awaited it after the
|
|
36
|
+
* handler had already built the response. So an exception argued for a client
|
|
37
|
+
* asking for markdown was being paid by a human loading HTML -- the one thing
|
|
38
|
+
* this SDK is built not to do -- on the one deployment that uses this module.
|
|
39
|
+
*
|
|
40
|
+
* Two rules now, and `lookup` exists so a caller can obey them:
|
|
41
|
+
*
|
|
42
|
+
* 1. **`lookup` is synchronous and never touches the network.** Anything
|
|
43
|
+
* running on a request that did NOT ask for markdown -- advertising, above
|
|
44
|
+
* all -- uses it, and a cold process simply advertises nothing.
|
|
45
|
+
* 2. **`resolve` may await on a cold process, and is now BOUNDED.** It is for
|
|
46
|
+
* requests that asked for markdown by `.md` suffix or `Accept`, where the
|
|
47
|
+
* alternative is never serving a twin on a serverless isolate at all. The
|
|
48
|
+
* bound is `coldTimeoutMs`; past it the answer is "no twin" and the
|
|
49
|
+
* customer's own handler runs. It had no bound before, which meant a
|
|
50
|
+
* hanging ingest held a customer's request for as long as it liked.
|
|
51
|
+
*
|
|
52
|
+
* ── ⚠️ Every failure resolves to "no twin" ───────────────────────────────────
|
|
53
|
+
* Before the first refresh completes, during an outage, after a 500: `resolve`
|
|
54
|
+
* returns undefined and the customer's own HTML is served unchanged. A twin
|
|
55
|
+
* that is missing is a page that works normally; there is no failure here that
|
|
56
|
+
* degrades to anything worse.
|
|
57
|
+
*/
|
|
58
|
+
export interface HostedTwinsOptions {
|
|
59
|
+
/** The SERVER key. This corpus is not public and a browser never reads it. */
|
|
60
|
+
serverKey: string;
|
|
61
|
+
/** Defaults to the SDK's configured ingest origin. */
|
|
62
|
+
ingestUrl?: string;
|
|
63
|
+
/**
|
|
64
|
+
* How stale the set may get. ⚠️ A page wins consensus over DAYS, so there is
|
|
65
|
+
* nothing to gain from a tight interval and a shared database to protect.
|
|
66
|
+
*/
|
|
67
|
+
refreshMs?: number;
|
|
68
|
+
/**
|
|
69
|
+
* The ceiling on the one await this module is allowed (see rule 2 above).
|
|
70
|
+
*
|
|
71
|
+
* ⚠️ **A default, not a measurement.** Nobody has measured p99 of a corpus
|
|
72
|
+
* pull from a serverless region yet, and this number lands on somebody's page
|
|
73
|
+
* load. Phase 49 records that measuring it is outstanding; until then it is
|
|
74
|
+
* short enough that the worst case is a page that renders normally.
|
|
75
|
+
*/
|
|
76
|
+
coldTimeoutMs?: number;
|
|
77
|
+
/**
|
|
78
|
+
* How long a failed refresh stops the next one being attempted.
|
|
79
|
+
*
|
|
80
|
+
* ⚠️ **The breaker the telemetry half has had since the beginning, and this
|
|
81
|
+
* half did not.** `refreshIfStale()` runs on every request, and a failure
|
|
82
|
+
* does not move `fetchedAt` -- so for the whole of an ingest outage, every
|
|
83
|
+
* request through the customer's server started another refresh the moment
|
|
84
|
+
* the last one failed. `breaker.ts` says exactly why that is the customer's
|
|
85
|
+
* problem and not ours: a fleet of their servers paying a DNS lookup, a
|
|
86
|
+
* connect and a timeout, continuously, because OUR endpoint is down.
|
|
87
|
+
*
|
|
88
|
+
* A held-back refresh is free: the corpus already in memory keeps serving.
|
|
89
|
+
*/
|
|
90
|
+
failureBackoffMs?: number;
|
|
91
|
+
/**
|
|
92
|
+
* The ceiling on a single refresh's network call.
|
|
93
|
+
*
|
|
94
|
+
* ⚠️ Nothing awaits this on a request, so this bound is not about latency --
|
|
95
|
+
* it is about a HANGING ingest. Without it a stalled connection kept
|
|
96
|
+
* `inFlight` set for as long as the runtime's own socket timeout (undici:
|
|
97
|
+
* five minutes), and `refreshIfStale()` returns that same promise, so the
|
|
98
|
+
* corpus stopped refreshing entirely for the duration.
|
|
99
|
+
*/
|
|
100
|
+
refreshTimeoutMs?: number;
|
|
101
|
+
/** Injected for tests. */
|
|
102
|
+
now?: () => number;
|
|
103
|
+
/** Injected for tests. */
|
|
104
|
+
fetchImpl?: typeof fetch;
|
|
105
|
+
}
|
|
106
|
+
export interface HostedTwins {
|
|
107
|
+
/**
|
|
108
|
+
* One twin, straight from ingest, for a COLD process.
|
|
109
|
+
*
|
|
110
|
+
* ⚠️ **A single row, never the corpus.** The bulk refresh can be megabytes
|
|
111
|
+
* and an isolate that will serve one request must not pay for it on that
|
|
112
|
+
* request. Bounded by the caller; fails to `undefined`, never throws.
|
|
113
|
+
*/
|
|
114
|
+
fetchOne: (path: string) => Promise<Twin | undefined>;
|
|
115
|
+
/**
|
|
116
|
+
* ⚠️ **Synchronous, and it never starts a fetch.** For every caller that runs
|
|
117
|
+
* on a request which has not asked for markdown. A cold process answers
|
|
118
|
+
* `undefined`, which is the truth: this isolate does not know of a twin.
|
|
119
|
+
*/
|
|
120
|
+
lookup: (path: string) => Twin | undefined;
|
|
121
|
+
resolve: (path: string) => Twin | undefined | Promise<Twin | undefined>;
|
|
122
|
+
/** Refresh unconditionally. Mostly for tests and for a deliberate warm-up. */
|
|
123
|
+
refresh: () => Promise<void>;
|
|
124
|
+
/**
|
|
125
|
+
* Refresh only if the set is older than `refreshMs`, and never more than one
|
|
126
|
+
* at a time.
|
|
127
|
+
*
|
|
128
|
+
* ⚠️ Safe to call on every request: it is a clock comparison in the common
|
|
129
|
+
* case. Hand it to `ctx.waitUntil` on a Worker so the refresh outlives the
|
|
130
|
+
* response without delaying it.
|
|
131
|
+
*/
|
|
132
|
+
refreshIfStale: () => Promise<void>;
|
|
133
|
+
readonly size: number;
|
|
134
|
+
}
|
|
135
|
+
export declare function hostedTwins(options: HostedTwinsOptions): HostedTwins;
|