@pentoshi/clai 0.10.5 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agent/runner.js +41 -3
- package/dist/agent/runner.js.map +1 -1
- package/dist/commands/providers.js +28 -0
- package/dist/commands/providers.js.map +1 -1
- package/dist/commands/search-providers.d.ts +50 -0
- package/dist/commands/search-providers.js +134 -0
- package/dist/commands/search-providers.js.map +1 -0
- package/dist/commands/update.js +1 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/provider.js +9 -6
- package/dist/llm/provider.js.map +1 -1
- package/dist/prompts/index.d.ts +1 -1
- package/dist/prompts/index.js +6 -0
- package/dist/prompts/index.js.map +1 -1
- package/dist/safety/classifier.js +40 -0
- package/dist/safety/classifier.js.map +1 -1
- package/dist/store/config.d.ts +5 -0
- package/dist/store/config.js +7 -0
- package/dist/store/config.js.map +1 -1
- package/dist/store/keys.d.ts +65 -0
- package/dist/store/keys.js +164 -28
- package/dist/store/keys.js.map +1 -1
- package/dist/tools/http.d.ts +12 -1
- package/dist/tools/http.js +8 -43
- package/dist/tools/http.js.map +1 -1
- package/dist/tools/registry.js +52 -0
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/shell.d.ts +25 -0
- package/dist/tools/shell.js +155 -6
- package/dist/tools/shell.js.map +1 -1
- package/dist/tools/web/audit.d.ts +154 -0
- package/dist/tools/web/audit.js +147 -0
- package/dist/tools/web/audit.js.map +1 -0
- package/dist/tools/web/budget.d.ts +76 -0
- package/dist/tools/web/budget.js +187 -0
- package/dist/tools/web/budget.js.map +1 -0
- package/dist/tools/web/capture.d.ts +201 -0
- package/dist/tools/web/capture.js +380 -0
- package/dist/tools/web/capture.js.map +1 -0
- package/dist/tools/web/fetch-core.d.ts +66 -0
- package/dist/tools/web/fetch-core.js +1123 -0
- package/dist/tools/web/fetch-core.js.map +1 -0
- package/dist/tools/web/fetch.d.ts +42 -0
- package/dist/tools/web/fetch.js +115 -0
- package/dist/tools/web/fetch.js.map +1 -0
- package/dist/tools/web/providers/brave.d.ts +46 -0
- package/dist/tools/web/providers/brave.js +263 -0
- package/dist/tools/web/providers/brave.js.map +1 -0
- package/dist/tools/web/providers/duckduckgo.d.ts +47 -0
- package/dist/tools/web/providers/duckduckgo.js +248 -0
- package/dist/tools/web/providers/duckduckgo.js.map +1 -0
- package/dist/tools/web/providers/provider.d.ts +99 -0
- package/dist/tools/web/providers/provider.js +38 -0
- package/dist/tools/web/providers/provider.js.map +1 -0
- package/dist/tools/web/providers/tavily.d.ts +52 -0
- package/dist/tools/web/providers/tavily.js +285 -0
- package/dist/tools/web/providers/tavily.js.map +1 -0
- package/dist/tools/web/readable.d.ts +67 -0
- package/dist/tools/web/readable.js +248 -0
- package/dist/tools/web/readable.js.map +1 -0
- package/dist/tools/web/redact.d.ts +120 -0
- package/dist/tools/web/redact.js +155 -0
- package/dist/tools/web/redact.js.map +1 -0
- package/dist/tools/web/search.d.ts +51 -0
- package/dist/tools/web/search.js +389 -0
- package/dist/tools/web/search.js.map +1 -0
- package/dist/tools/web/ssrf-guard.d.ts +85 -0
- package/dist/tools/web/ssrf-guard.js +265 -0
- package/dist/tools/web/ssrf-guard.js.map +1 -0
- package/dist/tools/web/types.d.ts +331 -0
- package/dist/tools/web/types.js +71 -0
- package/dist/tools/web/types.js.map +1 -0
- package/dist/ui/spinner.js +87 -14
- package/dist/ui/spinner.js.map +1 -1
- package/package.json +3 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"budget.js","sourceRoot":"","sources":["../../../src/tools/web/budget.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAML,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;GASG;AACH,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC;;;;;;;GAOG;AACH,MAAM,cAAc,GAAG,MAAM,CAAC;AAqC9B;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,KAAkB;IACxC,mEAAmE;IACnE,uDAAuD;IACvD,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACjE,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/D,MAAM,aAAa,GACjB,KAAK,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAE9B,MAAM,OAAO,GAAG,GAAW,EAAE,CAC3B,MAAM,CAAC,UAAU,CACf,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,EAChE,MAAM,CACP,CAAC;IAEJ,IAAI,aAAa,GAAG,OAAO,EAAE,CAAC;IAE9B,KACE,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,cAAc,IAAI,aAAa,GAAG,qBAAqB,EAC3D,CAAC,EAAE,EACH,CAAC;QACD,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,0DAA0D;YAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,KAAK,SAAS,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;YAChE,oEAAoE;YACpE,uBAAuB,CAAC,OAAO,CAAC,CAAC;YACjC,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,0DAA0D;YAC1D,aAAa,CAAC,GAAG,EAAE,CAAC;YACpB,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,MAAM;QACrB,aAAa,GAAG,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,OAAO,QAAQ,CAAC;QACd,OAAO;QACP,GAAG;QACH,MAAM;QACN,aAAa;QACb,OAAO;QACP,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,OAAkB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,kBAAkB;YAAE,OAAO,IAAI,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,uBAAuB,CAAC,OAAkB;IACjD,IAAI,OAA2B,CAAC;IAChC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,GAAG,GAAG,kBAAkB,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;YAC9C,OAAO,GAAG,GAAG,CAAC;YACd,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;IACH,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO;IAElC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO;IAElC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,MAAM,GAAG,iBAAiB,EAAE,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACtC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC;QACzD,CAAC,CAAC,KAAK,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,KAOjB;IACC,MAAM,MAAM,GAAiB;QAC3B,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,GAAG,EAAE,qBAAqB;KAC3B,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS;QAAE,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7D,IAAI,KAAK,CAAC,aAAa,KAAK,SAAS;QACnC,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Capture` — pure observation builder for a single `web.fetch` invocation.
|
|
3
|
+
*
|
|
4
|
+
* The fetch transport in `fetch-core.ts` feeds events into this builder
|
|
5
|
+
* (DNS resolution, TCP connect, TLS handshake, response headers, redirect
|
|
6
|
+
* hops, `Set-Cookie` lines) and the builder accumulates the structured
|
|
7
|
+
* fields the {@link WebFetchMetadata} envelope needs:
|
|
8
|
+
*
|
|
9
|
+
* - {@link TimingInfo}: `dnsMs`, `tcpMs`, `tlsMs`, `ttfbMs`, `totalMs`
|
|
10
|
+
* - {@link TlsInfo} from the final hop's `tls.TLSSocket`
|
|
11
|
+
* (`getProtocol()`, `getCipher().name`, `getPeerCertificate(true)`)
|
|
12
|
+
* - response headers (lowercased keys, repeat values joined with `, `)
|
|
13
|
+
* - the redirect chain (capped at {@link MAX_REDIRECT_HOPS})
|
|
14
|
+
* - the resolved IP and the final hostname
|
|
15
|
+
* - `Set-Cookie` cookies (capped at {@link MAX_COOKIES_CAPTURED})
|
|
16
|
+
*
|
|
17
|
+
* The builder performs **no I/O** — it never touches the network, the
|
|
18
|
+
* filesystem, or `auditLog`. It is a pure data sink so the transport
|
|
19
|
+
* layer in `fetch-core.ts` can be tested independently of the
|
|
20
|
+
* Node `https`/`tls` machinery.
|
|
21
|
+
*
|
|
22
|
+
* Per-hop semantics:
|
|
23
|
+
* - `setHopContext(hostname)` is called by the transport at the start of
|
|
24
|
+
* each hop; the value of the *last* hop becomes
|
|
25
|
+
* {@link CapturedFields.finalHostname}.
|
|
26
|
+
* - {@link Capture.markDnsResolved} replaces the running `dnsMs` /
|
|
27
|
+
* `resolvedIp` so the values surfaced in {@link CapturedFields}
|
|
28
|
+
* correspond to the *final* hop. Same for `tcpMs`, `tlsMs`, and
|
|
29
|
+
* `ttfbMs` — only the last hop's measurements are returned.
|
|
30
|
+
* - {@link Capture.addRedirectHop} appends one entry per intermediate
|
|
31
|
+
* 3xx hop; the array is bounded at {@link MAX_REDIRECT_HOPS}.
|
|
32
|
+
* - {@link Capture.addSetCookieHeader} parses every observed
|
|
33
|
+
* `Set-Cookie` line via {@link parseSetCookie}; the resulting array is
|
|
34
|
+
* bounded at {@link MAX_COOKIES_CAPTURED}.
|
|
35
|
+
*
|
|
36
|
+
* Redaction and 4096-char header-value truncation are applied later by
|
|
37
|
+
* `redact.applyToHeaders` / `redact.applyToCookies` and the 64 KiB
|
|
38
|
+
* metadata budget by `budget.enforce`. This module only collects.
|
|
39
|
+
*/
|
|
40
|
+
import type { IncomingHttpHeaders } from "node:http";
|
|
41
|
+
import type { TLSSocket } from "node:tls";
|
|
42
|
+
import { type CookieInfo, type HeaderMap, type RedirectChain, type TimingInfo, type TlsInfo } from "./types.js";
|
|
43
|
+
/**
|
|
44
|
+
* Snapshot of the structured fields the builder has accumulated when
|
|
45
|
+
* `fetch-core.ts` finalises the response. The shape matches the slice of
|
|
46
|
+
* {@link WebFetchMetadata} that depends on per-hop transport
|
|
47
|
+
* observation; the fetch handler combines this with `requestedUrl`,
|
|
48
|
+
* `finalUrl`, `mode`, `bytesReceived`, `truncated`, etc. before applying
|
|
49
|
+
* `redact.applyToHeaders` / `redact.applyToCookies` and `budget.enforce`.
|
|
50
|
+
*/
|
|
51
|
+
export interface CapturedFields {
|
|
52
|
+
/** IP address contacted on the final hop. */
|
|
53
|
+
resolvedIp: string;
|
|
54
|
+
/** Hostname of the final hop (the one whose body is returned). */
|
|
55
|
+
finalHostname: string;
|
|
56
|
+
/** Integer HTTP status of the final response. */
|
|
57
|
+
status: number;
|
|
58
|
+
/** Lowercased response headers from the final hop, repeats joined. */
|
|
59
|
+
headers: HeaderMap;
|
|
60
|
+
/** TLS session details from the final hop, when the scheme was https. */
|
|
61
|
+
tls?: TlsInfo;
|
|
62
|
+
/** Per-phase timings; `tlsMs` is omitted on http:// requests. */
|
|
63
|
+
timing: TimingInfo;
|
|
64
|
+
/** Up to {@link MAX_REDIRECT_HOPS} hops in chronological order. */
|
|
65
|
+
redirectChain: RedirectChain;
|
|
66
|
+
/** Up to {@link MAX_COOKIES_CAPTURED} cookies parsed from `Set-Cookie`. */
|
|
67
|
+
cookies: CookieInfo[];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Pure builder that records the per-hop observations made by
|
|
71
|
+
* `fetch-core.ts` and assembles them into a {@link CapturedFields}
|
|
72
|
+
* snapshot via {@link Capture.finalize}.
|
|
73
|
+
*
|
|
74
|
+
* The builder is single-use: callers should construct one `Capture` per
|
|
75
|
+
* `web.fetch` invocation and discard it after `finalize`.
|
|
76
|
+
*/
|
|
77
|
+
export declare class Capture {
|
|
78
|
+
/** Final-hop DNS-resolution time in milliseconds. */
|
|
79
|
+
private dnsMs;
|
|
80
|
+
/** Final-hop TCP-connect time in milliseconds. */
|
|
81
|
+
private tcpMs;
|
|
82
|
+
/** Final-hop TLS-handshake time in milliseconds (https only). */
|
|
83
|
+
private tlsMs;
|
|
84
|
+
/** Final-hop time-to-first-byte in milliseconds. */
|
|
85
|
+
private ttfbMs;
|
|
86
|
+
/** Final-hop resolved IP, set by {@link markDnsResolved}. */
|
|
87
|
+
private resolvedIp;
|
|
88
|
+
/** Final-hop hostname, set by {@link setHopContext}. */
|
|
89
|
+
private finalHostname;
|
|
90
|
+
/** Final-hop HTTP status, set by {@link markResponse}. */
|
|
91
|
+
private status;
|
|
92
|
+
/** Final-hop normalised headers, set by {@link markResponse}. */
|
|
93
|
+
private headers;
|
|
94
|
+
/** Final-hop TLS info (https only), set by {@link markTlsHandshaked}. */
|
|
95
|
+
private tls;
|
|
96
|
+
/** Redirect hops, capped at {@link MAX_REDIRECT_HOPS}. */
|
|
97
|
+
private readonly redirectChain;
|
|
98
|
+
/** Captured cookies, capped at {@link MAX_COOKIES_CAPTURED}. */
|
|
99
|
+
private readonly cookies;
|
|
100
|
+
/**
|
|
101
|
+
* Whether the current invocation is an `https://` fetch. When `false`,
|
|
102
|
+
* `tlsMs` is omitted from {@link TimingInfo} and `tls` from
|
|
103
|
+
* {@link CapturedFields} per Requirements 2.16, 2.24, and 2.25.
|
|
104
|
+
*/
|
|
105
|
+
private readonly isHttps;
|
|
106
|
+
/**
|
|
107
|
+
* Construct a fresh builder.
|
|
108
|
+
*
|
|
109
|
+
* @param opts.isHttps - Whether the request URL used `https://`.
|
|
110
|
+
* Controls whether `timing.tlsMs` and `tls` are populated.
|
|
111
|
+
* @param opts.finalHostname - Optional initial hostname; the transport
|
|
112
|
+
* will overwrite this via {@link setHopContext} as redirects are
|
|
113
|
+
* followed.
|
|
114
|
+
*/
|
|
115
|
+
constructor(opts: {
|
|
116
|
+
isHttps: boolean;
|
|
117
|
+
finalHostname?: string;
|
|
118
|
+
});
|
|
119
|
+
/**
|
|
120
|
+
* Record the hostname of the hop the transport is about to issue.
|
|
121
|
+
*
|
|
122
|
+
* Called once per hop, before {@link markDnsResolved}. The most-recent
|
|
123
|
+
* value becomes {@link CapturedFields.finalHostname}.
|
|
124
|
+
*/
|
|
125
|
+
setHopContext(hostname: string): void;
|
|
126
|
+
/**
|
|
127
|
+
* Record the DNS-resolution outcome for the current hop.
|
|
128
|
+
*
|
|
129
|
+
* The most-recent values overwrite any earlier ones so the values
|
|
130
|
+
* surfaced in {@link TimingInfo.dnsMs} and
|
|
131
|
+
* {@link CapturedFields.resolvedIp} correspond to the final hop.
|
|
132
|
+
*/
|
|
133
|
+
markDnsResolved(ms: number, ip: string): void;
|
|
134
|
+
/**
|
|
135
|
+
* Record the TCP-connect time for the current hop. The most-recent
|
|
136
|
+
* value wins (see class-level docstring for per-hop semantics).
|
|
137
|
+
*/
|
|
138
|
+
markTcpConnected(ms: number): void;
|
|
139
|
+
/**
|
|
140
|
+
* Record the TLS handshake time and extract {@link TlsInfo} from the
|
|
141
|
+
* final-hop `tls.TLSSocket`.
|
|
142
|
+
*
|
|
143
|
+
* - `protocol` ← `socket.getProtocol()` (`""` if null).
|
|
144
|
+
* - `cipher` ← `socket.getCipher().name` (`""` if missing).
|
|
145
|
+
* - `subjectCN/issuerCN` ← `cert.subject.CN` / `cert.issuer.CN`.
|
|
146
|
+
* - `subjectAltNames` ← parsed from `cert.subjectaltname`.
|
|
147
|
+
* - `notBefore/notAfter` ← `cert.valid_from` / `cert.valid_to` parsed
|
|
148
|
+
* to ISO 8601 (falls back to the raw value).
|
|
149
|
+
* - `fingerprintSha256` ← lowercase, colon-separated SHA-256 of
|
|
150
|
+
* `cert.raw` via `node:crypto`.
|
|
151
|
+
*
|
|
152
|
+
* Calling this method on an http:// fetch is harmless but pointless —
|
|
153
|
+
* the constructor's `isHttps=false` flag suppresses the field in
|
|
154
|
+
* {@link finalize} regardless.
|
|
155
|
+
*/
|
|
156
|
+
markTlsHandshaked(ms: number, socket: TLSSocket): void;
|
|
157
|
+
/**
|
|
158
|
+
* Record the final-hop response: HTTP status, raw headers
|
|
159
|
+
* (lowercased and joined into a {@link HeaderMap}), and TTFB.
|
|
160
|
+
*
|
|
161
|
+
* `Set-Cookie` is preserved in `headers` joined with `, ` like every
|
|
162
|
+
* other repeated header so the audit/redact passes can act on it.
|
|
163
|
+
* Per-cookie capture happens via {@link addSetCookieHeader}, which
|
|
164
|
+
* `fetch-core.ts` calls once per `Set-Cookie` line observed.
|
|
165
|
+
*/
|
|
166
|
+
markResponse(status: number, rawHeaders: IncomingHttpHeaders, ttfbMs: number): void;
|
|
167
|
+
/**
|
|
168
|
+
* Append a redirect hop to the chronological chain.
|
|
169
|
+
*
|
|
170
|
+
* Hops in excess of {@link MAX_REDIRECT_HOPS} are silently dropped so
|
|
171
|
+
* the array always satisfies the cap from Requirement 2.26.
|
|
172
|
+
* `fetch-core.ts` is responsible for surfacing the
|
|
173
|
+
* `redirect-limit` error when the cap is reached; this builder just
|
|
174
|
+
* stops accumulating.
|
|
175
|
+
*/
|
|
176
|
+
addRedirectHop(url: string, status: number, location?: string): void;
|
|
177
|
+
/**
|
|
178
|
+
* Parse one `Set-Cookie` header value via {@link parseSetCookie} and
|
|
179
|
+
* append the resulting {@link CookieInfo}, bounded at
|
|
180
|
+
* {@link MAX_COOKIES_CAPTURED}.
|
|
181
|
+
*
|
|
182
|
+
* Cookies in excess of the cap are silently dropped, matching
|
|
183
|
+
* Requirement 2.31.
|
|
184
|
+
*/
|
|
185
|
+
addSetCookieHeader(value: string): void;
|
|
186
|
+
/**
|
|
187
|
+
* Produce the {@link CapturedFields} snapshot.
|
|
188
|
+
*
|
|
189
|
+
* @param totalMs - Wall-clock duration of the whole invocation, in
|
|
190
|
+
* milliseconds. Stored in {@link TimingInfo.totalMs}.
|
|
191
|
+
*
|
|
192
|
+
* Optional fields obey the design's "include flags applied at the
|
|
193
|
+
* adapter, not at the builder" principle: this method always emits
|
|
194
|
+
* every field it observed, including `tls` (when `isHttps=true` and a
|
|
195
|
+
* handshake was captured). The fetch handler in `fetch.ts` is
|
|
196
|
+
* responsible for honouring `includeTls` / `includeTiming` /
|
|
197
|
+
* `includeRedirectChain` by stripping fields from the assembled
|
|
198
|
+
* {@link WebFetchMetadata} *after* this snapshot is produced.
|
|
199
|
+
*/
|
|
200
|
+
finalize(totalMs: number): CapturedFields;
|
|
201
|
+
}
|
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Capture` — pure observation builder for a single `web.fetch` invocation.
|
|
3
|
+
*
|
|
4
|
+
* The fetch transport in `fetch-core.ts` feeds events into this builder
|
|
5
|
+
* (DNS resolution, TCP connect, TLS handshake, response headers, redirect
|
|
6
|
+
* hops, `Set-Cookie` lines) and the builder accumulates the structured
|
|
7
|
+
* fields the {@link WebFetchMetadata} envelope needs:
|
|
8
|
+
*
|
|
9
|
+
* - {@link TimingInfo}: `dnsMs`, `tcpMs`, `tlsMs`, `ttfbMs`, `totalMs`
|
|
10
|
+
* - {@link TlsInfo} from the final hop's `tls.TLSSocket`
|
|
11
|
+
* (`getProtocol()`, `getCipher().name`, `getPeerCertificate(true)`)
|
|
12
|
+
* - response headers (lowercased keys, repeat values joined with `, `)
|
|
13
|
+
* - the redirect chain (capped at {@link MAX_REDIRECT_HOPS})
|
|
14
|
+
* - the resolved IP and the final hostname
|
|
15
|
+
* - `Set-Cookie` cookies (capped at {@link MAX_COOKIES_CAPTURED})
|
|
16
|
+
*
|
|
17
|
+
* The builder performs **no I/O** — it never touches the network, the
|
|
18
|
+
* filesystem, or `auditLog`. It is a pure data sink so the transport
|
|
19
|
+
* layer in `fetch-core.ts` can be tested independently of the
|
|
20
|
+
* Node `https`/`tls` machinery.
|
|
21
|
+
*
|
|
22
|
+
* Per-hop semantics:
|
|
23
|
+
* - `setHopContext(hostname)` is called by the transport at the start of
|
|
24
|
+
* each hop; the value of the *last* hop becomes
|
|
25
|
+
* {@link CapturedFields.finalHostname}.
|
|
26
|
+
* - {@link Capture.markDnsResolved} replaces the running `dnsMs` /
|
|
27
|
+
* `resolvedIp` so the values surfaced in {@link CapturedFields}
|
|
28
|
+
* correspond to the *final* hop. Same for `tcpMs`, `tlsMs`, and
|
|
29
|
+
* `ttfbMs` — only the last hop's measurements are returned.
|
|
30
|
+
* - {@link Capture.addRedirectHop} appends one entry per intermediate
|
|
31
|
+
* 3xx hop; the array is bounded at {@link MAX_REDIRECT_HOPS}.
|
|
32
|
+
* - {@link Capture.addSetCookieHeader} parses every observed
|
|
33
|
+
* `Set-Cookie` line via {@link parseSetCookie}; the resulting array is
|
|
34
|
+
* bounded at {@link MAX_COOKIES_CAPTURED}.
|
|
35
|
+
*
|
|
36
|
+
* Redaction and 4096-char header-value truncation are applied later by
|
|
37
|
+
* `redact.applyToHeaders` / `redact.applyToCookies` and the 64 KiB
|
|
38
|
+
* metadata budget by `budget.enforce`. This module only collects.
|
|
39
|
+
*/
|
|
40
|
+
import { createHash } from "node:crypto";
|
|
41
|
+
import { parseSetCookie } from "./readable.js";
|
|
42
|
+
import { MAX_COOKIES_CAPTURED, MAX_REDIRECT_HOPS, } from "./types.js";
|
|
43
|
+
/**
|
|
44
|
+
* Pure builder that records the per-hop observations made by
|
|
45
|
+
* `fetch-core.ts` and assembles them into a {@link CapturedFields}
|
|
46
|
+
* snapshot via {@link Capture.finalize}.
|
|
47
|
+
*
|
|
48
|
+
* The builder is single-use: callers should construct one `Capture` per
|
|
49
|
+
* `web.fetch` invocation and discard it after `finalize`.
|
|
50
|
+
*/
|
|
51
|
+
export class Capture {
|
|
52
|
+
/** Final-hop DNS-resolution time in milliseconds. */
|
|
53
|
+
dnsMs = 0;
|
|
54
|
+
/** Final-hop TCP-connect time in milliseconds. */
|
|
55
|
+
tcpMs = 0;
|
|
56
|
+
/** Final-hop TLS-handshake time in milliseconds (https only). */
|
|
57
|
+
tlsMs = undefined;
|
|
58
|
+
/** Final-hop time-to-first-byte in milliseconds. */
|
|
59
|
+
ttfbMs = 0;
|
|
60
|
+
/** Final-hop resolved IP, set by {@link markDnsResolved}. */
|
|
61
|
+
resolvedIp = "";
|
|
62
|
+
/** Final-hop hostname, set by {@link setHopContext}. */
|
|
63
|
+
finalHostname = "";
|
|
64
|
+
/** Final-hop HTTP status, set by {@link markResponse}. */
|
|
65
|
+
status = 0;
|
|
66
|
+
/** Final-hop normalised headers, set by {@link markResponse}. */
|
|
67
|
+
headers = {};
|
|
68
|
+
/** Final-hop TLS info (https only), set by {@link markTlsHandshaked}. */
|
|
69
|
+
tls = undefined;
|
|
70
|
+
/** Redirect hops, capped at {@link MAX_REDIRECT_HOPS}. */
|
|
71
|
+
redirectChain = [];
|
|
72
|
+
/** Captured cookies, capped at {@link MAX_COOKIES_CAPTURED}. */
|
|
73
|
+
cookies = [];
|
|
74
|
+
/**
|
|
75
|
+
* Whether the current invocation is an `https://` fetch. When `false`,
|
|
76
|
+
* `tlsMs` is omitted from {@link TimingInfo} and `tls` from
|
|
77
|
+
* {@link CapturedFields} per Requirements 2.16, 2.24, and 2.25.
|
|
78
|
+
*/
|
|
79
|
+
isHttps;
|
|
80
|
+
/**
|
|
81
|
+
* Construct a fresh builder.
|
|
82
|
+
*
|
|
83
|
+
* @param opts.isHttps - Whether the request URL used `https://`.
|
|
84
|
+
* Controls whether `timing.tlsMs` and `tls` are populated.
|
|
85
|
+
* @param opts.finalHostname - Optional initial hostname; the transport
|
|
86
|
+
* will overwrite this via {@link setHopContext} as redirects are
|
|
87
|
+
* followed.
|
|
88
|
+
*/
|
|
89
|
+
constructor(opts) {
|
|
90
|
+
this.isHttps = opts.isHttps;
|
|
91
|
+
if (typeof opts.finalHostname === "string") {
|
|
92
|
+
this.finalHostname = opts.finalHostname;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Record the hostname of the hop the transport is about to issue.
|
|
97
|
+
*
|
|
98
|
+
* Called once per hop, before {@link markDnsResolved}. The most-recent
|
|
99
|
+
* value becomes {@link CapturedFields.finalHostname}.
|
|
100
|
+
*/
|
|
101
|
+
setHopContext(hostname) {
|
|
102
|
+
if (typeof hostname === "string" && hostname.length > 0) {
|
|
103
|
+
this.finalHostname = hostname;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Record the DNS-resolution outcome for the current hop.
|
|
108
|
+
*
|
|
109
|
+
* The most-recent values overwrite any earlier ones so the values
|
|
110
|
+
* surfaced in {@link TimingInfo.dnsMs} and
|
|
111
|
+
* {@link CapturedFields.resolvedIp} correspond to the final hop.
|
|
112
|
+
*/
|
|
113
|
+
markDnsResolved(ms, ip) {
|
|
114
|
+
this.dnsMs = sanitiseMs(ms);
|
|
115
|
+
if (typeof ip === "string" && ip.length > 0) {
|
|
116
|
+
this.resolvedIp = ip;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Record the TCP-connect time for the current hop. The most-recent
|
|
121
|
+
* value wins (see class-level docstring for per-hop semantics).
|
|
122
|
+
*/
|
|
123
|
+
markTcpConnected(ms) {
|
|
124
|
+
this.tcpMs = sanitiseMs(ms);
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Record the TLS handshake time and extract {@link TlsInfo} from the
|
|
128
|
+
* final-hop `tls.TLSSocket`.
|
|
129
|
+
*
|
|
130
|
+
* - `protocol` ← `socket.getProtocol()` (`""` if null).
|
|
131
|
+
* - `cipher` ← `socket.getCipher().name` (`""` if missing).
|
|
132
|
+
* - `subjectCN/issuerCN` ← `cert.subject.CN` / `cert.issuer.CN`.
|
|
133
|
+
* - `subjectAltNames` ← parsed from `cert.subjectaltname`.
|
|
134
|
+
* - `notBefore/notAfter` ← `cert.valid_from` / `cert.valid_to` parsed
|
|
135
|
+
* to ISO 8601 (falls back to the raw value).
|
|
136
|
+
* - `fingerprintSha256` ← lowercase, colon-separated SHA-256 of
|
|
137
|
+
* `cert.raw` via `node:crypto`.
|
|
138
|
+
*
|
|
139
|
+
* Calling this method on an http:// fetch is harmless but pointless —
|
|
140
|
+
* the constructor's `isHttps=false` flag suppresses the field in
|
|
141
|
+
* {@link finalize} regardless.
|
|
142
|
+
*/
|
|
143
|
+
markTlsHandshaked(ms, socket) {
|
|
144
|
+
this.tlsMs = sanitiseMs(ms);
|
|
145
|
+
this.tls = extractTlsInfo(socket);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Record the final-hop response: HTTP status, raw headers
|
|
149
|
+
* (lowercased and joined into a {@link HeaderMap}), and TTFB.
|
|
150
|
+
*
|
|
151
|
+
* `Set-Cookie` is preserved in `headers` joined with `, ` like every
|
|
152
|
+
* other repeated header so the audit/redact passes can act on it.
|
|
153
|
+
* Per-cookie capture happens via {@link addSetCookieHeader}, which
|
|
154
|
+
* `fetch-core.ts` calls once per `Set-Cookie` line observed.
|
|
155
|
+
*/
|
|
156
|
+
markResponse(status, rawHeaders, ttfbMs) {
|
|
157
|
+
this.status = Number.isInteger(status) ? status : 0;
|
|
158
|
+
this.ttfbMs = sanitiseMs(ttfbMs);
|
|
159
|
+
this.headers = normaliseHeaders(rawHeaders);
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Append a redirect hop to the chronological chain.
|
|
163
|
+
*
|
|
164
|
+
* Hops in excess of {@link MAX_REDIRECT_HOPS} are silently dropped so
|
|
165
|
+
* the array always satisfies the cap from Requirement 2.26.
|
|
166
|
+
* `fetch-core.ts` is responsible for surfacing the
|
|
167
|
+
* `redirect-limit` error when the cap is reached; this builder just
|
|
168
|
+
* stops accumulating.
|
|
169
|
+
*/
|
|
170
|
+
addRedirectHop(url, status, location) {
|
|
171
|
+
if (this.redirectChain.length >= MAX_REDIRECT_HOPS)
|
|
172
|
+
return;
|
|
173
|
+
if (typeof url !== "string" || url.length === 0)
|
|
174
|
+
return;
|
|
175
|
+
const hop = {
|
|
176
|
+
url,
|
|
177
|
+
status: Number.isInteger(status) ? status : 0,
|
|
178
|
+
};
|
|
179
|
+
if (typeof location === "string" && location.length > 0) {
|
|
180
|
+
hop.location = location;
|
|
181
|
+
}
|
|
182
|
+
this.redirectChain.push(hop);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Parse one `Set-Cookie` header value via {@link parseSetCookie} and
|
|
186
|
+
* append the resulting {@link CookieInfo}, bounded at
|
|
187
|
+
* {@link MAX_COOKIES_CAPTURED}.
|
|
188
|
+
*
|
|
189
|
+
* Cookies in excess of the cap are silently dropped, matching
|
|
190
|
+
* Requirement 2.31.
|
|
191
|
+
*/
|
|
192
|
+
addSetCookieHeader(value) {
|
|
193
|
+
if (this.cookies.length >= MAX_COOKIES_CAPTURED)
|
|
194
|
+
return;
|
|
195
|
+
if (typeof value !== "string" || value.length === 0)
|
|
196
|
+
return;
|
|
197
|
+
this.cookies.push(parseSetCookie(value));
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Produce the {@link CapturedFields} snapshot.
|
|
201
|
+
*
|
|
202
|
+
* @param totalMs - Wall-clock duration of the whole invocation, in
|
|
203
|
+
* milliseconds. Stored in {@link TimingInfo.totalMs}.
|
|
204
|
+
*
|
|
205
|
+
* Optional fields obey the design's "include flags applied at the
|
|
206
|
+
* adapter, not at the builder" principle: this method always emits
|
|
207
|
+
* every field it observed, including `tls` (when `isHttps=true` and a
|
|
208
|
+
* handshake was captured). The fetch handler in `fetch.ts` is
|
|
209
|
+
* responsible for honouring `includeTls` / `includeTiming` /
|
|
210
|
+
* `includeRedirectChain` by stripping fields from the assembled
|
|
211
|
+
* {@link WebFetchMetadata} *after* this snapshot is produced.
|
|
212
|
+
*/
|
|
213
|
+
finalize(totalMs) {
|
|
214
|
+
const timing = {
|
|
215
|
+
dnsMs: this.dnsMs,
|
|
216
|
+
tcpMs: this.tcpMs,
|
|
217
|
+
ttfbMs: this.ttfbMs,
|
|
218
|
+
totalMs: sanitiseMs(totalMs),
|
|
219
|
+
};
|
|
220
|
+
if (this.isHttps && this.tlsMs !== undefined) {
|
|
221
|
+
timing.tlsMs = this.tlsMs;
|
|
222
|
+
}
|
|
223
|
+
const fields = {
|
|
224
|
+
resolvedIp: this.resolvedIp,
|
|
225
|
+
finalHostname: this.finalHostname,
|
|
226
|
+
status: this.status,
|
|
227
|
+
headers: this.headers,
|
|
228
|
+
timing,
|
|
229
|
+
redirectChain: this.redirectChain.slice(),
|
|
230
|
+
cookies: this.cookies.slice(),
|
|
231
|
+
};
|
|
232
|
+
if (this.isHttps && this.tls !== undefined) {
|
|
233
|
+
fields.tls = this.tls;
|
|
234
|
+
}
|
|
235
|
+
return fields;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
// ---------------------------------------------------------------------------
|
|
239
|
+
// Internals
|
|
240
|
+
// ---------------------------------------------------------------------------
|
|
241
|
+
/**
|
|
242
|
+
* Coerce an arbitrary millisecond value into a non-negative finite
|
|
243
|
+
* integer. Used to keep timing arithmetic resilient to clock skew or
|
|
244
|
+
* non-numeric inputs from a stubbed transport.
|
|
245
|
+
*/
|
|
246
|
+
function sanitiseMs(ms) {
|
|
247
|
+
if (typeof ms !== "number" || !Number.isFinite(ms))
|
|
248
|
+
return 0;
|
|
249
|
+
if (ms < 0)
|
|
250
|
+
return 0;
|
|
251
|
+
return Math.round(ms);
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Lower-case every header key and join repeat values per RFC 7230 with
|
|
255
|
+
* `, ` so the `redact`/`budget` passes downstream see a uniform
|
|
256
|
+
* `Record<string, string>` shape. Header-value length truncation is
|
|
257
|
+
* deferred to `redact.applyToHeaders` (4096-char cap from
|
|
258
|
+
* Requirement 2.21) so this builder stays purely observational.
|
|
259
|
+
*/
|
|
260
|
+
function normaliseHeaders(raw) {
|
|
261
|
+
const out = {};
|
|
262
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
263
|
+
if (value === undefined)
|
|
264
|
+
continue;
|
|
265
|
+
const lowerKey = key.toLowerCase();
|
|
266
|
+
if (Array.isArray(value)) {
|
|
267
|
+
out[lowerKey] = value.join(", ");
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
out[lowerKey] = String(value);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return out;
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Pull the TLS session details we surface from the leaf certificate.
|
|
277
|
+
*
|
|
278
|
+
* `getCipher()` returns `null` for sessions that have not completed the
|
|
279
|
+
* handshake — defensive `?? ""` keeps the field present rather than
|
|
280
|
+
* throwing when fed a half-initialised socket from a test stub.
|
|
281
|
+
*/
|
|
282
|
+
function extractTlsInfo(socket) {
|
|
283
|
+
const protocol = socket.getProtocol() ?? "";
|
|
284
|
+
const cipherInfo = socket.getCipher();
|
|
285
|
+
const cipher = cipherInfo?.name ?? "";
|
|
286
|
+
// `getPeerCertificate(true)` returns the detailed certificate object
|
|
287
|
+
// including the DER `raw` bytes we need for the SHA-256 fingerprint.
|
|
288
|
+
// Some test stubs return a plain object lacking `raw`; we guard for
|
|
289
|
+
// that with a typeof check below.
|
|
290
|
+
const cert = socket.getPeerCertificate(true);
|
|
291
|
+
const subjectCN = pickCN(cert?.subject?.CN);
|
|
292
|
+
const issuerCN = pickCN(cert?.issuer?.CN);
|
|
293
|
+
const subjectAltNames = parseSubjectAltName(cert?.subjectaltname);
|
|
294
|
+
const notBefore = isoFromCertDate(cert?.valid_from);
|
|
295
|
+
const notAfter = isoFromCertDate(cert?.valid_to);
|
|
296
|
+
const fingerprintSha256 = computeSha256Fingerprint(cert?.raw);
|
|
297
|
+
return {
|
|
298
|
+
protocol,
|
|
299
|
+
cipher,
|
|
300
|
+
subjectCN,
|
|
301
|
+
issuerCN,
|
|
302
|
+
subjectAltNames,
|
|
303
|
+
notBefore,
|
|
304
|
+
notAfter,
|
|
305
|
+
fingerprintSha256,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* `tls.Certificate.CN` is typed as `string | string[] | undefined`.
|
|
310
|
+
* When multiple CN values are present, we take the first to keep the
|
|
311
|
+
* field shape simple; SAN expansion already covers the multi-name case.
|
|
312
|
+
*/
|
|
313
|
+
function pickCN(cn) {
|
|
314
|
+
if (typeof cn === "string")
|
|
315
|
+
return cn;
|
|
316
|
+
if (Array.isArray(cn) && cn.length > 0) {
|
|
317
|
+
return typeof cn[0] === "string" ? cn[0] : "";
|
|
318
|
+
}
|
|
319
|
+
return "";
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Parse `cert.subjectaltname` (the comma-separated string emitted by
|
|
323
|
+
* Node's `getPeerCertificate`, e.g. `"DNS:example.com, IP Address:1.2.3.4"`)
|
|
324
|
+
* into a string array. The type prefix (`DNS:`, `IP Address:`, `URI:`,
|
|
325
|
+
* `email:`) is preserved so callers retain SAN-type information.
|
|
326
|
+
*
|
|
327
|
+
* Empty / undefined input yields an empty array.
|
|
328
|
+
*/
|
|
329
|
+
function parseSubjectAltName(san) {
|
|
330
|
+
if (typeof san !== "string" || san.length === 0)
|
|
331
|
+
return [];
|
|
332
|
+
return san
|
|
333
|
+
.split(",")
|
|
334
|
+
.map((s) => s.trim())
|
|
335
|
+
.filter((s) => s.length > 0);
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Convert a certificate `valid_from`/`valid_to` date string (which uses
|
|
339
|
+
* the OpenSSL `MMM DD HH:mm:ss YYYY GMT` format) into ISO 8601. Falls
|
|
340
|
+
* back to the raw input when parsing fails so the field remains useful
|
|
341
|
+
* for forensic review even if the format is unexpected.
|
|
342
|
+
*/
|
|
343
|
+
function isoFromCertDate(value) {
|
|
344
|
+
if (typeof value !== "string" || value.length === 0)
|
|
345
|
+
return "";
|
|
346
|
+
const ms = Date.parse(value);
|
|
347
|
+
if (!Number.isFinite(ms))
|
|
348
|
+
return value;
|
|
349
|
+
return new Date(ms).toISOString();
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Compute the SHA-256 digest of the certificate's DER bytes via
|
|
353
|
+
* `node:crypto` and format it as a lowercase, colon-separated hex
|
|
354
|
+
* string (`"ab:cd:ef:..."`), matching the `cert.fingerprint256` shape
|
|
355
|
+
* Node exposes for SHA-1 digests.
|
|
356
|
+
*
|
|
357
|
+
* Returns `""` when `raw` is missing or not a Buffer-like value (e.g. a
|
|
358
|
+
* test stub that omitted the field).
|
|
359
|
+
*/
|
|
360
|
+
function computeSha256Fingerprint(raw) {
|
|
361
|
+
if (raw === undefined || raw === null)
|
|
362
|
+
return "";
|
|
363
|
+
let bytes;
|
|
364
|
+
if (Buffer.isBuffer(raw)) {
|
|
365
|
+
bytes = raw;
|
|
366
|
+
}
|
|
367
|
+
else if (raw instanceof Uint8Array) {
|
|
368
|
+
bytes = Buffer.from(raw);
|
|
369
|
+
}
|
|
370
|
+
else {
|
|
371
|
+
return "";
|
|
372
|
+
}
|
|
373
|
+
const hex = createHash("sha256").update(bytes).digest("hex");
|
|
374
|
+
// Pair-up the hex string into colon-separated bytes: `abcdef…` →
|
|
375
|
+
// `ab:cd:ef:…`. The regex match is bounded by the deterministic
|
|
376
|
+
// 64-char output of SHA-256 so there is no unbounded work here.
|
|
377
|
+
const pairs = hex.match(/.{2}/g) ?? [];
|
|
378
|
+
return pairs.join(":");
|
|
379
|
+
}
|
|
380
|
+
//# sourceMappingURL=capture.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"capture.js","sourceRoot":"","sources":["../../../src/tools/web/capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EACL,oBAAoB,EACpB,iBAAiB,GAOlB,MAAM,YAAY,CAAC;AA6BpB;;;;;;;GAOG;AACH,MAAM,OAAO,OAAO;IAClB,qDAAqD;IAC7C,KAAK,GAAG,CAAC,CAAC;IAClB,kDAAkD;IAC1C,KAAK,GAAG,CAAC,CAAC;IAClB,iEAAiE;IACzD,KAAK,GAAuB,SAAS,CAAC;IAC9C,oDAAoD;IAC5C,MAAM,GAAG,CAAC,CAAC;IAEnB,6DAA6D;IACrD,UAAU,GAAG,EAAE,CAAC;IACxB,wDAAwD;IAChD,aAAa,GAAG,EAAE,CAAC;IAC3B,0DAA0D;IAClD,MAAM,GAAG,CAAC,CAAC;IACnB,iEAAiE;IACzD,OAAO,GAAc,EAAE,CAAC;IAChC,yEAAyE;IACjE,GAAG,GAAwB,SAAS,CAAC;IAC7C,0DAA0D;IACzC,aAAa,GAAkB,EAAE,CAAC;IACnD,gEAAgE;IAC/C,OAAO,GAAiB,EAAE,CAAC;IAE5C;;;;OAIG;IACc,OAAO,CAAU;IAElC;;;;;;;;OAQG;IACH,YAAY,IAAkD;QAC5D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;YAC3C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QAC1C,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,aAAa,CAAC,QAAgB;QAC5B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC;QAChC,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,eAAe,CAAC,EAAU,EAAE,EAAU;QACpC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,EAAU;QACzB,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,iBAAiB,CAAC,EAAU,EAAE,MAAiB;QAC7C,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;OAQG;IACH,YAAY,CACV,MAAc,EACd,UAA+B,EAC/B,MAAc;QAEd,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;OAQG;IACH,cAAc,CAAC,GAAW,EAAE,MAAc,EAAE,QAAiB;QAC3D,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,IAAI,iBAAiB;YAAE,OAAO;QAC3D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACxD,MAAM,GAAG,GAAgB;YACvB,GAAG;YACH,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,CAAC;QACF,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC1B,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;OAOG;IACH,kBAAkB,CAAC,KAAa;QAC9B,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,oBAAoB;YAAE,OAAO;QACxD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC5D,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,OAAe;QACtB,MAAM,MAAM,GAAe;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC;SAC7B,CAAC;QACF,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,MAAM,MAAM,GAAmB;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM;YACN,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE;YACzC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE;SAC9B,CAAC;QACF,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACxB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,UAAU,CAAC,EAAU;IAC5B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7D,IAAI,EAAE,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,GAAwB;IAChD,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,KAAK,KAAK,SAAS;YAAE,SAAS;QAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,MAAiB;IACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC;IAEtC,qEAAqE;IACrE,qEAAqE;IACrE,oEAAoE;IACpE,kCAAkC;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1C,MAAM,eAAe,GAAG,mBAAmB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACjD,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAE9D,OAAO;QACL,QAAQ;QACR,MAAM;QACN,SAAS;QACT,QAAQ;QACR,eAAe;QACf,SAAS;QACT,QAAQ;QACR,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,EAAiC;IAC/C,IAAI,OAAO,EAAE,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,mBAAmB,CAAC,GAAuB;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC3D,OAAO,GAAG;SACP,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CAAC,KAAyB;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC/D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACpC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,wBAAwB,CAAC,GAAY;IAC5C,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IACjD,IAAI,KAAa,CAAC;IAClB,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,GAAG,GAAG,CAAC;IACd,CAAC;SAAM,IAAI,GAAG,YAAY,UAAU,EAAE,CAAC;QACrC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7D,iEAAiE;IACjE,gEAAgE;IAChE,gEAAgE;IAChE,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IACvC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `web.fetch` core orchestration.
|
|
3
|
+
*
|
|
4
|
+
* This module is the deterministic, dependency-injected pipeline that
|
|
5
|
+
* `src/tools/web/fetch.ts` (added in task 4.x) wraps in a `ToolResult`
|
|
6
|
+
* adapter and audit-log emitter. The core itself returns a typed
|
|
7
|
+
* {@link WebFetchOutcome} and never touches the registry, the safety
|
|
8
|
+
* classifier, or `auditLog`.
|
|
9
|
+
*
|
|
10
|
+
* The pipeline implements the design's "Pipeline steps in detail"
|
|
11
|
+
* sequence (`.kiro/specs/web-search-and-fetch/design.md`):
|
|
12
|
+
*
|
|
13
|
+
* 1. argument validation
|
|
14
|
+
* 2. URL parse + SSRF pre-check on hostname literal
|
|
15
|
+
* 3. DNS resolve + IP pin
|
|
16
|
+
* 4. SSRF check on resolved IP
|
|
17
|
+
* 5. pinned `https.request` (or `http.request`) with custom `lookup`
|
|
18
|
+
* 6. TLS handshake capture
|
|
19
|
+
* 7. response headers + body stream with `maxBytes` cap
|
|
20
|
+
* 8. redirect handling (≤ {@link MAX_REDIRECT_HOPS}, re-running
|
|
21
|
+
* validation + SSRF + DNS at each hop)
|
|
22
|
+
* 9. body classification (binary / raw / readable)
|
|
23
|
+
* 10. metadata assembly + 64 KiB budget enforcement
|
|
24
|
+
*
|
|
25
|
+
* Every outbound transport call (DNS, HTTP, HTTPS) is injectable via
|
|
26
|
+
* {@link WebFetchCoreOptions} so tests in epics 3.x, 4.x and 6.x can
|
|
27
|
+
* stub the network deterministically without spinning up a real server.
|
|
28
|
+
*/
|
|
29
|
+
import { lookup as defaultDnsLookup } from "node:dns/promises";
|
|
30
|
+
import type { ClientRequest, IncomingMessage, RequestOptions } from "node:http";
|
|
31
|
+
import { type WebFetchArgs, type WebFetchErrorKind, type WebFetchOutcome } from "./types.js";
|
|
32
|
+
/** Signature of `node:dns/promises.lookup`. */
|
|
33
|
+
export type DnsLookupFn = typeof defaultDnsLookup;
|
|
34
|
+
/** Signature of `node:http.request` (the overload accepting URL + options). */
|
|
35
|
+
export type HttpRequestFn = (url: string | URL, options: RequestOptions, callback?: (res: IncomingMessage) => void) => ClientRequest;
|
|
36
|
+
/** Signature of `node:https.request` (same shape as {@link HttpRequestFn}). */
|
|
37
|
+
export type HttpsRequestFn = HttpRequestFn;
|
|
38
|
+
/**
|
|
39
|
+
* Injection points for {@link webFetchCore}. Each defaults to the
|
|
40
|
+
* corresponding `node:` built-in so production code does not need to
|
|
41
|
+
* supply anything; tests stub these to drive the pipeline without
|
|
42
|
+
* touching the network.
|
|
43
|
+
*/
|
|
44
|
+
export interface WebFetchCoreOptions {
|
|
45
|
+
/** HTTPS transport. Defaults to `https.request`. */
|
|
46
|
+
httpsRequest?: HttpsRequestFn;
|
|
47
|
+
/** HTTP transport. Defaults to `http.request`. */
|
|
48
|
+
httpRequest?: HttpRequestFn;
|
|
49
|
+
/** DNS resolver. Defaults to `dns/promises.lookup`. */
|
|
50
|
+
dnsLookup?: DnsLookupFn;
|
|
51
|
+
/** Wall-clock source for timing fields. Defaults to `Date.now`. */
|
|
52
|
+
now?: () => number;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Run the full `web.fetch` pipeline for the given arguments.
|
|
56
|
+
*
|
|
57
|
+
* Returns a typed {@link WebFetchOutcome}. The outcome is never thrown
|
|
58
|
+
* — argument validation failures, SSRF blocks, network errors, HTTP
|
|
59
|
+
* errors, and timeouts all surface as `ok=false` with a categorical
|
|
60
|
+
* `error.kind` and a human-readable message. The `metadata` field is
|
|
61
|
+
* always populated: pipeline stages that completed before the failure
|
|
62
|
+
* are surfaced (e.g. `resolvedIp` when DNS succeeded but a 4xx came
|
|
63
|
+
* back), and stages that did not run carry default zero/empty values.
|
|
64
|
+
*/
|
|
65
|
+
export declare function webFetchCore(args: WebFetchArgs, options?: WebFetchCoreOptions): Promise<WebFetchOutcome>;
|
|
66
|
+
export type { WebFetchErrorKind };
|