@pentoshi/clai 0.10.4 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -0
- package/dist/agent/runner.js +41 -3
- package/dist/agent/runner.js.map +1 -1
- package/dist/commands/providers.js +28 -0
- package/dist/commands/providers.js.map +1 -1
- package/dist/commands/search-providers.d.ts +50 -0
- package/dist/commands/search-providers.js +134 -0
- package/dist/commands/search-providers.js.map +1 -0
- package/dist/commands/update.js +1 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/provider.js +9 -6
- package/dist/llm/provider.js.map +1 -1
- package/dist/prompts/index.d.ts +1 -1
- package/dist/prompts/index.js +6 -0
- package/dist/prompts/index.js.map +1 -1
- package/dist/repl.d.ts +1 -0
- package/dist/repl.js +139 -113
- package/dist/repl.js.map +1 -1
- package/dist/safety/classifier.js +40 -0
- package/dist/safety/classifier.js.map +1 -1
- package/dist/store/config.d.ts +5 -0
- package/dist/store/config.js +7 -0
- package/dist/store/config.js.map +1 -1
- package/dist/store/keys.d.ts +65 -0
- package/dist/store/keys.js +164 -28
- package/dist/store/keys.js.map +1 -1
- package/dist/tools/http.d.ts +12 -1
- package/dist/tools/http.js +8 -43
- package/dist/tools/http.js.map +1 -1
- package/dist/tools/registry.js +52 -0
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/shell.d.ts +25 -0
- package/dist/tools/shell.js +155 -6
- package/dist/tools/shell.js.map +1 -1
- package/dist/tools/web/audit.d.ts +154 -0
- package/dist/tools/web/audit.js +147 -0
- package/dist/tools/web/audit.js.map +1 -0
- package/dist/tools/web/budget.d.ts +76 -0
- package/dist/tools/web/budget.js +187 -0
- package/dist/tools/web/budget.js.map +1 -0
- package/dist/tools/web/capture.d.ts +201 -0
- package/dist/tools/web/capture.js +380 -0
- package/dist/tools/web/capture.js.map +1 -0
- package/dist/tools/web/fetch-core.d.ts +66 -0
- package/dist/tools/web/fetch-core.js +1123 -0
- package/dist/tools/web/fetch-core.js.map +1 -0
- package/dist/tools/web/fetch.d.ts +42 -0
- package/dist/tools/web/fetch.js +115 -0
- package/dist/tools/web/fetch.js.map +1 -0
- package/dist/tools/web/providers/brave.d.ts +46 -0
- package/dist/tools/web/providers/brave.js +263 -0
- package/dist/tools/web/providers/brave.js.map +1 -0
- package/dist/tools/web/providers/duckduckgo.d.ts +47 -0
- package/dist/tools/web/providers/duckduckgo.js +248 -0
- package/dist/tools/web/providers/duckduckgo.js.map +1 -0
- package/dist/tools/web/providers/provider.d.ts +99 -0
- package/dist/tools/web/providers/provider.js +38 -0
- package/dist/tools/web/providers/provider.js.map +1 -0
- package/dist/tools/web/providers/tavily.d.ts +52 -0
- package/dist/tools/web/providers/tavily.js +285 -0
- package/dist/tools/web/providers/tavily.js.map +1 -0
- package/dist/tools/web/readable.d.ts +67 -0
- package/dist/tools/web/readable.js +248 -0
- package/dist/tools/web/readable.js.map +1 -0
- package/dist/tools/web/redact.d.ts +120 -0
- package/dist/tools/web/redact.js +155 -0
- package/dist/tools/web/redact.js.map +1 -0
- package/dist/tools/web/search.d.ts +51 -0
- package/dist/tools/web/search.js +389 -0
- package/dist/tools/web/search.js.map +1 -0
- package/dist/tools/web/ssrf-guard.d.ts +85 -0
- package/dist/tools/web/ssrf-guard.js +265 -0
- package/dist/tools/web/ssrf-guard.js.map +1 -0
- package/dist/tools/web/types.d.ts +331 -0
- package/dist/tools/web/types.js +71 -0
- package/dist/tools/web/types.js.map +1 -0
- package/dist/ui/keys.js +3 -2
- package/dist/ui/keys.js.map +1 -1
- package/dist/ui/spinner.js +87 -14
- package/dist/ui/spinner.js.map +1 -1
- package/package.json +3 -1
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 64 KiB metadata budget enforcement for `web.fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Requirement 2.35 caps the combined serialized size of the
|
|
5
|
+
* {@link HeaderMap}, {@link TlsInfo}, {@link TimingInfo},
|
|
6
|
+
* {@link RedirectChain}, and `cookies` array at
|
|
7
|
+
* {@link METADATA_BUDGET_BYTES} (64 KiB). When the assembled metadata
|
|
8
|
+
* exceeds the cap, this module reduces it deterministically using the
|
|
9
|
+
* order documented in `.kiro/specs/web-search-and-fetch/design.md`,
|
|
10
|
+
* "Truncation order at the 64 KiB cap":
|
|
11
|
+
*
|
|
12
|
+
* 1. Drop trailing cookies (last entry first) — Requirement 2.35.
|
|
13
|
+
* 2. Halve the longest header value (with the literal
|
|
14
|
+
* {@link TRUNCATION_MARKER} appended) until every header value's
|
|
15
|
+
* "content" portion is ≤ 1024 characters. Header keys are never
|
|
16
|
+
* removed so the agent always sees which headers were present.
|
|
17
|
+
* 3. Drop trailing redirect hops one at a time.
|
|
18
|
+
*
|
|
19
|
+
* `tls` and `timing` are well under 1 KiB combined and are never touched
|
|
20
|
+
* by this loop. The function is pure: caller inputs are not mutated; the
|
|
21
|
+
* returned object holds fresh copies of any arrays/maps that were
|
|
22
|
+
* shortened.
|
|
23
|
+
*
|
|
24
|
+
* The implementation is bounded to a fixed iteration count so a
|
|
25
|
+
* pathological input (for example, hundreds of header values that are
|
|
26
|
+
* all ≤ 1024 chars yet collectively still exceed the cap) cannot wedge
|
|
27
|
+
* the agent. In that edge case the returned `metadataBytes` may exceed
|
|
28
|
+
* {@link METADATA_BUDGET_BYTES} — the caller can decide whether to
|
|
29
|
+
* surface the overflow or treat it as a soft warning.
|
|
30
|
+
*/
|
|
31
|
+
import { type CookieInfo, type HeaderMap, type RedirectChain, type TimingInfo, type TlsInfo, METADATA_BUDGET_BYTES } from "./types.js";
|
|
32
|
+
/**
|
|
33
|
+
* Inputs accepted by {@link enforce}.
|
|
34
|
+
*
|
|
35
|
+
* Each field corresponds to one slice of {@link WebFetchMetadata}.
|
|
36
|
+
* Fields that the caller chose not to include (for example because the
|
|
37
|
+
* user passed `includeHeaders=false`) may be omitted or set to
|
|
38
|
+
* `undefined`; they are passed through to the result unchanged.
|
|
39
|
+
*/
|
|
40
|
+
export interface BudgetInput {
|
|
41
|
+
headers?: HeaderMap | undefined;
|
|
42
|
+
tls?: TlsInfo | undefined;
|
|
43
|
+
timing?: TimingInfo | undefined;
|
|
44
|
+
redirectChain?: RedirectChain | undefined;
|
|
45
|
+
cookies?: CookieInfo[] | undefined;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Result returned by {@link enforce}. Optional fields are present iff
|
|
49
|
+
* the corresponding input field was present (an empty array is still
|
|
50
|
+
* "present"). `metadataBytes` is the UTF-8 byte length of
|
|
51
|
+
* `JSON.stringify({headers, tls, timing, redirectChain, cookies})` after
|
|
52
|
+
* the truncation loop has finished. `cap` mirrors the constant from
|
|
53
|
+
* `types.ts` so consumers can render `metadataBytes / cap` without
|
|
54
|
+
* importing it themselves.
|
|
55
|
+
*/
|
|
56
|
+
export interface BudgetResult {
|
|
57
|
+
headers?: HeaderMap;
|
|
58
|
+
tls?: TlsInfo;
|
|
59
|
+
timing?: TimingInfo;
|
|
60
|
+
redirectChain?: RedirectChain;
|
|
61
|
+
cookies?: CookieInfo[];
|
|
62
|
+
metadataBytes: number;
|
|
63
|
+
cap: typeof METADATA_BUDGET_BYTES;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Enforce the 64 KiB metadata budget on the supplied fields.
|
|
67
|
+
*
|
|
68
|
+
* Returns a fresh {@link BudgetResult} containing (possibly shortened)
|
|
69
|
+
* copies of `headers`, `redirectChain`, and `cookies`, alongside `tls`
|
|
70
|
+
* and `timing` passed through unchanged, plus the final
|
|
71
|
+
* `metadataBytes` count.
|
|
72
|
+
*
|
|
73
|
+
* The returned arrays/maps are independent of the caller's inputs — the
|
|
74
|
+
* function does not mutate `input`.
|
|
75
|
+
*/
|
|
76
|
+
export declare function enforce(input: BudgetInput): BudgetResult;
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 64 KiB metadata budget enforcement for `web.fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Requirement 2.35 caps the combined serialized size of the
|
|
5
|
+
* {@link HeaderMap}, {@link TlsInfo}, {@link TimingInfo},
|
|
6
|
+
* {@link RedirectChain}, and `cookies` array at
|
|
7
|
+
* {@link METADATA_BUDGET_BYTES} (64 KiB). When the assembled metadata
|
|
8
|
+
* exceeds the cap, this module reduces it deterministically using the
|
|
9
|
+
* order documented in `.kiro/specs/web-search-and-fetch/design.md`,
|
|
10
|
+
* "Truncation order at the 64 KiB cap":
|
|
11
|
+
*
|
|
12
|
+
* 1. Drop trailing cookies (last entry first) — Requirement 2.35.
|
|
13
|
+
* 2. Halve the longest header value (with the literal
|
|
14
|
+
* {@link TRUNCATION_MARKER} appended) until every header value's
|
|
15
|
+
* "content" portion is ≤ 1024 characters. Header keys are never
|
|
16
|
+
* removed so the agent always sees which headers were present.
|
|
17
|
+
* 3. Drop trailing redirect hops one at a time.
|
|
18
|
+
*
|
|
19
|
+
* `tls` and `timing` are well under 1 KiB combined and are never touched
|
|
20
|
+
* by this loop. The function is pure: caller inputs are not mutated; the
|
|
21
|
+
* returned object holds fresh copies of any arrays/maps that were
|
|
22
|
+
* shortened.
|
|
23
|
+
*
|
|
24
|
+
* The implementation is bounded to a fixed iteration count so a
|
|
25
|
+
* pathological input (for example, hundreds of header values that are
|
|
26
|
+
* all ≤ 1024 chars yet collectively still exceed the cap) cannot wedge
|
|
27
|
+
* the agent. In that edge case the returned `metadataBytes` may exceed
|
|
28
|
+
* {@link METADATA_BUDGET_BYTES} — the caller can decide whether to
|
|
29
|
+
* surface the overflow or treat it as a soft warning.
|
|
30
|
+
*/
|
|
31
|
+
import { METADATA_BUDGET_BYTES, TRUNCATION_MARKER, } from "./types.js";
|
|
32
|
+
/**
|
|
33
|
+
* Per-header-value floor below which the budget loop will not halve any
|
|
34
|
+
* further. Once every header value's "content" portion (length excluding
|
|
35
|
+
* the trailing {@link TRUNCATION_MARKER}, when present) is at or below
|
|
36
|
+
* this number of characters, the loop moves on to dropping redirect
|
|
37
|
+
* hops.
|
|
38
|
+
*
|
|
39
|
+
* Matches the design pseudocode "any header value not yet truncated to
|
|
40
|
+
* 1024".
|
|
41
|
+
*/
|
|
42
|
+
const HEADER_VALUE_FLOOR = 1024;
|
|
43
|
+
/**
|
|
44
|
+
* Hard upper bound on the number of reduction steps the budget loop will
|
|
45
|
+
* perform per invocation. Each step either drops one cookie, halves one
|
|
46
|
+
* header value, or drops one redirect hop. With realistic inputs the
|
|
47
|
+
* loop terminates within a few dozen iterations; the cap exists purely
|
|
48
|
+
* to defend against pathological inputs that would otherwise loop
|
|
49
|
+
* forever.
|
|
50
|
+
*/
|
|
51
|
+
const MAX_ITERATIONS = 10_000;
|
|
52
|
+
/**
|
|
53
|
+
* Enforce the 64 KiB metadata budget on the supplied fields.
|
|
54
|
+
*
|
|
55
|
+
* Returns a fresh {@link BudgetResult} containing (possibly shortened)
|
|
56
|
+
* copies of `headers`, `redirectChain`, and `cookies`, alongside `tls`
|
|
57
|
+
* and `timing` passed through unchanged, plus the final
|
|
58
|
+
* `metadataBytes` count.
|
|
59
|
+
*
|
|
60
|
+
* The returned arrays/maps are independent of the caller's inputs — the
|
|
61
|
+
* function does not mutate `input`.
|
|
62
|
+
*/
|
|
63
|
+
export function enforce(input) {
|
|
64
|
+
// Shallow-copy any mutable structures so the caller's data is left
|
|
65
|
+
// untouched even when the loop pops or halves entries.
|
|
66
|
+
const headers = input.headers !== undefined ? { ...input.headers } : undefined;
|
|
67
|
+
const cookies = input.cookies !== undefined ? [...input.cookies] : undefined;
|
|
68
|
+
const redirectChain = input.redirectChain !== undefined ? [...input.redirectChain] : undefined;
|
|
69
|
+
const { tls, timing } = input;
|
|
70
|
+
const measure = () => Buffer.byteLength(JSON.stringify({ headers, tls, timing, redirectChain, cookies }), "utf8");
|
|
71
|
+
let metadataBytes = measure();
|
|
72
|
+
for (let i = 0; i < MAX_ITERATIONS && metadataBytes > METADATA_BUDGET_BYTES; i++) {
|
|
73
|
+
let progress = false;
|
|
74
|
+
if (cookies !== undefined && cookies.length > 0) {
|
|
75
|
+
// Step 1: drop trailing cookies first (Requirement 2.35).
|
|
76
|
+
cookies.pop();
|
|
77
|
+
progress = true;
|
|
78
|
+
}
|
|
79
|
+
else if (headers !== undefined && hasHalveableHeader(headers)) {
|
|
80
|
+
// Step 2: halve the longest header value with content > 1024 chars.
|
|
81
|
+
halveLongestHeaderValue(headers);
|
|
82
|
+
progress = true;
|
|
83
|
+
}
|
|
84
|
+
else if (redirectChain !== undefined && redirectChain.length > 0) {
|
|
85
|
+
// Step 3: drop trailing redirect hops as the last resort.
|
|
86
|
+
redirectChain.pop();
|
|
87
|
+
progress = true;
|
|
88
|
+
}
|
|
89
|
+
if (!progress)
|
|
90
|
+
break;
|
|
91
|
+
metadataBytes = measure();
|
|
92
|
+
}
|
|
93
|
+
return assemble({
|
|
94
|
+
headers,
|
|
95
|
+
tls,
|
|
96
|
+
timing,
|
|
97
|
+
redirectChain,
|
|
98
|
+
cookies,
|
|
99
|
+
metadataBytes,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
// Internals
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
/**
|
|
106
|
+
* Return `true` iff at least one header value's content portion (length
|
|
107
|
+
* excluding the trailing {@link TRUNCATION_MARKER}, when present)
|
|
108
|
+
* exceeds {@link HEADER_VALUE_FLOOR}.
|
|
109
|
+
*/
|
|
110
|
+
function hasHalveableHeader(headers) {
|
|
111
|
+
for (const value of Object.values(headers)) {
|
|
112
|
+
if (contentLength(value) > HEADER_VALUE_FLOOR)
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Find the header whose content portion is currently longest (ties
|
|
119
|
+
* resolved by iteration order, which is stable in V8 for string keys),
|
|
120
|
+
* halve that content in place, and append the
|
|
121
|
+
* {@link TRUNCATION_MARKER}. Header values whose content is already
|
|
122
|
+
* ≤ {@link HEADER_VALUE_FLOOR} are skipped; the function is a no-op when
|
|
123
|
+
* no such value exists.
|
|
124
|
+
*/
|
|
125
|
+
function halveLongestHeaderValue(headers) {
|
|
126
|
+
let bestKey;
|
|
127
|
+
let bestLen = -1;
|
|
128
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
129
|
+
const len = contentLength(value);
|
|
130
|
+
if (len > HEADER_VALUE_FLOOR && len > bestLen) {
|
|
131
|
+
bestKey = key;
|
|
132
|
+
bestLen = len;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (bestKey === undefined)
|
|
136
|
+
return;
|
|
137
|
+
const current = headers[bestKey];
|
|
138
|
+
if (current === undefined)
|
|
139
|
+
return;
|
|
140
|
+
const content = stripMarker(current);
|
|
141
|
+
const halved = content.slice(0, Math.floor(content.length / 2));
|
|
142
|
+
headers[bestKey] = `${halved}${TRUNCATION_MARKER}`;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Length of `value`'s "content" portion in characters: the full string
|
|
146
|
+
* length minus the {@link TRUNCATION_MARKER} suffix when one is
|
|
147
|
+
* present. Used so repeated halving of the same header value reduces
|
|
148
|
+
* the underlying content rather than treating the marker itself as
|
|
149
|
+
* shrinkable text.
|
|
150
|
+
*/
|
|
151
|
+
function contentLength(value) {
|
|
152
|
+
return stripMarker(value).length;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Strip a single trailing {@link TRUNCATION_MARKER} from `value` if one
|
|
156
|
+
* is present. Returns `value` unchanged otherwise. Used by the halving
|
|
157
|
+
* step so the marker is not duplicated when a header is truncated more
|
|
158
|
+
* than once across loop iterations.
|
|
159
|
+
*/
|
|
160
|
+
function stripMarker(value) {
|
|
161
|
+
return value.endsWith(TRUNCATION_MARKER)
|
|
162
|
+
? value.slice(0, value.length - TRUNCATION_MARKER.length)
|
|
163
|
+
: value;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Build the {@link BudgetResult} from the working state. Optional input
|
|
167
|
+
* fields that were not supplied are left absent on the result (rather
|
|
168
|
+
* than set to `undefined`) to satisfy `exactOptionalPropertyTypes`.
|
|
169
|
+
*/
|
|
170
|
+
function assemble(state) {
|
|
171
|
+
const result = {
|
|
172
|
+
metadataBytes: state.metadataBytes,
|
|
173
|
+
cap: METADATA_BUDGET_BYTES,
|
|
174
|
+
};
|
|
175
|
+
if (state.headers !== undefined)
|
|
176
|
+
result.headers = state.headers;
|
|
177
|
+
if (state.tls !== undefined)
|
|
178
|
+
result.tls = state.tls;
|
|
179
|
+
if (state.timing !== undefined)
|
|
180
|
+
result.timing = state.timing;
|
|
181
|
+
if (state.redirectChain !== undefined)
|
|
182
|
+
result.redirectChain = state.redirectChain;
|
|
183
|
+
if (state.cookies !== undefined)
|
|
184
|
+
result.cookies = state.cookies;
|
|
185
|
+
return result;
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=budget.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"budget.js","sourceRoot":"","sources":["../../../src/tools/web/budget.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAML,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;GASG;AACH,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC;;;;;;;GAOG;AACH,MAAM,cAAc,GAAG,MAAM,CAAC;AAqC9B;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,KAAkB;IACxC,mEAAmE;IACnE,uDAAuD;IACvD,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACjE,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/D,MAAM,aAAa,GACjB,KAAK,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAE9B,MAAM,OAAO,GAAG,GAAW,EAAE,CAC3B,MAAM,CAAC,UAAU,CACf,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,EAChE,MAAM,CACP,CAAC;IAEJ,IAAI,aAAa,GAAG,OAAO,EAAE,CAAC;IAE9B,KACE,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,cAAc,IAAI,aAAa,GAAG,qBAAqB,EAC3D,CAAC,EAAE,EACH,CAAC;QACD,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,0DAA0D;YAC1D,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,OAAO,KAAK,SAAS,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;YAChE,oEAAoE;YACpE,uBAAuB,CAAC,OAAO,CAAC,CAAC;YACjC,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,0DAA0D;YAC1D,aAAa,CAAC,GAAG,EAAE,CAAC;YACpB,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,QAAQ;YAAE,MAAM;QACrB,aAAa,GAAG,OAAO,EAAE,CAAC;IAC5B,CAAC;IAED,OAAO,QAAQ,CAAC;QACd,OAAO;QACP,GAAG;QACH,MAAM;QACN,aAAa;QACb,OAAO;QACP,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,OAAkB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,kBAAkB;YAAE,OAAO,IAAI,CAAC;IAC7D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,uBAAuB,CAAC,OAAkB;IACjD,IAAI,OAA2B,CAAC;IAChC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,GAAG,GAAG,kBAAkB,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;YAC9C,OAAO,GAAG,GAAG,CAAC;YACd,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;IACH,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO;IAElC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO;IAElC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,MAAM,GAAG,iBAAiB,EAAE,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,WAAW,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACnC,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACtC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC;QACzD,CAAC,CAAC,KAAK,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,KAOjB;IACC,MAAM,MAAM,GAAiB;QAC3B,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,GAAG,EAAE,qBAAqB;KAC3B,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS;QAAE,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7D,IAAI,KAAK,CAAC,aAAa,KAAK,SAAS;QACnC,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAChE,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `Capture` — pure observation builder for a single `web.fetch` invocation.
|
|
3
|
+
*
|
|
4
|
+
* The fetch transport in `fetch-core.ts` feeds events into this builder
|
|
5
|
+
* (DNS resolution, TCP connect, TLS handshake, response headers, redirect
|
|
6
|
+
* hops, `Set-Cookie` lines) and the builder accumulates the structured
|
|
7
|
+
* fields the {@link WebFetchMetadata} envelope needs:
|
|
8
|
+
*
|
|
9
|
+
* - {@link TimingInfo}: `dnsMs`, `tcpMs`, `tlsMs`, `ttfbMs`, `totalMs`
|
|
10
|
+
* - {@link TlsInfo} from the final hop's `tls.TLSSocket`
|
|
11
|
+
* (`getProtocol()`, `getCipher().name`, `getPeerCertificate(true)`)
|
|
12
|
+
* - response headers (lowercased keys, repeat values joined with `, `)
|
|
13
|
+
* - the redirect chain (capped at {@link MAX_REDIRECT_HOPS})
|
|
14
|
+
* - the resolved IP and the final hostname
|
|
15
|
+
* - `Set-Cookie` cookies (capped at {@link MAX_COOKIES_CAPTURED})
|
|
16
|
+
*
|
|
17
|
+
* The builder performs **no I/O** — it never touches the network, the
|
|
18
|
+
* filesystem, or `auditLog`. It is a pure data sink so the transport
|
|
19
|
+
* layer in `fetch-core.ts` can be tested independently of the
|
|
20
|
+
* Node `https`/`tls` machinery.
|
|
21
|
+
*
|
|
22
|
+
* Per-hop semantics:
|
|
23
|
+
* - `setHopContext(hostname)` is called by the transport at the start of
|
|
24
|
+
* each hop; the value of the *last* hop becomes
|
|
25
|
+
* {@link CapturedFields.finalHostname}.
|
|
26
|
+
* - {@link Capture.markDnsResolved} replaces the running `dnsMs` /
|
|
27
|
+
* `resolvedIp` so the values surfaced in {@link CapturedFields}
|
|
28
|
+
* correspond to the *final* hop. Same for `tcpMs`, `tlsMs`, and
|
|
29
|
+
* `ttfbMs` — only the last hop's measurements are returned.
|
|
30
|
+
* - {@link Capture.addRedirectHop} appends one entry per intermediate
|
|
31
|
+
* 3xx hop; the array is bounded at {@link MAX_REDIRECT_HOPS}.
|
|
32
|
+
* - {@link Capture.addSetCookieHeader} parses every observed
|
|
33
|
+
* `Set-Cookie` line via {@link parseSetCookie}; the resulting array is
|
|
34
|
+
* bounded at {@link MAX_COOKIES_CAPTURED}.
|
|
35
|
+
*
|
|
36
|
+
* Redaction and 4096-char header-value truncation are applied later by
|
|
37
|
+
* `redact.applyToHeaders` / `redact.applyToCookies` and the 64 KiB
|
|
38
|
+
* metadata budget by `budget.enforce`. This module only collects.
|
|
39
|
+
*/
|
|
40
|
+
import type { IncomingHttpHeaders } from "node:http";
|
|
41
|
+
import type { TLSSocket } from "node:tls";
|
|
42
|
+
import { type CookieInfo, type HeaderMap, type RedirectChain, type TimingInfo, type TlsInfo } from "./types.js";
|
|
43
|
+
/**
|
|
44
|
+
* Snapshot of the structured fields the builder has accumulated when
|
|
45
|
+
* `fetch-core.ts` finalises the response. The shape matches the slice of
|
|
46
|
+
* {@link WebFetchMetadata} that depends on per-hop transport
|
|
47
|
+
* observation; the fetch handler combines this with `requestedUrl`,
|
|
48
|
+
* `finalUrl`, `mode`, `bytesReceived`, `truncated`, etc. before applying
|
|
49
|
+
* `redact.applyToHeaders` / `redact.applyToCookies` and `budget.enforce`.
|
|
50
|
+
*/
|
|
51
|
+
export interface CapturedFields {
|
|
52
|
+
/** IP address contacted on the final hop. */
|
|
53
|
+
resolvedIp: string;
|
|
54
|
+
/** Hostname of the final hop (the one whose body is returned). */
|
|
55
|
+
finalHostname: string;
|
|
56
|
+
/** Integer HTTP status of the final response. */
|
|
57
|
+
status: number;
|
|
58
|
+
/** Lowercased response headers from the final hop, repeats joined. */
|
|
59
|
+
headers: HeaderMap;
|
|
60
|
+
/** TLS session details from the final hop, when the scheme was https. */
|
|
61
|
+
tls?: TlsInfo;
|
|
62
|
+
/** Per-phase timings; `tlsMs` is omitted on http:// requests. */
|
|
63
|
+
timing: TimingInfo;
|
|
64
|
+
/** Up to {@link MAX_REDIRECT_HOPS} hops in chronological order. */
|
|
65
|
+
redirectChain: RedirectChain;
|
|
66
|
+
/** Up to {@link MAX_COOKIES_CAPTURED} cookies parsed from `Set-Cookie`. */
|
|
67
|
+
cookies: CookieInfo[];
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Pure builder that records the per-hop observations made by
|
|
71
|
+
* `fetch-core.ts` and assembles them into a {@link CapturedFields}
|
|
72
|
+
* snapshot via {@link Capture.finalize}.
|
|
73
|
+
*
|
|
74
|
+
* The builder is single-use: callers should construct one `Capture` per
|
|
75
|
+
* `web.fetch` invocation and discard it after `finalize`.
|
|
76
|
+
*/
|
|
77
|
+
export declare class Capture {
|
|
78
|
+
/** Final-hop DNS-resolution time in milliseconds. */
|
|
79
|
+
private dnsMs;
|
|
80
|
+
/** Final-hop TCP-connect time in milliseconds. */
|
|
81
|
+
private tcpMs;
|
|
82
|
+
/** Final-hop TLS-handshake time in milliseconds (https only). */
|
|
83
|
+
private tlsMs;
|
|
84
|
+
/** Final-hop time-to-first-byte in milliseconds. */
|
|
85
|
+
private ttfbMs;
|
|
86
|
+
/** Final-hop resolved IP, set by {@link markDnsResolved}. */
|
|
87
|
+
private resolvedIp;
|
|
88
|
+
/** Final-hop hostname, set by {@link setHopContext}. */
|
|
89
|
+
private finalHostname;
|
|
90
|
+
/** Final-hop HTTP status, set by {@link markResponse}. */
|
|
91
|
+
private status;
|
|
92
|
+
/** Final-hop normalised headers, set by {@link markResponse}. */
|
|
93
|
+
private headers;
|
|
94
|
+
/** Final-hop TLS info (https only), set by {@link markTlsHandshaked}. */
|
|
95
|
+
private tls;
|
|
96
|
+
/** Redirect hops, capped at {@link MAX_REDIRECT_HOPS}. */
|
|
97
|
+
private readonly redirectChain;
|
|
98
|
+
/** Captured cookies, capped at {@link MAX_COOKIES_CAPTURED}. */
|
|
99
|
+
private readonly cookies;
|
|
100
|
+
/**
|
|
101
|
+
* Whether the current invocation is an `https://` fetch. When `false`,
|
|
102
|
+
* `tlsMs` is omitted from {@link TimingInfo} and `tls` from
|
|
103
|
+
* {@link CapturedFields} per Requirements 2.16, 2.24, and 2.25.
|
|
104
|
+
*/
|
|
105
|
+
private readonly isHttps;
|
|
106
|
+
/**
|
|
107
|
+
* Construct a fresh builder.
|
|
108
|
+
*
|
|
109
|
+
* @param opts.isHttps - Whether the request URL used `https://`.
|
|
110
|
+
* Controls whether `timing.tlsMs` and `tls` are populated.
|
|
111
|
+
* @param opts.finalHostname - Optional initial hostname; the transport
|
|
112
|
+
* will overwrite this via {@link setHopContext} as redirects are
|
|
113
|
+
* followed.
|
|
114
|
+
*/
|
|
115
|
+
constructor(opts: {
|
|
116
|
+
isHttps: boolean;
|
|
117
|
+
finalHostname?: string;
|
|
118
|
+
});
|
|
119
|
+
/**
|
|
120
|
+
* Record the hostname of the hop the transport is about to issue.
|
|
121
|
+
*
|
|
122
|
+
* Called once per hop, before {@link markDnsResolved}. The most-recent
|
|
123
|
+
* value becomes {@link CapturedFields.finalHostname}.
|
|
124
|
+
*/
|
|
125
|
+
setHopContext(hostname: string): void;
|
|
126
|
+
/**
|
|
127
|
+
* Record the DNS-resolution outcome for the current hop.
|
|
128
|
+
*
|
|
129
|
+
* The most-recent values overwrite any earlier ones so the values
|
|
130
|
+
* surfaced in {@link TimingInfo.dnsMs} and
|
|
131
|
+
* {@link CapturedFields.resolvedIp} correspond to the final hop.
|
|
132
|
+
*/
|
|
133
|
+
markDnsResolved(ms: number, ip: string): void;
|
|
134
|
+
/**
|
|
135
|
+
* Record the TCP-connect time for the current hop. The most-recent
|
|
136
|
+
* value wins (see class-level docstring for per-hop semantics).
|
|
137
|
+
*/
|
|
138
|
+
markTcpConnected(ms: number): void;
|
|
139
|
+
/**
|
|
140
|
+
* Record the TLS handshake time and extract {@link TlsInfo} from the
|
|
141
|
+
* final-hop `tls.TLSSocket`.
|
|
142
|
+
*
|
|
143
|
+
* - `protocol` ← `socket.getProtocol()` (`""` if null).
|
|
144
|
+
* - `cipher` ← `socket.getCipher().name` (`""` if missing).
|
|
145
|
+
* - `subjectCN/issuerCN` ← `cert.subject.CN` / `cert.issuer.CN`.
|
|
146
|
+
* - `subjectAltNames` ← parsed from `cert.subjectaltname`.
|
|
147
|
+
* - `notBefore/notAfter` ← `cert.valid_from` / `cert.valid_to` parsed
|
|
148
|
+
* to ISO 8601 (falls back to the raw value).
|
|
149
|
+
* - `fingerprintSha256` ← lowercase, colon-separated SHA-256 of
|
|
150
|
+
* `cert.raw` via `node:crypto`.
|
|
151
|
+
*
|
|
152
|
+
* Calling this method on an http:// fetch is harmless but pointless —
|
|
153
|
+
* the constructor's `isHttps=false` flag suppresses the field in
|
|
154
|
+
* {@link finalize} regardless.
|
|
155
|
+
*/
|
|
156
|
+
markTlsHandshaked(ms: number, socket: TLSSocket): void;
|
|
157
|
+
/**
|
|
158
|
+
* Record the final-hop response: HTTP status, raw headers
|
|
159
|
+
* (lowercased and joined into a {@link HeaderMap}), and TTFB.
|
|
160
|
+
*
|
|
161
|
+
* `Set-Cookie` is preserved in `headers` joined with `, ` like every
|
|
162
|
+
* other repeated header so the audit/redact passes can act on it.
|
|
163
|
+
* Per-cookie capture happens via {@link addSetCookieHeader}, which
|
|
164
|
+
* `fetch-core.ts` calls once per `Set-Cookie` line observed.
|
|
165
|
+
*/
|
|
166
|
+
markResponse(status: number, rawHeaders: IncomingHttpHeaders, ttfbMs: number): void;
|
|
167
|
+
/**
|
|
168
|
+
* Append a redirect hop to the chronological chain.
|
|
169
|
+
*
|
|
170
|
+
* Hops in excess of {@link MAX_REDIRECT_HOPS} are silently dropped so
|
|
171
|
+
* the array always satisfies the cap from Requirement 2.26.
|
|
172
|
+
* `fetch-core.ts` is responsible for surfacing the
|
|
173
|
+
* `redirect-limit` error when the cap is reached; this builder just
|
|
174
|
+
* stops accumulating.
|
|
175
|
+
*/
|
|
176
|
+
addRedirectHop(url: string, status: number, location?: string): void;
|
|
177
|
+
/**
|
|
178
|
+
* Parse one `Set-Cookie` header value via {@link parseSetCookie} and
|
|
179
|
+
* append the resulting {@link CookieInfo}, bounded at
|
|
180
|
+
* {@link MAX_COOKIES_CAPTURED}.
|
|
181
|
+
*
|
|
182
|
+
* Cookies in excess of the cap are silently dropped, matching
|
|
183
|
+
* Requirement 2.31.
|
|
184
|
+
*/
|
|
185
|
+
addSetCookieHeader(value: string): void;
|
|
186
|
+
/**
|
|
187
|
+
* Produce the {@link CapturedFields} snapshot.
|
|
188
|
+
*
|
|
189
|
+
* @param totalMs - Wall-clock duration of the whole invocation, in
|
|
190
|
+
* milliseconds. Stored in {@link TimingInfo.totalMs}.
|
|
191
|
+
*
|
|
192
|
+
* Optional fields obey the design's "include flags applied at the
|
|
193
|
+
* adapter, not at the builder" principle: this method always emits
|
|
194
|
+
* every field it observed, including `tls` (when `isHttps=true` and a
|
|
195
|
+
* handshake was captured). The fetch handler in `fetch.ts` is
|
|
196
|
+
* responsible for honouring `includeTls` / `includeTiming` /
|
|
197
|
+
* `includeRedirectChain` by stripping fields from the assembled
|
|
198
|
+
* {@link WebFetchMetadata} *after* this snapshot is produced.
|
|
199
|
+
*/
|
|
200
|
+
finalize(totalMs: number): CapturedFields;
|
|
201
|
+
}
|