@zhixuan92/multi-model-agent-core 4.7.19 → 4.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/config/schema.d.ts +2 -20
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +9 -80
- package/dist/config/schema.js.map +1 -1
- package/dist/events/task-envelope.d.ts +10 -1
- package/dist/events/task-envelope.d.ts.map +1 -1
- package/dist/events/task-envelope.js +4 -1
- package/dist/events/task-envelope.js.map +1 -1
- package/dist/events/wire-schema.d.ts +4 -0
- package/dist/events/wire-schema.d.ts.map +1 -1
- package/dist/events/wire-schema.js +3 -3
- package/dist/events/wire-schema.js.map +1 -1
- package/dist/journal/default-schema.d.ts +2 -0
- package/dist/journal/default-schema.d.ts.map +1 -0
- package/dist/journal/default-schema.js +27 -0
- package/dist/journal/default-schema.js.map +1 -0
- package/dist/journal/types.d.ts +22 -0
- package/dist/journal/types.d.ts.map +1 -0
- package/dist/journal/types.js +5 -0
- package/dist/journal/types.js.map +1 -0
- package/dist/lifecycle/annotate-parser.js +1 -1
- package/dist/lifecycle/annotate-parser.js.map +1 -1
- package/dist/lifecycle/derive-completion.js +1 -1
- package/dist/lifecycle/derive-completion.js.map +1 -1
- package/dist/lifecycle/handlers/annotate-stage.d.ts.map +1 -1
- package/dist/lifecycle/handlers/annotate-stage.js +9 -3
- package/dist/lifecycle/handlers/annotate-stage.js.map +1 -1
- package/dist/lifecycle/handlers/baseline-handlers.d.ts.map +1 -1
- package/dist/lifecycle/handlers/baseline-handlers.js +10 -0
- package/dist/lifecycle/handlers/baseline-handlers.js.map +1 -1
- package/dist/lifecycle/handlers/compose-commit-message.d.ts +15 -0
- package/dist/lifecycle/handlers/compose-commit-message.d.ts.map +1 -0
- package/dist/lifecycle/handlers/compose-commit-message.js +227 -0
- package/dist/lifecycle/handlers/compose-commit-message.js.map +1 -0
- package/dist/lifecycle/handlers/enrich-runtime-result.d.ts.map +1 -1
- package/dist/lifecycle/handlers/enrich-runtime-result.js +31 -0
- package/dist/lifecycle/handlers/enrich-runtime-result.js.map +1 -1
- package/dist/lifecycle/handlers/git-commit-handler.d.ts.map +1 -1
- package/dist/lifecycle/handlers/git-commit-handler.js +2 -15
- package/dist/lifecycle/handlers/git-commit-handler.js.map +1 -1
- package/dist/lifecycle/handlers/implement-stage.d.ts.map +1 -1
- package/dist/lifecycle/handlers/implement-stage.js +4 -1
- package/dist/lifecycle/handlers/implement-stage.js.map +1 -1
- package/dist/lifecycle/handlers/journal-review-prompt.d.ts +7 -0
- package/dist/lifecycle/handlers/journal-review-prompt.d.ts.map +1 -0
- package/dist/lifecycle/handlers/journal-review-prompt.js +54 -0
- package/dist/lifecycle/handlers/journal-review-prompt.js.map +1 -0
- package/dist/lifecycle/handlers/quality-review-prompt.d.ts +1 -0
- package/dist/lifecycle/handlers/quality-review-prompt.d.ts.map +1 -1
- package/dist/lifecycle/handlers/quality-review-prompt.js +8 -0
- package/dist/lifecycle/handlers/quality-review-prompt.js.map +1 -1
- package/dist/lifecycle/handlers/review-stage.d.ts.map +1 -1
- package/dist/lifecycle/handlers/review-stage.js +57 -13
- package/dist/lifecycle/handlers/review-stage.js.map +1 -1
- package/dist/lifecycle/handlers/rework-stage.d.ts.map +1 -1
- package/dist/lifecycle/handlers/rework-stage.js +3 -0
- package/dist/lifecycle/handlers/rework-stage.js.map +1 -1
- package/dist/lifecycle/handlers/spec-review-prompt.d.ts +1 -0
- package/dist/lifecycle/handlers/spec-review-prompt.d.ts.map +1 -1
- package/dist/lifecycle/handlers/spec-review-prompt.js +8 -0
- package/dist/lifecycle/handlers/spec-review-prompt.js.map +1 -1
- package/dist/lifecycle/handlers/terminal-handlers.d.ts +25 -1
- package/dist/lifecycle/handlers/terminal-handlers.d.ts.map +1 -1
- package/dist/lifecycle/handlers/terminal-handlers.js +55 -19
- package/dist/lifecycle/handlers/terminal-handlers.js.map +1 -1
- package/dist/lifecycle/perform-implementation.d.ts.map +1 -1
- package/dist/lifecycle/perform-implementation.js +7 -4
- package/dist/lifecycle/perform-implementation.js.map +1 -1
- package/dist/lifecycle/read-route-criteria.d.ts +3 -2
- package/dist/lifecycle/read-route-criteria.d.ts.map +1 -1
- package/dist/lifecycle/read-route-criteria.js +2 -0
- package/dist/lifecycle/read-route-criteria.js.map +1 -1
- package/dist/lifecycle/research-pre-loop.d.ts +5 -4
- package/dist/lifecycle/research-pre-loop.d.ts.map +1 -1
- package/dist/lifecycle/research-pre-loop.js +7 -10
- package/dist/lifecycle/research-pre-loop.js.map +1 -1
- package/dist/lifecycle/stage-io.d.ts +8 -4
- package/dist/lifecycle/stage-io.d.ts.map +1 -1
- package/dist/lifecycle/stage-io.js +2 -2
- package/dist/lifecycle/stage-io.js.map +1 -1
- package/dist/lifecycle/stage-plan-types.d.ts +1 -1
- package/dist/lifecycle/stage-plan-types.d.ts.map +1 -1
- package/dist/lifecycle/stage-progression.d.ts.map +1 -1
- package/dist/lifecycle/stage-progression.js +2 -0
- package/dist/lifecycle/stage-progression.js.map +1 -1
- package/dist/lifecycle/task-runner.d.ts.map +1 -1
- package/dist/lifecycle/task-runner.js +6 -1
- package/dist/lifecycle/task-runner.js.map +1 -1
- package/dist/lifecycle/worker-output-contract.d.ts +3 -1
- package/dist/lifecycle/worker-output-contract.d.ts.map +1 -1
- package/dist/lifecycle/worker-output-contract.js +4 -1
- package/dist/lifecycle/worker-output-contract.js.map +1 -1
- package/dist/reporting/headline-templates/journal-recall.d.ts +3 -0
- package/dist/reporting/headline-templates/journal-recall.d.ts.map +1 -0
- package/dist/reporting/headline-templates/journal-recall.js +9 -0
- package/dist/reporting/headline-templates/journal-recall.js.map +1 -0
- package/dist/reporting/headline-templates/journal.d.ts +3 -0
- package/dist/reporting/headline-templates/journal.d.ts.map +1 -0
- package/dist/reporting/headline-templates/journal.js +17 -0
- package/dist/reporting/headline-templates/journal.js.map +1 -0
- package/dist/reporting/report-parser-slots/journal-report.d.ts +8 -0
- package/dist/reporting/report-parser-slots/journal-report.d.ts.map +1 -0
- package/dist/reporting/report-parser-slots/journal-report.js +12 -0
- package/dist/reporting/report-parser-slots/journal-report.js.map +1 -0
- package/dist/reporting/terminal-report-markdown.d.ts +13 -0
- package/dist/reporting/terminal-report-markdown.d.ts.map +1 -0
- package/dist/reporting/terminal-report-markdown.js +31 -0
- package/dist/reporting/terminal-report-markdown.js.map +1 -0
- package/dist/research/adapters/index.d.ts +0 -1
- package/dist/research/adapters/index.d.ts.map +1 -1
- package/dist/research/adapters/index.js +0 -3
- package/dist/research/adapters/index.js.map +1 -1
- package/dist/research/adapters/types.d.ts +1 -1
- package/dist/research/adapters/types.d.ts.map +1 -1
- package/dist/research/evidence-pack.d.ts +19 -1
- package/dist/research/evidence-pack.d.ts.map +1 -1
- package/dist/research/evidence-pack.js +36 -1
- package/dist/research/evidence-pack.js.map +1 -1
- package/dist/research/orchestrator.d.ts +0 -4
- package/dist/research/orchestrator.d.ts.map +1 -1
- package/dist/research/orchestrator.js +4 -53
- package/dist/research/orchestrator.js.map +1 -1
- package/dist/research/query-plan.d.ts +0 -2
- package/dist/research/query-plan.d.ts.map +1 -1
- package/dist/research/query-plan.js +1 -8
- package/dist/research/query-plan.js.map +1 -1
- package/dist/research/user-agent.js +1 -1
- package/dist/research/user-agent.js.map +1 -1
- package/dist/research/web-search.d.ts +2 -1
- package/dist/research/web-search.d.ts.map +1 -1
- package/dist/research/web-search.js +31 -7
- package/dist/research/web-search.js.map +1 -1
- package/dist/stores/batch-registry.d.ts +1 -1
- package/dist/stores/batch-registry.d.ts.map +1 -1
- package/dist/stores/batch-registry.js +4 -4
- package/dist/stores/batch-registry.js.map +1 -1
- package/dist/tool-surface/register-all-tools.d.ts.map +1 -1
- package/dist/tool-surface/register-all-tools.js +4 -0
- package/dist/tool-surface/register-all-tools.js.map +1 -1
- package/dist/tools/delegate/brief-slot.d.ts +3 -0
- package/dist/tools/delegate/brief-slot.d.ts.map +1 -1
- package/dist/tools/delegate/brief-slot.js +1 -0
- package/dist/tools/delegate/brief-slot.js.map +1 -1
- package/dist/tools/delegate/tool-config.d.ts.map +1 -1
- package/dist/tools/delegate/tool-config.js +1 -0
- package/dist/tools/delegate/tool-config.js.map +1 -1
- package/dist/tools/execute-plan/tool-config.d.ts.map +1 -1
- package/dist/tools/execute-plan/tool-config.js +3 -0
- package/dist/tools/execute-plan/tool-config.js.map +1 -1
- package/dist/tools/journal/recall/brief-slot.d.ts +7 -0
- package/dist/tools/journal/recall/brief-slot.d.ts.map +1 -0
- package/dist/tools/journal/recall/brief-slot.js +5 -0
- package/dist/tools/journal/recall/brief-slot.js.map +1 -0
- package/dist/tools/journal/recall/implementer-criteria.d.ts +9 -0
- package/dist/tools/journal/recall/implementer-criteria.d.ts.map +1 -0
- package/dist/tools/journal/recall/implementer-criteria.js +23 -0
- package/dist/tools/journal/recall/implementer-criteria.js.map +1 -0
- package/dist/tools/journal/recall/schema.d.ts +54 -0
- package/dist/tools/journal/recall/schema.d.ts.map +1 -0
- package/dist/tools/journal/recall/schema.js +10 -0
- package/dist/tools/journal/recall/schema.js.map +1 -0
- package/dist/tools/journal/recall/subtypes.d.ts +4 -0
- package/dist/tools/journal/recall/subtypes.d.ts.map +1 -0
- package/dist/tools/journal/recall/subtypes.js +25 -0
- package/dist/tools/journal/recall/subtypes.js.map +1 -0
- package/dist/tools/journal/recall/tool-config.d.ts +8 -0
- package/dist/tools/journal/recall/tool-config.d.ts.map +1 -0
- package/dist/tools/journal/recall/tool-config.js +46 -0
- package/dist/tools/journal/recall/tool-config.js.map +1 -0
- package/dist/tools/journal/record/brief-slot.d.ts +14 -0
- package/dist/tools/journal/record/brief-slot.d.ts.map +1 -0
- package/dist/tools/journal/record/brief-slot.js +18 -0
- package/dist/tools/journal/record/brief-slot.js.map +1 -0
- package/dist/tools/journal/record/implementer-criteria.d.ts +6 -0
- package/dist/tools/journal/record/implementer-criteria.d.ts.map +1 -0
- package/dist/tools/journal/record/implementer-criteria.js +17 -0
- package/dist/tools/journal/record/implementer-criteria.js.map +1 -0
- package/dist/tools/journal/record/schema.d.ts +55 -0
- package/dist/tools/journal/record/schema.d.ts.map +1 -0
- package/dist/tools/journal/record/schema.js +13 -0
- package/dist/tools/journal/record/schema.js.map +1 -0
- package/dist/tools/journal/record/tool-config.d.ts +7 -0
- package/dist/tools/journal/record/tool-config.d.ts.map +1 -0
- package/dist/tools/journal/record/tool-config.js +40 -0
- package/dist/tools/journal/record/tool-config.js.map +1 -0
- package/dist/tools/research/brief-slot.d.ts +0 -2
- package/dist/tools/research/brief-slot.d.ts.map +1 -1
- package/dist/tools/research/brief-slot.js.map +1 -1
- package/dist/tools/research/implementer-criteria.d.ts +0 -1
- package/dist/tools/research/implementer-criteria.d.ts.map +1 -1
- package/dist/tools/research/implementer-criteria.js +3 -15
- package/dist/tools/research/implementer-criteria.js.map +1 -1
- package/dist/tools/research/tool-config.d.ts.map +1 -1
- package/dist/tools/research/tool-config.js +0 -1
- package/dist/tools/research/tool-config.js.map +1 -1
- package/dist/types/task-spec.d.ts +2 -1
- package/dist/types/task-spec.d.ts.map +1 -1
- package/package.json +17 -1
- package/dist/research/adapters/generic-rss.d.ts +0 -8
- package/dist/research/adapters/generic-rss.d.ts.map +0 -1
- package/dist/research/adapters/generic-rss.js +0 -26
- package/dist/research/adapters/generic-rss.js.map +0 -1
- package/dist/research/ssrf-guard.d.ts +0 -12
- package/dist/research/ssrf-guard.d.ts.map +0 -1
- package/dist/research/ssrf-guard.js +0 -209
- package/dist/research/ssrf-guard.js.map +0 -1
- package/dist/research/untrusted-content.d.ts +0 -13
- package/dist/research/untrusted-content.d.ts.map +0 -1
- package/dist/research/untrusted-content.js +0 -9
- package/dist/research/untrusted-content.js.map +0 -1
- package/dist/research/web-fetch-helpers.d.ts +0 -44
- package/dist/research/web-fetch-helpers.d.ts.map +0 -1
- package/dist/research/web-fetch-helpers.js +0 -209
- package/dist/research/web-fetch-helpers.js.map +0 -1
- package/dist/research/web-fetch.d.ts +0 -55
- package/dist/research/web-fetch.d.ts.map +0 -1
- package/dist/research/web-fetch.js +0 -236
- package/dist/research/web-fetch.js.map +0 -1
|
@@ -1,236 +0,0 @@
|
|
|
1
|
-
// packages/core/src/research/web-fetch.ts
|
|
2
|
-
//
|
|
3
|
-
// Replaces the previous IP-pinning dispatcher (which failed 100% of real
|
|
4
|
-
// requests on Node 25 / undici current) with a connect-callback re-validation
|
|
5
|
-
// SSRF guard: undici resolves the host normally; the connect callback compares
|
|
6
|
-
// the resolved IP against ssrf-guard's public-IP classification and aborts
|
|
7
|
-
// the connection if it's private/loopback/metadata. validateAndPinURL still
|
|
8
|
-
// runs first as the pre-request defense.
|
|
9
|
-
import { request, Agent } from 'undici';
|
|
10
|
-
import { USER_AGENT } from './user-agent.js';
|
|
11
|
-
import { wrapFetchedContent } from './untrusted-content.js';
|
|
12
|
-
import { classifyIP } from './ssrf-guard.js';
|
|
13
|
-
import { REDIRECT_ERR_CODE_MAP, validateAndPinURL, extractContentType, isRedirect, extractLocation, extractBodyFromHTML, stripCredentialsFromURL, readBody, drainBody, mapRequestError, } from './web-fetch-helpers.js';
|
|
14
|
-
const ALLOWED_CT = new Set([
|
|
15
|
-
'text/html', 'text/plain',
|
|
16
|
-
'application/xml', 'application/atom+xml', 'application/rss+xml',
|
|
17
|
-
'application/json',
|
|
18
|
-
]);
|
|
19
|
-
const RETURNED_TEXT_CAP = 64 * 1024;
|
|
20
|
-
/**
|
|
21
|
-
* Build the SSRF-revalidating `connect.lookup` for the guard agent.
|
|
22
|
-
*
|
|
23
|
-
* undici invokes `connect.lookup` with `{ all: true }` and expects the callback
|
|
24
|
-
* to receive an ARRAY of `{ address, family }` entries — NOT the single-result
|
|
25
|
-
* `dns.lookup(host, (err, address, family) => ...)` form. Returning a bare
|
|
26
|
-
* address string makes undici read `addresses[0].address === undefined`, throw
|
|
27
|
-
* `ERR_INVALID_IP_ADDRESS`, and surface as `web_fetch_request_failed`. Every
|
|
28
|
-
* callback path here therefore returns the array form.
|
|
29
|
-
*
|
|
30
|
-
* Typed as `net.LookupFunction` — the exact type undici's `connect.lookup`
|
|
31
|
-
* field accepts. On error paths we pass an empty address array (undici reads
|
|
32
|
-
* `err` first and never consumes the addresses), which keeps the runtime
|
|
33
|
-
* contract while satisfying the callback's required address argument.
|
|
34
|
-
*
|
|
35
|
-
* Exported for unit testing — it locks the undici lookup contract without
|
|
36
|
-
* requiring real network (see tests/research/web-fetch.test.ts).
|
|
37
|
-
*/
|
|
38
|
-
export function makeConnectGuardLookup(allowPrivateNetwork, testResolvedIp) {
|
|
39
|
-
return (host, opts, cb) => {
|
|
40
|
-
// If test seam present, return that IP; otherwise let Node resolve.
|
|
41
|
-
if (testResolvedIp) {
|
|
42
|
-
const fam = testResolvedIp.includes(':') ? 6 : 4;
|
|
43
|
-
// Re-validate test IP via ssrf-guard classification.
|
|
44
|
-
if (!allowPrivateNetwork) {
|
|
45
|
-
if (classifyIP(testResolvedIp) !== 'public') {
|
|
46
|
-
cb(new Error('web_fetch_ssrf_postresolve_block'), []);
|
|
47
|
-
return;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
cb(null, [{ address: testResolvedIp, family: fam }]);
|
|
51
|
-
return;
|
|
52
|
-
}
|
|
53
|
-
// Production path: forward undici's options (which carry `all: true`) to
|
|
54
|
-
// Node's resolver, re-validate EVERY resolved address via the SSRF
|
|
55
|
-
// classifier, then return the array form undici expects.
|
|
56
|
-
import('node:dns').then(({ lookup }) => {
|
|
57
|
-
lookup(host, { ...opts, all: true }, (err, addresses) => {
|
|
58
|
-
if (err) {
|
|
59
|
-
cb(err, []);
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
const list = addresses;
|
|
63
|
-
if (!allowPrivateNetwork) {
|
|
64
|
-
for (const a of list) {
|
|
65
|
-
if (classifyIP(a.address) !== 'public') {
|
|
66
|
-
cb(new Error('web_fetch_ssrf_postresolve_block'), []);
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
cb(null, list);
|
|
72
|
-
});
|
|
73
|
-
}).catch(e => cb(e, []));
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
/** Build a shared agent that re-validates the resolved IP at connect time. */
|
|
77
|
-
function makeConnectGuardAgent(allowPrivateNetwork, testResolvedIp, connectTimeoutMs) {
|
|
78
|
-
const lookup = makeConnectGuardLookup(allowPrivateNetwork, testResolvedIp);
|
|
79
|
-
return new Agent({
|
|
80
|
-
connect: { lookup },
|
|
81
|
-
connectTimeout: connectTimeoutMs,
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
export async function webFetch(input) {
|
|
85
|
-
const { cfg, hostAllowlist } = input;
|
|
86
|
-
const privateNetworkHosts = input.privateNetworkHosts ?? new Set();
|
|
87
|
-
let credentialsStripped = false;
|
|
88
|
-
let initial;
|
|
89
|
-
try {
|
|
90
|
-
initial = new URL(input.url);
|
|
91
|
-
}
|
|
92
|
-
catch {
|
|
93
|
-
return { status: 'error', reasonCode: 'web_fetch_invalid_url', credentialsStripped };
|
|
94
|
-
}
|
|
95
|
-
credentialsStripped = stripCredentialsFromURL(initial);
|
|
96
|
-
const totalCtrl = new AbortController();
|
|
97
|
-
const totalTimer = setTimeout(() => totalCtrl.abort(), cfg.totalDeadlineMs);
|
|
98
|
-
try {
|
|
99
|
-
let currentURL = initial.toString();
|
|
100
|
-
let redirects = 0;
|
|
101
|
-
while (true) {
|
|
102
|
-
let v;
|
|
103
|
-
try {
|
|
104
|
-
v = await validateAndPinURL(currentURL, hostAllowlist, privateNetworkHosts, input.resolveIP, totalCtrl.signal);
|
|
105
|
-
}
|
|
106
|
-
catch (e) {
|
|
107
|
-
if (e instanceof DOMException && e.name === 'AbortError') {
|
|
108
|
-
return { status: 'error', reasonCode: 'web_fetch_timeout', credentialsStripped };
|
|
109
|
-
}
|
|
110
|
-
return { status: 'error', reasonCode: 'web_fetch_dns_resolution_failed', credentialsStripped };
|
|
111
|
-
}
|
|
112
|
-
if (!v.ok) {
|
|
113
|
-
if (redirects > 0) {
|
|
114
|
-
const mapped = REDIRECT_ERR_CODE_MAP[v.reasonCode] ?? v.reasonCode;
|
|
115
|
-
return { status: 'error', reasonCode: mapped, host: v.host, credentialsStripped };
|
|
116
|
-
}
|
|
117
|
-
return { status: 'error', reasonCode: v.reasonCode, host: v.host, credentialsStripped };
|
|
118
|
-
}
|
|
119
|
-
// Honor createDispatcher hook for tests: when present, use whatever it
|
|
120
|
-
// returns (or skip dispatcher entirely if undefined → MockAgent
|
|
121
|
-
// intercepts via global). In production, fall back to the connect-guard
|
|
122
|
-
// agent for post-resolve SSRF re-validation.
|
|
123
|
-
let agent;
|
|
124
|
-
if (input.createDispatcher !== undefined) {
|
|
125
|
-
agent = input.createDispatcher(v.host, v.pinnedIP, cfg);
|
|
126
|
-
}
|
|
127
|
-
else {
|
|
128
|
-
agent = makeConnectGuardAgent(cfg.allowPrivateNetwork ?? false, input._testConnectResolvedIp, cfg.connectTimeoutMs);
|
|
129
|
-
}
|
|
130
|
-
const closeAgent = async () => {
|
|
131
|
-
if (agent && typeof agent.close === 'function') {
|
|
132
|
-
try {
|
|
133
|
-
await agent.close();
|
|
134
|
-
}
|
|
135
|
-
catch { /* ignore */ }
|
|
136
|
-
}
|
|
137
|
-
};
|
|
138
|
-
let res;
|
|
139
|
-
try {
|
|
140
|
-
res = await request(v.url.toString(), {
|
|
141
|
-
method: 'GET',
|
|
142
|
-
headersTimeout: cfg.connectTimeoutMs,
|
|
143
|
-
headers: { 'user-agent': USER_AGENT },
|
|
144
|
-
...(agent ? { dispatcher: agent } : {}),
|
|
145
|
-
signal: totalCtrl.signal,
|
|
146
|
-
});
|
|
147
|
-
}
|
|
148
|
-
catch (e) {
|
|
149
|
-
await closeAgent();
|
|
150
|
-
// Map our connect-callback abort to a stable reasonCode.
|
|
151
|
-
const msg = e?.message ?? '';
|
|
152
|
-
if (msg.includes('web_fetch_ssrf_postresolve_block')) {
|
|
153
|
-
return { status: 'error', reasonCode: 'web_fetch_ssrf_postresolve_block', host: v.host, credentialsStripped };
|
|
154
|
-
}
|
|
155
|
-
return { ...mapRequestError(e, totalCtrl.signal, v.host), credentialsStripped };
|
|
156
|
-
}
|
|
157
|
-
if (isRedirect(res.statusCode)) {
|
|
158
|
-
redirects++;
|
|
159
|
-
if (redirects > cfg.maxRedirects) {
|
|
160
|
-
await closeAgent();
|
|
161
|
-
return { status: 'error', reasonCode: 'web_fetch_too_many_redirects', host: v.host, credentialsStripped };
|
|
162
|
-
}
|
|
163
|
-
const location = extractLocation(res.headers);
|
|
164
|
-
if (!location) {
|
|
165
|
-
await closeAgent();
|
|
166
|
-
return { status: 'error', reasonCode: 'web_fetch_redirect_missing_location', host: v.host, credentialsStripped };
|
|
167
|
-
}
|
|
168
|
-
let nextURL;
|
|
169
|
-
try {
|
|
170
|
-
nextURL = new URL(location, v.url);
|
|
171
|
-
}
|
|
172
|
-
catch {
|
|
173
|
-
await closeAgent();
|
|
174
|
-
return { status: 'error', reasonCode: 'web_fetch_redirect_invalid_url', host: v.host, credentialsStripped };
|
|
175
|
-
}
|
|
176
|
-
credentialsStripped = stripCredentialsFromURL(nextURL) || credentialsStripped;
|
|
177
|
-
currentURL = nextURL.toString();
|
|
178
|
-
await drainBody(res.body, totalCtrl.signal);
|
|
179
|
-
await closeAgent();
|
|
180
|
-
if (totalCtrl.signal.aborted) {
|
|
181
|
-
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
182
|
-
}
|
|
183
|
-
continue;
|
|
184
|
-
}
|
|
185
|
-
const contentType = extractContentType(res.headers);
|
|
186
|
-
if (contentType && !ALLOWED_CT.has(contentType)) {
|
|
187
|
-
await drainBody(res.body, totalCtrl.signal);
|
|
188
|
-
await closeAgent();
|
|
189
|
-
if (totalCtrl.signal.aborted) {
|
|
190
|
-
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
191
|
-
}
|
|
192
|
-
return { status: 'error', reasonCode: 'web_fetch_unsupported_content_type', host: v.host, credentialsStripped };
|
|
193
|
-
}
|
|
194
|
-
let rawText;
|
|
195
|
-
let bytesReturned;
|
|
196
|
-
let truncated;
|
|
197
|
-
try {
|
|
198
|
-
const rawBody = res.body;
|
|
199
|
-
const result = await readBody(rawBody, cfg.maxBodyBytes, totalCtrl.signal);
|
|
200
|
-
rawText = result.text;
|
|
201
|
-
bytesReturned = result.bytesReturned;
|
|
202
|
-
truncated = result.truncated;
|
|
203
|
-
}
|
|
204
|
-
catch (e) {
|
|
205
|
-
await closeAgent();
|
|
206
|
-
if (e instanceof DOMException && e.name === 'AbortError') {
|
|
207
|
-
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
208
|
-
}
|
|
209
|
-
if (totalCtrl.signal.aborted) {
|
|
210
|
-
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
211
|
-
}
|
|
212
|
-
return { status: 'error', reasonCode: 'web_fetch_body_read_failed', host: v.host, credentialsStripped };
|
|
213
|
-
}
|
|
214
|
-
await closeAgent();
|
|
215
|
-
let extracted = rawText;
|
|
216
|
-
if (contentType === 'text/html')
|
|
217
|
-
extracted = extractBodyFromHTML(rawText);
|
|
218
|
-
let textTruncated = false;
|
|
219
|
-
if (extracted.length > RETURNED_TEXT_CAP) {
|
|
220
|
-
extracted = extracted.slice(0, RETURNED_TEXT_CAP);
|
|
221
|
-
textTruncated = true;
|
|
222
|
-
}
|
|
223
|
-
const wrapped = wrapFetchedContent({
|
|
224
|
-
url: v.url.toString(), host: v.host, content: extracted,
|
|
225
|
-
});
|
|
226
|
-
return {
|
|
227
|
-
status: 'ok', body: wrapped, rawText, host: v.host,
|
|
228
|
-
bytesReturned, truncated, textTruncated, credentialsStripped,
|
|
229
|
-
};
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
finally {
|
|
233
|
-
clearTimeout(totalTimer);
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
//# sourceMappingURL=web-fetch.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"web-fetch.js","sourceRoot":"","sources":["../../src/research/web-fetch.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,EAAE;AACF,yEAAyE;AACzE,8EAA8E;AAC9E,+EAA+E;AAC/E,2EAA2E;AAC3E,4EAA4E;AAC5E,yCAAyC;AAEzC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAIxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EACL,qBAAqB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,EACxE,eAAe,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,QAAQ,EACvE,SAAS,EAAE,eAAe,GAE3B,MAAM,wBAAwB,CAAC;AA8BhC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,WAAW,EAAE,YAAY;IACzB,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB;IAChE,kBAAkB;CACnB,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAEpC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,sBAAsB,CACpC,mBAA4B,EAC5B,cAAkC;IAElC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;QACxB,oEAAoE;QACpE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,qDAAqD;YACrD,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACzB,IAAI,UAAU,CAAC,cAAc,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAC5C,EAAE,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAA0B,EAAE,EAAE,CAAC,CAAC;oBAC/E,OAAO;gBACT,CAAC;YACH,CAAC;YACD,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QACD,yEAAyE;QACzE,mEAAmE;QACnE,yDAAyD;QACzD,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACrC,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;gBACtD,IAAI,GAAG,EAAE,CAAC;oBAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBACjC,MAAM,IAAI,GAAG,SAA4B,CAAC;gBAC1C,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;wBACrB,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;4BACvC,EAAE,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAA0B,EAAE,EAAE,CAAC,CAAC;4BAC/E,OAAO;wBACT,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,SAAS,qBAAqB,CAC5B,mBAA4B,EAC5B,cAAkC,EAClC,gBAAwB;IAExB,MAAM,MAAM,GAAG,sBAAsB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC3E,OAAO,IAAI,KAAK,CAAC;QACf,OAAO,EAAE,EAAE,MAAM,EAAE;QACnB,cAAc,EAAE,gBAAgB;KACjC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAoB;IACjD,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;IACrC,MAAM,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,IAAI,IAAI,GAAG,EAAU,CAAC;IAC3E,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAEhC,IAAI,OAAY,CAAC;IACjB,IAAI,CAAC;QAAC,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;IACrC,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,CAAC;IAAC,CAAC;IAC/F,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;IAE5E,IAAI,CAAC;QACH,IAAI,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAkC,CAAC;YACvC,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,iBAAiB,CACzB,UAAU,EAAE,aAAa,EAAE,mBAAmB,EAC9C,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAClC,CAAC;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,CAAC;gBACnF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,CAAC;YACjG,CAAC;YACD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACV,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;oBACnE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACpF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1F,CAAC;YAED,uEAAuE;YACvE,gEAAgE;YAChE,wEAAwE;YACxE,6CAA6C;YAC7C,IAAI,KAA8C,CAAC;YACnD,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;gBACzC,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,qBAAqB,CAC3B,GAAG,CAAC,mBAAmB,IAAI,KAAK,EAChC,KAAK,CAAC,sBAAsB,EAC5B,GAAG,CAAC,gBAAgB,CACrB,CAAC;YACJ,CAAC;YACD,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;gBAC5B,IAAI,KAAK,IAAI,OAAQ,KAAyC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBACpF,IAAI,CAAC;wBAAC,MAAO,KAAwC,CAAC,KAAK,EAAE,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBACzF,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,GAAG,CAAC;YACR,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;oBACpC,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,GAAG,CAAC,gBAAgB;oBACpC,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;oBACrC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACvC,MAAM,EAAE,SAAS,CAAC,MAAM;iBACzB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAU,EAAE,CAAC;gBACpB,MAAM,UAAU,EAAE,CAAC;gBACnB,yDAAyD;gBACzD,MAAM,GAAG,GAAI,CAA0B,EAAE,OAAO,IAAI,EAAE,CAAC;gBACvD,IAAI,GAAG,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;oBACrD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,kCAAkC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAChH,CAAC;gBACD,OAAO,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,mBAAmB,EAAE,CAAC;YAClF,CAAC;YAED,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,SAAS,EAAE,CAAC;gBACZ,IAAI,SAAS,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;oBACjC,MAAM,UAAU,EAAE,CAAC;oBACnB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,8BAA8B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAC5G,CAAC;gBACD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;gBACnF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,UAAU,EAAE,CAAC;oBACnB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,qCAAqC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACnH,CAAC;gBACD,IAAI,OAAY,CAAC;gBACjB,IAAI,CAAC;oBAAC,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBAAC,CAAC;gBAC3C,MAAM,CAAC;oBAAC,MAAM,UAAU,EAAE,CAAC;oBAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,gCAAgC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAAC,CAAC;gBAC1I,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,mBAAmB,CAAC;gBAC9E,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAChC,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;YACzF,IAAI,WAAW,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChD,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,oCAAoC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAClH,CAAC;YAED,IAAI,OAAe,CAAC;YAAC,IAAI,aAAqB,CAAC;YAAC,IAAI,SAAkB,CAAC;YACvE,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,GAAG,CAAC,IAAwC,CAAC;gBAC7D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC3E,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;gBAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;gBAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC5F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1G,CAAC;YACD,MAAM,UAAU,EAAE,CAAC;YAEnB,IAAI,SAAS,GAAG,OAAO,CAAC;YACxB,IAAI,WAAW,KAAK,WAAW;gBAAE,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAE1E,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACzC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;gBAClD,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,CAAC;gBACjC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS;aACxD,CAAC,CAAC;YACH,OAAO;gBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI;gBAClD,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB;aAC7D,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}
|