@zhixuan92/multi-model-agent-core 4.7.19 → 4.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (219) hide show
  1. package/README.md +4 -4
  2. package/dist/config/schema.d.ts +2 -20
  3. package/dist/config/schema.d.ts.map +1 -1
  4. package/dist/config/schema.js +9 -80
  5. package/dist/config/schema.js.map +1 -1
  6. package/dist/events/task-envelope.d.ts +10 -1
  7. package/dist/events/task-envelope.d.ts.map +1 -1
  8. package/dist/events/task-envelope.js +4 -1
  9. package/dist/events/task-envelope.js.map +1 -1
  10. package/dist/events/wire-schema.d.ts +4 -0
  11. package/dist/events/wire-schema.d.ts.map +1 -1
  12. package/dist/events/wire-schema.js +3 -3
  13. package/dist/events/wire-schema.js.map +1 -1
  14. package/dist/journal/default-schema.d.ts +2 -0
  15. package/dist/journal/default-schema.d.ts.map +1 -0
  16. package/dist/journal/default-schema.js +27 -0
  17. package/dist/journal/default-schema.js.map +1 -0
  18. package/dist/journal/types.d.ts +22 -0
  19. package/dist/journal/types.d.ts.map +1 -0
  20. package/dist/journal/types.js +5 -0
  21. package/dist/journal/types.js.map +1 -0
  22. package/dist/lifecycle/annotate-parser.js +1 -1
  23. package/dist/lifecycle/annotate-parser.js.map +1 -1
  24. package/dist/lifecycle/derive-completion.js +1 -1
  25. package/dist/lifecycle/derive-completion.js.map +1 -1
  26. package/dist/lifecycle/handlers/annotate-stage.d.ts.map +1 -1
  27. package/dist/lifecycle/handlers/annotate-stage.js +9 -3
  28. package/dist/lifecycle/handlers/annotate-stage.js.map +1 -1
  29. package/dist/lifecycle/handlers/baseline-handlers.d.ts.map +1 -1
  30. package/dist/lifecycle/handlers/baseline-handlers.js +10 -0
  31. package/dist/lifecycle/handlers/baseline-handlers.js.map +1 -1
  32. package/dist/lifecycle/handlers/compose-commit-message.d.ts +15 -0
  33. package/dist/lifecycle/handlers/compose-commit-message.d.ts.map +1 -0
  34. package/dist/lifecycle/handlers/compose-commit-message.js +227 -0
  35. package/dist/lifecycle/handlers/compose-commit-message.js.map +1 -0
  36. package/dist/lifecycle/handlers/enrich-runtime-result.d.ts.map +1 -1
  37. package/dist/lifecycle/handlers/enrich-runtime-result.js +31 -0
  38. package/dist/lifecycle/handlers/enrich-runtime-result.js.map +1 -1
  39. package/dist/lifecycle/handlers/git-commit-handler.d.ts.map +1 -1
  40. package/dist/lifecycle/handlers/git-commit-handler.js +2 -15
  41. package/dist/lifecycle/handlers/git-commit-handler.js.map +1 -1
  42. package/dist/lifecycle/handlers/implement-stage.d.ts.map +1 -1
  43. package/dist/lifecycle/handlers/implement-stage.js +4 -1
  44. package/dist/lifecycle/handlers/implement-stage.js.map +1 -1
  45. package/dist/lifecycle/handlers/journal-review-prompt.d.ts +7 -0
  46. package/dist/lifecycle/handlers/journal-review-prompt.d.ts.map +1 -0
  47. package/dist/lifecycle/handlers/journal-review-prompt.js +54 -0
  48. package/dist/lifecycle/handlers/journal-review-prompt.js.map +1 -0
  49. package/dist/lifecycle/handlers/quality-review-prompt.d.ts +1 -0
  50. package/dist/lifecycle/handlers/quality-review-prompt.d.ts.map +1 -1
  51. package/dist/lifecycle/handlers/quality-review-prompt.js +8 -0
  52. package/dist/lifecycle/handlers/quality-review-prompt.js.map +1 -1
  53. package/dist/lifecycle/handlers/review-stage.d.ts.map +1 -1
  54. package/dist/lifecycle/handlers/review-stage.js +57 -13
  55. package/dist/lifecycle/handlers/review-stage.js.map +1 -1
  56. package/dist/lifecycle/handlers/rework-stage.d.ts.map +1 -1
  57. package/dist/lifecycle/handlers/rework-stage.js +3 -0
  58. package/dist/lifecycle/handlers/rework-stage.js.map +1 -1
  59. package/dist/lifecycle/handlers/spec-review-prompt.d.ts +1 -0
  60. package/dist/lifecycle/handlers/spec-review-prompt.d.ts.map +1 -1
  61. package/dist/lifecycle/handlers/spec-review-prompt.js +8 -0
  62. package/dist/lifecycle/handlers/spec-review-prompt.js.map +1 -1
  63. package/dist/lifecycle/handlers/terminal-handlers.d.ts +25 -1
  64. package/dist/lifecycle/handlers/terminal-handlers.d.ts.map +1 -1
  65. package/dist/lifecycle/handlers/terminal-handlers.js +55 -19
  66. package/dist/lifecycle/handlers/terminal-handlers.js.map +1 -1
  67. package/dist/lifecycle/perform-implementation.d.ts.map +1 -1
  68. package/dist/lifecycle/perform-implementation.js +7 -4
  69. package/dist/lifecycle/perform-implementation.js.map +1 -1
  70. package/dist/lifecycle/read-route-criteria.d.ts +3 -2
  71. package/dist/lifecycle/read-route-criteria.d.ts.map +1 -1
  72. package/dist/lifecycle/read-route-criteria.js +2 -0
  73. package/dist/lifecycle/read-route-criteria.js.map +1 -1
  74. package/dist/lifecycle/research-pre-loop.d.ts +5 -4
  75. package/dist/lifecycle/research-pre-loop.d.ts.map +1 -1
  76. package/dist/lifecycle/research-pre-loop.js +7 -10
  77. package/dist/lifecycle/research-pre-loop.js.map +1 -1
  78. package/dist/lifecycle/stage-io.d.ts +8 -4
  79. package/dist/lifecycle/stage-io.d.ts.map +1 -1
  80. package/dist/lifecycle/stage-io.js +2 -2
  81. package/dist/lifecycle/stage-io.js.map +1 -1
  82. package/dist/lifecycle/stage-plan-types.d.ts +1 -1
  83. package/dist/lifecycle/stage-plan-types.d.ts.map +1 -1
  84. package/dist/lifecycle/stage-progression.d.ts.map +1 -1
  85. package/dist/lifecycle/stage-progression.js +2 -0
  86. package/dist/lifecycle/stage-progression.js.map +1 -1
  87. package/dist/lifecycle/task-runner.d.ts.map +1 -1
  88. package/dist/lifecycle/task-runner.js +6 -1
  89. package/dist/lifecycle/task-runner.js.map +1 -1
  90. package/dist/lifecycle/worker-output-contract.d.ts +3 -1
  91. package/dist/lifecycle/worker-output-contract.d.ts.map +1 -1
  92. package/dist/lifecycle/worker-output-contract.js +4 -1
  93. package/dist/lifecycle/worker-output-contract.js.map +1 -1
  94. package/dist/reporting/headline-templates/journal-recall.d.ts +3 -0
  95. package/dist/reporting/headline-templates/journal-recall.d.ts.map +1 -0
  96. package/dist/reporting/headline-templates/journal-recall.js +9 -0
  97. package/dist/reporting/headline-templates/journal-recall.js.map +1 -0
  98. package/dist/reporting/headline-templates/journal.d.ts +3 -0
  99. package/dist/reporting/headline-templates/journal.d.ts.map +1 -0
  100. package/dist/reporting/headline-templates/journal.js +17 -0
  101. package/dist/reporting/headline-templates/journal.js.map +1 -0
  102. package/dist/reporting/report-parser-slots/journal-report.d.ts +8 -0
  103. package/dist/reporting/report-parser-slots/journal-report.d.ts.map +1 -0
  104. package/dist/reporting/report-parser-slots/journal-report.js +12 -0
  105. package/dist/reporting/report-parser-slots/journal-report.js.map +1 -0
  106. package/dist/reporting/terminal-report-markdown.d.ts +13 -0
  107. package/dist/reporting/terminal-report-markdown.d.ts.map +1 -0
  108. package/dist/reporting/terminal-report-markdown.js +31 -0
  109. package/dist/reporting/terminal-report-markdown.js.map +1 -0
  110. package/dist/research/adapters/index.d.ts +0 -1
  111. package/dist/research/adapters/index.d.ts.map +1 -1
  112. package/dist/research/adapters/index.js +0 -3
  113. package/dist/research/adapters/index.js.map +1 -1
  114. package/dist/research/adapters/types.d.ts +1 -1
  115. package/dist/research/adapters/types.d.ts.map +1 -1
  116. package/dist/research/evidence-pack.d.ts +19 -1
  117. package/dist/research/evidence-pack.d.ts.map +1 -1
  118. package/dist/research/evidence-pack.js +36 -1
  119. package/dist/research/evidence-pack.js.map +1 -1
  120. package/dist/research/orchestrator.d.ts +0 -4
  121. package/dist/research/orchestrator.d.ts.map +1 -1
  122. package/dist/research/orchestrator.js +4 -53
  123. package/dist/research/orchestrator.js.map +1 -1
  124. package/dist/research/query-plan.d.ts +0 -2
  125. package/dist/research/query-plan.d.ts.map +1 -1
  126. package/dist/research/query-plan.js +1 -8
  127. package/dist/research/query-plan.js.map +1 -1
  128. package/dist/research/user-agent.js +1 -1
  129. package/dist/research/user-agent.js.map +1 -1
  130. package/dist/research/web-search.d.ts +2 -1
  131. package/dist/research/web-search.d.ts.map +1 -1
  132. package/dist/research/web-search.js +31 -7
  133. package/dist/research/web-search.js.map +1 -1
  134. package/dist/stores/batch-registry.d.ts +1 -1
  135. package/dist/stores/batch-registry.d.ts.map +1 -1
  136. package/dist/stores/batch-registry.js +4 -4
  137. package/dist/stores/batch-registry.js.map +1 -1
  138. package/dist/tool-surface/register-all-tools.d.ts.map +1 -1
  139. package/dist/tool-surface/register-all-tools.js +4 -0
  140. package/dist/tool-surface/register-all-tools.js.map +1 -1
  141. package/dist/tools/delegate/brief-slot.d.ts +3 -0
  142. package/dist/tools/delegate/brief-slot.d.ts.map +1 -1
  143. package/dist/tools/delegate/brief-slot.js +1 -0
  144. package/dist/tools/delegate/brief-slot.js.map +1 -1
  145. package/dist/tools/delegate/tool-config.d.ts.map +1 -1
  146. package/dist/tools/delegate/tool-config.js +1 -0
  147. package/dist/tools/delegate/tool-config.js.map +1 -1
  148. package/dist/tools/execute-plan/tool-config.d.ts.map +1 -1
  149. package/dist/tools/execute-plan/tool-config.js +3 -0
  150. package/dist/tools/execute-plan/tool-config.js.map +1 -1
  151. package/dist/tools/journal/recall/brief-slot.d.ts +7 -0
  152. package/dist/tools/journal/recall/brief-slot.d.ts.map +1 -0
  153. package/dist/tools/journal/recall/brief-slot.js +5 -0
  154. package/dist/tools/journal/recall/brief-slot.js.map +1 -0
  155. package/dist/tools/journal/recall/implementer-criteria.d.ts +9 -0
  156. package/dist/tools/journal/recall/implementer-criteria.d.ts.map +1 -0
  157. package/dist/tools/journal/recall/implementer-criteria.js +23 -0
  158. package/dist/tools/journal/recall/implementer-criteria.js.map +1 -0
  159. package/dist/tools/journal/recall/schema.d.ts +54 -0
  160. package/dist/tools/journal/recall/schema.d.ts.map +1 -0
  161. package/dist/tools/journal/recall/schema.js +10 -0
  162. package/dist/tools/journal/recall/schema.js.map +1 -0
  163. package/dist/tools/journal/recall/subtypes.d.ts +4 -0
  164. package/dist/tools/journal/recall/subtypes.d.ts.map +1 -0
  165. package/dist/tools/journal/recall/subtypes.js +25 -0
  166. package/dist/tools/journal/recall/subtypes.js.map +1 -0
  167. package/dist/tools/journal/recall/tool-config.d.ts +8 -0
  168. package/dist/tools/journal/recall/tool-config.d.ts.map +1 -0
  169. package/dist/tools/journal/recall/tool-config.js +46 -0
  170. package/dist/tools/journal/recall/tool-config.js.map +1 -0
  171. package/dist/tools/journal/record/brief-slot.d.ts +14 -0
  172. package/dist/tools/journal/record/brief-slot.d.ts.map +1 -0
  173. package/dist/tools/journal/record/brief-slot.js +18 -0
  174. package/dist/tools/journal/record/brief-slot.js.map +1 -0
  175. package/dist/tools/journal/record/implementer-criteria.d.ts +6 -0
  176. package/dist/tools/journal/record/implementer-criteria.d.ts.map +1 -0
  177. package/dist/tools/journal/record/implementer-criteria.js +17 -0
  178. package/dist/tools/journal/record/implementer-criteria.js.map +1 -0
  179. package/dist/tools/journal/record/schema.d.ts +55 -0
  180. package/dist/tools/journal/record/schema.d.ts.map +1 -0
  181. package/dist/tools/journal/record/schema.js +13 -0
  182. package/dist/tools/journal/record/schema.js.map +1 -0
  183. package/dist/tools/journal/record/tool-config.d.ts +7 -0
  184. package/dist/tools/journal/record/tool-config.d.ts.map +1 -0
  185. package/dist/tools/journal/record/tool-config.js +40 -0
  186. package/dist/tools/journal/record/tool-config.js.map +1 -0
  187. package/dist/tools/research/brief-slot.d.ts +0 -2
  188. package/dist/tools/research/brief-slot.d.ts.map +1 -1
  189. package/dist/tools/research/brief-slot.js.map +1 -1
  190. package/dist/tools/research/implementer-criteria.d.ts +0 -1
  191. package/dist/tools/research/implementer-criteria.d.ts.map +1 -1
  192. package/dist/tools/research/implementer-criteria.js +3 -15
  193. package/dist/tools/research/implementer-criteria.js.map +1 -1
  194. package/dist/tools/research/tool-config.d.ts.map +1 -1
  195. package/dist/tools/research/tool-config.js +0 -1
  196. package/dist/tools/research/tool-config.js.map +1 -1
  197. package/dist/types/task-spec.d.ts +2 -1
  198. package/dist/types/task-spec.d.ts.map +1 -1
  199. package/package.json +17 -1
  200. package/dist/research/adapters/generic-rss.d.ts +0 -8
  201. package/dist/research/adapters/generic-rss.d.ts.map +0 -1
  202. package/dist/research/adapters/generic-rss.js +0 -26
  203. package/dist/research/adapters/generic-rss.js.map +0 -1
  204. package/dist/research/ssrf-guard.d.ts +0 -12
  205. package/dist/research/ssrf-guard.d.ts.map +0 -1
  206. package/dist/research/ssrf-guard.js +0 -209
  207. package/dist/research/ssrf-guard.js.map +0 -1
  208. package/dist/research/untrusted-content.d.ts +0 -13
  209. package/dist/research/untrusted-content.d.ts.map +0 -1
  210. package/dist/research/untrusted-content.js +0 -9
  211. package/dist/research/untrusted-content.js.map +0 -1
  212. package/dist/research/web-fetch-helpers.d.ts +0 -44
  213. package/dist/research/web-fetch-helpers.d.ts.map +0 -1
  214. package/dist/research/web-fetch-helpers.js +0 -209
  215. package/dist/research/web-fetch-helpers.js.map +0 -1
  216. package/dist/research/web-fetch.d.ts +0 -55
  217. package/dist/research/web-fetch.d.ts.map +0 -1
  218. package/dist/research/web-fetch.js +0 -236
  219. package/dist/research/web-fetch.js.map +0 -1
@@ -1,236 +0,0 @@
1
- // packages/core/src/research/web-fetch.ts
2
- //
3
- // Replaces the previous IP-pinning dispatcher (which failed 100% of real
4
- // requests on Node 25 / undici current) with a connect-callback re-validation
5
- // SSRF guard: undici resolves the host normally; the connect callback compares
6
- // the resolved IP against ssrf-guard's public-IP classification and aborts
7
- // the connection if it's private/loopback/metadata. validateAndPinURL still
8
- // runs first as the pre-request defense.
9
- import { request, Agent } from 'undici';
10
- import { USER_AGENT } from './user-agent.js';
11
- import { wrapFetchedContent } from './untrusted-content.js';
12
- import { classifyIP } from './ssrf-guard.js';
13
- import { REDIRECT_ERR_CODE_MAP, validateAndPinURL, extractContentType, isRedirect, extractLocation, extractBodyFromHTML, stripCredentialsFromURL, readBody, drainBody, mapRequestError, } from './web-fetch-helpers.js';
14
- const ALLOWED_CT = new Set([
15
- 'text/html', 'text/plain',
16
- 'application/xml', 'application/atom+xml', 'application/rss+xml',
17
- 'application/json',
18
- ]);
19
- const RETURNED_TEXT_CAP = 64 * 1024;
20
- /**
21
- * Build the SSRF-revalidating `connect.lookup` for the guard agent.
22
- *
23
- * undici invokes `connect.lookup` with `{ all: true }` and expects the callback
24
- * to receive an ARRAY of `{ address, family }` entries — NOT the single-result
25
- * `dns.lookup(host, (err, address, family) => ...)` form. Returning a bare
26
- * address string makes undici read `addresses[0].address === undefined`, throw
27
- * `ERR_INVALID_IP_ADDRESS`, and surface as `web_fetch_request_failed`. Every
28
- * callback path here therefore returns the array form.
29
- *
30
- * Typed as `net.LookupFunction` — the exact type undici's `connect.lookup`
31
- * field accepts. On error paths we pass an empty address array (undici reads
32
- * `err` first and never consumes the addresses), which keeps the runtime
33
- * contract while satisfying the callback's required address argument.
34
- *
35
- * Exported for unit testing — it locks the undici lookup contract without
36
- * requiring real network (see tests/research/web-fetch.test.ts).
37
- */
38
- export function makeConnectGuardLookup(allowPrivateNetwork, testResolvedIp) {
39
- return (host, opts, cb) => {
40
- // If test seam present, return that IP; otherwise let Node resolve.
41
- if (testResolvedIp) {
42
- const fam = testResolvedIp.includes(':') ? 6 : 4;
43
- // Re-validate test IP via ssrf-guard classification.
44
- if (!allowPrivateNetwork) {
45
- if (classifyIP(testResolvedIp) !== 'public') {
46
- cb(new Error('web_fetch_ssrf_postresolve_block'), []);
47
- return;
48
- }
49
- }
50
- cb(null, [{ address: testResolvedIp, family: fam }]);
51
- return;
52
- }
53
- // Production path: forward undici's options (which carry `all: true`) to
54
- // Node's resolver, re-validate EVERY resolved address via the SSRF
55
- // classifier, then return the array form undici expects.
56
- import('node:dns').then(({ lookup }) => {
57
- lookup(host, { ...opts, all: true }, (err, addresses) => {
58
- if (err) {
59
- cb(err, []);
60
- return;
61
- }
62
- const list = addresses;
63
- if (!allowPrivateNetwork) {
64
- for (const a of list) {
65
- if (classifyIP(a.address) !== 'public') {
66
- cb(new Error('web_fetch_ssrf_postresolve_block'), []);
67
- return;
68
- }
69
- }
70
- }
71
- cb(null, list);
72
- });
73
- }).catch(e => cb(e, []));
74
- };
75
- }
76
- /** Build a shared agent that re-validates the resolved IP at connect time. */
77
- function makeConnectGuardAgent(allowPrivateNetwork, testResolvedIp, connectTimeoutMs) {
78
- const lookup = makeConnectGuardLookup(allowPrivateNetwork, testResolvedIp);
79
- return new Agent({
80
- connect: { lookup },
81
- connectTimeout: connectTimeoutMs,
82
- });
83
- }
84
- export async function webFetch(input) {
85
- const { cfg, hostAllowlist } = input;
86
- const privateNetworkHosts = input.privateNetworkHosts ?? new Set();
87
- let credentialsStripped = false;
88
- let initial;
89
- try {
90
- initial = new URL(input.url);
91
- }
92
- catch {
93
- return { status: 'error', reasonCode: 'web_fetch_invalid_url', credentialsStripped };
94
- }
95
- credentialsStripped = stripCredentialsFromURL(initial);
96
- const totalCtrl = new AbortController();
97
- const totalTimer = setTimeout(() => totalCtrl.abort(), cfg.totalDeadlineMs);
98
- try {
99
- let currentURL = initial.toString();
100
- let redirects = 0;
101
- while (true) {
102
- let v;
103
- try {
104
- v = await validateAndPinURL(currentURL, hostAllowlist, privateNetworkHosts, input.resolveIP, totalCtrl.signal);
105
- }
106
- catch (e) {
107
- if (e instanceof DOMException && e.name === 'AbortError') {
108
- return { status: 'error', reasonCode: 'web_fetch_timeout', credentialsStripped };
109
- }
110
- return { status: 'error', reasonCode: 'web_fetch_dns_resolution_failed', credentialsStripped };
111
- }
112
- if (!v.ok) {
113
- if (redirects > 0) {
114
- const mapped = REDIRECT_ERR_CODE_MAP[v.reasonCode] ?? v.reasonCode;
115
- return { status: 'error', reasonCode: mapped, host: v.host, credentialsStripped };
116
- }
117
- return { status: 'error', reasonCode: v.reasonCode, host: v.host, credentialsStripped };
118
- }
119
- // Honor createDispatcher hook for tests: when present, use whatever it
120
- // returns (or skip dispatcher entirely if undefined → MockAgent
121
- // intercepts via global). In production, fall back to the connect-guard
122
- // agent for post-resolve SSRF re-validation.
123
- let agent;
124
- if (input.createDispatcher !== undefined) {
125
- agent = input.createDispatcher(v.host, v.pinnedIP, cfg);
126
- }
127
- else {
128
- agent = makeConnectGuardAgent(cfg.allowPrivateNetwork ?? false, input._testConnectResolvedIp, cfg.connectTimeoutMs);
129
- }
130
- const closeAgent = async () => {
131
- if (agent && typeof agent.close === 'function') {
132
- try {
133
- await agent.close();
134
- }
135
- catch { /* ignore */ }
136
- }
137
- };
138
- let res;
139
- try {
140
- res = await request(v.url.toString(), {
141
- method: 'GET',
142
- headersTimeout: cfg.connectTimeoutMs,
143
- headers: { 'user-agent': USER_AGENT },
144
- ...(agent ? { dispatcher: agent } : {}),
145
- signal: totalCtrl.signal,
146
- });
147
- }
148
- catch (e) {
149
- await closeAgent();
150
- // Map our connect-callback abort to a stable reasonCode.
151
- const msg = e?.message ?? '';
152
- if (msg.includes('web_fetch_ssrf_postresolve_block')) {
153
- return { status: 'error', reasonCode: 'web_fetch_ssrf_postresolve_block', host: v.host, credentialsStripped };
154
- }
155
- return { ...mapRequestError(e, totalCtrl.signal, v.host), credentialsStripped };
156
- }
157
- if (isRedirect(res.statusCode)) {
158
- redirects++;
159
- if (redirects > cfg.maxRedirects) {
160
- await closeAgent();
161
- return { status: 'error', reasonCode: 'web_fetch_too_many_redirects', host: v.host, credentialsStripped };
162
- }
163
- const location = extractLocation(res.headers);
164
- if (!location) {
165
- await closeAgent();
166
- return { status: 'error', reasonCode: 'web_fetch_redirect_missing_location', host: v.host, credentialsStripped };
167
- }
168
- let nextURL;
169
- try {
170
- nextURL = new URL(location, v.url);
171
- }
172
- catch {
173
- await closeAgent();
174
- return { status: 'error', reasonCode: 'web_fetch_redirect_invalid_url', host: v.host, credentialsStripped };
175
- }
176
- credentialsStripped = stripCredentialsFromURL(nextURL) || credentialsStripped;
177
- currentURL = nextURL.toString();
178
- await drainBody(res.body, totalCtrl.signal);
179
- await closeAgent();
180
- if (totalCtrl.signal.aborted) {
181
- return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
182
- }
183
- continue;
184
- }
185
- const contentType = extractContentType(res.headers);
186
- if (contentType && !ALLOWED_CT.has(contentType)) {
187
- await drainBody(res.body, totalCtrl.signal);
188
- await closeAgent();
189
- if (totalCtrl.signal.aborted) {
190
- return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
191
- }
192
- return { status: 'error', reasonCode: 'web_fetch_unsupported_content_type', host: v.host, credentialsStripped };
193
- }
194
- let rawText;
195
- let bytesReturned;
196
- let truncated;
197
- try {
198
- const rawBody = res.body;
199
- const result = await readBody(rawBody, cfg.maxBodyBytes, totalCtrl.signal);
200
- rawText = result.text;
201
- bytesReturned = result.bytesReturned;
202
- truncated = result.truncated;
203
- }
204
- catch (e) {
205
- await closeAgent();
206
- if (e instanceof DOMException && e.name === 'AbortError') {
207
- return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
208
- }
209
- if (totalCtrl.signal.aborted) {
210
- return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
211
- }
212
- return { status: 'error', reasonCode: 'web_fetch_body_read_failed', host: v.host, credentialsStripped };
213
- }
214
- await closeAgent();
215
- let extracted = rawText;
216
- if (contentType === 'text/html')
217
- extracted = extractBodyFromHTML(rawText);
218
- let textTruncated = false;
219
- if (extracted.length > RETURNED_TEXT_CAP) {
220
- extracted = extracted.slice(0, RETURNED_TEXT_CAP);
221
- textTruncated = true;
222
- }
223
- const wrapped = wrapFetchedContent({
224
- url: v.url.toString(), host: v.host, content: extracted,
225
- });
226
- return {
227
- status: 'ok', body: wrapped, rawText, host: v.host,
228
- bytesReturned, truncated, textTruncated, credentialsStripped,
229
- };
230
- }
231
- }
232
- finally {
233
- clearTimeout(totalTimer);
234
- }
235
- }
236
- //# sourceMappingURL=web-fetch.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"web-fetch.js","sourceRoot":"","sources":["../../src/research/web-fetch.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,EAAE;AACF,yEAAyE;AACzE,8EAA8E;AAC9E,+EAA+E;AAC/E,2EAA2E;AAC3E,4EAA4E;AAC5E,yCAAyC;AAEzC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAIxC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EACL,qBAAqB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,EACxE,eAAe,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,QAAQ,EACvE,SAAS,EAAE,eAAe,GAE3B,MAAM,wBAAwB,CAAC;AA8BhC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,WAAW,EAAE,YAAY;IACzB,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB;IAChE,kBAAkB;CACnB,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAEpC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,sBAAsB,CACpC,mBAA4B,EAC5B,cAAkC;IAElC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;QACxB,oEAAoE;QACpE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,qDAAqD;YACrD,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBACzB,IAAI,UAAU,CAAC,cAAc,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAC5C,EAAE,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAA0B,EAAE,EAAE,CAAC,CAAC;oBAC/E,OAAO;gBACT,CAAC;YACH,CAAC;YACD,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrD,OAAO;QACT,CAAC;QACD,yEAAyE;QACzE,mEAAmE;QACnE,yDAAyD;QACzD,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE;YACrC,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE;gBACtD,IAAI,GAAG,EAAE,CAAC;oBAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAAC,OAAO;gBAAC,CAAC;gBACjC,MAAM,IAAI,GAAG,SAA4B,CAAC;gBAC1C,IAAI,CAAC,mBAAmB,EAAE,CAAC;oBACzB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;wBACrB,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;4BACvC,EAAE,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAA0B,EAAE,EAAE,CAAC,CAAC;4BAC/E,OAAO;wBACT,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,SAAS,qBAAqB,CAC5B,mBAA4B,EAC5B,cAAkC,EAClC,gBAAwB;IAExB,MAAM,MAAM,GAAG,sBAAsB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;IAC3E,OAAO,IAAI,KAAK,CAAC;QACf,OAAO,EAAE,EAAE,MAAM,EAAE;QACnB,cAAc,EAAE,gBAAgB;KACjC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAoB;IACjD,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;IACrC,MAAM,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,IAAI,IAAI,GAAG,EAAU,CAAC;IAC3E,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAEhC,IAAI,OAAY,CAAC;IACjB,IAAI,CAAC;QAAC,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAAC,CAAC;IACrC,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,CAAC;IAAC,CAAC;IAC/F,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvD,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;IAE5E,IAAI,CAAC;QACH,IAAI,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,CAAkC,CAAC;YACvC,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,iBAAiB,CACzB,UAAU,EAAE,aAAa,EAAE,mBAAmB,EAC9C,KAAK,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAClC,CAAC;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,CAAC;gBACnF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,CAAC;YACjG,CAAC;YACD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACV,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;oBACnE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACpF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1F,CAAC;YAED,uEAAuE;YACvE,gEAAgE;YAChE,wEAAwE;YACxE,6CAA6C;YAC7C,IAAI,KAA8C,CAAC;YACnD,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;gBACzC,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,qBAAqB,CAC3B,GAAG,CAAC,mBAAmB,IAAI,KAAK,EAChC,KAAK,CAAC,sBAAsB,EAC5B,GAAG,CAAC,gBAAgB,CACrB,CAAC;YACJ,CAAC;YACD,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE;gBAC5B,IAAI,KAAK,IAAI,OAAQ,KAAyC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;oBACpF,IAAI,CAAC;wBAAC,MAAO,KAAwC,CAAC,KAAK,EAAE,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;gBACzF,CAAC;YACH,CAAC,CAAC;YAEF,IAAI,GAAG,CAAC;YACR,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;oBACpC,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,GAAG,CAAC,gBAAgB;oBACpC,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;oBACrC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACvC,MAAM,EAAE,SAAS,CAAC,MAAM;iBACzB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAU,EAAE,CAAC;gBACpB,MAAM,UAAU,EAAE,CAAC;gBACnB,yDAAyD;gBACzD,MAAM,GAAG,GAAI,CAA0B,EAAE,OAAO,IAAI,EAAE,CAAC;gBACvD,IAAI,GAAG,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;oBACrD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,kCAAkC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAChH,CAAC;gBACD,OAAO,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,mBAAmB,EAAE,CAAC;YAClF,CAAC;YAED,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,SAAS,EAAE,CAAC;gBACZ,IAAI,SAAS,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;oBACjC,MAAM,UAAU,EAAE,CAAC;oBACnB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,8BAA8B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAC5G,CAAC;gBACD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;gBACnF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,UAAU,EAAE,CAAC;oBACnB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,qCAAqC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACnH,CAAC;gBACD,IAAI,OAAY,CAAC;gBACjB,IAAI,CAAC;oBAAC,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBAAC,CAAC;gBAC3C,MAAM,CAAC;oBAAC,MAAM,UAAU,EAAE,CAAC;oBAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,gCAAgC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAAC,CAAC;gBAC1I,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,mBAAmB,CAAC;gBAC9E,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAChC,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;YACzF,IAAI,WAAW,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChD,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,oCAAoC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAClH,CAAC;YAED,IAAI,OAAe,CAAC;YAAC,IAAI,aAAqB,CAAC;YAAC,IAAI,SAAkB,CAAC;YACvE,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,GAAG,CAAC,IAAwC,CAAC;gBAC7D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC3E,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;gBAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;gBAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC5F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,UAAU,EAAE,CAAC;gBACnB,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1G,CAAC;YACD,MAAM,UAAU,EAAE,CAAC;YAEnB,IAAI,SAAS,GAAG,OAAO,CAAC;YACxB,IAAI,WAAW,KAAK,WAAW;gBAAE,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAE1E,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACzC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;gBAClD,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,CAAC;gBACjC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS;aACxD,CAAC,CAAC;YACH,OAAO;gBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI;gBAClD,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB;aAC7D,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}