@lincoln504/pi-research 1.4.1 → 1.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/cli.mjs +271 -156
  2. package/dist/thread-worker.mjs +91 -59
  3. package/docs/CHANGELOG.md +76 -0
  4. package/docs/CONFIGURATION.md +1 -1
  5. package/package.json +1 -1
  6. package/skills/pi-research/SKILL.md +1 -1
  7. package/src/config.ts +10 -4
  8. package/src/core/planning-service.ts +55 -8
  9. package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
  10. package/src/infrastructure/browser/config.ts +40 -2
  11. package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
  12. package/src/infrastructure/browser/thread-worker-messaging.ts +61 -4
  13. package/src/infrastructure/browser/thread-worker.mjs +91 -59
  14. package/src/infrastructure/browser/thread-worker.ts +13 -5
  15. package/src/infrastructure/embedding/embedding-server.ts +34 -0
  16. package/src/knowledge/embedder-init.ts +3 -2
  17. package/src/knowledge/embedder.ts +11 -3
  18. package/src/knowledge/index.ts +36 -10
  19. package/src/knowledge/store.ts +29 -5
  20. package/src/orchestration/deep-research-orchestrator.ts +3 -1
  21. package/src/orchestration/session-state.ts +16 -0
  22. package/src/sdk.ts +10 -0
  23. package/src/skill-install/skill-installer.ts +5 -1
  24. package/src/stackexchange/index.ts +3 -0
  25. package/src/stackexchange/rest-client.ts +7 -12
  26. package/src/tools/research-tool-definition.ts +38 -7
  27. package/src/tools/security.ts +13 -10
  28. package/src/utils/coverage-digest.ts +1 -1
  29. package/src/utils/host-config.ts +12 -1
  30. package/src/utils/log-utils.ts +10 -0
  31. package/src/web-research/browser-search.ts +11 -4
  32. package/src/web-research/web-scraper.ts +27 -16
  33. package/src/youtube/potoken.ts +8 -1
  34. package/src/youtube/transcript-client.ts +56 -5
package/src/sdk.ts CHANGED
@@ -369,6 +369,16 @@ async function _doInit(options: ResearchSDKOptions = {}): Promise<void> {
369
369
  * Public initialization. Guarantees the SDK is ready for use.
370
370
  */
371
371
  export async function initResearchSDK(options: ResearchSDKOptions = {}): Promise<void> {
372
+ // A shutdown may be in flight: isInitialized only flips to false near the END
373
+ // of _doShutdown (after container disposal), so without this a caller racing a
374
+ // shutdown would hit the isInitialized branch below, log its "already
375
+ // initialized" warning, and return believing the SDK is ready — while the
376
+ // container it would use is mid-teardown or about to be nulled out from under
377
+ // it. Wait the shutdown out first so this call observes the real post-shutdown
378
+ // state, mirroring shutdownResearchSDK's symmetric wait on _initPromise.
379
+ if (_shutdownPromise) {
380
+ await _shutdownPromise.catch(() => { /* a failed shutdown must not block a fresh init */ });
381
+ }
372
382
  if (isInitialized) {
373
383
  logger.warn('[SDK] Research SDK already initialized. Call shutdownResearchSDK() first if you want to re-initialize with new options.');
374
384
  return;
@@ -480,7 +480,11 @@ export function uninstallSkill(toolIds?: string[], opts: InstallOptions = {}): U
480
480
  }
481
481
 
482
482
  for (const { tool, path: skillPath } of targets.values()) {
483
- if (toolIds && toolIds.length && !toolIds.includes(tool)) {
483
+ // undefined = every manifest entry; an explicit (including empty) list means
484
+ // exactly those tools. `toolIds.length` used to gate this, so `[]` was
485
+ // indistinguishable from `undefined` and silently uninstalled everything —
486
+ // inconsistent with this file's other list params, where empty means none.
487
+ if (toolIds !== undefined && !toolIds.includes(tool)) {
484
488
  const e = manifest.entries.find(x => x.path === skillPath);
485
489
  if (e) keep.push(e);
486
490
  continue;
@@ -27,6 +27,7 @@ import {
27
27
  } from './output/compact.ts';
28
28
  import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
29
29
  import { logger } from '../logger.ts';
30
+ import { metrics } from '../utils/metrics.ts';
30
31
 
31
32
  function notify(
32
33
  ctx: ExtensionContext,
@@ -86,6 +87,7 @@ export async function stackexchangeCommand(options: {
86
87
  try {
87
88
  // Check quota before making requests
88
89
  if (client.isQuotaExhausted()) {
90
+ metrics.increment('stackexchange_quota_exhausted_total', 1);
89
91
  const quota = client.getQuotaInfo();
90
92
  notify(
91
93
  ctx,
@@ -103,6 +105,7 @@ export async function stackexchangeCommand(options: {
103
105
 
104
106
  // Warn if quota is low
105
107
  if (client.isQuotaLow()) {
108
+ metrics.increment('stackexchange_quota_low_total', 1);
106
109
  const quota = client.getQuotaInfo();
107
110
  notify(
108
111
  ctx,
@@ -193,24 +193,19 @@ export class StackExchangeClient {
193
193
  });
194
194
  }
195
195
 
196
+ // Pure accessor — no metrics side effects. `stackexchangeCommand()` calls this
197
+ // repeatedly per invocation (pre-request gate checks, success footer, error
198
+ // footer) purely to read current state for display; incrementing counters here
199
+ // used to record 2-3x the real number of quota-low/exhausted events per call.
200
+ // The counters are incremented once, at the single gate check that actually
201
+ // observes each event (see stackexchange/index.ts).
196
202
  getQuotaInfo(): { remaining: number; max: number; requestCount: number; lastBackoff: number | null } {
197
- const info = {
203
+ return {
198
204
  remaining: this.quotaRemaining,
199
205
  max: this.quotaMax,
200
206
  requestCount: this.requestCount,
201
207
  lastBackoff: this.lastBackoff,
202
208
  };
203
-
204
- // Track quota exhaustion events
205
- if (this.isQuotaExhausted()) {
206
- metrics.increment('stackexchange_quota_exhausted_total', 1);
207
- }
208
-
209
- if (this.isQuotaLow()) {
210
- metrics.increment('stackexchange_quota_low_total', 1);
211
- }
212
-
213
- return info;
214
209
  }
215
210
 
216
211
  isQuotaExhausted(): boolean {
@@ -17,6 +17,7 @@ import type {
17
17
  import type { ExtendedExtensionContext } from '../types/extension-context.ts';
18
18
  import type { ResearchDepth, CleanupContext } from '../types/index.ts';
19
19
  import { Type, type Static } from 'typebox';
20
+ import { Value } from 'typebox/value';
20
21
  import { validateConfig, getConfig, type ConfigInterface } from '../config.ts';
21
22
  import { tryGetServiceContainerFromCtx, getService } from '../core/service-registry.ts';
22
23
 
@@ -28,7 +29,7 @@ import { redactSecrets } from '../utils/log-utils.ts';
28
29
  import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
29
30
  import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
30
31
  import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
31
- import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages, getPiActivePanels } from '../orchestration/session-state.ts';
32
+ import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages, getActiveResearchRunCount } from '../orchestration/session-state.ts';
32
33
  import { createResearchTuiManager, hideWorkingIndicator } from '../tui/research-tui-manager.ts';
33
34
  import { createCleanupFunction } from '../cleanup/research-cleanup.ts';
34
35
  import { createResearchObserver, createObserverState, stopObserverWaveAnimation } from '../observers/research-observer-impl.ts';
@@ -118,6 +119,21 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
118
119
 
119
120
  type ResearchParams = Static<typeof parameters>;
120
121
 
122
+ // Same field shapes as `parameters`, but WITHOUT the initialLinks count/length
123
+ // bounds — used only for the runtime execute() gate below. validateInitialLinks
124
+ // already enforces those same two bounds with a specific, friendly message
125
+ // ("at most N URLs...", "at most N characters..."); checking them again here
126
+ // would just produce the generic 'invalid_parameters' rejection first and mask
127
+ // that better message. Structural type-checking (initialLinks must still be an
128
+ // array of strings) is kept, since validateInitialLinks assumes that shape.
129
+ const executeShapeCheck = Type.Object({
130
+ query: Type.String(),
131
+ depth: Type.Optional(Type.Integer({ minimum: 1, maximum: 3 })),
132
+ model: Type.Optional(Type.String()),
133
+ excludeTools: Type.Optional(Type.Array(Type.String())),
134
+ initialLinks: Type.Optional(Type.Array(Type.String())),
135
+ });
136
+
121
137
  return {
122
138
  name: 'research',
123
139
  label: 'Research',
@@ -169,6 +185,17 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
169
185
  };
170
186
  }
171
187
 
188
+ // Every other tool in this codebase re-validates params against its own
189
+ // schema before trusting it (the host does not enforce this) — this was the
190
+ // one exception, relying entirely on incidental downstream checks (a
191
+ // typeof on query, a warn-gate on excludeTools) for fields that skip both.
192
+ if (!Value.Check(executeShapeCheck, params)) {
193
+ return {
194
+ content: [{ type: 'text', text: 'Error: invalid parameters for research tool.' }],
195
+ details: { error: 'invalid_parameters' },
196
+ };
197
+ }
198
+
172
199
  const { query, depth: rawDepth, model: modelId, excludeTools: paramExcludeTools, initialLinks } = params as ResearchParams;
173
200
 
174
201
  // Enforce the same bounds the CLI applies to --initial-links: these links are
@@ -227,11 +254,13 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
227
254
  }
228
255
  // Steering messages are shared per pi-session. Only reset them when THIS is
229
256
  // the last active run — otherwise a finishing run wipes a concurrent sibling
230
- // run's queued/active steering mid-flight. This run's panel is still
231
- // registered here (endResearchSession runs later in cleanup()), so a count
232
- // of <= 1 means no other run is active.
257
+ // run's queued/active steering mid-flight. This run's abort registration is
258
+ // still in place here (endResearchSession runs later in cleanup()), so a
259
+ // count of <= 1 means no other run is active. getActiveResearchRunCount
260
+ // (not getPiActivePanels) because headless runs never register a panel —
261
+ // a panel-based check here was always a no-op for headless callers.
233
262
  try {
234
- if (getPiActivePanels(piSessionId).length <= 1) clearSteeringMessages(piSessionId);
263
+ if (getActiveResearchRunCount(piSessionId) <= 1) clearSteeringMessages(piSessionId);
235
264
  } catch (e) { logger.error('[research] Clear steering messages failed:', e); }
236
265
  if (detachAbortListener) {
237
266
  try { detachAbortListener(); } catch { /* best-effort */ } finally { detachAbortListener = undefined; }
@@ -344,9 +373,11 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
344
373
  // module-level piSessions map keyed by the STABLE pi session id, so
345
374
  // without this every headless run leaked them for the process
346
375
  // lifetime (the TUI branch frees them via createCleanupFunction, the
347
- // SDK in its own finally; this branch freed only steering).
376
+ // SDK in its own finally). Steering is NOT cleared here — teardownUi()
377
+ // already does that, gated on getActiveResearchRunCount so a finishing
378
+ // run doesn't wipe a concurrent sibling headless run's steering; an
379
+ // unconditional clear here bypassed that guard entirely.
348
380
  endResearchSession(piSessionId, sessionResearchId);
349
- clearSteeringMessages(piSessionId);
350
381
  };
351
382
 
352
383
  // Minimal stub — only stopObserverWaveAnimation touches panelState in the
@@ -98,16 +98,6 @@ export function createSecuritySearchTool(options: {
98
98
  };
99
99
  }
100
100
 
101
- // Record call in tracker - returns false if limit reached
102
- const allowed = options.tracker.recordCall('security_search');
103
- if (!allowed) {
104
- metrics.increment('tool_security_search_calls_total', 1, { status: 'rate_limited' });
105
- return {
106
- content: [{ type: 'text', text: options.tracker.getLimitMessage('security_search') }],
107
- details: { blocked: true, reason: 'limit_reached' },
108
- };
109
- }
110
-
111
101
  const p = params as SecuritySearchParamsType;
112
102
 
113
103
  const terms = p.terms;
@@ -160,6 +150,19 @@ export function createSecuritySearchTool(options: {
160
150
  }
161
151
  const maxResults = p.maxResults ?? 20;
162
152
 
153
+ // Record call in tracker - returns false if limit reached. Deliberately AFTER
154
+ // every semantic-validation reject above (terms/databases/severity), same
155
+ // reasoning as the schema check: a rejected call did no work and must not
156
+ // permanently consume one of the researcher's shared MAX_GATHERING_CALLS slots.
157
+ const allowed = options.tracker.recordCall('security_search');
158
+ if (!allowed) {
159
+ metrics.increment('tool_security_search_calls_total', 1, { status: 'rate_limited' });
160
+ return {
161
+ content: [{ type: 'text', text: options.tracker.getLimitMessage('security_search') }],
162
+ details: { blocked: true, reason: 'limit_reached' },
163
+ };
164
+ }
165
+
163
166
  metrics.increment('tool_security_search_terms_total', terms.length);
164
167
  metrics.increment('tool_security_search_databases_total', databases.length);
165
168
 
@@ -109,7 +109,7 @@ export interface SplitReport {
109
109
  * better than one that reaches it short.
110
110
  */
111
111
  export function splitCoverageDigest(report: string): SplitReport {
112
- if (!report) return { digest: '', body: report ?? '' };
112
+ if (!report) return { digest: '', body: report };
113
113
 
114
114
  const headerMatch = delimiterRegex(COVERAGE_DIGEST_HEADER).exec(report);
115
115
  if (!headerMatch) return { digest: '', body: report };
@@ -27,11 +27,22 @@ let _configDirName: string | undefined;
27
27
  * Resolution order: explicit env override (`PI_RESEARCH_CONFIG_DIR_NAME`) →
28
28
  * the host `@earendil-works/pi-coding-agent` package (only where it is actually
29
29
  * resolvable, i.e. the main pi process) → the stable default `.pi`.
30
+ *
31
+ * The override is a directory NAME, not a path: it is joined onto
32
+ * `os.homedir()` by every caller (getGlobalConfigDir() in config.ts), which in
33
+ * turn is the base for config.env (API keys, chmod 0600), the state dir
34
+ * (locks, run semaphores), and the knowledge_db. A value containing a path
35
+ * separator or `..` traverses out of HOME entirely — scripts/cleanup.cjs
36
+ * already rejects exactly that in its own copy of this resolution (see its
37
+ * `rawDirName`/`dirName` guard); this must reject it the same way, since this
38
+ * is the copy every actual run resolves secrets and state through.
30
39
  */
31
40
  export function getConfigDirName(): string {
32
41
  if (_configDirName !== undefined) return _configDirName;
33
42
  const override = process.env['PI_RESEARCH_CONFIG_DIR_NAME'];
34
- if (override) return (_configDirName = override);
43
+ if (override && !override.includes('/') && !override.includes('\\') && override !== '.' && override !== '..') {
44
+ return (_configDirName = override);
45
+ }
35
46
  try {
36
47
  const require = createRequire(import.meta.url);
37
48
  const host = require('@earendil-works/pi-coding-agent') as { CONFIG_DIR_NAME?: string };
@@ -255,6 +255,16 @@ export function redactSecrets(message: string): string {
255
255
  // never negative; it still prefers the larger rawLength when the
256
256
  // REDACT_SCAN_LENGTH cut discarded raw content redaction then shrank.
257
257
  out = `${out.slice(0, keep)}…[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
258
+ } else if (rawLength > scanEnd) {
259
+ // The scanEnd cut (line 220) already dropped `rawLength - scanEnd` raw
260
+ // chars before redaction ever ran. If redaction then net-shrinks the
261
+ // scanned prefix enough that `out.length` no longer exceeds
262
+ // MAX_LOG_MESSAGE_LENGTH (e.g. many long hex/JWT runs each collapsed to
263
+ // the short literal '[REDACTED]'), the branch above never fires and this
264
+ // message would otherwise read as complete — silently hiding the tail of
265
+ // a large scraped page or provider error body with no indication
266
+ // anything was cut.
267
+ out = `${out}…[truncated ${rawLength - scanEnd} chars]`;
258
268
  }
259
269
  return out;
260
270
  }
@@ -5,7 +5,7 @@
5
5
  */
6
6
 
7
7
  import { runWorkerSearch } from '../infrastructure/browser/task-execution-service.ts';
8
- import { getMaxWorkers } from '../infrastructure/browser/config.ts';
8
+ import { getMaxWorkers, COLD_START_ALLOWANCE_MS } from '../infrastructure/browser/config.ts';
9
9
  import { isTaskTimeoutError } from '../infrastructure/browser/browser-error-utils.ts';
10
10
  import { logger } from '../logger.ts';
11
11
  import { safeUnref } from '../utils/safe-unref.ts';
@@ -82,8 +82,9 @@ export async function performSearch(
82
82
  // cannot stall the entire Promise.all burst.
83
83
  //
84
84
  // The budget mirrors the scheduler's own task ceiling in
85
- // BrowserTaskScheduler.runSearch (= SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS)
86
- // so this layer never preempts the worker's deadline with a shorter one.
85
+ // BrowserTaskScheduler.runSearch (= SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS +
86
+ // COLD_START_ALLOWANCE_MS) so this layer never preempts the worker's deadline with
87
+ // a shorter one.
87
88
  //
88
89
  // Both clocks now start at DISPATCH (see startQueryDeadline below), which is
89
90
  // what makes mirroring the value correct. While this one started at enqueue,
@@ -95,7 +96,13 @@ export async function performSearch(
95
96
  // query's worth of wait, and a burst four times the pool's reach exhausts any
96
97
  // constant. Measuring execution instead of waiting is the fix; the sum
97
98
  // survives as the budget because a search legitimately costs nav plus overhead.
98
- const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45_000) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 10_000);
99
+ //
100
+ // COLD_START_ALLOWANCE_MS is part of the sum for the same reason it is part of
101
+ // the scheduler's own ceiling: the first query dispatched to a freshly created or
102
+ // just-reset worker pays a real browser launch + context creation inline before
103
+ // any navigation starts, and this layer's budget must not fire before the
104
+ // scheduler's own (correct, larger) deadline gets a chance to.
105
+ const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45_000) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 10_000) + COLD_START_ALLOWANCE_MS;
99
106
 
100
107
  // Deduplicated: resultMap and the failures map are both keyed by the query
101
108
  // string, so two tasks for the same string race last-writer-wins — instance A
@@ -123,23 +123,34 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
123
123
 
124
124
  try {
125
125
  const doc = new WasmPdfDocument(bytes);
126
- const pageCount = doc.pageCount();
127
-
128
- let markdown = `# PDF Document\n\n**Pages:** ${pageCount}\n\n`;
129
-
126
+ // free() is native/WASM-side memory, not GC-reclaimed — it must run on
127
+ // every exit, including the case toMarkdownAll() AND the per-page fallback
128
+ // both throw (a malformed/encrypted PDF the codebase already treats as
129
+ // routine on the open web, so this path fires regularly). It used to sit
130
+ // after the fallback loop, reachable only when nothing below it threw —
131
+ // the one case that needed the fallback landing squarely on the one case
132
+ // that skipped the free(), leaking one WasmPdfDocument per occurrence in a
133
+ // browser-server process that stays warm across many research sessions.
130
134
  try {
131
- markdown += doc.toMarkdownAll();
132
- } catch {
133
- for (let i = 0; i < pageCount; i++) {
134
- markdown += `## Page ${i + 1}\n\n${doc.toMarkdown(i)}\n\n`;
135
+ const pageCount = doc.pageCount();
136
+
137
+ let markdown = `# PDF Document\n\n**Pages:** ${pageCount}\n\n`;
138
+
139
+ try {
140
+ markdown += doc.toMarkdownAll();
141
+ } catch {
142
+ for (let i = 0; i < pageCount; i++) {
143
+ markdown += `## Page ${i + 1}\n\n${doc.toMarkdown(i)}\n\n`;
144
+ }
135
145
  }
146
+
147
+ const pdfExtractionDuration = Date.now() - pdfExtractionStart;
148
+ metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
149
+ metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
150
+ return markdown;
151
+ } finally {
152
+ doc.free();
136
153
  }
137
-
138
- doc.free();
139
- const pdfExtractionDuration = Date.now() - pdfExtractionStart;
140
- metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
141
- metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
142
- return markdown;
143
154
  } catch (e) {
144
155
  const msg = e instanceof Error ? e.message : String(e);
145
156
  // debug, not error: a malformed/encrypted/scanned-image-only PDF is an
@@ -300,7 +311,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
300
311
  const contentLength = response.headers.get('content-length');
301
312
  if (contentLength) {
302
313
  const size = parseInt(contentLength, 10);
303
- if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
314
+ if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
304
315
  if (size > MAX_PDF_SIZE) {
305
316
  const sizeMB = Math.round(size / 1024 / 1024);
306
317
  logger.warn(`[Scrapers] PDF too large (Content-Length: ${sizeMB}MB, max 100MB), skipping`);
@@ -319,7 +330,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
319
330
  }
320
331
  }
321
332
 
322
- if (contentType.includes('application/pdf') || url.toLowerCase().endsWith('.pdf')) {
333
+ if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
323
334
  let pdfBytes: Uint8Array;
324
335
  try {
325
336
  pdfBytes = await readBodyCapped(response, MAX_PDF_SIZE);
@@ -44,6 +44,7 @@
44
44
  import { logger } from '../logger.ts';
45
45
  import { safeUnref } from '../utils/safe-unref.ts';
46
46
  import { metrics } from '../utils/metrics.ts';
47
+ import { readJsonCapped } from '../utils/http-body.ts';
47
48
 
48
49
  /**
49
50
  * Well-known request key for the YouTube web PoToken program. This is a public
@@ -229,7 +230,13 @@ async function doMint(
229
230
  body: JSON.stringify([requestKey, botguardResponse]),
230
231
  signal: opts.signal,
231
232
  });
232
- const integrityJson = (await integrityResponse.json()) as unknown[];
233
+ if (!integrityResponse.ok) {
234
+ throw new Error(`HTTP ${integrityResponse.status} from attestation server`);
235
+ }
236
+ // The attestation response is a short JSON array; readJsonCapped bounds it the
237
+ // same way as every other externally-controlled body this codebase reads (the
238
+ // default 32MB cap is already generous for this endpoint).
239
+ const integrityJson = await readJsonCapped<unknown[]>(integrityResponse);
233
240
  const integrityToken = integrityJson?.[0] as string | undefined;
234
241
  if (!integrityToken) {
235
242
  throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');
@@ -18,6 +18,7 @@
18
18
  */
19
19
 
20
20
  import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
21
+ import { getSsrfSafeFetcher } from '../web-research/scraper-utils.ts';
21
22
  import { CircuitBreaker } from '../utils/circuit-breaker.ts';
22
23
  import { logger } from '../logger.ts';
23
24
  import { redactSecrets } from '../utils/log-utils.ts';
@@ -109,7 +110,31 @@ export async function fetchVideoTranscripts(
109
110
  } catch {
110
111
  // non-fatal: logging stays at default if the API ever changes
111
112
  }
112
- const fetchImpl: typeof fetch = (input, init) => fetch(input, init);
113
+ // Connect-time SSRF pin (see scraper-utils.ts's getSsrfSafeFetcher): closes the
114
+ // gap between assertTrustedTimedtextUrl's host check below and fetch's own
115
+ // re-resolution at connect time. The host allowlist only constrains WHICH
116
+ // hostname a redirect can name; it says nothing about what that hostname
117
+ // resolves to at the moment the socket opens, and this codebase already
118
+ // treats a resolver answering differently between validation and connect as
119
+ // a real threat elsewhere (thread-worker-messaging.ts's DNS-rebinding
120
+ // backstop applies even to already-trusted hosts). One fetchImpl closure
121
+ // covers every network call in this batch: the BotGuard/attestation fetches
122
+ // inside mintPoTokens() and the timedtext fetch below. Falls back to the
123
+ // plain global fetch for non-string/URL input (never seen in practice here)
124
+ // since the dispatcher pairing requires a URL string; see getSsrfSafeFetcher's
125
+ // own doc comment for why the fetch and its dispatcher must travel together.
126
+ const ssrfFetcher = await getSsrfSafeFetcher();
127
+ const fetchImpl: typeof fetch = ssrfFetcher
128
+ ? (input, init) => {
129
+ if (typeof input === 'string' || input instanceof URL) {
130
+ return ssrfFetcher.fetch(String(input), {
131
+ ...(init ?? {}),
132
+ dispatcher: ssrfFetcher.dispatcher,
133
+ }) as unknown as Promise<Response>;
134
+ }
135
+ return fetch(input, init);
136
+ }
137
+ : (input, init) => fetch(input, init);
113
138
 
114
139
  // 1) Throwaway session to obtain stable visitor data for the PoToken binding.
115
140
  let visitorData: string;
@@ -245,10 +270,36 @@ async function fetchOne(
245
270
  // response headers AND the body read are bounded and tear the socket
246
271
  // down on timeout/cancellation (withTimeout alone would not bound text()).
247
272
  const fetchSignal = createTimeoutSignal(timeoutMs, signal);
248
- const response = await fetchImpl(url, {
249
- signal: fetchSignal,
250
- headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
251
- });
273
+ // Manual redirect following, re-validating each hop against the same
274
+ // trusted-host allowlist as the first hop. `redirect: 'follow' (the
275
+ // default) would carry the PoToken-bearing request to whatever host a
276
+ // 3xx Location header names with no revalidation — assertTrustedTimedtextUrl
277
+ // above only ever checked base_url, the pre-redirect URL. fetchImpl
278
+ // carries the connect-time IP pin (see its construction above), so each
279
+ // hop is also protected against a resolver answering differently
280
+ // between this host check and the moment the socket actually opens.
281
+ const MAX_REDIRECTS = 5;
282
+ let currentUrl = url;
283
+ let response!: Response;
284
+ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
285
+ response = await fetchImpl(currentUrl, {
286
+ signal: fetchSignal,
287
+ redirect: 'manual',
288
+ headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
289
+ });
290
+ if (response.status >= 300 && response.status < 400) {
291
+ const location = response.headers.get('location');
292
+ if (!location) break; // No Location header — treat as final response
293
+ const resolved = new URL(location, currentUrl).href;
294
+ assertTrustedTimedtextUrl(resolved);
295
+ // Drain the redirect response before following: an unconsumed body
296
+ // pins its socket (undici keeps it open until read/cancelled or GC).
297
+ void response.body?.cancel()?.catch(() => { /* best-effort */ });
298
+ currentUrl = resolved;
299
+ continue;
300
+ }
301
+ break; // Non-redirect response
302
+ }
252
303
  // Surface the real HTTP status. Without this, a 403 (age-restricted) or
253
304
  // 404 (removed) returns HTML that parseJson3 yields '' for, which would
254
305
  // be misreported below as a bot-protection/PoToken failure.