@lincoln504/pi-research 1.4.1 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.mjs +271 -156
- package/dist/thread-worker.mjs +91 -59
- package/docs/CHANGELOG.md +76 -0
- package/docs/CONFIGURATION.md +1 -1
- package/package.json +1 -1
- package/skills/pi-research/SKILL.md +1 -1
- package/src/config.ts +10 -4
- package/src/core/planning-service.ts +55 -8
- package/src/infrastructure/browser/browser-task-scheduler.ts +16 -9
- package/src/infrastructure/browser/config.ts +40 -2
- package/src/infrastructure/browser/thread-worker-browser.ts +69 -48
- package/src/infrastructure/browser/thread-worker-messaging.ts +61 -4
- package/src/infrastructure/browser/thread-worker.mjs +91 -59
- package/src/infrastructure/browser/thread-worker.ts +13 -5
- package/src/infrastructure/embedding/embedding-server.ts +34 -0
- package/src/knowledge/embedder-init.ts +3 -2
- package/src/knowledge/embedder.ts +11 -3
- package/src/knowledge/index.ts +36 -10
- package/src/knowledge/store.ts +29 -5
- package/src/orchestration/deep-research-orchestrator.ts +3 -1
- package/src/orchestration/session-state.ts +16 -0
- package/src/sdk.ts +10 -0
- package/src/skill-install/skill-installer.ts +5 -1
- package/src/stackexchange/index.ts +3 -0
- package/src/stackexchange/rest-client.ts +7 -12
- package/src/tools/research-tool-definition.ts +38 -7
- package/src/tools/security.ts +13 -10
- package/src/utils/coverage-digest.ts +1 -1
- package/src/utils/host-config.ts +12 -1
- package/src/utils/log-utils.ts +10 -0
- package/src/web-research/browser-search.ts +11 -4
- package/src/web-research/web-scraper.ts +27 -16
- package/src/youtube/potoken.ts +8 -1
- package/src/youtube/transcript-client.ts +56 -5
package/src/sdk.ts
CHANGED
|
@@ -369,6 +369,16 @@ async function _doInit(options: ResearchSDKOptions = {}): Promise<void> {
|
|
|
369
369
|
* Public initialization. Guarantees the SDK is ready for use.
|
|
370
370
|
*/
|
|
371
371
|
export async function initResearchSDK(options: ResearchSDKOptions = {}): Promise<void> {
|
|
372
|
+
// A shutdown may be in flight: isInitialized only flips to false near the END
|
|
373
|
+
// of _doShutdown (after container disposal), so without this a caller racing a
|
|
374
|
+
// shutdown would hit the isInitialized branch below, log its "already
|
|
375
|
+
// initialized" warning, and return believing the SDK is ready — while the
|
|
376
|
+
// container it would use is mid-teardown or about to be nulled out from under
|
|
377
|
+
// it. Wait the shutdown out first so this call observes the real post-shutdown
|
|
378
|
+
// state, mirroring shutdownResearchSDK's symmetric wait on _initPromise.
|
|
379
|
+
if (_shutdownPromise) {
|
|
380
|
+
await _shutdownPromise.catch(() => { /* a failed shutdown must not block a fresh init */ });
|
|
381
|
+
}
|
|
372
382
|
if (isInitialized) {
|
|
373
383
|
logger.warn('[SDK] Research SDK already initialized. Call shutdownResearchSDK() first if you want to re-initialize with new options.');
|
|
374
384
|
return;
|
|
@@ -480,7 +480,11 @@ export function uninstallSkill(toolIds?: string[], opts: InstallOptions = {}): U
|
|
|
480
480
|
}
|
|
481
481
|
|
|
482
482
|
for (const { tool, path: skillPath } of targets.values()) {
|
|
483
|
-
|
|
483
|
+
// undefined = every manifest entry; an explicit (including empty) list means
|
|
484
|
+
// exactly those tools. `toolIds.length` used to gate this, so `[]` was
|
|
485
|
+
// indistinguishable from `undefined` and silently uninstalled everything —
|
|
486
|
+
// inconsistent with this file's other list params, where empty means none.
|
|
487
|
+
if (toolIds !== undefined && !toolIds.includes(tool)) {
|
|
484
488
|
const e = manifest.entries.find(x => x.path === skillPath);
|
|
485
489
|
if (e) keep.push(e);
|
|
486
490
|
continue;
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
} from './output/compact.ts';
|
|
28
28
|
import type { ExtensionContext, AgentToolResult } from '@earendil-works/pi-coding-agent';
|
|
29
29
|
import { logger } from '../logger.ts';
|
|
30
|
+
import { metrics } from '../utils/metrics.ts';
|
|
30
31
|
|
|
31
32
|
function notify(
|
|
32
33
|
ctx: ExtensionContext,
|
|
@@ -86,6 +87,7 @@ export async function stackexchangeCommand(options: {
|
|
|
86
87
|
try {
|
|
87
88
|
// Check quota before making requests
|
|
88
89
|
if (client.isQuotaExhausted()) {
|
|
90
|
+
metrics.increment('stackexchange_quota_exhausted_total', 1);
|
|
89
91
|
const quota = client.getQuotaInfo();
|
|
90
92
|
notify(
|
|
91
93
|
ctx,
|
|
@@ -103,6 +105,7 @@ export async function stackexchangeCommand(options: {
|
|
|
103
105
|
|
|
104
106
|
// Warn if quota is low
|
|
105
107
|
if (client.isQuotaLow()) {
|
|
108
|
+
metrics.increment('stackexchange_quota_low_total', 1);
|
|
106
109
|
const quota = client.getQuotaInfo();
|
|
107
110
|
notify(
|
|
108
111
|
ctx,
|
|
@@ -193,24 +193,19 @@ export class StackExchangeClient {
|
|
|
193
193
|
});
|
|
194
194
|
}
|
|
195
195
|
|
|
196
|
+
// Pure accessor — no metrics side effects. `stackexchangeCommand()` calls this
|
|
197
|
+
// repeatedly per invocation (pre-request gate checks, success footer, error
|
|
198
|
+
// footer) purely to read current state for display; incrementing counters here
|
|
199
|
+
// used to record 2-3x the real number of quota-low/exhausted events per call.
|
|
200
|
+
// The counters are incremented once, at the single gate check that actually
|
|
201
|
+
// observes each event (see stackexchange/index.ts).
|
|
196
202
|
getQuotaInfo(): { remaining: number; max: number; requestCount: number; lastBackoff: number | null } {
|
|
197
|
-
|
|
203
|
+
return {
|
|
198
204
|
remaining: this.quotaRemaining,
|
|
199
205
|
max: this.quotaMax,
|
|
200
206
|
requestCount: this.requestCount,
|
|
201
207
|
lastBackoff: this.lastBackoff,
|
|
202
208
|
};
|
|
203
|
-
|
|
204
|
-
// Track quota exhaustion events
|
|
205
|
-
if (this.isQuotaExhausted()) {
|
|
206
|
-
metrics.increment('stackexchange_quota_exhausted_total', 1);
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
if (this.isQuotaLow()) {
|
|
210
|
-
metrics.increment('stackexchange_quota_low_total', 1);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
return info;
|
|
214
209
|
}
|
|
215
210
|
|
|
216
211
|
isQuotaExhausted(): boolean {
|
|
@@ -17,6 +17,7 @@ import type {
|
|
|
17
17
|
import type { ExtendedExtensionContext } from '../types/extension-context.ts';
|
|
18
18
|
import type { ResearchDepth, CleanupContext } from '../types/index.ts';
|
|
19
19
|
import { Type, type Static } from 'typebox';
|
|
20
|
+
import { Value } from 'typebox/value';
|
|
20
21
|
import { validateConfig, getConfig, type ConfigInterface } from '../config.ts';
|
|
21
22
|
import { tryGetServiceContainerFromCtx, getService } from '../core/service-registry.ts';
|
|
22
23
|
|
|
@@ -28,7 +29,7 @@ import { redactSecrets } from '../utils/log-utils.ts';
|
|
|
28
29
|
import { exportResearchReport, appendExportMessage } from '../utils/research-export.ts';
|
|
29
30
|
import { validateAndSanitizeQuery } from '../utils/input-validation.ts';
|
|
30
31
|
import { validateInitialLinks, MAX_INITIAL_LINKS, MAX_INITIAL_LINK_CHARS } from '../utils/url-utils.ts';
|
|
31
|
-
import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages,
|
|
32
|
+
import { startResearchSession, endResearchSession, registerSessionAbort, clearSteeringMessages, getActiveResearchRunCount } from '../orchestration/session-state.ts';
|
|
32
33
|
import { createResearchTuiManager, hideWorkingIndicator } from '../tui/research-tui-manager.ts';
|
|
33
34
|
import { createCleanupFunction } from '../cleanup/research-cleanup.ts';
|
|
34
35
|
import { createResearchObserver, createObserverState, stopObserverWaveAnimation } from '../observers/research-observer-impl.ts';
|
|
@@ -118,6 +119,21 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
118
119
|
|
|
119
120
|
type ResearchParams = Static<typeof parameters>;
|
|
120
121
|
|
|
122
|
+
// Same field shapes as `parameters`, but WITHOUT the initialLinks count/length
|
|
123
|
+
// bounds — used only for the runtime execute() gate below. validateInitialLinks
|
|
124
|
+
// already enforces those same two bounds with a specific, friendly message
|
|
125
|
+
// ("at most N URLs...", "at most N characters..."); checking them again here
|
|
126
|
+
// would just produce the generic 'invalid_parameters' rejection first and mask
|
|
127
|
+
// that better message. Structural type-checking (initialLinks must still be an
|
|
128
|
+
// array of strings) is kept, since validateInitialLinks assumes that shape.
|
|
129
|
+
const executeShapeCheck = Type.Object({
|
|
130
|
+
query: Type.String(),
|
|
131
|
+
depth: Type.Optional(Type.Integer({ minimum: 1, maximum: 3 })),
|
|
132
|
+
model: Type.Optional(Type.String()),
|
|
133
|
+
excludeTools: Type.Optional(Type.Array(Type.String())),
|
|
134
|
+
initialLinks: Type.Optional(Type.Array(Type.String())),
|
|
135
|
+
});
|
|
136
|
+
|
|
121
137
|
return {
|
|
122
138
|
name: 'research',
|
|
123
139
|
label: 'Research',
|
|
@@ -169,6 +185,17 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
169
185
|
};
|
|
170
186
|
}
|
|
171
187
|
|
|
188
|
+
// Every other tool in this codebase re-validates params against its own
|
|
189
|
+
// schema before trusting it (the host does not enforce this) — this was the
|
|
190
|
+
// one exception, relying entirely on incidental downstream checks (a
|
|
191
|
+
// typeof on query, a warn-gate on excludeTools) for fields that skip both.
|
|
192
|
+
if (!Value.Check(executeShapeCheck, params)) {
|
|
193
|
+
return {
|
|
194
|
+
content: [{ type: 'text', text: 'Error: invalid parameters for research tool.' }],
|
|
195
|
+
details: { error: 'invalid_parameters' },
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
172
199
|
const { query, depth: rawDepth, model: modelId, excludeTools: paramExcludeTools, initialLinks } = params as ResearchParams;
|
|
173
200
|
|
|
174
201
|
// Enforce the same bounds the CLI applies to --initial-links: these links are
|
|
@@ -227,11 +254,13 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
227
254
|
}
|
|
228
255
|
// Steering messages are shared per pi-session. Only reset them when THIS is
|
|
229
256
|
// the last active run — otherwise a finishing run wipes a concurrent sibling
|
|
230
|
-
// run's queued/active steering mid-flight. This run's
|
|
231
|
-
//
|
|
232
|
-
// of <= 1 means no other run is active.
|
|
257
|
+
// run's queued/active steering mid-flight. This run's abort registration is
|
|
258
|
+
// still in place here (endResearchSession runs later in cleanup()), so a
|
|
259
|
+
// count of <= 1 means no other run is active. getActiveResearchRunCount
|
|
260
|
+
// (not getPiActivePanels) because headless runs never register a panel —
|
|
261
|
+
// a panel-based check here was always a no-op for headless callers.
|
|
233
262
|
try {
|
|
234
|
-
if (
|
|
263
|
+
if (getActiveResearchRunCount(piSessionId) <= 1) clearSteeringMessages(piSessionId);
|
|
235
264
|
} catch (e) { logger.error('[research] Clear steering messages failed:', e); }
|
|
236
265
|
if (detachAbortListener) {
|
|
237
266
|
try { detachAbortListener(); } catch { /* best-effort */ } finally { detachAbortListener = undefined; }
|
|
@@ -344,9 +373,11 @@ export function createResearchTool(iface?: ConfigInterface): ToolDefinition {
|
|
|
344
373
|
// module-level piSessions map keyed by the STABLE pi session id, so
|
|
345
374
|
// without this every headless run leaked them for the process
|
|
346
375
|
// lifetime (the TUI branch frees them via createCleanupFunction, the
|
|
347
|
-
// SDK in its own finally
|
|
376
|
+
// SDK in its own finally). Steering is NOT cleared here — teardownUi()
|
|
377
|
+
// already does that, gated on getActiveResearchRunCount so a finishing
|
|
378
|
+
// run doesn't wipe a concurrent sibling headless run's steering; an
|
|
379
|
+
// unconditional clear here bypassed that guard entirely.
|
|
348
380
|
endResearchSession(piSessionId, sessionResearchId);
|
|
349
|
-
clearSteeringMessages(piSessionId);
|
|
350
381
|
};
|
|
351
382
|
|
|
352
383
|
// Minimal stub — only stopObserverWaveAnimation touches panelState in the
|
package/src/tools/security.ts
CHANGED
|
@@ -98,16 +98,6 @@ export function createSecuritySearchTool(options: {
|
|
|
98
98
|
};
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
// Record call in tracker - returns false if limit reached
|
|
102
|
-
const allowed = options.tracker.recordCall('security_search');
|
|
103
|
-
if (!allowed) {
|
|
104
|
-
metrics.increment('tool_security_search_calls_total', 1, { status: 'rate_limited' });
|
|
105
|
-
return {
|
|
106
|
-
content: [{ type: 'text', text: options.tracker.getLimitMessage('security_search') }],
|
|
107
|
-
details: { blocked: true, reason: 'limit_reached' },
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
|
|
111
101
|
const p = params as SecuritySearchParamsType;
|
|
112
102
|
|
|
113
103
|
const terms = p.terms;
|
|
@@ -160,6 +150,19 @@ export function createSecuritySearchTool(options: {
|
|
|
160
150
|
}
|
|
161
151
|
const maxResults = p.maxResults ?? 20;
|
|
162
152
|
|
|
153
|
+
// Record call in tracker - returns false if limit reached. Deliberately AFTER
|
|
154
|
+
// every semantic-validation reject above (terms/databases/severity), same
|
|
155
|
+
// reasoning as the schema check: a rejected call did no work and must not
|
|
156
|
+
// permanently consume one of the researcher's shared MAX_GATHERING_CALLS slots.
|
|
157
|
+
const allowed = options.tracker.recordCall('security_search');
|
|
158
|
+
if (!allowed) {
|
|
159
|
+
metrics.increment('tool_security_search_calls_total', 1, { status: 'rate_limited' });
|
|
160
|
+
return {
|
|
161
|
+
content: [{ type: 'text', text: options.tracker.getLimitMessage('security_search') }],
|
|
162
|
+
details: { blocked: true, reason: 'limit_reached' },
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
163
166
|
metrics.increment('tool_security_search_terms_total', terms.length);
|
|
164
167
|
metrics.increment('tool_security_search_databases_total', databases.length);
|
|
165
168
|
|
|
@@ -109,7 +109,7 @@ export interface SplitReport {
|
|
|
109
109
|
* better than one that reaches it short.
|
|
110
110
|
*/
|
|
111
111
|
export function splitCoverageDigest(report: string): SplitReport {
|
|
112
|
-
if (!report) return { digest: '', body: report
|
|
112
|
+
if (!report) return { digest: '', body: report };
|
|
113
113
|
|
|
114
114
|
const headerMatch = delimiterRegex(COVERAGE_DIGEST_HEADER).exec(report);
|
|
115
115
|
if (!headerMatch) return { digest: '', body: report };
|
package/src/utils/host-config.ts
CHANGED
|
@@ -27,11 +27,22 @@ let _configDirName: string | undefined;
|
|
|
27
27
|
* Resolution order: explicit env override (`PI_RESEARCH_CONFIG_DIR_NAME`) →
|
|
28
28
|
* the host `@earendil-works/pi-coding-agent` package (only where it is actually
|
|
29
29
|
* resolvable, i.e. the main pi process) → the stable default `.pi`.
|
|
30
|
+
*
|
|
31
|
+
* The override is a directory NAME, not a path: it is joined onto
|
|
32
|
+
* `os.homedir()` by every caller (getGlobalConfigDir() in config.ts), which in
|
|
33
|
+
* turn is the base for config.env (API keys, chmod 0600), the state dir
|
|
34
|
+
* (locks, run semaphores), and the knowledge_db. A value containing a path
|
|
35
|
+
* separator or `..` traverses out of HOME entirely — scripts/cleanup.cjs
|
|
36
|
+
* already rejects exactly that in its own copy of this resolution (see its
|
|
37
|
+
* `rawDirName`/`dirName` guard); this must reject it the same way, since this
|
|
38
|
+
* is the copy every actual run resolves secrets and state through.
|
|
30
39
|
*/
|
|
31
40
|
export function getConfigDirName(): string {
|
|
32
41
|
if (_configDirName !== undefined) return _configDirName;
|
|
33
42
|
const override = process.env['PI_RESEARCH_CONFIG_DIR_NAME'];
|
|
34
|
-
if (override)
|
|
43
|
+
if (override && !override.includes('/') && !override.includes('\\') && override !== '.' && override !== '..') {
|
|
44
|
+
return (_configDirName = override);
|
|
45
|
+
}
|
|
35
46
|
try {
|
|
36
47
|
const require = createRequire(import.meta.url);
|
|
37
48
|
const host = require('@earendil-works/pi-coding-agent') as { CONFIG_DIR_NAME?: string };
|
package/src/utils/log-utils.ts
CHANGED
|
@@ -255,6 +255,16 @@ export function redactSecrets(message: string): string {
|
|
|
255
255
|
// never negative; it still prefers the larger rawLength when the
|
|
256
256
|
// REDACT_SCAN_LENGTH cut discarded raw content redaction then shrank.
|
|
257
257
|
out = `${out.slice(0, keep)}…[truncated ${Math.max(rawLength, out.length) - keep} chars]`;
|
|
258
|
+
} else if (rawLength > scanEnd) {
|
|
259
|
+
// The scanEnd cut (line 220) already dropped `rawLength - scanEnd` raw
|
|
260
|
+
// chars before redaction ever ran. If redaction then net-shrinks the
|
|
261
|
+
// scanned prefix enough that `out.length` no longer exceeds
|
|
262
|
+
// MAX_LOG_MESSAGE_LENGTH (e.g. many long hex/JWT runs each collapsed to
|
|
263
|
+
// the short literal '[REDACTED]'), the branch above never fires and this
|
|
264
|
+
// message would otherwise read as complete — silently hiding the tail of
|
|
265
|
+
// a large scraped page or provider error body with no indication
|
|
266
|
+
// anything was cut.
|
|
267
|
+
out = `${out}…[truncated ${rawLength - scanEnd} chars]`;
|
|
258
268
|
}
|
|
259
269
|
return out;
|
|
260
270
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import { runWorkerSearch } from '../infrastructure/browser/task-execution-service.ts';
|
|
8
|
-
import { getMaxWorkers } from '../infrastructure/browser/config.ts';
|
|
8
|
+
import { getMaxWorkers, COLD_START_ALLOWANCE_MS } from '../infrastructure/browser/config.ts';
|
|
9
9
|
import { isTaskTimeoutError } from '../infrastructure/browser/browser-error-utils.ts';
|
|
10
10
|
import { logger } from '../logger.ts';
|
|
11
11
|
import { safeUnref } from '../utils/safe-unref.ts';
|
|
@@ -82,8 +82,9 @@ export async function performSearch(
|
|
|
82
82
|
// cannot stall the entire Promise.all burst.
|
|
83
83
|
//
|
|
84
84
|
// The budget mirrors the scheduler's own task ceiling in
|
|
85
|
-
// BrowserTaskScheduler.runSearch (= SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS
|
|
86
|
-
// so this layer never preempts the worker's deadline with
|
|
85
|
+
// BrowserTaskScheduler.runSearch (= SEARCH_TIMEOUT_MS + BROWSER_TASK_TIMEOUT_MS +
|
|
86
|
+
// COLD_START_ALLOWANCE_MS) so this layer never preempts the worker's deadline with
|
|
87
|
+
// a shorter one.
|
|
87
88
|
//
|
|
88
89
|
// Both clocks now start at DISPATCH (see startQueryDeadline below), which is
|
|
89
90
|
// what makes mirroring the value correct. While this one started at enqueue,
|
|
@@ -95,7 +96,13 @@ export async function performSearch(
|
|
|
95
96
|
// query's worth of wait, and a burst four times the pool's reach exhausts any
|
|
96
97
|
// constant. Measuring execution instead of waiting is the fix; the sum
|
|
97
98
|
// survives as the budget because a search legitimately costs nav plus overhead.
|
|
98
|
-
|
|
99
|
+
//
|
|
100
|
+
// COLD_START_ALLOWANCE_MS is part of the sum for the same reason it is part of
|
|
101
|
+
// the scheduler's own ceiling: the first query dispatched to a freshly created or
|
|
102
|
+
// just-reset worker pays a real browser launch + context creation inline before
|
|
103
|
+
// any navigation starts, and this layer's budget must not fire before the
|
|
104
|
+
// scheduler's own (correct, larger) deadline gets a chance to.
|
|
105
|
+
const QUERY_TIMEOUT_MS = (config?.SEARCH_TIMEOUT_MS ?? 45_000) + (config?.BROWSER_TASK_TIMEOUT_MS ?? 10_000) + COLD_START_ALLOWANCE_MS;
|
|
99
106
|
|
|
100
107
|
// Deduplicated: resultMap and the failures map are both keyed by the query
|
|
101
108
|
// string, so two tasks for the same string race last-writer-wins — instance A
|
|
@@ -123,23 +123,34 @@ async function extractPdfToMarkdown(bytes: Uint8Array): Promise<string> {
|
|
|
123
123
|
|
|
124
124
|
try {
|
|
125
125
|
const doc = new WasmPdfDocument(bytes);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
126
|
+
// free() is native/WASM-side memory, not GC-reclaimed — it must run on
|
|
127
|
+
// every exit, including the case toMarkdownAll() AND the per-page fallback
|
|
128
|
+
// both throw (a malformed/encrypted PDF the codebase already treats as
|
|
129
|
+
// routine on the open web, so this path fires regularly). It used to sit
|
|
130
|
+
// after the fallback loop, reachable only when nothing below it threw —
|
|
131
|
+
// the one case that needed the fallback landing squarely on the one case
|
|
132
|
+
// that skipped the free(), leaking one WasmPdfDocument per occurrence in a
|
|
133
|
+
// browser-server process that stays warm across many research sessions.
|
|
130
134
|
try {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
+
const pageCount = doc.pageCount();
|
|
136
|
+
|
|
137
|
+
let markdown = `# PDF Document\n\n**Pages:** ${pageCount}\n\n`;
|
|
138
|
+
|
|
139
|
+
try {
|
|
140
|
+
markdown += doc.toMarkdownAll();
|
|
141
|
+
} catch {
|
|
142
|
+
for (let i = 0; i < pageCount; i++) {
|
|
143
|
+
markdown += `## Page ${i + 1}\n\n${doc.toMarkdown(i)}\n\n`;
|
|
144
|
+
}
|
|
135
145
|
}
|
|
146
|
+
|
|
147
|
+
const pdfExtractionDuration = Date.now() - pdfExtractionStart;
|
|
148
|
+
metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
|
|
149
|
+
metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
|
|
150
|
+
return markdown;
|
|
151
|
+
} finally {
|
|
152
|
+
doc.free();
|
|
136
153
|
}
|
|
137
|
-
|
|
138
|
-
doc.free();
|
|
139
|
-
const pdfExtractionDuration = Date.now() - pdfExtractionStart;
|
|
140
|
-
metrics.observe('scrape_pdf_conversion_ms', pdfExtractionDuration);
|
|
141
|
-
metrics.increment('scrape_pdf_conversions_total', 1, { status: 'success', pages: String(pageCount) });
|
|
142
|
-
return markdown;
|
|
143
154
|
} catch (e) {
|
|
144
155
|
const msg = e instanceof Error ? e.message : String(e);
|
|
145
156
|
// debug, not error: a malformed/encrypted/scanned-image-only PDF is an
|
|
@@ -300,7 +311,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
|
|
|
300
311
|
const contentLength = response.headers.get('content-length');
|
|
301
312
|
if (contentLength) {
|
|
302
313
|
const size = parseInt(contentLength, 10);
|
|
303
|
-
if (contentType.includes('application/pdf') ||
|
|
314
|
+
if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
|
|
304
315
|
if (size > MAX_PDF_SIZE) {
|
|
305
316
|
const sizeMB = Math.round(size / 1024 / 1024);
|
|
306
317
|
logger.warn(`[Scrapers] PDF too large (Content-Length: ${sizeMB}MB, max 100MB), skipping`);
|
|
@@ -319,7 +330,7 @@ async function scrapeWithFetch(url: string, signal?: AbortSignal): Promise<Scrap
|
|
|
319
330
|
}
|
|
320
331
|
}
|
|
321
332
|
|
|
322
|
-
if (contentType.includes('application/pdf') ||
|
|
333
|
+
if (contentType.includes('application/pdf') || currentUrl.toLowerCase().endsWith('.pdf')) {
|
|
323
334
|
let pdfBytes: Uint8Array;
|
|
324
335
|
try {
|
|
325
336
|
pdfBytes = await readBodyCapped(response, MAX_PDF_SIZE);
|
package/src/youtube/potoken.ts
CHANGED
|
@@ -44,6 +44,7 @@
|
|
|
44
44
|
import { logger } from '../logger.ts';
|
|
45
45
|
import { safeUnref } from '../utils/safe-unref.ts';
|
|
46
46
|
import { metrics } from '../utils/metrics.ts';
|
|
47
|
+
import { readJsonCapped } from '../utils/http-body.ts';
|
|
47
48
|
|
|
48
49
|
/**
|
|
49
50
|
* Well-known request key for the YouTube web PoToken program. This is a public
|
|
@@ -229,7 +230,13 @@ async function doMint(
|
|
|
229
230
|
body: JSON.stringify([requestKey, botguardResponse]),
|
|
230
231
|
signal: opts.signal,
|
|
231
232
|
});
|
|
232
|
-
|
|
233
|
+
if (!integrityResponse.ok) {
|
|
234
|
+
throw new Error(`HTTP ${integrityResponse.status} from attestation server`);
|
|
235
|
+
}
|
|
236
|
+
// The attestation response is a short JSON array; readJsonCapped bounds it the
|
|
237
|
+
// same way as every other externally-controlled body this codebase reads (the
|
|
238
|
+
// default 32MB cap is already generous for this endpoint).
|
|
239
|
+
const integrityJson = await readJsonCapped<unknown[]>(integrityResponse);
|
|
233
240
|
const integrityToken = integrityJson?.[0] as string | undefined;
|
|
234
241
|
if (!integrityToken) {
|
|
235
242
|
throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { withTimeout, retryWithBackoff, createTimeoutSignal } from '../web-research/retry-utils.ts';
|
|
21
|
+
import { getSsrfSafeFetcher } from '../web-research/scraper-utils.ts';
|
|
21
22
|
import { CircuitBreaker } from '../utils/circuit-breaker.ts';
|
|
22
23
|
import { logger } from '../logger.ts';
|
|
23
24
|
import { redactSecrets } from '../utils/log-utils.ts';
|
|
@@ -109,7 +110,31 @@ export async function fetchVideoTranscripts(
|
|
|
109
110
|
} catch {
|
|
110
111
|
// non-fatal: logging stays at default if the API ever changes
|
|
111
112
|
}
|
|
112
|
-
|
|
113
|
+
// Connect-time SSRF pin (see scraper-utils.ts's getSsrfSafeFetcher): closes the
|
|
114
|
+
// gap between assertTrustedTimedtextUrl's host check below and fetch's own
|
|
115
|
+
// re-resolution at connect time. The host allowlist only constrains WHICH
|
|
116
|
+
// hostname a redirect can name; it says nothing about what that hostname
|
|
117
|
+
// resolves to at the moment the socket opens, and this codebase already
|
|
118
|
+
// treats a resolver answering differently between validation and connect as
|
|
119
|
+
// a real threat elsewhere (thread-worker-messaging.ts's DNS-rebinding
|
|
120
|
+
// backstop applies even to already-trusted hosts). One fetchImpl closure
|
|
121
|
+
// covers every network call in this batch: the BotGuard/attestation fetches
|
|
122
|
+
// inside mintPoTokens() and the timedtext fetch below. Falls back to the
|
|
123
|
+
// plain global fetch for non-string/URL input (never seen in practice here)
|
|
124
|
+
// since the dispatcher pairing requires a URL string; see getSsrfSafeFetcher's
|
|
125
|
+
// own doc comment for why the fetch and its dispatcher must travel together.
|
|
126
|
+
const ssrfFetcher = await getSsrfSafeFetcher();
|
|
127
|
+
const fetchImpl: typeof fetch = ssrfFetcher
|
|
128
|
+
? (input, init) => {
|
|
129
|
+
if (typeof input === 'string' || input instanceof URL) {
|
|
130
|
+
return ssrfFetcher.fetch(String(input), {
|
|
131
|
+
...(init ?? {}),
|
|
132
|
+
dispatcher: ssrfFetcher.dispatcher,
|
|
133
|
+
}) as unknown as Promise<Response>;
|
|
134
|
+
}
|
|
135
|
+
return fetch(input, init);
|
|
136
|
+
}
|
|
137
|
+
: (input, init) => fetch(input, init);
|
|
113
138
|
|
|
114
139
|
// 1) Throwaway session to obtain stable visitor data for the PoToken binding.
|
|
115
140
|
let visitorData: string;
|
|
@@ -245,10 +270,36 @@ async function fetchOne(
|
|
|
245
270
|
// response headers AND the body read are bounded and tear the socket
|
|
246
271
|
// down on timeout/cancellation (withTimeout alone would not bound text()).
|
|
247
272
|
const fetchSignal = createTimeoutSignal(timeoutMs, signal);
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
273
|
+
// Manual redirect following, re-validating each hop against the same
|
|
274
|
+
// trusted-host allowlist as the first hop. `redirect: 'follow' (the
|
|
275
|
+
// default) would carry the PoToken-bearing request to whatever host a
|
|
276
|
+
// 3xx Location header names with no revalidation — assertTrustedTimedtextUrl
|
|
277
|
+
// above only ever checked base_url, the pre-redirect URL. fetchImpl
|
|
278
|
+
// carries the connect-time IP pin (see its construction above), so each
|
|
279
|
+
// hop is also protected against a resolver answering differently
|
|
280
|
+
// between this host check and the moment the socket actually opens.
|
|
281
|
+
const MAX_REDIRECTS = 5;
|
|
282
|
+
let currentUrl = url;
|
|
283
|
+
let response!: Response;
|
|
284
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
285
|
+
response = await fetchImpl(currentUrl, {
|
|
286
|
+
signal: fetchSignal,
|
|
287
|
+
redirect: 'manual',
|
|
288
|
+
headers: { 'User-Agent': 'Mozilla/5.0', 'Accept-Language': lang },
|
|
289
|
+
});
|
|
290
|
+
if (response.status >= 300 && response.status < 400) {
|
|
291
|
+
const location = response.headers.get('location');
|
|
292
|
+
if (!location) break; // No Location header — treat as final response
|
|
293
|
+
const resolved = new URL(location, currentUrl).href;
|
|
294
|
+
assertTrustedTimedtextUrl(resolved);
|
|
295
|
+
// Drain the redirect response before following: an unconsumed body
|
|
296
|
+
// pins its socket (undici keeps it open until read/cancelled or GC).
|
|
297
|
+
void response.body?.cancel()?.catch(() => { /* best-effort */ });
|
|
298
|
+
currentUrl = resolved;
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
break; // Non-redirect response
|
|
302
|
+
}
|
|
252
303
|
// Surface the real HTTP status. Without this, a 403 (age-restricted) or
|
|
253
304
|
// 404 (removed) returns HTML that parseJson3 yields '' for, which would
|
|
254
305
|
// be misreported below as a bot-protection/PoToken failure.
|