@nijaru/pi-search 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +165 -0
  3. package/THIRD_PARTY_NOTICES.md +31 -0
  4. package/package.json +69 -0
  5. package/src/anthropic.ts +350 -0
  6. package/src/anydoc-worker.mjs +35 -0
  7. package/src/anydoc.ts +161 -0
  8. package/src/brave-answers.ts +396 -0
  9. package/src/brave.ts +441 -0
  10. package/src/codex.ts +314 -0
  11. package/src/contracts.ts +644 -0
  12. package/src/direct-transport.ts +141 -0
  13. package/src/errors.ts +185 -0
  14. package/src/exa.ts +204 -0
  15. package/src/fetch-errors.ts +134 -0
  16. package/src/fetch-extractor-worker.mjs +85 -0
  17. package/src/fetch-tool.ts +166 -0
  18. package/src/fetcher.ts +451 -0
  19. package/src/fixtures/sample.csv +2 -0
  20. package/src/fixtures/sample.docx +0 -0
  21. package/src/fixtures/sample.epub +0 -0
  22. package/src/fixtures/sample.odt +0 -0
  23. package/src/fixtures/sample.pdf +32 -0
  24. package/src/fixtures/sample.pptx +0 -0
  25. package/src/fixtures/sample.rtf +1 -0
  26. package/src/fixtures/sample.xlsx +0 -0
  27. package/src/gemini.ts +298 -0
  28. package/src/grounding.ts +167 -0
  29. package/src/http.ts +78 -0
  30. package/src/index.ts +101 -0
  31. package/src/local-llama-compat.ts +134 -0
  32. package/src/meta.ts +137 -0
  33. package/src/model-selection.ts +134 -0
  34. package/src/openai.ts +850 -0
  35. package/src/parallel-responses.ts +260 -0
  36. package/src/parallel.ts +173 -0
  37. package/src/pdf.ts +153 -0
  38. package/src/provider-availability.ts +35 -0
  39. package/src/provider-http.ts +203 -0
  40. package/src/render-text.ts +8 -0
  41. package/src/research-tool.ts +487 -0
  42. package/src/router.ts +302 -0
  43. package/src/search-cleanup.ts +219 -0
  44. package/src/search-tool.ts +551 -0
  45. package/src/search.ts +420 -0
  46. package/src/ssrf.ts +302 -0
  47. package/src/url-rendering.ts +21 -0
  48. package/src/x.ts +188 -0
  49. package/src/xai.ts +254 -0
  50. package/src/youtube.ts +224 -0
package/src/xai.ts ADDED
@@ -0,0 +1,254 @@
1
+ import type {
2
+ Provider,
3
+ ProviderCapabilities,
4
+ ProviderContext,
5
+ ProviderModel,
6
+ ProviderProfile,
7
+ SearchOption,
8
+ SearchRequest,
9
+ SearchResponse,
10
+ SearchResult,
11
+ SearchWarning,
12
+ } from "./contracts";
13
+ import { createProviderError } from "./errors";
14
+ import { executeGroundedSearch } from "./grounding";
15
+ import { httpSource, objectValue, optionalString, type SearchHttpFetch } from "./provider-http";
16
+ import { modelAuthHeaders, type ModelExecution } from "./model-selection";
17
+ import { validateSearchRequest } from "./search";
18
+
19
+ export const XAI_RESPONSES_ENDPOINT = "https://api.x.ai/v1";
20
+ export const DEFAULT_XAI_RESPONSE_BYTES = 4 * 1024 * 1024;
21
+ const MAX_XAI_ANSWER_LENGTH = 8_000;
22
+
23
+ type XAITool = "web_search" | "x_search";
24
+
25
+ export interface XAIAdapterOptions {
26
+ readonly tool: XAITool;
27
+ readonly endpoint?: string;
28
+ readonly fetchImpl?: SearchHttpFetch;
29
+ readonly maxResponseBytes?: number;
30
+ }
31
+
32
+ const profiles: Record<XAITool, ProviderProfile> = {
33
+ web_search: { auth: "modelRegistry", costModel: "usage-based" },
34
+ x_search: { auth: "modelRegistry", costModel: "usage-based" },
35
+ };
36
+
37
+ function capabilities(tool: XAITool): ProviderCapabilities {
38
+ return tool === "x_search"
39
+ ? { semantic: true, freshness: true, social: true, dateFilter: true, handleFilter: true, mediaUnderstanding: true, nativeGrounding: true }
40
+ : { semantic: true, freshness: true, domainFilter: true, nativeGrounding: true };
41
+ }
42
+
43
+ export interface XAIRequestPlan {
44
+ readonly body: Record<string, unknown>;
45
+ readonly appliedOptions: readonly SearchOption[];
46
+ readonly warnings: readonly SearchWarning[];
47
+ }
48
+
49
+ export function buildXAIRequest(request: SearchRequest, tool: XAITool): XAIRequestPlan {
50
+ const normalized = validateSearchRequest(request);
51
+ if (tool === "x_search" && (normalized.domains?.include?.length || normalized.domains?.exclude?.length)) {
52
+ throw createProviderError({ provider: "xai-x", kind: "unsupported", message: "xAI X search does not accept web domain filters", retryable: false });
53
+ }
54
+ if (tool === "web_search" && (normalized.dateRange !== undefined || normalized.social !== undefined)) {
55
+ throw createProviderError({ provider: "xai", kind: "unsupported", message: "xAI web search does not accept X date, handle, or media constraints", retryable: false });
56
+ }
57
+ if (tool === "x_search" && normalized.social?.includeHandles !== undefined && normalized.social.excludeHandles !== undefined) {
58
+ throw createProviderError({ provider: "xai-x", kind: "unsupported", message: "xAI X search cannot combine included and excluded handles", retryable: false });
59
+ }
60
+ const warnings: SearchWarning[] = [];
61
+ if (normalized.mode === "keyword") warnings.push({ code: "unsupported-option", option: "mode", message: `xAI ${tool === "x_search" ? "X" : "web"} search is semantic and does not guarantee keyword-only ranking` });
62
+ if (normalized.mode === "fresh") warnings.push({ code: "unsupported-option", option: "mode", message: `xAI ${tool === "x_search" ? "X" : "web"} search can use current sources but does not guarantee a freshness-only ranking` });
63
+ const searchTool: Record<string, unknown> = { type: tool };
64
+ if (tool === "web_search") {
65
+ const filters = {
66
+ ...(normalized.domains?.include?.length ? { allowed_domains: [...normalized.domains.include] } : {}),
67
+ ...(normalized.domains?.exclude?.length ? { excluded_domains: [...normalized.domains.exclude] } : {}),
68
+ };
69
+ if (Object.keys(filters).length > 0) searchTool.filters = filters;
70
+ } else {
71
+ if (normalized.social?.includeHandles !== undefined) searchTool.allowed_x_handles = [...normalized.social.includeHandles];
72
+ if (normalized.social?.excludeHandles !== undefined) searchTool.excluded_x_handles = [...normalized.social.excludeHandles];
73
+ if (normalized.dateRange?.from !== undefined) searchTool.from_date = normalized.dateRange.from;
74
+ if (normalized.dateRange?.to !== undefined) searchTool.to_date = normalized.dateRange.to;
75
+ if (normalized.social?.understandImages === true) searchTool.enable_image_understanding = true;
76
+ if (normalized.social?.understandVideos === true) searchTool.enable_video_understanding = true;
77
+ }
78
+ return {
79
+ body: {
80
+ model: "",
81
+ input: [{ role: "user", content: normalized.mode === "fresh" ? `Prefer current sources. ${normalized.query}` : normalized.query }],
82
+ tools: [searchTool],
83
+ },
84
+ appliedOptions: [
85
+ "maxResults",
86
+ "mode",
87
+ ...(normalized.domains?.include?.length || normalized.domains?.exclude?.length ? ["domains" as const] : []),
88
+ ...(normalized.dateRange === undefined ? [] : ["dateRange" as const]),
89
+ ...(normalized.social === undefined ? [] : ["social" as const]),
90
+ ],
91
+ warnings,
92
+ };
93
+ }
94
+
95
+ function endpointFor(model: ProviderModel, override?: string): string {
96
+ const base = (override ?? (model.baseUrl.trim().length > 0 ? model.baseUrl : XAI_RESPONSES_ENDPOINT)).replace(/\/+$/, "");
97
+ return base.endsWith("/responses") ? base : `${base}/responses`;
98
+ }
99
+
100
+ function authHeaders(execution: ModelExecution, provider: "xai" | "xai-x"): Readonly<Record<string, string>> {
101
+ const headers = modelAuthHeaders(execution);
102
+ if (!headers.has("authorization")) throw createProviderError({ provider, kind: "auth", message: "xAI authentication returned no authorization header", retryable: false });
103
+ return Object.fromEntries(headers.entries());
104
+ }
105
+
106
+ function citationTitle(value: unknown): string | undefined {
107
+ const title = optionalString(value, 500);
108
+ // Responses API inline-citation labels are often numeric display indexes
109
+ // ("1", "2", ...), not useful source titles. Let the renderer fall back to
110
+ // the source domain instead of exposing those labels as titles.
111
+ return title !== undefined && !/^\d+$/.test(title.trim()) ? title : undefined;
112
+ }
113
+
114
+ function appendCitation(results: SearchResult[], seen: Set<string>, value: unknown, query: string, title?: string): boolean {
115
+ const raw = typeof value === "string" ? value : value !== null && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>).url : undefined;
116
+ const parsed = httpSource(raw, "xai");
117
+ if (parsed === undefined) return false;
118
+ if (seen.has(parsed.url)) return true;
119
+ seen.add(parsed.url);
120
+ results.push({ url: parsed.url, ...(title === undefined ? {} : { title }), domain: parsed.domain, provider: "xai", searchQuery: query });
121
+ return true;
122
+ }
123
+
124
+ function normalizeXAIResponse(payload: unknown, request: SearchRequest, tool: XAITool): SearchResponse {
125
+ const normalized = validateSearchRequest(request);
126
+ const root = objectValue(payload, "response", "xai");
127
+ const status = optionalString(root.status);
128
+ if (status !== "completed") {
129
+ throw createProviderError({ provider: tool === "x_search" ? "xai-x" : "xai", kind: status === undefined ? "malformed" : "http", message: status === undefined ? "xAI response has no terminal status" : `xAI response was ${status}`, retryable: status === "incomplete" || status === "in_progress" });
130
+ }
131
+ const results: SearchResult[] = [];
132
+ const answerParts: string[] = [];
133
+ const answerCitationUrls = new Set<string>();
134
+ const seen = new Set<string>();
135
+ let discarded = 0;
136
+ if (Array.isArray(root.citations)) {
137
+ for (const citation of root.citations) {
138
+ if (!appendCitation(results, seen, citation, normalized.query)) discarded += 1;
139
+ }
140
+ }
141
+ if (Array.isArray(root.output)) {
142
+ for (const outputValue of root.output) {
143
+ if (typeof outputValue !== "object" || outputValue === null || Array.isArray(outputValue)) continue;
144
+ const output = outputValue as Record<string, unknown>;
145
+ if (output.type === "web_search_call" || output.type === "x_search_call") {
146
+ const action = output.action !== null && typeof output.action === "object" && !Array.isArray(output.action) ? output.action as Record<string, unknown> : undefined;
147
+ const callStatus = optionalString(output.status) ?? optionalString(action?.status);
148
+ if (callStatus === undefined) {
149
+ throw createProviderError({ provider: tool === "x_search" ? "xai-x" : "xai", kind: "malformed", message: `xAI ${tool} call has no terminal status`, retryable: true });
150
+ }
151
+ if (callStatus !== "completed") {
152
+ throw createProviderError({ provider: tool === "x_search" ? "xai-x" : "xai", kind: "http", message: `xAI ${tool} call was ${callStatus}`, retryable: callStatus === "incomplete" || callStatus === "in_progress" });
153
+ }
154
+ }
155
+ const content = output.content;
156
+ if (!Array.isArray(content)) continue;
157
+ for (const partValue of content) {
158
+ if (typeof partValue === "object" && partValue !== null && !Array.isArray(partValue) && typeof (partValue as Record<string, unknown>).text === "string") answerParts.push((partValue as Record<string, unknown>).text as string);
159
+ if (typeof partValue !== "object" || partValue === null || Array.isArray(partValue)) continue;
160
+ const part = partValue as Record<string, unknown>;
161
+ if (!Array.isArray(part.annotations)) continue;
162
+ for (const annotationValue of part.annotations) {
163
+ if (typeof annotationValue !== "object" || annotationValue === null || Array.isArray(annotationValue)) {
164
+ discarded += 1;
165
+ continue;
166
+ }
167
+ const annotation = annotationValue as Record<string, unknown>;
168
+ const parsed = httpSource(annotation.url, "xai");
169
+ if (parsed === undefined || !appendCitation(results, seen, parsed.url, normalized.query, citationTitle(annotation.title))) {
170
+ discarded += 1;
171
+ } else {
172
+ answerCitationUrls.add(parsed.url);
173
+ }
174
+ }
175
+ }
176
+ }
177
+ }
178
+ if (results.length === 0) {
179
+ throw createProviderError({ provider: tool === "x_search" ? "xai-x" : "xai", kind: "malformed", message: "xAI returned no parseable citation URLs", retryable: false });
180
+ }
181
+ const provider = tool === "x_search" ? "xai-x" : "xai";
182
+ const warnings: SearchWarning[] = discarded > 0 ? [{ code: "partial-results", message: `xAI discarded ${discarded} malformed citation entr${discarded === 1 ? "y" : "ies"}` }] : [];
183
+ const normalizedResults = results.slice(0, normalized.maxResults ?? 10).map((result) => ({ ...result, provider }));
184
+ const answerText = answerParts.join(" ").replace(/\s+/g, " ").trim().slice(0, MAX_XAI_ANSWER_LENGTH);
185
+ const answerCitations = normalizedResults.filter((result) => answerCitationUrls.has(result.url)).map((result) => ({ url: result.url, ...(result.title === undefined ? {} : { title: result.title }) }));
186
+ const answer = normalized.answerMode !== "evidence" && answerText.length > 0 && answerCitations.length > 0
187
+ ? { text: answerText, contentTrust: "untrusted" as const, provider, citations: answerCitations }
188
+ : undefined;
189
+ const usage = root.usage;
190
+ const usageRecord = usage !== null && typeof usage === "object" && !Array.isArray(usage) ? usage as Record<string, unknown> : undefined;
191
+ const inputTokens = typeof usageRecord?.input_tokens === "number" && Number.isFinite(usageRecord.input_tokens) ? usageRecord.input_tokens : undefined;
192
+ const outputTokens = typeof usageRecord?.output_tokens === "number" && Number.isFinite(usageRecord.output_tokens) ? usageRecord.output_tokens : undefined;
193
+ const totalTokens = typeof usageRecord?.total_tokens === "number" && Number.isFinite(usageRecord.total_tokens) ? usageRecord.total_tokens : undefined;
194
+ const usageDetails = inputTokens === undefined && outputTokens === undefined && totalTokens === undefined
195
+ ? undefined
196
+ : {
197
+ ...(inputTokens === undefined ? {} : { inputTokens }),
198
+ ...(outputTokens === undefined ? {} : { outputTokens }),
199
+ ...(totalTokens === undefined ? {} : { totalTokens, billedUnits: totalTokens, billedUnit: "tokens" }),
200
+ };
201
+ return {
202
+ query: normalized.query,
203
+ results: normalizedResults,
204
+ ...(answer === undefined ? {} : { answer }),
205
+ provider,
206
+ appliedOptions: [],
207
+ warnings,
208
+ ...(optionalString(root.id, 500) === undefined ? {} : { requestId: optionalString(root.id, 500) }),
209
+ ...(usageDetails === undefined ? {} : { usage: usageDetails }),
210
+ };
211
+ }
212
+
213
+ export class XAIProvider implements Provider {
214
+ readonly id: "xai" | "xai-x";
215
+ readonly capabilities: ProviderCapabilities;
216
+ readonly profile: ProviderProfile;
217
+ private readonly tool: XAITool;
218
+ private readonly endpoint?: string;
219
+ private readonly fetchImpl: SearchHttpFetch;
220
+ private readonly maxResponseBytes: number;
221
+
222
+ constructor(options: XAIAdapterOptions) {
223
+ this.tool = options.tool;
224
+ this.id = options.tool === "x_search" ? "xai-x" : "xai";
225
+ this.capabilities = capabilities(options.tool);
226
+ this.profile = profiles[options.tool];
227
+ this.endpoint = options.endpoint;
228
+ this.fetchImpl = options.fetchImpl ?? (fetch as SearchHttpFetch);
229
+ this.maxResponseBytes = options.maxResponseBytes ?? DEFAULT_XAI_RESPONSE_BYTES;
230
+ }
231
+
232
+ async search(request: SearchRequest, signal: AbortSignal, context: ProviderContext): Promise<SearchResponse> {
233
+ const normalized = validateSearchRequest(request);
234
+ const plan = buildXAIRequest(normalized, this.tool);
235
+ return executeGroundedSearch({
236
+ provider: this.id,
237
+ modelProvider: "xai",
238
+ api: "openai-responses",
239
+ request: normalized,
240
+ signal,
241
+ context,
242
+ fetchImpl: this.fetchImpl,
243
+ maxResponseBytes: this.maxResponseBytes,
244
+ endpointFor: (model) => endpointFor(model, this.endpoint),
245
+ headersFor: (execution) => authHeaders(execution, this.id),
246
+ plan,
247
+ normalize: (payload, current) => normalizeXAIResponse(payload, current, this.tool),
248
+ });
249
+ }
250
+ }
251
+
252
+ export function createXAIProvider(options: XAIAdapterOptions): XAIProvider {
253
+ return new XAIProvider(options);
254
+ }
package/src/youtube.ts ADDED
@@ -0,0 +1,224 @@
1
+ import { spawn } from "node:child_process";
2
+ import { mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import { SafeFetchError } from "./fetch-errors";
6
+
7
+ export const DEFAULT_YOUTUBE_LANGUAGE = "en";
8
+ export const MAX_YOUTUBE_LANGUAGE_LENGTH = 32;
9
+ export const MAX_YOUTUBE_TRANSCRIPT_BYTES = 4 * 1024 * 1024;
10
+
11
+ export interface YouTubeUrl {
12
+ readonly videoId: string;
13
+ }
14
+
15
+ export interface YouTubeExtractorOptions {
16
+ readonly command?: string;
17
+ readonly language?: string;
18
+ readonly maxOutputBytes?: number;
19
+ readonly signal: AbortSignal;
20
+ readonly spawnImpl?: (command: string, args: readonly string[], options: Record<string, unknown>) => any;
21
+ }
22
+
23
+ export interface YouTubeTranscriptResult {
24
+ readonly text: string;
25
+ readonly bytesRead: number;
26
+ readonly videoId: string;
27
+ }
28
+
29
+ export function parseYouTubeUrl(value: string): YouTubeUrl | undefined {
30
+ try {
31
+ const url = new URL(value);
32
+ if (url.protocol !== "https:" || url.username.length > 0 || url.password.length > 0 || url.port.length > 0) return undefined;
33
+ const hostname = url.hostname.toLowerCase();
34
+ if (hostname === "youtu.be") {
35
+ const id = url.pathname.slice(1).split("/", 1)[0];
36
+ return validVideoId(id);
37
+ }
38
+ if (!["youtube.com", "www.youtube.com", "m.youtube.com", "youtube-nocookie.com", "www.youtube-nocookie.com"].includes(hostname)) return undefined;
39
+ const id = url.searchParams.get("v") ?? url.pathname.match(/^\/(?:shorts|live|embed|v)\/([^/?]+)/)?.[1];
40
+ return validVideoId(id ?? "");
41
+ } catch {
42
+ return undefined;
43
+ }
44
+ }
45
+
46
+ function validVideoId(value: string): YouTubeUrl | undefined {
47
+ return /^[A-Za-z0-9_-]{6,24}$/.test(value) ? { videoId: value } : undefined;
48
+ }
49
+
50
+ function extraction(message: string, cause?: unknown): SafeFetchError {
51
+ return new SafeFetchError({ kind: "extraction", message, cause });
52
+ }
53
+
54
+ function canceled(): SafeFetchError {
55
+ return new SafeFetchError({ kind: "canceled", message: "YouTube transcript extraction canceled" });
56
+ }
57
+
58
+ function boundedCommand(command: string, args: readonly string[], maxOutputBytes: number): { command: string; args: readonly string[]; options: Record<string, unknown> } {
59
+ if (process.platform === "win32") return { command, args, options: { stdio: ["ignore", "ignore", "pipe"] } };
60
+ const blocks = Math.max(1, Math.ceil(maxOutputBytes / 512));
61
+ return {
62
+ command: "/bin/sh",
63
+ args: ["-c", `ulimit -f ${blocks}; exec "$@"`, "pi-search-yt-dlp", command, ...args],
64
+ options: { stdio: ["ignore", "ignore", "pipe"], detached: true },
65
+ };
66
+ }
67
+
68
+ async function runYtDlp(
69
+ command: string,
70
+ args: readonly string[],
71
+ directory: string,
72
+ maxOutputBytes: number,
73
+ signal: AbortSignal,
74
+ spawnImpl: NonNullable<YouTubeExtractorOptions["spawnImpl"]>,
75
+ ): Promise<void> {
76
+ if (signal.aborted) throw canceled();
77
+ await new Promise<void>((resolve, reject) => {
78
+ let child: any;
79
+ let settled = false;
80
+ let terminationError: SafeFetchError | undefined;
81
+ let fallbackTimer: ReturnType<typeof setTimeout> | undefined;
82
+ let monitorTimer: ReturnType<typeof setInterval> | undefined;
83
+ const finish = (error?: unknown): void => {
84
+ if (settled) return;
85
+ settled = true;
86
+ signal.removeEventListener("abort", onAbort);
87
+ if (fallbackTimer !== undefined) clearTimeout(fallbackTimer);
88
+ if (monitorTimer !== undefined) clearInterval(monitorTimer);
89
+ if (error === undefined) resolve(); else reject(error);
90
+ };
91
+ const kill = (): void => {
92
+ try {
93
+ if (child?.pid && process.platform !== "win32") {
94
+ try { process.kill(-child.pid, "SIGKILL"); } catch { /* group may already be gone */ }
95
+ }
96
+ child?.kill?.("SIGKILL");
97
+ } catch { /* best effort */ }
98
+ };
99
+ const terminate = (error: SafeFetchError): void => {
100
+ if (settled || terminationError !== undefined) return;
101
+ terminationError = error;
102
+ kill();
103
+ fallbackTimer = setTimeout(() => finish(error), 1_000);
104
+ };
105
+ const onAbort = (): void => terminate(canceled());
106
+ try {
107
+ const bounded = boundedCommand(command, args, maxOutputBytes);
108
+ child = spawnImpl(bounded.command, bounded.args, bounded.options);
109
+ signal.addEventListener("abort", onAbort, { once: true });
110
+ let diagnostics = "";
111
+ child.stderr?.on("data", (chunk: unknown) => {
112
+ if (diagnostics.length < 8_192) diagnostics += String(chunk).slice(0, 8_192 - diagnostics.length);
113
+ });
114
+ monitorTimer = setInterval(() => {
115
+ void (async () => {
116
+ if (settled || terminationError !== undefined) return;
117
+ try {
118
+ const files = await readdir(directory);
119
+ let total = 0;
120
+ for (const file of files) total += (await stat(join(directory, file))).size;
121
+ if (total > maxOutputBytes) terminate(new SafeFetchError({ kind: "responseTooLarge", message: "YouTube captions exceed the configured extraction limit" }));
122
+ } catch {
123
+ // The process may create the directory entries between polls.
124
+ }
125
+ })();
126
+ }, 50);
127
+ child.once("error", (error: unknown) => finish(extraction(`yt-dlp could not start: ${error instanceof Error ? error.message : String(error)}`, error)));
128
+ child.once("close", (code: number | null) => {
129
+ if (settled) return;
130
+ if (terminationError !== undefined) return finish(terminationError);
131
+ if (signal.aborted) return finish(canceled());
132
+ if (code !== 0) return finish(extraction(diagnostics.trim().slice(0, 500) || `yt-dlp exited with ${code}`));
133
+ finish();
134
+ });
135
+ if (signal.aborted) onAbort();
136
+ } catch (error) {
137
+ finish(extraction(`yt-dlp could not start: ${error instanceof Error ? error.message : String(error)}`, error));
138
+ }
139
+ });
140
+ }
141
+
142
+ /** Strip VTT cue metadata and adjacent duplicate caption lines. */
143
+ export function parseVtt(value: string): string {
144
+ const lines = value.replace(/^\uFEFF?WEBVTT[^\n]*\n?/i, "").split(/\r?\n/);
145
+ const output: string[] = [];
146
+ let inBlock = false;
147
+ let inCue = false;
148
+ for (const raw of lines) {
149
+ const line = raw.trim();
150
+ if (line === "") {
151
+ inBlock = false;
152
+ inCue = false;
153
+ continue;
154
+ }
155
+ if (!inCue && (line === "NOTE" || line.startsWith("NOTE ") || line === "STYLE" || line === "REGION")) {
156
+ inBlock = true;
157
+ inCue = false;
158
+ continue;
159
+ }
160
+ if (inBlock) continue;
161
+ if (isVttCueTimingLine(line)) {
162
+ inCue = true;
163
+ continue;
164
+ }
165
+ if (!inCue) continue;
166
+ const clean = line.replace(/<\/?[^>]+>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").trim();
167
+ if (clean.length > 0 && output.at(-1) !== clean) output.push(clean);
168
+ }
169
+ return output.join("\n");
170
+ }
171
+
172
+ function isVttCueTimingLine(line: string): boolean {
173
+ return /^\d{2}:\d{2}(?::\d{2})?\.\d{3}\s+-->\s+\d{2}:\d{2}(?::\d{2})?\.\d{3}(?:\s|$)/.test(line);
174
+ }
175
+
176
+ export async function extractYouTubeTranscript(url: string, options: YouTubeExtractorOptions): Promise<YouTubeTranscriptResult> {
177
+ const parsed = parseYouTubeUrl(url);
178
+ if (parsed === undefined) throw new SafeFetchError({ kind: "invalidRequest", message: "URL is not a supported YouTube video URL" });
179
+ if (options.signal.aborted) throw canceled();
180
+ if (process.platform === "win32") throw extraction("Bounded YouTube caption extraction requires a POSIX process limit on this runtime");
181
+ const language = options.language ?? DEFAULT_YOUTUBE_LANGUAGE;
182
+ if (!/^[A-Za-z0-9,._*-]{1,32}$/.test(language) || language.split(",").length > 3) throw new SafeFetchError({ kind: "invalidRequest", message: "captionLanguage is invalid" });
183
+ const maxOutputBytes = options.maxOutputBytes ?? MAX_YOUTUBE_TRANSCRIPT_BYTES;
184
+ if (!Number.isInteger(maxOutputBytes) || maxOutputBytes < 1 || maxOutputBytes > MAX_YOUTUBE_TRANSCRIPT_BYTES) {
185
+ throw new SafeFetchError({ kind: "invalidRequest", message: "YouTube transcript limit is outside the supported bound" });
186
+ }
187
+ const directory = await mkdtemp(join(tmpdir(), "pi-search-youtube-"));
188
+ try {
189
+ const outputTemplate = join(directory, "captions.%(ext)s");
190
+ const canonicalUrl = `https://www.youtube.com/watch?v=${parsed.videoId}`;
191
+ await runYtDlp(options.command ?? "yt-dlp", [
192
+ "--ignore-config",
193
+ "--no-playlist",
194
+ "--skip-download",
195
+ "--write-subs",
196
+ "--write-auto-subs",
197
+ "--sub-langs", language,
198
+ "--sub-format", "vtt",
199
+ "--output", outputTemplate,
200
+ "--no-warnings",
201
+ canonicalUrl,
202
+ ], directory, maxOutputBytes, options.signal, options.spawnImpl ?? (spawn as unknown as NonNullable<YouTubeExtractorOptions["spawnImpl"]>));
203
+ if (options.signal.aborted) throw canceled();
204
+ const files = (await readdir(directory)).filter((file) => file.endsWith(".vtt")).sort();
205
+ if (files.length === 0) throw extraction("No YouTube captions were available for the requested language");
206
+ if (files.length > 4) throw new SafeFetchError({ kind: "responseTooLarge", message: "Too many YouTube caption tracks were returned" });
207
+ let bytesRead = 0;
208
+ const parts: string[] = [];
209
+ for (const file of files) {
210
+ if (options.signal.aborted) throw canceled();
211
+ const size = (await stat(join(directory, file))).size;
212
+ if (size > maxOutputBytes - bytesRead) throw new SafeFetchError({ kind: "responseTooLarge", message: "YouTube captions exceed the configured extraction limit" });
213
+ const bytes = await readFile(join(directory, file));
214
+ bytesRead += bytes.byteLength;
215
+ if (bytesRead > maxOutputBytes) throw new SafeFetchError({ kind: "responseTooLarge", message: "YouTube captions exceed the configured extraction limit" });
216
+ parts.push(parseVtt(bytes.toString("utf8")));
217
+ }
218
+ const text = parts.filter((part) => part.length > 0).join("\n").trim();
219
+ if (text.length === 0) throw extraction("YouTube captions contained no extractable text");
220
+ return { text, bytesRead, videoId: parsed.videoId };
221
+ } finally {
222
+ await rm(directory, { recursive: true, force: true }).catch(() => undefined);
223
+ }
224
+ }