@steel-dev/atlas 0.1.0 → 0.1.2

package/README.md

@@ -14,6 +14,14 @@ const { report } = await atlas.research(
 );
 ```
 
+## Try it
+
+One-off query, no install:
+
+```bash
+ANTHROPIC_API_KEY=sk-ant-... npx @steel-dev/atlas "How do reasoning models differ from standard LLMs?"
+```
+
 ## Install
 
 ```bash
@@ -195,14 +203,6 @@ The real backstops are **price-independent** — each defaults to the effort row
 
 `result.stats.stopReason` folds those into one value — `"completed"`, `"finished"` (`run.finish()`), or a binding cap (`"budget"`, `"tokens"`, `"timeout"`). When several apply, the most proximate wins.
 
-## Safety
-
-Untrusted web content is quarantined (data, not instructions). Fetches pass SSRF guards hop-by-hop; `run_code` runs in a memory-capped V8 isolate with no network, filesystem, or host access. Direct fetch honors robots.txt.
-
-The isolate needs the optional `isolated-vm` dependency; without it, `run_code` is dropped from the toolset and the run proceeds without it — Atlas never falls back to an unsandboxed evaluator.
-
-The SSRF guard validates DNS at check time but can't pin the connection, so an attacker controlling DNS can defeat it via rebinding. Treat it as defense-in-depth — for hostile targets, run behind network-level egress controls that block private ranges.
-
 ## Dev
 
 ```bash

package/dist/cli.js

@@ -22867,6 +22867,21 @@ function extractionMetadataFromHtml(opts) {
     leadNote: "Fetched with direct HTML text extraction."
   });
 }
+function extractionMetadataFromYoutube(opts) {
+  return buildExtractionMetadata({
+    markdownChars: opts.markdownChars,
+    method: "youtube_transcript",
+    leadNote: "Fetched the YouTube caption track (timed text) for this video.",
+    ...opts.finalUrl ? { finalUrl: opts.finalUrl } : {},
+    ...opts.attempts ? { attempts: opts.attempts } : {},
+    ...opts.notes ? { notes: opts.notes } : {},
+    pageMetadata: {
+      ...opts.author ? { author: opts.author } : {},
+      ...opts.language ? { language: opts.language } : {},
+      ...opts.description ? { description: opts.description } : {}
+    }
+  });
+}
 function extractionMetadataFromExa(opts) {
   return buildExtractionMetadata({
     ...opts,
@@ -23105,6 +23120,232 @@ function quoteSource(document, start, end) {
   );
 }
 
+// src/youtube.ts
+var VIDEO_ID_RE = /^[A-Za-z0-9_-]{11}$/;
+var PATH_ID_RE = /^\/(?:shorts|embed|v|live)\/([A-Za-z0-9_-]{11})/;
+var INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
+var INNERTUBE_CLIENT = {
+  clientName: "IOS",
+  clientVersion: "20.10.4",
+  deviceModel: "iPhone16,2",
+  osName: "iPhone",
+  osVersion: "18.3.2.22D82",
+  hl: "en",
+  gl: "US"
+};
+var INNERTUBE_USER_AGENT = "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X)";
+var PLAYER_TIMEOUT_MS = 15e3;
+var TRANSCRIPT_TIMEOUT_MS = 15e3;
+var DESCRIPTION_CAP = 2e3;
+function youtubeVideoId(url) {
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return null;
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+  const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
+  if (host === "youtu.be") {
+    const id = parsed.pathname.split("/").filter(Boolean)[0] ?? "";
+    return VIDEO_ID_RE.test(id) ? id : null;
+  }
+  if (host === "youtube.com" || host.endsWith(".youtube.com")) {
+    if (parsed.pathname === "/watch") {
+      const v = parsed.searchParams.get("v") ?? "";
+      return VIDEO_ID_RE.test(v) ? v : null;
+    }
+    const match = PATH_ID_RE.exec(parsed.pathname);
+    if (match) return match[1] ?? null;
+  }
+  return null;
+}
+async function fetchYoutubeTranscript(url, options = {}) {
+  const videoId = youtubeVideoId(url);
+  if (!videoId) return null;
+  const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+  const player = await fetchPlayerResponse(videoId, fetchImpl, options.signal);
+  if (!player) return null;
+  const track = pickCaptionTrack(player, options.preferLang ?? "en");
+  if (!track?.baseUrl) return null;
+  const text2 = await fetchTranscriptText(
+    track.baseUrl,
+    fetchImpl,
+    options.signal
+  );
+  const trimmed = text2.trim();
+  if (!trimmed) return null;
+  const details = player.videoDetails ?? {};
+  const lengthRaw = Number(details.lengthSeconds);
+  return {
+    videoId,
+    title: stringOr(details.title, `YouTube video ${videoId}`),
+    author: nonEmpty(details.author),
+    languageCode: track.languageCode ?? "und",
+    kind: track.kind === "asr" ? "asr" : "manual",
+    text: trimmed,
+    segmentCount: trimmed.split("\n").filter(Boolean).length,
+    lengthSeconds: Number.isFinite(lengthRaw) && lengthRaw > 0 ? lengthRaw : null,
+    description: capDescription(nonEmpty(details.shortDescription))
+  };
+}
+function youtubeTranscriptToMarkdown(t) {
+  const header = [`# ${t.title}`];
+  if (t.author) header.push(`**Channel:** ${t.author}`);
+  const lang = t.kind === "asr" ? `${t.languageCode} (auto-generated)` : t.languageCode;
+  header.push(`**Transcript language:** ${lang}`);
+  if (t.lengthSeconds)
+    header.push(`**Length:** ${formatDuration(t.lengthSeconds)}`);
+  header.push("**Source:** YouTube caption track");
+  const parts = [header.join("\n"), "", "## Transcript", "", t.text];
+  if (t.description) {
+    parts.push("", "## Description", "", t.description);
+  }
+  return parts.join("\n");
+}
+async function fetchPlayerResponse(videoId, fetchImpl, signal) {
+  const resp = await fetchImpl(INNERTUBE_PLAYER_URL, {
+    method: "POST",
+    signal: withTimeout2(signal, PLAYER_TIMEOUT_MS),
+    headers: {
+      "content-type": "application/json",
+      "user-agent": INNERTUBE_USER_AGENT,
+      "accept-language": "en-US,en"
+    },
+    body: JSON.stringify({
+      context: { client: INNERTUBE_CLIENT },
+      videoId,
+      contentCheckOk: true,
+      racyCheckOk: true
+    })
+  });
+  if (!resp.ok) throw new Error(`youtube player HTTP ${resp.status}`);
+  try {
+    return JSON.parse(await resp.text());
+  } catch {
+    return null;
+  }
+}
+async function fetchTranscriptText(baseUrl, fetchImpl, signal) {
+  const direct = await requestTimedText(baseUrl, fetchImpl, signal);
+  const parsed = parseTranscriptBody(direct);
+  if (parsed) return parsed;
+  const json2 = await requestTimedText(
+    appendQuery(baseUrl, "fmt", "json3"),
+    fetchImpl,
+    signal
+  );
+  return parseTranscriptBody(json2);
+}
+async function requestTimedText(url, fetchImpl, signal) {
+  const resp = await fetchImpl(url, {
+    signal: withTimeout2(signal, TRANSCRIPT_TIMEOUT_MS),
+    headers: {
+      "user-agent": INNERTUBE_USER_AGENT,
+      "accept-language": "en-US,en"
+    }
+  });
+  if (!resp.ok) throw new Error(`youtube timedtext HTTP ${resp.status}`);
+  return resp.text();
+}
+function pickCaptionTrack(player, preferLang = "en") {
+  const tracks = player?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+  if (!Array.isArray(tracks) || tracks.length === 0) return null;
+  const lang = preferLang.toLowerCase();
+  const inLang = tracks.filter(
+    (t) => (t.languageCode ?? "").toLowerCase().startsWith(lang)
+  );
+  const manual = tracks.filter((t) => t.kind !== "asr");
+  return inLang.find((t) => t.kind !== "asr") ?? inLang[0] ?? manual[0] ?? tracks[0] ?? null;
+}
+function parseTranscriptBody(raw) {
+  const trimmed = raw.trim();
+  if (!trimmed) return "";
+  if (trimmed.startsWith("{")) return parseTimedTextJson(trimmed);
+  const fromText = parseTagLines(trimmed, "text");
+  if (fromText) return fromText;
+  return parseTagLines(trimmed, "p");
+}
+function parseTagLines(xml, tag) {
+  const re2 = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "g");
+  const lines = [];
+  let match;
+  while ((match = re2.exec(xml)) !== null) {
+    const inner = (match[1] ?? "").replace(/<[^>]+>/g, "");
+    const decoded = decodeEntities(inner).replace(/\s+/g, " ").trim();
+    if (decoded) lines.push(decoded);
+  }
+  return lines.join("\n");
+}
+function parseTimedTextJson(raw) {
+  let data;
+  try {
+    data = JSON.parse(raw);
+  } catch {
+    return "";
+  }
+  const lines = [];
+  for (const event of data.events ?? []) {
+    const text2 = (event.segs ?? []).map((seg) => seg.utf8 ?? "").join("").replace(/\s+/g, " ").trim();
+    if (text2) lines.push(text2);
+  }
+  return lines.join("\n");
+}
+function decodeEntities(s) {
+  const once = decodeEntitiesOnce(s);
+  if (once === s || !once.includes("&")) return once;
+  return decodeEntitiesOnce(once);
+}
+function decodeEntitiesOnce(s) {
+  return s.replace(
+    /&#x([0-9a-fA-F]+);/g,
+    (_, hex) => fromCodePoint(Number.parseInt(hex, 16))
+  ).replace(/&#(\d+);/g, (_, dec) => fromCodePoint(Number.parseInt(dec, 10))).replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&");
+}
+function fromCodePoint(cp) {
+  if (!Number.isFinite(cp) || cp < 0 || cp > 1114111) return "";
+  try {
+    return String.fromCodePoint(cp);
+  } catch {
+    return "";
+  }
+}
+function appendQuery(url, key, value) {
+  try {
+    const parsed = new URL(url);
+    parsed.searchParams.set(key, value);
+    return parsed.toString();
+  } catch {
+    const sep = url.includes("?") ? "&" : "?";
+    return `${url}${sep}${key}=${value}`;
+  }
+}
+function withTimeout2(signal, ms) {
+  const timeout = AbortSignal.timeout(ms);
+  return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function formatDuration(seconds) {
+  const total = Math.floor(seconds);
+  const h = Math.floor(total / 3600);
+  const m = Math.floor(total % 3600 / 60);
+  const s = total % 60;
+  const mm = String(m).padStart(h > 0 ? 2 : 1, "0");
+  const ss = String(s).padStart(2, "0");
+  return h > 0 ? `${h}:${mm}:${ss}` : `${mm}:${ss}`;
+}
+function capDescription(value) {
+  if (!value) return null;
+  const trimmed = value.trim();
+  if (!trimmed) return null;
+  return trimmed.length > DESCRIPTION_CAP ? `${trimmed.slice(0, DESCRIPTION_CAP)}\u2026` : trimmed;
+}
+function stringOr(value, fallback) {
+  return typeof value === "string" && value.trim() ? value.trim() : fallback;
+}
+function nonEmpty(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
 // src/providers/fetch.ts
 var DIRECT_PDF_MAX_BYTES = 25 * 1024 * 1024;
 var DIRECT_HTML_MAX_BYTES = 5 * 1024 * 1024;
@@ -23294,6 +23535,10 @@ async function directFetch({ url, signal, guardRedirect: guardRedirect2, dispatc
       );
     }
   }
+  if (youtubeVideoId(url)) {
+    const transcript = await tryYoutubeTranscript(url, signal);
+    if (transcript) return transcript;
+  }
   let currentUrl = url;
   let response;
   for (let hop = 0; ; hop++) {
@@ -23445,6 +23690,42 @@ function extractHtml(data, contentType, finalUrl) {
     }
   };
 }
+var YOUTUBE_TRANSCRIPT_MIN_CHARS = 40;
+async function tryYoutubeTranscript(url, signal) {
+  let transcript;
+  try {
+    transcript = await fetchYoutubeTranscript(url, { signal });
+  } catch {
+    return null;
+  }
+  if (!transcript) return null;
+  const markdown = youtubeTranscriptToMarkdown(transcript).trim();
+  if (markdown.length < YOUTUBE_TRANSCRIPT_MIN_CHARS) return null;
+  const finalUrl = `https://www.youtube.com/watch?v=${transcript.videoId}`;
+  const attempt = {
+    method: "youtube_transcript",
+    ok: true,
+    note: `youtube_transcript: extracted ${markdown.length} text chars (${transcript.kind} ${transcript.languageCode}, ${transcript.segmentCount} segments)`
+  };
+  return {
+    ok: true,
+    attempt,
+    page: {
+      finalUrl,
+      title: transcript.title,
+      markdown,
+      renderedWith: "youtube_transcript",
+      metadata: extractionMetadataFromYoutube({
+        markdownChars: markdown.length,
+        finalUrl,
+        attempts: [attempt],
+        ...transcript.author ? { author: transcript.author } : {},
+        language: transcript.languageCode,
+        ...transcript.description ? { description: transcript.description } : {}
+      })
+    }
+  };
+}
 var PDF_PARSE_TIMEOUT_MS = 3e4;
 async function extractPdf(data, contentType, finalUrl, signal) {
   try {
@@ -32208,36 +32489,44 @@ Run one web search and list the ${limit} most relevant distinct result pages. Do
         maxOutputTokens: 1500,
         abortSignal: signal
       });
-      const snippets = snippetsByUrl(result.text);
+      const lines = parseResultLines(result.text);
+      const summaryByKey = /* @__PURE__ */ new Map();
+      for (const line of lines) {
+        summaryByKey.set(normalizeUrlForSource(line.url), line.summary);
+      }
       const seen = /* @__PURE__ */ new Set();
       const results = [];
+      const add = (url, title, snippet) => {
+        if (results.length >= limit || typeof url !== "string") return;
+        const key = normalizeUrlForSource(url);
+        if (seen.has(key)) return;
+        const parsed = toResult(results.length, url, title, snippet);
+        if (!parsed) return;
+        seen.add(key);
+        results.push(parsed);
+      };
       for (const source of result.sources) {
         if (source.sourceType !== "url") continue;
         const key = normalizeUrlForSource(source.url);
-        if (seen.has(key)) continue;
-        seen.add(key);
-        const parsed = toResult(
-          results.length,
-          source.url,
-          source.title,
-          snippets.get(key) ?? ""
-        );
-        if (parsed) results.push(parsed);
-        if (results.length >= limit) break;
+        add(source.url, source.title, summaryByKey.get(key) ?? "");
       }
+      for (const line of lines) add(line.url, void 0, line.summary);
       return results;
     }
   };
 }
-function snippetsByUrl(text2) {
-  const map = /* @__PURE__ */ new Map();
+function parseResultLines(text2) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
   for (const line of text2.split("\n")) {
     const match = /(https?:\/\/\S+?)[)\]>.,]*\s*::\s*(\S.*)/.exec(line);
     if (!match) continue;
     const key = normalizeUrlForSource(match[1]);
-    if (!map.has(key)) map.set(key, match[2].trim().slice(0, 500));
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({ url: match[1], summary: match[2].trim().slice(0, 500) });
   }
-  return map;
+  return out;
 }
 async function importProvider(pkg, load4) {
   try {

package/dist/providers/fetch.js

@@ -5,7 +5,8 @@ import { htmlToMarkdown } from "../html-extract.js";
 import { extractPdfText } from "../pdf-extract.js";
 import { createRobotsCache } from "../robots.js";
 import { guardRedirect as guardRedirectUrl } from "../safety.js";
-import { extractionMetadataFromExa, extractionMetadataFromHtml, extractionMetadataFromPdf, extractionMetadataFromScrape, extractionMetadataFromText, } from "../source-documents.js";
+import { extractionMetadataFromExa, extractionMetadataFromHtml, extractionMetadataFromPdf, extractionMetadataFromScrape, extractionMetadataFromText, extractionMetadataFromYoutube, } from "../source-documents.js";
+import { fetchYoutubeTranscript, youtubeTranscriptToMarkdown, youtubeVideoId, } from "../youtube.js";
 const DIRECT_PDF_MAX_BYTES = 25 * 1024 * 1024;
 const DIRECT_HTML_MAX_BYTES = 5 * 1024 * 1024;
 const DIRECT_FETCH_TIMEOUT_MS = 15_000;
@@ -197,6 +198,11 @@ async function directFetch({ url, signal, guardRedirect, dispatcher }, robots) {
             return failed("direct_http", `blocked_url: fetch of ${url} blocked: ${initial.reason}`, false);
         }
     }
+    if (youtubeVideoId(url)) {
+        const transcript = await tryYoutubeTranscript(url, signal);
+        if (transcript)
+            return transcript;
+    }
     let currentUrl = url;
     let response;
     for (let hop = 0;; hop++) {
@@ -317,6 +323,47 @@ function extractHtml(data, contentType, finalUrl) {
         },
     };
 }
+const YOUTUBE_TRANSCRIPT_MIN_CHARS = 40;
+async function tryYoutubeTranscript(url, signal) {
+    let transcript;
+    try {
+        transcript = await fetchYoutubeTranscript(url, { signal });
+    }
+    catch {
+        return null;
+    }
+    if (!transcript)
+        return null;
+    const markdown = youtubeTranscriptToMarkdown(transcript).trim();
+    if (markdown.length < YOUTUBE_TRANSCRIPT_MIN_CHARS)
+        return null;
+    const finalUrl = `https://www.youtube.com/watch?v=${transcript.videoId}`;
+    const attempt = {
+        method: "youtube_transcript",
+        ok: true,
+        note: `youtube_transcript: extracted ${markdown.length} text chars (${transcript.kind} ${transcript.languageCode}, ${transcript.segmentCount} segments)`,
+    };
+    return {
+        ok: true,
+        attempt,
+        page: {
+            finalUrl,
+            title: transcript.title,
+            markdown,
+            renderedWith: "youtube_transcript",
+            metadata: extractionMetadataFromYoutube({
+                markdownChars: markdown.length,
+                finalUrl,
+                attempts: [attempt],
+                ...(transcript.author ? { author: transcript.author } : {}),
+                language: transcript.languageCode,
+                ...(transcript.description
+                    ? { description: transcript.description }
+                    : {}),
+            }),
+        },
+    };
+}
 const PDF_PARSE_TIMEOUT_MS = 30_000;
 async function extractPdf(data, contentType, finalUrl, signal) {
     try {

package/dist/providers/search.js

@@ -264,37 +264,51 @@ export function nativeModelSearch(opts) {
                 maxOutputTokens: 1_500,
                 abortSignal: signal,
             });
-            const snippets = snippetsByUrl(result.text);
+            const lines = parseResultLines(result.text);
+            const summaryByKey = new Map();
+            for (const line of lines) {
+                summaryByKey.set(normalizeUrlForSource(line.url), line.summary);
+            }
             const seen = new Set();
             const results = [];
+            const add = (url, title, snippet) => {
+                if (results.length >= limit || typeof url !== "string")
+                    return;
+                const key = normalizeUrlForSource(url);
+                if (seen.has(key))
+                    return;
+                const parsed = toResult(results.length, url, title, snippet);
+                if (!parsed)
+                    return;
+                seen.add(key);
+                results.push(parsed);
+            };
             for (const source of result.sources) {
                 if (source.sourceType !== "url")
                     continue;
                 const key = normalizeUrlForSource(source.url);
-                if (seen.has(key))
-                    continue;
-                seen.add(key);
-                const parsed = toResult(results.length, source.url, source.title, snippets.get(key) ?? "");
-                if (parsed)
-                    results.push(parsed);
-                if (results.length >= limit)
-                    break;
+                add(source.url, source.title, summaryByKey.get(key) ?? "");
             }
+            for (const line of lines)
+                add(line.url, undefined, line.summary);
             return results;
         },
     };
 }
-function snippetsByUrl(text) {
-    const map = new Map();
+function parseResultLines(text) {
+    const out = [];
+    const seen = new Set();
     for (const line of text.split("\n")) {
         const match = /(https?:\/\/\S+?)[)\]>.,]*\s*::\s*(\S.*)/.exec(line);
         if (!match)
             continue;
         const key = normalizeUrlForSource(match[1]);
-        if (!map.has(key))
-            map.set(key, match[2].trim().slice(0, 500));
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({ url: match[1], summary: match[2].trim().slice(0, 500) });
     }
-    return map;
+    return out;
 }
 async function importProvider(pkg, load) {
     try {

package/dist/source-documents.d.ts

@@ -43,6 +43,15 @@ export declare function extractionMetadataFromHtml(opts: {
     discoveredLinks?: SourceDiscoveredLink[];
     pageMetadata?: HtmlPageMetadata;
 }): SourceExtractionMetadata;
+export declare function extractionMetadataFromYoutube(opts: {
+    markdownChars: number;
+    finalUrl?: string;
+    attempts?: SourceExtractionAttempt[];
+    author?: string;
+    language?: string;
+    description?: string;
+    notes?: string[];
+}): SourceExtractionMetadata;
 export declare function extractionMetadataFromExa(opts: {
     markdownChars: number;
     finalUrl?: string;

package/dist/source-documents.js

@@ -95,6 +95,21 @@ export function extractionMetadataFromHtml(opts) {
         leadNote: "Fetched with direct HTML text extraction.",
     });
 }
+export function extractionMetadataFromYoutube(opts) {
+    return buildExtractionMetadata({
+        markdownChars: opts.markdownChars,
+        method: "youtube_transcript",
+        leadNote: "Fetched the YouTube caption track (timed text) for this video.",
+        ...(opts.finalUrl ? { finalUrl: opts.finalUrl } : {}),
+        ...(opts.attempts ? { attempts: opts.attempts } : {}),
+        ...(opts.notes ? { notes: opts.notes } : {}),
+        pageMetadata: {
+            ...(opts.author ? { author: opts.author } : {}),
+            ...(opts.language ? { language: opts.language } : {}),
+            ...(opts.description ? { description: opts.description } : {}),
+        },
+    });
+}
 export function extractionMetadataFromExa(opts) {
     return buildExtractionMetadata({
         ...opts,

package/dist/youtube.d.ts

@@ -0,0 +1,44 @@
+type FetchImpl = (input: string, init?: {
+    method?: string;
+    signal?: AbortSignal;
+    headers?: Record<string, string>;
+    body?: string;
+}) => Promise<{
+    ok: boolean;
+    status: number;
+    text(): Promise<string>;
+}>;
+export interface YoutubeTranscriptOptions {
+    preferLang?: string;
+    fetchImpl?: FetchImpl;
+    signal?: AbortSignal | undefined;
+}
+export interface YoutubeTranscript {
+    videoId: string;
+    title: string;
+    author: string | null;
+    languageCode: string;
+    kind: "asr" | "manual";
+    text: string;
+    segmentCount: number;
+    lengthSeconds: number | null;
+    description: string | null;
+}
+interface CaptionTrack {
+    baseUrl?: string;
+    languageCode?: string;
+    kind?: string;
+    name?: {
+        simpleText?: string;
+        runs?: Array<{
+            text?: string;
+        }>;
+    };
+}
+export declare function youtubeVideoId(url: string): string | null;
+export declare function isYoutubeWatchUrl(url: string): boolean;
+export declare function fetchYoutubeTranscript(url: string, options?: YoutubeTranscriptOptions): Promise<YoutubeTranscript | null>;
+export declare function youtubeTranscriptToMarkdown(t: YoutubeTranscript): string;
+export declare function pickCaptionTrack(player: Record<string, any>, preferLang?: string): CaptionTrack | null;
+export declare function parseTranscriptBody(raw: string): string;
+export {};

package/dist/youtube.js

@@ -0,0 +1,257 @@
+const VIDEO_ID_RE = /^[A-Za-z0-9_-]{11}$/;
+const PATH_ID_RE = /^\/(?:shorts|embed|v|live)\/([A-Za-z0-9_-]{11})/;
+const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player?prettyPrint=false";
+const INNERTUBE_CLIENT = {
+    clientName: "IOS",
+    clientVersion: "20.10.4",
+    deviceModel: "iPhone16,2",
+    osName: "iPhone",
+    osVersion: "18.3.2.22D82",
+    hl: "en",
+    gl: "US",
+};
+const INNERTUBE_USER_AGENT = "com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X)";
+const PLAYER_TIMEOUT_MS = 15_000;
+const TRANSCRIPT_TIMEOUT_MS = 15_000;
+const DESCRIPTION_CAP = 2_000;
+export function youtubeVideoId(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
+        return null;
+    const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
+    if (host === "youtu.be") {
+        const id = parsed.pathname.split("/").filter(Boolean)[0] ?? "";
+        return VIDEO_ID_RE.test(id) ? id : null;
+    }
+    if (host === "youtube.com" || host.endsWith(".youtube.com")) {
+        if (parsed.pathname === "/watch") {
+            const v = parsed.searchParams.get("v") ?? "";
+            return VIDEO_ID_RE.test(v) ? v : null;
+        }
+        const match = PATH_ID_RE.exec(parsed.pathname);
+        if (match)
+            return match[1] ?? null;
+    }
+    return null;
+}
+export function isYoutubeWatchUrl(url) {
+    return youtubeVideoId(url) !== null;
+}
+export async function fetchYoutubeTranscript(url, options = {}) {
+    const videoId = youtubeVideoId(url);
+    if (!videoId)
+        return null;
+    const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+    const player = await fetchPlayerResponse(videoId, fetchImpl, options.signal);
+    if (!player)
+        return null;
+    const track = pickCaptionTrack(player, options.preferLang ?? "en");
+    if (!track?.baseUrl)
+        return null;
+    const text = await fetchTranscriptText(track.baseUrl, fetchImpl, options.signal);
+    const trimmed = text.trim();
+    if (!trimmed)
+        return null;
+    const details = (player.videoDetails ?? {});
+    const lengthRaw = Number(details.lengthSeconds);
+    return {
+        videoId,
+        title: stringOr(details.title, `YouTube video ${videoId}`),
+        author: nonEmpty(details.author),
+        languageCode: track.languageCode ?? "und",
+        kind: track.kind === "asr" ? "asr" : "manual",
+        text: trimmed,
+        segmentCount: trimmed.split("\n").filter(Boolean).length,
+        lengthSeconds: Number.isFinite(lengthRaw) && lengthRaw > 0 ? lengthRaw : null,
+        description: capDescription(nonEmpty(details.shortDescription)),
+    };
+}
+export function youtubeTranscriptToMarkdown(t) {
+    const header = [`# ${t.title}`];
+    if (t.author)
+        header.push(`**Channel:** ${t.author}`);
+    const lang = t.kind === "asr" ? `${t.languageCode} (auto-generated)` : t.languageCode;
+    header.push(`**Transcript language:** ${lang}`);
+    if (t.lengthSeconds)
+        header.push(`**Length:** ${formatDuration(t.lengthSeconds)}`);
+    header.push("**Source:** YouTube caption track");
+    const parts = [header.join("\n"), "", "## Transcript", "", t.text];
+    if (t.description) {
+        parts.push("", "## Description", "", t.description);
+    }
+    return parts.join("\n");
+}
+async function fetchPlayerResponse(videoId, fetchImpl, signal) {
+    const resp = await fetchImpl(INNERTUBE_PLAYER_URL, {
+        method: "POST",
+        signal: withTimeout(signal, PLAYER_TIMEOUT_MS),
+        headers: {
+            "content-type": "application/json",
+            "user-agent": INNERTUBE_USER_AGENT,
+            "accept-language": "en-US,en",
+        },
+        body: JSON.stringify({
+            context: { client: INNERTUBE_CLIENT },
+            videoId,
+            contentCheckOk: true,
+            racyCheckOk: true,
+        }),
+    });
+    if (!resp.ok)
+        throw new Error(`youtube player HTTP ${resp.status}`);
+    try {
+        return JSON.parse(await resp.text());
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchTranscriptText(baseUrl, fetchImpl, signal) {
+    const direct = await requestTimedText(baseUrl, fetchImpl, signal);
+    const parsed = parseTranscriptBody(direct);
+    if (parsed)
+        return parsed;
+    const json = await requestTimedText(appendQuery(baseUrl, "fmt", "json3"), fetchImpl, signal);
+    return parseTranscriptBody(json);
+}
+async function requestTimedText(url, fetchImpl, signal) {
+    const resp = await fetchImpl(url, {
+        signal: withTimeout(signal, TRANSCRIPT_TIMEOUT_MS),
+        headers: {
+            "user-agent": INNERTUBE_USER_AGENT,
+            "accept-language": "en-US,en",
+        },
+    });
+    if (!resp.ok)
+        throw new Error(`youtube timedtext HTTP ${resp.status}`);
+    return resp.text();
+}
+export function pickCaptionTrack(player, preferLang = "en") {
+    const tracks = player?.captions?.playerCaptionsTracklistRenderer?.captionTracks;
+    if (!Array.isArray(tracks) || tracks.length === 0)
+        return null;
+    const lang = preferLang.toLowerCase();
+    const inLang = tracks.filter((t) => (t.languageCode ?? "").toLowerCase().startsWith(lang));
+    const manual = tracks.filter((t) => t.kind !== "asr");
+    return (inLang.find((t) => t.kind !== "asr") ??
+        inLang[0] ??
+        manual[0] ??
+        tracks[0] ??
+        null);
+}
+export function parseTranscriptBody(raw) {
+    const trimmed = raw.trim();
+    if (!trimmed)
+        return "";
+    if (trimmed.startsWith("{"))
+        return parseTimedTextJson(trimmed);
+    const fromText = parseTagLines(trimmed, "text");
+    if (fromText)
+        return fromText;
+    return parseTagLines(trimmed, "p");
+}
+function parseTagLines(xml, tag) {
+    const re = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "g");
+    const lines = [];
+    let match;
+    while ((match = re.exec(xml)) !== null) {
+        const inner = (match[1] ?? "").replace(/<[^>]+>/g, "");
+        const decoded = decodeEntities(inner).replace(/\s+/g, " ").trim();
+        if (decoded)
+            lines.push(decoded);
+    }
+    return lines.join("\n");
+}
+function parseTimedTextJson(raw) {
+    let data;
+    try {
+        data = JSON.parse(raw);
+    }
+    catch {
+        return "";
+    }
+    const lines = [];
+    for (const event of data.events ?? []) {
+        const text = (event.segs ?? [])
+            .map((seg) => seg.utf8 ?? "")
+            .join("")
+            .replace(/\s+/g, " ")
+            .trim();
+        if (text)
+            lines.push(text);
+    }
+    return lines.join("\n");
+}
+function decodeEntities(s) {
+    const once = decodeEntitiesOnce(s);
+    if (once === s || !once.includes("&"))
+        return once;
+    return decodeEntitiesOnce(once);
+}
+function decodeEntitiesOnce(s) {
+    return s
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => fromCodePoint(Number.parseInt(hex, 16)))
+        .replace(/&#(\d+);/g, (_, dec) => fromCodePoint(Number.parseInt(dec, 10)))
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&nbsp;/g, " ")
+        .replace(/&amp;/g, "&");
+}
+function fromCodePoint(cp) {
+    if (!Number.isFinite(cp) || cp < 0 || cp > 0x10ffff)
+        return "";
+    try {
+        return String.fromCodePoint(cp);
+    }
+    catch {
+        return "";
+    }
+}
+function appendQuery(url, key, value) {
+    try {
+        const parsed = new URL(url);
+        parsed.searchParams.set(key, value);
+        return parsed.toString();
+    }
+    catch {
+        const sep = url.includes("?") ? "&" : "?";
+        return `${url}${sep}${key}=${value}`;
+    }
+}
+function withTimeout(signal, ms) {
+    const timeout = AbortSignal.timeout(ms);
+    return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function formatDuration(seconds) {
+    const total = Math.floor(seconds);
+    const h = Math.floor(total / 3600);
+    const m = Math.floor((total % 3600) / 60);
+    const s = total % 60;
+    const mm = String(m).padStart(h > 0 ? 2 : 1, "0");
+    const ss = String(s).padStart(2, "0");
+    return h > 0 ? `${h}:${mm}:${ss}` : `${mm}:${ss}`;
+}
+function capDescription(value) {
+    if (!value)
+        return null;
+    const trimmed = value.trim();
+    if (!trimmed)
+        return null;
+    return trimmed.length > DESCRIPTION_CAP
+        ? `${trimmed.slice(0, DESCRIPTION_CAP)}…`
+        : trimmed;
+}
+function stringOr(value, fallback) {
+    return typeof value === "string" && value.trim() ? value.trim() : fallback;
+}
+function nonEmpty(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : null;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steel-dev/atlas",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Deep research from your terminal or your code. Backed by Steel Browser and model providers.",
   "type": "module",
   "main": "./dist/index.js",