@mannyflussnpm/adapter-youtube-history 1.0.0

package/fixtures/watch-history.json

@@ -0,0 +1,41 @@
+[
+  {
+    "id": "wl-001",
+    "snippet": {
+      "publishedAt": "2023-11-14T10:00:00.000Z",
+      "title": "How to Build a Data Vault in TypeScript",
+      "channelId": "UCtechwithtim",
+      "channelTitle": "Tech With Tim",
+      "resourceId": {
+        "kind": "youtube#video",
+        "videoId": "dQw4w9WgXcQ"
+      }
+    }
+  },
+  {
+    "id": "wl-002",
+    "snippet": {
+      "publishedAt": "2023-11-14T14:30:00.000Z",
+      "title": "The Future of Local-First Software",
+      "channelId": "UCfireship",
+      "channelTitle": "Fireship",
+      "resourceId": {
+        "kind": "youtube#video",
+        "videoId": "jNQXAC9IVRw"
+      }
+    }
+  },
+  {
+    "id": "wl-003",
+    "snippet": {
+      "publishedAt": "2023-11-15T09:00:00.000Z",
+      "title": "Understanding Zero-Knowledge Proofs",
+      "channelId": "UC3blue1brown",
+      "channelTitle": "3Blue1Brown",
+      "resourceId": {
+        "kind": "youtube#video",
+        "videoId": "abc123def45"
+      }
+    }
+  }
+]

package/index.ts

@@ -0,0 +1,644 @@
+/**
+ * youtube-history — live YouTube watch history via the internal API.
+ *
+ * Uses YouTube's `youtubei/v1/browse` endpoint (the same one that powers
+ * youtube.com/feed/history) to pull the user's full watch history.
+ *
+ * Emits `video_watched` envelopes with title, channel, videoId, URL, and
+ * watch date (derived from section headers like "Monday", "Jun 18").
+ *
+ * **Transcript fetching (opt-in):** set `YOUTUBE_HISTORY_FETCH_TRANSCRIPTS=1`
+ * to also pull captions via YouTube's public timedtext API (no auth needed).
+ * Transcripts are stored as content-addressed assets.
+ *
+ * Cursor: JSON `{ continuation, newestVideoId }`. During backfill, the
+ * continuation token drives pagination. Once backfill is complete, only
+ * videos newer than `newestVideoId` are kept.
+ *
+ * ## Setup (one-time)
+ *
+ * Export your YouTube cookies using a browser extension like
+ * "Get cookies.txt LOCALLY" (Chrome/Firefox). Save the Netscape-format file to:
+ *   ~/.personal-data-vault/youtube-cookies.txt
+ *
+ * Alternatively, set the raw Cookie header string as:
+ *   export YOUTUBE_COOKIE='...'
+ *
+ * Cookies expire after a few weeks/months — re-export when pulls start
+ * returning 0 events.
+ *
+ * Falls back to synthetic fixture when no cookies are available (conformance).
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { createHash } from "node:crypto";
+import {
+  ENVELOPE_VERSION,
+  type Adapter,
+  type AdapterContext,
+  type Cursor,
+  type Envelope,
+  type PullResult,
+} from "@mannyflussnpm/spec";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+const FIXTURE_PATH = fileURLToPath(
+  new URL("./fixtures/watch-history.json", import.meta.url),
+);
+
+const COOKIES_PATH = join(
+  homedir(),
+  ".personal-data-vault",
+  "youtube-cookies.txt",
+);
+
+const BROWSE_URL = "https://www.youtube.com/youtubei/v1/browse";
+const MAX_PER_PULL = 500;
+
+const FETCH_TRANSCRIPTS =
+  process.env.YOUTUBE_HISTORY_FETCH_TRANSCRIPTS === "1";
+
+/** YouTube's web client API key (public, embedded in youtube.com). */
+const YT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+
+// ---------------------------------------------------------------------------
+// Cookie loading & auth
+// ---------------------------------------------------------------------------
+
+interface CookieJar {
+  header: string;
+  sapisid: string;
+}
+
+function loadCookies(): CookieJar | null {
+  // 1. Raw cookie header via env var
+  const envCookie = process.env.YOUTUBE_COOKIE;
+  if (envCookie) {
+    const sapisid = extractSapisidFromHeader(envCookie);
+    return { header: envCookie, sapisid };
+  }
+
+  // 2. Netscape-format file
+  if (!existsSync(COOKIES_PATH)) return null;
+
+  const raw = readFileSync(COOKIES_PATH, "utf8");
+  const cookies: string[] = [];
+  let sapisid = "";
+
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+
+    const parts = trimmed.split("\t");
+    if (parts.length < 7) continue;
+
+    const [domain, , , , , name, value] = parts;
+    if (!domain.includes("youtube.com")) continue;
+
+    cookies.push(`${name}=${value}`);
+    if (name === "SAPISID") sapisid = value;
+  }
+
+  return cookies.length > 0
+    ? { header: cookies.join("; "), sapisid }
+    : null;
+}
+
+function extractSapisidFromHeader(header: string): string {
+  const match = header.match(/SAPISID=([^;]+)/);
+  return match ? match[1] : "";
+}
+
+function makeAuthHeader(sapisid: string): string {
+  const hash = createHash("sha1")
+    .update(sapisid + " https://www.youtube.com")
+    .digest("hex");
+  return `SAPISIDHASH ${hash}`;
+}
+
+// ---------------------------------------------------------------------------
+// Cursor helpers
+// ---------------------------------------------------------------------------
+
+interface YoutubeHistoryCursor {
+  continuation: string | null;
+  newestVideoId: string | null;
+}
+
+function parseCursor(cursor: Cursor): YoutubeHistoryCursor {
+  if (!cursor) return { continuation: null, newestVideoId: null };
+  try {
+    return JSON.parse(cursor) as YoutubeHistoryCursor;
+  } catch {
+    return { continuation: null, newestVideoId: cursor };
+  }
+}
+
+function encodeCursor(c: YoutubeHistoryCursor): string {
+  return JSON.stringify(c);
+}
+
+// ---------------------------------------------------------------------------
+// API call
+// ---------------------------------------------------------------------------
+
+async function browseRequest(
+  body: Record<string, unknown>,
+  jar: CookieJar,
+): Promise<Record<string, unknown> | null> {
+  try {
+    const res = await fetch(`${BROWSE_URL}?key=${YT_API_KEY}`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-YouTube-Client-Name": "1",
+        "X-YouTube-Client-Version": "2.20250625.00.00",
+        Cookie: jar.header,
+        Authorization: makeAuthHeader(jar.sapisid),
+        Origin: "https://www.youtube.com",
+        Referer: "https://www.youtube.com/feed/history",
+      },
+      body: JSON.stringify(body),
+    });
+
+    if (!res.ok) {
+      console.error(
+        `[youtube-history] API returned ${res.status} ${res.statusText}`,
+      );
+      return null;
+    }
+
+    return (await res.json()) as Record<string, unknown>;
+  } catch (err) {
+    console.error(`[youtube-history] API request failed:`, err);
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Response parsing
+// ---------------------------------------------------------------------------
+
+interface ParsedEntry {
+  videoId: string;
+  title: string;
+  channel: string;
+  /** Date string from the section header, e.g. "Monday", "Jun 18", "Apr 15". */
+  dateLabel: string;
+}
+
+/**
+ * Parse the watch history browse response into flat video entries.
+ * Handles both the initial page structure and continuation responses.
+ */
+function parseHistoryResponse(
+  data: Record<string, unknown>,
+): ParsedEntry[] {
+  const entries: ParsedEntry[] = [];
+
+  try {
+    // Try initial page structure first
+    const tabs =
+      (data as any).contents?.twoColumnBrowseResultsRenderer?.tabs;
+    if (tabs?.[0]) {
+      const slr = tabs[0].tabRenderer?.content?.sectionListRenderer;
+      if (slr?.contents) {
+        parseSections(slr.contents, entries);
+      }
+    }
+
+    // Try continuation response structure
+    const actions = (data as any).onResponseReceivedActions;
+    if (actions?.[0]?.appendContinuationItemsAction?.continuationItems) {
+      parseSections(
+        actions[0].appendContinuationItemsAction.continuationItems,
+        entries,
+      );
+    }
+  } catch {
+    // Response shape changed — return whatever we parsed so far
+  }
+
+  return entries;
+}
+
+function parseSections(
+  sections: any[],
+  entries: ParsedEntry[],
+): void {
+  let currentDate = "Unknown";
+
+  for (const section of sections) {
+    // Check for date header
+    const header = section.itemSectionRenderer?.header;
+    if (header?.itemSectionHeaderRenderer?.title?.simpleText) {
+      currentDate = header.itemSectionHeaderRenderer.title.simpleText;
+    }
+
+    const items = section.itemSectionRenderer?.contents;
+    if (!items) continue;
+
+    for (const item of items) {
+      // --- Regular video: lockupViewModel ---
+      const lvm = item.lockupViewModel;
+      if (lvm) {
+        const videoId =
+          lvm.rendererContext?.commandContext?.onTap?.innertubeCommand
+            ?.watchEndpoint?.videoId ?? lvm.contentId;
+        const title = lvm.metadata?.lockupMetadataViewModel?.title?.content;
+        const channelLabel =
+          lvm.metadata?.lockupMetadataViewModel?.image
+            ?.decoratedAvatarViewModel?.a11yLabel;
+        const channel = channelLabel
+          ? channelLabel.replace(/^Go to channel /, "")
+          : "";
+
+        if (videoId && title) {
+          entries.push({ videoId, title, channel, dateLabel: currentDate });
+        }
+        continue;
+      }
+
+      // --- Shorts shelf: reelShelfRenderer ---
+      const shelf = item.reelShelfRenderer;
+      if (shelf?.items) {
+        for (const short of shelf.items) {
+          const slvm = short.shortsLockupViewModel;
+          if (!slvm) continue;
+
+          const videoId =
+            slvm.onTap?.innertubeCommand?.reelWatchEndpoint?.videoId;
+          const title = slvm.overlayMetadata?.primaryText?.content;
+          if (videoId && title) {
+            entries.push({
+              videoId,
+              title,
+              channel: "",
+              dateLabel: currentDate,
+            });
+          }
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Find a continuation token in the response.
+ */
+function findContinuation(
+  obj: unknown,
+  depth = 0,
+): string | null {
+  if (obj == null || depth > 20) return null;
+
+  if (Array.isArray(obj)) {
+    for (const item of obj) {
+      const c = findContinuation(item, depth + 1);
+      if (c) return c;
+    }
+    return null;
+  }
+
+  if (typeof obj !== "object") return null;
+
+  const record = obj as Record<string, unknown>;
+
+  if (record.nextContinuationData) {
+    const ncd = record.nextContinuationData as Record<string, unknown>;
+    if (typeof ncd.continuation === "string") return ncd.continuation;
+  }
+
+  if (record.continuationItemRenderer) {
+    const cir = record.continuationItemRenderer as Record<string, unknown>;
+    if (cir.continuationEndpoint) {
+      const ce = cir.continuationEndpoint as Record<string, unknown>;
+      if (ce.continuationCommand) {
+        const cc = ce.continuationCommand as Record<string, unknown>;
+        if (typeof cc.token === "string") return cc.token;
+      }
+    }
+  }
+
+  for (const value of Object.values(record)) {
+    const c = findContinuation(value, depth + 1);
+    if (c) return c;
+  }
+
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Date parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a section header date label into an ISO timestamp.
+ * Handles: "Monday", "Tuesday", ..., "Yesterday", "Today",
+ * "Jun 18", "Apr 15", "Mar 3", etc.
+ */
+function parseDateLabel(label: string, now: Date): Date {
+  const days: Record<string, number> = {
+    sunday: 0,
+    monday: 1,
+    tuesday: 2,
+    wednesday: 3,
+    thursday: 4,
+    friday: 5,
+    saturday: 6,
+  };
+
+  const lower = label.toLowerCase().trim();
+
+  // Day names: "Monday" = most recent occurrence of that day
+  const dayIndex = days[lower];
+  if (dayIndex !== undefined) {
+    const today = now.getDay();
+    let diff = today - dayIndex;
+    if (diff < 0) diff += 7;
+    if (diff === 0) diff = 0; // Today would be "Today", not the day name
+    // Actually, "Monday" on a Monday = today. But YouTube uses "Today" for that.
+    // "Monday" on any other day = the most recent Monday.
+    if (diff === 0) diff = 7; // Same day name = last week
+    const d = new Date(now);
+    d.setDate(d.getDate() - diff);
+    d.setHours(12, 0, 0, 0);
+    return d;
+  }
+
+  if (lower === "yesterday") {
+    const d = new Date(now);
+    d.setDate(d.getDate() - 1);
+    d.setHours(12, 0, 0, 0);
+    return d;
+  }
+
+  if (lower === "today") {
+    return now;
+  }
+
+  // "Jun 18", "Apr 15" — month abbreviation + day
+  // "Jun 25, 2025" — month abbreviation + day + year (continuation pages)
+  const months: Record<string, number> = {
+    jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5,
+    jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11,
+  };
+
+  const match = lower.match(/^([a-z]{3})\s+(\d{1,2})(?:,\s*(\d{4}))?$/);
+  if (match) {
+    const month = months[match[1]];
+    const day = parseInt(match[2]);
+    const explicitYear = match[3] ? parseInt(match[3]) : undefined;
+    if (month !== undefined && day > 0 && day <= 31) {
+      const year =
+        explicitYear ??
+        (month > now.getMonth() ? now.getFullYear() - 1 : now.getFullYear());
+      return new Date(year, month, day, 12, 0, 0);
+    }
+  }
+
+  return now;
+}
+
+// ---------------------------------------------------------------------------
+// Transcript fetching (public timedtext API, no auth)
+// ---------------------------------------------------------------------------
+
+async function fetchTranscript(videoId: string): Promise<string | null> {
+  try {
+    const url = `https://www.youtube.com/api/timedtext?v=${videoId}&lang=en`;
+    const res = await fetch(url);
+    if (!res.ok) return null;
+
+    const xml = await res.text();
+    if (!xml || !xml.includes("<text ")) return null;
+
+    const texts: string[] = [];
+    const textRegex = /<text[^>]*>([^<]*)<\/text>/g;
+    let match;
+    while ((match = textRegex.exec(xml)) !== null) {
+      const decoded = match[1]
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .trim();
+      if (decoded) texts.push(decoded);
+    }
+
+    return texts.length > 0 ? texts.join(" ") : null;
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Payload type
+// ---------------------------------------------------------------------------
+
+interface VideoWatchedPayload {
+  title: string;
+  channel: string;
+  channelId: string;
+  videoId: string;
+  url: string;
+  transcript: string | null;
+}
+
+// ---------------------------------------------------------------------------
+// Adapter
+// ---------------------------------------------------------------------------
+
+const adapter: Adapter = {
+  name: "youtube-history",
+
+  async pull(cursor: Cursor, ctx: AdapterContext): Promise<PullResult> {
+    const jar = loadCookies();
+    const useFixture = process.env.PDV_FIXTURE_MODE === "1" || !jar;
+
+    if (useFixture) {
+      return pullFromFixture(cursor);
+    }
+
+    // --- Live mode ---
+    const cur = parseCursor(cursor);
+    const events: Envelope<VideoWatchedPayload>[] = [];
+    const now = new Date();
+    let continuation: string | null = cur.continuation;
+    let newestVideoId = cur.newestVideoId;
+    let capped = false;
+
+    do {
+      const body: Record<string, unknown> = {
+        context: {
+          client: {
+            clientName: "WEB",
+            clientVersion: "2.20250625.00.00",
+            hl: "en",
+            gl: "US",
+          },
+        },
+      };
+
+      if (continuation) {
+        body.continuation = continuation;
+      } else {
+        body.browseId = "FEhistory";
+      }
+
+      const data = await browseRequest(body, jar);
+      if (!data) break;
+
+      if (data.error) {
+        console.error(
+          `[youtube-history] API error:`,
+          JSON.stringify(data.error).slice(0, 500),
+        );
+        break;
+      }
+
+      const entries = parseHistoryResponse(data);
+
+      for (const entry of entries) {
+        if (events.length >= MAX_PER_PULL) {
+          capped = true;
+          break;
+        }
+
+        // On re-pull (no continuation), stop at the first video we've
+        // already seen.
+        if (
+          !cur.continuation &&
+          newestVideoId &&
+          entry.videoId === newestVideoId
+        ) {
+          capped = true;
+          break;
+        }
+
+        const watchedAt = parseDateLabel(entry.dateLabel, now).toISOString();
+
+        events.push({
+          v: ENVELOPE_VERSION,
+          ts: watchedAt,
+          source: adapter.name,
+          type: "video_watched",
+          id: entry.videoId,
+          payload: {
+            title: entry.title,
+            channel: entry.channel,
+            channelId: "",
+            videoId: entry.videoId,
+            url: `https://youtube.com/watch?v=${entry.videoId}`,
+            transcript: null,
+          },
+        });
+
+        if (!newestVideoId && !cur.continuation && events.length === 1) {
+          newestVideoId = entry.videoId;
+        }
+      }
+
+      if (capped) break;
+
+      continuation = findContinuation(data);
+    } while (continuation);
+
+    // --- Transcript phase (opt-in) ---
+    if (FETCH_TRANSCRIPTS && events.length > 0) {
+      for (const event of events) {
+        const raw = await fetchTranscript(event.payload.videoId);
+        if (!raw) continue;
+
+        const { uri } = ctx.storeAsset(Buffer.from(raw, "utf-8"), {
+          mime: "text/plain; source=youtube-transcript",
+        });
+        event.payload.transcript = uri;
+      }
+    }
+
+    return {
+      events,
+      cursor:
+        events.length > 0 || continuation
+          ? encodeCursor({
+              continuation: continuation ?? null,
+              newestVideoId,
+            })
+          : cursor,
+      hasMore: !!continuation || capped,
+    };
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Fixture path
+// ---------------------------------------------------------------------------
+
+async function pullFromFixture(cursor: Cursor): Promise<PullResult> {
+  let raw: string;
+  try {
+    raw = readFileSync(FIXTURE_PATH, "utf8");
+  } catch {
+    return { events: [], cursor, hasMore: false };
+  }
+
+  const items = JSON.parse(raw) as {
+    id: string;
+    snippet: {
+      publishedAt: string;
+      title: string;
+      channelId: string;
+      channelTitle: string;
+      resourceId: { kind: string; videoId: string };
+    };
+  }[];
+
+  const cur = parseCursor(cursor);
+  const events: Envelope[] = [];
+  let firstVideoId: string | null = null;
+
+  for (const item of items) {
+    if (
+      cur.newestVideoId &&
+      item.snippet.resourceId.videoId === cur.newestVideoId
+    )
+      break;
+
+    if (!firstVideoId) firstVideoId = item.snippet.resourceId.videoId;
+
+    events.push({
+      v: ENVELOPE_VERSION,
+      ts: item.snippet.publishedAt,
+      source: adapter.name,
+      type: "video_watched",
+      id: item.id,
+      payload: {
+        title: item.snippet.title,
+        channel: item.snippet.channelTitle,
+        channelId: item.snippet.channelId,
+        videoId: item.snippet.resourceId.videoId,
+        url: `https://youtube.com/watch?v=${item.snippet.resourceId.videoId}`,
+        transcript: null,
+      },
+    });
+  }
+
+  return {
+    events,
+    cursor:
+      events.length > 0
+        ? encodeCursor({ continuation: null, newestVideoId: firstVideoId })
+        : cursor,
+    hasMore: false,
+  };
+}
+
+export default adapter;

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@mannyflussnpm/adapter-youtube-history",
+  "version": "1.0.0",
+  "description": "adapter for Personal Data Vault",
+  "type": "module",
+  "main": "./index.ts",
+  "types": "./index.ts",
+  "exports": {
+    ".": "./index.ts"
+  },
+  "pdv": {
+    "type": "adapter"
+  },
+  "peerDependencies": {
+    "@mannyflussnpm/spec": "^1.0.0"
+  },
+  "license": "MIT"
+}