@chronicle.app/youtube 0.3.0

package/dist/connectors/YouTubeTransformer.js

@@ -0,0 +1,220 @@
+import { ChronicleTransformer, createImageObject } from '@chronicle.app/etl';
+import { ytTime } from '../utils/time.js';
+import { channelUrl, playlistUrl, videoUrl } from '../utils/urls.js';
+// YouTube returns thumbnails at several sizes; pick the largest available.
+function largestThumbnail(thumbnails) {
+    if (!thumbnails)
+        return undefined;
+    const order = ['maxres', 'standard', 'high', 'medium', 'default'];
+    for (const size of order) {
+        const url = thumbnails[size]?.url;
+        if (url)
+            return url;
+    }
+    return undefined;
+}
+// When a liked video has been deleted or made private, videos.list returns
+// nothing and the playlist item carries these literal placeholder strings in
+// place of the video's real title — never assert them as data.
+const PLACEHOLDER_TITLES = new Set(['Deleted video', 'Private video']);
+export default class YouTubeTransformer extends ChronicleTransformer {
+    async transform(record) {
+        switch (record.extraction.recordType) {
+            case 'likes': {
+                return this.buildLikeAction(record);
+            }
+            case 'subscriptions': {
+                return this.buildFollowAction(record);
+            }
+            case 'playlists': {
+                return this.buildPlaylistCreateAction(record);
+            }
+            case 'playlist-items': {
+                return this.buildPlaylistAddAction(record);
+            }
+            case 'uploads': {
+                return this.buildUploadPublishAction(record);
+            }
+            default: {
+                return [];
+            }
+        }
+    }
+    buildLikeAction(record) {
+        const liked = record.data;
+        const action = {
+            '@type': 'LikeAction',
+            '@key': ['@type', 'source', 'agent.handle', 'object.sourceId', 'timestamp'],
+            source: 'youtube',
+            timestamp: ytTime(liked.item.snippet.publishedAt),
+            agent: this.resolveSelf(record),
+            object: this.buildVideo(liked),
+        };
+        return [action];
+    }
+    // A subscription is a state row (current channel title/avatar at read time),
+    // but its publishedAt is the real subscribe instant. Keyed on the followed
+    // channel's id so snapshot re-reads supersede rather than duplicate.
+    buildFollowAction(record) {
+        const subscription = record.data;
+        const { channelId } = subscription.snippet.resourceId;
+        const action = {
+            '@type': 'FollowAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: channelId,
+            timestamp: ytTime(subscription.snippet.publishedAt),
+            agent: this.resolveSelf(record),
+            object: this.buildChannelAgent(channelId, {
+                name: subscription.snippet.title,
+                description: subscription.snippet.description,
+                image: largestThumbnail(subscription.snippet.thumbnails),
+            }),
+        };
+        return [action];
+    }
+    // A playlist's coming-into-being: real creation timestamp from the API, the
+    // account owner as agent, the Collection as origin-grade result.
+    buildPlaylistCreateAction(record) {
+        const playlist = record.data;
+        const action = {
+            '@type': 'CreateAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: playlist.id,
+            timestamp: ytTime(playlist.snippet.publishedAt),
+            agent: this.resolveSelf(record),
+            result: this.buildCollection(playlist),
+        };
+        return [action];
+    }
+    // Adding a video to a playlist. The membership's identity is (video,
+    // playlist, add instant) — the only identity every reader of the account
+    // shares (a Takeout archive has no playlist-item ids), so assertions from
+    // any path converge on one action. A re-added video is a new instant, hence
+    // a new action; the item id rides along as provenance.
+    buildPlaylistAddAction(record) {
+        const { playlist, item, video } = record.data;
+        const action = {
+            '@type': 'AddAction',
+            '@key': ['@type', 'source', 'object.sourceId', 'target.sourceId', 'timestamp'],
+            source: 'youtube',
+            sourceId: item.id,
+            timestamp: ytTime(item.snippet.publishedAt),
+            agent: this.resolveSelf(record),
+            object: this.buildVideo({ item, video }),
+            target: this.buildCollection(playlist),
+        };
+        return [action];
+    }
+    // An upload is the video's lifecycle occurrence: the action's own timestamp
+    // IS datePublished (carrying the scalar on the video too would make the fold
+    // derive a duplicate agentless occurrence), the video rides in `object`
+    // (where the lifecycle render-back and edge facets look), and the agent is
+    // routed onto the video's `publisher` edge by the fold.
+    buildUploadPublishAction(record) {
+        const upload = record.data;
+        const video = this.buildVideo(upload);
+        delete video.datePublished;
+        const action = {
+            '@type': 'PublishAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: upload.item.snippet.resourceId.videoId,
+            timestamp: ytTime(upload.item.snippet.publishedAt),
+            agent: this.resolveSelf(record),
+            object: video,
+        };
+        return [action];
+    }
+    buildCollection(playlist) {
+        const thumbnail = largestThumbnail(playlist.snippet.thumbnails);
+        return {
+            '@type': 'Collection',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: playlist.id,
+            url: playlistUrl(playlist.id),
+            name: playlist.snippet.title,
+            ...(playlist.snippet.description && {
+                description: playlist.snippet.description,
+            }),
+            ...(playlist.status?.privacyStatus && {
+                visibility: playlist.status.privacyStatus,
+            }),
+            ...(thumbnail && { emblem: createImageObject({ url: thumbnail }) }),
+        };
+    }
+    buildVideo(liked) {
+        const { item, video } = liked;
+        const { videoId } = item.snippet.resourceId;
+        // The hydrated videos.list response is canonical; the playlist item is the
+        // fallback for videos that have since gone private or been deleted — but
+        // its placeholder title/description are YouTube boilerplate, not the
+        // video's metadata, so an unavailable video keeps only its id and url.
+        const snippet = video?.snippet ?? item.snippet;
+        const unavailable = !video && PLACEHOLDER_TITLES.has(item.snippet.title);
+        const uploaderChannelId = video?.snippet.channelId ?? item.snippet.videoOwnerChannelId;
+        const uploaderChannelTitle = video?.snippet.channelTitle ?? item.snippet.videoOwnerChannelTitle;
+        const thumbnail = largestThumbnail(snippet.thumbnails);
+        const videoObject = {
+            '@type': 'VideoObject',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: videoId,
+            url: videoUrl(videoId),
+            ...(!unavailable && { name: snippet.title }),
+            ...(!unavailable && snippet.description && { description: snippet.description }),
+            ...(video?.contentDetails?.duration && {
+                duration: video.contentDetails.duration,
+            }),
+            ...(video?.snippet.publishedAt && {
+                datePublished: video.snippet.publishedAt,
+            }),
+            ...(thumbnail && { emblem: createImageObject({ url: thumbnail }) }),
+            ...(uploaderChannelId && {
+                author: [
+                    this.buildChannelAgent(uploaderChannelId, {
+                        name: uploaderChannelTitle,
+                    }),
+                ],
+            }),
+        };
+        return videoObject;
+    }
+    // A YouTube channel as an Agent, keyed on its channel id. Shared by the
+    // account owner and subscribed/uploading channels so all merge on the id.
+    buildChannelAgent(channelId, opts) {
+        const agent = {
+            '@type': 'Agent',
+            '@key': ['@type', 'source', 'handle'],
+            source: 'youtube',
+            handle: channelId,
+            url: channelUrl(channelId),
+        };
+        if (opts.name) {
+            agent.name = opts.name;
+        }
+        if (opts.description) {
+            agent.description = opts.description;
+        }
+        if (opts.image) {
+            agent.emblem = createImageObject({ url: opts.image });
+        }
+        return agent;
+    }
+    // The account owner, from the self channel resolved in extractor setup.
+    // Only this agent is tagged @me; other channels route through
+    // buildChannelAgent and must not.
+    resolveSelf(record) {
+        const user = record.context?.user;
+        if (!user?.id) {
+            throw new Error('YouTube self channel could not be resolved for the account owner');
+        }
+        const self = this.buildChannelAgent(user.id, {
+            name: user.snippet?.title,
+            image: largestThumbnail(user.snippet?.thumbnails),
+        });
+        return { ...self, sameAs: [...(self.sameAs ?? []), '@me'] };
+    }
+}

package/dist/connectors/YouTubeUploadsExtractor.d.ts

@@ -0,0 +1,11 @@
+import { Record } from '@chronicle.app/etl';
+import { YouTubeBaseExtractor } from './YouTubeBaseExtractor.js';
+export declare class YouTubeUploadsExtractor extends YouTubeBaseExtractor<typeof YouTubeUploadsExtractor> {
+    static description: string;
+    static recordTypes: string[];
+    private uploadsPlaylistId;
+    setup(): Promise<void>;
+    extract(): AsyncGenerator<Record>;
+    determineCount(): Promise<number | null>;
+}
+//# sourceMappingURL=YouTubeUploadsExtractor.d.ts.map

package/dist/connectors/YouTubeUploadsExtractor.js

@@ -0,0 +1,50 @@
+import { YouTubeBaseExtractor } from './YouTubeBaseExtractor.js';
+export class YouTubeUploadsExtractor extends YouTubeBaseExtractor {
+    static description = 'Videos uploaded to your channel';
+    static recordTypes = ['uploads'];
+    uploadsPlaylistId;
+    async setup() {
+        await super.setup();
+        const uploadsPlaylistId = this.user?.contentDetails?.relatedPlaylists?.uploads;
+        if (!uploadsPlaylistId) {
+            throw new Error('YouTube API did not expose an uploads playlist for this account');
+        }
+        this.uploadsPlaylistId = uploadsPlaylistId;
+    }
+    async *extract() {
+        const { since, until } = this.config;
+        let items = await this.proxy.getPlaylistItems(this.uploadsPlaylistId, {
+            limit: since ? undefined : (this.getEffectiveLimit() ?? undefined),
+        });
+        items = items.filter(item => {
+            const publishedAt = new Date(item.snippet.publishedAt);
+            if (since && publishedAt <= since)
+                return false;
+            if (until && publishedAt > until)
+                return false;
+            return true;
+        });
+        const limit = this.getEffectiveLimit();
+        if (limit !== null) {
+            items = items.slice(0, limit);
+        }
+        const details = await this.proxy.getVideoDetails(items.map(item => item.snippet.resourceId.videoId));
+        const detailsById = new Map(details.map(video => [video.id, video]));
+        this.logger.info(`Retrieved ${items.length} uploads from YouTube`);
+        for (const item of items) {
+            const upload = {
+                item,
+                video: detailsById.get(item.snippet.resourceId.videoId),
+            };
+            yield this.createRecordWithUser(upload, { recordType: 'uploads' });
+        }
+    }
+    async determineCount() {
+        try {
+            return await this.proxy.getPlaylistItemCount(this.uploadsPlaylistId);
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export { YouTubeBaseExtractor } from './connectors/YouTubeBaseExtractor.js';
+export { YouTubeLikesExtractor } from './connectors/YouTubeLikesExtractor.js';
+export { YouTubeSubscriptionsExtractor } from './connectors/YouTubeSubscriptionsExtractor.js';
+export { YouTubePlaylistsExtractor } from './connectors/YouTubePlaylistsExtractor.js';
+export { YouTubeUploadsExtractor } from './connectors/YouTubeUploadsExtractor.js';
+export { YouTubeTakeoutExtractor } from './connectors/YouTubeTakeoutExtractor.js';
+export { default as YouTubeTransformer } from './connectors/YouTubeTransformer.js';
+export { default as YouTubeTakeoutTransformer } from './connectors/YouTubeTakeoutTransformer.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,12 @@
+export { YouTubeBaseExtractor } from './connectors/YouTubeBaseExtractor.js';
+export { YouTubeLikesExtractor } from './connectors/YouTubeLikesExtractor.js';
+export { YouTubeSubscriptionsExtractor } from './connectors/YouTubeSubscriptionsExtractor.js';
+export { YouTubePlaylistsExtractor } from './connectors/YouTubePlaylistsExtractor.js';
+export { YouTubeUploadsExtractor } from './connectors/YouTubeUploadsExtractor.js';
+export { YouTubeTakeoutExtractor } from './connectors/YouTubeTakeoutExtractor.js';
+export { default as YouTubeTransformer } from './connectors/YouTubeTransformer.js';
+export { default as YouTubeTakeoutTransformer } from './connectors/YouTubeTakeoutTransformer.js';
+// Register OAuth provider
+import { OAuthProviderRegistry } from '@chronicle.app/auth';
+import { GoogleOAuthProvider } from './auth/GoogleOAuthProvider.js';
+OAuthProviderRegistry.register(GoogleOAuthProvider);

package/dist/takeout/csv.d.ts

@@ -0,0 +1,4 @@
+/** A Takeout CSV as header-keyed rows. Fields regularly carry embedded
+ * newlines (descriptions, comment JSON), so this is never line-splittable. */
+export declare function readTakeoutCsv(path: string): Promise<Record<string, string>[]>;
+//# sourceMappingURL=csv.d.ts.map

package/dist/takeout/csv.js

@@ -0,0 +1,12 @@
+import { readFile } from 'node:fs/promises';
+import { parse } from 'csv-parse/sync';
+/** A Takeout CSV as header-keyed rows. Fields regularly carry embedded
+ * newlines (descriptions, comment JSON), so this is never line-splittable. */
+export async function readTakeoutCsv(path) {
+    const content = await readFile(path, 'utf-8');
+    return parse(content, {
+        columns: true,
+        bom: true,
+        skip_empty_lines: true,
+    });
+}

package/dist/takeout/parseActivityHtml.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Parser for Google Takeout "My Activity" HTML files (watch-history.html,
+ * search-history.html). The file is machine-generated Material Design Lite
+ * markup: one `outer-cell` div per activity entry, whose first body cell
+ * carries a verb ("Watched", "Searched for", …) followed by a non-breaking
+ * space, the linked title/query, an optional channel link, and a rendered
+ * local-time date line; a caption cell carries provenance, including the
+ * "From Google Ads" marker on ad impressions.
+ *
+ * Timestamps are rendered in the account's timezone with a single fixed
+ * abbreviation for the whole file (e.g. every stamp `EDT`), so parsing maps
+ * the abbreviation to its UTC offset and reconstructs the instant.
+ */
+export interface ActivityEntry {
+    verb: string;
+    /** First link's text — the video title or the search query. */
+    titleText?: string;
+    /** First link's href — the watch URL or the results URL. */
+    titleHref?: string;
+    channelName?: string;
+    channelId?: string;
+    time: Date;
+    /** "From Google Ads" in the caption cell — an ad impression, not activity. */
+    isAd: boolean;
+}
+export declare function parseActivityDate(text: string): Date;
+export declare function parseActivityHtml(html: string): ActivityEntry[];
+/** The 11-char video id from a watch URL (www or music host). */
+export declare function videoIdFromUrl(url: string): string | undefined;
+//# sourceMappingURL=parseActivityHtml.d.ts.map

package/dist/takeout/parseActivityHtml.js

@@ -0,0 +1,116 @@
+/**
+ * Parser for Google Takeout "My Activity" HTML files (watch-history.html,
+ * search-history.html). The file is machine-generated Material Design Lite
+ * markup: one `outer-cell` div per activity entry, whose first body cell
+ * carries a verb ("Watched", "Searched for", …) followed by a non-breaking
+ * space, the linked title/query, an optional channel link, and a rendered
+ * local-time date line; a caption cell carries provenance, including the
+ * "From Google Ads" marker on ad impressions.
+ *
+ * Timestamps are rendered in the account's timezone with a single fixed
+ * abbreviation for the whole file (e.g. every stamp `EDT`), so parsing maps
+ * the abbreviation to its UTC offset and reconstructs the instant.
+ */
+import { decodeEntities } from '@chronicle.app/etl';
+// Google's renderer separates tokens with U+00A0 (and in newer exports
+// U+202F before AM/PM) rather than plain spaces.
+const SPACERS = /[\u00A0\u202F]/g;
+const BODY_CELL = /<div class="content-cell mdl-cell mdl-cell--6-col mdl-typography--body-1">([\s\S]*?)<\/div>/;
+const CAPTION_CELL = /mdl-typography--caption">([\s\S]*?)<\/div>/;
+const LINK = /<a href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g;
+const DATE_LINE = /([A-Z][a-z]{2} \d{1,2}, \d{4}, \d{1,2}:\d{2}:\d{2}[\u00A0\u202F ]?(?:AM|PM)[\u00A0\u202F ][A-Z]{2,5})(?:<br|$)/g;
+const CHANNEL_ID_IN_URL = /\/channel\/([^/?#"]+)/;
+const MONTHS = {
+    Jan: 0,
+    Feb: 1,
+    Mar: 2,
+    Apr: 3,
+    May: 4,
+    Jun: 5,
+    Jul: 6,
+    Aug: 7,
+    Sep: 8,
+    Oct: 9,
+    Nov: 10,
+    Dec: 11,
+};
+// North-American + UTC abbreviations Google renders My Activity in. An
+// unmapped abbreviation throws rather than silently misplacing a history.
+const TZ_OFFSET_MINUTES = {
+    UTC: 0,
+    GMT: 0,
+    NST: -210,
+    NDT: -150,
+    AST: -240,
+    ADT: -180,
+    EST: -300,
+    EDT: -240,
+    CST: -360,
+    CDT: -300,
+    MST: -420,
+    MDT: -360,
+    PST: -480,
+    PDT: -420,
+    AKST: -540,
+    AKDT: -480,
+    HST: -600,
+};
+/** Decoded link/verb text, with layout non-breaking spaces normalized. */
+function cleanText(text) {
+    return decodeEntities(text).replaceAll(SPACERS, ' ');
+}
+export function parseActivityDate(text) {
+    const normalized = text.replaceAll(SPACERS, ' ');
+    const match = normalized.match(/^([A-Z][a-z]{2}) (\d{1,2}), (\d{4}), (\d{1,2}):(\d{2}):(\d{2}) (AM|PM) ([A-Z]{2,5})$/);
+    if (!match) {
+        throw new Error(`Unparseable My Activity timestamp: "${text}"`);
+    }
+    const [, mon, day, year, hour, minute, second, meridiem, tz] = match;
+    const offset = TZ_OFFSET_MINUTES[tz];
+    if (offset === undefined) {
+        throw new Error(`Unknown timezone abbreviation "${tz}" in "${text}"`);
+    }
+    let h = Number(hour) % 12;
+    if (meridiem === 'PM')
+        h += 12;
+    const utc = Date.UTC(Number(year), MONTHS[mon], Number(day), h, Number(minute), Number(second));
+    return new Date(utc - offset * 60_000);
+}
+export function parseActivityHtml(html) {
+    const entries = [];
+    for (const cell of html.split('<div class="outer-cell').slice(1)) {
+        const body = cell.match(BODY_CELL)?.[1];
+        if (!body)
+            continue;
+        const verb = cleanText(body.match(/^([^<]*)/)?.[1] ?? '').trim();
+        // The date line is the body's last text segment; ad entries carry earlier
+        // time-like segments ("Watched at 8:33 AM") that lack a date and don't
+        // match the full pattern — take the last full match.
+        let dateText;
+        for (const match of body.matchAll(DATE_LINE))
+            dateText = match[1];
+        if (!dateText)
+            continue;
+        const links = [...body.matchAll(LINK)];
+        const [firstHref, firstText] = links[0] ? [links[0][1], links[0][2]] : [];
+        const channelLink = links.slice(1).find(link => CHANNEL_ID_IN_URL.test(link[1]));
+        entries.push({
+            verb,
+            ...(firstHref !== undefined && {
+                titleHref: decodeEntities(firstHref),
+                titleText: cleanText(firstText ?? ''),
+            }),
+            ...(channelLink && {
+                channelName: cleanText(channelLink[2]),
+                channelId: channelLink[1].match(CHANNEL_ID_IN_URL)?.[1],
+            }),
+            time: parseActivityDate(dateText),
+            isAd: (cell.match(CAPTION_CELL)?.[1] ?? '').includes('From Google Ads'),
+        });
+    }
+    return entries;
+}
+/** The 11-char video id from a watch URL (www or music host). */
+export function videoIdFromUrl(url) {
+    return url.match(/[?&]v=([\w-]{11})/)?.[1];
+}

package/dist/utils/YouTubeProxy.d.ts

@@ -0,0 +1,138 @@
+import { ApiProxy } from '@chronicle.app/etl';
+export interface YouTubeThumbnails {
+    default?: {
+        url: string;
+        width?: number;
+        height?: number;
+    };
+    medium?: {
+        url: string;
+        width?: number;
+        height?: number;
+    };
+    high?: {
+        url: string;
+        width?: number;
+        height?: number;
+    };
+    standard?: {
+        url: string;
+        width?: number;
+        height?: number;
+    };
+    maxres?: {
+        url: string;
+        width?: number;
+        height?: number;
+    };
+}
+export interface YouTubeSelfChannel {
+    id: string;
+    snippet: {
+        title: string;
+        description?: string;
+        customUrl?: string;
+        thumbnails?: YouTubeThumbnails;
+    };
+    contentDetails?: {
+        relatedPlaylists?: {
+            likes?: string;
+            uploads?: string;
+        };
+    };
+}
+export interface YouTubePlaylistItem {
+    id: string;
+    snippet: {
+        publishedAt: string;
+        title: string;
+        description?: string;
+        thumbnails?: YouTubeThumbnails;
+        channelId: string;
+        channelTitle: string;
+        resourceId: {
+            kind: string;
+            videoId: string;
+        };
+        videoOwnerChannelId?: string;
+        videoOwnerChannelTitle?: string;
+    };
+}
+export interface YouTubeVideo {
+    id: string;
+    snippet: {
+        publishedAt: string;
+        channelId: string;
+        channelTitle: string;
+        title: string;
+        description?: string;
+        thumbnails?: YouTubeThumbnails;
+        tags?: string[];
+    };
+    contentDetails?: {
+        duration?: string;
+        definition?: string;
+    };
+}
+export interface YouTubePlaylist {
+    id: string;
+    snippet: {
+        publishedAt: string;
+        title: string;
+        description?: string;
+        thumbnails?: YouTubeThumbnails;
+        channelId: string;
+    };
+    status?: {
+        privacyStatus?: string;
+    };
+    contentDetails?: {
+        itemCount?: number;
+    };
+}
+export interface YouTubeSubscription {
+    id: string;
+    snippet: {
+        publishedAt: string;
+        title: string;
+        description?: string;
+        thumbnails?: YouTubeThumbnails;
+        resourceId: {
+            kind: string;
+            channelId: string;
+        };
+    };
+}
+export default class YouTubeProxy extends ApiProxy {
+    constructor(accessToken?: string);
+    initialize(): Promise<void>;
+    private ensureAuthenticated;
+    protected mapError(error: unknown): unknown;
+    private get;
+    /** Self channel (drives the @me agent) plus the likes playlist id. */
+    getUser(): Promise<YouTubeSelfChannel>;
+    /**
+     * Liked videos, newest first, from the account's likes playlist.
+     * snippet.publishedAt on a playlist item is when the video was liked.
+     */
+    getLikes(options: {
+        likesPlaylistId: string;
+        limit?: number;
+    }): Promise<YouTubePlaylistItem[]>;
+    /** Items of any playlist. snippet.publishedAt is when the item was added. */
+    getPlaylistItems(playlistId: string, options?: {
+        limit?: number;
+    }): Promise<YouTubePlaylistItem[]>;
+    /** The account's own playlists (created playlists, not saved ones). */
+    getPlaylists(options?: {
+        limit?: number;
+    }): Promise<YouTubePlaylist[]>;
+    /** Hydrate video details (duration, tags, canonical channel) in batches. */
+    getVideoDetails(ids: string[]): Promise<YouTubeVideo[]>;
+    getSubscriptions(options: {
+        limit?: number;
+    }): Promise<YouTubeSubscription[]>;
+    getPlaylistItemCount(playlistId: string): Promise<number>;
+    getSubscriptionsCount(): Promise<number>;
+}
+//# sourceMappingURL=YouTubeProxy.d.ts.map