@chronicle.app/youtube 0.3.0

package/dist/connectors/YouTubeTakeoutExtractor.js

@@ -0,0 +1,315 @@
+import { Extractor } from '@chronicle.app/etl';
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import { z } from 'zod';
+import { readTakeoutCsv } from '../takeout/csv.js';
+import { parseActivityHtml, videoIdFromUrl } from '../takeout/parseActivityHtml.js';
+import YouTubeTakeoutTransformer from './YouTubeTakeoutTransformer.js';
+// Relative paths of the archive files this extractor reads.
+const FILES = {
+    watch: 'history/watch-history.html',
+    search: 'history/search-history.html',
+    subscriptions: 'subscriptions/subscriptions.csv',
+    playlists: 'playlists/playlists.csv',
+    uploads: 'video metadata/videos.csv',
+    comments: 'comments/comments.csv',
+    channel: 'channels/channel.csv',
+};
+export class YouTubeTakeoutExtractor extends Extractor {
+    static source = 'youtube';
+    static description = 'Watch and search history and library from Takeout';
+    static delivery = 'export';
+    static strategy = 'takeout';
+    static recordTypes = [
+        'watches',
+        'searches',
+        'subscriptions',
+        'playlists',
+        'playlist-items',
+        'uploads',
+        'comments',
+    ];
+    static default = false;
+    // A Takeout archive is a full re-read of account state: re-exporting
+    // re-reports the same rows with their latest values (current titles,
+    // current membership), so attributes are sighted at export time while
+    // actions keep their own occurrence timestamps.
+    static temporality = 'snapshot';
+    static defaultTransformer = YouTubeTakeoutTransformer;
+    // `input` is validated in setup() rather than by the schema: the CLI unions
+    // flags across a source's extractors, so a schema-required flag here would
+    // also gate the API extractors that share the youtube source.
+    static schema = Extractor.schema.extend({
+        input: z.string().optional().describe('Path to the Takeout "YouTube and YouTube Music" folder'),
+    });
+    archiveAsOf = null;
+    selfChannel = null;
+    inputPath(relative) {
+        const config = this.config;
+        if (!config.input) {
+            throw new Error('The takeout extractor needs --input: the path to the Takeout "YouTube and YouTube Music" folder');
+        }
+        return join(config.input, relative);
+    }
+    async fileExists(path) {
+        try {
+            await stat(path);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async setup() {
+        await super.setup();
+        // The archive is a static artifact: its sighting time is the export's,
+        // not this run's, so re-running over an unchanged tree is a byte-identical
+        // no-op. Newest mtime across the files read stands in for the export time.
+        const paths = [
+            ...Object.values(FILES).map(f => this.inputPath(f)),
+            ...(await this.playlistItemFiles()).map(([, path]) => path),
+        ];
+        let newest = 0;
+        for (const path of paths) {
+            try {
+                newest = Math.max(newest, (await stat(path)).mtimeMs);
+            }
+            catch {
+                // Missing sections are fine; each is skipped with a warning later.
+            }
+        }
+        if (newest > 0)
+            this.archiveAsOf = new Date(newest);
+        const channelPath = this.inputPath(FILES.channel);
+        if (await this.fileExists(channelPath)) {
+            const [row] = await readTakeoutCsv(channelPath);
+            const id = row?.['Channel ID'];
+            if (id) {
+                this.selfChannel = {
+                    id,
+                    title: row['Channel Title (Original)'] || undefined,
+                };
+            }
+        }
+        if (!this.selfChannel) {
+            this.logger.warn('channels/channel.csv missing or empty — the account owner falls back to the per-source singleton self');
+        }
+    }
+    asOfTime() {
+        return this.archiveAsOf ?? super.asOfTime();
+    }
+    inRange(time) {
+        const { since, until } = this.config;
+        if (since && time < since)
+            return false;
+        if (until && time > until)
+            return false;
+        return true;
+    }
+    /** `[playlist title, csv path]` for each per-playlist membership file. */
+    async playlistItemFiles() {
+        const dir = this.inputPath('playlists');
+        try {
+            const files = await readdir(dir);
+            return files
+                .filter(f => f.endsWith('-videos.csv'))
+                .map(f => [basename(f, '-videos.csv'), join(dir, f)]);
+        }
+        catch {
+            return [];
+        }
+    }
+    async *extract() {
+        let count = 0;
+        const self = this.selfChannel;
+        const emit = (recordType, data) => this.createRecord(data, { recordType, selfChannel: self });
+        for (const section of [
+            this.extractWatches(),
+            this.extractSearches(),
+            this.extractSubscriptions(),
+            this.extractPlaylists(),
+            this.extractUploads(),
+            this.extractComments(),
+        ]) {
+            for await (const [recordType, data] of section) {
+                if (this.shouldStopExtracting(count))
+                    return;
+                yield emit(recordType, data);
+                count++;
+            }
+        }
+    }
+    async *extractWatches() {
+        const path = this.inputPath(FILES.watch);
+        if (!(await this.fileExists(path))) {
+            this.logger.warn(`No watch history at ${FILES.watch} — skipping`);
+            return;
+        }
+        const entries = parseActivityHtml(await readFile(path, 'utf-8'));
+        let ads = 0;
+        let noise = 0;
+        let unparseable = 0;
+        let emitted = 0;
+        for (const entry of entries) {
+            if (entry.isAd) {
+                ads++;
+                continue;
+            }
+            // "Visited" (ad clicks) and "Used Shorts creation tools" aren't watches.
+            if (entry.verb !== 'Watched') {
+                noise++;
+                continue;
+            }
+            const videoId = entry.titleHref && videoIdFromUrl(entry.titleHref);
+            if (!videoId) {
+                unparseable++;
+                continue;
+            }
+            if (!this.inRange(entry.time))
+                continue;
+            const record = {
+                videoId,
+                title: entry.titleText,
+                titleUrl: entry.titleHref,
+                channelId: entry.channelId,
+                channelName: entry.channelName,
+                time: entry.time.toISOString(),
+            };
+            yield ['watches', record];
+            emitted++;
+        }
+        this.logger.info(`Watch history: ${emitted} watches (skipped ${ads} ads, ${noise} non-watch entries, ${unparseable} without a video id)`);
+    }
+    async *extractSearches() {
+        const path = this.inputPath(FILES.search);
+        if (!(await this.fileExists(path))) {
+            this.logger.warn(`No search history at ${FILES.search} — skipping`);
+            return;
+        }
+        const entries = parseActivityHtml(await readFile(path, 'utf-8'));
+        let skipped = 0;
+        let emitted = 0;
+        for (const entry of entries) {
+            // The search file also carries ad-impression "Watched" rows and ad-click
+            // "Visited" rows; only real searches are activity.
+            if (entry.isAd || entry.verb !== 'Searched for' || !entry.titleText) {
+                skipped++;
+                continue;
+            }
+            if (!this.inRange(entry.time))
+                continue;
+            const record = {
+                query: entry.titleText,
+                time: entry.time.toISOString(),
+            };
+            yield ['searches', record];
+            emitted++;
+        }
+        this.logger.info(`Search history: ${emitted} searches (skipped ${skipped} ads/non-search entries)`);
+    }
+    async *extractSubscriptions() {
+        const path = this.inputPath(FILES.subscriptions);
+        if (!(await this.fileExists(path)))
+            return;
+        for (const row of await readTakeoutCsv(path)) {
+            const channelId = row['Channel Id'];
+            if (!channelId)
+                continue;
+            const record = {
+                channelId,
+                channelTitle: row['Channel Title'] || undefined,
+            };
+            yield ['subscriptions', record];
+        }
+    }
+    async *extractPlaylists() {
+        const path = this.inputPath(FILES.playlists);
+        if (!(await this.fileExists(path)))
+            return;
+        const byTitle = new Map();
+        for (const row of await readTakeoutCsv(path)) {
+            const playlistId = row['Playlist ID'];
+            const createdAt = row['Playlist Create Timestamp'];
+            if (!playlistId || !createdAt)
+                continue;
+            const record = {
+                playlistId,
+                title: row['Playlist Title (Original)'] ?? '',
+                createdAt,
+                visibility: row['Playlist Visibility'] || undefined,
+            };
+            byTitle.set(record.title, record);
+            yield ['playlists', record];
+        }
+        // Membership files are named after the playlist title; a file whose title
+        // no longer matches playlists.csv has no playlist id to key against.
+        for (const [title, itemPath] of await this.playlistItemFiles()) {
+            const playlist = byTitle.get(title);
+            if (!playlist) {
+                this.logger.warn(`Playlist file "${basename(itemPath)}" has no matching row in playlists.csv — skipping`);
+                continue;
+            }
+            for (const row of await readTakeoutCsv(itemPath)) {
+                const videoId = row['Video ID'];
+                const addedAt = row['Playlist Video Creation Timestamp'];
+                if (!videoId || !addedAt)
+                    continue;
+                const record = {
+                    playlistId: playlist.playlistId,
+                    playlistTitle: playlist.title,
+                    videoId,
+                    addedAt,
+                };
+                yield ['playlist-items', record];
+            }
+        }
+    }
+    async *extractUploads() {
+        const path = this.inputPath(FILES.uploads);
+        if (!(await this.fileExists(path)))
+            return;
+        let drafts = 0;
+        for (const row of await readTakeoutCsv(path)) {
+            const videoId = row['Video ID'];
+            if (!videoId)
+                continue;
+            // A row without a publish timestamp is an unpublished draft/private
+            // upload — there is no publication occurrence to assert.
+            const publishedAt = row['Video Publish Timestamp'];
+            if (!publishedAt) {
+                drafts++;
+                continue;
+            }
+            const record = {
+                videoId,
+                title: row['Video Title (Original)'] || undefined,
+                description: row['Video Description (Original)'] || undefined,
+                durationMs: row['Approx Duration (ms)'] || undefined,
+                publishedAt,
+            };
+            yield ['uploads', record];
+        }
+        if (drafts > 0) {
+            this.logger.info(`Uploads: skipped ${drafts} unpublished draft(s)`);
+        }
+    }
+    async *extractComments() {
+        const path = this.inputPath(FILES.comments);
+        if (!(await this.fileExists(path)))
+            return;
+        for (const row of await readTakeoutCsv(path)) {
+            const commentId = row['Comment ID'];
+            const videoId = row['Video ID'];
+            const createdAt = row['Comment Create Timestamp'];
+            if (!commentId || !videoId || !createdAt)
+                continue;
+            const record = {
+                commentId,
+                videoId,
+                rawText: row['Comment Text'] ?? '',
+                createdAt,
+            };
+            yield ['comments', record];
+        }
+    }
+}

package/dist/connectors/YouTubeTakeoutTransformer.d.ts

@@ -0,0 +1,27 @@
+import { ChronicleTransformer, Record } from '@chronicle.app/etl';
+import { ActionAndChildren } from '@chronicle.app/schema';
+/**
+ * Transforms a Google Takeout "YouTube and YouTube Music" archive into the
+ * same shapes the API transformer emits — identical entity keysets, URL
+ * builders, and second-precision timestamps — so a full archive import and
+ * API crawls fold onto the same actions and entities instead of duplicating
+ * them.
+ */
+export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
+    transform(record: Record): Promise<ActionAndChildren[]>;
+    private buildWatchAction;
+    private buildSearchAction;
+    private buildFollowAction;
+    private buildPlaylistCreateAction;
+    private buildPlaylistAddAction;
+    private buildUploadPublishAction;
+    private buildCommentAnnotateAction;
+    private commentBody;
+    private buildVideo;
+    private buildBareVideo;
+    private buildCollection;
+    private buildChannelAgent;
+    private agentScopedKey;
+    private resolveSelf;
+}
+//# sourceMappingURL=YouTubeTakeoutTransformer.d.ts.map

package/dist/connectors/YouTubeTakeoutTransformer.js

@@ -0,0 +1,260 @@
+import { ChronicleTransformer, selfAgent } from '@chronicle.app/etl';
+import { msToIsoDuration, ytTime } from '../utils/time.js';
+import { channelUrl, playlistUrl, videoUrl } from '../utils/urls.js';
+/**
+ * Transforms a Google Takeout "YouTube and YouTube Music" archive into the
+ * same shapes the API transformer emits — identical entity keysets, URL
+ * builders, and second-precision timestamps — so a full archive import and
+ * API crawls fold onto the same actions and entities instead of duplicating
+ * them.
+ */
+export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
+    async transform(record) {
+        switch (record.extraction.recordType) {
+            case 'watches': {
+                return this.buildWatchAction(record);
+            }
+            case 'searches': {
+                return this.buildSearchAction(record);
+            }
+            case 'subscriptions': {
+                return this.buildFollowAction(record);
+            }
+            case 'playlists': {
+                return this.buildPlaylistCreateAction(record);
+            }
+            case 'playlist-items': {
+                return this.buildPlaylistAddAction(record);
+            }
+            case 'uploads': {
+                return this.buildUploadPublishAction(record);
+            }
+            case 'comments': {
+                return this.buildCommentAnnotateAction(record);
+            }
+            default: {
+                return [];
+            }
+        }
+    }
+    buildWatchAction(record) {
+        const watch = record.data;
+        const agent = this.resolveSelf(record);
+        const action = {
+            '@type': 'WatchAction',
+            '@key': this.agentScopedKey(agent, 'object.sourceId'),
+            source: 'youtube',
+            timestamp: ytTime(watch.time),
+            agent,
+            object: this.buildVideo(watch),
+        };
+        return [action];
+    }
+    buildSearchAction(record) {
+        const search = record.data;
+        const agent = this.resolveSelf(record);
+        const query = {
+            '@type': 'Query',
+            '@key': ['@type', 'source', 'body'],
+            source: 'youtube',
+            name: search.query,
+            body: search.query,
+        };
+        const action = {
+            '@type': 'FindAction',
+            '@key': this.agentScopedKey(agent, 'object.body'),
+            source: 'youtube',
+            timestamp: ytTime(search.time),
+            agent,
+            object: query,
+        };
+        return [action];
+    }
+    // Takeout reports which channels are followed but not since when — the
+    // record carries no occurrence, only the snapshot's @assertedAt. Keyed on
+    // the channel id exactly like the API FollowAction, so both paths converge
+    // on one action and the API's real subscribe date owns the occurrence.
+    buildFollowAction(record) {
+        const subscription = record.data;
+        const action = {
+            '@type': 'FollowAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: subscription.channelId,
+            agent: this.resolveSelf(record),
+            object: this.buildChannelAgent(subscription.channelId, subscription.channelTitle),
+        };
+        return [action];
+    }
+    buildPlaylistCreateAction(record) {
+        const playlist = record.data;
+        const action = {
+            '@type': 'CreateAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: playlist.playlistId,
+            timestamp: ytTime(playlist.createdAt),
+            agent: this.resolveSelf(record),
+            result: this.buildCollection(playlist.playlistId, {
+                name: playlist.title,
+                visibility: playlist.visibility,
+            }),
+        };
+        return [action];
+    }
+    // Same membership key as the API path — (video, playlist, add instant) —
+    // so an archive import and an API crawl of the same playlist fold onto one
+    // AddAction.
+    buildPlaylistAddAction(record) {
+        const item = record.data;
+        const action = {
+            '@type': 'AddAction',
+            '@key': ['@type', 'source', 'object.sourceId', 'target.sourceId', 'timestamp'],
+            source: 'youtube',
+            timestamp: ytTime(item.addedAt),
+            agent: this.resolveSelf(record),
+            object: this.buildBareVideo(item.videoId),
+            target: this.buildCollection(item.playlistId, {
+                name: item.playlistTitle,
+            }),
+        };
+        return [action];
+    }
+    // Mirrors the API upload mapping: the action's timestamp IS datePublished, so
+    // the video carries no datePublished scalar of its own.
+    buildUploadPublishAction(record) {
+        const upload = record.data;
+        const video = {
+            ...this.buildBareVideo(upload.videoId),
+            ...(upload.title && { name: upload.title }),
+            ...(upload.description && { description: upload.description }),
+            ...(upload.durationMs && {
+                duration: msToIsoDuration(Number(upload.durationMs)),
+            }),
+        };
+        const action = {
+            '@type': 'PublishAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: upload.videoId,
+            timestamp: ytTime(upload.publishedAt),
+            agent: this.resolveSelf(record),
+            object: video,
+        };
+        return [action];
+    }
+    buildCommentAnnotateAction(record) {
+        const data = record.data;
+        const agent = this.resolveSelf(record);
+        const video = this.buildBareVideo(data.videoId);
+        const comment = {
+            '@type': 'Comment',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: data.commentId,
+            body: this.commentBody(data.rawText),
+            author: [agent],
+            about: [video],
+        };
+        const action = {
+            '@type': 'AnnotateAction',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: data.commentId,
+            timestamp: ytTime(data.createdAt),
+            agent,
+            object: video,
+            result: comment,
+        };
+        return [action];
+    }
+    // Takeout wraps comment text as JSON ({"text": "..."}); the raw string is
+    // the fallback for rows that don't parse.
+    commentBody(rawText) {
+        try {
+            const parsed = JSON.parse(rawText);
+            if (typeof parsed?.text === 'string')
+                return parsed.text;
+        }
+        catch {
+            // fall through
+        }
+        return rawText;
+    }
+    buildVideo(watch) {
+        // A deleted/private video's entry degrades to the bare watch URL in place
+        // of a title — boilerplate, not the video's name (mirrors the API path's
+        // placeholder handling).
+        const unavailable = !watch.title || watch.title === watch.titleUrl;
+        return {
+            ...this.buildBareVideo(watch.videoId),
+            ...(!unavailable && { name: watch.title }),
+            ...(watch.channelId && {
+                author: [this.buildChannelAgent(watch.channelId, watch.channelName)],
+            }),
+        };
+    }
+    buildBareVideo(videoId) {
+        return {
+            '@type': 'VideoObject',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: videoId,
+            url: videoUrl(videoId),
+        };
+    }
+    buildCollection(playlistId, opts) {
+        return {
+            '@type': 'Collection',
+            '@key': ['@type', 'source', 'sourceId'],
+            source: 'youtube',
+            sourceId: playlistId,
+            url: playlistUrl(playlistId),
+            ...(opts.name && { name: opts.name }),
+            // The API reports privacyStatus lowercase; Takeout capitalizes. The
+            // schema's visibility tokens are lowercase, and a casing mismatch would
+            // flip-flop the run on every alternating import.
+            ...(opts.visibility && { visibility: opts.visibility.toLowerCase() }),
+        };
+    }
+    // A YouTube channel as an Agent — the same shape (and so the same keyset)
+    // as the API transformer's channels, with the canonical https url in place
+    // of Takeout's http:// links.
+    buildChannelAgent(channelId, name) {
+        return {
+            '@type': 'Agent',
+            '@key': ['@type', 'source', 'handle'],
+            source: 'youtube',
+            handle: channelId,
+            url: channelUrl(channelId),
+            ...(name && { name }),
+        };
+    }
+    // Watch/search actions key on the acting handle like the API LikeAction; an
+    // archive without channels/channel.csv has no handle, so the key drops to
+    // the remaining components rather than referencing a missing field.
+    agentScopedKey(agent, objectComponent) {
+        return agent.handle
+            ? ['@type', 'source', 'agent.handle', objectComponent, 'timestamp']
+            : ['@type', 'source', objectComponent, 'timestamp'];
+    }
+    // The account owner from channels/channel.csv (a real channel id, the same
+    // handle keyset as the API self, so the two merge directly); an archive
+    // missing that file falls back to the per-source singleton, still folded
+    // into the person through @me.
+    resolveSelf(record) {
+        const self = record.context?.selfChannel;
+        if (self?.id) {
+            return {
+                ...selfAgent({
+                    type: 'Agent',
+                    source: 'youtube',
+                    handle: self.id,
+                    name: self.title,
+                }),
+                url: channelUrl(self.id),
+            };
+        }
+        return selfAgent({ type: 'Agent', source: 'youtube' });
+    }
+}

package/dist/connectors/YouTubeTransformer.d.ts

@@ -0,0 +1,15 @@
+import { ChronicleTransformer, Record } from '@chronicle.app/etl';
+import { ActionAndChildren } from '@chronicle.app/schema';
+export default class YouTubeTransformer extends ChronicleTransformer {
+    transform(record: Record): Promise<ActionAndChildren[]>;
+    private buildLikeAction;
+    private buildFollowAction;
+    private buildPlaylistCreateAction;
+    private buildPlaylistAddAction;
+    private buildUploadPublishAction;
+    private buildCollection;
+    private buildVideo;
+    private buildChannelAgent;
+    private resolveSelf;
+}
+//# sourceMappingURL=YouTubeTransformer.d.ts.map