@chronicle.app/youtube 0.4.0 → 0.5.0

package/README.md

@@ -10,7 +10,18 @@ Extract liked videos and channel subscriptions from YouTube via the Data API v3
 - **Uploads**: Extract videos published to your own channel
 - **OAuth Authentication**: Google OAuth 2.0 with offline refresh tokens
 
-Watch and search history are not exposed by the Data API (Google Takeout only) and are out of scope for this plugin.
+Watch and search history aren't in the Data API. The `takeout` strategy reads
+them, with the rest of the library, from a Google Takeout:
+
+```bash
+chronicle extract youtube --strategy takeout --input ~/Downloads/takeout-20260725T000000Z-001.zip
+```
+
+It takes the Takeout as a zip, one part of a split Takeout, the unzipped folder,
+or its `YouTube and YouTube Music` folder, and reads a zip in place.
+`chronicle scan <path>` finds a Takeout in a folder of exports. It reads history
+in HTML, as Takeout writes it by default; a Takeout with its history in JSON
+isn't read yet.
 
 ## Setup
 
@@ -47,12 +58,12 @@ chronicle extract youtube --type uploads
 ## Data Mapping
 
 - Liked video → `LikeAction` by the account owner (`@me`) on a `VideoObject` (title, description, duration, thumbnail, uploading channel as author)
-- Subscription → `FollowAction` on the channel as an `Agent`, keyed on the channel id so re-runs supersede
+- Subscription → `FollowAction` on the channel as an `Agent`, keyed by the channel it follows so re-runs supersede
 - Playlist → `Collection` (with `visibility` from the privacy status), created via a `CreateAction` at the playlist's real creation time; each video in it → `AddAction` keyed on the playlist item id with the `Collection` as `targetCollection`
 - Upload → `PublishAction` with the account owner as agent and the `VideoObject` as result
 - Channels are `Agent`s keyed on their YouTube channel id
 
 Tests run the pipeline against a fake Data API served on 127.0.0.1 with a fake
-access token, and against a small synthetic Takeout folder in a temporary
-directory. They never call Google, read stored credentials, or read the host's
+access token, and against a small synthetic Takeout, unzipped and zipped, in a
+temporary directory. They never call Google, read stored credentials, or read the host's
 data or accounts.

package/dist/connectors/YouTubeTakeoutExtractor.d.ts

@@ -1,4 +1,4 @@
-import { Extractor, Record } from '@chronicle.app/etl';
+import { Extractor, Record, type Survey, type SurveyOptions } from '@chronicle.app/etl';
 import { z } from 'zod';
 import YouTubeTakeoutTransformer from './YouTubeTakeoutTransformer.js';
 export interface TakeoutSelfChannel {
@@ -72,16 +72,30 @@ export declare class YouTubeTakeoutExtractor extends Extractor<typeof YouTubeTak
         until?: Date | undefined;
         input?: string | undefined;
     }>;
+    private files;
+    /** Where the YouTube folder is in the export, as a path prefix. */
+    private prefix;
     private archiveAsOf;
     private selfChannel;
+    /** The path in the export of a file in the YouTube folder. */
     private inputPath;
     private fileExists;
+    private readCsv;
     setup(): Promise<void>;
+    teardown(): Promise<void>;
     protected asOfTime(): Date;
     private inRange;
     /** `[playlist title, csv path]` for each per-playlist membership file. */
     private playlistItemFiles;
     extract(): AsyncGenerator<Record>;
+    /**
+     * The records a run reads and when each happened, from every section: the
+     * archive has no index, so each file is read. Subscriptions are as of the
+     * archive, and watches say how many videos they were of.
+     */
+    survey(options?: SurveyOptions): Promise<Survey>;
+    /** Each section's records, in the order a run reads them, as `[record type, data]`. */
+    private readSections;
     private extractWatches;
     private extractSearches;
     private extractSubscriptions;

package/dist/connectors/YouTubeTakeoutExtractor.js

@@ -1,10 +1,14 @@
-import { Extractor } from '@chronicle.app/etl';
-import { readdir, readFile, stat } from 'node:fs/promises';
-import { basename, join } from 'node:path';
+import { Extractor, InputNotFound, countMonth, openExport, } from '@chronicle.app/etl';
 import { z } from 'zod';
-import { readTakeoutCsv } from '../takeout/csv.js';
+import { parseTakeoutCsv } from '../takeout/csv.js';
 import { parseActivityHtml, videoIdFromUrl } from '../takeout/parseActivityHtml.js';
+import { ytTime } from '../utils/time.js';
 import YouTubeTakeoutTransformer from './YouTubeTakeoutTransformer.js';
+// The folder Takeout puts YouTube's files in.
+const PRODUCT = 'YouTube and YouTube Music';
+// Where that folder can be in an export: the export itself, as an unpacked
+// folder was passed before, in the Takeout folder, or in the zip that holds it.
+const PREFIXES = ['', `${PRODUCT}/`, `Takeout/${PRODUCT}/`];
 // Relative paths of the archive files this extractor reads.
 const FILES = {
     watch: 'history/watch-history.html',
@@ -15,6 +19,16 @@ const FILES = {
     comments: 'comments/comments.csv',
     channel: 'channels/channel.csv',
 };
+// The time the transformer dates each kind by. Takeout doesn't say when a
+// subscription began, so it has none.
+const DATED = {
+    watches: data => data.time,
+    searches: data => data.time,
+    playlists: data => data.createdAt,
+    'playlist-items': data => data.addedAt,
+    uploads: data => data.publishedAt,
+    comments: data => data.createdAt,
+};
 export class YouTubeTakeoutExtractor extends Extractor {
     static source = 'youtube';
     static description = 'Watch and search history and library from Takeout';
@@ -40,49 +54,63 @@ export class YouTubeTakeoutExtractor extends Extractor {
     // flags across a source's extractors, so a schema-required flag here would
     // also gate the API extractors that share the youtube source.
     static schema = Extractor.schema.extend({
-        input: z.string().optional().describe('Path to the Takeout "YouTube and YouTube Music" folder'),
+        input: z
+            .string()
+            .optional()
+            .describe(`The Takeout, as a folder or zip, or its "${PRODUCT}" folder`),
     });
+    files = null;
+    /** Where the YouTube folder is in the export, as a path prefix. */
+    prefix = '';
     archiveAsOf = null;
     selfChannel = null;
+    /** The path in the export of a file in the YouTube folder. */
     inputPath(relative) {
-        const config = this.config;
-        if (!config.input) {
-            throw new Error('The takeout extractor needs --input: the path to the Takeout "YouTube and YouTube Music" folder');
-        }
-        return join(config.input, relative);
+        return this.prefix + relative;
     }
     async fileExists(path) {
-        try {
-            await stat(path);
-            return true;
-        }
-        catch {
-            return false;
-        }
+        return this.files.has(path);
+    }
+    async readCsv(path) {
+        return parseTakeoutCsv(await this.files.readText(path));
     }
     async setup() {
         await super.setup();
+        const config = this.config;
+        if (!config.input) {
+            throw new Error(`The takeout extractor needs --input: the Takeout, as a folder or zip, or its "${PRODUCT}" folder`);
+        }
+        this.files = await openExport(config.input);
+        const listed = new Set(await this.files.list());
+        const prefix = PREFIXES.find(p => Object.values(FILES).some(f => listed.has(p + f)));
+        if (prefix === undefined) {
+            await this.teardown();
+            throw new InputNotFound(`No ${PRODUCT} folder in the export`, {
+                path: config.input,
+                hint: `Pass the Takeout, as a folder or zip, or its "${PRODUCT}" folder with \`--input <path>\`.`,
+            });
+        }
+        this.prefix = prefix;
         // The archive is a static artifact: its sighting time is the export's,
         // not this run's, so re-running over an unchanged tree is a byte-identical
-        // no-op. Newest mtime across the files read stands in for the export time.
+        // no-op. Newest mtime across the files read stands in for the export time;
+        // in a zip, the time its index gives each file.
         const paths = [
             ...Object.values(FILES).map(f => this.inputPath(f)),
             ...(await this.playlistItemFiles()).map(([, path]) => path),
         ];
         let newest = 0;
         for (const path of paths) {
-            try {
-                newest = Math.max(newest, (await stat(path)).mtimeMs);
-            }
-            catch {
-                // Missing sections are fine; each is skipped with a warning later.
-            }
+            // Missing sections are fine; each is skipped with a warning later.
+            const modified = await this.files.modified(path);
+            if (modified)
+                newest = Math.max(newest, modified.getTime());
         }
         if (newest > 0)
             this.archiveAsOf = new Date(newest);
         const channelPath = this.inputPath(FILES.channel);
         if (await this.fileExists(channelPath)) {
-            const [row] = await readTakeoutCsv(channelPath);
+            const [row] = await this.readCsv(channelPath);
             const id = row?.['Channel ID'];
             if (id) {
                 this.selfChannel = {
@@ -95,6 +123,11 @@ export class YouTubeTakeoutExtractor extends Extractor {
             this.logger.warn('channels/channel.csv missing or empty — the account owner falls back to the per-source singleton self');
         }
     }
+    async teardown() {
+        await this.files?.close();
+        this.files = null;
+        await super.teardown();
+    }
     asOfTime() {
         return this.archiveAsOf ?? super.asOfTime();
     }
@@ -108,44 +141,74 @@ export class YouTubeTakeoutExtractor extends Extractor {
     }
     /** `[playlist title, csv path]` for each per-playlist membership file. */
     async playlistItemFiles() {
-        const dir = this.inputPath('playlists');
-        try {
-            const files = await readdir(dir);
-            return files
-                .filter(f => f.endsWith('-videos.csv'))
-                .map(f => [basename(f, '-videos.csv'), join(dir, f)]);
-        }
-        catch {
-            return [];
-        }
+        const dir = this.inputPath('playlists/');
+        return (await this.files.list(dir))
+            .map(path => path.slice(dir.length))
+            .filter(name => !name.includes('/') && name.endsWith('-videos.csv'))
+            .map(name => [name.slice(0, -'-videos.csv'.length), dir + name]);
     }
     async *extract() {
         let count = 0;
         const self = this.selfChannel;
         const emit = (recordType, data) => this.createRecord(data, { recordType, selfChannel: self });
-        for (const section of [
-            this.extractWatches(),
-            this.extractSearches(),
-            this.extractSubscriptions(),
-            this.extractPlaylists(),
-            this.extractUploads(),
-            this.extractComments(),
-        ]) {
-            for await (const [recordType, data] of section) {
-                if (this.shouldStopExtracting(count))
-                    return;
-                yield emit(recordType, data);
-                count++;
-            }
+        for await (const [recordType, data] of this.readSections()) {
+            if (this.shouldStopExtracting(count))
+                return;
+            yield emit(recordType, data);
+            count++;
         }
     }
+    /**
+     * The records a run reads and when each happened, from every section: the
+     * archive has no index, so each file is read. Subscriptions are as of the
+     * archive, and watches say how many videos they were of.
+     */
+    async survey(options = {}) {
+        const kinds = {};
+        for (const recordType of YouTubeTakeoutExtractor.recordTypes) {
+            kinds[recordType] = {
+                count: 0,
+                ...(options.byMonth && DATED[recordType] && { byMonth: {} }),
+            };
+        }
+        const videos = new Set();
+        for await (const [recordType, data] of this.readSections()) {
+            const kind = kinds[recordType];
+            kind.count++;
+            if (recordType === 'watches')
+                videos.add(data.videoId);
+            const dated = DATED[recordType];
+            if (!dated)
+                continue;
+            const time = ytTime(dated(data));
+            if (!kind.earliest || time < kind.earliest)
+                kind.earliest = time;
+            if (!kind.latest || time > kind.latest)
+                kind.latest = time;
+            if (kind.byMonth)
+                countMonth(kind.byMonth, time);
+        }
+        if (this.archiveAsOf)
+            kinds.subscriptions.asOf = this.archiveAsOf;
+        kinds.watches.measures = [{ key: 'videos', label: 'videos', value: videos.size }];
+        return kinds;
+    }
+    /** Each section's records, in the order a run reads them, as `[record type, data]`. */
+    async *readSections() {
+        yield* this.extractWatches();
+        yield* this.extractSearches();
+        yield* this.extractSubscriptions();
+        yield* this.extractPlaylists();
+        yield* this.extractUploads();
+        yield* this.extractComments();
+    }
     async *extractWatches() {
         const path = this.inputPath(FILES.watch);
         if (!(await this.fileExists(path))) {
             this.logger.warn(`No watch history at ${FILES.watch} — skipping`);
             return;
         }
-        const entries = parseActivityHtml(await readFile(path, 'utf-8'));
+        const entries = parseActivityHtml(await this.files.readText(path));
         let ads = 0;
         let noise = 0;
         let unparseable = 0;
@@ -186,7 +249,7 @@ export class YouTubeTakeoutExtractor extends Extractor {
             this.logger.warn(`No search history at ${FILES.search} — skipping`);
             return;
         }
-        const entries = parseActivityHtml(await readFile(path, 'utf-8'));
+        const entries = parseActivityHtml(await this.files.readText(path));
         let skipped = 0;
         let emitted = 0;
         for (const entry of entries) {
@@ -211,7 +274,7 @@ export class YouTubeTakeoutExtractor extends Extractor {
         const path = this.inputPath(FILES.subscriptions);
         if (!(await this.fileExists(path)))
             return;
-        for (const row of await readTakeoutCsv(path)) {
+        for (const row of await this.readCsv(path)) {
             const channelId = row['Channel Id'];
             if (!channelId)
                 continue;
@@ -227,7 +290,7 @@ export class YouTubeTakeoutExtractor extends Extractor {
         if (!(await this.fileExists(path)))
             return;
         const byTitle = new Map();
-        for (const row of await readTakeoutCsv(path)) {
+        for (const row of await this.readCsv(path)) {
             const playlistId = row['Playlist ID'];
             const createdAt = row['Playlist Create Timestamp'];
             if (!playlistId || !createdAt)
@@ -246,10 +309,10 @@ export class YouTubeTakeoutExtractor extends Extractor {
         for (const [title, itemPath] of await this.playlistItemFiles()) {
             const playlist = byTitle.get(title);
             if (!playlist) {
-                this.logger.warn(`Playlist file "${basename(itemPath)}" has no matching row in playlists.csv — skipping`);
+                this.logger.warn(`Playlist file "${itemPath.split('/').at(-1)}" has no matching row in playlists.csv — skipping`);
                 continue;
             }
-            for (const row of await readTakeoutCsv(itemPath)) {
+            for (const row of await this.readCsv(itemPath)) {
                 const videoId = row['Video ID'];
                 const addedAt = row['Playlist Video Creation Timestamp'];
                 if (!videoId || !addedAt)
@@ -269,7 +332,7 @@ export class YouTubeTakeoutExtractor extends Extractor {
         if (!(await this.fileExists(path)))
             return;
         let drafts = 0;
-        for (const row of await readTakeoutCsv(path)) {
+        for (const row of await this.readCsv(path)) {
             const videoId = row['Video ID'];
             if (!videoId)
                 continue;
@@ -297,7 +360,7 @@ export class YouTubeTakeoutExtractor extends Extractor {
         const path = this.inputPath(FILES.comments);
         if (!(await this.fileExists(path)))
             return;
-        for (const row of await readTakeoutCsv(path)) {
+        for (const row of await this.readCsv(path)) {
             const commentId = row['Comment ID'];
             const videoId = row['Video ID'];
             const createdAt = row['Comment Create Timestamp'];

package/dist/connectors/YouTubeTakeoutTransformer.js

@@ -78,9 +78,9 @@ export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
         const subscription = record.data;
         const action = {
             '@type': 'FollowAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the channel it follows, as the API path keys it.
+            '@key': ['@type', 'source', 'object.handle'],
             source: 'youtube',
-            sourceId: subscription.channelId,
             agent: this.resolveSelf(record),
             object: this.buildChannelAgent(subscription.channelId, subscription.channelTitle),
         };
@@ -90,9 +90,9 @@ export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
         const playlist = record.data;
         const action = {
             '@type': 'CreateAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the playlist it made, as the API path keys it.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'youtube',
-            sourceId: playlist.playlistId,
             timestamp: ytTime(playlist.createdAt),
             agent: this.resolveSelf(record),
             result: this.buildCollection(playlist.playlistId, {
@@ -134,12 +134,12 @@ export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
         };
         const action = {
             '@type': 'PublishAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the video it published, as the API path keys it.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'youtube',
-            sourceId: upload.videoId,
             timestamp: ytTime(upload.publishedAt),
             agent: this.resolveSelf(record),
-            object: video,
+            result: video,
         };
         return [action];
     }
@@ -158,9 +158,9 @@ export default class YouTubeTakeoutTransformer extends ChronicleTransformer {
         };
         const action = {
             '@type': 'AnnotateAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the comment it made.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'youtube',
-            sourceId: data.commentId,
             timestamp: ytTime(data.createdAt),
             agent,
             object: video,

package/dist/connectors/YouTubeTransformer.js

@@ -60,9 +60,9 @@ export default class YouTubeTransformer extends ChronicleTransformer {
         const { channelId } = subscription.snippet.resourceId;
         const action = {
             '@type': 'FollowAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the channel it follows.
+            '@key': ['@type', 'source', 'object.handle'],
             source: 'youtube',
-            sourceId: channelId,
             timestamp: ytTime(subscription.snippet.publishedAt),
             agent: this.resolveSelf(record),
             object: this.buildChannelAgent(channelId, {
@@ -79,9 +79,9 @@ export default class YouTubeTransformer extends ChronicleTransformer {
         const playlist = record.data;
         const action = {
             '@type': 'CreateAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the playlist it made.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'youtube',
-            sourceId: playlist.id,
             timestamp: ytTime(playlist.snippet.publishedAt),
             agent: this.resolveSelf(record),
             result: this.buildCollection(playlist),
@@ -118,12 +118,12 @@ export default class YouTubeTransformer extends ChronicleTransformer {
         delete video.datePublished;
         const action = {
             '@type': 'PublishAction',
-            '@key': ['@type', 'source', 'sourceId'],
+            // Keyed by the video it published.
+            '@key': ['@type', 'source', 'result.sourceId'],
             source: 'youtube',
-            sourceId: upload.item.snippet.resourceId.videoId,
             timestamp: ytTime(upload.item.snippet.publishedAt),
             agent: this.resolveSelf(record),
-            object: video,
+            result: video,
         };
         return [action];
     }

package/dist/takeout/csv.d.ts

@@ -1,4 +1,4 @@
 /** A Takeout CSV as header-keyed rows. Fields regularly carry embedded
  * newlines (descriptions, comment JSON), so this is never line-splittable. */
-export declare function readTakeoutCsv(path: string): Promise<Record<string, string>[]>;
+export declare function parseTakeoutCsv(content: string): Record<string, string>[];
 //# sourceMappingURL=csv.d.ts.map

package/dist/takeout/csv.js

@@ -1,9 +1,7 @@
-import { readFile } from 'node:fs/promises';
 import { parse } from 'csv-parse/sync';
 /** A Takeout CSV as header-keyed rows. Fields regularly carry embedded
  * newlines (descriptions, comment JSON), so this is never line-splittable. */
-export async function readTakeoutCsv(path) {
-    const content = await readFile(path, 'utf-8');
+export function parseTakeoutCsv(content) {
     return parse(content, {
         columns: true,
         bom: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chronicle.app/youtube",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "files": [
@@ -22,9 +22,9 @@
   "type": "module",
   "description": "Chronicle plugin for extracting data from YouTube",
   "devDependencies": {
-    "@chronicle.app/etl": "0.4.0",
-    "@chronicle.app/schema": "0.4.0",
-    "@chronicle.app/tsconfig": "0.4.0",
+    "@chronicle.app/etl": "0.5.0",
+    "@chronicle.app/schema": "0.5.0",
+    "@chronicle.app/tsconfig": "0.5.0",
     "@types/node": "^22.10.1",
     "typescript": "^5.7.2"
   },
@@ -39,7 +39,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
-    "@chronicle.app/auth": "0.4.0",
+    "@chronicle.app/auth": "0.5.0",
     "axios": "^1.7.9",
     "csv-parse": "^5.6.0",
     "zod": "^3.24.1"
@@ -75,6 +75,20 @@
           },
           "takeout": {
             "delivery": "export",
+            "layouts": [
+              {
+                "files": [
+                  "YouTube and YouTube Music/history/watch-history.html"
+                ]
+              },
+              {
+                "name": "JSON",
+                "files": [
+                  "YouTube and YouTube Music/history/watch-history.json"
+                ],
+                "supported": false
+              }
+            ],
             "recordTypes": [
               "watches",
               "searches",