@pagepocket/plugin-yt-dlp 0.8.0

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./yt-dlp-plugin.js";
+export { default as YtDlpPlugin } from "./yt-dlp-plugin.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from "./yt-dlp-plugin.js";
+export { default as YtDlpPlugin } from "./yt-dlp-plugin.js";

package/dist/utils/vtree.d.ts

@@ -0,0 +1,3 @@
+import type { FileTree, FileTreeDirectory, FileTreeEntry } from "@pagepocket/lib";
+export declare const withInsertedFile: (root: FileTreeDirectory, relPath: string, fileEntry: FileTreeEntry) => FileTreeDirectory;
+export declare const emptyFileTree: () => FileTree;

package/dist/utils/vtree.js

@@ -0,0 +1,39 @@
+const ensureDirectory = (dir, name) => {
+    const existingIndex = dir.entries.findIndex((e) => e.kind === "directory" && e.path === name);
+    if (existingIndex >= 0) {
+        return { dir: dir.entries[existingIndex], entryIndex: existingIndex };
+    }
+    const nextDir = { kind: "directory", path: name, entries: [] };
+    return { dir: nextDir, entryIndex: -1 };
+};
+export const withInsertedFile = (root, relPath, fileEntry) => {
+    const clean = relPath.replace(/^\/+/, "");
+    const parts = clean.split("/").filter(Boolean);
+    if (parts.length === 0) {
+        return root;
+    }
+    const filename = parts[parts.length - 1];
+    const dirs = parts.slice(0, -1);
+    const insert = (dir, remainingDirs) => {
+        if (remainingDirs.length === 0) {
+            const nextEntries = dir.entries.filter((e) => !(e.kind === "file" && e.path === filename));
+            nextEntries.push(fileEntry);
+            return { ...dir, entries: nextEntries };
+        }
+        const nextName = remainingDirs[0];
+        const { dir: childDir, entryIndex } = ensureDirectory(dir, nextName);
+        const nextChild = insert(childDir, remainingDirs.slice(1));
+        const nextEntries = [...dir.entries];
+        if (entryIndex >= 0) {
+            nextEntries[entryIndex] = nextChild;
+        }
+        else {
+            nextEntries.push(nextChild);
+        }
+        return { ...dir, entries: nextEntries };
+    };
+    return insert(root, dirs);
+};
+export const emptyFileTree = () => ({
+    root: { kind: "directory", path: "", entries: [] }
+});

package/dist/utils/yt-dlp-job-manager.d.ts

@@ -0,0 +1,19 @@
+type SetupValue = {
+    dateString: string;
+    dirPrefix: string;
+};
+export type YoutubeJob = {
+    id: string;
+    url: string;
+    relPath: string;
+};
+export declare const parseYoutubeEmbedSrc: (src: string) => {
+    id: string;
+    url: string;
+} | null;
+export declare class YtDlpJobManager {
+    createSetupValue(now?: Date): SetupValue;
+    buildVideoRelPath(setupValue: SetupValue, videoId: string): string;
+    discoverJobsFromHtml(htmlString: string, setupValue: SetupValue): YoutubeJob[];
+}
+export type { SetupValue };

package/dist/utils/yt-dlp-job-manager.js

@@ -0,0 +1,65 @@
+import { formatCompactDateTime } from "@pagepocket/shared";
+export const parseYoutubeEmbedSrc = (src) => {
+    try {
+        const url = new URL(src, "https://www.youtube.com");
+        const host = url.hostname;
+        if (!host.endsWith("youtube.com") && !host.endsWith("youtube-nocookie.com")) {
+            return null;
+        }
+        const parts = url.pathname.split("/").filter(Boolean);
+        const embedIndex = parts.indexOf("embed");
+        if (embedIndex === -1) {
+            return null;
+        }
+        const id = parts[embedIndex + 1];
+        if (!id) {
+            return null;
+        }
+        return { id, url: `https://www.youtube.com/watch?v=${id}` };
+    }
+    catch {
+        return null;
+    }
+};
+const uniqueById = (jobs) => {
+    const seen = new Set();
+    const out = [];
+    for (const job of jobs) {
+        if (seen.has(job.id)) {
+            continue;
+        }
+        seen.add(job.id);
+        out.push(job);
+    }
+    return out;
+};
+export class YtDlpJobManager {
+    createSetupValue(now = new Date()) {
+        const dateString = formatCompactDateTime(now);
+        const dirPrefix = `yt-videos/${dateString}`;
+        return { dateString, dirPrefix };
+    }
+    buildVideoRelPath(setupValue, videoId) {
+        return `${setupValue.dirPrefix}/${videoId}.mp4`;
+    }
+    discoverJobsFromHtml(htmlString, setupValue) {
+        const jobs = [];
+        const iframeSrcRegex = /<iframe\b[^>]*\bsrc\s*=\s*"([^"]+)"[^>]*>/gi;
+        for (const m of htmlString.matchAll(iframeSrcRegex)) {
+            const src = m[1];
+            if (!src) {
+                continue;
+            }
+            const parsed = parseYoutubeEmbedSrc(src);
+            if (!parsed) {
+                continue;
+            }
+            jobs.push({
+                id: parsed.id,
+                url: parsed.url,
+                relPath: this.buildVideoRelPath(setupValue, parsed.id)
+            });
+        }
+        return uniqueById(jobs);
+    }
+}

package/dist/utils/ytdlp.d.ts

@@ -0,0 +1,2 @@
+import type { YoutubeJob } from "./yt-dlp-job-manager.js";
+export declare const downloadVideosAsBytes: (jobs: YoutubeJob[]) => Promise<Map<string, Uint8Array<ArrayBufferLike>>>;

package/dist/utils/ytdlp.js

@@ -0,0 +1,33 @@
+export const downloadVideosAsBytes = async (jobs) => {
+    const { YtDlp, helpers } = await import("ytdlp-nodejs");
+    const fs = await import("node:fs/promises");
+    const pathMod = await import("node:path");
+    const os = await import("node:os");
+    const ytdlp = new YtDlp();
+    await helpers.downloadYtDlp();
+    await helpers.downloadFFmpeg();
+    const bytesById = new Map();
+    await Promise.all(jobs.map(async (job) => {
+        const tmpDir = await fs.mkdtemp(pathMod.join(os.tmpdir(), "pagepocket-ytdlp-"));
+        try {
+            const outTemplate = pathMod.join(tmpDir, "%(id)s.%(ext)s");
+            await ytdlp.downloadAsync(job.url, {
+                format: { filter: "mergevideo", quality: "highest", type: "mp4" },
+                output: outTemplate
+            });
+            const entries = await fs.readdir(tmpDir);
+            const prefix = `${job.id}.`;
+            const filename = entries.find((e) => e.startsWith(prefix));
+            if (!filename) {
+                return;
+            }
+            const abs = pathMod.join(tmpDir, filename);
+            const bytes = await fs.readFile(abs);
+            bytesById.set(job.id, new Uint8Array(bytes));
+        }
+        finally {
+            await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => { });
+        }
+    }));
+    return bytesById;
+};

package/dist/yt-dlp-plugin.d.ts

@@ -0,0 +1,9 @@
+import type { Plugin, PluginHost, UnitContributeContext, UnitPatch, UnitRuntime } from "@pagepocket/lib";
+import { type SetupValue } from "./utils/yt-dlp-job-manager.js";
+export default class YtDlpPlugin implements Plugin<SetupValue> {
+    readonly name = "plugin:yt-dlp";
+    static readonly unitId = "buildSnapshot";
+    private jobManager;
+    setup(host: PluginHost): SetupValue;
+    contribute(ctx: UnitContributeContext, _rt: UnitRuntime): Promise<UnitPatch>;
+}

package/dist/yt-dlp-plugin.js

@@ -0,0 +1,81 @@
+import { emptyFileTree, withInsertedFile } from "./utils/vtree.js";
+import { YtDlpJobManager, parseYoutubeEmbedSrc } from "./utils/yt-dlp-job-manager.js";
+import { downloadVideosAsBytes } from "./utils/ytdlp.js";
+const buildFilesForVideos = (input) => {
+    let root = emptyFileTree().root;
+    for (const job of input.jobs) {
+        const bytes = input.bytesById.get(job.id);
+        if (!bytes) {
+            continue;
+        }
+        const filename = job.relPath.split("/").pop() ?? `${job.id}.mp4`;
+        const entry = {
+            kind: "file",
+            path: filename,
+            source: { kind: "bytes", data: bytes }
+        };
+        root = withInsertedFile(root, job.relPath, entry);
+    }
+    return { root };
+};
+class YtDlpPlugin {
+    constructor() {
+        this.name = "plugin:yt-dlp";
+        this.jobManager = new YtDlpJobManager();
+    }
+    setup(host) {
+        const setupValue = this.jobManager.createSetupValue();
+        const rules = [
+            {
+                name: "yt-dlp:replace-youtube-embeds",
+                query: "iframe[src*='youtube.com/embed'], iframe[src*='youtube-nocookie.com/embed']",
+                run: async (ctx) => {
+                    const $el = ctx.$el;
+                    const src = $el.attr("src");
+                    if (!src) {
+                        return;
+                    }
+                    const parsed = parseYoutubeEmbedSrc(src);
+                    if (!parsed) {
+                        return;
+                    }
+                    const relPath = this.jobManager.buildVideoRelPath(setupValue, parsed.id);
+                    return {
+                        type: "replaceWithHtml",
+                        html: `<video controls src="/${relPath}"></video>`
+                    };
+                }
+            }
+        ];
+        host.elements.contribute({ type: "plugin", name: this.name }, rules);
+        return setupValue;
+    }
+    async contribute(ctx, _rt) {
+        const setupValue = ctx.setupValue;
+        if (!setupValue) {
+            return {};
+        }
+        const html = ctx.value.html;
+        if (!html?.htmlString) {
+            return {};
+        }
+        const jobs = this.jobManager.discoverJobsFromHtml(html.htmlString, setupValue);
+        if (jobs.length === 0) {
+            return {};
+        }
+        try {
+            const bytesById = await downloadVideosAsBytes(jobs);
+            if (bytesById.size === 0) {
+                return {};
+            }
+            const files = buildFilesForVideos({ jobs, bytesById });
+            return { files };
+        }
+        catch (e) {
+            console.error(e);
+        }
+        return {};
+    }
+}
+YtDlpPlugin.unitId = "buildSnapshot";
+export default YtDlpPlugin;

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@pagepocket/plugin-yt-dlp",
+  "version": "0.8.0",
+  "description": "PagePocket plugin: download YouTube embeds and replace with <video>",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "ISC",
+  "dependencies": {
+    "ytdlp-nodejs": "^3.4.2",
+    "@pagepocket/lib": "0.8.0",
+    "@pagepocket/shared": "0.8.0",
+    "@pagepocket/contracts": "0.8.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.5"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "node -e \"process.exit(0)\""
+  }
+}