@wrenautomation/birdwatch 0.0.0-stage → 0.30.0

package/dist/sources/sites.js

@@ -0,0 +1,285 @@
+/**
+ * Readers for one site each, over that site's own read route on an autobrowse worker (the same
+ * wire as the ladder). Each claims only its site's URLs and goes before the general openers
+ * (`first`): the route answers the thread or the post as data, where a fetched page is a wall or a
+ * JavaScript shell.
+ *
+ * `reddit`: a thread (`/comments/{id}`) through `reddit-public` (signed out, the RSS feed), then,
+ * when the caller named an account (`BIRDWATCH_ACCOUNTS=reddit=reddit@alt`), through `reddit`
+ * signed in as it. Signed out goes first: read volume on a signed-in account is what Reddit
+ * flags. A follow from a user's page (`/user/{name}`) lists their top posts and comments in one
+ * read (`/user/{name}/overview`), signed out. `facebook`: one public group post through `fb-public`, always signed out (Wren's Meta
+ * login runs the ads). Design: autobrowse `designs/2026-10-28-signed-in-reads.md`.
+ *
+ * A route that answered no (a 404, a spent cap, a refusal) is a miss with its reason and the
+ * worker's access code; a worker that did not answer, or a busy profile (503), is a Transient.
+ * A signed-out miss reads signed in, except a spent cap or a broken site (`rate_limited`,
+ * `upstream_broken`): signing in fixes neither.
+ */
+import { Transient } from "@wrenautomation/models";
+import { junkReason } from "./junk.js";
+import { LadderCallError } from "./ladder.js";
+/** `reddit=reddit@alt,linkedin=linkedin@wren`: the account each site reader signs in as. */
+export function accountsFrom(spec) {
+    const out = new Map();
+    for (const pair of (spec ?? "").split(",")) {
+        const [site, account] = pair.split("=").map((s) => s.trim());
+        if (site && account)
+            out.set(site, account);
+    }
+    return out;
+}
+const hostOf = (url) => {
+    try {
+        return new URL(url);
+    }
+    catch {
+        return null;
+    }
+};
+/** A Reddit thread's id, and the comment's when the URL is one comment's permalink. */
+export function redditThread(url) {
+    const u = hostOf(url);
+    if (!u)
+        return null;
+    const host = u.hostname.toLowerCase();
+    if (host === "redd.it") {
+        const m = /^\/([a-z0-9]{1,12})\/?$/i.exec(u.pathname);
+        return m?.[1] ? { article: m[1].toLowerCase() } : null;
+    }
+    if (host !== "reddit.com" && !host.endsWith(".reddit.com"))
+        return null;
+    // `/comments/{id}`, then the title's slug, then a comment's id: each after the id is optional.
+    const m = /\/comments\/([a-z0-9]{1,12})(?:\/[^/]*(?:\/([a-z0-9]{1,12}))?)?\/?$/i.exec(u.pathname);
+    if (!m?.[1])
+        return null;
+    return { article: m[1].toLowerCase(), ...(m[2] ? { comment: m[2].toLowerCase() } : {}) };
+}
+/** A Reddit user's name from their page (`/user/{name}`, `/u/{name}`). */
+export function redditUser(url) {
+    const u = hostOf(url);
+    if (!u)
+        return null;
+    const host = u.hostname.toLowerCase();
+    if (host !== "reddit.com" && !host.endsWith(".reddit.com"))
+        return null;
+    const m = /^\/(?:user|u)\/([A-Za-z0-9_-]{3,20})(?:\/|$)/.exec(u.pathname);
+    return m?.[1] ?? null;
+}
+/** A user's overview (posts and comments mixed) as hits: each its permalink, title and words. */
+export function userHits(body) {
+    return things(body).flatMap((t) => {
+        const link = str(t.permalink);
+        if (!link)
+            return [];
+        const words = (str(t.selftext) || str(t.body)).trim();
+        const title = str(t.title) || str(t.link_title);
+        const created = typeof t.created_utc === "number" ? t.created_utc : null;
+        return [
+            {
+                url: `https://www.reddit.com${link}`,
+                ...(title ? { title } : {}),
+                ...(str(t.subreddit) ? { channel: `r/${str(t.subreddit)}` } : {}),
+                ...(created ? { date: new Date(created * 1000).toISOString().slice(0, 10) } : {}),
+                ...(words ? { snippet: words.slice(0, 300) } : {}),
+            },
+        ];
+    });
+}
+/** A Facebook group post's group and post ids. */
+export function facebookPost(url) {
+    const u = hostOf(url);
+    if (!u)
+        return null;
+    const host = u.hostname.toLowerCase();
+    if (host !== "facebook.com" && !host.endsWith(".facebook.com"))
+        return null;
+    const m = /^\/groups\/([\w.-]{2,100})\/(?:posts|permalink)\/(\d{5,25})\/?$/.exec(u.pathname);
+    return m?.[1] && m[2] ? { group: m[1], post: m[2] } : null;
+}
+const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+const str = (v) => (typeof v === "string" ? v : "");
+const things = (listing) => {
+    const data = isObj(listing) && isObj(listing.data) ? listing.data : {};
+    const children = Array.isArray(data.children) ? data.children : [];
+    return children.filter(isObj).map((c) => (isObj(c.data) ? c.data : {}));
+};
+/** Reddit says the thread is not there (404): no account would read it either. */
+class Gone extends Error {
+}
+/** The worker refused the call; `code` is its access code when it sent one. */
+class Refused extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+    }
+}
+/** Codes a signed-in read would not fix: a spent cap (it would spend the account instead) or a broken site. */
+const NOT_FOR_SIGN_IN = new Set(["rate_limited", "upstream_broken"]);
+/** A thread (`[post listing, comments listing]`) as text: the post, then each comment. */
+export function threadDocument(url, body) {
+    if (isObj(body) && "error" in body) {
+        const said = `reddit: ${str(body.message) || `error ${String(body.error)}`}`;
+        throw body.error === 404 ? new Gone(said) : new Error(said);
+    }
+    const [postListing, commentListing] = Array.isArray(body) ? body : [body, null];
+    const post = things(postListing)[0];
+    if (!post)
+        throw new Error("reddit: the thread came back without its post");
+    const head = [
+        `r/${str(post.subreddit)} · u/${str(post.author)}`,
+        typeof post.score === "number" ? `${post.score} points` : "",
+        typeof post.num_comments === "number" ? `${post.num_comments} comments` : "",
+    ]
+        .filter(Boolean)
+        .join(" · ");
+    const selftext = str(post.selftext).trim();
+    const link = post.is_self === false && str(post.url) ? `Link: ${str(post.url)}` : "";
+    const comments = things(commentListing)
+        .filter((c) => str(c.body).trim())
+        .map((c) => {
+        const score = typeof c.score === "number" ? ` (${c.score})` : "";
+        return `u/${str(c.author) || "[deleted]"}${score}: ${str(c.body).trim()}`;
+    });
+    const text = [
+        str(post.title),
+        head,
+        selftext || link,
+        comments.length ? `Comments:\n\n${comments.join("\n\n")}` : "",
+    ]
+        .filter(Boolean)
+        .join("\n\n");
+    const created = typeof post.created_utc === "number" ? post.created_utc : null;
+    return {
+        url,
+        media: "text",
+        ...(str(post.title) ? { title: str(post.title) } : {}),
+        text,
+        ...(str(post.subreddit) ? { channel: `r/${str(post.subreddit)}` } : {}),
+        ...(created ? { date: new Date(created * 1000).toISOString().slice(0, 10) } : {}),
+    };
+}
+const POST_KEYS = ["author", "badge", "time", "title"];
+/** fb-public's `{ post, comments }` as text: the post's whole text, then each comment. */
+export function groupPostDocument(url, body) {
+    const post = isObj(body) && isObj(body.post) ? body.post : null;
+    if (!post)
+        throw new Error("facebook: the walk read no post (private, removed or a wall)");
+    const whole = str(post.all).trim() || str(post.text).trim();
+    const head = POST_KEYS.map((k) => str(post[k]).trim())
+        .filter(Boolean)
+        .join(" · ");
+    const comments = (isObj(body) && Array.isArray(body.comments) ? body.comments : [])
+        .filter(isObj)
+        .map((c) => `${str(c.author) || "someone"}: ${str(c.text).trim()}`)
+        .filter((line) => !line.endsWith(": "));
+    const text = [head, whole, comments.length ? `Comments:\n\n${comments.join("\n\n")}` : ""]
+        .filter(Boolean)
+        .join("\n\n");
+    return {
+        url,
+        media: "text",
+        ...(str(post.title) ? { title: str(post.title) } : {}),
+        text,
+    };
+}
+/** One route call; a refusal is a miss with the worker's words, no answer a Transient. */
+export async function routeCall(sites, who, site, path, input, account) {
+    try {
+        return await sites.call(site, "GET", path, input, account ? { account } : undefined);
+    }
+    catch (err) {
+        if (err instanceof LadderCallError) {
+            if (err.status === 429)
+                throw new Refused(`${who}: ${site}'s cap is spent for today`, err.code ?? "rate_limited");
+            // The profile stayed busy past the worker's wait: the read was never sent.
+            if (err.status === 503)
+                throw new Transient(`${who}: ${err.message}`);
+            throw new Refused(`${who}: ${err.code ? `${err.code}: ` : ""}${err.message}`, err.code);
+        }
+        throw new Transient(`${who}: the worker did not answer (${err.message})`);
+    }
+}
+const usable = (who, doc) => {
+    const junk = junkReason(doc);
+    if (junk)
+        throw new Error(`${who}: ${junk}`);
+    return doc;
+};
+/** Top comments asked of a thread: enough to read the room, inside the route's 500. */
+const COMMENTS = 200;
+export function redditSource(o) {
+    const read = async (url, signedIn) => {
+        const t = redditThread(url);
+        if (!t)
+            throw new Error(`reddit: not a thread: ${url}`);
+        const input = {
+            article: t.article,
+            limit: COMMENTS,
+            sort: "top",
+            ...(t.comment ? { comment: t.comment, context: 3 } : {}),
+        };
+        const who = signedIn ? `reddit as ${o.account}` : "reddit signed out";
+        const body = signedIn
+            ? await routeCall(o.sites, who, "reddit", `/comments/${t.article}`, input, o.account)
+            : await routeCall(o.sites, who, "reddit-public", `/comments/${t.article}`, input);
+        return usable(who, threadDocument(url, body));
+    };
+    return {
+        name: "reddit",
+        kind: "open",
+        cost: "browser",
+        first: true,
+        opens: (url) => redditThread(url) !== null,
+        follows: (as, url) => as === "posts" && redditUser(url) !== null,
+        async follow(_as, url, n) {
+            const username = redditUser(url);
+            if (!username)
+                throw new Error(`reddit: not a user's page: ${url}`);
+            const input = { username, limit: Math.min(n, 100), sort: "top" };
+            const body = await routeCall(o.sites, "reddit signed out", "reddit-public", `/user/${username}/overview`, input);
+            return userHits(body);
+        },
+        async open(h) {
+            if (!o.account)
+                return read(h.url, false);
+            try {
+                return await read(h.url, false);
+            }
+            catch (err) {
+                if (err instanceof Transient || err instanceof Gone)
+                    throw err;
+                if (err instanceof Refused && err.code && NOT_FOR_SIGN_IN.has(err.code))
+                    throw err;
+                // Say it: a signed-in read spends the account's standing, and the agent counts it.
+                console.warn(`${h.url}: ${err.message}; reading as ${o.account}`);
+                try {
+                    return await read(h.url, true);
+                }
+                catch (again) {
+                    if (again instanceof Transient)
+                        throw again;
+                    throw new Error(`${err.message}; ${again.message}`);
+                }
+            }
+        },
+    };
+}
+export function facebookSource(o) {
+    return {
+        name: "facebook",
+        kind: "open",
+        cost: "browser",
+        first: true,
+        opens: (url) => facebookPost(url) !== null,
+        async open(h) {
+            const p = facebookPost(h.url);
+            if (!p)
+                throw new Error(`facebook: not a group post: ${h.url}`);
+            const body = await routeCall(o.sites, "facebook", "fb-public", `/groups/${p.group}/posts/${p.post}`, p);
+            return usable("facebook", groupPostDocument(h.url, body));
+        },
+    };
+}
+//# sourceMappingURL=sites.js.map

package/dist/sources/sites.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sites.js","sourceRoot":"","sources":["../../src/sources/sites.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAKnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,4FAA4F;AAC5F,MAAM,UAAU,YAAY,CAAC,IAAwB;IACnD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI,IAAI,OAAO;YAAE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,GAAW,EAAc,EAAE;IACzC,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF,uFAAuF;AACvF,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,0BAA0B,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACtD,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,CAAC;IACD,IAAI,IAAI,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IACxE,+FAA+F;IAC/F,MAAM,CAAC,GAAG,sEAAsE,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAClG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;AAC3F,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IACxE,MAAM,CAAC,GAAG,8CAA8C,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1E,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACxB,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,QAAQ,CAAC,IAAa;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;QACzE,OAAO;YACL;gBACE,GAAG,EAAE,yBAAyB,IAAI,EAAE;gBACpC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjF,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACtC,IAAI,IAAI,KAAK,cAAc,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5E,MAAM,CAAC,GAAG,iEAAiE,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC7F,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAGD,MAAM,KAAK,GAAG,CAAC,CAAU,EAAY,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACjG,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACrE,MAAM,MAAM,GAAG,CAAC,OAAgB,EAAS,EAAE;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,OAAO,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1E,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,IAAK,SAAQ,KAAK;CAAG;AAE3B,+EAA+E;AAC/E,MAAM,OAAQ,SAAQ,KAAK;IAGd;IAFX,YACE,OAAe,EACN,IAAmB;QAE5B,KAAK,CAAC,OAAO,CAAC,CAAC;QAFN,SAAI,GAAJ,IAAI,CAAe;IAG9B,CAAC;CACF;AAED,+GAA+G;AAC/G,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC,CAAC;AAErE,0FAA0F;AAC1F,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,IAAa;IACvD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,SAAS,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;QAC7E,MAAM,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChF,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG;QACX,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;QAClD,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE;QAC5D,OAAO,IAAI,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,WAAW,CAAC,CAAC,CAAC,EAAE;KAC7E;SACE,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACjE,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,WAAW,GAAG,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAC5E,CAAC,CAAC,CAAC;IACL,MAAM,IAAI,GAAG;QACX,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;QACf,IAAI;QACJ,QAAQ,IAAI,IAAI;QAChB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;KAC/D;SACE,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,MAAM,CAAC,CAAC;IAChB,MAAM,OAAO,GAAG,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/E,OAAO;QACL,GAAG;QACH,KAAK,EAAE,MAAM;QACb,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,IAAI;QACJ,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAU,CAAC;AAEhE,0FAA0F;AAC1F,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,IAAa;IAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACnD,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,KAAK,CAAC,CAAC;IACf,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;SAChF,MAAM,CAAC,KAAK,CAAC;SACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;SAClE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvF,MAAM,CAAC,OAAO,CAAC;SACf,IAAI,CAAC,MAAM,CAAC,CAAC;IAChB,OAAO;QACL,GAAG;QACH,KAAK,EAAE,MAAM;QACb,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,IAAI;KACL,CAAC;AACJ,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAmB,EACnB,GAAW,EACX,IAAY,EACZ,IAAY,EACZ,KAAU,EACV,OAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,eAAe,EAAE,CAAC;YACnC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;gBACpB,MAAM,IAAI,OAAO,CAAC,GAAG,GAAG,KAAK,IAAI,2BAA2B,EAAE,GAAG,CAAC,IAAI,IAAI,cAAc,CAAC,CAAC;YAC5F,2EAA2E;YAC3E,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;gBAAE,MAAM,IAAI,SAAS,CAAC,GAAG,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACtE,MAAM,IAAI,OAAO,CAAC,GAAG,GAAG,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,GAAG,GAAG,gCAAiC,GAAa,CAAC,OAAO,GAAG,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,GAAa,EAAY,EAAE;IACtD,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,uFAAuF;AACvF,MAAM,QAAQ,GAAG,GAAG,CAAC;AAErB,MAAM,UAAU,YAAY,CAAC,CAA4C;IACvE,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAE,QAAiB,EAAqB,EAAE;QACvE,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QACxD,MAAM,KAAK,GAAQ;YACjB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,KAAK,EAAE,QAAQ;YACf,IAAI,EAAE,KAAK;YACX,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAC;QACF,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC;QACtE,MAAM,IAAI,GAAG,QAAQ;YACnB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC;YACrF,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;QACpF,OAAO,MAAM,CAAC,GAAG,EAAE,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IAChD,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,IAAI;QAC1C,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,IAAI;QAChE,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;YACpE,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACjE,MAAM,IAAI,GAAG,MAAM,SAAS,CAC1B,CAAC,CAAC,KAAK,EACP,mBAAmB,EACnB,eAAe,EACf,SAAS,QAAQ,WAAW,EAC5B,KAAK,CACN,CAAC;YACF,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC;YACV,IAAI,CAAC,CAAC,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC1C,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,GAAG,YAAY,SAAS,IAAI,GAAG,YAAY,IAAI;oBAAE,MAAM,GAAG,CAAC;gBAC/D,IAAI,GAAG,YAAY,OAAO,IAAI,GAAG,CAAC,IAAI,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,MAAM,GAAG,CAAC;gBACnF,mFAAmF;gBACnF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAM,GAAa,CAAC,OAAO,gBAAgB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7E,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACjC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,SAAS;wBAAE,MAAM,KAAK,CAAC;oBAC5C,MAAM,IAAI,KAAK,CAAC,GAAI,GAAa,CAAC,OAAO,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAA0B;IACvD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,IAAI;QAC1C,KAAK,CAAC,IAAI,CAAC,CAAC;YACV,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,MAAM,SAAS,CAC1B,CAAC,CAAC,KAAK,EACP,UAAU,EACV,WAAW,EACX,WAAW,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,IAAI,EAAE,EACpC,CAAC,CACF,CAAC;YACF,OAAO,MAAM,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/sources/ytdlp.d.ts

@@ -0,0 +1,43 @@
+/**
+ * yt-dlp as a source: `ytsearchN:<terms>` for hits (what the listing shows for free), and a
+ * YouTube video read whole (captions, else Gemini hears it) as one document. Other hosts' videos
+ * need the watcher's keys: not here yet. Metered: a read without captions costs a model call.
+ */
+import type { Model } from "@wrenautomation/models";
+import type { VideoRead } from "@wrenautomation/read/video";
+import { flatPlaylist, pageInfo, readYoutube, type YtDlpEntry } from "@wrenautomation/read/video";
+import type { Document } from "../document/index.js";
+import type { Source } from "../engine/ports.js";
+export declare const videoDocument: (r: VideoRead) => Document;
+/** One listed entry as a hit; `youtube` builds the watch URL from the id, other sites keep theirs. */
+export declare const entryHit: (e: YtDlpEntry, site?: "youtube" | "other") => {
+    snippet?: string;
+    date?: string;
+    length?: number;
+    channel?: string;
+    title?: string;
+    url: string;
+} | null;
+/** A YouTube watch URL (or youtu.be): what the `youtube` source opens. */
+export declare function isYoutube(url: string): boolean;
+/**
+ * The page yt-dlp lists a URL's own posts from: a YouTube channel's videos (a video resolves
+ * to its channel first), a TikTok profile. Null: not a page it lists.
+ */
+export declare function postsOf(url: string): {
+    target: string;
+    site: "youtube" | "other";
+} | "video" | null;
+export declare function ytdlpSource(o: {
+    ytDlp: string;
+    /**
+     * The model that reads what a video shows, made on first open (`geminiVideo` checks its keys
+     * over the network): listing sources costs no call.
+     */
+    video?: () => Promise<Model | null>;
+    n?: number;
+    /** For tests: stand-ins for the yt-dlp calls. */
+    list?: typeof flatPlaylist;
+    read?: typeof readYoutube;
+    info?: typeof pageInfo;
+}): Source;

package/dist/sources/ytdlp.js

@@ -0,0 +1,117 @@
+/**
+ * yt-dlp as a source: `ytsearchN:<terms>` for hits (what the listing shows for free), and a
+ * YouTube video read whole (captions, else Gemini hears it) as one document. Other hosts' videos
+ * need the watcher's keys: not here yet. Metered: a read without captions costs a model call.
+ */
+import { flatPlaylist, pageInfo, readYoutube } from "@wrenautomation/read/video";
+import { youtubeId } from "../engine/canonical.js";
+export const videoDocument = (r) => ({
+    url: r.url,
+    media: "video",
+    title: r.title,
+    text: r.body,
+    ...(r.channel ? { channel: r.channel } : {}),
+    ...(r.uploaded ? { date: r.uploaded } : {}),
+    ...(r.duration !== undefined ? { length: r.duration } : {}),
+});
+/** One listed entry as a hit; `youtube` builds the watch URL from the id, other sites keep theirs. */
+export const entryHit = (e, site = "youtube") => {
+    const own = e.webpage_url ?? e.url;
+    const url = site === "youtube" && e.id ? `https://www.youtube.com/watch?v=${e.id}` : own;
+    if (!url)
+        return null;
+    const by = e.channel ?? e.uploader;
+    return {
+        url,
+        ...(e.title ? { title: e.title } : {}),
+        ...(by ? { channel: by } : {}),
+        ...(e.duration !== undefined ? { length: Math.round(e.duration) } : {}),
+        ...(e.timestamp ? { date: new Date(e.timestamp * 1000).toISOString().slice(0, 10) } : {}),
+        ...(e.description ? { snippet: e.description.slice(0, 300) } : {}),
+    };
+};
+/** A YouTube watch URL (or youtu.be): what the `youtube` source opens. */
+export function isYoutube(url) {
+    try {
+        return Boolean(youtubeId(new URL(url)));
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * The page yt-dlp lists a URL's own posts from: a YouTube channel's videos (a video resolves
+ * to its channel first), a TikTok profile. Null: not a page it lists.
+ */
+export function postsOf(url) {
+    let u;
+    try {
+        u = new URL(url);
+    }
+    catch {
+        return null;
+    }
+    const host = u.hostname.toLowerCase().replace(/^(www|m)\./, "");
+    const [first, second] = u.pathname.split("/").filter(Boolean);
+    if (host === "youtube.com" || host === "youtu.be") {
+        if (isYoutube(url))
+            return "video";
+        if (first?.startsWith("@"))
+            return { target: `https://www.youtube.com/${first}/videos`, site: "youtube" };
+        if ((first === "channel" || first === "c" || first === "user") && second)
+            return { target: `https://www.youtube.com/${first}/${second}/videos`, site: "youtube" };
+        return null;
+    }
+    if (host === "tiktok.com" && first?.startsWith("@"))
+        return { target: `https://www.tiktok.com/${first}`, site: "other" };
+    return null;
+}
+export function ytdlpSource(o) {
+    const list = o.list ?? flatPlaylist;
+    const read = o.read ?? readYoutube;
+    const info = o.info ?? pageInfo;
+    let video;
+    const videoModel = (make) => {
+        video ??= make();
+        return video;
+    };
+    return {
+        name: "youtube",
+        kind: "both",
+        cost: "metered",
+        // YouTube's own search knows no other site.
+        keeps: (site) => /(^|\.)(youtube\.com|youtu\.be)$/i.test(site),
+        async search(q) {
+            const entries = await list(o.ytDlp, `ytsearch${o.n ?? 10}:${q.terms}`);
+            return entries.map((e) => entryHit(e)).filter((h) => h !== null);
+        },
+        follows: (as, url) => as === "posts" && postsOf(url) !== null,
+        async follow(_as, url, n) {
+            let at = postsOf(url);
+            if (at === "video") {
+                const v = await info(o.ytDlp, url);
+                const channel = [v.channel_url, v.uploader_url].find((x) => typeof x === "string");
+                at = typeof channel === "string" ? postsOf(channel) : null;
+                if (!at || at === "video")
+                    throw new Error(`youtube: no channel for ${url}`);
+            }
+            if (!at)
+                throw new Error(`youtube: no posts page for ${url}`);
+            const site = at.site;
+            const entries = await list(o.ytDlp, at.target, { start: 1, end: n });
+            return entries
+                .map((e) => entryHit(e, site))
+                .filter((h) => h !== null);
+        },
+        opens: isYoutube,
+        async open(h) {
+            if (!isYoutube(h.url))
+                throw new Error(`youtube: not a YouTube page: ${h.url}`);
+            return videoDocument(await read(h.url, {
+                ytDlp: o.ytDlp,
+                ...(o.video ? { video: await videoModel(o.video) } : {}),
+            }));
+        },
+    };
+}
+//# sourceMappingURL=ytdlp.js.map

package/dist/sources/ytdlp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ytdlp.js","sourceRoot":"","sources":["../../src/sources/ytdlp.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAmB,MAAM,4BAA4B,CAAC;AAElG,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAGnD,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAY,EAAY,EAAE,CAAC,CAAC;IACxD,GAAG,EAAE,CAAC,CAAC,GAAG;IACV,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,CAAC,CAAC,KAAK;IACd,IAAI,EAAE,CAAC,CAAC,IAAI;IACZ,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CAC5D,CAAC,CAAC;AAEH,sGAAsG;AACtG,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAa,EAAE,OAA4B,SAAS,EAAE,EAAE;IAC/E,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACzF,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,QAAQ,CAAC;IACnC,OAAO;QACL,GAAG;QACH,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzF,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CACrB,GAAW;IAEX,IAAI,CAAM,CAAC;IACX,IAAI,CAAC;QACH,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IAChE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,IAAI,IAAI,KAAK,aAAa,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QAClD,IAAI,SAAS,CAAC,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;QACnC,IAAI,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;YACxB,OAAO,EAAE,MAAM,EAAE,2BAA2B,KAAK,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAChF,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,MAAM,CAAC,IAAI,MAAM;YACtE,OAAO,EAAE,MAAM,EAAE,2BAA2B,KAAK,IAAI,MAAM,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC1F,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,KAAK,YAAY,IAAI,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC;QACjD,OAAO,EAAE,MAAM,EAAE,0BAA0B,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACtE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,CAY3B;IACC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC;IACpC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,WAAW,CAAC;IACnC,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC;IAChC,IAAI,KAAwC,CAAC;IAC7C,MAAM,UAAU,GAAG,CAAC,IAAiC,EAAE,EAAE;QACvD,KAAK,KAAK,IAAI,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,SAAS;QACf,4CAA4C;QAC5C,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9D,KAAK,CAAC,MAAM,CAAC,CAAC;YACZ,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YACvE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAA8B,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC/F,CAAC;QACD,OAAO,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,KAAK,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI;QAC7D,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtB,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACtB,IAAI,EAAE,KAAK,OAAO,EAAE,CAAC;gBACnB,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;gBACnF,EAAE,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAC3D,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO,OAAO;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;iBAC7B,MAAM,CAAC,CAAC,CAAC,EAA8B,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC3D,CAAC;QACD,KAAK,EAAE,SAAS;QAChB,KAAK,CAAC,IAAI,CAAC,CAAC;YACV,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAChF,OAAO,aAAa,CAClB,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE;gBAChB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzD,CAAC,CACH,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,64 @@
 {
   "name": "@wrenautomation/birdwatch",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.30.0",
+  "description": "Research from the open web, driven by an agent one step per command: plan, search, triage, open, extract behind a quote gate, check, review, every record in a ledger. The agent decides; the engine runs the fixed steps. The ask and plan files, the document side (Document, Finding, Proof, the claim extractor), the sources (SearXNG, Exa, Google, arXiv, YouTube, Reddit, Facebook, the fetch ladder) and the ledger. One CLI, `birdwatch`, with the readers of @wrenautomation/read on it.",
+  "license": "PolyForm-Strict-1.0.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/wrenautomation/engine.git",
+    "directory": "packages/birdwatch"
+  },
+  "type": "module",
+  "files": [
+    "dist",
+    "!dist/**/*.test.*",
+    "README.md",
+    "LICENSE.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./document": {
+      "types": "./dist/document/index.d.ts",
+      "import": "./dist/document/index.js"
+    },
+    "./engine": {
+      "types": "./dist/engine/index.d.ts",
+      "import": "./dist/engine/index.js"
+    },
+    "./ledger": {
+      "types": "./dist/ledger/index.d.ts",
+      "import": "./dist/ledger/index.js"
+    },
+    "./sources": {
+      "types": "./dist/sources/index.d.ts",
+      "import": "./dist/sources/index.js"
+    }
+  },
+  "bin": {
+    "birdwatch": "dist/app/cli.js"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "dependencies": {
+    "@hono/node-server": "2.1.3",
+    "commander": "15.0.0",
+    "hono": "4.13.12",
+    "htmlparser2": "12.0.0",
+    "minisearch": "7.2.0",
+    "ulid": "3.0.2",
+    "zod": "4.6.5",
+    "@wrenautomation/models": "0.30.0",
+    "@wrenautomation/read": "0.30.0"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "build": "sh ../../scripts/build.sh app/cli.js"
+  }
 }