@wrenautomation/birdwatch 0.0.0-stage → 0.30.0

package/dist/sources/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * The sources: one file each, registered by name. `searxng` and `exa` search the web, `arxiv`
+ * searches and reads papers, `youtube`
+ * searches and reads videos, `fetcher` and `ladder` open pages, `reddit` and `facebook` read a
+ * thread or a group post through the site's own route, `paste` is a person's list,
+ * `ledger` is what earlier runs saw.
+ */
+export * from "./arxiv.js";
+export * from "./exa.js";
+export * from "./fetcher.js";
+export * from "./google.js";
+export * from "./ladder.js";
+export * from "./ledger.js";
+export * from "./paste.js";
+export * from "./searxng.js";
+export * from "./sites.js";
+export * from "./ytdlp.js";

package/dist/sources/index.js

@@ -0,0 +1,18 @@
+/**
+ * The sources: one file each, registered by name. `searxng` and `exa` search the web, `arxiv`
+ * searches and reads papers, `youtube`
+ * searches and reads videos, `fetcher` and `ladder` open pages, `reddit` and `facebook` read a
+ * thread or a group post through the site's own route, `paste` is a person's list,
+ * `ledger` is what earlier runs saw.
+ */
+export * from "./arxiv.js";
+export * from "./exa.js";
+export * from "./fetcher.js";
+export * from "./google.js";
+export * from "./ladder.js";
+export * from "./ledger.js";
+export * from "./paste.js";
+export * from "./searxng.js";
+export * from "./sites.js";
+export * from "./ytdlp.js";
+//# sourceMappingURL=index.js.map

package/dist/sources/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sources/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC"}

package/dist/sources/junk.d.ts

@@ -0,0 +1,10 @@
+/**
+ * A page that answered 200 and still holds nothing: a captcha wall, a 404 shell, an empty body.
+ * Both open sources ask before they return a document, so such a page is a miss with a reason
+ * (spend ok:false, a gap), never a read.
+ */
+import type { Document } from "../document/index.js";
+/** Fewer characters than this is no page: a shell, a redirect stub, a bare title. */
+export declare const FLOOR = 200;
+/** Why a document is junk, or null when it reads as a page. */
+export declare function junkReason(doc: Document, status?: number | null): string | null;

package/dist/sources/junk.js

@@ -0,0 +1,16 @@
+/** Fewer characters than this is no page: a shell, a redirect stub, a bare title. */
+export const FLOOR = 200;
+const WALLS = /prove your humanity|just a moment|are you a (?:robot|human)|verify you are human|attention required|access denied|page not found|not found\b|403 forbidden|404\b|blocked|captcha|security check|enable javascript/i;
+/** Why a document is junk, or null when it reads as a page. */
+export function junkReason(doc, status) {
+    if (status !== undefined && status !== null && status >= 400)
+        return `HTTP ${status}`;
+    const title = doc.title?.trim() ?? "";
+    if (WALLS.test(title))
+        return `a wall, not a page: "${title}"`;
+    const n = doc.text.trim().length;
+    if (n < FLOOR)
+        return `only ${n} characters of text, no page`;
+    return null;
+}
+//# sourceMappingURL=junk.js.map

package/dist/sources/junk.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junk.js","sourceRoot":"","sources":["../../src/sources/junk.ts"],"names":[],"mappings":"AAOA,qFAAqF;AACrF,MAAM,CAAC,MAAM,KAAK,GAAG,GAAG,CAAC;AAEzB,MAAM,KAAK,GACT,oNAAoN,CAAC;AAEvN,+DAA+D;AAC/D,MAAM,UAAU,UAAU,CAAC,GAAa,EAAE,MAAsB;IAC9D,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,QAAQ,MAAM,EAAE,CAAC;IACtF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACtC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,wBAAwB,KAAK,GAAG,CAAC;IAC/D,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC;IACjC,IAAI,CAAC,GAAG,KAAK;QAAE,OAAO,QAAQ,CAAC,8BAA8B,CAAC;IAC9D,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/sources/ladder.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The fetch ladder as an open source: `web GET /fetch` on a site client, up to a browser. Only
+ * worth a call after the plain fetcher came back empty or blocked, so it sits after `fetcher` in
+ * a plan file's sources. Browser cost, and one of the worker's daily `fetch` bucket per call. A
+ * block the last rung still hit, a spent bucket or a page that is not HTML is a miss (a throw
+ * with the reason); a worker that did not answer is a Transient.
+ *
+ * Two wires to an autobrowse worker for a terminal run: `httpLadderClient` for its HTTP face
+ * (`GET $url/api/sites/web/fetch?url=&from=`, a bearer token); `ingressLadderClient` for the
+ * Restate ingress the desk worker sits behind (`POST $url/desk/call`, the same bearer).
+ */
+import { type LadderClient, type LadderTier } from "@wrenautomation/read/fetch";
+import { type Media, type Reader } from "../document/index.js";
+import type { Source } from "../engine/ports.js";
+export declare function ladderSource(o: {
+    sites: LadderClient;
+    from?: LadderTier;
+    readers?: Map<Media, Reader>;
+}): Source;
+export declare class LadderCallError extends Error {
+    readonly status: number;
+    /** The worker's access code (`need_login`, `platform_blocked`, `rate_limited`, `upstream_broken`), when it sent one. */
+    readonly code: string | null;
+    constructor(message: string, status: number, 
+    /** The worker's access code (`need_login`, `platform_blocked`, `rate_limited`, `upstream_broken`), when it sent one. */
+    code?: string | null);
+}
+/** The HTTP wire to an autobrowse worker, enough for the ladder. wren has the whole client. */
+export declare function httpLadderClient(o: {
+    url: string;
+    token?: string | null;
+    fetchFn?: typeof fetch;
+}): LadderClient;
+/**
+ * The Restate ingress wire to an autobrowse worker: `POST $url/$service/call` with the site
+ * call as its body, the way wren's CLI reaches the desk. `service` is `desk` (the Mac) or
+ * `sites` (a box); `caller` names who asks, for the worker's per-caller usage. Restate answers a
+ * refused call as `{"code":429,"message":"…"}` under that status: a 429 is the ladder's "cap".
+ * A coded refusal adds `"metadata":{"code":"need_login"}`.
+ */
+export declare function ingressLadderClient(o: {
+    url: string;
+    token?: string | null;
+    service?: string;
+    caller?: string;
+    fetchFn?: typeof fetch;
+}): LadderClient;

package/dist/sources/ladder.js

@@ -0,0 +1,158 @@
+/**
+ * The fetch ladder as an open source: `web GET /fetch` on a site client, up to a browser. Only
+ * worth a call after the plain fetcher came back empty or blocked, so it sits after `fetcher` in
+ * a plan file's sources. Browser cost, and one of the worker's daily `fetch` bucket per call. A
+ * block the last rung still hit, a spent bucket or a page that is not HTML is a miss (a throw
+ * with the reason); a worker that did not answer is a Transient.
+ *
+ * Two wires to an autobrowse worker for a terminal run: `httpLadderClient` for its HTTP face
+ * (`GET $url/api/sites/web/fetch?url=&from=`, a bearer token); `ingressLadderClient` for the
+ * Restate ingress the desk worker sits behind (`POST $url/desk/call`, the same bearer).
+ */
+import { Transient } from "@wrenautomation/models";
+import { fetchLadder } from "@wrenautomation/read/fetch";
+import { defaultReaders } from "../document/index.js";
+import { isPage } from "./fetcher.js";
+import { junkReason } from "./junk.js";
+export function ladderSource(o) {
+    const readers = o.readers ?? defaultReaders();
+    return {
+        name: "ladder",
+        kind: "open",
+        cost: "browser",
+        opens: isPage,
+        async open(h) {
+            if (!isPage(h.url))
+                throw new Error(`ladder: not a page: ${h.url}`);
+            const r = await fetchLadder(o.sites, h.url, o.from ?? "impersonate");
+            // offline: the worker did not answer, worth another try. The rest is a miss, like the
+            // fetcher's 403: the read step writes a gap and goes on to the next hit.
+            if ("failed" in r) {
+                if (r.failed === "offline")
+                    throw new Transient("ladder: offline");
+                throw new Error("ladder: the worker's fetch bucket is spent for today");
+            }
+            if (r.fetched.block)
+                throw new Error(`ladder: ${r.fetched.block} at ${r.fetched.tier}`);
+            // The ladder answers the body as text: only HTML reads right that way.
+            const type = r.fetched.type?.split(";")[0]?.trim().toLowerCase();
+            if (type && type !== "text/html" && type !== "application/xhtml+xml")
+                throw new Error(`ladder: ${type} is not html`);
+            const html = readers.get("html");
+            if (!html)
+                throw new Error("ladder: no html reader");
+            const doc = await html.read({
+                url: r.fetched.finalUrl || h.url,
+                text: r.fetched.html,
+                ...(h.title ? { title: h.title } : {}),
+            });
+            if (!doc)
+                throw new Error("ladder: nothing to read in the page");
+            const junk = junkReason(doc, r.fetched.status);
+            if (junk)
+                throw new Error(`ladder: ${junk} at ${r.fetched.tier}`);
+            return doc;
+        },
+    };
+}
+export class LadderCallError extends Error {
+    status;
+    code;
+    constructor(message, status, 
+    /** The worker's access code (`need_login`, `platform_blocked`, `rate_limited`, `upstream_broken`), when it sent one. */
+    code = null) {
+        super(message);
+        this.status = status;
+        this.code = code;
+        this.name = "LadderCallError";
+    }
+}
+/** The HTTP wire to an autobrowse worker, enough for the ladder. wren has the whole client. */
+export function httpLadderClient(o) {
+    const fetchFn = o.fetchFn ?? fetch;
+    const base = o.url.replace(/\/$/, "");
+    return {
+        async call(site, method, path, input, as) {
+            const read = method === "GET" || method === "DELETE";
+            let url = `${base}/api/sites/${site}${path}`;
+            const q = new URLSearchParams();
+            if (read && input && typeof input === "object")
+                for (const [k, v] of Object.entries(input))
+                    if (v !== undefined)
+                        q.set(k, String(v));
+            // autobrowse takes `account` beside the query, for writes too.
+            if (as?.account)
+                q.set("account", as.account);
+            if (q.size)
+                url += `?${q}`;
+            const res = await fetchFn(url, {
+                method,
+                headers: {
+                    ...(read ? {} : { "content-type": "application/json" }),
+                    ...(o.token ? { authorization: `Bearer ${o.token}` } : {}),
+                },
+                ...(read ? {} : { body: JSON.stringify(input ?? {}) }),
+                signal: AbortSignal.timeout(180_000),
+            });
+            const text = await res.text();
+            const body = text ? JSON.parse(text) : null;
+            if (!res.ok) {
+                const b = body;
+                throw new LadderCallError(`${site} ${method} ${path}: ${res.status} ${b?.error ?? res.statusText}`, res.status, b?.code ?? null);
+            }
+            return body;
+        },
+    };
+}
+/**
+ * The Restate ingress wire to an autobrowse worker: `POST $url/$service/call` with the site
+ * call as its body, the way wren's CLI reaches the desk. `service` is `desk` (the Mac) or
+ * `sites` (a box); `caller` names who asks, for the worker's per-caller usage. Restate answers a
+ * refused call as `{"code":429,"message":"…"}` under that status: a 429 is the ladder's "cap".
+ * A coded refusal adds `"metadata":{"code":"need_login"}`.
+ */
+export function ingressLadderClient(o) {
+    const fetchFn = o.fetchFn ?? fetch;
+    const base = o.url.replace(/\/$/, "");
+    const service = o.service ?? "desk";
+    const caller = o.caller ?? "birdwatch";
+    return {
+        async call(site, method, path, input, as) {
+            const res = await fetchFn(`${base}/${service}/call`, {
+                method: "POST",
+                headers: {
+                    "content-type": "application/json",
+                    "idempotency-key": crypto.randomUUID(),
+                    ...(o.token ? { authorization: `Bearer ${o.token}` } : {}),
+                },
+                body: JSON.stringify({
+                    site,
+                    method,
+                    path,
+                    input: input ?? {},
+                    caller,
+                    ...(as?.account ? { account: as.account } : {}),
+                }),
+                signal: AbortSignal.timeout(180_000),
+            });
+            const text = await res.text();
+            let body = null;
+            try {
+                body = text ? JSON.parse(text) : null;
+            }
+            catch {
+                body = text;
+            }
+            if (!res.ok) {
+                const said = body && typeof body === "object" && "message" in body
+                    ? String(body.message)
+                    : text || res.statusText;
+                // A coded refusal: autobrowse puts the access code in the terminal error's metadata.
+                const code = body?.metadata?.code;
+                throw new LadderCallError(`${site} ${method} ${path}: ${res.status} ${said}`, res.status, typeof code === "string" ? code : null);
+            }
+            return body;
+        },
+    };
+}
+//# sourceMappingURL=ladder.js.map

package/dist/sources/ladder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ladder.js","sourceRoot":"","sources":["../../src/sources/ladder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAsC,MAAM,4BAA4B,CAAC;AAC7F,OAAO,EAAE,cAAc,EAA2B,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,MAAM,UAAU,YAAY,CAAC,CAI5B;IACC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,cAAc,EAAE,CAAC;IAC9C,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,MAAM;QACb,KAAK,CAAC,IAAI,CAAC,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,aAAa,CAAC,CAAC;YACrE,sFAAsF;YACtF,yEAAyE;YACzE,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAClB,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;oBAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,CAAC,CAAC;gBACnE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACxF,uEAAuE;YACvE,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACjE,IAAI,IAAI,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,uBAAuB;gBAClE,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,cAAc,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,CAAC,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YACrD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;gBAC1B,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG;gBAChC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;gBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC,CAAC;YACH,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;YACjE,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,IAAI;gBAAE,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAClE,OAAO,GAAG,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAG7B;IAEA;IAJX,YACE,OAAe,EACN,MAAc;IACvB,wHAAwH;IAC/G,OAAsB,IAAI;QAEnC,KAAK,CAAC,OAAO,CAAC,CAAC;QAJN,WAAM,GAAN,MAAM,CAAQ;QAEd,SAAI,GAAJ,IAAI,CAAsB;QAGnC,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED,+FAA+F;AAC/F,MAAM,UAAU,gBAAgB,CAAC,CAIhC;IACC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO;QACL,KAAK,CAAC,IAAI,CACR,IAAY,EACZ,MAAc,EACd,IAAY,EACZ,KAAe,EACf,EAAyB;YAEzB,MAAM,IAAI,GAAG,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,QAAQ,CAAC;YACrD,IAAI,GAAG,GAAG,GAAG,IAAI,cAAc,IAAI,GAAG,IAAI,EAAE,CAAC;YAC7C,MAAM,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;YAChC,IAAI,IAAI,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAC5C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC;oBACnE,IAAI,CAAC,KAAK,SAAS;wBAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,+DAA+D;YAC/D,IAAI,EAAE,EAAE,OAAO;gBAAE,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,CAAC,CAAC,IAAI;gBAAE,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;gBAC7B,MAAM;gBACN,OAAO,EAAE;oBACP,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;oBACvD,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3D;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;gBACtD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC,CAAC,CAAC,IAAI,CAAC;YACzD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,CAAC,GAAG,IAAgD,CAAC;gBAC3D,MAAM,IAAI,eAAe,CACvB,GAAG,IAAI,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,KAAK,IAAI,GAAG,CAAC,UAAU,EAAE,EACxE,GAAG,CAAC,MAAM,EACV,CAAC,EAAE,IAAI,IAAI,IAAI,CAChB,CAAC;YACJ,CAAC;YACD,OAAO,IAAS,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,CAMnC;IACC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,WAAW,CAAC;IACvC,OAAO;QACL,KAAK,CAAC,IAAI,CACR,IAAY,EACZ,MAAc,EACd,IAAY,EACZ,KAAe,EACf,EAAyB;YAEzB,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,IAAI,IAAI,OAAO,OAAO,EAAE;gBACnD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,iBAAiB,EAAE,MAAM,CAAC,UAAU,EAAE;oBACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC3D;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI;oBACJ,MAAM;oBACN,IAAI;oBACJ,KAAK,EAAE,KAAK,IAAI,EAAE;oBAClB,MAAM;oBACN,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAChD,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;aACrC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,IAAI,GAAY,IAAI,CAAC;YACzB,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,GAAG,IAAI,CAAC;YACd,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GACR,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,SAAS,IAAI,IAAI;oBACnD,CAAC,CAAC,MAAM,CAAE,IAA6B,CAAC,OAAO,CAAC;oBAChD,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;gBAC7B,qFAAqF;gBACrF,MAAM,IAAI,GAAI,IAAiD,EAAE,QAAQ,EAAE,IAAI,CAAC;gBAChF,MAAM,IAAI,eAAe,CACvB,GAAG,IAAI,IAAI,MAAM,IAAI,IAAI,KAAK,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,EAClD,GAAG,CAAC,MAAM,EACV,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CACvC,CAAC;YACJ,CAAC;YACD,OAAO,IAAS,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/sources/ledger.d.ts

@@ -0,0 +1,7 @@
+/**
+ * The ledger as a source: hits earlier runs saw whose title or snippet holds every word of the
+ * query. A run that starts where the last one got to. Free.
+ */
+import type { Source } from "../engine/ports.js";
+import type { Ledger } from "../ledger/ledger.js";
+export declare function ledgerSource(ledger: Ledger, n?: number): Source;

package/dist/sources/ledger.js

@@ -0,0 +1,35 @@
+import { ofType } from "../ledger/ledger.js";
+const wordsOf = (s) => s
+    .toLowerCase()
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((w) => w.length > 2);
+export function ledgerSource(ledger, n = 10) {
+    return {
+        name: "ledger",
+        kind: "search",
+        cost: "free",
+        async search(q) {
+            const want = wordsOf(q.terms);
+            if (!want.length)
+                return [];
+            const hits = ofType(await ledger.scan({ type: "hit" }), "hit").filter((h) => h.run !== q.run);
+            const out = new Map();
+            for (const h of hits) {
+                const have = new Set(wordsOf(`${h.title ?? ""} ${h.snippet ?? ""}`));
+                if (want.every((w) => have.has(w)) && !out.has(h.key))
+                    out.set(h.key, h);
+            }
+            return [...out.values()]
+                .slice(0, n)
+                .map(({ url, title, channel, length, date, snippet }) => ({
+                url,
+                ...(title ? { title } : {}),
+                ...(channel ? { channel } : {}),
+                ...(length !== undefined ? { length } : {}),
+                ...(date ? { date } : {}),
+                ...(snippet ? { snippet } : {}),
+            }));
+        },
+    };
+}
+//# sourceMappingURL=ledger.js.map

package/dist/sources/ledger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ledger.js","sourceRoot":"","sources":["../../src/sources/ledger.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE7C,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,EAAE,CAC5B,CAAC;KACE,WAAW,EAAE;KACb,KAAK,CAAC,iBAAiB,CAAC;KACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAEjC,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,CAAC,GAAG,EAAE;IACjD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,KAAK,CAAC,MAAM,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAC9F,MAAM,GAAG,GAAG,IAAI,GAAG,EAAiC,CAAC;YACrD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;gBACrE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;iBACrB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;gBACxD,GAAG;gBACH,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/B,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3C,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChC,CAAC,CAAC,CAAC;QACR,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/sources/paste.d.ts

@@ -0,0 +1,8 @@
+/**
+ * A person's URLs as a source: the hits they pasted come back on the first search of a run,
+ * marked `from: human` and `by` them. Every query after that gets nothing: the list was given
+ * once.
+ */
+import type { Source } from "../engine/ports.js";
+import type { HitViewIn } from "../engine/types.js";
+export declare function paste(hits: readonly HitViewIn[], by: string): Source;

package/dist/sources/paste.js

@@ -0,0 +1,22 @@
+/**
+ * A person's URLs as a source: the hits they pasted come back on the first search of a run,
+ * marked `from: human` and `by` them. Every query after that gets nothing: the list was given
+ * once.
+ */
+export function paste(hits, by) {
+    let given = false;
+    return {
+        name: "paste",
+        kind: "search",
+        cost: "free",
+        from: "human",
+        by,
+        async search() {
+            if (given)
+                return [];
+            given = true;
+            return hits.map((h) => ({ ...h }));
+        },
+    };
+}
+//# sourceMappingURL=paste.js.map

package/dist/sources/paste.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paste.js","sourceRoot":"","sources":["../../src/sources/paste.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,MAAM,UAAU,KAAK,CAAC,IAA0B,EAAE,EAAU;IAC1D,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,OAAO;QACL,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,OAAO;QACb,EAAE;QACF,KAAK,CAAC,MAAM;YACV,IAAI,KAAK;gBAAE,OAAO,EAAE,CAAC;YACrB,KAAK,GAAG,IAAI,CAAC;YACb,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/sources/searxng.d.ts

@@ -0,0 +1,11 @@
+/**
+ * SearXNG: a self-hosted metasearch, `GET $url/search?q=&format=json`. Free. `token` goes as a
+ * bearer for an instance behind an authenticating proxy (Wren's, on the VPS).
+ */
+import type { Source } from "../engine/ports.js";
+export declare function searxng(o: {
+    url: string;
+    token?: string;
+    fetchFn?: typeof fetch;
+    n?: number;
+}): Source;

package/dist/sources/searxng.js

@@ -0,0 +1,38 @@
+/**
+ * SearXNG: a self-hosted metasearch, `GET $url/search?q=&format=json`. Free. `token` goes as a
+ * bearer for an instance behind an authenticating proxy (Wren's, on the VPS).
+ */
+import { httpError, transport } from "@wrenautomation/models";
+export function searxng(o) {
+    const fetchFn = o.fetchFn ?? fetch;
+    const base = o.url.replace(/\/$/, "");
+    return {
+        name: "searxng",
+        kind: "search",
+        cost: "free",
+        async search(q) {
+            const u = new URL(`${base}/search`);
+            u.searchParams.set("q", q.site ? `site:${q.site} ${q.terms}` : q.terms);
+            u.searchParams.set("format", "json");
+            const res = await fetchFn(u, {
+                signal: AbortSignal.timeout(30_000),
+                ...(o.token ? { headers: { authorization: `Bearer ${o.token}` } } : {}),
+            }).catch((err) => {
+                throw transport("searxng", err);
+            });
+            if (!res.ok)
+                throw httpError("searxng", res);
+            const body = (await res.json());
+            return (body.results ?? [])
+                .filter((r) => typeof r.url === "string")
+                .slice(0, o.n ?? 10)
+                .map((r) => ({
+                url: r.url,
+                ...(r.title ? { title: r.title } : {}),
+                ...(r.content ? { snippet: r.content } : {}),
+                ...(r.publishedDate ? { date: r.publishedDate.slice(0, 10) } : {}),
+            }));
+        },
+    };
+}
+//# sourceMappingURL=searxng.js.map

package/dist/sources/searxng.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../src/sources/searxng.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAU9D,MAAM,UAAU,OAAO,CAAC,CAKvB;IACC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtC,OAAO;QACL,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,MAAM;QACZ,KAAK,CAAC,MAAM,CAAC,CAAC;YACZ,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,IAAI,SAAS,CAAC,CAAC;YACpC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACxE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,CAAC,EAAE;gBAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;gBACnC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACf,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAgC,CAAC;YAC/D,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;iBACxB,MAAM,CAAC,CAAC,CAAC,EAAsC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC;iBAC5E,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnE,CAAC,CAAC,CAAC;QACR,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/sources/sites.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Readers for one site each, over that site's own read route on an autobrowse worker (the same
+ * wire as the ladder). Each claims only its site's URLs and goes before the general openers
+ * (`first`): the route answers the thread or the post as data, where a fetched page is a wall or a
+ * JavaScript shell.
+ *
+ * `reddit`: a thread (`/comments/{id}`) through `reddit-public` (signed out, the RSS feed), then,
+ * when the caller named an account (`BIRDWATCH_ACCOUNTS=reddit=reddit@alt`), through `reddit`
+ * signed in as it. Signed out goes first: read volume on a signed-in account is what Reddit
+ * flags. A follow from a user's page (`/user/{name}`) lists their top posts and comments in one
+ * read (`/user/{name}/overview`), signed out. `facebook`: one public group post through `fb-public`, always signed out (Wren's Meta
+ * login runs the ads). Design: autobrowse `designs/2026-10-28-signed-in-reads.md`.
+ *
+ * A route that answered no (a 404, a spent cap, a refusal) is a miss with its reason and the
+ * worker's access code; a worker that did not answer, or a busy profile (503), is a Transient.
+ * A signed-out miss reads signed in, except a spent cap or a broken site (`rate_limited`,
+ * `upstream_broken`): signing in fixes neither.
+ */
+import type { LadderClient } from "@wrenautomation/read/fetch";
+import type { Document } from "../document/index.js";
+import type { Source } from "../engine/ports.js";
+import type { HitViewIn } from "../engine/types.js";
+/** `reddit=reddit@alt,linkedin=linkedin@wren`: the account each site reader signs in as. */
+export declare function accountsFrom(spec: string | undefined): Map<string, string>;
+/** A Reddit thread's id, and the comment's when the URL is one comment's permalink. */
+export declare function redditThread(url: string): {
+    article: string;
+    comment?: string;
+} | null;
+/** A Reddit user's name from their page (`/user/{name}`, `/u/{name}`). */
+export declare function redditUser(url: string): string | null;
+/** A user's overview (posts and comments mixed) as hits: each its permalink, title and words. */
+export declare function userHits(body: unknown): HitViewIn[];
+/** A Facebook group post's group and post ids. */
+export declare function facebookPost(url: string): {
+    group: string;
+    post: string;
+} | null;
+type Obj = Record<string, unknown>;
+/** A thread (`[post listing, comments listing]`) as text: the post, then each comment. */
+export declare function threadDocument(url: string, body: unknown): Document;
+/** fb-public's `{ post, comments }` as text: the post's whole text, then each comment. */
+export declare function groupPostDocument(url: string, body: unknown): Document;
+/** One route call; a refusal is a miss with the worker's words, no answer a Transient. */
+export declare function routeCall(sites: LadderClient, who: string, site: string, path: string, input: Obj, account?: string): Promise<unknown>;
+export declare function redditSource(o: {
+    sites: LadderClient;
+    account?: string;
+}): Source;
+export declare function facebookSource(o: {
+    sites: LadderClient;
+}): Source;
+export {};