@vfrey/vfrey 0.0.1 → 1.0.0-beta.0

package/vendor/@vfrey/contracts/dist/log.js

@@ -0,0 +1,10 @@
+import { redact } from "./domain/redact.js";
+export function log(level, message, fields = {}) {
+    const line = redact(JSON.stringify({
+        ts: new Date().toISOString(),
+        level,
+        message,
+        ...fields,
+    }));
+    process.stdout.write(line + "\n");
+}

package/vendor/@vfrey/contracts/dist/mail.d.ts

@@ -0,0 +1,55 @@
+import type { BusinessContact, CreatorSummary } from "./domain/creators.js";
+export interface MailContext {
+    videoTitle?: string | null;
+    publicationUrl?: string | null;
+}
+/** Spec 018. `generic` is the original free-form draft. */
+export type MailKind = "generic" | "sponsor" | "brand" | "creator_collab" | "client_update";
+/** The only creator data a draft may use. Built from what the Creators service returns. */
+export interface MailCreatorContext {
+    summary: CreatorSummary;
+    /** One recent top post. */
+    topPost?: {
+        title?: string;
+        url?: string;
+    } | null;
+    /** The published business contact. Never a bio, never a phone number. */
+    businessContact?: BusinessContact | null;
+}
+/** Who the message identifies itself as. Feeds the compliance footer. */
+export interface MailSender {
+    senderName: string;
+    senderOrg: string;
+    postalAddress: string;
+    unsubscribeUrl: string;
+}
+/** What the operator asks for. This is not a pipeline event. */
+export interface MailRequest {
+    accountId: string;
+    to: string;
+    need: string;
+    tone: string;
+    channelId?: string | null;
+    context?: MailContext | null;
+    /** Spec 018. Absent means `generic`. */
+    kind?: MailKind;
+    /** Spec 018. Personalisation for outreach kinds. */
+    creator?: MailCreatorContext | null;
+    /** Spec 018. Name of the person addressed, when known. */
+    recipientName?: string | null;
+    /** Spec 018. Required for outreach kinds. Appended as the compliance footer. */
+    sender?: MailSender | null;
+}
+export interface MailDraft {
+    accountId: string;
+    channelId: string | null;
+    to: string;
+    subject: string;
+    body: string;
+    text: string;
+    /** Spec 018. Escaped HTML twin of `text`. No tracking pixel, no remote image. */
+    html?: string;
+    /** Spec 018. */
+    kind?: MailKind;
+    status: "draft";
+}

package/vendor/@vfrey/contracts/dist/mail.js

@@ -0,0 +1 @@
+export {};

package/vendor/@vfrey/contracts/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@vfrey/contracts",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Shared types and pure rules for vfrey. No server.",
+  "engines": {
+    "node": ">=22.13"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ]
+}

package/vendor/@vfrey/ingest/dist/index.d.ts

@@ -0,0 +1,41 @@
+import type { Account, AppConfig, Candidate } from "@vfrey/contracts";
+export { fingerprint, parseFeed, parseRedditListing, parseYouTubePopular } from "./parse.js";
+/** The tags the scorer is known to emit. Any other non-empty tag is also treated as compliance. */
+export declare const COMPLIANCE_TAGS: Set<string>;
+export interface ScoreResult {
+    score: number;
+    safety: number;
+    reason: string;
+    riskClass: "ok" | "quality" | "compliance";
+    complianceTags: string[];
+}
+export interface SourceError {
+    source: string;
+    code: string;
+    message: string;
+    retryable: boolean;
+    retryAfterMs?: number;
+}
+export interface CollectInput {
+    config: AppConfig;
+    account: Account;
+    fetchImpl: typeof fetch;
+    manual: Candidate[];
+}
+export interface CollectResult {
+    candidates: Candidate[];
+    errors: SourceError[];
+}
+/**
+ * Collects every source on its own. One failing source never discards the
+ * others or the manual candidates; the failures come back in `errors`.
+ */
+export declare function collectCandidatesDetailed(input: CollectInput): Promise<CollectResult>;
+/**
+ * Returns the candidates. Failed sources are logged; when nothing at all was
+ * collected and a source failed, the first failure is rethrown so the worker
+ * can retry or stop. Use collectCandidatesDetailed to read the errors.
+ */
+export declare function collectCandidates(input: CollectInput): Promise<Candidate[]>;
+export declare function interpretScore(raw: unknown): ScoreResult;
+export declare function assertScore(result: ScoreResult): void;

package/vendor/@vfrey/ingest/dist/index.js

@@ -0,0 +1,221 @@
+import { AppError, ComplianceError, ConfigError, RetryableError } from "@vfrey/contracts";
+import { requestJson, retryAfterFrom } from "@vfrey/contracts";
+import { log } from "@vfrey/contracts";
+import { parseFeed, parseRedditListing, parseYouTubePopular } from "./parse.js";
+export { fingerprint, parseFeed, parseRedditListing, parseYouTubePopular } from "./parse.js";
+/** The tags the scorer is known to emit. Any other non-empty tag is also treated as compliance. */
+export const COMPLIANCE_TAGS = new Set([
+    "medical_advice_persona",
+    "legal_advice_persona",
+    "financial_advice_persona",
+    "impersonation",
+    "minor_safety",
+    "election_manipulation",
+    "scam",
+    "copyrighted_transcript",
+    "graphic_shock_template",
+]);
+const YOUTUBE_RETRY_REASONS = new Set(["quotaexceeded", "ratelimitexceeded", "userratelimitexceeded", "dailylimitexceeded"]);
+function toSourceError(source, error) {
+    if (error instanceof AppError) {
+        const row = { source, code: error.code, message: error.message, retryable: error.retryable };
+        if (error instanceof RetryableError && error.retryAfterMs !== undefined)
+            row.retryAfterMs = error.retryAfterMs;
+        return row;
+    }
+    return { source, code: "source_failed", message: error instanceof Error ? error.message : String(error), retryable: false };
+}
+/**
+ * Collects every source on its own. One failing source never discards the
+ * others or the manual candidates; the failures come back in `errors`.
+ */
+export async function collectCandidatesDetailed(input) {
+    const { candidates, errors } = await collectAll(input);
+    return { candidates, errors };
+}
+async function collectAll(input) {
+    const thrown = [];
+    const found = [...input.manual];
+    const errors = [];
+    const guard = async (source, run) => {
+        try {
+            await run();
+        }
+        catch (error) {
+            thrown.push(error);
+            errors.push(toSourceError(source, error));
+        }
+    };
+    for (const url of input.account.sources.rss) {
+        if (!isHttpSource(url)) {
+            log("warn", "source url rejected", { accountId: input.account.id, code: "source_url_rejected", url });
+            continue;
+        }
+        await guard(`rss:${url}`, async () => {
+            const result = await requestJson(input.fetchImpl, url, { method: "GET", timeoutMs: 20_000 });
+            if (result.status >= 500)
+                throw new RetryableError("rss", `RSS ${result.status} ${url}`);
+            if (result.status < 400)
+                found.push(...parseFeed(result.text, "rss"));
+        });
+    }
+    for (const channelId of input.account.sources.youtubeChannelIds) {
+        await guard(`youtube-channel:${channelId}`, async () => {
+            const url = `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
+            const result = await requestJson(input.fetchImpl, url, { method: "GET" });
+            if (result.status < 400)
+                found.push(...parseFeed(result.text, "youtube"));
+        });
+    }
+    if (input.account.sources.subreddits.length > 0) {
+        let token = null;
+        await guard("reddit:token", async () => {
+            token = await redditToken(input.config, input.fetchImpl);
+        });
+        if (token !== null) {
+            for (const sub of input.account.sources.subreddits) {
+                await guard(`reddit:${sub}`, async () => {
+                    const result = await requestJson(input.fetchImpl, `https://oauth.reddit.com/r/${encodeURIComponent(sub)}/hot?limit=15`, {
+                        method: "GET",
+                        headers: {
+                            authorization: `Bearer ${token}`,
+                            "user-agent": input.config.reddit.userAgent,
+                        },
+                    });
+                    if (result.status === 429) {
+                        throw new RetryableError("reddit_rate", "Reddit rate limit", retryAfterFrom(result, Date.now()));
+                    }
+                    if (result.status < 400 && result.json)
+                        found.push(...parseRedditListing(result.json));
+                });
+            }
+        }
+    }
+    if (input.account.sources.youtubeTrendingRegion && input.config.googleApiKey) {
+        const region = input.account.sources.youtubeTrendingRegion;
+        const apiKey = input.config.googleApiKey;
+        await guard("youtube-trending", async () => {
+            const url = `https://www.googleapis.com/youtube/v3/videos?part=snippet&chart=mostPopular&regionCode=${encodeURIComponent(region)}` +
+                `&maxResults=10&key=${encodeURIComponent(apiKey)}`;
+            const result = await requestJson(input.fetchImpl, url, { method: "GET" });
+            if (result.status === 400 || result.status === 403 || result.status === 429) {
+                const reason = youtubeReason(result.json);
+                if (YOUTUBE_RETRY_REASONS.has(reason) || (result.status === 429 && reason === "")) {
+                    // Quota is a transport pause, not a reason to scrape an unofficial endpoint.
+                    throw new RetryableError("youtube_quota", "YouTube Data API quota blocked trending lookup", 60 * 60_000);
+                }
+                if (result.status !== 429) {
+                    throw new ConfigError("youtube_key", `YouTube Data API refused the key (${result.status} ${reason || "no reason"})`);
+                }
+            }
+            if (result.status < 400 && result.json)
+                found.push(...parseYouTubePopular(result.json));
+        });
+    }
+    // TikTok has no official public trend endpoint for this app class.
+    // video.list on the owner's account is an analytics signal, not a topic feed,
+    // and third-party footage is never downloaded. See the vfreyai/vfrey-pipeline docs.
+    const deduped = new Map();
+    // First wins, and manual candidates are first, so a feed item never replaces one.
+    for (const candidate of found)
+        if (!deduped.has(candidate.fingerprint))
+            deduped.set(candidate.fingerprint, candidate);
+    return { candidates: [...deduped.values()].slice(0, 30), errors, thrown };
+}
+/**
+ * Returns the candidates. Failed sources are logged; when nothing at all was
+ * collected and a source failed, the first failure is rethrown so the worker
+ * can retry or stop. Use collectCandidatesDetailed to read the errors.
+ */
+export async function collectCandidates(input) {
+    const result = await collectAll(input);
+    for (const error of result.errors) {
+        log("warn", "source failed", { accountId: input.account.id, code: error.code, source: error.source });
+    }
+    if (result.candidates.length === 0 && result.errors.length > 0)
+        throw result.thrown[0];
+    return result.candidates;
+}
+function isHttpSource(url) {
+    try {
+        const protocol = new URL(url).protocol;
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+function youtubeReason(json) {
+    const error = json?.error;
+    const reason = error?.errors?.[0]?.reason;
+    return typeof reason === "string" ? reason.toLowerCase() : "";
+}
+async function redditToken(config, fetchImpl) {
+    if (!config.reddit.clientId || !config.reddit.clientSecret) {
+        throw new ConfigError("reddit_auth", "REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET are required for subreddit sources");
+    }
+    if (!config.reddit.userAgent || config.reddit.userAgent.trim() === "") {
+        throw new ConfigError("reddit_user_agent", "A non-empty Reddit User-Agent is required for subreddit sources");
+    }
+    const basic = Buffer.from(`${config.reddit.clientId}:${config.reddit.clientSecret}`).toString("base64");
+    const result = await requestJson(fetchImpl, "https://www.reddit.com/api/v1/access_token", {
+        method: "POST",
+        headers: {
+            authorization: `Basic ${basic}`,
+            "content-type": "application/x-www-form-urlencoded",
+            "user-agent": config.reddit.userAgent,
+        },
+        body: "grant_type=client_credentials",
+    });
+    if (result.status === 401 || result.status === 403) {
+        throw new ConfigError("reddit_auth", `Reddit token HTTP ${result.status}: check the client id and secret`);
+    }
+    if (result.status >= 400) {
+        throw new RetryableError("reddit_auth", `Reddit token HTTP ${result.status}`, retryAfterFrom(result, Date.now()));
+    }
+    const token = result.json?.access_token;
+    if (!token)
+        throw new RetryableError("reddit_auth", "Reddit token response had no access_token");
+    return token;
+}
+/** Placeholders a model emits to mean "no tags". They are not tags. */
+const NO_TAG_SENTINELS = new Set(["none", "n_a", "na", "null", "nil", "ok", "safe", "no", "false", "empty"]);
+function normalizeTag(tag) {
+    return tag
+        .normalize("NFKC")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "_")
+        .replace(/^_+|_+$/g, "");
+}
+export function interpretScore(raw) {
+    const row = (raw ?? {});
+    const tags = Array.isArray(row.complianceTags) ? row.complianceTags.map(String) : [];
+    const score = clamp(row.score);
+    const safety = clamp(row.safety);
+    let riskClass = row.riskClass === "compliance" || row.riskClass === "quality" ? row.riskClass : "ok";
+    // Compliance only escalates. A quality verdict never overwrites it.
+    // An unrecognized non-empty tag fails closed as compliance.
+    const flagged = tags.map(normalizeTag).some((tag) => tag !== "" && !NO_TAG_SENTINELS.has(tag));
+    if (riskClass !== "compliance") {
+        if (flagged || safety < 15)
+            riskClass = "compliance";
+        else if (score < 70)
+            riskClass = "quality";
+    }
+    return {
+        score,
+        safety,
+        reason: String(row.reason ?? ""),
+        riskClass,
+        complianceTags: tags,
+    };
+}
+function clamp(value) {
+    const number = typeof value === "number" ? value : 0;
+    return Math.max(0, Math.min(100, number));
+}
+export function assertScore(result) {
+    if (result.riskClass === "compliance") {
+        throw new ComplianceError("source_compliance", result.reason || result.complianceTags.join(","));
+    }
+}

package/vendor/@vfrey/ingest/dist/parse.d.ts

@@ -0,0 +1,5 @@
+import type { Candidate } from "@vfrey/contracts";
+export declare function fingerprint(provider: string, url: string, title: string): string;
+export declare function parseFeed(xml: string, provider: Candidate["provider"]): Candidate[];
+export declare function parseRedditListing(json: unknown): Candidate[];
+export declare function parseYouTubePopular(json: unknown): Candidate[];

package/vendor/@vfrey/ingest/dist/parse.js

@@ -0,0 +1,165 @@
+import { createHash } from "node:crypto";
+const TRACKING_PARAM = /^(utm_[^=]*|fbclid|gclid)$/i;
+/** Lowercase scheme and host only, drop tracking params, keep path and query case. */
+function normalizeUrl(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return url;
+    }
+    const kept = parsed.search
+        .replace(/^\?/, "")
+        .split("&")
+        .filter((pair) => pair !== "" && !TRACKING_PARAM.test(pair.split("=")[0] ?? ""));
+    const search = kept.length > 0 ? `?${kept.join("&")}` : "";
+    return `${parsed.protocol}//${parsed.host}${parsed.pathname}${search}${parsed.hash}`;
+}
+export function fingerprint(provider, url, title) {
+    const key = `${provider.toLowerCase()}|${normalizeUrl(url)}|${title.toLowerCase()}`;
+    return createHash("sha256").update(key).digest("hex").slice(0, 24);
+}
+const NAMED_ENTITIES = {
+    amp: "&",
+    lt: "<",
+    gt: ">",
+    quot: '"',
+    apos: "'",
+    nbsp: " ",
+    rsquo: "\u2019",
+    lsquo: "\u2018",
+    rdquo: "\u201d",
+    ldquo: "\u201c",
+    ndash: "\u2013",
+    mdash: "\u2014",
+    hellip: "\u2026",
+};
+function codePoint(value, original) {
+    if (!Number.isInteger(value) || value < 0 || value > 0x10ffff)
+        return original;
+    try {
+        return String.fromCodePoint(value);
+    }
+    catch {
+        return original;
+    }
+}
+function decodeEntities(value) {
+    return value.replace(/&(#x[0-9a-f]+|#[0-9]+|[a-z]+);/gi, (whole, body) => {
+        if (body[0] === "#") {
+            const hex = body[1] === "x" || body[1] === "X";
+            return codePoint(parseInt(body.slice(hex ? 2 : 1), hex ? 16 : 10), whole);
+        }
+        return NAMED_ENTITIES[body.toLowerCase()] ?? whole;
+    });
+}
+function decode(value) {
+    return decodeEntities(value.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, "$1"))
+        .replace(/<[^>]+>/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+function tag(block, name) {
+    const match = new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)</${name}>`, "i").exec(block);
+    return match ? decode(match[1] ?? "") : "";
+}
+function isHttpUrl(value) {
+    try {
+        const protocol = new URL(value).protocol;
+        return protocol === "http:" || protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+function attrOf(tagText, name) {
+    const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, "i").exec(tagText);
+    return match?.[1] ?? match?.[2] ?? "";
+}
+/** Atom links: prefer rel="alternate" (or no rel) over replies, enclosures, and the like. */
+function atomLink(block) {
+    for (const match of block.matchAll(/<link\b[^>]*>/gi)) {
+        const href = attrOf(match[0], "href");
+        const rel = attrOf(match[0], "rel").toLowerCase();
+        if (href && (rel === "" || rel === "alternate"))
+            return decodeEntities(href);
+    }
+    return "";
+}
+function isoDate(value) {
+    if (!value)
+        return null;
+    const time = Date.parse(value);
+    return Number.isFinite(time) ? new Date(time).toISOString() : null;
+}
+export function parseFeed(xml, provider) {
+    const blocks = xml.split(/<item\b|<entry\b/i).slice(1);
+    const candidates = [];
+    for (const block of blocks) {
+        const title = tag(block, "title");
+        const link = atomLink(block) || tag(block, "link") || tag(block, "id");
+        const summary = tag(block, "description") ||
+            tag(block, "summary") ||
+            tag(block, "content:encoded") ||
+            tag(block, "content") ||
+            tag(block, "media:description");
+        const published = tag(block, "pubDate") || tag(block, "published") || tag(block, "updated");
+        if (!title || !link || !isHttpUrl(link))
+            continue;
+        candidates.push({
+            fingerprint: fingerprint(provider, link, title),
+            provider,
+            title: title.slice(0, 300),
+            url: link.slice(0, 500),
+            excerpt: summary.slice(0, 500),
+            publishedAt: isoDate(published),
+        });
+    }
+    return candidates;
+}
+export function parseRedditListing(json) {
+    const data = json;
+    const children = data.data?.children ?? [];
+    const candidates = [];
+    for (const child of children) {
+        const row = child.data ?? {};
+        if (row.stickied === true || row.over_18 === true)
+            continue;
+        const title = String(row.title ?? "");
+        const permalink = String(row.permalink ?? "");
+        if (!title || !permalink)
+            continue;
+        const url = `https://www.reddit.com${permalink}`;
+        const selftext = String(row.selftext ?? "");
+        candidates.push({
+            fingerprint: fingerprint("reddit", url, title),
+            provider: "reddit",
+            title: title.slice(0, 300),
+            url,
+            excerpt: selftext.slice(0, 500),
+            publishedAt: typeof row.created_utc === "number" ? new Date(row.created_utc * 1000).toISOString() : null,
+        });
+    }
+    return candidates;
+}
+export function parseYouTubePopular(json) {
+    const data = json;
+    const candidates = [];
+    for (const item of data.items ?? []) {
+        const id = item.id ?? "";
+        const title = String(item.snippet?.title ?? "");
+        if (!id || !title)
+            continue;
+        const url = `https://www.youtube.com/watch?v=${id}`;
+        candidates.push({
+            fingerprint: fingerprint("youtube", url, title),
+            provider: "youtube",
+            title: title.slice(0, 300),
+            url,
+            excerpt: String(item.snippet?.description ?? "").slice(0, 500),
+            publishedAt: typeof item.snippet?.publishedAt === "string" ? item.snippet.publishedAt : null,
+        });
+    }
+    return candidates;
+}

package/vendor/@vfrey/ingest/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@vfrey/ingest",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Official RSS, YouTube, and Reddit collectors for vfrey. No upload tokens.",
+  "engines": {
+    "node": ">=22.13"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@vfrey/contracts": "file:../contracts"
+  }
+}