@vfrey/vfrey 0.0.1 → 1.0.0-beta.1

package/src/pipeline/run.ts

@@ -0,0 +1,443 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { AppError, ComplianceError, QualityError, RetryableError, asAppError } from "@vfrey/contracts";
+import { localDayKey } from "@vfrey/contracts";
+import { cadenceDecision } from "@vfrey/contracts";
+import { backoffMs } from "@vfrey/contracts";
+import type { Account, AppConfig, Candidate, ContentPackage } from "@vfrey/contracts";
+import type { LlmClient } from "../llm/client.js";
+import { log } from "../log.js";
+import { applyDesk } from "../desk/overlay.js";
+import { configWithSecrets, secretEnvFor } from "../desk/secrets.js";
+import { deskAccounts, deskIsEmpty, seedDesk } from "../desk/series.js";
+import { LeaseLost, emptyPayload, type Store, type JobRecord } from "../queue/store.js";
+import { fingerprint } from "@vfrey/ingest";
+import { toRenderProps } from "../render/props.js";
+import type { ScriptFile } from "../generate/script.js";
+import type { EngineId, EngineProbes } from "../product/engines.js";
+import { retimeToVoice } from "../render/framey-template.js";
+import { forcedFailure, renderHtmlEngine, voiceSeconds } from "../render/html-engine.js";
+import { usableSeries } from "./series-guard.js";
+import { StopRequested, noPlatformKeys, runAnalytics, runEngage, stageGenerate, stagePublish, stageRender, stageSource } from "./stages.js";
+
+type Env = Record<string, string | undefined>;
+
+/** What a run records about its picture: an agent-written scene (kept with the run) or the default template. */
+export interface SceneRecord {
+  source: "agent" | "default";
+  /** vfrey's copy of the scene folder, `$VFREY_HOME/scenes/<job id>`. */
+  dir: string | null;
+  kind: "html" | "tsx" | null;
+  bytes: number;
+  files: number;
+}
+
+/** What the render stage hands the renderer when the run has an agent-written scene. */
+export interface SceneRenderContext {
+  dir: string;
+  warn: (message: string) => void;
+}
+
+import type { Project } from "./../project/marker.js";
+
+export interface Runtime {
+  config: AppConfig;
+  env: Env;
+  store: Store;
+  llm: LlmClient;
+  fetchImpl: typeof fetch;
+  now: () => Date;
+  dryRun: boolean;
+  skipRender: boolean;
+  /** The person asked for a video now. Ignore the posting window for this tick. */
+  forceNow?: boolean;
+  /** Limit this tick to one series id. */
+  onlyAccountId?: string;
+  /** A subject the person named. It becomes the one manual candidate of this run; feeds are not read. */
+  about?: string;
+  /** A script the coding agent wrote. It replaces the writer for the one-off job `tick` makes. */
+  script?: ScriptFile;
+  /** Why a series made nothing this tick. tick fills it in; the CLI prints it. */
+  notes?: Array<{ series: string; message: string }>;
+  /** Renders with the engine the stage chose. Test fakes may ignore the third argument. */
+  render: (content: ContentPackage, outputPath: string, engine: EngineId, scene?: SceneRenderContext) => Promise<void>;
+  /** A scene folder the agent wrote (`--scene DIR`). The render stage copies it into the run. */
+  scene?: string;
+  /** An engine the person named for this run (`--engine`). It beats the series and the machine default. */
+  engine?: string;
+  /** What each engine needs on this machine. Defaults to probing the real machine. */
+  probes?: () => Promise<EngineProbes> | EngineProbes;
+  /** The project whose `videos/` folder gets the finished file under a readable name. Null or absent: `renders/<jobId>.mp4`. */
+  project?: Project | null;
+  /** How long one claim holds a job. The worker renews it while the job runs. */
+  leaseMs?: number;
+  /** True once the worker was asked to stop. A job stops at the next safe point and its lease is released. */
+  shouldStop?: () => boolean;
+  /** What the after-render quality checks read the file with. Defaults to ffprobe and ffmpeg on this machine. */
+  review?: import("./review.js").ReviewTools;
+  /** No bytes for this long aborts a binary upload. Default 120 s. */
+  uploadIdleMs?: number;
+}
+
+const NO_KEYS_MESSAGE = "No platform key is set, so no comments were read and no replies were sent.";
+
+export const LEASE_MS = 20 * 60_000;
+
+/** The job's own view: desk settings, plus desk secrets in a local env. process.env is never touched. */
+function deskView(runtime: Runtime, account: Account, withSecrets: boolean): { runtime: Runtime; account: Account } {
+  const overlay = runtime.store.deskOverlay(account.id);
+  const applied = overlay ? applyDesk(account, overlay, runtime.config) : { account, config: runtime.config };
+  if (!withSecrets) return { runtime: { ...runtime, config: applied.config }, account: applied.account };
+  const secretEnv = secretEnvFor(runtime.store, applied.account, runtime.env.CREDENTIALS_KEY);
+  return {
+    runtime: { ...runtime, config: configWithSecrets(applied.config, secretEnv), env: { ...runtime.env, ...secretEnv } },
+    account: applied.account,
+  };
+}
+
+export async function tick(runtime: Runtime): Promise<number> {
+  return (await tickJobs(runtime)).length;
+}
+
+/** The manual candidate for a subject the person named. Each ask is its own, so asking twice is not "seen". */
+export function aboutCandidate(subject: string, now: Date): Candidate {
+  const url = `manual:about/${now.getTime()}`;
+  return { fingerprint: fingerprint("manual", url, subject), provider: "manual", title: subject, url, excerpt: subject, publishedAt: now.toISOString() };
+}
+
+/** Runs a tick and returns the ids of the jobs it processed, in order. */
+export async function tickJobs(runtime: Runtime): Promise<string[]> {
+  const now = runtime.now();
+  seedDesk(runtime.store, runtime.config.accounts);
+  const accounts = usableSeries(
+    runtime.store,
+    deskAccounts(runtime.store).filter((account) => !runtime.onlyAccountId || account.id === runtime.onlyAccountId),
+  );
+  const oneOff = Boolean(runtime.about || runtime.script);
+  const own: string[] = [];
+  for (const account of accounts) {
+    if (oneOff && runtime.script && own.length > 0) break;
+    const view = deskView(runtime, account, false);
+    // A job that waits for review holds no worker and does not stop the cadence: the next due job is made, and waits too.
+    for (const id of runtime.store.awaitingReviewJobs(account.id)) {
+      runtime.notes?.push({ series: account.id, message: `job ${id} waits for review — npx vfrey desk job approve --id ${id}` });
+    }
+    const decision = dueDecision(view.runtime, view.account, now);
+    if (!decision.due) {
+      log("info", "account not due", { accountId: account.id, reason: decision.reason });
+      runtime.notes?.push({ series: account.id, message: decision.reason === "paused" ? "paused" : `not due: ${decision.reason}` });
+      continue;
+    }
+    if (oneOff) {
+      // A video the person asked for is its own job. A series job that is running or waiting does not block it.
+      const subject = (runtime.about ?? runtime.script?.subject ?? runtime.script?.title ?? "").trim();
+      const manual = aboutCandidate(subject, now);
+      const payload = { ...emptyPayload(), candidates: [manual], ...(runtime.script ? { script: runtime.script } : {}) };
+      own.push(runtime.store.enqueue({ accountId: account.id, kind: "pipeline", payload, standalone: true }).id);
+      continue;
+    }
+    const job = runtime.store.enqueue({ accountId: account.id, kind: "pipeline" });
+    if (job.status === "leased" && job.leaseUntil) {
+      runtime.notes?.push({
+        series: account.id,
+        message: `job ${job.id} in progress until ${job.leaseUntil} — npx vfrey desk job cancel --id ${job.id}`,
+      });
+    } else if (job.status === "pending" && new Date(job.runAfter).getTime() > now.getTime()) {
+      runtime.notes?.push({ series: account.id, message: `job ${job.id} waits to retry at ${job.runAfter} — npx vfrey desk job cancel --id ${job.id}` });
+    }
+  }
+  const processed: string[] = [];
+  if (oneOff) {
+    // Only the jobs this ask made run. Series jobs left over from earlier ticks wait for a plain tick.
+    for (const id of own) {
+      if (runtime.shouldStop?.()) break;
+      const job = runtime.store.claim(runtime.now(), runtime.leaseMs ?? LEASE_MS, id);
+      if (!job) continue;
+      await processJob(runtime, job);
+      processed.push(job.id);
+    }
+    return processed;
+  }
+  for (;;) {
+    if (runtime.shouldStop?.()) break;
+    const job = runtime.store.claim(runtime.now(), runtime.leaseMs ?? LEASE_MS);
+    if (!job) break;
+    await processJob(runtime, job);
+    processed.push(job.id);
+    if (processed.length >= 3) break;
+  }
+  return processed;
+}
+
+export function dueDecision(runtime: Pick<Runtime, "store" | "forceNow">, account: Account, now: Date) {
+  if (runtime.store.seriesPaused(account.id)) return { due: false, reason: "paused" };
+  // The person asked for a video now: no window, gap, or daily count. A paused series was refused above.
+  if (runtime.forceNow) return { due: true, reason: "asked" };
+  // The cadence counts videos made, so a series that cannot publish does not render forever.
+  const times = runtime.store.videoTimes(account.id);
+  const day = localDayKey(now, account.cadence.timezone);
+  const madeOnLocalDay = times.filter((time) => localDayKey(time, account.cadence.timezone) === day).length;
+  return cadenceDecision({
+    cadence: account.cadence,
+    now,
+    lastPublishedAt: times[0] ?? null,
+    postsOnLocalDay: madeOnLocalDay,
+  });
+}
+
+export async function processJob(parent: Runtime, job: JobRecord): Promise<void> {
+  if (deskIsEmpty(parent.store)) seedDesk(parent.store, parent.config.accounts);
+  const found = deskAccounts(parent.store).find((item) => item.id === job.accountId);
+  if (!found) {
+    const message = `Series ${job.accountId} is not in the desk.`;
+    parent.store.recordEvent(job.id, { stage: job.stage, status: "blocked", code: "unknown_series", message, durationMs: null, attempt: job.attempt });
+    parent.store.finish(job, "blocked", `unknown_series: ${message}`);
+    return;
+  }
+  const view = deskView(parent, found, true);
+  const runtime = view.runtime;
+  const account = view.account;
+  const leaseMs = runtime.leaseMs ?? LEASE_MS;
+  const renew = setInterval(() => {
+    try {
+      if (!runtime.store.renewLease(job, leaseMs, runtime.now())) log("warn", "lease lost", { jobId: job.id });
+    } catch (error) {
+      log("warn", "lease renewal failed", { jobId: job.id, message: error instanceof Error ? error.message : String(error) });
+    }
+  }, Math.max(10, Math.floor(leaseMs / 3)));
+  renew.unref();
+  try {
+    await runJob(runtime, account, job);
+  } catch (error) {
+    // The claim was cancelled or taken over. The new owner of the job owns its stored sessions; this worker writes nothing more.
+    if (!(error instanceof LeaseLost)) throw error;
+    log("warn", "claim lost", { jobId: job.id });
+    runtime.store.releaseLeaseOwner(job);
+  } finally {
+    clearInterval(renew);
+  }
+}
+
+async function runJob(runtime: Runtime, account: Account, job: JobRecord): Promise<void> {
+  try {
+    if (job.kind === "analytics") {
+      await runAnalytics(runtime, account);
+      runtime.store.finish(job, "succeeded", null);
+      return;
+    }
+    if (job.kind === "engage") {
+      if (noPlatformKeys(runtime, account)) {
+        runtime.store.recordTerminal(job.id, { stage: "engage", status: "skipped", code: "no_keys", message: NO_KEYS_MESSAGE, attempt: job.attempt });
+      } else {
+        await runEngage(runtime, account, null);
+      }
+      runtime.store.finish(job, "succeeded", null);
+      return;
+    }
+    await runPipeline(runtime, account, job);
+  } catch (error) {
+    if (error instanceof LeaseLost) throw error;
+    if (error instanceof StopRequested) {
+      runtime.store.releaseWithoutAttempt(job, runtime.now(), null);
+      runtime.store.recordTerminal(job.id, {
+        stage: job.stage,
+        status: "retrying",
+        code: "shutdown",
+        message: "The worker stopped. The job resumes from this stage on the next tick.",
+        attempt: job.attempt,
+      });
+      return;
+    }
+    const appError = asAppError(error);
+    log(appError.retryable ? "warn" : "error", "job stopped", {
+      jobId: job.id,
+      accountId: account.id,
+      stage: job.stage,
+      code: appError.code,
+      risk: appError.risk,
+      message: appError.message,
+    });
+    if (appError instanceof QualityError) {
+      runtime.store.recordTerminal(job.id, {
+        stage: job.stage,
+        status: "rejected",
+        code: appError.code,
+        message: appError.message,
+        attempt: job.attempt,
+      });
+      runtime.store.finish(job, "rejected", `${appError.code}: ${appError.message}`);
+      return;
+    }
+    if (appError instanceof ComplianceError || (appError instanceof AppError && appError.risk === "config")) {
+      runtime.store.recordTerminal(job.id, {
+        stage: job.stage,
+        status: "blocked",
+        code: appError.code,
+        message: appError.message,
+        attempt: job.attempt,
+      });
+      runtime.store.finish(job, "blocked", `${appError.code}: ${appError.message}`);
+      return;
+    }
+    const wait = backoffMs({
+      attempt: job.attempt,
+      retryAfterMs: appError instanceof RetryableError ? appError.retryAfterMs : undefined,
+      random: () => 0.5,
+    });
+    const retryAt = new Date(runtime.now().getTime() + wait);
+    runtime.store.releaseRetry(job, retryAt, `${appError.code}: ${appError.message}`);
+    const exhausted = job.status === "dead";
+    runtime.store.recordTerminal(job.id, {
+      stage: job.stage,
+      status: exhausted ? "dead" : "retrying",
+      code: appError.code,
+      message: exhausted
+        ? `${appError.message} (attempts exhausted)`
+        : `${appError.message} (next attempt after ${retryAt.toISOString()})`,
+      attempt: job.attempt,
+    });
+  }
+}
+
+async function runPipeline(runtime: Runtime, account: Account, job: JobRecord): Promise<void> {
+  const data = job.payload;
+  const { store } = runtime;
+  const begin = (stage: string) =>
+    store.recordEvent(job.id, { stage, status: "running", code: null, message: null, durationMs: null, attempt: job.attempt });
+  const end = (stage: string, status: string, code: string | null = null, message: string | null = null) =>
+    store.recordTerminal(job.id, { stage, status, code, message, attempt: job.attempt });
+  const safePoint = () => {
+    if (runtime.shouldStop?.()) throw new StopRequested();
+  };
+  if (job.stage === "source") {
+    safePoint();
+    begin("source");
+    await stageSource(runtime, account, data, job);
+    end("source", "succeeded");
+    job.stage = "generate";
+    store.save(job);
+  }
+  if (job.stage === "generate") {
+    safePoint();
+    begin("generate");
+    await stageGenerate(runtime, account, data);
+    end("generate", "succeeded");
+    job.stage = "render";
+    store.save(job);
+  }
+  if (job.stage === "render") {
+    safePoint();
+    begin("render");
+    if ((await stageRender(runtime, account, job, data)) === "awaiting_review") {
+      // The series asked to see this before it renders. Hand back the claim: nothing holds a worker while the person looks.
+      (data as { review?: { state: string; at?: string } }).review = { state: "waiting", at: runtime.now().toISOString() };
+      store.holdForReview(job);
+      end("render", "awaiting_review", "awaiting_review", `Waiting for review. Approve: npx vfrey desk job approve --id ${job.id}`);
+      return;
+    }
+    const used = (data as { engine?: { label?: string } }).engine;
+    end("render", "succeeded", null, used?.label ? `Engine: ${used.label}` : null);
+    job.stage = "publish";
+    store.save(job);
+  }
+  if (job.stage === "publish") {
+    safePoint();
+    begin("publish");
+    const deferral = await stagePublish(runtime, account, job, data);
+    if (deferral) {
+      const note = `${deferral.code}: ${deferral.message}`;
+      if (deferral.countsAttempt) {
+        store.releaseRetry(job, deferral.until, note);
+      } else {
+        store.releaseWithoutAttempt(job, deferral.until, note);
+      }
+      const dead = job.status === "dead";
+      end(
+        "publish",
+        dead ? "dead" : "retrying",
+        deferral.code,
+        dead ? `${deferral.message} (attempts exhausted)` : `${deferral.message} (next try after ${deferral.until.toISOString()})`,
+      );
+      return;
+    }
+    const sessions = Object.entries(data.platforms);
+    const skipCodes = sessions.flatMap(([, item]) => (item?.phase === "skipped" && item.errorCode ? [item.errorCode] : []));
+    const skipCode = skipCodes.includes("publish_off") ? "publish_off" : skipCodes.includes("daily_cap") ? "daily_cap" : skipCodes.includes("missing_key") ? "missing_key" : null;
+    const onlySkips = sessions.every(([, item]) => !item || item.phase === "skipped");
+    const live = store.seriesGates(account.id).live || runtime.dryRun;
+    const summary = (live ? "" : "dry-run, the series is not live: ") + sessions.map(([platform, item]) => `${platform}: ${item?.errorCode ?? item?.phase ?? "none"}`).join("; ");
+    end("publish", skipCode && onlySkips ? "skipped" : "succeeded", skipCode && onlySkips ? skipCode : null, summary || null);
+    job.stage = "engage";
+    store.save(job);
+  }
+  if (job.stage === "engage") {
+    safePoint();
+    begin("engage");
+    let skippedWith: string | null = null;
+    try {
+      if (noPlatformKeys(runtime, account)) skippedWith = "no_keys";
+      else data.engageReplied = await runEngage(runtime, account, data);
+    } catch (error) {
+      // The videos are published. Replies are never worth ending a published job dead.
+      const appError = asAppError(error);
+      log("warn", "engage skipped after publish", { jobId: job.id, code: appError.code, risk: appError.risk });
+      skippedWith = appError.code;
+    }
+    job.payload = data;
+    end("engage", skippedWith ? "skipped" : "succeeded", skippedWith, skippedWith === "no_keys" ? NO_KEYS_MESSAGE : null);
+    store.finish(job, "succeeded", null, runtime.now());
+    store.recordEvent(job.id, { stage: "done", status: "succeeded", code: null, message: null, durationMs: 0, attempt: job.attempt });
+  }
+}
+
+export async function defaultRender(
+  content: ContentPackage,
+  outputPath: string,
+  dataDir: string,
+  engine: EngineId = "remotion",
+  env: Env = process.env,
+  scene?: SceneRenderContext,
+): Promise<void> {
+  const forced = forcedFailure(engine, env);
+  if (forced) throw new Error(forced);
+  if (scene) {
+    if (engine !== "remotion" && engine !== "vfrey-framey" && engine !== "hyperframes") throw new Error(`${engine} does not render scenes.`);
+    const { renderScene } = await import("../scene/render.js");
+    await renderScene({
+      engine,
+      sceneDir: scene.dir,
+      title: content.title,
+      lines: content.voiceoverScript.split("\n").map((line) => line.trim()).filter(Boolean),
+      beats: content.beats.map((beat) => ({ role: beat.role, text: beat.text, caption: beat.caption })),
+      voiceoverPath: content.voiceoverPath,
+      outputPath,
+      workDir: path.join(dataDir, "public", path.basename(outputPath, ".mp4"), `scene-${engine}`),
+      env,
+      warn: scene.warn,
+    });
+    return;
+  }
+  const publicDir = path.join(dataDir, "public", path.basename(outputPath, ".mp4"));
+  if (engine === "vfrey-framey" || engine === "hyperframes") {
+    await renderHtmlEngine({
+      engine,
+      props: toRenderProps(content, null),
+      voiceoverPath: content.voiceoverPath,
+      outputPath,
+      workDir: path.join(publicDir, engine),
+      env,
+    });
+    return;
+  }
+  if (engine !== "remotion") throw new Error(`${engine} has no renderer yet.`);
+  const { renderMp4 } = await import("../render/worker.js");
+  // The picture follows the voice, as it does for the HTML engines, so a short voice is not padded with silence.
+  const props = retimeToVoice(toRenderProps(content, content.voiceoverPath ? "voice.mp3" : null), content.voiceoverPath ? await voiceSeconds(content.voiceoverPath, env) : null);
+  await renderMp4({
+    props,
+    voiceoverPath: content.voiceoverPath,
+    outputPath,
+    publicDir,
+    entryPoint: path.join(path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."), "src/remotion/index.ts"),
+  });
+}

package/src/pipeline/series-guard.ts

@@ -0,0 +1,26 @@
+import type { Account } from "@vfrey/contracts";
+import { log } from "../log.js";
+import { validateCadence } from "../config.js";
+import type { Store } from "../queue/store.js";
+
+/**
+ * Keeps the series whose cadence can run. A series with a bad timezone or window is left out of
+ * this tick with one blocked row that says why, and the others go on.
+ */
+export function usableSeries(store: Store, accounts: Account[]): Account[] {
+  return accounts.filter((account) => {
+    const fault = validateCadence(account.cadence);
+    if (!fault) return true;
+    const error = `invalid_cadence: ${fault}`;
+    log("warn", "series skipped", { accountId: account.id, reason: error });
+    const seen = store.db
+      .prepare(`SELECT 1 FROM jobs WHERE account_id = ? AND status = 'blocked' AND last_error = ? LIMIT 1`)
+      .get(account.id, error);
+    if (!seen) {
+      const job = store.enqueue({ accountId: account.id, kind: "pipeline" });
+      store.recordEvent(job.id, { stage: job.stage, status: "blocked", code: "invalid_cadence", message: fault, durationMs: null, attempt: 0 });
+      store.finish(job, "blocked", error);
+    }
+    return false;
+  });
+}

package/src/pipeline/source-fetch.ts

@@ -0,0 +1,69 @@
+// Fetches a series' feed, subreddit and YouTube sources one at a time, so each source gets its own result:
+// n candidates, or the error code. The result is stored on the source row for the Sources screen.
+import { collectCandidates } from "@vfrey/ingest";
+import { log } from "@vfrey/contracts";
+import type { Account, Candidate } from "@vfrey/contracts";
+import type { Runtime } from "./run.js";
+
+const isHttp = (url: string): boolean => {
+  try {
+    const protocol = new URL(url).protocol;
+    return protocol === "http:" || protocol === "https:";
+  } catch {
+    return false;
+  }
+};
+
+type Sources = Account["sources"];
+const alone = (account: Account, sources: Partial<Sources>): Account => ({
+  ...account,
+  sources: { rss: [], subreddits: [], youtubeChannelIds: [], youtubeTrendingRegion: null, ...sources },
+});
+
+/** The error code of a thrown source failure. */
+export const codeOf = (error: unknown): string => {
+  const code = (error as { code?: unknown } | null)?.code;
+  return typeof code === "string" && code ? code : "source_failed";
+};
+
+/**
+ * Collects every non-Topics source of the series on its own and records each result. Behaves like one
+ * `collectCandidates` call: a failed source never discards the others, and when nothing was collected and a
+ * source failed, the first failure is rethrown so the worker can retry or stop.
+ */
+export async function collectPerSource(runtime: Pick<Runtime, "config" | "fetchImpl" | "now" | "store">, account: Account): Promise<Candidate[]> {
+  // What to read comes from the account (the desk rows, overlaid on the base account); the row id is only for the record.
+  const rows = runtime.store.db.prepare(`SELECT id, kind, value FROM desk_sources WHERE series_id = ?`).all(account.id) as Array<{ id: string; kind: string; value: string }>;
+  const rowId = (kind: string, value: string) => rows.find((row) => row.kind === kind && row.value === value)?.id ?? null;
+  const plan: Array<{ kind: string; value: string; sources: Partial<Sources> }> = [
+    ...account.sources.rss.map((value) => ({ kind: "rss", value, sources: { rss: [value] } })),
+    ...account.sources.subreddits.map((value) => ({ kind: "subreddit", value, sources: { subreddits: [value] } })),
+    ...account.sources.youtubeChannelIds.map((value) => ({ kind: "youtubeChannel", value, sources: { youtubeChannelIds: [value] } })),
+    ...(account.sources.youtubeTrendingRegion ? [{ kind: "youtubeTrending", value: account.sources.youtubeTrendingRegion, sources: { youtubeTrendingRegion: account.sources.youtubeTrendingRegion } }] : []),
+  ];
+  const found: Candidate[] = [];
+  const thrown: unknown[] = [];
+  const record = (kind: string, value: string, count: number | null, code: string | null) => {
+    const id = rowId(kind, value);
+    if (id) runtime.store.recordSourceFetch(id, { at: runtime.now(), count, code, status: code ? "error" : "ok" });
+  };
+  for (const item of plan) {
+    if (item.kind === "rss" && !isHttp(item.value)) {
+      log("warn", "source url rejected", { accountId: account.id, code: "source_url_rejected", url: item.value });
+      record(item.kind, item.value, null, "source_url_rejected");
+      continue;
+    }
+    try {
+      const items = await collectCandidates({ config: runtime.config, account: alone(account, item.sources), fetchImpl: runtime.fetchImpl, manual: [] });
+      found.push(...items);
+      record(item.kind, item.value, items.length, null);
+    } catch (error) {
+      thrown.push(error);
+      record(item.kind, item.value, null, codeOf(error));
+    }
+  }
+  if (found.length === 0 && thrown.length > 0) throw thrown[0];
+  const deduped = new Map<string, Candidate>();
+  for (const candidate of found) if (!deduped.has(candidate.fingerprint)) deduped.set(candidate.fingerprint, candidate);
+  return [...deduped.values()].slice(0, 30);
+}