@foldrun/core 0.3.0 → 0.4.0

package/src/platform.ts

@@ -12,13 +12,34 @@
 // and calls registerPlatform() once at boot. A process that forgets to is a
 // local install — which is exactly the failure that is safe.
 
-import type { FlowStep, RunRecord } from "./store.ts";
+import type { FlowInfo, FlowStep, RunRecord } from "./store.ts";
 import type { RunInContainerArgs, ContainerStepOutcome } from "./run-container.ts";
 import type { EgressHooks } from "./egress.ts";
 
 export type IsolatedStepRunner = (args: RunInContainerArgs) => Promise<ContainerStepOutcome>;
 
+/** Who this install is for. `hosted` means someone is paying the operator
+ *  for it — a wallet, a plan, a bill. `self-hosted` is everything else: a
+ *  laptop, a company's own box, the open-source download. */
+export type Edition = "self-hosted" | "hosted";
+
+/** What a trigger gate decided. `admit: false` is never an error: the
+ *  trigger was received and understood, and the flow's own file says this
+ *  one does not become a run. `detail` is what the sender and the log are
+ *  told, because a silent drop reads exactly like a broken hook. */
+export type TriggerAdmission =
+  | { admit: true }
+  | { admit: false; reason: "duplicate" | "throttled" | "quarantined" | "debounced"; detail: string; firesAt?: number };
+
 export interface PlatformHooks {
+  /** Is this a hosted install? The ONE question every commercial surface —
+   *  the wallet, top-ups, plan limits, whatever is added next that assumes
+   *  a customer — asks before it renders or answers. Default: self-hosted,
+   *  which is what a process that never registered a platform is, and what
+   *  the open-source image is unless its operator configures billing. One
+   *  predicate, one place, so "does the community build show X" is a test
+   *  and not an audit. */
+  edition(): Edition;
   /** Put a flow run where a worker will pick it up. Default: start it here, now. */
   enqueueFlowRun(
     tenant: string,
@@ -27,10 +48,60 @@ export interface PlatformHooks {
     flowName: string,
     modelOverride?: string | null,
     tags?: string[],
+    /** The person who asked, when one did — their email. A schedule, a
+     *  webhook and a chained flow leave it unset, which is what makes
+     *  "nobody approves their own run" a rule the record can enforce. */
+    startedBy?: string | null,
+    /** `test: true` marks the run a test run (see test-mode.ts). Unset
+     *  lets the platform decide — a preview workspace's runs are tests
+     *  unless the flow says `live: true`. */
+    opts?: { test?: boolean },
   ): Promise<RunRecord>;
   /** A parked run was approved and has no driver; line it up. Default: nothing —
    *  locally the starter that parked it is still polling the record. */
   enqueueResume(tenant: string, workspace: string, runId: string): Promise<void>;
+  /**
+   * What a run has spent so far, in the money the customer is actually
+   * billed — tokens AND the sandbox seconds behind them.
+   *
+   * `budget:` says "the most one run may spend", and for a long time it
+   * only ever compared token cost. A step that called no model — a script
+   * looping on a page that never loads, a browser waiting on a selector
+   * that never appears — spent nothing by that reading and was capped by
+   * nothing, while the sandbox billed by the second. The platform has no
+   * clock of its own by design, so the cap is the only backstop there is,
+   * and it has to count the whole bill for that to be true.
+   *
+   * Default: token cost, which is the entire bill on an install that
+   * prices no compute.
+   */
+  runSpend(run: RunRecord): number;
+  /**
+   * Should a trigger become a run? The flow file's `idempotency:`,
+   * `throttle:`, `debounce:` and `disable_after:` are all answered here,
+   * because all four need state that outlives one delivery — what has been
+   * seen, when the last run started, what is still being waited out.
+   *
+   * Default: yes, always. A laptop has no fleet to protect and no duplicate
+   * deliveries to remember; the local runner is the one asking, and it asked
+   * because a person did.
+   */
+  admitTrigger(
+    tenant: string,
+    workspace: string,
+    flow: FlowInfo,
+    opts?: { deliveryKey?: string | null; body?: string; tag?: string },
+  ): Promise<TriggerAdmission>;
+  /** An admitted trigger whose start then threw. Undoes what admitTrigger
+   *  recorded — the delivery as seen, the throttle window — so the sender's
+   *  retry is a fresh event rather than a "duplicate" of a run that never
+   *  existed. Default: nothing, because the default gate records nothing. */
+  rollbackTrigger(
+    tenant: string,
+    workspace: string,
+    flow: FlowInfo,
+    opts?: { deliveryKey?: string | null; reason?: string },
+  ): Promise<void>;
   /** Step executors by FOLDRUN_RUN_ISOLATION value, beyond the `container`
    *  one core ships. The platform adds `k8s`. */
   isolation: Record<string, IsolatedStepRunner>;
@@ -55,15 +126,35 @@ export interface PlatformHooks {
    *  on its record is resumed rather than destroyed and run again. Default:
    *  nothing is. */
   sandboxResumable(kind: string): boolean;
+  /**
+   * Where the platform's built-in tools sit on disk, materialised at boot —
+   * a shelf every account reads, beneath its own library. A name found in
+   * the workspace wins, then the account's copy, then this. Default: none,
+   * which is what a laptop has; there the gallery is a dashboard feature and
+   * not a directory, and an agent grants only what it or its account owns.
+   */
+  galleryDir(): string | null;
+  /**
+   * An oauth2 secret was just refreshed — `error` null when it worked, the
+   * provider's reason when it did not. The platform keeps each connection's
+   * health, emails once when a grant is dead, and answers with the one-click
+   * reconnect address, which the failing step's error then quotes. Default:
+   * nothing to say, which is a laptop.
+   */
+  noteOAuthRefresh(ctx: { tenant: string; workspace?: string }, name: string, error: string | null): Promise<string | undefined>;
 }
 
 const local: PlatformHooks = {
-  async enqueueFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = []) {
+  edition: () => "self-hosted",
+  async enqueueFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = [], startedBy = null, opts = {}) {
     // Imported here, not at the top: runner.ts imports this file.
     const { startFlowRun } = await import("./runner.ts");
-    return startFlowRun(tenant, workspace, steps, flowName, modelOverride ?? null, tags);
+    return startFlowRun(tenant, workspace, steps, flowName, modelOverride ?? null, tags, null, startedBy, opts);
   },
   async enqueueResume() {},
+  admitTrigger: async () => ({ admit: true }),
+  async rollbackTrigger() {},
+  runSpend: (run) => run.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0),
   isolation: {},
   killRunSandboxes() {},
   syncPublicShares: () => ({ added: [] }),
@@ -72,6 +163,8 @@ const local: PlatformHooks = {
   egress: { lease: async () => null },
   workspaceChanged() {},
   sandboxResumable: () => false,
+  galleryDir: () => null,
+  noteOAuthRefresh: async () => undefined,
 };
 
 // One object per PROCESS, not per module instance. A bundler that compiles
@@ -93,6 +186,17 @@ export function registerPlatform(hooks: Partial<PlatformHooks>): void {
   });
 }
 
+/**
+ * Does a platform own the queue? True once something registered its own
+ * enqueueFlowRun — whatever process asks. An eval used to decide this from
+ * FOLDRUN_ROLE === "web", so on a worker it started the run in-process and
+ * walked around the queue the platform was counting on (the account cap,
+ * the lanes, the per-pod lease). The seam is the fact; the role is a guess.
+ */
+export function platformQueues(): boolean {
+  return platform.enqueueFlowRun !== local.enqueueFlowRun;
+}
+
 /** Back to the local defaults — for tests that register and must not leak. */
 export function resetPlatform(): void {
   Object.assign(platform, local, { isolation: {} });

package/src/provider-health.ts

@@ -0,0 +1,274 @@
+// Is the key still good?
+//
+// A customer's own model key can be revoked, expire, run out of credit or
+// have its base URL moved, and today the platform finds out the way the
+// customer does: a desk fails at five in the morning. Twenty-seven of the
+// thirty provider presets have never had an agent run through them from
+// this runtime at all, so "it worked when we set it up" is doing a lot of
+// work in that sentence.
+//
+// This is deliberately NOT the tool-loop probe next door. `foldrun probe`
+// asks "can this model drive an agent", which is a question about a model
+// and is asked once. This asks "does this endpoint still take our key",
+// which is a question about a credential and has to be asked repeatedly —
+// so it has to be cheap. One request, one token of output, no SDK, no tool
+// loop, no agent. A fraction of a cent, and it answers the thing that
+// actually breaks.
+//
+// It reads the same provider block a run reads, resolved the same way, so a
+// pass here means a run would reach the same place with the same key.
+
+import fs from "node:fs";
+import path from "node:path";
+import matter from "gray-matter";
+import { getSecret } from "./secrets.ts";
+import { noteSecretUse, healthKey } from "./secret-health.ts";
+import { accountDir, workspaceDir, listWorkspaces, listAgents, parseProvider, type ProviderSpec } from "./store.ts";
+
+export type HealthVerdict = "ok" | "credential" | "not-found" | "busy" | "unreachable" | "provider-error";
+
+export interface ProviderCheck {
+  /** How this provider is written in the file: its preset name, or its host. */
+  provider: string;
+  /** The model id actually asked for — what a run would send. */
+  model: string;
+  verdict: HealthVerdict;
+  ok: boolean;
+  status: number | null;
+  ms: number;
+  /** One sentence a person can act on, in the provider's own words where
+   *  they said anything useful. */
+  detail: string;
+}
+
+/** How long to wait. A provider slower than this is not one a step should
+ *  be waiting on either, and the check must never hold a tick. */
+const TIMEOUT_MS = 15_000;
+
+/** What each status means for a credential, and what a person does about
+ *  it. Kept as a table because the mapping IS the feature: "401" tells you
+ *  nothing you can act on, "the key is no longer accepted" tells you
+ *  everything. */
+function verdictFor(status: number): { verdict: HealthVerdict; detail: string } {
+  if (status >= 200 && status < 300) return { verdict: "ok", detail: "the key was accepted and the model answered" };
+  if (status === 401 || status === 403) {
+    return { verdict: "credential", detail: "the key is no longer accepted — revoked, rotated, or wrong for this endpoint" };
+  }
+  if (status === 402) return { verdict: "credential", detail: "the account behind this key is out of credit" };
+  if (status === 404) {
+    return { verdict: "not-found", detail: "the endpoint or the model id does not exist there — check base_url and models:" };
+  }
+  if (status === 429) {
+    // Not a health problem. A busy provider is a provider that is working,
+    // and reporting it as broken would train people to ignore the alert.
+    return { verdict: "busy", detail: "rate limited right now — the key is fine" };
+  }
+  if (status >= 500) return { verdict: "provider-error", detail: "the provider is failing at its end" };
+  return { verdict: "provider-error", detail: `unexpected HTTP ${status}` };
+}
+
+/** `${SECRET}` in a token or header, filled from this account's vault. */
+function resolve(tenant: string, workspace: string | null, raw: string): string {
+  return raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
+    const hit = getSecret(tenant, name, workspace ?? undefined);
+    return hit ? hit.value : whole;
+  });
+}
+
+/**
+ * One request at the provider, in the wire format it speaks, asking for a
+ * single token. Never throws: an unreachable host is a verdict, not an
+ * exception, because this runs on a tick.
+ */
+export async function checkProvider(
+  tenant: string,
+  workspace: string | null,
+  spec: ProviderSpec,
+  model: string,
+): Promise<ProviderCheck> {
+  const label = spec.name || (() => {
+    try {
+      return new URL(spec.baseUrl).host;
+    } catch {
+      return spec.baseUrl || "provider";
+    }
+  })();
+  const began = Date.now();
+  const fail = (verdict: HealthVerdict, detail: string, status: number | null = null): ProviderCheck => ({
+    provider: label,
+    model,
+    verdict,
+    ok: false,
+    status,
+    ms: Date.now() - began,
+    detail,
+  });
+
+  const token = resolve(tenant, workspace, spec.token);
+  if (!token || token.includes("${")) {
+    return fail("credential", `the secret named in provider.token is not in this account's vault (${spec.token})`);
+  }
+  if (!spec.baseUrl) return fail("unreachable", "this provider block has no base_url");
+
+  // The three wire formats, each asking for the smallest possible answer.
+  const base = spec.baseUrl.replace(/\/+$/, "");
+  const anthropic = spec.format === "anthropic";
+  const url = anthropic
+    ? `${base}/v1/messages`
+    : spec.format === "responses"
+      ? `${base}/responses`
+      : `${base}/chat/completions`;
+  const body = anthropic
+    ? { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] }
+    : spec.format === "responses"
+      ? { model, max_output_tokens: 16, input: "hi" }
+      : { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] };
+
+  const headers: Record<string, string> = { "content-type": "application/json" };
+  // An Anthropic-format endpoint takes the key in whichever header its
+  // `auth:` says; everything else is a bearer token.
+  if (anthropic && spec.auth === "x-api-key") {
+    headers["x-api-key"] = token;
+    headers["anthropic-version"] = "2023-06-01";
+  } else {
+    headers.authorization = `Bearer ${token}`;
+    if (anthropic) headers["anthropic-version"] = "2023-06-01";
+  }
+  for (const [k, v] of Object.entries(spec.headers ?? {})) headers[k.toLowerCase()] = resolve(tenant, workspace, v);
+
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(TIMEOUT_MS),
+    });
+    const { verdict, detail } = verdictFor(res.status);
+    // The check is itself a use of the credential, and the most informative
+    // one there is: it was made for exactly this purpose.
+    const named = spec.token.match(/^\$\{([A-Z][A-Z0-9_]*)\}$/)?.[1];
+    if (named) {
+      let host = spec.baseUrl;
+      try {
+        host = new URL(spec.baseUrl).host;
+      } catch {
+        // an unparseable base_url is its own problem, reported above
+      }
+      const hit = getSecret(tenant, named, workspace ?? undefined);
+      noteSecretUse(tenant, healthKey(named, hit?.scope === "workspace" ? "workspace" : "account", workspace), { host, status: res.status });
+    }
+    // The provider's own words when it refused. They are the difference
+    // between "fix your key" and "fix your model id", and they are only
+    // ever in the body.
+    let said = "";
+    if (!res.ok) {
+      said = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 300);
+    }
+    return {
+      provider: label,
+      model,
+      verdict,
+      // Busy counts as healthy: the key worked, the provider is simply full.
+      ok: verdict === "ok" || verdict === "busy",
+      status: res.status,
+      ms: Date.now() - began,
+      detail: said ? `${detail} — ${said}` : detail,
+    };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return fail("unreachable", /timeout|aborted/i.test(message) ? `no answer within ${TIMEOUT_MS / 1000}s` : message);
+  }
+}
+
+
+/** A provider block in use, and where it is declared — because "which file
+ *  do I edit" is the next question after "this key is dead". */
+export interface ProviderInUse {
+  spec: ProviderSpec;
+  /** The workspace whose vault resolves this block's secrets. Null for the
+   *  account's own block, which resolves against the account vault. */
+  workspace: string | null;
+  /** Human-readable origin: "account", "rank-desk", "rank-desk/writer". */
+  declaredIn: string;
+  /** The model id a run would send for the cheapest tier, which is what a
+   *  health check should spend. */
+  model: string;
+}
+
+/** The cheapest tier's id for a provider block: what it calls `fast`, or
+ *  our own fast model when it remaps nothing. */
+function cheapestModel(spec: ProviderSpec): string {
+  return spec.models.fast || spec.models.default || spec.models.max || "claude-haiku-4-5-20251001";
+}
+
+function frontmatterOf(file: string): Record<string, unknown> {
+  try {
+    return (matter(fs.readFileSync(file, "utf8")).data ?? {}) as Record<string, unknown>;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Every distinct provider block this account would actually use, read from
+ * the same three places a run reads them: the account, each workspace, each
+ * agent.
+ *
+ * Deduplicated by endpoint and token, because ten agents sharing one gateway
+ * are one credential and checking it ten times spends ten times as much to
+ * learn the same fact. The origin recorded is the first one seen, which is
+ * the broadest — an account block covers everything under it.
+ */
+export function providersInUse(tenant: string): ProviderInUse[] {
+  const out: ProviderInUse[] = [];
+  const seen = new Set<string>();
+
+  const add = (raw: unknown, workspace: string | null, declaredIn: string) => {
+    const spec = parseProvider(raw);
+    if (!spec || !spec.baseUrl) return;
+    // The token is part of the identity: the same gateway with two different
+    // keys is two credentials, and either can die on its own.
+    const key = `${workspace ?? ""}|${spec.baseUrl}|${spec.token}|${cheapestModel(spec)}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    out.push({ spec, workspace, declaredIn, model: cheapestModel(spec) });
+  };
+
+  add(frontmatterOf(path.join(accountDir(tenant), "AGENTS.md")).provider, null, "account");
+  for (const ws of safeList(() => listWorkspaces(tenant))) {
+    const dir = workspaceDir(tenant, ws.name);
+    add(frontmatterOf(path.join(dir, "AGENTS.md")).provider, ws.name, ws.name);
+    for (const agent of safeList(() => listAgents(tenant, ws.name))) {
+      add(
+        frontmatterOf(path.join(dir, "agents", agent.name, "agent.md")).provider,
+        ws.name,
+        `${ws.name}/${agent.name}`,
+      );
+    }
+  }
+  return out;
+}
+
+function safeList<T>(fn: () => T[]): T[] {
+  try {
+    return fn();
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Check every provider this account uses, one at a time.
+ *
+ * Serial on purpose: this is housekeeping on a tick, the whole point is that
+ * it is cheap, and firing twelve model requests at once to save eight
+ * seconds nobody is waiting for would be a strange trade.
+ */
+export async function checkAccountProviders(tenant: string): Promise<(ProviderCheck & { declaredIn: string })[]> {
+  const results: (ProviderCheck & { declaredIn: string })[] = [];
+  for (const use of providersInUse(tenant)) {
+    const check = await checkProvider(tenant, use.workspace, use.spec, use.model);
+    results.push({ ...check, declaredIn: use.declaredIn });
+  }
+  return results;
+}