@framers/agentos 0.12.3 → 0.12.5

package/dist/api/generateText.js

@@ -12,7 +12,7 @@
  * the system prompt so the tool loop executes with awareness of the strategy.
  */
 import { randomUUID } from 'node:crypto';
-import { resolveModelOption, resolveProvider, createProviderManager } from './model.js';
+import { resolveModelOption, resolveProvider, createProviderManager, knownProviderPrefixOf } from './model.js';
 import { attachGenAiAttributes, attachUsageAttributes, toTurnMetricUsage } from './observability.js';
 import { fireLlmUsageObserver } from './observers.js';
 import { hostPolicyToRouteParams, mergeRequiredCapabilities, } from './runtime/hostPolicy.js';
@@ -24,6 +24,9 @@ import { recordAgentOSTurnMetrics, withAgentOSSpan } from '../safety/evaluation/
 import { createLogger } from '../core/logging/loggerFactory.js';
 import { globalLLMProviderHealth } from '../core/safety/LLMProviderHealthRegistry.js';
 import { CONTEXT_WINDOW_EXCEEDED_CODE } from '../core/llm/providers/errors/errorCodes.js';
+import { ContextWindowExceededError } from '../core/llm/providers/errors/ContextWindowExceededError.js';
+import { catalogEntryHasCapability, createUncensoredModelCatalog, findCatalogTextModel, } from '../core/llm/routing/UncensoredModelCatalog.js';
+import { checkContextFit } from './runtime/contextWindowFit.js';
 import { describeResponseFormatShape } from './runtime/responseFormatForProvider.js';
 const fallbackLogger = createLogger('fallback');
 /**
@@ -324,7 +327,7 @@ export function addModelUsage(target, usage) {
  * The usage a provider error reports for the request it ended, such as the
  * billed tokens of a refused turn (`details.usage`).
  *
- * @internal Shared with streamText.
+ * @internal Shared with streamText and the completion gateway.
  */
 export function usageOfError(error) {
     return error?.details?.usage;
@@ -662,81 +665,193 @@ export function fallbackHopOverrides(opts, entry) {
     };
 }
 /**
- * Build a policy-tier-aware fallback chain. Used by callers that pass
- * `policyTier: 'mature' | 'private-adult'` so refusals from the
- * primary model (typically gpt-4o, Claude, Gemini: all of which
- * moderate explicit content) re-route to an uncensored OpenRouter
- * model instead of hard-failing the request.
+ * The policy tier a call's fallback chain is built for: the explicit route
+ * tier, the call's tier, the host policy's tier (`standard` when a host
+ * policy names none), then the router's default. The primary's route block
+ * sends the first three terms only, so a delegated base router still sees no
+ * tier when no explicit source set one.
+ */
+export function resolvePolicyTier(opts) {
+    return (opts.routerParams?.policyTier ??
+        opts.policyTier ??
+        hostPolicyToRouteParams(opts.hostPolicy).policyTier ??
+        opts.router?.policyTier);
+}
+/** @internal */
+export const INITIAL_FALLBACK_WALK = Object.freeze({
+    refillsOwed: 0,
+    refusalSeen: false,
+});
+/** Claude, direct or through OpenRouter. */
+function isClaudeEntry(entry) {
+    return (entry.provider === 'anthropic' ||
+        (entry.provider === 'openrouter' && (entry.model ?? '').startsWith('anthropic/')));
+}
+/**
+ * The capabilities a call names explicitly (host policy and route params).
+ * Native tool calling is not inferred from the presence of tools: a model
+ * without it serves a tool-carrying call through the prompt shim.
  *
- * Chain order for mature / private-adult:
- *   1. `nousresearch/hermes-3-llama-3.1-405b` on OpenRouter: leads
- *      the uncensored leaderboard for instruction-following + long-
- *      context comprehension. Same model the wilds-ai
- *      companion-pipeline uses for identity generation on mature+
- *      companions; battle-tested on real workloads.
- *   2. `anthropic/claude-sonnet-4` on OpenRouter: Claude refuses
- *      hard NSFW but is markedly more permissive than gpt-4o for
- *      narrative analysis of explicit fiction (extracting characters
- *      from a CAI-export with mild adult content, etc.). Acts as the
- *      headroom band when Hermes 3 is rate-limited or down.
- *   3. The standard {@link buildFallbackChain} suffix: keeps
- *      availability fallback on top of the policy fallback so a
- *      mature request that hits a Hermes 3 outage AND a Sonnet
- *      outage still has gpt-4o-mini etc. to fall back to (which
- *      will refuse on the explicit case but at least surfaces a
- *      moderation error rather than a network error).
+ * @internal
+ */
+export function explicitRequiredCapabilities(opts) {
+    return (mergeRequiredCapabilities(hostPolicyToRouteParams(opts.hostPolicy).requiredCapabilities, opts.routerParams?.requiredCapabilities) ?? []);
+}
+/**
+ * The provider and model a fallback entry is sent as: resolved the way the
+ * leg's own call resolves it (a provider's default model when the entry
+ * names none, a `provider:model` id split), or the entry as written when it
+ * does not resolve.
  *
- * For `safe` / `standard` tiers, this is identical to
- * {@link buildFallbackChain}: no uncensored prefix needed. Callers
- * that don't pass a tier should keep using the original builder.
+ * @internal Shared with streamText.
+ */
+export function fallbackEntrySentAs(entry) {
+    try {
+        const { providerId, modelId } = resolveModelOption({ provider: entry.provider, model: entry.model }, 'text');
+        return { provider: providerId, model: modelId };
+    }
+    catch {
+        return { provider: entry.provider, model: entry.model };
+    }
+}
+/**
+ * Resolve a call's fallback chain once, at the start of the top-level walk.
+ * Policy-chain entries (`origin: 'policy-default'`) naming the failed first
+ * model are dropped; every entry is checked, as the provider and model it
+ * will be sent as, against the call's explicitly required capabilities and
+ * excluded models (a model outside the catalog counts as capable); the
+ * uncensored group's first two remaining entries become standing legs and
+ * the rest refills. Caller-written entries keep their order.
  *
- * Auto-built fallbacks always require their own env keys; missing
- * keys silently drop the entry rather than throwing, so a partial
- * deploy still produces a usable (shorter) chain.
+ * @internal
+ */
+export function resolveFallbackChain(chain, ctx) {
+    const required = ctx.requiredCapabilities ?? [];
+    // An exclusion names a model as written or as a `provider:model` id; a leg
+    // matches by its model as written and as it is sent.
+    const excluded = new Set();
+    for (const id of ctx.excludedModelIds ?? []) {
+        excluded.add(id);
+        const prefix = knownProviderPrefixOf(id);
+        if (prefix)
+            excluded.add(id.slice(prefix.length + 1));
+    }
+    const isExcluded = (model) => model !== undefined && excluded.has(model);
+    const resolved = [];
+    let uncensoredLegs = 0;
+    for (const entry of chain) {
+        if (entry.origin === 'policy-default' &&
+            entry.provider === ctx.primary.provider &&
+            entry.model === ctx.primary.model) {
+            ctx.onSkip?.(entry, 'failed_primary');
+            continue;
+        }
+        const sentAs = fallbackEntrySentAs(entry);
+        if (isExcluded(entry.model) || isExcluded(sentAs.model)) {
+            ctx.onSkip?.(entry, 'excluded_model');
+            continue;
+        }
+        const catalogEntry = sentAs.model !== undefined ? findCatalogTextModel(sentAs.model, sentAs.provider) : undefined;
+        if (catalogEntry && required.some((capability) => !catalogEntryHasCapability(catalogEntry, capability))) {
+            ctx.onSkip?.(entry, 'missing_capability');
+            continue;
+        }
+        if (entry.origin === 'policy-default' && entry.group === 'uncensored') {
+            resolved.push({ ...entry, walkRole: uncensoredLegs < 2 ? 'standing' : 'refill' });
+            uncensoredLegs += 1;
+        }
+        else {
+            resolved.push(entry);
+        }
+    }
+    return resolved;
+}
+/**
+ * Whether the walk runs `entry` now: a refill only while a standing leg is
+ * owed one, and none of the policy chain's Claude legs after a refusal.
  *
- * @param tier - Caller's intended content tier. Mature/private-adult
- *   triggers the uncensored prefix; safe/standard returns the
- *   availability-only chain.
- * @param excludeProvider - Provider to omit (typically the primary
- *   that already failed). Mirrors {@link buildFallbackChain}.
- * @returns Ordered fallback entries: uncensored prefix (when tier
- *   warrants it) + availability suffix.
+ * @internal
+ */
+export function gateFallbackEntry(state, entry) {
+    if (state.refusalSeen && entry.origin === 'policy-default' && isClaudeEntry(entry)) {
+        return { run: false, reason: 'claude_after_refusal' };
+    }
+    if (entry.walkRole === 'refill') {
+        if (state.refillsOwed < 1)
+            return { run: false, reason: 'refill_not_owed' };
+        return { run: true, state: { ...state, refillsOwed: state.refillsOwed - 1 } };
+    }
+    return { run: true, state };
+}
+/**
+ * The walk's state after an attempt failed with `error`: a standing leg that
+ * failed for anything but a content decline (a timeout, an open breaker and
+ * the context check's refusal included) is owed a refill, and a model
+ * refusal (code `content_filter`; a filter's `content_policy_violation`
+ * does not count) is recorded for the rest of the walk.
+ *
+ * @internal
+ */
+export function advanceFallbackWalk(state, role, error) {
+    const owed = role === 'standing' && !isContentPolicyRefusal(error) ? 1 : 0;
+    const refusal = error?.code === 'content_filter';
+    return {
+        refillsOwed: state.refillsOwed + owed,
+        refusalSeen: state.refusalSeen || refusal,
+    };
+}
+/** The catalog the policy chain reads its ladders from. */
+const POLICY_CATALOG = createUncensoredModelCatalog();
+/** A catalog model never used as a fallback leg: too weak for a rescue reply. */
+const NEVER_A_LEG_MODEL = 'meta-llama/llama-3.1-8b-instruct';
+/**
+ * Build a policy-tier-aware fallback chain, for callers that pass
+ * `policyTier: 'mature' | 'private-adult'`: a refusal or an outage of the
+ * primary walks to an uncensored model before the availability legs.
+ *
+ * Mature and private-adult: the tier's catalog ladder
+ * (the catalog's `getFallbackLadder`; private-adult keeps the
+ * models that permit `erotic`) as OpenRouter legs in the `uncensored`
+ * group, llama-3.1-8b left out, then the {@link buildFallbackChain} suffix.
+ * Every entry is tagged `origin: 'policy-default'`, which lets the walk drop
+ * the leg naming the failed first model, keep two standing uncensored legs,
+ * and pass over Claude legs after a refusal. Safe, standard and an absent
+ * tier get {@link buildFallbackChain} unchanged and untagged.
+ *
+ * Each leg pins `cache: false`. A missing key drops its legs instead of
+ * throwing, so a partial deploy still gets a shorter usable chain.
+ *
+ * @param tier - The call's content tier.
+ * @param excludeProvider - Provider to leave out of the availability suffix
+ *   (typically the failed primary's). The ladder legs ignore it: another
+ *   model on the same provider is a valid rescue, and the walk drops the
+ *   exact failed model.
  */
 export function buildPolicyAwareFallbackChain(tier, excludeProvider) {
-    const isMatureTier = tier === 'mature' || tier === 'private-adult';
-    if (!isMatureTier) {
+    if (tier !== 'mature' && tier !== 'private-adult') {
         return buildFallbackChain(excludeProvider);
     }
     const chain = [];
-    // Hermes 3 405B leads: uncensored, large, instruction-following
-    // proven on the wilds-ai identity-generation path. Skipped when
-    // OPENROUTER_API_KEY is absent rather than throwing; the suffix
-    // chain may still produce a usable fallback.
     if (process.env.OPENROUTER_API_KEY) {
-        chain.push({
-            provider: 'openrouter',
-            model: 'nousresearch/hermes-3-llama-3.1-405b',
-            cache: false,
-        });
-        // Sonnet via OpenRouter as the second uncensored band. We keep
-        // it on OpenRouter (not direct Anthropic) because the chain's
-        // `excludeProvider` semantics treat each entry as a provider
-        // ID: using `anthropic` here would lock out the suffix's
-        // Anthropic fallback. OpenRouter routes Claude under its own
-        // billing surface, so the slot is independent.
-        chain.push({
-            provider: 'openrouter',
-            model: 'anthropic/claude-sonnet-4',
-            cache: false,
-        });
+        const ladder = tier === 'private-adult'
+            ? POLICY_CATALOG.getFallbackLadder('private-adult', { contentIntent: 'erotic' })
+            : POLICY_CATALOG.getFallbackLadder('mature');
+        for (const entry of ladder) {
+            if (entry.modelId === NEVER_A_LEG_MODEL)
+                continue;
+            chain.push({
+                provider: entry.providerId,
+                model: entry.modelId,
+                cache: false,
+                origin: 'policy-default',
+                group: 'uncensored',
+            });
+        }
     }
-    // Append the standard availability chain. Filter out duplicates
-    // since the uncensored prefix may have already added openrouter.
-    const availability = buildFallbackChain(excludeProvider);
-    for (const entry of availability) {
-        const alreadyInChain = chain.some((existing) => existing.provider === entry.provider && existing.model === entry.model);
-        if (!alreadyInChain)
-            chain.push(entry);
+    for (const entry of buildFallbackChain(excludeProvider)) {
+        const listed = chain.some((e) => e.provider === entry.provider && e.model === entry.model);
+        if (!listed)
+            chain.push({ ...entry, origin: 'policy-default' });
     }
     return chain;
 }
@@ -947,6 +1062,28 @@ export async function generateText(opts) {
                     function: { name: t.name, description: t.description, parameters: t.inputSchema },
                 }))
                 : undefined;
+            // A catalog model is sent only a request it can hold: otherwise the
+            // call throws in place of the send and the walk moves on. Checked on
+            // the first native send and on the prompt shim's first send.
+            const assertFitsContextWindow = (sent) => {
+                const fit = checkContextFit({
+                    provider: resolved.providerId,
+                    model: resolved.modelId,
+                    messages: sent.messages,
+                    tools: sent.tools,
+                    maxTokens: opts.maxTokens,
+                    customModelParams: opts.customModelParams,
+                });
+                if (!fit.fits && fit.contextWindow !== undefined) {
+                    throw new ContextWindowExceededError({
+                        provider: resolved.providerId,
+                        model: resolved.modelId,
+                        contextWindow: fit.contextWindow,
+                        estimatedInputTokens: fit.estimatedInputTokens,
+                        outputTokens: fit.outputTokens,
+                    });
+                }
+            };
             const allToolCalls = [];
             const totalUsage = attemptUsage;
             // Provider-reported model id of the final step (spec batch-1 C1);
@@ -1003,6 +1140,7 @@ export async function generateText(opts) {
             // provider's tool-unsupported error (see the catch after the loop).
             const toolMode = opts.toolMode ?? 'auto';
             const shimMaxRoundtrips = opts.maxSteps ?? 5;
+            let shimSendChecked = false;
             const runShim = async () => {
                 const loopResult = await runEmulatedToolLoop({
                     tools: Array.from(toolMap.values()),
@@ -1018,6 +1156,12 @@ export async function generateText(opts) {
                     messages: toShimMessages(messages),
                     maxRoundtrips: shimMaxRoundtrips,
                     callModel: async (msgs) => {
+                        // The shim sends rendered tool text in place of native schemas,
+                        // so its first send is checked on its own.
+                        if (!shimSendChecked) {
+                            shimSendChecked = true;
+                            assertFitsContextWindow({ messages: msgs });
+                        }
                         const r = await provider.generateCompletion(resolved.modelId, msgs, {
                             temperature: opts.temperature,
                             ...(opts.topP !== undefined ? { topP: opts.topP } : {}),
@@ -1176,6 +1320,10 @@ export async function generateText(opts) {
                             console.warn('[agentos] onBeforeGeneration hook error:', hookErr);
                         }
                     }
+                    // A continuation leg's first send carries its completed tool rounds.
+                    // Later steps are not checked again.
+                    if (step === 0)
+                        assertFitsContextWindow({ messages: effectiveMessages, tools: toolSchemas });
                     const response = await withAgentOSSpan('agentos.api.generate_text.step', async (stepSpan) => {
                         stepSpan?.setAttribute('llm.provider', resolved.providerId);
                         stepSpan?.setAttribute('llm.model', resolved.modelId);
@@ -1668,40 +1816,76 @@ export async function generateText(opts) {
             { provider: primaryProviderId ?? 'unknown', model: primaryModelId, ok: false },
         ];
         // ── Fallback chain ────────────────────────────────────────────────
-        // Resolve fallback chain: caller-supplied wins, undefined triggers
-        // auto-build from env keys, empty array explicitly opts out.
-        // When `policyTier` is mature/private-adult, the auto-build picks
-        // the policy-aware chain (Hermes 3 + Sonnet uncensored prefix +
-        // standard availability suffix) instead of the availability-only
-        // chain. Caller-supplied chains are respected verbatim regardless
-        // of tier: explicit beats implicit.
-        const effectiveFallbacks = opts.fallbackProviders === undefined
-            ? buildPolicyAwareFallbackChain(opts.policyTier, metricProviderId)
+        // Caller-supplied wins, undefined auto-builds from env keys for the
+        // call's resolved tier (the policy-aware chain on mature and
+        // private-adult), an empty array opts out. The top-level walk resolves
+        // the chain once: it drops the policy chain's entry for the failed first
+        // model, applies the call's explicit capability and exclusion
+        // requirements, and marks the uncensored group's standing legs and
+        // refills. A leg receives its resolved slice and never resolves it again.
+        const walkTier = resolvePolicyTier(opts);
+        const chainEntries = opts.fallbackProviders === undefined
+            ? buildPolicyAwareFallbackChain(walkTier, metricProviderId)
             : opts.fallbackProviders;
+        const logLegSkip = (entry, reason) => fallbackLogger.info('provider fallback skipped', {
+            event: 'fallback_leg_skipped',
+            api: 'generateText',
+            reason,
+            primaryProvider: metricProviderId,
+            fallbackProvider: entry.provider,
+            fallbackModel: entry.model,
+        });
+        // Resolved only when the walk runs, so its skip lines describe a walk.
         // A call whose prompt-shim tools already ran cannot be continued, and a
         // restart would run them again; it surfaces the error instead.
-        if (effectiveFallbacks.length &&
-            isRetryableError(error) &&
-            !toolProgress.shimRanTool) {
+        const walks = isRetryableError(error) && !toolProgress.shimRanTool;
+        const effectiveFallbacks = !walks
+            ? []
+            : opts.__fallbackWalk
+                ? chainEntries
+                : resolveFallbackChain(chainEntries, {
+                    primary: { provider: metricProviderId, model: metricModelId },
+                    requiredCapabilities: explicitRequiredCapabilities(opts),
+                    excludedModelIds: opts.routerParams?.excludedModelIds,
+                    onSkip: logLegSkip,
+                });
+        // This attempt's own failure updates the walk: a refusal is recorded,
+        // and a standing leg that failed on availability is owed a refill.
+        let walkState = advanceFallbackWalk(opts.__fallbackWalk?.state ?? INITIAL_FALLBACK_WALK, opts.__fallbackWalk?.role, error);
+        if (walks && effectiveFallbacks.length) {
             let lastError = error;
             let attempt = 0;
             for (const fb of effectiveFallbacks) {
                 attempt += 1;
+                // A refill runs only while a standing leg is owed one, and the
+                // policy chain's Claude legs are passed over after a refusal.
+                const gate = gateFallbackEntry(walkState, fb);
+                if (!gate.run) {
+                    logLegSkip(fb, gate.reason);
+                    continue;
+                }
+                walkState = gate.state;
                 // Skip fallback entries whose breaker is already open. Without
                 // this check, the loop would still spend a full network round-
                 // trip on every dead fallback in the chain before reaching the
                 // next healthy one. The recursive `generateText` call below would
                 // also short-circuit at the same isOpen() check, but the outer
                 // skip avoids the recursion overhead + the extra log line.
-                if (globalLLMProviderHealth.isOpen(fb.provider)) {
+                // The breaker read is the provider the leg is sent to: an
+                // `openrouter:` id under another provider goes to OpenRouter.
+                const legProvider = fallbackEntrySentAs(fb).provider;
+                if (globalLLMProviderHealth.isOpen(legProvider)) {
                     fallbackLogger.info('provider fallback skipped (circuit open)', {
                         event: 'fallback_skipped_circuit_open',
                         api: 'generateText',
                         primaryProvider: metricProviderId,
-                        fallbackProvider: fb.provider,
+                        fallbackProvider: legProvider,
                         fallbackModel: fb.model,
                         attempt,
                     });
+                    // An open breaker is an availability failure: a standing leg is
+                    // owed a refill.
+                    walkState = advanceFallbackWalk(walkState, fb.walkRole, undefined);
                     continue;
                 }
                 try {
@@ -1735,6 +1919,12 @@ export async function generateText(opts) {
                         ...opts,
                         provider: fb.provider,
                         model: fb.model,
+                        // Legs run as named: no router re-picks the model, and the call's
+                        // resolved tier travels with them.
+                        router: undefined,
+                        policyTier: walkTier,
+                        // The walk's state before this leg, and its role.
+                        __fallbackWalk: { state: walkState, role: fb.walkRole },
                         // When a builder exists, ALWAYS override _responseFormat —
                         // including with an explicit `undefined` on builder failure or
                         // decline. Merely omitting the key would let the `...opts`
@@ -1764,7 +1954,7 @@ export async function generateText(opts) {
                         // fallback provider rather than the primary's overrides.
                         apiKey: undefined,
                         baseUrl: undefined,
-                        // Preserve the REMAINING explicit chain (entries AFTER the current
+                        // Preserve the REMAINING resolved chain (entries AFTER the current
                         // fb; `attempt` is 1-indexed so slice(attempt) drops fb and all
                         // already-tried entries). This stops the recursion from rebuilding
                         // the default cheap chain (which includes gpt-4o-mini) when a
@@ -1844,6 +2034,7 @@ export async function generateText(opts) {
                     // The leg walked every entry after it and all of them failed.
                     if (chainWalkedBefore(fbError))
                         break;
+                    walkState = advanceFallbackWalk(walkState, fb.walkRole, fbError);
                 }
             }
             // All fallbacks exhausted: fall through to throw