@camera.ui/server 2.2.5-beta.3 → 2.2.5

package/dist/server/src/assistant/manager.js

@@ -3,6 +3,7 @@ import { chat, chatParamsFromRequestBody, EventType, maxIterations, modelMessage
 import { clearToolResults, composeStrategies, evictOldest, withCompaction } from '@tanstack/ai-compaction';
 import { memoryMiddleware } from '@tanstack/ai-memory';
 import { withPersistence } from '@tanstack/ai-persistence';
+import { withSkills } from '@tanstack/ai-skills';
 import { toolCacheMiddleware } from '@tanstack/ai/middlewares';
 import { randomUUID } from 'node:crypto';
 import { hostname } from 'node:os';
@@ -13,24 +14,32 @@ import { PluginsService } from '../api/services/plugins.service.js';
 import { RoomsService } from '../api/services/rooms.service.js';
 import { UsersService } from '../api/services/users.service.js';
 import { decryptPassword, encryptPassword } from '../api/utils/encryption.js';
+import { ALWAYS_EAGER, planRun, promote, toolTokens } from './budget.js';
+import { clearDiscoveryResults, estimateMessage, keepQuestion, keepSkills, PICTURE_TOKENS, trimToolResults } from './compaction.js';
 import { secretGuard } from './guard.js';
-import { repairHistory } from './history.js';
+import { flattenToolHistory, repairHistory } from './history.js';
 import { ASK_USER_INTERRUPT } from './interrupts.js';
 import { AssistantMcp } from './mcp.js';
 import { AssistantMemoryStore, memoryAdapter } from './memory.js';
 import { abortStaleRuns, lmdbPersistence } from './persistence.js';
+import { pluginOfProvider } from './plugin-models.js';
 import { estimateCost, warmPrices } from './pricing.js';
 import { AssistantProfileStore } from './profiles.js';
-import { buildSystemPrompt } from './prompt.js';
+import { composePrompt, promptSections } from './prompt.js';
 import { createAdapter, listModels, providerNeedsKey } from './providers.js';
 import { modelOptionsFor } from './reasoning.js';
+import { RECAP_TOOL, recapCard } from './recap.js';
 import { AssistantToolRegistry, toolGroup } from './registry.js';
 import { withEmptyTurnRetry } from './retry.js';
+import { questionText, ROUTE_TIMEOUT_MS, ROUTED_TOOLS, routeTools } from './router.js';
 import { AssistantScheduler, PUSH_BODY_MAX } from './scheduler.js';
 import { SEARCH_SCHEMA, searchPrompt, toSearchResult } from './search.js';
+import { skillSources, toolsNamedBySkills, withNextSteps } from './skills.js';
 import { AssistantThreadStore, contentParts } from './threads.js';
 import { withToolNameRepair } from './tool-names.js';
+import { leanTools } from './tool-schema.js';
 import { isBrowserTool } from './tools/index.js';
+import { ASSISTANT_CARD_EVENT } from './tools/shared.js';
 import { describeUploads, extractUploads, messagesForModel } from './uploads.js';
 import { AssistantUsageStore } from './usage.js';
 const DEFAULT_CONTEXT_TOKENS = 48_000;
@@ -50,12 +59,25 @@ const CACHED_TOOLS = [
     'automation_catalog',
 ];
 const USAGE_EVENT = 'assistant.usage';
+// what a model without a declared window is assumed to hold on top of the history budget
+const ASSUMED_HEADROOM_TOKENS = 16_384;
+const MIN_CONTEXT_TOKENS = 2_000;
+// a single tool result never takes more than this share of the history budget
+const TOOL_RESULT_SHARE = 3;
+const MIN_TOOL_RESULT_TOKENS = 600;
+const SKILL_TOOLS_SHARE = 0.25;
+const SKILL_TOOLS_TOKENS = 1_200;
+// discovery costs a round of its own, so a plan that hides tools gets its rounds back
+const DISCOVERY_ITERATIONS = 4;
+const MODEL_REFRESH_MS = 5_000;
 const ASK_TIMEOUT_MS = 45_000;
 const ASK_TIMEOUT_MAX_MS = 300_000;
 const ASK_CONCURRENCY = 4;
 const EMPTY_TURN_LOG = 'Assistant: the model ended a turn without text or tool call, asking once more';
 const EXTRACT_TIMEOUT_MS = 30_000;
 const EXTRACT_MAX_CHARS = 1500;
+// a small model stores the questions themselves as facts and they crowd its window, only remembering on request stays
+const EXTRACT_MIN_CONTEXT_TOKENS = 16_000;
 // prettier-ignore
 const EXTRACT_PROMPT = 'You maintain a short list of durable facts about the user of a home camera system: names of people and pets, which camera watches what, ' +
     'how they want answers, routines. From the exchange you get, output JSON {"add": [...], "remove": [...]}: add = new facts as short third-person ' +
@@ -69,7 +91,10 @@ const REPLAY_RETENTION_MS = 5 * 60_000;
 const NO_MIDDLEWARE = { name: 'camera.ui-memory-off' };
 const SEARCH_TIMEOUT_MS = 45_000;
 const REFERENCE_KINDS = new Set(['event', 'episode', 'camera', 'download']);
-const FINAL_TURN_PROMPT = 'The tool budget of this run is used up. Answer now with what you already know and say plainly what you could not check.';
+// prettier-ignore
+const FINAL_TURN_PROMPT = 'The tool budget of this run is used up and no tool is available anymore. Write the answer now as plain text from what the tools already returned, ' +
+    'and say plainly what you could not check. Do not write a tool call or a tool name with arguments into the answer.';
+const DUPLICATE_CALL_RESULT = 'Same call as the one before in this answer, its result is above.';
 const TEST_TIMEOUT_MS = 45_000;
 const PROBE_IMAGE = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII=';
 export class AssistantManager {
@@ -82,6 +107,7 @@ export class AssistantManager {
     mcp;
     runs = new Map();
     askSlots = new Map();
+    modelProvidersCheckedAt = 0;
     memoryAdapter;
     memoryMiddleware;
     logger;
@@ -169,6 +195,13 @@ export class AssistantManager {
             pluginToolCount: this.registry.pluginToolCount,
         };
     }
+    // the settings page asks for this on every open, the plugins answer from their own state
+    async refreshModelProviders() {
+        if (Date.now() - this.modelProvidersCheckedAt < MODEL_REFRESH_MS)
+            return;
+        this.modelProvidersCheckedAt = Date.now();
+        await this.registry.refreshModels();
+    }
     info(role) {
         const { apiKey: _apiKey, provider: _provider, baseURL: _baseURL, model: _model, sendImages: _sendImages, mcpServers, models, ...settings } = this.settings();
         const servers = mcpServers.map(({ token, ...server }) => ({ ...server, tokenSet: token !== null }));
@@ -180,6 +213,7 @@ export class AssistantManager {
                 .listPlugins()
                 .filter((plugin) => hasInterface(plugin.contract, PluginInterface.AssistantTools))
                 .map((plugin) => ({ id: plugin.id, name: plugin.displayName })),
+            modelProviders: this.registry.modelProviders,
             tools: this.registry.describe(role).filter((tool) => settings.terminalEnabled ?? tool.group !== TERMINAL_GROUP),
             external: this.registry.externalStatus(this.externalServers(this.settings())),
         };
@@ -296,6 +330,7 @@ export class AssistantManager {
     async ask(request) {
         const settings = this.settings();
         const entry = this.pluginModel(request.pluginId);
+        await this.ensureModel(entry);
         if (!entry)
             return { ok: false, reason: 'not_allowed', message: `Plugin ${request.pluginId} may not use the assistant model, allow it under Settings, Assistant` };
         if (!settings.enabled || !entryConfigured(entry))
@@ -309,15 +344,15 @@ export class AssistantManager {
         const images = entry.capabilities?.vision !== false
             ? (request.images ?? []).map((img) => ({ type: 'image', source: { type: 'data', value: Buffer.from(img.data).toString('base64'), mimeType: img.mimeType } }))
             : [];
-        const base = {
-            adapter: createAdapter(model, this.decryptKey(entry)),
-            systemPrompts: request.system ? [request.system] : [],
-            messages: [{ role: 'user', content: [{ type: 'text', content: request.prompt }, ...images] }],
-            modelOptions: modelOptionsFor(model),
-            middleware: [countUsage(stats)],
-            abortController: abort,
-        };
         try {
+            const base = {
+                adapter: this.baseAdapter(model, entry, 'en', timeoutMs),
+                systemPrompts: request.system ? [request.system] : [],
+                messages: [{ role: 'user', content: [{ type: 'text', content: request.prompt }, ...images] }],
+                modelOptions: modelOptionsFor(model),
+                middleware: [countUsage(stats)],
+                abortController: abort,
+            };
             if (request.outputSchema) {
                 const json = await chat({ ...base, outputSchema: request.outputSchema });
                 return { ok: true, text: JSON.stringify(json), json, usage: { promptTokens: stats.promptTokens, completionTokens: stats.completionTokens } };
@@ -354,7 +389,15 @@ export class AssistantManager {
         if (providerNeedsKey(entry.provider) && !apiKey) {
             return { ok: false, toolCalling: false, vision: null, latencyMs: 0, model, error: 'This provider needs an API key' };
         }
-        const adapter = createAdapter({ ...entry, model }, apiKey);
+        await this.ensureModel({ provider: entry.provider, model });
+        const spec = this.modelSpec(entry.provider, model);
+        let adapter;
+        try {
+            adapter = this.baseAdapter({ ...entry, model }, { ...entry, model }, 'en', TEST_TIMEOUT_MS);
+        }
+        catch (error) {
+            return { ok: false, toolCalling: false, vision: null, latencyMs: 0, model, error: describeError(error) };
+        }
         let toolCalled = false;
         const probe = toolDefinition({
             name: 'assistant_probe',
@@ -366,43 +409,51 @@ export class AssistantManager {
         });
         const abort = new AbortController();
         const timer = setTimeout(() => abort.abort(), TEST_TIMEOUT_MS);
+        const probeTools = spec?.toolCalling !== false;
         try {
             await this.drain(chat({
                 adapter,
                 systemPrompts: ['You are a connectivity probe. Follow the instruction literally.'],
-                messages: [{ role: 'user', content: 'Call the tool assistant_probe with value 42, then answer with the single word OK.' }],
-                tools: [probe],
-                agentLoopStrategy: maxIterations(3),
-                abortController: abort,
-            }));
-        }
-        catch (error) {
-            clearTimeout(timer);
-            return { ok: false, toolCalling: false, vision: null, latencyMs: Date.now() - started, model, error: describeError(error) };
-        }
-        let vision = null;
-        try {
-            await this.drain(chat({
-                adapter,
                 messages: [
                     {
                         role: 'user',
-                        content: [
-                            { type: 'text', content: 'Answer with one word: what color is this image?' },
-                            { type: 'image', source: { type: 'data', value: PROBE_IMAGE, mimeType: 'image/png' } },
-                        ],
+                        content: probeTools ? 'Call the tool assistant_probe with value 42, then answer with the single word OK.' : 'Answer with the single word OK.',
                     },
                 ],
-                agentLoopStrategy: maxIterations(1),
+                tools: probeTools ? [probe] : [],
+                agentLoopStrategy: maxIterations(3),
                 abortController: abort,
             }));
-            vision = true;
         }
-        catch {
-            vision = false;
+        catch (error) {
+            clearTimeout(timer);
+            return { ok: false, toolCalling: false, vision: null, latencyMs: Date.now() - started, model, error: describeError(error) };
+        }
+        let vision = spec?.vision === false ? false : null;
+        if (vision === null) {
+            try {
+                await this.drain(chat({
+                    adapter,
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                { type: 'text', content: 'Answer with one word: what color is this image?' },
+                                { type: 'image', source: { type: 'data', value: PROBE_IMAGE, mimeType: 'image/png' } },
+                            ],
+                        },
+                    ],
+                    agentLoopStrategy: maxIterations(1),
+                    abortController: abort,
+                }));
+                vision = true;
+            }
+            catch {
+                vision = false;
+            }
         }
         clearTimeout(timer);
-        return { ok: true, toolCalling: toolCalled, vision, latencyMs: Date.now() - started, model };
+        return { ok: true, toolCalling: probeTools && toolCalled, vision, latencyMs: Date.now() - started, model };
     }
     async models(entry, rawKey) {
         try {
@@ -422,6 +473,7 @@ export class AssistantManager {
         const params = await chatParamsFromRequestBody(request.body);
         const forwarded = params.forwardedProps ?? {};
         const entry = this.resolveModel(request.user, { modelId: pickString(forwarded.modelId), profileId: pickString(forwarded.profileId) });
+        await this.ensureModel(entry);
         const model = this.modelSettings(settings, entry);
         const ctx = {
             userId: request.user._id,
@@ -450,18 +502,36 @@ export class AssistantManager {
         if (params.resume?.length ? !resume.length : forwarded.regenerate !== true && repeatsStored(stored?.messages, params.messages)) {
             throw Object.assign(new Error('The conversation changed since this device loaded it'), { statusCode: 409 });
         }
-        const adapter = this.chatAdapter(model, entry);
+        const adapter = this.chatAdapter(model, entry, ctx.language);
         const disabledGroups = new Set(Array.isArray(forwarded.disabledGroups) ? forwarded.disabledGroups.filter((group) => typeof group === 'string') : []);
         if (!settings.terminalEnabled)
             disabledGroups.add(TERMINAL_GROUP);
         const available = entry.capabilities?.toolCalling === false ? [] : this.registry.toolsFor(ctx.role).filter((tool) => !disabledGroups.has(toolGroup(tool)));
-        const tools = resume.length ? available.map((tool) => (tool.lazy ? { ...tool, lazy: false } : tool)) : available;
+        const sections = promptSections(ctx, this.promptFacts(request.user));
+        const plan = this.plan(settings, entry, sections, available);
+        const stats = newStats();
+        const picks = !resume.length && this.routesTools(entry) ? await this.route(model, entry, params.messages, available, plan, stats) : null;
+        const routed = picks !== null;
+        // a routed run has no catalog, so what follows a search has to be there from the start
+        if (routed)
+            promote(plan, available, [...ROUTED_TOOLS, ...picks]);
+        const planned = applyPlan(available, plan);
+        // a routed model gets the few tools that matter and no catalog: it calls what it sees and never opened one
+        const offered = routed ? planned.filter((tool) => !tool.lazy) : planned;
+        const hiddenTools = planned.filter((tool) => tool.lazy).map((tool) => ({ ...tool, lazy: false }));
+        const tools = resume.length ? offered.map((tool) => (tool.lazy ? { ...tool, lazy: false } : tool)) : offered;
+        const prompt = composePrompt(sections, routed ? { ...plan, demoted: 0, hidden: [], routed: picks } : plan);
+        // a procedure names tools the router did not pick, the room for them comes off the history
+        const skillRoom = routed && plan.skillsOnDemand ? Math.min(SKILL_TOOLS_TOKENS, Math.floor(plan.historyTokens * SKILL_TOOLS_SHARE)) : 0;
+        const skillTools = skillRoom ? this.skillTools(model, entry, params.messages, hiddenTools, skillRoom, stats) : undefined;
+        const contextBudget = Math.min(settings.contextTokens, plan.historyTokens - skillRoom);
+        const rounds = settings.maxIterations + (plan.demoted && !routed ? DISCOVERY_ITERATIONS : 0);
         const thread = await this.threads.ensure(request.user._id, params.threadId || randomUUID(), params.messages);
         ctx.threadId = thread._id;
+        await this.threads.setModel(ctx.userId, thread._id, entry._id);
         await this.storeUploads(ctx, thread._id, params.messages.length - 1);
         const abortController = new AbortController();
         request.signal?.addEventListener('abort', () => abortController.abort(), { once: true });
-        const stats = newStats();
         warmPrices(model.provider);
         const runId = params.runId ?? randomUUID();
         this.runs.set(runId, { userId: ctx.userId, threadId: thread._id, abort: abortController });
@@ -475,24 +545,28 @@ export class AssistantManager {
             tools,
             interrupts: [ASK_USER_INTERRUPT],
             context: ctx,
-            systemPrompts: [...cachedPrompts(buildSystemPrompt(ctx, this.promptFacts(request.user)), model.provider), describeUploads(ctx.uploads, ctx.sendImages)].filter(Boolean),
-            agentLoopStrategy: maxIterations(settings.maxIterations + 1),
-            lazyToolsConfig: { includeDescription: 'first-sentence' },
+            systemPrompts: [...cachedPrompts(prompt, model.provider), describeUploads(ctx.uploads, ctx.sendImages)].filter(Boolean),
+            agentLoopStrategy: maxIterations(rounds + 1),
+            lazyToolsConfig: { includeDescription: plan.catalogDescriptions },
             modelOptions: modelOptionsFor(model),
             middleware: [
                 withPersistence(persistence),
                 settings.memoryEnabled ? this.memoryMiddleware : NO_MIDDLEWARE,
+                plan.skillsOnDemand ? withSkills(skillSources()) : NO_MIDDLEWARE,
                 withCompaction({
-                    maxTokens: settings.contextTokens,
-                    strategy: composeStrategies(clearToolResults({ keepRecentToolResults: 6 }), evictOldest({ keepRecentTokens: Math.floor(settings.contextTokens / 2) })),
+                    maxTokens: contextBudget,
+                    estimateTokens: estimateMessage,
+                    strategyKey: `discovery-trim-clear-evict-skills:${contextBudget}`,
+                    strategy: composeStrategies(clearDiscoveryResults(), trimToolResults({ maxTokens: toolResultTokens(contextBudget) }), keepSkills(clearToolResults({ keepRecentToolResults: 6 })), keepQuestion(keepSkills(evictOldest({ keepRecentTokens: Math.floor(contextBudget / 2) })))),
                 }),
-                this.runMiddleware(model, ctx, thread._id, params.messages, stats),
+                this.runMiddleware({ ...model, maxIterations: rounds }, ctx, thread._id, params.messages, stats, pictureLimit(contextBudget), skillTools),
                 toolCacheMiddleware({ ttl: TOOL_CACHE_TTL_MS, toolNames: CACHED_TOOLS }),
                 secretGuard(this.logger),
             ],
             abortController,
         });
         const attachments = {};
+        const recap = tools.some((tool) => tool.name === 'show_report') ? { language: ctx.language, timezone: ctx.timezone } : undefined;
         const observed = this.collectAttachments(stream, attachments, ctx.userId, thread._id, () => {
             const event = usageEvent(model, stats);
             this.usage.record(ctx.userId, event).catch((error) => this.logger.warn(`Assistant: could not record usage: ${error.message}`));
@@ -502,7 +576,7 @@ export class AssistantManager {
             if (run)
                 run.finishedAt = Date.now();
             setTimeout(() => this.runs.delete(runId), REPLAY_RETENTION_MS).unref();
-        });
+        }, recap);
         return toServerSentEventsResponse(observed, { abortController, ...(request.durability ? { durability: { adapter: request.durability } } : {}) });
     }
     async runMcp(request, reply, opts) {
@@ -522,6 +596,7 @@ export class AssistantManager {
         if (!settings.enabled || this.status(user.role).state !== 'ready')
             throw new Error('The assistant is not ready');
         const entry = this.resolveModel(user, { profileId: opts.profile?._id });
+        await this.ensureModel(entry);
         const model = this.modelSettings(settings, entry);
         const ctx = {
             userId: user._id,
@@ -531,11 +606,14 @@ export class AssistantManager {
             sendImages: model.sendImages,
         };
         const disabledGroups = new Set(opts.profile?.disabledGroups ?? []);
-        const tools = entry.capabilities?.toolCalling === false
+        const available = entry.capabilities?.toolCalling === false
             ? []
             : this.registry.toolsFor(ctx.role).filter((tool) => !tool.needsApproval && !isBrowserTool(tool) && !disabledGroups.has(toolGroup(tool)));
+        const sections = promptSections(ctx, this.promptFacts(user));
+        const plan = this.plan(settings, entry, sections, available);
+        const tools = applyPlan(available, plan);
         const systemPrompts = [
-            ...cachedPrompts(buildSystemPrompt(ctx, this.promptFacts(user)), model.provider),
+            ...cachedPrompts(composePrompt(sections, plan), model.provider),
             // prettier-ignore
             'This run is scheduled and unattended: nobody reads along and nobody can confirm anything. ' +
                 'Answer completely in one go, as a report the user reads later on the phone. No questions back, no markdown headings.',
@@ -555,14 +633,15 @@ export class AssistantManager {
         const stats = newStats();
         try {
             const stream = chat({
-                adapter: this.chatAdapter(model, entry),
+                adapter: this.chatAdapter(model, entry, ctx.language),
                 messages: [{ id: `${Date.now()}-user`, role: 'user', parts: parts }],
                 tools,
                 context: ctx,
                 systemPrompts,
                 agentLoopStrategy: maxIterations(settings.maxIterations),
+                lazyToolsConfig: { includeDescription: plan.catalogDescriptions },
                 modelOptions: modelOptionsFor(model),
-                middleware: [countUsage(stats)],
+                middleware: [countUsage(stats), plan.skillsOnDemand ? withSkills(skillSources()) : NO_MIDDLEWARE],
                 abortController,
             });
             for await (const chunk of stream) {
@@ -586,6 +665,7 @@ export class AssistantManager {
         if (!settings.enabled || this.status(user.role).state !== 'ready')
             throw new Error('The assistant is not ready');
         const entry = this.resolveModel(user);
+        await this.ensureModel(entry);
         const model = this.modelSettings(settings, entry);
         const rooms = new RoomsService();
         const cameras = new CamerasService().list().map((camera) => ({ id: camera._id, name: camera.name, room: rooms.label(camera.roomId) ?? camera.room ?? undefined }));
@@ -601,7 +681,7 @@ export class AssistantManager {
         const stats = newStats();
         try {
             const output = await chat({
-                adapter: createAdapter(model, this.decryptKey(entry)),
+                adapter: this.baseAdapter(model, entry, 'en'),
                 systemPrompts: [searchPrompt(cameras, ctx.language === 'auto' ? 'en' : ctx.language, ctx.timezone)],
                 messages: [{ role: 'user', content: text }],
                 outputSchema: SEARCH_SCHEMA,
@@ -616,9 +696,10 @@ export class AssistantManager {
             this.usage.record(user._id, usageEvent(model, stats)).catch((error) => this.logger.warn(`Assistant: could not record usage: ${error.message}`));
         }
     }
-    runMiddleware(settings, runCtx, threadId, incoming, stats) {
+    runMiddleware(settings, runCtx, threadId, incoming, stats, maxPictures = Infinity, skillTools) {
         const log = this.logger;
         const userId = runCtx.userId;
+        const calls = new Set();
         return {
             name: 'camera.ui-assistant',
             onInterruptBoundary: (ctx) => {
@@ -640,14 +721,22 @@ export class AssistantManager {
                 }
                 return undefined;
             },
-            onConfig: (ctx, config) => {
+            onConfig: async (ctx, config) => {
                 const exhausted = ctx.iteration >= settings.maxIterations || stats.toolCalls >= settings.maxToolCalls;
-                const providerMessages = messagesForModel(repairHistory(config.providerMessages ?? config.messages), settings.sendImages);
-                if (!exhausted || !config.tools.length)
-                    return { providerMessages };
-                return { providerMessages, tools: [], systemPrompts: [...config.systemPrompts, FINAL_TURN_PROMPT] };
+                const providerMessages = messagesForModel(repairHistory(config.providerMessages ?? config.messages), settings.sendImages, maxPictures);
+                if (exhausted && config.tools.length) {
+                    return { providerMessages: flattenToolHistory(providerMessages), tools: [], systemPrompts: [...config.systemPrompts, FINAL_TURN_PROMPT] };
+                }
+                const offered = new Set(config.tools.map((tool) => tool.name));
+                const named = ((await skillTools?.(config.messages)) ?? []).filter((tool) => !offered.has(tool.name));
+                return { providerMessages: withNextSteps(providerMessages, offered), tools: leanTools([...config.tools, ...named]) };
             },
-            onBeforeToolCall: (_ctx, hook) => {
+            onBeforeToolCall: (ctx, hook) => {
+                // small models repeat the same call many times in one answer, every copy of the result would fill the window
+                const key = `${ctx.iteration}:${hook.toolName}:${JSON.stringify(hook.args ?? {})}`;
+                if (calls.has(key))
+                    return { type: 'skip', result: DUPLICATE_CALL_RESULT };
+                calls.add(key);
                 stats.toolCalls += 1;
                 if (stats.toolCalls > settings.maxToolCalls) {
                     return { type: 'skip', result: { error: `Tool call budget of ${settings.maxToolCalls} per run is used up, answer with what you have.` } };
@@ -677,15 +766,18 @@ export class AssistantManager {
     }
     async extractFacts(userId, turn, existing) {
         const entry = this.resolveModel({ _id: userId, role: new UsersService().findById(userId)?.role ?? 'user' });
+        await this.ensureModel(entry);
         if (!entry)
             return { add: [], remove: [] };
         const model = this.modelSettings(this.settings(), entry);
+        if (model.contextTokens < EXTRACT_MIN_CONTEXT_TOKENS)
+            return { add: [], remove: [] };
         const abort = new AbortController();
         const timer = setTimeout(() => abort.abort(), EXTRACT_TIMEOUT_MS);
         let text = '';
         try {
             const stream = chat({
-                adapter: createAdapter(model, this.decryptKey(entry)),
+                adapter: this.baseAdapter(model, entry, 'en'),
                 systemPrompts: [EXTRACT_PROMPT],
                 messages: [
                     {
@@ -721,11 +813,36 @@ export class AssistantManager {
             return;
         await this.threads.addAttachments(ctx.userId, threadId, attachments).catch((error) => this.logger.warn(`Assistant: could not store uploads: ${error.message}`));
     }
-    async *collectAttachments(stream, attachments, userId, threadId, usage, done) {
+    async *collectAttachments(stream, attachments, userId, threadId, usage, done, recap) {
+        const names = new Map();
+        let summary;
+        let carded = false;
         try {
             for await (const chunk of stream) {
                 collectAttachment(chunk, attachments);
+                if (chunk.type === EventType.TOOL_CALL_START)
+                    names.set(chunk.toolCallId, chunk.toolCallName);
+                else if (recap && chunk.type === EventType.TOOL_CALL_RESULT && names.get(chunk.toolCallId) === RECAP_TOOL) {
+                    // a later summary of an empty day must not replace one that had moments
+                    const card = recapCard(chunk.content, recap.language, recap.timezone);
+                    if (card)
+                        summary = { toolCallId: chunk.toolCallId, card };
+                }
+                else if (chunk.type === EventType.CUSTOM && chunk.name === ASSISTANT_CARD_EVENT)
+                    carded = true;
                 if (chunk.type === EventType.RUN_FINISHED && finishReasonOf(chunk) !== 'tool_calls') {
+                    // many models answer a recap with a text list although the procedure asks for the card, the episodes are there to build it
+                    if (summary && !carded) {
+                        const event = {
+                            type: EventType.CUSTOM,
+                            name: ASSISTANT_CARD_EVENT,
+                            value: { toolCallId: summary.toolCallId, card: summary.card },
+                            timestamp: Date.now(),
+                        };
+                        collectAttachment(event, attachments);
+                        carded = true;
+                        yield event;
+                    }
                     yield { type: EventType.CUSTOM, name: USAGE_EVENT, value: usage(), timestamp: Date.now() };
                 }
                 yield chunk;
@@ -785,12 +902,82 @@ export class AssistantManager {
         const access = settings.plugins.find((row) => row.pluginId === pluginId);
         return access ? (settings.models.find((entry) => entry._id === access.modelId) ?? defaultEntry(settings)) : undefined;
     }
-    chatAdapter(model, entry) {
-        const adapter = withEmptyTurnRetry(createAdapter(model, this.decryptKey(entry)), () => this.logger.debug(EMPTY_TURN_LOG));
+    chatAdapter(model, entry, language = 'en') {
+        const adapter = withEmptyTurnRetry(this.baseAdapter(model, entry, language), () => this.logger.debug(EMPTY_TURN_LOG));
         return withToolNameRepair(adapter, (from, to) => this.logger.debug(`Assistant: repaired tool name ${from} to ${to}`));
     }
+    modelSpec(provider, model) {
+        return this.registry.modelProvider(provider)?.models.find((candidate) => candidate.id === model);
+    }
+    plan(settings, entry, sections, tools) {
+        const plan = planRun(this.window(settings, entry), sections, tools);
+        if (plan.compactPrompt || plan.demoted) {
+            const steps = `${plan.demoted} tools behind discovery${plan.skillsOnDemand ? ', skills on demand' : ''}`;
+            this.logger.debug(`Assistant: ${entry.name} holds ${plan.window} tokens, the request takes ${plan.overheadTokens} (${steps})`);
+        }
+        return plan;
+    }
+    async route(model, entry, messages, available, plan, stats) {
+        const hidden = available.filter((tool) => !plan.eagerTools.has(tool.name) && !ROUTED_TOOLS.includes(tool.name));
+        const adapter = this.baseAdapter(model, entry, 'en', ROUTE_TIMEOUT_MS);
+        const inFront = available.filter((tool) => tool.name === 'api_search' || (plan.eagerTools.has(tool.name) && !ROUTED_TOOLS.includes(tool.name) && !ALWAYS_EAGER.includes(tool.name)));
+        const picks = await routeTools(adapter, questionText(messages), hidden, inFront, [countUsage(stats)], (message) => this.logger.debug(`Assistant: could not route tools for ${entry.name}: ${message}`));
+        if (picks?.length)
+            this.logger.debug(`Assistant: routed ${picks.join(', ')} in front of ${entry.name}`);
+        return picks;
+    }
+    skillTools(model, entry, messages, hidden, room, stats) {
+        const chosen = new Map();
+        return async (history) => {
+            const named = toolsNamedBySkills(history, hidden);
+            const key = named.map((tool) => tool.name).join();
+            const known = chosen.get(key);
+            if (known || !named.length)
+                return known ?? [];
+            let tools = named;
+            if (toolTokens(named) > room) {
+                const adapter = this.baseAdapter(model, entry, 'en', ROUTE_TIMEOUT_MS);
+                const picks = await routeTools(adapter, questionText(messages), named, [], [countUsage(stats)], (message) => this.logger.debug(`Assistant: could not pick the tools of a procedure for ${entry.name}: ${message}`));
+                tools = picks ? named.filter((tool) => picks.includes(tool.name)) : named.filter((_, index) => toolTokens(named.slice(0, index + 1)) <= room);
+            }
+            chosen.set(key, tools);
+            return tools;
+        };
+    }
+    routesTools(entry) {
+        return entry.toolRouting ?? this.modelSpec(entry.provider, entry.model)?.toolRouting ?? false;
+    }
+    window(settings, entry) {
+        const spec = this.modelSpec(entry.provider, entry.model);
+        if (spec)
+            return Math.max(MIN_CONTEXT_TOKENS, spec.contextTokens);
+        return entry.contextTokens ?? settings.contextTokens + ASSUMED_HEADROOM_TOKENS;
+    }
+    // a plugin is asked for its models once when it starts, an answer from a bad moment must not outlive the next question
+    async ensureModel(entry) {
+        if (!entry || !pluginOfProvider(entry.provider) || this.modelSpec(entry.provider, entry.model))
+            return;
+        await this.refreshModelProviders();
+    }
+    baseAdapter(model, entry, language, timeoutMs) {
+        const provider = this.registry.modelProvider(entry.provider);
+        const spec = this.modelSpec(entry.provider, entry.model);
+        if (provider && spec)
+            return this.registry.modelAdapter(provider, spec, language, timeoutMs);
+        if (pluginOfProvider(entry.provider))
+            throw new Error(provider?.status?.message ?? `The plugin behind the model ${entry.name || entry.model} is not running`);
+        return createAdapter(model, this.decryptKey(entry));
+    }
     modelSettings(settings, entry) {
-        return { ...settings, provider: entry.provider, baseURL: entry.baseURL, apiKey: entry.apiKey, model: entry.model, sendImages: sendsImages(entry) };
+        return {
+            ...settings,
+            provider: entry.provider,
+            baseURL: entry.baseURL,
+            apiKey: entry.apiKey,
+            model: entry.model,
+            sendImages: sendsImages(entry),
+            contextTokens: this.window(settings, entry),
+        };
     }
     askSlot(pluginId) {
         const slot = this.askSlots.get(pluginId) ?? { active: 0, waiters: [] };
@@ -935,6 +1122,18 @@ function imageParts(content) {
     }
     return images;
 }
+function applyPlan(tools, plan) {
+    return tools.map((tool) => {
+        const lazy = !plan.eagerTools.has(tool.name);
+        return lazy === Boolean(tool.lazy) ? tool : { ...tool, lazy };
+    });
+}
+function pictureLimit(contextBudget) {
+    return Math.max(1, Math.floor(contextBudget / 2 / PICTURE_TOKENS));
+}
+function toolResultTokens(contextBudget) {
+    return Math.max(MIN_TOOL_RESULT_TOKENS, Math.floor(contextBudget / TOOL_RESULT_SHARE));
+}
 function cachedPrompts(prompts, provider) {
     if (provider !== 'anthropic')
         return prompts;