@m6d/cortex-server 1.8.0 → 2.0.0

package/src/lib/ai/finish-turn.ts

@@ -0,0 +1,75 @@
+import type { ResolvedCortexAgentConfig } from "@/config";
+import type { ChatMessage, Thread } from "@/types";
+import { notify } from "@/ws/connections";
+import { toThreadSummary } from "@/types";
+import { optimizeThreadContext } from "./context/optimize-thread-context";
+import type { CapturedStep } from "./inspector";
+
+type FinishTurnOptions = {
+    /** What this run produced — the only messages the turn writes. */
+    messages: ChatMessage[];
+    /** The thread including them, which is what a summary has to be judged against. */
+    threadMessages: ChatMessage[];
+    isAborted: boolean;
+    assistantMessageId: string | undefined;
+    thread: Thread;
+    userId: string;
+    capturedSteps: CapturedStep[];
+    config: ResolvedCortexAgentConfig;
+};
+
+/**
+ * Persists everything a completed turn produced, then tells the client to refetch.
+ *
+ * The thread is re-read first: it may have been deleted mid-stream, in which case
+ * there is nothing to write to and the whole turn is discarded.
+ */
+export async function finishTurn(options: FinishTurnOptions) {
+    const { messages, threadMessages, isAborted, assistantMessageId, thread, userId } = options;
+    const { capturedSteps, config } = options;
+
+    const persistedThread = await config.db.threads.getById(userId, thread.id);
+    if (!persistedThread) return;
+
+    await config.db.messages.upsert(thread.id, messages);
+    await persistCapturedSteps(capturedSteps, assistantMessageId, config);
+
+    config.onStreamFinish?.({ messages: threadMessages, isAborted });
+
+    // The turn's terminal event is held back until this write lands, so a client
+    // that refetches on stream end already sees committed rows. This event exists
+    // for the other tabs and devices, which have no stream of their own.
+    notify(userId, persistedThread.agentId, {
+        type: "thread:messages-updated",
+        payload: { threadId: thread.id, thread: toThreadSummary(persistedThread, false) },
+    });
+
+    // Fire-and-forget: the terminal event is waiting on this function, and
+    // summarizing the thread must not be what a user waits for.
+    optimizeThreadContext(thread, threadMessages, config).catch((err: unknown) => {
+        console.error("[cortex-server] Context optimization failed:", err);
+    });
+}
+
+/** Debug data only — losing it must never fail an otherwise-good turn. */
+async function persistCapturedSteps(
+    capturedSteps: CapturedStep[],
+    messageId: string | undefined,
+    config: ResolvedCortexAgentConfig,
+) {
+    if (capturedSteps.length === 0 || !messageId) return;
+
+    try {
+        await config.db.llmRequests.insert(
+            capturedSteps.map((step, index) => ({
+                messageId,
+                stepNumber: index,
+                prompt: step.prompt,
+                output: step.output,
+                tokenUsage: step.tokenUsage,
+            })),
+        );
+    } catch (err) {
+        console.error("[cortex-server] Failed to persist AI requests:", err);
+    }
+}

package/src/lib/ai/helpers.ts

@@ -1,24 +1,61 @@
-import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
-import type { ResolvedCortexAgentConfig } from "../config.ts";
+import { OpenAICompatibleChatAdapter } from "@tanstack/ai-openai/compatible";
+import OpenAI from "openai";
 
-export function createModel(config: ResolvedCortexAgentConfig["model"]) {
-    const provider = createOpenAICompatible({
-        name: config.providerName ?? "default",
-        baseURL: config.baseURL,
-        apiKey: config.apiKey,
-    });
+/** Everything cortex needs to reach one OpenAI-compatible chat endpoint. */
+export type ProviderConfig = {
+    baseURL: string;
+    apiKey: string;
+    modelName: string;
+    providerName?: string;
+};
 
-    return provider(config.modelName);
+/**
+ * `openaiCompatible()` drops streamed reasoning: its `extractReasoning` hook is
+ * a no-op and nothing shipped overrides it. Cortex renders thinking parts, so it
+ * takes the sanctioned subclass seam and reads the `reasoning_content` delta.
+ */
+class ReasoningChatAdapter extends OpenAICompatibleChatAdapter<string> {
+    protected override extractReasoning(chunk: unknown) {
+        const delta = (chunk as { choices?: { delta?: { reasoning_content?: string } }[] })
+            .choices?.[0]?.delta;
+        return delta?.reasoning_content ? { text: delta.reasoning_content } : undefined;
+    }
 }
 
-export function createEmbeddingModel(config: ResolvedCortexAgentConfig["embedding"]) {
-    const provider = createOpenAICompatible({
-        name: "embedding",
+/**
+ * `fetch` is how the caller observes or rewrites the raw HTTP exchange — the LLM
+ * inspector captures through it, the vision model adapts `max_tokens` through it.
+ */
+export function createModel(
+    config: ProviderConfig,
+    options: { name?: string; fetch?: typeof globalThis.fetch } = {},
+) {
+    const client = new OpenAI({
         baseURL: config.baseURL,
         apiKey: config.apiKey,
+        ...(options.fetch ? { fetch: options.fetch } : {}),
     });
 
-    return provider.textEmbeddingModel(config.modelName);
+    return new ReasoningChatAdapter(
+        client,
+        config.modelName,
+        config.providerName ?? options.name ?? "default",
+    );
+}
+
+const visionModels = new WeakMap<object, ReturnType<typeof createModel>>();
+
+/** Vision calls cap output tokens, so they need the max_tokens parameter shim. */
+export function createVisionModel(config: ProviderConfig) {
+    const cached = visionModels.get(config);
+    if (cached) return cached;
+
+    const model = createModel(config, {
+        name: "vision",
+        fetch: createMaxTokensAdaptingFetch(),
+    });
+    visionModels.set(config, model);
+    return model;
 }
 
 export async function streamToBase64(stream: ReadableStream<Uint8Array>) {
@@ -27,6 +64,37 @@ export async function streamToBase64(stream: ReadableStream<Uint8Array>) {
         .then((buffer) => Buffer.from(buffer).toString("base64"));
 }
 
-export function tokenToUserId(token: string) {
-    return JSON.parse(Buffer.from(token.split(".")[1]!, "base64").toString())["sub"] as string;
+/**
+ * OpenAI's newer models reject `max_tokens` in favor of `max_completion_tokens`,
+ * while most OpenAI-compatible gateways only accept `max_tokens`. Retry once on
+ * that specific rejection and remember the answer for this fetch's lifetime.
+ */
+export function createMaxTokensAdaptingFetch() {
+    let renameMaxTokens = false;
+
+    function renamed(init?: RequestInit) {
+        if (typeof init?.body !== "string") return init;
+        const body = JSON.parse(init.body) as Record<string, unknown>;
+        if (!("max_tokens" in body)) return init;
+        body["max_completion_tokens"] = body["max_tokens"];
+        delete body["max_tokens"];
+        return { ...init, body: JSON.stringify(body) };
+    }
+
+    async function adaptiveFetch(input: string | URL | Request, init?: RequestInit) {
+        if (renameMaxTokens) return fetch(input, renamed(init));
+
+        const response = await fetch(input, init);
+        if (response.status === 400) {
+            const text = await response.clone().text();
+            if (text.includes("max_completion_tokens")) {
+                renameMaxTokens = true;
+                return fetch(input, renamed(init));
+            }
+        }
+        return response;
+    }
+
+    // Bun's fetch type carries preconnect; delegate so the provider accepts the shim.
+    return Object.assign(adaptiveFetch, { preconnect: fetch.preconnect });
 }