@m6d/cortex-server 1.7.0 → 2.0.0

package/src/lib/ai/context/compressor.ts

@@ -0,0 +1,267 @@
+import type { MessagePart } from "@tanstack/ai";
+import { estimateTokens, CHARS_PER_TOKEN } from "./token-estimator";
+import type { ChatMessage } from "@/types";
+
+/**
+ * Returns a new array of messages with large tool outputs truncated
+ * to `maxTokensPerResult`. Does not mutate the input messages.
+ */
+export function compressToolResults(messages: ChatMessage[], maxTokensPerResult: number) {
+    return messages.map((message) => {
+        const parts = message.parts.map((part) => compressPart(part, maxTokensPerResult));
+        const changed = parts.some((part, index) => part !== message.parts[index]);
+
+        return changed ? { ...message, parts } : message;
+    });
+}
+
+/**
+ * Both halves of a tool call can carry a payload big enough to crowd the window:
+ * the result the tool returned, and the output stashed on the call itself when the
+ * client executed it.
+ */
+function compressPart(part: MessagePart, maxTokensPerResult: number) {
+    if (part.type === "tool-result") {
+        const compressed = compressPayload(part.content, maxTokensPerResult);
+        return compressed === null ? part : { ...part, content: compressed };
+    }
+
+    if (part.type === "tool-call" && part.output !== undefined) {
+        const compressed = compressPayload(part.output, maxTokensPerResult);
+        return compressed === null ? part : { ...part, output: compressed };
+    }
+
+    return part;
+}
+
+/** Returns the compressed form, or null when the payload already fits. */
+function compressPayload(value: unknown, maxTokens: number) {
+    if (estimateTokens(JSON.stringify(value)) <= maxTokens) return null;
+
+    return smartStructuralCompress(value, maxTokens * CHARS_PER_TOKEN);
+}
+
+/**
+ * Structurally compresses a value to fit within a character budget.
+ * Understands JSON arrays, error objects, and nested objects — preserving
+ * semantic meaning better than naive string truncation.
+ */
+// eslint-disable-next-line m6d/no-explicit-return-type -- mutually recursive with compressString, so the return type cannot be inferred
+function smartStructuralCompress(value: unknown, charBudget: number): string {
+    if (typeof value === "string") {
+        return compressString(value, charBudget);
+    }
+
+    if (Array.isArray(value)) {
+        return compressArray(value, charBudget);
+    }
+
+    if (typeof value === "object" && value !== null) {
+        return compressObject(value as Record<string, unknown>, charBudget);
+    }
+
+    // Primitives
+    const str = String(value);
+    if (str.length <= charBudget) return str;
+    return str.slice(0, charBudget - 20) + "\n[...truncated]";
+}
+
+function compressString(value: string, charBudget: number) {
+    if (value.length <= charBudget) return value;
+
+    // Try to parse as JSON first — if it's a stringified structure, compress structurally
+    try {
+        const parsed: unknown = JSON.parse(value);
+        if (typeof parsed === "object" && parsed !== null) {
+            const result = smartStructuralCompress(parsed, charBudget);
+            return result;
+        }
+    } catch {
+        // Not JSON, just truncate
+    }
+
+    return value.slice(0, charBudget - 20) + "\n[...truncated]";
+}
+
+function compressArray(value: unknown[], charBudget: number) {
+    const totalCount = value.length;
+
+    if (totalCount === 0) return "[]";
+
+    // Infer schema from first object element (if items are objects)
+    const firstObj = value.find(
+        (item): item is Record<string, unknown> =>
+            typeof item === "object" && item !== null && !Array.isArray(item),
+    );
+    const schemaKeys = firstObj ? Object.keys(firstObj) : null;
+
+    // Keep as many items as fit within ~60% of budget, leaving room for the summary footer
+    const itemBudget = Math.floor(charBudget * 0.6);
+    const items: unknown[] = [];
+    let accumulated = 0;
+
+    for (const item of value) {
+        const itemStr = JSON.stringify(item);
+        if (accumulated + itemStr.length > itemBudget && items.length > 0) break;
+        accumulated += itemStr.length;
+        items.push(item);
+    }
+
+    const shownStr = JSON.stringify(items, null, 2);
+    const remaining = totalCount - items.length;
+
+    if (remaining === 0 && shownStr.length <= charBudget) return shownStr;
+
+    const schemaHint = schemaKeys ? ` Schema: {${schemaKeys.join(", ")}}` : "";
+    const footer = `\n[COMPRESSED: ${remaining} more items of ${totalCount} total not shown.${schemaHint} Re-execute with filters or pagination to access specific items.]`;
+
+    const result = shownStr + footer;
+    if (result.length <= charBudget) return result;
+
+    // If still too large, fall back to a tighter representation
+    const tightStr = JSON.stringify(items);
+    return tightStr.slice(0, charBudget - footer.length - 20) + footer;
+}
+
+function compressObject(value: Record<string, unknown>, charBudget: number) {
+    // Check if it looks like an error — keep message, drop stack
+    if (isErrorLike(value)) {
+        return compressErrorObject(value, charBudget);
+    }
+
+    const fullStr = JSON.stringify(value, null, 2);
+    if (fullStr.length <= charBudget) return fullStr;
+
+    const allKeys = Object.keys(value);
+
+    // Phase 1: Partition keys into scalar (cheap) and complex (expensive)
+    const scalarEntries: string[] = [];
+    const complexKeys: string[] = [];
+    let scalarCharsUsed = 0;
+
+    for (const key of allKeys) {
+        const val = value[key];
+        const valStr = JSON.stringify(val);
+
+        if (valStr === undefined) {
+            const entry = `  "${key}": null`;
+            scalarEntries.push(entry);
+            scalarCharsUsed += entry.length + 2; // +2 for comma and newline
+        } else if (isScalar(val)) {
+            const entry = `  "${key}": ${valStr}`;
+            scalarEntries.push(entry);
+            scalarCharsUsed += entry.length + 2;
+        } else {
+            complexKeys.push(key);
+        }
+    }
+
+    // Phase 2: Distribute remaining budget to complex values
+    const compressionHeader = `  "__compressed": "Some values were compressed. If you need the full data for a specific key, re-execute the tool returning only that key."`;
+    const headerCost = compressionHeader.length + 2;
+    const structuralOverhead = 4; // { } and newlines
+    const remainingBudget = charBudget - scalarCharsUsed - headerCost - structuralOverhead;
+    const complexBudget = Math.floor(
+        Math.max(remainingBudget, 100) / Math.max(complexKeys.length, 1),
+    );
+
+    const complexEntries: string[] = [];
+    const truncatedKeys: string[] = [];
+
+    for (const key of complexKeys) {
+        const val = value[key];
+        const valStr = JSON.stringify(val);
+
+        if (valStr !== undefined && valStr.length <= complexBudget) {
+            complexEntries.push(`  "${key}": ${valStr}`);
+        } else {
+            truncatedKeys.push(key);
+            complexEntries.push(`  "${key}": ${summarizeValue(val)}`);
+        }
+    }
+
+    // Build result with compression notice
+    const allEntries = [...scalarEntries, ...complexEntries];
+
+    if (truncatedKeys.length > 0) {
+        allEntries.push(compressionHeader);
+        allEntries.push(`  "__truncatedKeys": ${JSON.stringify(truncatedKeys)}`);
+    }
+
+    const result = `{\n${allEntries.join(",\n")}\n}`;
+    if (result.length <= charBudget) return result;
+
+    // Final fallback: keep scalars + summaries, trim from the complex entries end
+    return result.slice(0, charBudget - 20) + "\n[...truncated]";
+}
+
+function isScalar(val: unknown) {
+    return (
+        val === null ||
+        typeof val === "string" ||
+        typeof val === "number" ||
+        typeof val === "boolean"
+    );
+}
+
+function isErrorLike(value: Record<string, unknown>) {
+    return "error" in value || "message" in value || "stack" in value || "stackTrace" in value;
+}
+
+function compressErrorObject(value: Record<string, unknown>, charBudget: number) {
+    // Keep error/message fields, drop stack traces
+    const { stack: _stack, stackTrace: _stackTrace, ...rest } = value;
+    const compressed = JSON.stringify(rest, null, 2);
+
+    if (compressed.length <= charBudget) return compressed;
+    return compressed.slice(0, charBudget - 20) + "\n[...truncated]";
+}
+
+/**
+ * Produces a short human-readable summary of a value's shape and key data.
+ * Used both for inline object-key placeholders and for tool result one-liners.
+ */
+/** The first element stands in for the rest, which is the point of a summary. */
+function summarizeArray(val: unknown[]) {
+    const first = val[0];
+    const schemaHint =
+        typeof first === "object" && first !== null
+            ? ` {${Object.keys(first).slice(0, 5).join(", ")}}`
+            : "";
+
+    return `[Array(${val.length})${schemaHint}]`;
+}
+
+/** A status/success envelope says more as its status plus a count than as its keys. */
+function summarizeStatusEnvelope(obj: Record<string, unknown>, keys: string[]) {
+    const status = obj["status"] ?? obj["success"];
+    const dataKey = keys.find((k) => k === "data" || k === "result" || k === "results");
+    const data = dataKey ? obj[dataKey] : null;
+    const count = Array.isArray(data) ? `, ${data.length} items` : "";
+
+    return `{${String(status)}${count}}`;
+}
+
+function summarizeObject(obj: Record<string, unknown>) {
+    const keys = Object.keys(obj);
+
+    if ("status" in obj || "success" in obj) {
+        return summarizeStatusEnvelope(obj, keys);
+    }
+
+    const shown = keys.slice(0, 5).join(", ");
+    const ellipsis = keys.length > 5 ? ", ..." : "";
+    return `{Object(${keys.length} keys: ${shown}${ellipsis})}`;
+}
+
+export function summarizeValue(val: unknown) {
+    if (Array.isArray(val)) return summarizeArray(val);
+    if (typeof val === "object" && val !== null) {
+        return summarizeObject(val as Record<string, unknown>);
+    }
+    if (typeof val === "string") {
+        return val.length > 80 ? `"${val.slice(0, 77)}..."` : JSON.stringify(val);
+    }
+
+    return String(val);
+}

package/src/lib/ai/context/intra-turn-compressor.ts

@@ -0,0 +1,181 @@
+import { chat } from "@tanstack/ai";
+import type { ModelMessage } from "@tanstack/ai";
+import { createModel } from "@/ai/helpers";
+import { estimateTokens } from "./token-estimator";
+import { summarizeValue } from "./compressor";
+import type { ContextConfig } from "./types";
+
+type SummarizationModelConfig = NonNullable<ContextConfig["summarizationModel"]>;
+
+type ToolCallOrigin = { name: string; step: number };
+
+/**
+ * A tool message names only the call it answers, so both the tool's own name and the
+ * step it belongs to have to be recovered from the assistant message that asked for
+ * it. One tool message carries one result, so a step that issued three calls in
+ * parallel arrives as three messages; regrouping them into their step is what stops
+ * a step's own results from ageing each other out.
+ */
+function toolStepsOf(messages: ModelMessage[]) {
+    const calls = new Map<string, ToolCallOrigin>();
+    const steps: number[][] = [];
+
+    for (const [index, message] of messages.entries()) {
+        for (const call of message.toolCalls ?? []) {
+            calls.set(call.id, { name: call.function.name, step: steps.length });
+        }
+        if (message.role !== "tool") continue;
+
+        // A result whose call was never announced belongs to the step in progress;
+        // filing it anywhere else would invent a step that issued nothing.
+        const step = calls.get(message.toolCallId ?? "")?.step ?? steps.length;
+        (steps[step] ??= []).push(index);
+    }
+
+    return { calls, steps };
+}
+
+function toolNameOf(message: ModelMessage, calls: Map<string, ToolCallOrigin>) {
+    return calls.get(message.toolCallId ?? "")?.name ?? "tool";
+}
+
+/**
+ * A message's content is either a plain string or content parts, of which only the
+ * text ones can be digested — which is all a one-line stand-in was going to keep.
+ */
+function contentText(content: ModelMessage["content"]) {
+    if (typeof content === "string") return content;
+    if (content === null) return "";
+
+    return content.flatMap((part) => (part.type === "text" ? [part.content] : [])).join(" ");
+}
+
+function extractPreview(text: string) {
+    // Try to parse as JSON for structured summary
+    try {
+        const parsed: unknown = JSON.parse(text);
+        if (typeof parsed === "object" && parsed !== null) {
+            return summarizeValue(parsed);
+        }
+    } catch {
+        // Not JSON
+    }
+
+    if (text.length <= 150) return text;
+    return text.slice(0, 147) + "...";
+}
+
+/**
+ * Compresses tool results from older steps in the prompt.
+ *
+ * On step N, tool results from steps 1..N-2 are aggressively compressed
+ * to short one-liners. Only the most recent step's results stay in full.
+ * Does not mutate the input — returns a new array.
+ */
+export function compressIntraTurnToolResults(
+    messages: ModelMessage[],
+    toolResultMaxTokens: number,
+) {
+    const { calls, steps } = toolStepsOf(messages);
+
+    // Below three there is nothing genuinely "old" to compress once the most
+    // recent step is kept intact.
+    if (steps.length < 3) return messages;
+
+    const result = [...messages];
+
+    for (const index of steps.slice(0, -1).flat()) {
+        const message = messages[index]!;
+        const text = contentText(message.content);
+        if (estimateTokens(text) <= toolResultMaxTokens) continue;
+
+        const tag = `[COMPRESSED prior ${toolNameOf(message, calls)} result]`;
+        result[index] = { ...message, content: `${tag} ${extractPreview(text)}` };
+    }
+
+    return result;
+}
+
+/**
+ * Estimates the total token count of tool results in the prompt.
+ */
+export function estimateToolResultTokens(messages: ModelMessage[]) {
+    let total = 0;
+    for (const message of messages) {
+        if (message.role === "tool") total += estimateTokens(contentText(message.content));
+    }
+
+    return total;
+}
+
+const SUMMARIZER_SYSTEM_PROMPT = `You are a tool result summarizer. Given previous tool call results from an AI agent's workflow, produce a concise summary preserving:
+- Key data values, IDs, and counts
+- Success/failure status of each operation
+- Any error messages
+- Data that subsequent tool calls may need
+
+Maximum 300 tokens. Use bullet points. No preamble.`;
+
+function requestSummary(resultTexts: string[], modelConfig: SummarizationModelConfig) {
+    return chat({
+        adapter: createModel(modelConfig, { name: "summarization" }),
+        systemPrompts: [SUMMARIZER_SYSTEM_PROMPT],
+        messages: [
+            {
+                role: "user",
+                content: `Summarize these tool results:\n\n${resultTexts.join("\n\n")}`,
+            },
+        ],
+        stream: false,
+    });
+}
+
+/**
+ * When accumulated tool result tokens exceed the threshold, summarizes
+ * older step results using a lightweight LLM call.
+ *
+ * Replaces every tool result before the last two steps with a single
+ * synthetic summary in the oldest tool message.
+ */
+export async function summarizeOldStepResults(
+    messages: ModelMessage[],
+    thresholdTokens: number,
+    modelConfig: SummarizationModelConfig,
+) {
+    if (estimateToolResultTokens(messages) <= thresholdTokens) return messages;
+
+    const { calls, steps } = toolStepsOf(messages);
+
+    // Summarization keeps the last two steps intact — one more than compression
+    // does, because a summary loses more detail than a truncated one-liner.
+    if (steps.length < 3) return messages;
+
+    // An emptied tool message has nothing to contribute to the digest, and nothing
+    // to gain from being pointed at one.
+    const indices = steps
+        .slice(0, -2)
+        .flat()
+        .filter((index) => contentText(messages[index]!.content) !== "");
+    if (indices.length === 0) return messages;
+
+    const resultTexts = indices.map((index) => {
+        const message = messages[index]!;
+        return `[${toolNameOf(message, calls)}]: ${contentText(message.content)}`;
+    });
+
+    const summary = await requestSummary(resultTexts, modelConfig);
+
+    // The first old tool message carries the summary and the rest point at it, so
+    // the model reads the digest once rather than once per collapsed message.
+    const result = [...messages];
+    indices.forEach((index, position) => {
+        const content =
+            position === 0
+                ? `[Summary of ${indices.length} earlier tool results]:\n${summary}`
+                : "[see summary above]";
+
+        result[index] = { ...messages[index]!, content };
+    });
+
+    return result;
+}

package/src/lib/ai/context/optimize-thread-context.ts

@@ -0,0 +1,50 @@
+import type { ResolvedCortexAgentConfig } from "@/config";
+import type { ChatMessage, Thread } from "@/types";
+import type { ThreadContextMeta } from "./types";
+import { estimateMessagesTokens } from "./token-estimator";
+import { summarizeMessages } from "./summarizer";
+
+/**
+ * Post-response context optimization, called fire-and-forget from onFinish.
+ *
+ * Below the summarization threshold this only refreshes the token estimate —
+ * summarizing early would burn a model call to compress context that still fits.
+ */
+export async function optimizeThreadContext(
+    thread: Thread,
+    messages: ChatMessage[],
+    config: ResolvedCortexAgentConfig,
+) {
+    const contextConfig = config.context;
+    const estimates = estimateMessagesTokens(messages);
+    const totalEstimatedTokens = estimates.reduce((sum, e) => sum + e.tokens, 0);
+    const threshold = contextConfig.maxContextTokens * contextConfig.summarizationThreshold;
+
+    if (totalEstimatedTokens <= threshold) {
+        await config.db.threads.updateContextMeta(thread.id, {
+            summary: thread.contextMeta?.summary ?? null,
+            summaryUpToMessageId: thread.contextMeta?.summaryUpToMessageId ?? null,
+            totalEstimatedTokens,
+            lastOptimizedAt: new Date().toISOString(),
+        } satisfies ThreadContextMeta);
+        return;
+    }
+
+    // The most recent messages stay unsummarized — they're the hot context.
+    const recentCount = Math.min(contextConfig.recentMessagesToKeep, messages.length);
+    const messagesToSummarize = messages.slice(0, messages.length - recentCount);
+    if (messagesToSummarize.length === 0) return;
+
+    const summary = await summarizeMessages(
+        messagesToSummarize,
+        thread.contextMeta?.summary ?? null,
+        contextConfig.summarizationModel ?? config.model,
+    );
+
+    await config.db.threads.updateContextMeta(thread.id, {
+        summary,
+        summaryUpToMessageId: messagesToSummarize.at(-1)?.id ?? null,
+        totalEstimatedTokens,
+        lastOptimizedAt: new Date().toISOString(),
+    } satisfies ThreadContextMeta);
+}

package/src/lib/ai/context/summarizer.ts

@@ -0,0 +1,48 @@
+import { chat } from "@tanstack/ai";
+import { createModel } from "@/ai/helpers";
+import type { ChatMessage } from "@/types";
+import type { ContextConfig } from "./types";
+
+type SummarizationModelConfig = NonNullable<ContextConfig["summarizationModel"]>;
+
+const SYSTEM_PROMPT = `You are a precise conversation summarizer. Produce a concise summary that preserves:
+- Key decisions and conclusions
+- Important entities (names, IDs, URLs, values)
+- User intent and goals
+- Any unresolved questions or next steps
+
+Maximum 500 tokens. Use bullet points. Do not include preamble.`;
+
+export function summarizeMessages(
+    messages: ChatMessage[],
+    existingSummary: string | null,
+    modelConfig: SummarizationModelConfig,
+) {
+    const conversationText = messages
+        .map(function (msg) {
+            const text = msg.parts
+                .flatMap((part) => (part.type === "text" ? [part.content] : []))
+                .join(" ");
+            return `[${msg.role}]: ${text}`;
+        })
+        .join("\n");
+
+    const summaryContext = existingSummary
+        ? `\nPrior summary of earlier messages:\n${existingSummary}\n`
+        : "";
+
+    return chat({
+        adapter: createModel(modelConfig, { name: "summarization" }),
+        systemPrompts: [SYSTEM_PROMPT],
+        messages: [
+            {
+                role: "user",
+                content: `${summaryContext}
+Summarize the following conversation:
+
+${conversationText}`,
+            },
+        ],
+        stream: false,
+    });
+}

package/src/lib/ai/context/token-estimator.ts

@@ -0,0 +1,65 @@
+import type { MessagePart } from "@tanstack/ai";
+import type { ChatMessage } from "@/types";
+
+/** Average characters per token for English text. Used by the heuristic estimator. */
+export const CHARS_PER_TOKEN = 4;
+
+/**
+ * Estimates token count for a string using the chars/4 heuristic.
+ * ~10% accuracy for English text — good enough for budget decisions.
+ */
+export function estimateTokens(text: string) {
+    return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+
+/**
+ * Media and UI parts carry a source, not text. What the provider actually bills
+ * for them is set by its own tokenizer — tiles for an image, pages for a document —
+ * and the length of a base64 payload says nothing useful about it, so they are
+ * charged a flat rate instead.
+ */
+const OPAQUE_PART_TOKENS = 5;
+
+function estimatePartTokens(part: MessagePart) {
+    switch (part.type) {
+        case "text":
+        case "thinking":
+            return estimateTokens(part.content);
+        case "tool-call": {
+            // `arguments` is already the JSON the model emitted. `output` is only
+            // set for client-executed tools and for calls resolved after approval.
+            const output: unknown = part.output;
+            const outputTokens = output === undefined ? 0 : estimateTokens(JSON.stringify(output));
+
+            return estimateTokens(part.arguments) + outputTokens;
+        }
+        case "tool-result":
+            return estimateTokens(
+                typeof part.content === "string" ? part.content : JSON.stringify(part.content),
+            );
+        case "structured-output":
+            return estimateTokens(part.raw);
+        default:
+            return OPAQUE_PART_TOKENS;
+    }
+}
+
+export function estimateMessageTokens(message: ChatMessage) {
+    const parts = message.parts.reduce((sum, part) => sum + estimatePartTokens(part), 0);
+
+    // Per-message overhead (role, metadata framing)
+    return parts + 4;
+}
+
+/**
+ * Estimates token counts for an array of messages.
+ * Returns per-message estimates in the same order.
+ */
+export function estimateMessagesTokens(messages: ChatMessage[]) {
+    return messages.map(function (message) {
+        return {
+            message,
+            tokens: estimateMessageTokens(message),
+        };
+    });
+}

package/src/{ai → lib/ai}/context/types.ts

@@ -10,6 +10,8 @@ export type ContextConfig = {
     };
     toolResultMaxTokens: number;
     recentMessagesToKeep: number;
+    /** Token threshold for triggering intra-turn summarization of older tool results */
+    intraTurnSummarizationThresholdTokens: number;
 };
 
 export type ThreadContextMeta = {
@@ -23,6 +25,7 @@ export const DEFAULT_CONTEXT_CONFIG: ContextConfig = {
     maxContextTokens: 120_000,
     reservedTokenBudget: 8_000,
     summarizationThreshold: 0.75,
-    toolResultMaxTokens: 2_000,
+    toolResultMaxTokens: 1_000,
     recentMessagesToKeep: 6,
+    intraTurnSummarizationThresholdTokens: 20_000,
 };

package/src/{ai → lib/ai}/fetch.ts

@@ -1,4 +1,5 @@
-import type { ResolvedCortexAgentConfig } from "../config.ts";
+import type { ResolvedCortexAgentConfig } from "@/config";
+import { resolveBackendUrl } from "./backend-url";
 
 export async function fetchBackend(
     path: string,
@@ -14,18 +15,22 @@ export async function fetchBackend(
         finalOptions?.body &&
         typeof finalOptions.body === "string"
     ) {
+        let parsed: Record<string, unknown> | undefined;
         try {
-            const parsed = JSON.parse(finalOptions.body) as Record<string, unknown>;
+            parsed = JSON.parse(finalOptions.body) as Record<string, unknown>;
+        } catch {
+            // Non-JSON bodies pass through unchanged.
+        }
+
+        if (parsed !== undefined) {
             const transformed = await backendFetch.transformRequestBody(parsed, {
                 token,
             });
             finalOptions = { ...finalOptions, body: JSON.stringify(transformed) };
-        } catch {
-            // If body isn't valid JSON, pass through as-is
         }
     }
 
-    return fetch(`${backendFetch.baseUrl}${path}`, {
+    return fetch(resolveBackendUrl(backendFetch.baseUrl, path), {
         ...finalOptions,
         headers: {
             "Content-Type": "application/json",