larkup 0.1.14 → 0.1.21

package/app/analytics/page.tsx

@@ -0,0 +1,19 @@
+import { AnalyticsDashboard } from '@/components/analytics/analytics-dashboard';
+
+export default function AnalyticsPage() {
+  return (
+    <div className="flex flex-col h-full overflow-hidden">
+      <div className="px-6 pt-6 pb-2 md:px-8 mx-auto max-w-7xl w-full">
+        <h1 className="text-xl font-semibold tracking-tight text-foreground">Analytics</h1>
+        <p className="text-sm text-muted-foreground mt-1">
+          Track usage, requests, and performance of your AI server.
+        </p>
+      </div>
+      <div className="flex-1 min-h-0 overflow-y-auto">
+        <div className="p-6 md:p-8 mx-auto max-w-7xl w-full">
+          <AnalyticsDashboard />
+        </div>
+      </div>
+    </div>
+  );
+}

package/app/api/analytics/route.ts

@@ -0,0 +1,31 @@
+import { NextResponse } from 'next/server';
+import { getAnalyticsSummary } from '@larkup/core/analytics-store';
+import { runWithServer } from '@larkup/core/workspace';
+
+export const dynamic = 'force-dynamic';
+
+export async function GET(req: Request) {
+  const url = new URL(req.url);
+  const serverId = url.searchParams.get('serverId');
+  const timeframe = url.searchParams.get('timeframe') || '30d';
+
+  let days = 30;
+  if (timeframe === '7d') days = 7;
+  else if (timeframe === '14d') days = 14;
+  else if (timeframe === '30d') days = 30;
+  else if (timeframe === '90d') days = 90;
+  else if (timeframe === 'all') days = 0;
+
+  const getSummary = async () => {
+    return await getAnalyticsSummary(days);
+  };
+
+  try {
+    const summary = serverId ? await runWithServer(serverId, getSummary) : await getSummary();
+
+    return NextResponse.json(summary);
+  } catch (error: any) {
+    console.error('Analytics fetch error:', error);
+    return NextResponse.json({ error: error.message }, { status: 500 });
+  }
+}

package/app/api/chat/route.ts

@@ -1,192 +1,378 @@
-import {
-  streamText,
-  convertToModelMessages,
-  tool,
-  stepCountIs,
-  type UIMessage,
-} from "ai";
-import { z } from "zod";
-import { readConfig } from "@larkup/core/config-store";
-import { readRun } from "@larkup/core/index-store";
-import { refreshServerStatus } from "@larkup/core/generator/server-runtime";
-import { createAdapter } from "@larkup/vector-stores/factory";
-import { embedQuery } from "@larkup/core/indexing/embedder";
-import { runWithServer } from "@larkup/core/workspace";
-import {
-  getDefaultChatModel,
-  getChatModel,
-} from "@larkup/core/chat-models/registry";
-
-import { createOpenAI } from "@ai-sdk/openai";
-import { createGoogleGenerativeAI } from "@ai-sdk/google";
-import { createCohere } from "@ai-sdk/cohere";
-import { createMistral } from "@ai-sdk/mistral";
-import { createDeepSeek } from "@ai-sdk/deepseek";
-import { createGateway } from "@ai-sdk/gateway";
+import { streamText, convertToModelMessages, stepCountIs, type UIMessage } from 'ai';
+import { readConfig } from '@larkup/core/config-store';
+import { getModelsByType } from '@larkup/core/models-cache';
+import { toChatDescriptor, getDefaultChatModel } from '@larkup/core/chat-models/registry';
+import { listTabularDatasets } from '@larkup/core/tabular-store';
 
-export const maxDuration = 60;
-
-const DEFAULT_SYSTEM_PROMPT = `You are a helpful research assistant powered by a knowledge base.
+import { createOpenAI } from '@ai-sdk/openai';
+import { createGoogleGenerativeAI } from '@ai-sdk/google';
+import { createCohere } from '@ai-sdk/cohere';
+import { createMistral } from '@ai-sdk/mistral';
+import { createDeepSeek } from '@ai-sdk/deepseek';
+import { createAnthropic } from '@ai-sdk/anthropic';
+import { createGateway } from '@ai-sdk/gateway';
+import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
+import type { CustomModelConfig } from '@larkup/core/types';
+import { DEFAULT_SYSTEM_PROMPT } from '@larkup/core/types';
 
-You have one tool:
-- "searchKnowledgeBase" — searches a private RAG knowledge base.
+import { getChatTools } from './tools';
 
-Guidelines:
-- ONLY skip the tool if the user's message is a basic greeting (e.g., "hi", "hello") or simple conversational filler.
-- For ALL OTHER messages (especially questions about facts, personal preferences, instructions, or specific topics), you MUST call the searchKnowledgeBase tool FIRST before answering. Do not assume you cannot answer a question (like "what do I like?") without checking the knowledge base first!
-- Synthesize a clear, well-structured answer based on the retrieved documents.
-- Cite sources inline using markdown links to their URLs when available.
-- Be concise and accurate. Never fabricate sources or facts.
-- Don't reply to the user with a question — try to give the answer unless you truly need clarification.
-- IMPORTANT: If the searchKnowledgeBase tool returns empty results, inform the user that you currently have no knowledge in your database for their specific question. However, you may still attempt to answer general questions using your general knowledge.
-`;
+export const maxDuration = 60;
 
 /**
  * Creates an AI SDK language model instance based on the provider and model ID.
  */
-function createChatModel(provider: string, modelId: string, apiKey?: string) {
-  const modelName = modelId.includes("/")
-    ? modelId.split("/").slice(1).join("/")
-    : modelId;
-  const key = apiKey;
+function createChatModel(
+  provider: string,
+  modelId: string,
+  apiKey?: string,
+  customChatModels?: CustomModelConfig[],
+) {
+  if (modelId.startsWith('custom:')) {
+    const customName = modelId.slice('custom:'.length);
+    const custom = (customChatModels ?? []).find((m) => m.modelName === customName);
+    if (custom) {
+      const customProvider = createOpenAICompatible({
+        name: 'custom_chat_provider',
+        baseURL: custom.baseUrl,
+        apiKey: custom.apiKey || apiKey || undefined,
+      });
+      return customProvider(custom.modelName);
+    }
+  }
+
+  const modelName = modelId.includes('/') ? modelId.split('/').slice(1).join('/') : modelId;
 
   switch (provider) {
-    case "google":
-      return createGoogleGenerativeAI({ apiKey: key })(modelName);
-    case "cohere":
-      return createCohere({ apiKey: key })(modelName);
-    case "mistral":
-      return createMistral({ apiKey: key })(modelName);
-    case "deepseek":
-      return createDeepSeek({ apiKey: key })(modelName);
-    case "vercel_ai_gateway":
-      return createGateway({ apiKey: key })(modelId);
-    case "openai":
+    case 'google':
+      return createGoogleGenerativeAI({ apiKey })(modelName);
+    case 'cohere':
+      return createCohere({ apiKey })(modelName);
+    case 'mistral':
+      return createMistral({ apiKey })(modelName);
+    case 'deepseek':
+      return createDeepSeek({ apiKey })(modelName);
+    case 'anthropic':
+      return createAnthropic({ apiKey })(modelName);
+    case 'openai':
+      return createOpenAI({ apiKey })(modelName);
+    case 'vercel_ai_gateway':
     default:
-      return createOpenAI({ apiKey: key })(modelName);
+      return createGateway({ apiKey })(modelId);
   }
 }
 
-/**
- * Retrieves documents from the knowledge base — either via the running
- * generated server or directly from the local vector store.
- */
-async function queryKnowledgeBase(
-  query: string,
-  topK: number,
-  serverId: string | null,
-) {
-  const doRetrieve = async () => {
-    const config = await readConfig();
-
-    // 1) Try running generated server first
-    const server = await refreshServerStatus();
-    if (server.running) {
-      try {
-        const res = await fetch(`${server.endpoint}/query`, {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ query, topK }),
-          signal: AbortSignal.timeout(15_000),
-        });
-        const data = await res.json();
-        if (res.ok && data.hits) {
-          return {
-            query,
-            hits: (data.hits as any[]).map((h: any) => ({
-              title: h.title ?? "Untitled",
-              url: h.url ?? "",
-              score: Number((h.score ?? 0).toFixed(3)),
-              text: (h.text ?? "").slice(0, 1200),
-            })),
-          };
-        }
-      } catch {
-        // Fall through to direct retrieval
-      }
-    }
+const SMART_RETRIEVAL_POLICY = `
 
-    // 2) Direct retrieval from local vector store
-    const run = await readRun();
-    if (!run || run.status !== "completed" || (run.totalChunks ?? 0) === 0) {
-      return { query, hits: [] };
-    }
-
-    const vector = await embedQuery(config, query);
-    const adapter = await createAdapter(config);
-    const hits = await adapter.query(vector, topK, query);
-
-    return {
-      query,
-      hits: hits.map((h) => ({
-        title: h.title ?? "Untitled",
-        url: h.url ?? "",
-        score: Number((h.score ?? 0).toFixed(3)),
-        text: (h.text ?? "").slice(0, 1200),
-      })),
-    };
-  };
-
-  return serverId ? runWithServer(serverId, doRetrieve) : doRetrieve();
-}
+CONVERSATION, RETRIEVAL, AND MEDIA POLICY:
+- Default to a short, direct answer. One to three sentences is usually enough; use structure only when the task is genuinely complex.
+- Do not search for greetings, casual conversation, writing help, brainstorming, explanations you can answer reliably from general knowledge, or questions unrelated to the user's indexed data.
+- Use searchKnowledgeBase only when the answer depends on the user's indexed/private content, an exact source-specific fact, or a topic that plausibly refers to something the user indexed.
+- Search at most once per user request unless the first search is clearly irrelevant. Never repeat a search when a prior tool result in this conversation already contains enough evidence.
+- For a follow-up such as "show me that part", "play the clip", "show the image", or "let me hear it", reuse the mediaAssetId and timestamps already present in the earlier search result and call presentMedia directly. Do not search again just to rediscover the same media.
+- Use presentMedia to embed exactly one best video, image, or audio citation when the user asks to see/hear it or when one compact preview materially supports the answer. Do not present every search result.
+- Keep the written answer minimal around a media citation. Do not print internal media reference markers.
+`;
 
 export async function POST(req: Request) {
   const {
     messages,
     serverId,
     chatModelId: requestedModelId,
+    docSessionId,
+    docFields,
   }: {
     messages: UIMessage[];
     serverId?: string;
     chatModelId?: string;
+    docSessionId?: string;
+    docFields?: {
+      id: string;
+      name: string;
+      type: string;
+      value?: string;
+      context?: string;
+      placeholder?: string;
+    }[];
   } = await req.json();
 
   const config = await readConfig();
   const provider = config.chatProvider || config.embeddingProvider;
 
-  // Resolve chat model: explicit request > config > default for provider
+  // Fetch dynamic models to resolve defaults
+  const gatewayModels = await getModelsByType('language');
+  const allChatModels = gatewayModels.map(toChatDescriptor);
+
   const chatModelId =
     requestedModelId ||
     config.chatModelId ||
-    getDefaultChatModel(provider)?.id ||
-    "openai/gpt-4o-mini";
+    getDefaultChatModel(allChatModels, provider)?.id ||
+    'openai/gpt-4o-mini';
 
-  const chatModelDescriptor = getChatModel(chatModelId);
+  const modelProvider = chatModelId.split('/')[0];
   const resolvedProvider =
-    provider === "vercel_ai_gateway"
-      ? "vercel_ai_gateway"
-      : chatModelDescriptor?.provider || provider;
+    provider === 'vercel_ai_gateway' ? 'vercel_ai_gateway' : modelProvider || provider;
 
   const apiKey = config.chatApiKey || config.embeddingApiKey || undefined;
-  console.log("Using API Key for chat:", apiKey ? `${apiKey.substring(0, 10)}...` : "NONE", "Provider:", resolvedProvider);
-  const model = createChatModel(resolvedProvider, chatModelId, apiKey);
+  console.log(
+    'Using API Key for chat:',
+    apiKey ? `${apiKey.substring(0, 10)}...` : 'NONE',
+    'Provider:',
+    resolvedProvider,
+  );
+  const model = createChatModel(
+    resolvedProvider,
+    chatModelId,
+    apiKey,
+    config.customChatModels,
+  ) as any;
+
+  // Strip large PDF base64 attachments and heavy doc-tool results from messages to prevent context window explosion.
+  // Also limit message history to last 20 messages to prevent unbounded growth.
+  const MAX_HISTORY_MESSAGES = 20;
+  const messagesToProcess =
+    messages.length > MAX_HISTORY_MESSAGES
+      ? messages.slice(messages.length - MAX_HISTORY_MESSAGES)
+      : messages;
+
+  const safeMessages = messagesToProcess.map((m) => {
+    const anyM = { ...m } as any;
+
+    // Helper: strip heavy keys from a tool result object
+    const stripHeavyResult = (result: any, toolName?: string): any => {
+      let resultObj = result;
+      let isString = false;
+      if (typeof result === 'string') {
+        try {
+          resultObj = JSON.parse(result);
+          isString = true;
+        } catch {
+          return result;
+        }
+      }
+      if (typeof resultObj !== 'object' || resultObj === null) return result;
+      const isDocTool =
+        toolName &&
+        ['fillDocumentForm', 'editDocument', 'requestDocumentSignature'].includes(toolName);
+      if (isDocTool || resultObj.fileBase64 || resultObj.pages) {
+        const copy = { ...resultObj };
+        delete copy.fileBase64;
+        delete copy.pages;
+        if (isDocTool) {
+          delete copy.fields;
+          if (copy.rawText) copy.rawText = copy.rawText.slice(0, 500);
+        }
+        return isString ? JSON.stringify(copy) : copy;
+      }
+      return result;
+    };
+
+    // Clean toolInvocations on historical messages — strip fileBase64, pages, and heavy field arrays
+    if (anyM.role === 'assistant' && anyM.toolInvocations) {
+      anyM.toolInvocations = anyM.toolInvocations
+        .filter(
+          (ti: any) =>
+            ti.state === 'result' || ti.state === 'output' || ti.state === 'output-available',
+        )
+        .map((ti: any) => {
+          if (ti.result) {
+            return { ...ti, result: stripHeavyResult(ti.result, ti.toolName) };
+          }
+          return ti;
+        });
+    }
+
+    // Clean parts (AI SDK v5 UIMessage format) — strip heavy data from tool results and file parts
+    if (Array.isArray(anyM.parts)) {
+      anyM.parts = anyM.parts.map((part: any) => {
+        // Tool invocation parts — strip heavy results
+        if (part.type === 'tool-invocation' && part.toolInvocation) {
+          const ti = part.toolInvocation;
+          if (ti.result !== undefined) {
+            return {
+              ...part,
+              toolInvocation: {
+                ...ti,
+                result: stripHeavyResult(ti.result, ti.toolName),
+              },
+            };
+          }
+        }
+        // New-format tool parts (tool-result)
+        if (part.type === 'tool-result' && part.result !== undefined) {
+          return { ...part, result: stripHeavyResult(part.result, part.toolName) };
+        }
+        // File parts — strip large non-image files (PDF base64)
+        if (part.type === 'file') {
+          const data = part.data || part.url || '';
+          if (typeof data === 'string' && data.length > 5000) {
+            const mimeType = (part.mimeType || part.mediaType || '').toLowerCase();
+            const isImage = mimeType.startsWith('image/');
+            if (!isImage) {
+              // Replace with a placeholder
+              return {
+                type: 'text',
+                text: `[File attachment: ${
+                  mimeType || 'document'
+                } — removed from context for size]`,
+              };
+            }
+          }
+        }
+        return part;
+      });
+    }
+
+    // Strip PDF attachments from experimental_attachments (base64 data URLs can be many MB)
+    if (anyM.experimental_attachments) {
+      anyM.experimental_attachments = anyM.experimental_attachments.filter((att: any) => {
+        if (att.url && att.url.length > 5000) {
+          const isPdf =
+            (att.contentType && att.contentType.toLowerCase().includes('pdf')) ||
+            (att.name && att.name.toLowerCase().endsWith('.pdf')) ||
+            att.url.substring(0, 50).toLowerCase().includes('pdf');
+
+          if (isPdf || !(att.contentType && att.contentType.toLowerCase().startsWith('image/'))) {
+            return false;
+          }
+        }
+        return true;
+      });
+    }
+
+    if (Array.isArray(anyM.content)) {
+      anyM.content = anyM.content.filter((part: any) => {
+        const data = part.data || part.url || part.text;
+        if (typeof data === 'string' && data.length > 5000) {
+          const isPdf =
+            (part.mimeType && part.mimeType.toLowerCase().includes('pdf')) ||
+            (part.contentType && part.contentType.toLowerCase().includes('pdf')) ||
+            data.substring(0, 50).toLowerCase().includes('pdf');
+
+          if (
+            isPdf ||
+            !(part.mimeType?.startsWith('image/') || part.contentType?.startsWith('image/'))
+          ) {
+            return false;
+          }
+        }
+        return true;
+      });
+    }
+
+    return anyM;
+  });
+
+  let tabularContext = '';
+  try {
+    const datasets = await listTabularDatasets();
+    if (datasets.length > 0) {
+      tabularContext = `\n\nAvailable tabular datasets:\n${datasets
+        .map((d) => {
+          const colDescriptions = d.columns
+            .map((c) => {
+              let desc = `${c.name} (${c.type})`;
+              if (c.type === 'date' && c.dateRange) {
+                desc += ` [format: ${c.dateRange.format}, range: ${c.dateRange.min} to ${c.dateRange.max}]`;
+              }
+              if (c.sampleValues && c.sampleValues.length > 0) {
+                desc += ` [samples: ${c.sampleValues.slice(0, 3).join(', ')}]`;
+              }
+              return desc;
+            })
+            .join(', ');
+          const sizeHint =
+            d.rowCount > 10000
+              ? ' ⚠️ LARGE DATASET — prefer executeAnalysis with pandas for complex queries'
+              : '';
+          return `- Dataset "${d.fileName}" (ID: ${d.id}): ${d.rowCount} rows, ${d.summary.totalColumns} columns.${sizeHint}\n  Columns: ${colDescriptions}`;
+        })
+        .join('\n')}`;
+    }
+  } catch {
+    /* no tabular data */
+  }
+
+  let docContext = '';
+  if (docSessionId) {
+    // Build a rich field listing so the LLM can correctly match semantic meaning to field IDs.
+    // Include: ID, name/label, type, current value (if any), and surrounding context text.
+    const fieldLines =
+      docFields && docFields.length > 0
+        ? (docFields as any[])
+            .map((f: any) => {
+              let line = `- ID: "${f.id}" | Label: "${f.name}" | Type: ${f.type}`;
+              if (f.value) line += ` | Current value: "${f.value}"`;
+              if (f.context) line += ` | Surrounding text: "${String(f.context).slice(0, 120)}"`;
+              if (f.placeholder) line += ` | Placeholder: "${f.placeholder}"`;
+              return line;
+            })
+            .join('\n')
+        : 'None detected.';
+
+    docContext = `\n\n[Active Document Session: ${docSessionId}]\nYou are currently editing a document in the Canvas.
+The user may ask you to fill out form fields or edit content.
+IMPORTANT: Use the exact field IDs listed below when calling "fillDocumentForm". Do NOT invent field IDs.
+Available Form Fields (${docFields?.length ?? 0} total):
+${fieldLines}`;
+  }
+
+  const systemPrompt =
+    (config.systemPrompt || DEFAULT_SYSTEM_PROMPT) +
+    SMART_RETRIEVAL_POLICY +
+    tabularContext +
+    docContext;
+  const tools = await getChatTools({ serverId, docSessionId, config });
+
+  // Debug: log payload sizes to console in development
+  if (process.env.NODE_ENV === 'development') {
+    const stringifiedMsgs = JSON.stringify(safeMessages);
+    console.log(
+      `[chat] Payload size — System Prompt: ${systemPrompt.length} chars, safeMessages: ${stringifiedMsgs.length} chars`,
+    );
+  }
 
   const result = streamText({
     model,
-    system: config.systemPrompt || DEFAULT_SYSTEM_PROMPT,
-    messages: await convertToModelMessages(messages),
-    stopWhen: stepCountIs(5),
-    tools: {
-      searchKnowledgeBase: tool({
-        description:
-          "Search the private RAG knowledge base for relevant documents. Use this for factual questions about the indexed content.",
-        inputSchema: z.object({
-          query: z
-            .string()
-            .describe("The search query for the knowledge base."),
-        }),
-        execute: async ({ query }) => {
-          return queryKnowledgeBase(query, 5, serverId ?? null);
-        },
-      }),
+    system: systemPrompt,
+    messages: await convertToModelMessages(safeMessages, { tools }),
+    maxOutputTokens: 8192,
+    stopWhen: stepCountIs(8),
+    onFinish: async ({ usage, response }) => {
+      const { trackUsageEvent, estimateCost } = await import('@larkup/core/analytics-store');
+      const u = usage as any;
+      void trackUsageEvent({
+        type: 'chat',
+        modelId: chatModelId,
+        provider: resolvedProvider,
+        promptTokens: u?.promptTokens ?? 0,
+        completionTokens: u?.completionTokens ?? 0,
+        totalTokens: u?.totalTokens ?? 0,
+        estimatedCost: estimateCost(chatModelId, u?.promptTokens ?? 0, u?.completionTokens ?? 0),
+        timestamp: new Date().toISOString(),
+      });
     },
+    tools,
   });
 
   return result.toUIMessageStreamResponse({
-    onError: (error) => {
-      console.error("[chat] stream error:", error);
+    onError: (error: any) => {
+      console.error('[chat] stream error:', error);
+      if (error && typeof error === 'object') {
+        if (error.message) {
+          return error.message;
+        }
+        if (error.error && error.error.message) {
+          return error.error.message;
+        }
+        try {
+          return JSON.stringify(error);
+        } catch {
+          return 'Something went wrong while generating a response.';
+        }
+      }
       const message = error instanceof Error ? error.message : String(error);
-      return message || "Something went wrong while generating a response.";
+      return message || 'Something went wrong while generating a response.';
     },
   });
 }