larkup 0.1.14

package/.env

@@ -0,0 +1,2 @@
+SERPER_API_KEY="15cd177b8da4d70f262f0c116cf1164f40ca32c4"
+AI_GATEWAY_API_KEY="vck_7qkeRibvoKg3fbVJNOVY1OjSDuixxryKYeeK9jsyaJDn65UrWD4PnzNa"

package/app/actions/vercel.ts

@@ -0,0 +1,417 @@
+"use server";
+
+import fs from "fs";
+import path from "path";
+import { getActiveServer } from "@larkup/core/workspace";
+
+const VERCEL_API = "https://api.vercel.com";
+
+interface VercelProject {
+  id: string;
+  name: string;
+  link?: {
+    type: string;
+    repo?: string;
+    repoId?: number;
+    org?: string;
+    gitCredentialId?: string;
+    productionBranch?: string;
+  };
+}
+
+interface VercelDeployment {
+  id: string;
+  uid: string;
+  name: string;
+  url: string;
+  readyState: string;
+  target?: string;
+}
+
+async function vercelFetch(
+  path: string,
+  token: string,
+  options: RequestInit = {},
+) {
+  const res = await fetch(`${VERCEL_API}${path}`, {
+    ...options,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Content-Type": "application/json",
+      ...(options.headers || {}),
+    },
+  });
+
+  const text = await res.text();
+  let body: any;
+  try {
+    body = JSON.parse(text);
+  } catch {
+    body = { rawText: text };
+  }
+
+  if (!res.ok) {
+    const msg =
+      body?.error?.message ||
+      body?.message ||
+      `API error occurred: Status ${res.status} Content-Type "${res.headers.get("content-type")}". Body: ${text}`;
+    throw new Error(msg);
+  }
+
+  return body;
+}
+
+/** Look up a project by name or ID. Returns null if not found. */
+async function getProject(
+  token: string,
+  nameOrId: string,
+): Promise<VercelProject | null> {
+  try {
+    const data = await vercelFetch(
+      `/v9/projects/${encodeURIComponent(nameOrId)}`,
+      token,
+    );
+    return data as VercelProject;
+  } catch (e: any) {
+    if (
+      e.message?.toLowerCase().includes("not found") ||
+      e.message?.toLowerCase().includes("project not found")
+    ) {
+      return null;
+    }
+    throw e;
+  }
+}
+
+/** Create a new blank Vercel project. */
+async function createProject(
+  token: string,
+  name: string,
+): Promise<VercelProject> {
+  const data = await vercelFetch(`/v9/projects`, token, {
+    method: "POST",
+    body: JSON.stringify({ name, framework: null }),
+  });
+  return data as VercelProject;
+}
+
+/** Set an environment variable for the project */
+async function setProjectEnv(
+  token: string,
+  projectId: string,
+  key: string,
+  value: string,
+) {
+  // Try to set it. We use the /v10/projects/:id/env endpoint.
+  // First, check if it exists so we can patch or post
+  const envData = await vercelFetch(`/v9/projects/${projectId}/env`, token);
+  const existing = envData.envs?.find((e: any) => e.key === key);
+
+  if (existing) {
+    await vercelFetch(`/v9/projects/${projectId}/env/${existing.id}`, token, {
+      method: "PATCH",
+      body: JSON.stringify({
+        value,
+        target: ["production", "preview", "development"],
+      }),
+    });
+  } else {
+    await vercelFetch(`/v10/projects/${projectId}/env`, token, {
+      method: "POST",
+      body: JSON.stringify([
+        {
+          key,
+          value,
+          target: ["production", "preview", "development"],
+          type: "plain",
+        },
+      ]),
+    });
+  }
+}
+
+function getFilesRecursively(dir: string, baseDir: string): any[] {
+  let results: any[] = [];
+  if (!fs.existsSync(dir)) return results;
+  const list = fs.readdirSync(dir);
+  for (const file of list) {
+    if (file === "node_modules" || file === ".git" || file === "server.log")
+      continue;
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat && stat.isDirectory()) {
+      results = results.concat(getFilesRecursively(filePath, baseDir));
+    } else {
+      const data = fs.readFileSync(filePath);
+      const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
+      results.push({
+        file: relativePath,
+        data: data.toString("base64"),
+        encoding: "base64",
+      });
+    }
+  }
+  return results;
+}
+
+/** Deploy by sending files directly. */
+async function triggerDeploymentWithFiles(
+  token: string,
+  project: VercelProject,
+  files: any[],
+): Promise<VercelDeployment> {
+  const body: Record<string, any> = {
+    name: project.name,
+    target: "production",
+    files,
+  };
+
+  const data = await vercelFetch(`/v13/deployments`, token, {
+    method: "POST",
+    body: JSON.stringify(body),
+  });
+
+  return data as VercelDeployment;
+}
+
+export async function getServerEnvRequirements(serverId: string) {
+  let activeId = serverId;
+  if (activeId === "default") {
+    const server = await getActiveServer();
+    if (server) activeId = server.id;
+  }
+
+  const cwd = process.cwd();
+
+  // ── Read the LIVE config (not the stale .env.example on disk).
+  // This ensures that switching vector stores in the UI is immediately
+  // reflected in the env-vars sheet without requiring a server regeneration.
+  let vectorStore: string = "lancedb";
+  let storeConfig: Record<string, string> = {};
+  const configCandidates = [
+    path.join(cwd, ".larkup", "servers", activeId, "config.json"),
+    path.join(cwd, ".larkup", "config.json"),
+  ];
+  for (const cfgPath of configCandidates) {
+    if (fs.existsSync(cfgPath)) {
+      try {
+        const cfg = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+        vectorStore = cfg.vectorStore ?? "lancedb";
+        storeConfig = cfg.storeConfig ?? {};
+        break;
+      } catch {
+        // continue to next candidate
+      }
+    }
+  }
+
+  // ── Map storeConfig fields → env var names.
+  // The configure page stores credentials as storeConfig.apiKey, storeConfig.indexName, etc.
+  // Mirror the same mapping used in server-runtime.ts so configured values are
+  // automatically pre-filled as defaults in the env-vars sheet.
+  const storeDefaults: Record<string, string> = {};
+  if (vectorStore === "pinecone") {
+    if (storeConfig.apiKey)          storeDefaults["PINECONE_API_KEY"]       = storeConfig.apiKey;
+    if (storeConfig.indexName)       storeDefaults["PINECONE_INDEX"]         = storeConfig.indexName;
+    if (storeConfig.namespace)       storeDefaults["PINECONE_NAMESPACE"]     = storeConfig.namespace;
+    if (storeConfig.sparseModel)     storeDefaults["PINECONE_SPARSE_MODEL"]  = storeConfig.sparseModel;
+    if (storeConfig.sparseIndexName) storeDefaults["PINECONE_SPARSE_INDEX"]  = storeConfig.sparseIndexName;
+  } else {
+    if (storeConfig.tableName) storeDefaults["LANCEDB_TABLE"]   = storeConfig.tableName;
+    if (storeConfig.uri)       storeDefaults["LANCEDB_URI"]     = storeConfig.uri;
+    if (storeConfig.apiKey)    storeDefaults["LANCEDB_API_KEY"] = storeConfig.apiKey;
+  }
+
+  // ── Canonical env-var list — mirrors generateServer() in generate-server.ts.
+  // SERVER_API_KEY, PORT, LANCEDB_PATH, LANCEDB_MODE are handled internally
+  // and intentionally omitted here.
+  const envVarDefs: { key: string; required: boolean; help: string }[] = [
+    {
+      key: "EMBEDDING_API_KEY",
+      required: true,
+      help: "API key used to embed incoming queries.",
+    },
+  ];
+
+  if (vectorStore === "pinecone") {
+    envVarDefs.push(
+      { key: "PINECONE_API_KEY", required: true, help: "Pinecone API key." },
+      { key: "PINECONE_INDEX", required: true, help: "Pinecone index name to query." },
+      { key: "PINECONE_NAMESPACE", required: false, help: "Pinecone namespace (default 'default')." },
+      { key: "PINECONE_SPARSE_MODEL", required: false, help: "Pinecone sparse model (for hybrid search)." },
+      { key: "PINECONE_SPARSE_INDEX", required: false, help: "Pinecone sparse index name (for hybrid search)." },
+    );
+  } else {
+    // lancedb
+    envVarDefs.push(
+      { key: "LANCEDB_TABLE", required: false, help: "Table name holding the embedded chunks (default 'documents')." },
+      { key: "LANCEDB_URI", required: false, help: "LanceDB Cloud database URI (cloud mode)." },
+      { key: "LANCEDB_API_KEY", required: false, help: "LanceDB Cloud API key (cloud mode)." },
+    );
+  }
+
+  // ── Load .env file defaults (lowest priority).
+  const rootEnvPath = path.join(cwd, ".env");
+  const monoRootEnvPath = path.join(cwd, "../..", ".env");
+  const envFileDefaults: Record<string, string> = {};
+  for (const envFile of [monoRootEnvPath, rootEnvPath]) {
+    if (fs.existsSync(envFile)) {
+      const content = fs.readFileSync(envFile, "utf-8");
+      for (const line of content.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed && !trimmed.startsWith("#") && trimmed.includes("=")) {
+          const [key, ...rest] = trimmed.split("=");
+          let val = rest.join("=").trim();
+          if (val.startsWith('"') && val.endsWith('"')) val = val.slice(1, -1);
+          else if (val.startsWith("'") && val.endsWith("'")) val = val.slice(1, -1);
+          envFileDefaults[key.trim()] = val;
+        }
+      }
+    }
+  }
+
+  // Priority: config > storeConfig (configure page) > .env file > empty
+  return envVarDefs.map((e) => {
+    let defaultValue = storeDefaults[e.key] || envFileDefaults[e.key] || "";
+    if (e.key === "EMBEDDING_API_KEY" && !defaultValue) {
+      // try to read from config directly
+      const cwd = process.cwd();
+      const configCandidates = [
+        path.join(cwd, ".larkup", "servers", activeId, "config.json"),
+        path.join(cwd, ".larkup", "config.json"),
+      ];
+      for (const cfgPath of configCandidates) {
+        if (fs.existsSync(cfgPath)) {
+          try {
+            const cfg = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+            if (cfg.embeddingApiKey) defaultValue = cfg.embeddingApiKey;
+            break;
+          } catch {}
+        }
+      }
+    }
+    return {
+      key: e.key,
+      help: e.help,
+      required: e.required,
+      defaultValue,
+    };
+  });
+}
+
+
+export async function deployToVercel(
+  token: string,
+  projectIdOrName: string,
+  serverId: string,
+  envVars: Record<string, string> = {}
+) {
+  let activeId = serverId;
+  if (activeId === "default") {
+    const server = await getActiveServer();
+    if (server) activeId = server.id;
+  }
+
+  if (!token || !projectIdOrName) {
+    return {
+      success: false,
+      error: "Vercel token and project ID/name are required.",
+    };
+  }
+
+  if (!activeId) {
+    return {
+      success: false,
+      error: "Server ID is required to read deployment files.",
+    };
+  }
+
+  try {
+    // 1. Find existing project or create one
+    let project = await getProject(token, projectIdOrName);
+    const wasNew = !project;
+
+    if (!project) {
+      console.log(`Project "${projectIdOrName}" not found. Creating it...`);
+      project = await createProject(token, projectIdOrName);
+      console.log(`Created project: ${project.id} (${project.name})`);
+    }
+
+    // 2. Set environment variables
+    for (const [key, value] of Object.entries(envVars)) {
+      if (value) {
+        await setProjectEnv(token, project.id, key, value);
+      }
+    }
+
+    // 3. Read the LIVE config to ensure we deploy the latest configuration
+    const cwd = process.cwd();
+    let config: any = null;
+    const configCandidates = [
+      path.join(cwd, ".larkup", "servers", activeId, "config.json"),
+      path.join(cwd, ".larkup", "config.json"),
+    ];
+    for (const cfgPath of configCandidates) {
+      if (fs.existsSync(cfgPath)) {
+        try {
+          config = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+          break;
+        } catch {}
+      }
+    }
+
+    if (!config) {
+      return { success: false, error: "Configuration not found. Please save your settings first." };
+    }
+
+    // 4. Generate the server files in memory based on current config
+    const { generateServer } = await import("@larkup/core/generator/generate-server");
+    const generated = generateServer(config);
+    
+    const files = generated.files.map((f) => ({
+      file: f.path,
+      data: f.encoding === "base64" ? f.contents : Buffer.from(f.contents).toString("base64"),
+      encoding: "base64",
+    }));
+
+    // 5. Attach LanceDB local data folder ONLY if LanceDB local is actually configured
+    const isLanceLocal = config.vectorStore === "lancedb" && config.storeConfig?.mode !== "cloud";
+    if (isLanceLocal) {
+      await setProjectEnv(token, project.id, "LANCEDB_PATH", "./lancedb");
+      const lancedbDir = path.join(cwd, ".larkup", "servers", activeId, "lancedb");
+      if (fs.existsSync(lancedbDir)) {
+        const lancedbFiles = getFilesRecursively(
+          lancedbDir,
+          path.join(cwd, ".larkup", "servers", activeId)
+        );
+        files.push(...lancedbFiles);
+      }
+    }
+
+    if (files.length === 0) {
+      return {
+        success: false,
+        error: "No files found to deploy in the server directory.",
+      };
+    }
+
+    // 4. Create new deployment with files
+    const deployRes = await triggerDeploymentWithFiles(token, project, files);
+
+    // 5. Build canonical production URL
+    const url = `https://${project.name}.vercel.app`;
+
+    return {
+      success: true,
+      url,
+      deploymentId: deployRes?.id,
+      projectCreated: wasNew,
+    };
+  } catch (error: any) {
+    console.error("Vercel Deploy Error:", error);
+    return {
+      success: false,
+      error: error.message || "Failed to trigger deployment on Vercel.",
+    };
+  }
+}

package/app/api/chat/route.ts

@@ -0,0 +1,192 @@
+import {
+  streamText,
+  convertToModelMessages,
+  tool,
+  stepCountIs,
+  type UIMessage,
+} from "ai";
+import { z } from "zod";
+import { readConfig } from "@larkup/core/config-store";
+import { readRun } from "@larkup/core/index-store";
+import { refreshServerStatus } from "@larkup/core/generator/server-runtime";
+import { createAdapter } from "@larkup/vector-stores/factory";
+import { embedQuery } from "@larkup/core/indexing/embedder";
+import { runWithServer } from "@larkup/core/workspace";
+import {
+  getDefaultChatModel,
+  getChatModel,
+} from "@larkup/core/chat-models/registry";
+
+import { createOpenAI } from "@ai-sdk/openai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createCohere } from "@ai-sdk/cohere";
+import { createMistral } from "@ai-sdk/mistral";
+import { createDeepSeek } from "@ai-sdk/deepseek";
+import { createGateway } from "@ai-sdk/gateway";
+
+export const maxDuration = 60;
+
+const DEFAULT_SYSTEM_PROMPT = `You are a helpful research assistant powered by a knowledge base.
+
+You have one tool:
+- "searchKnowledgeBase" — searches a private RAG knowledge base.
+
+Guidelines:
+- ONLY skip the tool if the user's message is a basic greeting (e.g., "hi", "hello") or simple conversational filler.
+- For ALL OTHER messages (especially questions about facts, personal preferences, instructions, or specific topics), you MUST call the searchKnowledgeBase tool FIRST before answering. Do not assume you cannot answer a question (like "what do I like?") without checking the knowledge base first!
+- Synthesize a clear, well-structured answer based on the retrieved documents.
+- Cite sources inline using markdown links to their URLs when available.
+- Be concise and accurate. Never fabricate sources or facts.
+- Don't reply to the user with a question — try to give the answer unless you truly need clarification.
+- IMPORTANT: If the searchKnowledgeBase tool returns empty results, inform the user that you currently have no knowledge in your database for their specific question. However, you may still attempt to answer general questions using your general knowledge.
+`;
+
+/**
+ * Creates an AI SDK language model instance based on the provider and model ID.
+ */
+function createChatModel(provider: string, modelId: string, apiKey?: string) {
+  const modelName = modelId.includes("/")
+    ? modelId.split("/").slice(1).join("/")
+    : modelId;
+  const key = apiKey;
+
+  switch (provider) {
+    case "google":
+      return createGoogleGenerativeAI({ apiKey: key })(modelName);
+    case "cohere":
+      return createCohere({ apiKey: key })(modelName);
+    case "mistral":
+      return createMistral({ apiKey: key })(modelName);
+    case "deepseek":
+      return createDeepSeek({ apiKey: key })(modelName);
+    case "vercel_ai_gateway":
+      return createGateway({ apiKey: key })(modelId);
+    case "openai":
+    default:
+      return createOpenAI({ apiKey: key })(modelName);
+  }
+}
+
+/**
+ * Retrieves documents from the knowledge base — either via the running
+ * generated server or directly from the local vector store.
+ */
+async function queryKnowledgeBase(
+  query: string,
+  topK: number,
+  serverId: string | null,
+) {
+  const doRetrieve = async () => {
+    const config = await readConfig();
+
+    // 1) Try running generated server first
+    const server = await refreshServerStatus();
+    if (server.running) {
+      try {
+        const res = await fetch(`${server.endpoint}/query`, {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ query, topK }),
+          signal: AbortSignal.timeout(15_000),
+        });
+        const data = await res.json();
+        if (res.ok && data.hits) {
+          return {
+            query,
+            hits: (data.hits as any[]).map((h: any) => ({
+              title: h.title ?? "Untitled",
+              url: h.url ?? "",
+              score: Number((h.score ?? 0).toFixed(3)),
+              text: (h.text ?? "").slice(0, 1200),
+            })),
+          };
+        }
+      } catch {
+        // Fall through to direct retrieval
+      }
+    }
+
+    // 2) Direct retrieval from local vector store
+    const run = await readRun();
+    if (!run || run.status !== "completed" || (run.totalChunks ?? 0) === 0) {
+      return { query, hits: [] };
+    }
+
+    const vector = await embedQuery(config, query);
+    const adapter = await createAdapter(config);
+    const hits = await adapter.query(vector, topK, query);
+
+    return {
+      query,
+      hits: hits.map((h) => ({
+        title: h.title ?? "Untitled",
+        url: h.url ?? "",
+        score: Number((h.score ?? 0).toFixed(3)),
+        text: (h.text ?? "").slice(0, 1200),
+      })),
+    };
+  };
+
+  return serverId ? runWithServer(serverId, doRetrieve) : doRetrieve();
+}
+
+export async function POST(req: Request) {
+  const {
+    messages,
+    serverId,
+    chatModelId: requestedModelId,
+  }: {
+    messages: UIMessage[];
+    serverId?: string;
+    chatModelId?: string;
+  } = await req.json();
+
+  const config = await readConfig();
+  const provider = config.chatProvider || config.embeddingProvider;
+
+  // Resolve chat model: explicit request > config > default for provider
+  const chatModelId =
+    requestedModelId ||
+    config.chatModelId ||
+    getDefaultChatModel(provider)?.id ||
+    "openai/gpt-4o-mini";
+
+  const chatModelDescriptor = getChatModel(chatModelId);
+  const resolvedProvider =
+    provider === "vercel_ai_gateway"
+      ? "vercel_ai_gateway"
+      : chatModelDescriptor?.provider || provider;
+
+  const apiKey = config.chatApiKey || config.embeddingApiKey || undefined;
+  console.log("Using API Key for chat:", apiKey ? `${apiKey.substring(0, 10)}...` : "NONE", "Provider:", resolvedProvider);
+  const model = createChatModel(resolvedProvider, chatModelId, apiKey);
+
+  const result = streamText({
+    model,
+    system: config.systemPrompt || DEFAULT_SYSTEM_PROMPT,
+    messages: await convertToModelMessages(messages),
+    stopWhen: stepCountIs(5),
+    tools: {
+      searchKnowledgeBase: tool({
+        description:
+          "Search the private RAG knowledge base for relevant documents. Use this for factual questions about the indexed content.",
+        inputSchema: z.object({
+          query: z
+            .string()
+            .describe("The search query for the knowledge base."),
+        }),
+        execute: async ({ query }) => {
+          return queryKnowledgeBase(query, 5, serverId ?? null);
+        },
+      }),
+    },
+  });
+
+  return result.toUIMessageStreamResponse({
+    onError: (error) => {
+      console.error("[chat] stream error:", error);
+      const message = error instanceof Error ? error.message : String(error);
+      return message || "Something went wrong while generating a response.";
+    },
+  });
+}

package/app/api/chat/status/route.ts

@@ -0,0 +1,55 @@
+import { NextResponse } from "next/server";
+import { readConfig } from "@larkup/core/config-store";
+import { readRun } from "@larkup/core/index-store";
+import { runWithServer } from "@larkup/core/workspace";
+import {
+  getChatModelsForProvider,
+  getDefaultChatModel,
+} from "@larkup/core/chat-models/registry";
+
+export const dynamic = "force-dynamic";
+
+function withServer<T>(serverId: string | null, fn: () => Promise<T>) {
+  return serverId ? runWithServer(serverId, fn) : fn();
+}
+
+/**
+ * GET /api/chat/status — readiness snapshot for the Chat stage.
+ *
+ * Reports whether there's an index ready to chat against, plus
+ * the available chat models for the user's selected provider.
+ */
+export async function GET(req: Request) {
+  const serverId = new URL(req.url).searchParams.get("serverId");
+  return withServer(serverId, async () => {
+    const config = await readConfig();
+    const run = await readRun();
+
+    const indexed = run?.status === "completed" && (run.totalChunks ?? 0) > 0;
+    const hasApiKey = !!(config.chatApiKey || config.embeddingApiKey);
+
+    const blockers: string[] = [];
+    if (!hasApiKey) {
+      blockers.push(
+        "Set an API Key in Settings.",
+      );
+    }
+
+    const provider = config.chatProvider || config.embeddingProvider;
+    const models = getChatModelsForProvider(provider);
+    const defaultModel = getDefaultChatModel(provider);
+    const chatModelId =
+      config.chatModelId || defaultModel?.id || "openai/gpt-4o-mini";
+
+    return NextResponse.json({
+      ready: hasApiKey,
+      indexed,
+      blockers,
+      provider,
+      chatModelId,
+      availableModels: models.map((m) => ({ id: m.id, label: m.label, provider: m.provider })),
+      suggestions: config.chatSuggestions || [],
+    });
+  });
+}
+