larkup 0.1.14

package/app/api/jobs/route.ts

@@ -0,0 +1,116 @@
+import { NextResponse } from "next/server"
+import { randomUUID } from "node:crypto"
+import { readJobs, saveJob } from "@larkup/core/jobs-store"
+import { syncJob } from "@larkup/scraper/job-runner"
+import { isFirecrawlConfigured } from "@larkup/scraper/firecrawl"
+import { readDocuments } from "@larkup/core/documents-store"
+import type { CrawlJob, CrawlScope, CrawlTarget } from "@larkup/core/types"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/** GET → all crawl jobs, newest first. */
+export async function GET() {
+  const jobs = await readJobs()
+  return NextResponse.json({ jobs, configured: await isFirecrawlConfigured() })
+}
+
+/**
+ * POST → start a new ETL job.
+ * Body: { keywords, targets: [{ url, scope }], pageLimit }
+ */
+export async function POST(req: Request) {
+  if (!(await isFirecrawlConfigured())) {
+    return NextResponse.json(
+      {
+        error:
+          "No Firecrawl available. Launch a local instance or set FIRECRAWL_API_KEY to run scraping jobs.",
+      },
+      { status: 401 },
+    )
+  }
+
+  let body: {
+    keywords?: string
+    targets?: Array<{ url?: string; scope?: CrawlScope }>
+    pageLimit?: number
+  }
+  try {
+    body = await req.json()
+  } catch {
+    return NextResponse.json({ error: "Invalid request body." }, { status: 400 })
+  }
+
+  const docs = await readDocuments()
+  const existingUrls = new Set(docs.filter(d => d.url).map(d => d.url!))
+
+  const baseMap = new Map<string, CrawlTarget>()
+  const pathMap = new Map<string, CrawlTarget>()
+
+  ;(body.targets ?? []).forEach((t) => {
+    if (!t.url || !/^https?:\/\//i.test(t.url)) return
+
+    try {
+      const u = new URL(t.url)
+      const origin = u.origin
+
+      // Add base if not existing
+      if (!existingUrls.has(origin) && !existingUrls.has(origin + '/')) {
+        baseMap.set(origin, {
+          url: origin,
+          scope: t.scope === "domain" ? "domain" : "page",
+          status: "queued",
+          pagesCrawled: 0,
+        })
+      }
+
+      // Add specific path if not origin and not existing
+      if (t.url !== origin && t.url !== origin + '/') {
+        if (!existingUrls.has(t.url)) {
+          pathMap.set(t.url, {
+            url: t.url,
+            scope: t.scope === "domain" ? "domain" : "page",
+            status: "queued",
+            pagesCrawled: 0,
+          })
+        }
+      }
+    } catch {
+      // Fallback for weird URLs
+      if (!existingUrls.has(t.url)) {
+        pathMap.set(t.url, {
+          url: t.url,
+          scope: t.scope === "domain" ? "domain" : "page",
+          status: "queued",
+          pagesCrawled: 0,
+        })
+      }
+    }
+  })
+
+  const cleaned = [...Array.from(baseMap.values()), ...Array.from(pathMap.values())]
+
+  if (cleaned.length === 0) {
+    return NextResponse.json(
+      { error: "Select at least one valid new URL to scrape." },
+      { status: 400 },
+    )
+  }
+
+  const job: CrawlJob = {
+    id: randomUUID(),
+    keywords: body.keywords?.trim() || cleaned[0].url,
+    targets: cleaned,
+    status: "queued",
+    pageLimit: Math.min(Math.max(body.pageLimit ?? 2000, 1), 2000),
+    pagesCrawled: 0,
+    docCount: 0,
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+  }
+
+  await saveJob(job)
+  // Kick the job once so domain crawls start and single pages scrape immediately.
+  const advanced = await syncJob(job.id)
+  return NextResponse.json({ job: advanced ?? job }, { status: 201 })
+}

package/app/api/parse-file/route.ts

@@ -0,0 +1,39 @@
+import { NextResponse } from "next/server";
+// @ts-expect-error -- import from lib to avoid pdf-parse's test-file-loading bug
+import pdfParse from "pdf-parse/lib/pdf-parse.js";
+import mammoth from "mammoth";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function POST(req: Request) {
+  try {
+    const formData = await req.formData();
+    const file = formData.get("file") as File | null;
+    
+    if (!file) {
+      return NextResponse.json({ error: "No file provided" }, { status: 400 });
+    }
+
+    const buffer = Buffer.from(await file.arrayBuffer());
+    let text = "";
+
+    if (file.type === "application/pdf" || file.name.endsWith(".pdf")) {
+      const data = await pdfParse(buffer);
+      text = data.text;
+    } else if (
+      file.type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || 
+      file.name.endsWith(".docx")
+    ) {
+      const result = await mammoth.extractRawText({ buffer });
+      text = result.value;
+    } else {
+      return NextResponse.json({ error: "Unsupported file type" }, { status: 400 });
+    }
+
+    return NextResponse.json({ text });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Failed to parse file.";
+    return NextResponse.json({ error: message }, { status: 500 });
+  }
+}

package/app/api/search/google/route.ts

@@ -0,0 +1,162 @@
+import { NextResponse } from "next/server";
+import { readConfig } from "@larkup/core/config-store";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export interface SerperSearchItem {
+  url: string;
+  title: string;
+  description?: string;
+}
+
+export interface SerperSearchResponse {
+  items: SerperSearchItem[];
+  totalResults: number;
+  currentPage: number;
+  totalPages: number;
+  hasMore: boolean;
+  query: string;
+}
+
+/**
+ * POST { query, page } → paginated Serper.dev Google Search results with total count.
+ *
+ * Uses the Serper.dev API: https://serper.dev
+ * Much simpler than Google CSE — only one env var needed.
+ *
+ * Required env var:
+ *   SERPER_API_KEY  – from https://serper.dev → API Key tab
+ *
+ * Serper returns up to 100 results (10 per page, page 1–10).
+ */
+export async function POST(req: Request) {
+  const config = await readConfig();
+  const apiKey = config.serperApiKey;
+  console.log({ apiKey });
+
+  if (!apiKey) {
+    return NextResponse.json(
+      {
+        error:
+          "Serper search is not configured. Set SERPER_API_KEY in your environment.",
+        configured: false,
+      },
+      { status: 503 },
+    );
+  }
+
+  let body: { query?: string; page?: number };
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json(
+      { error: "Invalid request body." },
+      { status: 400 },
+    );
+  }
+
+  const { query, page = 1 } = body;
+  if (!query?.trim()) {
+    return NextResponse.json(
+      { error: "A search query is required." },
+      { status: 400 },
+    );
+  }
+
+  const clampedPage = Math.max(1, Math.min(page, 10));
+
+  try {
+    const res = await fetch("https://google.serper.dev/search", {
+      method: "POST",
+      headers: {
+        "X-API-KEY": apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        q: query.trim(),
+        num: 10,
+        page: clampedPage,
+      }),
+      cache: "no-store",
+    });
+
+    const json = (await res.json()) as {
+      error?: string;
+      /** Serper free plan does NOT include searchInformation — we derive pagination from organic count. */
+      searchInformation?: {
+        totalResults?: number | string;
+        formattedTotalResults?: string;
+        timeTakenDisplayed?: number;
+      };
+      organic?: Array<{
+        link?: string;
+        title?: string;
+        snippet?: string;
+        position?: number;
+      }>;
+    };
+
+    if (!res.ok || json.error) {
+      const msg = json.error ?? `Serper API error (${res.status})`;
+      return NextResponse.json({ error: msg }, { status: res.status });
+    }
+
+    const items: SerperSearchItem[] = (json.organic ?? [])
+      .filter((r) => r.link)
+      .map((r) => ({
+        url: r.link!,
+        title: r.title ?? r.link!,
+        description: r.snippet,
+      }));
+
+    // Serper's free plan omits searchInformation.totalResults.
+    // We derive pagination heuristically:
+    //   • A full page (10 results) means there are likely more pages.
+    //   • We cap at 10 pages (100 results) — Serper's max accessible window.
+    //   • If fewer than 10 results returned, this is the last page.
+    const gotFullPage = items.length >= 10;
+    const hasMore = gotFullPage && clampedPage < 10;
+
+    // Derive a synthetic total: current page offset + items on this page,
+    // plus "+?" if we believe there are more pages.
+    const minTotal = (clampedPage - 1) * 10 + items.length;
+    // If searchInformation is present (cloud/enterprise plan), use it.
+    const siTotal = json.searchInformation?.totalResults;
+    const rawTotal =
+      siTotal != null
+        ? typeof siTotal === "string"
+          ? parseInt(siTotal, 10) || minTotal
+          : siTotal
+        : minTotal;
+
+    // Total pages: if we know the real count, cap at 10; otherwise estimate.
+    const totalPages =
+      siTotal != null
+        ? Math.min(Math.max(1, Math.ceil(Math.min(rawTotal, 100) / 10)), 10)
+        : hasMore
+          ? 10 // unknown upper bound — show up to 10 pages
+          : clampedPage; // last page reached
+
+    return NextResponse.json({
+      items,
+      totalResults: rawTotal,
+      // Pass along whether totalResults is an estimate
+      totalResultsIsEstimate: siTotal == null,
+      currentPage: clampedPage,
+      totalPages,
+      hasMore,
+      query: query.trim(),
+    } satisfies SerperSearchResponse & { totalResultsIsEstimate: boolean });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : "Serper search failed.";
+    return NextResponse.json({ error: msg }, { status: 500 });
+  }
+}
+
+/** GET → whether Serper search is configured. */
+export async function GET() {
+  const config = await readConfig();
+  const configured = Boolean(config.serperApiKey);
+  return NextResponse.json({ configured });
+}

package/app/api/search/route.ts

@@ -0,0 +1,29 @@
+import { NextResponse } from "next/server"
+import { searchWeb, isFirecrawlConfigured, FirecrawlError } from "@larkup/scraper/firecrawl"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/** GET → report whether the scraper is configured (drives the setup notice). */
+export async function GET() {
+  return NextResponse.json({ configured: await isFirecrawlConfigured() })
+}
+
+/** POST { query, limit } → keyword web search returning candidate URLs. */
+export async function POST(req: Request) {
+  try {
+    const { query, limit } = (await req.json()) as {
+      query?: string
+      limit?: number
+    }
+    if (!query || !query.trim()) {
+      return NextResponse.json({ error: "A search query is required." }, { status: 400 })
+    }
+    const results = await searchWeb(query.trim(), Math.min(limit ?? 10, 25))
+    return NextResponse.json({ results })
+  } catch (err) {
+    const status = err instanceof FirecrawlError ? err.status ?? 500 : 500
+    const message = err instanceof Error ? err.message : "Search failed."
+    return NextResponse.json({ error: message }, { status })
+  }
+}

package/app/api/server/download/route.ts

@@ -0,0 +1,112 @@
+import { NextRequest, NextResponse } from "next/server";
+import fs from "fs";
+import path from "path";
+import JSZip from "jszip";
+import { getActiveServer } from "@larkup/core/workspace";
+
+function getFilesRecursively(dir: string, baseDir: string): any[] {
+  let results: any[] = [];
+  if (!fs.existsSync(dir)) return results;
+  const list = fs.readdirSync(dir);
+  for (const file of list) {
+    if (file === "node_modules" || file === ".git" || file === "server.log")
+      continue;
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat && stat.isDirectory()) {
+      results = results.concat(getFilesRecursively(filePath, baseDir));
+    } else {
+      results.push({
+        file: path.relative(baseDir, filePath).replace(/\\/g, "/"),
+        fullPath: filePath,
+      });
+    }
+  }
+  return results;
+}
+
+export async function GET(req: NextRequest) {
+  try {
+    const url = new URL(req.url);
+    const serverId = url.searchParams.get("serverId") || "default";
+    
+    let activeId = serverId;
+    if (activeId === "default") {
+      const server = await getActiveServer();
+      if (server) activeId = server.id;
+    }
+
+    const cwd = process.cwd();
+    let config: any = null;
+    const configCandidates = [
+      path.join(cwd, ".larkup", "servers", activeId, "config.json"),
+      path.join(cwd, ".larkup", "config.json"),
+    ];
+    for (const cfgPath of configCandidates) {
+      if (fs.existsSync(cfgPath)) {
+        try {
+          config = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+          break;
+        } catch {}
+      }
+    }
+
+    if (!config) {
+      return NextResponse.json({ error: "Configuration not found." }, { status: 404 });
+    }
+
+    const { generateServer } = await import("@larkup/core/generator/generate-server");
+    const generated = generateServer(config);
+    
+    const zip = new JSZip();
+
+    // Add generated files
+    for (const f of generated.files) {
+      zip.file(f.path, f.contents);
+    }
+
+    // Add .env.example as .env
+    const envExample = generated.files.find(f => f.path === ".env.example");
+    if (envExample) {
+      zip.file(".env", envExample.contents);
+    }
+
+    // Add LanceDB data if local
+    const isLanceLocal = config.vectorStore === "lancedb" && config.storeConfig?.mode !== "cloud";
+    if (isLanceLocal) {
+      const lancedbDir = path.join(cwd, ".larkup", "servers", activeId, "lancedb");
+      if (fs.existsSync(lancedbDir)) {
+        const lancedbFiles = getFilesRecursively(
+          lancedbDir,
+          path.join(cwd, ".larkup", "servers", activeId)
+        );
+        for (const f of lancedbFiles) {
+          zip.file(f.file, fs.readFileSync(f.fullPath));
+        }
+      }
+    }
+
+    // Add CI/CD templates based on provider if provided
+    const provider = url.searchParams.get("provider");
+    if (provider === "azure-container-apps") {
+      zip.file("DEPLOY_GUIDE.md", "# Azure Container Apps Deployment Guide\n\n1. Install Azure CLI.\n2. Run `az containerapp up --source . --name my-rag-server`.");
+    } else if (provider === "aws-app-runner") {
+      zip.file("DEPLOY_GUIDE.md", "# AWS App Runner Deployment Guide\n\n1. Push this folder to a GitHub repository.\n2. Connect the repository in AWS App Runner.\n3. Provide the Build Command: `npm install` and Start Command: `npm start`.");
+    } else if (provider === "gcp-cloud-run") {
+      zip.file("DEPLOY_GUIDE.md", "# GCP Cloud Run Deployment Guide\n\n1. Install gcloud CLI.\n2. Run `gcloud run deploy my-rag-server --source .`.");
+    }
+
+    const buffer = await zip.generateAsync({ type: "nodebuffer" });
+
+    return new NextResponse(buffer, {
+      headers: {
+        "Content-Type": "application/zip",
+        "Content-Disposition": `attachment; filename="${config.projectName || "rag-server"}.zip"`,
+      },
+    });
+
+  } catch (error: any) {
+    console.error("ZIP Generation Error:", error);
+    return NextResponse.json({ error: error.message }, { status: 500 });
+  }
+}

package/app/api/server/generate/route.ts

@@ -0,0 +1,46 @@
+import { NextResponse } from "next/server"
+import JSZip from "jszip"
+import { readConfig } from "@larkup/core/config-store"
+import { generateServer } from "@larkup/core/generator/generate-server"
+import { getActiveServer } from "@larkup/core/workspace"
+
+export const dynamic = "force-dynamic"
+
+/**
+ * Phase 4 — server generation.
+ *
+ *   GET /api/server/generate            -> JSON manifest (files + deps + env)
+ *   GET /api/server/generate?download=1 -> a .zip of the generated server
+ *
+ * The generator reads the saved RagConfig and emits a standalone, dependency-
+ * minimal RAG server tailored to the selected vector store.
+ */
+export async function GET(req: Request) {
+  const config = await readConfig()
+  const server = generateServer(config)
+  const activeServer = await getActiveServer()
+  const serverId = activeServer?.id ?? "default"
+
+  const url = new URL(req.url)
+  if (url.searchParams.get("download") === "1") {
+    const zip = new JSZip()
+    const root = server.projectName || "rag-server"
+    for (const file of server.files) {
+      if (file.encoding === "base64") {
+        zip.file(`${root}/${file.path}`, file.contents, { base64: true })
+      } else {
+        zip.file(`${root}/${file.path}`, file.contents)
+      }
+    }
+    const buf = await zip.generateAsync({ type: "nodebuffer" })
+    return new NextResponse(new Uint8Array(buf), {
+      headers: {
+        "Content-Type": "application/zip",
+        "Content-Disposition": `attachment; filename="${root}.zip"`,
+        "Cache-Control": "no-store",
+      },
+    })
+  }
+
+  return NextResponse.json({ config, server, serverId })
+}

package/app/api/server/local/route.ts

@@ -0,0 +1,42 @@
+import { NextResponse } from "next/server"
+import { readConfig } from "@larkup/core/config-store"
+import {
+  readServerState,
+  refreshServerStatus,
+  startServer,
+  stopServer,
+} from "@larkup/core/generator/server-runtime"
+
+export const dynamic = "force-dynamic"
+export const maxDuration = 300
+
+/** GET — current local server status (re-checked against the live process). */
+export async function GET() {
+  const state = await refreshServerStatus()
+  return NextResponse.json({ state })
+}
+
+/** POST { action: "start" | "stop" } — control the local generated server. */
+export async function POST(req: Request) {
+  let action = "start"
+  let serverApiKey = ""
+  try {
+    const body = await req.json()
+    if (body?.action) action = String(body.action)
+    if (body?.serverApiKey) serverApiKey = String(body.serverApiKey)
+  } catch {
+    // default to start
+  }
+
+  if (action === "stop") {
+    return NextResponse.json({ state: await stopServer() })
+  }
+
+  if (action === "status") {
+    return NextResponse.json({ state: await readServerState() })
+  }
+
+  const config = await readConfig()
+  const state = await startServer(config, serverApiKey)
+  return NextResponse.json({ state })
+}

package/app/api/servers/route.ts

@@ -0,0 +1,137 @@
+import { NextResponse } from "next/server"
+import {
+  createServer,
+  deleteServer,
+  getWorkspace,
+  renameServer,
+  runWithServer,
+  setActiveServer,
+  setUsername,
+  setMode,
+  type ServerMeta,
+  type WorkspaceMode,
+} from "@larkup/core/workspace"
+import { corpusStats } from "@larkup/core/documents-store"
+import { readRun } from "@larkup/core/index-store"
+import { readServerState } from "@larkup/core/generator/server-runtime"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/** A server entry enriched with lightweight, per-server status for the UI. */
+async function decorate(server: ServerMeta) {
+  return runWithServer(server.id, async () => {
+    const [stats, run, state] = await Promise.all([
+      corpusStats(),
+      readRun(),
+      readServerState(),
+    ])
+    return {
+      ...server,
+      docCount: stats.docCount,
+      indexed: run?.status === "completed" && (run.totalChunks ?? 0) > 0,
+      running: state.running,
+      endpoint: state.endpoint,
+    }
+  })
+}
+
+/** GET → the full workspace: username, active server, and decorated servers. */
+export async function GET() {
+  const ws = await getWorkspace()
+  const servers = await Promise.all(ws.servers.map(decorate))
+  return NextResponse.json({
+    username: ws.username,
+    activeServerId: ws.activeServerId,
+    servers,
+    mode: ws.mode,
+  })
+}
+
+/** POST { name } → create a new server (and make it active). */
+export async function POST(req: Request) {
+  let body: { name?: string } = {}
+  try {
+    body = await req.json()
+  } catch {
+    // empty body is fine — server gets a default name
+  }
+  const { server } = await createServer(body.name ?? "Untitled server")
+  return NextResponse.json({ server }, { status: 201 })
+}
+
+/**
+ * PATCH → workspace mutations.
+ * Body: { action: "activate" | "rename" | "setUsername", serverId?, name?, username? }
+ */
+export async function PATCH(req: Request) {
+  let body: {
+    action?: string
+    serverId?: string
+    name?: string
+    username?: string
+    mode?: WorkspaceMode
+  }
+  try {
+    body = await req.json()
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 })
+  }
+
+  switch (body.action) {
+    case "activate": {
+      if (!body.serverId) {
+        return NextResponse.json(
+          { error: "serverId is required." },
+          { status: 400 },
+        )
+      }
+      return NextResponse.json({ workspace: await setActiveServer(body.serverId) })
+    }
+    case "rename": {
+      if (!body.serverId || !body.name?.trim()) {
+        return NextResponse.json(
+          { error: "serverId and name are required." },
+          { status: 400 },
+        )
+      }
+      return NextResponse.json({
+        workspace: await renameServer(body.serverId, body.name),
+      })
+    }
+    case "setUsername": {
+      return NextResponse.json({
+        workspace: await setUsername(body.username ?? ""),
+      })
+    }
+    case "setMode": {
+      if (!body.mode || !(["tech", "simple"] as string[]).includes(body.mode)) {
+        return NextResponse.json(
+          { error: "mode must be 'tech' or 'simple'." },
+          { status: 400 },
+        )
+      }
+      return NextResponse.json({
+        workspace: await setMode(body.mode),
+      })
+    }
+    default:
+      return NextResponse.json({ error: "Unknown action." }, { status: 400 })
+  }
+}
+
+/** DELETE ?id=<id> — stop the server's instance, then remove all its data. */
+export async function DELETE(req: Request) {
+  const id = new URL(req.url).searchParams.get("id")
+  if (!id) {
+    return NextResponse.json({ error: "id is required." }, { status: 400 })
+  }
+  // Stop any running instance in the target server's context first.
+  try {
+    const { stopServer } = await import("@larkup/core/generator/server-runtime")
+    await runWithServer(id, () => stopServer())
+  } catch {
+    // best effort — still remove the data
+  }
+  return NextResponse.json({ workspace: await deleteServer(id) })
+}