larkup 0.1.14

package/app/api/deploy/ssh/route.ts

@@ -0,0 +1,340 @@
+import { NextRequest } from "next/server";
+import fs from "fs";
+import path from "path";
+import { getActiveServer } from "@larkup/core/workspace";
+import { NodeSSH } from "node-ssh";
+import crypto from "crypto";
+
+function getFilesRecursively(dir: string, baseDir: string): any[] {
+  let results: any[] = [];
+  if (!fs.existsSync(dir)) return results;
+  const list = fs.readdirSync(dir);
+  for (const file of list) {
+    if (file === "node_modules" || file === ".git" || file === "server.log") continue;
+    const filePath = path.join(dir, file);
+    const stat = fs.statSync(filePath);
+    if (stat && stat.isDirectory()) {
+      results = results.concat(getFilesRecursively(filePath, baseDir));
+    } else {
+      const data = fs.readFileSync(filePath);
+      const relativePath = path.relative(baseDir, filePath).replace(/\\/g, "/");
+      results.push({ file: relativePath, data: data.toString("utf8") });
+    }
+  }
+  return results;
+}
+
+export async function POST(req: NextRequest) {
+  const body = await req.json();
+  let { serverId, host, username, privateKeyOrPassword, envVars, newPassword: userNewPassword } = body;
+
+  const stream = new ReadableStream({
+    async start(controller) {
+      function sendEvent(data: any) {
+        try {
+          controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`));
+        } catch (e) {}
+      }
+      function log(msg: string) {
+        sendEvent({ type: "log", message: msg });
+      }
+      function done(result: any) {
+        sendEvent({ type: "done", ...result });
+        try { controller.close(); } catch (e) {}
+      }
+      function errorMsg(err: string) {
+        sendEvent({ type: "error", error: err });
+        try { controller.close(); } catch (e) {}
+      }
+
+      let activeId = serverId;
+      if (activeId === "default") {
+        const server = await getActiveServer();
+        if (server) activeId = server.id;
+      }
+
+      if (!host || !username || !privateKeyOrPassword.value) {
+        return errorMsg("Host, username, and credentials are required.");
+      }
+
+      try {
+        log(`Preparing deployment for server ${activeId}...`);
+        const cwd = process.cwd();
+        let config: any = null;
+        const configCandidates = [
+          path.join(cwd, ".larkup", "servers", activeId, "config.json"),
+          path.join(cwd, ".larkup", "config.json"),
+        ];
+        for (const cfgPath of configCandidates) {
+          if (fs.existsSync(cfgPath)) {
+            try {
+              config = JSON.parse(fs.readFileSync(cfgPath, "utf-8"));
+              break;
+            } catch {}
+          }
+        }
+
+        if (!config) {
+          return errorMsg("Configuration not found. Please save your settings first.");
+        }
+
+        const { generateServer } = await import("@larkup/core/generator/generate-server");
+        const generated = generateServer(config);
+        
+        const filesToUpload: { localPath?: string; contents?: string; remotePath: string }[] = [];
+
+        for (const f of generated.files) {
+          filesToUpload.push({ contents: f.contents, remotePath: f.path });
+        }
+
+        const envLines = Object.entries(envVars).map(([k, v]) => `${k}=${v}`);
+        const isLanceLocal = config.vectorStore === "lancedb" && config.storeConfig?.mode !== "cloud";
+        if (isLanceLocal) {
+          envLines.push("LANCEDB_PATH=./.larkup/lancedb");
+        }
+        filesToUpload.push({ contents: envLines.join("\n"), remotePath: ".env" });
+
+        if (isLanceLocal) {
+          const lancedbDir = path.join(cwd, ".larkup", "servers", activeId, "lancedb");
+          if (fs.existsSync(lancedbDir)) {
+            const lancedbFiles = getFilesRecursively(
+              lancedbDir,
+              path.join(cwd, ".larkup", "servers", activeId)
+            );
+            for (const f of lancedbFiles) {
+              filesToUpload.push({
+                localPath: path.join(cwd, ".larkup", "servers", activeId, f.file),
+                remotePath: f.file
+              });
+            }
+          }
+        }
+
+        let updatedPassword = userNewPassword || "";
+        let ssh = new NodeSSH();
+        log(`Connecting to ${username}@${host} via SSH...`);
+        
+        const connectOpts = {
+          host,
+          username,
+          privateKey: privateKeyOrPassword.type === "key" ? privateKeyOrPassword.value : undefined,
+          password: privateKeyOrPassword.type === "password" ? privateKeyOrPassword.value : undefined,
+          readyTimeout: 20000,
+          tryKeyboard: true,
+          onKeyboardInteractive: (
+            _name: string,
+            _instructions: string,
+            _instructionsLang: string,
+            prompts: { prompt: string; echo: boolean }[],
+            finish: (responses: string[]) => void
+          ) => {
+            if (prompts.length > 0 && privateKeyOrPassword.type === "password") {
+              const answers = prompts.map((p) => {
+                return privateKeyOrPassword.value;
+              });
+              finish(answers);
+            } else {
+              finish([]);
+            }
+          }
+        };
+
+        await ssh.connect(connectOpts);
+        log(`Connected successfully.`);
+
+        // Helper to run commands and stream output
+        const runCommand = async (cmd: string, opts?: { cwd?: string; ignoreErrors?: boolean }) => {
+           log(`$ ${cmd}`);
+           const res = await ssh.execCommand(cmd, { 
+             cwd: opts?.cwd, 
+             onStdout: (chunk: Buffer) => log(chunk.toString('utf8').trimEnd()),
+             onStderr: (chunk: Buffer) => log(chunk.toString('utf8').trimEnd())
+           });
+           const combined = `${res.stdout} ${res.stderr}`.toLowerCase();
+           if (combined.includes("password change required") || combined.includes("password has expired")) {
+              throw new Error("PASSWORD_EXPIRED");
+           }
+           if (res.code !== 0 && !opts?.ignoreErrors) {
+             throw new Error(`Command failed (exit ${res.code}): ${cmd}\n${res.stderr}`);
+           }
+           return res;
+        };
+
+        const remoteDir = `/opt/buddyhere-rag-${activeId}`;
+
+        // ── Step 1: Test if the session works or if password change is needed ──
+        let needsPasswordChange = false;
+        try {
+          await runCommand("echo ok");
+        } catch (err: any) {
+          if (err.message === "PASSWORD_EXPIRED") {
+            needsPasswordChange = true;
+          } else {
+            throw err;
+          }
+        }
+
+        // ── Step 2: If password expired, handle password change ──
+        if (needsPasswordChange) {
+          // If the user hasn't provided a new password yet, ask the frontend
+          if (!updatedPassword) {
+            log(`⚠️ Password has expired! A new password is required.`);
+            sendEvent({ type: "password_change_required" });
+            try { controller.close(); } catch (e) {}
+            return;
+          }
+
+          // User provided a new password — change it via PTY
+          log(`⚠️ Password has expired! Changing password...`);
+          
+          await new Promise<void>((resolve, reject) => {
+            const timeout = setTimeout(() => {
+              reject(new Error("Password change timed out after 30 seconds."));
+            }, 30000);
+
+            ssh.requestShell({ term: 'xterm' }).then(shell => {
+              let buffer = "";
+              let stage = 0; // 0=waiting for current, 1=waiting for new, 2=waiting for retype, 3=done
+
+              shell.on('data', (data: Buffer) => {
+                const text = data.toString('utf8');
+                buffer += text;
+                // Stream raw shell output to the UI (clean up control chars)
+                const clean = text.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '').trim();
+                if (clean) log(clean);
+                
+                const lower = buffer.toLowerCase();
+                
+                if (stage === 0) {
+                  if ((lower.includes("current") || lower.includes("old")) && lower.includes("password")) {
+                    shell.write(privateKeyOrPassword.value + "\n");
+                    buffer = "";
+                    stage = 1;
+                  } else if (lower.includes("new") && lower.includes("password")) {
+                    // Hetzner sometimes skips asking for the current password
+                    shell.write(updatedPassword + "\n");
+                    buffer = "";
+                    stage = 2;
+                  }
+                } else if (stage === 1 && lower.includes("new") && lower.includes("password")) {
+                  shell.write(updatedPassword + "\n");
+                  buffer = "";
+                  stage = 2;
+                } else if (stage === 2 && (lower.includes("retype") || lower.includes("re-enter") || lower.includes("confirm") || (lower.includes("new") && lower.includes("password")))) {
+                  shell.write(updatedPassword + "\n");
+                  buffer = "";
+                  stage = 3;
+                } else if (stage === 3 && (lower.includes("updated successfully") || lower.includes("password changed") || lower.includes("$") || lower.includes("#"))) {
+                  clearTimeout(timeout);
+                  shell.end();
+                  resolve();
+                }
+              });
+              
+              shell.on('close', () => {
+                clearTimeout(timeout);
+                resolve();
+              });
+              shell.on('error', (err: Error) => {
+                clearTimeout(timeout);
+                reject(err);
+              });
+            }).catch((err: Error) => {
+              clearTimeout(timeout);
+              reject(err);
+            });
+          });
+
+          log("✅ Password changed successfully.");
+          sendEvent({ type: "password_changed", newPassword: updatedPassword });
+
+          // Disconnect and reconnect with the new password
+          ssh.dispose();
+          log("Reconnecting with new password...");
+          ssh = new NodeSSH();
+          await ssh.connect({
+            ...connectOpts,
+            password: updatedPassword,
+            onKeyboardInteractive: (
+              _name: string,
+              _instructions: string,
+              _instructionsLang: string,
+              prompts: { prompt: string; echo: boolean }[],
+              finish: (responses: string[]) => void
+            ) => {
+              if (prompts.length > 0) {
+                finish(prompts.map(() => updatedPassword));
+              } else {
+                finish([]);
+              }
+            }
+          });
+          log("Reconnected successfully.");
+        }
+
+        // ── Step 3: Ensure Docker is installed ──
+        log("Checking for Docker...");
+        const dockerCheck = await runCommand("docker --version", { ignoreErrors: true });
+        if (dockerCheck.code !== 0) {
+          log("Docker not found. Installing Docker (this may take a few minutes)...");
+          await runCommand("curl -fsSL https://get.docker.com -o /tmp/get-docker.sh && sh /tmp/get-docker.sh");
+        }
+
+        // ── Step 4: Upload files ──
+        log(`Creating remote directory ${remoteDir}...`);
+        await runCommand(`mkdir -p ${remoteDir}`);
+
+        log(`Uploading ${filesToUpload.length} files...`);
+        for (const f of filesToUpload) {
+          const dir = path.posix.dirname(f.remotePath);
+          if (dir && dir !== ".") {
+            await runCommand(`mkdir -p ${remoteDir}/${dir}`, { ignoreErrors: true });
+          }
+          
+          const target = `${remoteDir}/${f.remotePath}`;
+          if (f.localPath) {
+            log(`  ↑ ${f.remotePath}`);
+            await ssh.putFile(f.localPath, target);
+          } else if (f.contents) {
+            log(`  ↑ ${f.remotePath}`);
+            const tempPath = path.join("/tmp", `buddyhere-upload-${Date.now()}-${Math.random().toString(36).substring(7)}`);
+            fs.writeFileSync(tempPath, f.contents);
+            await ssh.putFile(tempPath, target);
+            fs.unlinkSync(tempPath);
+          }
+        }
+
+        // ── Step 5: Docker Compose ──
+        log(`Starting Docker containers...`);
+        await runCommand("docker compose down 2>/dev/null || true", { cwd: remoteDir, ignoreErrors: true });
+        
+        const composeRes = await runCommand("docker compose up -d --build", { cwd: remoteDir, ignoreErrors: true });
+        if (composeRes.code !== 0) {
+          log("docker compose v2 failed, trying docker-compose v1...");
+          await runCommand("docker-compose up -d --build", { cwd: remoteDir });
+        }
+
+        ssh.dispose();
+        log(`🎉 Deployment completed successfully!`);
+
+        done({
+          success: true,
+          url: `http://${host}:8080`,
+          newPassword: updatedPassword || undefined,
+        });
+
+      } catch (error: any) {
+        console.error("SSH Deploy Error:", error);
+        errorMsg(error.message || "Failed to deploy via SSH.");
+      }
+    }
+  });
+
+  return new Response(stream, {
+    headers: {
+      "Content-Type": "text/event-stream",
+      "Cache-Control": "no-cache",
+      "Connection": "keep-alive",
+    },
+  });
+}

package/app/api/documents/route.ts

@@ -0,0 +1,90 @@
+import { NextResponse } from "next/server"
+import {
+  addDocument,
+  clearDocuments,
+  corpusStats,
+  deleteDocument,
+  deleteDocuments,
+  readDocuments,
+  updateDocument,
+} from "@larkup/core/documents-store"
+import type { DocumentSource } from "@larkup/core/types"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/** GET → the full corpus plus summary stats. */
+export async function GET() {
+  const [documents, stats] = await Promise.all([readDocuments(), corpusStats()])
+  return NextResponse.json({ documents, stats })
+}
+
+/**
+ * POST → ingest pasted text or an uploaded file's contents.
+ * Body: { title, content, source: "paste" | "upload", url? }
+ */
+export async function POST(req: Request) {
+  try {
+    const body = (await req.json()) as {
+      title?: string
+      content?: string
+      source?: DocumentSource
+      url?: string
+      metadata?: Record<string, any>
+    }
+    if (!body.content || !body.content.trim()) {
+      return NextResponse.json({ error: "Content is empty." }, { status: 400 })
+    }
+    const doc = await addDocument({
+      title: body.title ?? "Untitled",
+      content: body.content,
+      source: body.source === "upload" ? "upload" : "paste",
+      url: body.url,
+      metadata: body.metadata,
+    })
+    return NextResponse.json({ document: doc }, { status: 201 })
+  } catch (err) {
+    const message = err instanceof Error ? err.message : "Failed to add document."
+    return NextResponse.json({ error: message }, { status: 500 })
+  }
+}
+
+/**
+ * PATCH → edit a document in place.
+ * Body: { id, title?, content?, url? }
+ */
+export async function PATCH(req: Request) {
+  let body: { id?: string; title?: string; content?: string; url?: string; metadata?: Record<string, any> }
+  try {
+    body = await req.json()
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 })
+  }
+  if (!body.id) {
+    return NextResponse.json({ error: "id is required." }, { status: 400 })
+  }
+  if (body.content !== undefined && !body.content.trim()) {
+    return NextResponse.json({ error: "Content is empty." }, { status: 400 })
+  }
+  const doc = await updateDocument(body.id, {
+    title: body.title,
+    content: body.content,
+    url: body.url,
+    metadata: body.metadata,
+  })
+  if (!doc) {
+    return NextResponse.json({ error: "Document not found." }, { status: 404 })
+  }
+  return NextResponse.json({ document: doc })
+}
+
+/** DELETE ?id=<id> removes one doc; DELETE ?ids=1,2 removes many; DELETE with no id clears the corpus. */
+export async function DELETE(req: Request) {
+  const url = new URL(req.url)
+  const id = url.searchParams.get("id")
+  const ids = url.searchParams.get("ids")
+  if (id) await deleteDocument(id)
+  else if (ids) await deleteDocuments(ids.split(","))
+  else await clearDocuments()
+  return NextResponse.json({ ok: true })
+}

package/app/api/firecrawl/local/route.ts

@@ -0,0 +1,46 @@
+import { NextResponse } from "next/server"
+import {
+  checkDocker,
+  readLocalState,
+  refreshLocalStatus,
+  startLocal,
+  stopLocal,
+} from "@larkup/scraper/local-runtime"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/** GET → current local instance state + docker availability. */
+export async function GET() {
+  const [state, docker] = await Promise.all([
+    refreshLocalStatus(),
+    checkDocker(),
+  ])
+  // Never leak the bearer token to the client; just report whether one exists.
+  const { apiKey, ...safe } = state
+  return NextResponse.json({
+    state: { ...safe, hasKey: Boolean(apiKey) },
+    docker,
+  })
+}
+
+/** POST { action: "start" | "stop" } → control the local Firecrawl container. */
+export async function POST(req: Request) {
+  let action: string | undefined
+  try {
+    ;({ action } = (await req.json()) as { action?: string })
+  } catch {
+    return NextResponse.json({ error: "Invalid request body." }, { status: 400 })
+  }
+
+  if (action !== "start" && action !== "stop") {
+    return NextResponse.json(
+      { error: 'action must be "start" or "stop".' },
+      { status: 400 },
+    )
+  }
+
+  const state = action === "start" ? await startLocal() : await stopLocal()
+  const { apiKey, ...safe } = state
+  return NextResponse.json({ state: { ...safe, hasKey: Boolean(apiKey) } })
+}

package/app/api/index/route.ts

@@ -0,0 +1,113 @@
+import { NextResponse } from "next/server"
+import { readConfig } from "@larkup/core/config-store"
+import { corpusStats } from "@larkup/core/documents-store"
+import { isRunning, readRun } from "@larkup/core/index-store"
+import { createRun, runIndexer } from "@larkup/core/indexing/indexer"
+import { getEmbeddingModel } from "@larkup/core/embeddings/registry"
+import type { RagConfig } from "@larkup/core/types"
+
+export const dynamic = "force-dynamic"
+
+/**
+ * Assess whether indexing can run with the current config + corpus.
+ * Returned to the UI so it can explain exactly what is missing before enabling
+ * the "Build index" button.
+ */
+function assessReadiness(config: RagConfig, docCount: number) {
+  const blockers: string[] = []
+
+  const model = getEmbeddingModel(config.embeddingModelId)
+  if (!model) blockers.push("No embedding model is selected.")
+
+  if (docCount === 0)
+    blockers.push("The corpus is empty — load documents in the Data stage.")
+
+  // Pinecone needs a hosted index + key; LanceDB is local and always ready.
+  if (config.vectorStore === "pinecone") {
+    if (!config.storeConfig?.apiKey?.trim())
+      blockers.push("PINECONE_API_KEY is not set.")
+    if (!config.storeConfig?.indexName?.trim())
+      blockers.push("A Pinecone index name is required.")
+  }
+
+  return { ready: blockers.length === 0, blockers }
+}
+
+export async function GET() {
+  const [config, stats, run] = await Promise.all([
+    readConfig(),
+    corpusStats(),
+    readRun(),
+  ])
+  const { ready, blockers } = assessReadiness(config, stats.docCount)
+
+  let unindexedCount = 0
+  if (run?.status === "completed") {
+    const { readDocuments } = await import("@larkup/core/documents-store")
+    const docs = await readDocuments()
+    unindexedCount = docs.filter(d => d.createdAt > run.startedAt).length
+  } else {
+    unindexedCount = stats.docCount
+  }
+
+  return NextResponse.json({
+    run,
+    running: await isRunning(),
+    docCount: stats.docCount,
+    charCount: stats.charCount,
+    ready,
+    blockers,
+    unindexedCount,
+    config: {
+      embeddingModelId: config.embeddingModelId,
+      vectorStore: config.vectorStore,
+      indexType: config.indexType,
+      chunking: config.chunking,
+    },
+  })
+}
+
+export async function POST(req: Request) {
+  if (await isRunning()) {
+    return NextResponse.json(
+      { error: "An indexing run is already in progress." },
+      { status: 409 },
+    )
+  }
+
+  const [config, stats] = await Promise.all([readConfig(), corpusStats()])
+  const { ready, blockers } = assessReadiness(config, stats.docCount)
+  if (!ready) {
+    return NextResponse.json(
+      { error: blockers.join(" ") || "Indexing is not ready.", blockers },
+      { status: 400 },
+    )
+  }
+
+  let body: any = {}
+  try {
+    body = await req.json()
+  } catch {}
+  
+  const incremental = body.incremental === true
+  const previousRun = await readRun()
+
+  const run = await createRun(config)
+  // Fire-and-forget: the request returns immediately and the UI polls progress.
+  void runIndexer(run.id, config, incremental ? previousRun : null)
+
+  return NextResponse.json({ run }, { status: 202 })
+}
+
+export async function DELETE() {
+  const { readRun, patchRun } = await import("@larkup/core/index-store")
+  const run = await readRun()
+  if (run && ["chunking", "embedding", "upserting"].includes(run.status)) {
+    await patchRun({
+      status: "failed",
+      error: "Cancelled by user.",
+      finishedAt: new Date().toISOString(),
+    })
+  }
+  return NextResponse.json({ success: true })
+}

package/app/api/jobs/[id]/route.ts

@@ -0,0 +1,44 @@
+import { NextResponse } from "next/server"
+import { deleteJob, getJob } from "@larkup/core/jobs-store"
+import { cancelJob, syncJob } from "@larkup/scraper/job-runner"
+
+export const runtime = "nodejs"
+export const dynamic = "force-dynamic"
+
+/**
+ * GET → advance the job one increment (pull newly-scraped pages, persist them)
+ * and return the latest state. The UI polls this on an interval, which is how a
+ * long-running crawl streams documents into the corpus.
+ */
+export async function GET(
+  _req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) {
+  const { id } = await params
+  const job = await syncJob(id)
+  if (!job) {
+    return NextResponse.json({ error: "Job not found." }, { status: 404 })
+  }
+  return NextResponse.json({ job })
+}
+
+/** DELETE → cancel a running job, or remove a finished one (?remove=1). */
+export async function DELETE(
+  req: Request,
+  { params }: { params: Promise<{ id: string }> },
+) {
+  const { id } = await params
+  const remove = new URL(req.url).searchParams.get("remove")
+
+  if (remove) {
+    await deleteJob(id)
+    return NextResponse.json({ ok: true })
+  }
+
+  const existing = await getJob(id)
+  if (!existing) {
+    return NextResponse.json({ error: "Job not found." }, { status: 404 })
+  }
+  const job = await cancelJob(id)
+  return NextResponse.json({ job })
+}