larkup 0.1.14

package/components/data/data-workspace.tsx

@@ -0,0 +1,272 @@
+"use client";
+
+import { useState, useEffect, useRef } from "react";
+import useSWR, { useSWRConfig } from "swr";
+import { ClipboardPaste, FileUp, Globe, Layers, Plus } from "lucide-react";
+import { toast } from "sonner";
+import type { CrawlJob, SourceDocument } from "@larkup/core/types";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import { Skeleton } from "@/components/ui/skeleton";
+import { Button } from "@/components/ui/button";
+
+import { ScrapePanel } from "@/components/data/scrape-panel";
+import { PastePanel } from "@/components/data/paste-panel";
+import { UploadPanel } from "@/components/data/upload-panel";
+import { JobsPanel } from "@/components/data/jobs-panel";
+import { CorpusPanel } from "@/components/data/corpus-panel";
+import { FirecrawlNotice } from "@/components/data/firecrawl-notice";
+
+const fetcher = (url: string) => fetch(url).then((r) => r.json());
+
+interface DocsResponse {
+  documents: SourceDocument[];
+  stats: {
+    docCount: number;
+    charCount: number;
+    bySource: Record<string, number>;
+  };
+}
+
+async function fetchJobsWithSync(url: string): Promise<{
+  jobs: CrawlJob[];
+  configured: boolean;
+}> {
+  const { jobs, configured } = (await fetcher(url)) as {
+    jobs: CrawlJob[];
+    configured: boolean;
+  };
+  const active = jobs.filter(
+    (j) => j.status === "running" || j.status === "queued",
+  );
+  if (active.length === 0) return { jobs, configured };
+
+  const advanced = await Promise.all(
+    active.map((j) =>
+      fetch(`/api/jobs/${j.id}`)
+        .then((r) => r.json())
+        .then((d) => d.job as CrawlJob)
+        .catch(() => j),
+    ),
+  );
+  const map = new Map(advanced.map((j) => [j.id, j]));
+  return { jobs: jobs.map((j) => map.get(j.id) ?? j), configured };
+}
+
+export function DataWorkspace() {
+  const [activeTab, setActiveTab] = useState<"add" | "corpus" | "etl">("add");
+  const [scrapeHasError, setScrapeHasError] = useState(false);
+  const prevJobsRef = useRef<CrawlJob[]>([]);
+
+  const jobsQuery = useSWR("/api/jobs", fetchJobsWithSync, {
+    refreshInterval: (data) =>
+      data?.jobs.some((j) => j.status === "running" || j.status === "queued")
+        ? 4000
+        : 0,
+  });
+  const jobs = jobsQuery.data?.jobs ?? [];
+  const configured = jobsQuery.data?.configured ?? true;
+  const hasActive = jobs.some(
+    (j) => j.status === "running" || j.status === "queued",
+  );
+
+  useEffect(() => {
+    const prevJobs = prevJobsRef.current;
+    if (prevJobs.length > 0 && jobs.length > 0) {
+      const justCompleted = jobs.filter(
+        (j) =>
+          j.status === "completed" &&
+          prevJobs.some(
+            (pj) =>
+              pj.id === j.id &&
+              (pj.status === "running" || pj.status === "queued"),
+          ),
+      );
+
+      if (justCompleted.length > 0) {
+        toast.success(
+          "Indexing these corpus just added successfully completed.",
+        );
+      }
+    }
+    prevJobsRef.current = jobs;
+  }, [jobs]);
+
+  const docsQuery = useSWR<DocsResponse>("/api/documents", fetcher, {
+    refreshInterval: hasActive ? 5000 : 0,
+  });
+  const documents = docsQuery.data?.documents ?? [];
+  const stats = docsQuery.data?.stats;
+
+  const { mutate: mutateGlobal } = useSWRConfig();
+
+  const refreshAll = () => {
+    jobsQuery.mutate();
+    docsQuery.mutate();
+    mutateGlobal("/api/index");
+  };
+
+  const handleDataAdded = () => {
+    refreshAll();
+    setActiveTab("etl");
+  };
+
+  return (
+    <div className="px-6  md:px-8">
+      <Tabs
+        value={activeTab}
+        onValueChange={(v) => setActiveTab(v as "add" | "corpus" | "etl")}
+        className="w-full"
+      >
+        <TabsList className="mb-3 flex w-full h-10 justify-start rounded-none border-b border-border bg-transparent p-0! ">
+          <TabsTrigger
+            value="add"
+            className="h-full flex-none rounded-none border-b-2 border-transparent bg-transparent! px-4 font-medium text-muted-foreground shadow-none! transition-none data-active:border-transparent data-[state=active]:border-transparent data-active:border-b-foreground data-[state=active]:border-b-foreground data-active:text-foreground data-[state=active]:text-foreground hover:text-foreground"
+          >
+            {/* <Plus className="size-4 mr-2" /> */}
+            Add Data
+          </TabsTrigger>
+          <TabsTrigger
+            value="corpus"
+            className="h-full flex-none rounded-none border-b-2 border-transparent bg-transparent! px-4 font-medium text-muted-foreground shadow-none! transition-none data-active:border-transparent data-[state=active]:border-transparent data-active:border-b-foreground data-[state=active]:border-b-foreground data-active:text-foreground data-[state=active]:text-foreground hover:text-foreground"
+          >
+            Corpus
+          </TabsTrigger>
+          <TabsTrigger
+            value="etl"
+            className="h-full flex-none rounded-none border-b-2 border-transparent bg-transparent! px-4 font-medium text-muted-foreground shadow-none! transition-none data-active:border-transparent data-[state=active]:border-transparent data-active:border-b-foreground data-[state=active]:border-b-foreground data-active:text-foreground data-[state=active]:text-foreground hover:text-foreground"
+          >
+            ETL Jobs
+            {hasActive && (
+              <span className="ml-2 flex items-center gap-1.5 text-xs text-green-600">
+                <span className="relative flex size-2">
+                  <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-green-600 opacity-75" />
+                  <span className="relative inline-flex size-2 rounded-full bg-green-600" />
+                </span>
+              </span>
+            )}
+          </TabsTrigger>
+        </TabsList>
+
+        <TabsContent
+          value="add"
+          className="m-0 focus-visible:outline-none focus-visible:ring-0 p-0"
+        >
+          <Card className="bg-transparent border-none! ring-0 p-0 m-0! pl-0! shadow-none">
+            <CardContent className="border-none shadow-none  m-0! p-0.5">
+              <Tabs defaultValue="scrape" className="flex flex-col w-full">
+                <div className="shrink-0 mb-6">
+                  <TabsList className="inline-flex h-10! items-center justify-center rounded-lg bg-muted/30! border p-1 text-muted-foreground">
+                    <TabsTrigger
+                      value="scrape"
+                      className="inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-sm font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm hover:text-foreground"
+                    >
+                      <Globe className="size-4 mr-2" />
+                      Web scrape
+                    </TabsTrigger>
+                    <TabsTrigger
+                      value="paste"
+                      className="inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-sm font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm hover:text-foreground"
+                    >
+                      <ClipboardPaste className="size-4 mr-2" />
+                      Paste text
+                    </TabsTrigger>
+                    <TabsTrigger
+                      value="upload"
+                      className="inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-sm font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm hover:text-foreground"
+                    >
+                      <FileUp className="size-4 mr-2" />
+                      Upload files
+                    </TabsTrigger>
+                  </TabsList>
+                </div>
+
+                <div className="flex-1 w-full">
+                  <TabsContent
+                    value="scrape"
+                    className="m-0 h-full focus-visible:outline-none focus-visible:ring-0"
+                  >
+                    <div className="mb-4">
+                      <FirecrawlNotice
+                        cloudConfigured={configured}
+                        onChange={refreshAll}
+                        onErrorChange={setScrapeHasError}
+                      />
+                    </div>
+                    {!scrapeHasError && (
+                      <ScrapePanel
+                        disabled={!configured}
+                        onStarted={handleDataAdded}
+                      />
+                    )}
+                  </TabsContent>
+                  <TabsContent
+                    value="paste"
+                    className="m-0 h-full focus-visible:outline-none focus-visible:ring-0"
+                  >
+                    <PastePanel onAdded={handleDataAdded} />
+                  </TabsContent>
+                  <TabsContent
+                    value="upload"
+                    className="m-0 h-full focus-visible:outline-none focus-visible:ring-0"
+                  >
+                    <UploadPanel onAdded={handleDataAdded} />
+                  </TabsContent>
+                </div>
+              </Tabs>
+            </CardContent>
+          </Card>
+        </TabsContent>
+
+        <TabsContent
+          value="corpus"
+          className="m-0 focus-visible:outline-none focus-visible:ring-0"
+        >
+          <Card>
+            <CardHeader className="flex-row flex-wrap items-center justify-between gap-3">
+              <CardTitle className="flex items-center gap-2 text-base">
+                <Layers className="size-4 text-primary" />
+                Corpus
+              </CardTitle>
+              {stats && (
+                <div className="flex items-center gap-4 text-xs text-muted-foreground tabular-nums">
+                  <span>{stats.docCount.toLocaleString()} docs</span>
+                  <span>{stats.charCount.toLocaleString()} chars</span>
+                </div>
+              )}
+            </CardHeader>
+            <CardContent>
+              {docsQuery.isLoading ? (
+                <Skeleton className="h-40 w-full" />
+              ) : (
+                <CorpusPanel documents={documents} onChanged={refreshAll} />
+              )}
+            </CardContent>
+          </Card>
+        </TabsContent>
+
+        <TabsContent
+          value="etl"
+          className="m-0 focus-visible:outline-none focus-visible:ring-0"
+        >
+          <Card>
+            <CardHeader className="flex-row items-center justify-between shrink-0">
+              <CardTitle className="text-base">ETL jobs</CardTitle>
+            </CardHeader>
+            <CardContent>
+              {jobsQuery.isLoading ? (
+                <div className="space-y-3">
+                  <Skeleton className="h-8 w-full" />
+                  <Skeleton className="h-8 w-full" />
+                  <Skeleton className="h-8 w-full" />
+                </div>
+              ) : (
+                <JobsPanel jobs={jobs} onChanged={refreshAll} />
+              )}
+            </CardContent>
+          </Card>
+        </TabsContent>
+      </Tabs>
+    </div>
+  );
+}

package/components/data/firecrawl-notice.tsx

@@ -0,0 +1,236 @@
+"use client";
+
+import { useState, useEffect } from "react";
+import useSWR from "swr";
+import {
+  CheckCircle2,
+  KeyRound,
+  Loader2,
+  Play,
+  Server,
+  Square,
+  TriangleAlert,
+} from "lucide-react";
+import { toast } from "sonner";
+import { formatErrorMessage } from "@/lib/error-formatter";
+import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
+import { Button } from "@/components/ui/button";
+import { Badge } from "@/components/ui/badge";
+
+interface LocalState {
+  running: boolean;
+  endpoint: string;
+  port: number;
+  hasKey: boolean;
+  startedAt?: string;
+  lastError?: string;
+}
+interface DockerState {
+  docker: boolean;
+  compose: boolean;
+  message: string;
+}
+interface LocalResponse {
+  state: LocalState;
+  docker: DockerState;
+}
+
+const fetcher = (url: string) => fetch(url).then((r) => r.json());
+
+/**
+ * Local Firecrawl launcher.
+ *
+ * Lets the user spin up a self-hosted Firecrawl via Docker straight from the
+ * UI — no manual API key needed. We generate a bearer token automatically and
+ * wire the local endpoint into the scraper, so web search + crawling work
+ * against localhost. If the cloud key is already set, this stays collapsed.
+ */
+export function FirecrawlNotice({
+  cloudConfigured = false,
+  onChange,
+  onErrorChange,
+}: {
+  cloudConfigured?: boolean;
+  onChange?: () => void;
+  onErrorChange?: (hasError: boolean) => void;
+}) {
+  const { data, mutate, isLoading } = useSWR<LocalResponse>(
+    "/api/firecrawl/local",
+    fetcher,
+    {
+      refreshInterval: (d) =>
+        d?.state.running || d?.state.startedAt ? 8000 : 0,
+    },
+  );
+  const [busy, setBusy] = useState<"start" | "stop" | null>(null);
+
+  const state = data?.state;
+  const docker = data?.docker;
+  const running = state?.running ?? false;
+  const dockerReady = docker?.compose ?? false;
+  const hasError = !cloudConfigured && (!dockerReady || !!state?.lastError);
+
+  useEffect(() => {
+    if (!isLoading) {
+      onErrorChange?.(hasError);
+    }
+  }, [hasError, isLoading, onErrorChange]);
+
+  async function control(action: "start" | "stop") {
+    setBusy(action);
+    if (action === "start") {
+      toast.info("Launching local Firecrawl…", {
+        description:
+          "Pulling images and starting containers. This can take a minute on first run.",
+      });
+    }
+    try {
+      const res = await fetch("/api/firecrawl/local", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ action }),
+      });
+      const json = (await res.json()) as { state?: LocalState; error?: string };
+      if (!res.ok) throw new Error(json.error || "Request failed");
+
+      if (action === "start") {
+        if (json.state?.running) {
+          toast.success("Local Firecrawl is running", {
+            description: "Web scraping is now wired to your local instance.",
+          });
+        } else {
+          toast.warning("Containers started", {
+            description:
+              json.state?.lastError ?? "Waiting for the API to become healthy.",
+          });
+        }
+      } else {
+        toast.success("Local Firecrawl stopped");
+      }
+      await mutate();
+      onChange?.();
+    } catch (err) {
+      toast.error("Could not control local Firecrawl", {
+        description: formatErrorMessage(err),
+      });
+    } finally {
+      setBusy(null);
+    }
+  }
+
+  // Cloud key present and no local instance running → nothing to show.
+  if (cloudConfigured && !running && !state?.startedAt) return null;
+
+  return (
+    <Alert>
+      {running ? (
+        <Server className="size-4 text-primary" />
+      ) : (
+        <KeyRound className="size-4" />
+      )}
+      <AlertTitle className="flex items-center gap-2">
+        {running ? "Web Crawler is running" : "Launch Web Crawler"}
+        {running && (
+          <Badge
+            variant="secondary"
+            className="gap-1 font-normal text-green-600"
+          >
+            <span className="relative flex size-1.5">
+              <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-green-500 opacity-75" />
+              <span className="relative inline-flex size-1.5 rounded-full bg-green-500" />
+            </span>
+            live
+          </Badge>
+        )}
+      </AlertTitle>
+      <AlertDescription className="flex flex-col gap-0">
+        {running ? (
+          <p className="text-sm">
+            Connected to{" "}
+            <code className="rounded bg-muted px-1 py-0.5 font-mono text-xs">
+              {state?.endpoint}
+            </code>
+            . A bearer token was generated automatically — web search and
+            crawling use it without any setup.
+          </p>
+        ) : (
+          <p className="text-sm">
+            {cloudConfigured
+              ? "You have a cloud key set. You can also run Firecrawl locally in Docker."
+              : "Launch a self-hosted Firecrawl in Docker. A key is generated for you automatically, no signup required. Pasting and uploading work without it."}
+          </p>
+        )}
+
+        {/* Docker readiness hint */}
+        {!isLoading && !dockerReady && !running && docker?.message && (
+          <div className="flex items-start gap-2 rounded-md border border-border bg-muted/40 px-3 py-2 text-xs text-muted-foreground">
+            <TriangleAlert className="mt-0.5 size-3.5 shrink-0 text-destructive" />
+            <span>{formatErrorMessage(docker.message)}</span>
+          </div>
+        )}
+
+        {state?.lastError && !running && (
+          <div className="flex items-start mb-3 gap-2 max-h-[200px]! overflow-auto rounded-md border border-destructive/30 bg-destructive/5 px-3 py-2 text-xs text-destructive">
+            <TriangleAlert className="mt-0.5 size-3.5 shrink-0" />
+            <span>{formatErrorMessage(state.lastError)}</span>
+          </div>
+        )}
+
+        <div className="flex items-center gap-2 pb-2">
+          {running ? (
+            <Button
+              size="sm"
+              variant="destructive"
+              className={"bg-red-500 hover:bg-red-500/80 text-white "}
+              onClick={() => control("stop")}
+              disabled={busy !== null}
+            >
+              {busy === "stop" ? (
+                <Loader2 className="size-4 animate-spin text-white" />
+              ) : (
+                <Square className="size-4 text-white!" />
+              )}
+              Stop
+            </Button>
+          ) : (
+            <Button
+              size="sm"
+              onClick={() => control("start")}
+              disabled={busy !== null || isLoading || !dockerReady}
+              className="
+    inline-flex items-center gap-2 px-3 py-1.5 text-sm font-medium
+    bg-[#F97316] hover:bg-[#EA6C0A]
+    text-white border-none
+    rounded-md transition-all duration-200
+    disabled:opacity-70 disabled:cursor-not-allowed
+  "
+            >
+              {busy === "start" ? (
+                <Loader2 className="size-4 animate-spin text-white" />
+              ) : (
+                <img
+                  src="/icons/firecrawl2.png"
+                  alt="Firecrawl"
+                  className="size-4 object-contain"
+                />
+              )}
+              Launch Web Crawler
+            </Button>
+          )}
+
+          {/* {dockerReady && !running && (
+            <span className="flex items-center gap-1 text-xs text-cyan-700  bg-white px-3 p-1 rounded-md">
+              <img
+                src={"/docker.png"}
+                alt="docker"
+                width={20}
+                height={20}
+              />
+              Docker ready
+            </span>
+          )} */}
+        </div>
+      </AlertDescription>
+    </Alert>
+  );
+}