larkup 0.1.14 → 0.1.21

package/components/data/scrape-panel.tsx

@@ -1,12 +1,11 @@
-"use client";
+'use client';
 
-import { useMemo, useState, useEffect, useRef } from "react";
-import { toast } from "sonner";
-import { formatErrorMessage } from "@/lib/error-formatter";
+import { useMemo, useState, useEffect, useRef } from 'react';
+import { toast } from 'sonner';
+import { formatErrorMessage } from '@/lib/error-formatter';
 import {
   Globe,
   Loader2,
-  Plus,
   Rocket,
   Search,
   ChevronDown,
@@ -14,32 +13,32 @@ import {
   Info,
   Clock,
   X,
-} from "lucide-react";
-import type { CrawlScope, SearchResultItem } from "@larkup/core/types";
-import { Button } from "@/components/ui/button";
-import { Input } from "@/components/ui/input";
-import { Label } from "@/components/ui/label";
-import { Checkbox } from "@/components/ui/checkbox";
-import { Badge } from "@/components/ui/badge";
-
-import { Switch } from "@/components/ui/switch";
+  Trash2,
+} from 'lucide-react';
+import { useScrapeStore, type SearchState } from '@/store/scrape-store';
+import type { CrawlScope, SearchResultItem } from '@larkup/core/types';
+import { Button } from '@/components/ui/button';
+import { Input } from '@/components/ui/input';
+import { Label } from '@/components/ui/label';
+import { Checkbox } from '@/components/ui/checkbox';
+import { Badge } from '@/components/ui/badge';
+
+import { Switch } from '@/components/ui/switch';
 import {
   Select,
   SelectContent,
   SelectItem,
   SelectTrigger,
   SelectValue,
-} from "@/components/ui/select";
-import {
-  Tooltip,
-  TooltipContent,
-  TooltipProvider,
-  TooltipTrigger,
-} from "@/components/ui/tooltip";
-import { GenericAlert } from "@/components/alerts/generic-alert";
-import { cn } from "@/lib/utils";
+} from '@/components/ui/select';
+import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip';
+import { GenericAlert } from '@/components/alerts/generic-alert';
+import { cn } from '@/lib/utils';
+import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs';
+import useSWR from 'swr';
+import type { RagConfig } from '@larkup/core/types';
 
-import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
+const fetcher = (url: string) => fetch(url).then((r) => r.json() as Promise<{ config: RagConfig }>);
 
 function domainOf(url: string) {
   try {
@@ -70,61 +69,58 @@ function estimateEtlDuration(
   scope: CrawlScope,
   pageLimit: number,
 ): { totalPages: number; estimatedSeconds: number } {
-  // ~3s per page scrape, ~5s per domain page (crawl overhead)
-  const pagesPerUrl = scope === "domain" ? pageLimit : 1;
+  const pagesPerUrl = scope === 'domain' ? pageLimit : 1;
   const totalPages = urlCount * pagesPerUrl;
-  const perPageSeconds = scope === "domain" ? 5 : 3;
+  const perPageSeconds = scope === 'domain' ? 5 : 3;
   const estimatedSeconds = totalPages * perPageSeconds;
   return { totalPages, estimatedSeconds };
 }
 
-interface SearchState {
-  results: SearchResultItem[];
-  totalResults: number;
-  totalResultsIsEstimate: boolean;
-  currentPage: number;
-  totalPages: number;
-  hasMore: boolean;
-  query: string;
-  searchProvider: "firecrawl" | "serper";
-}
-
 export function ScrapePanel({
   disabled,
   onStarted,
+  crawlerControl,
 }: {
-  disabled: boolean;
+  disabled?: boolean;
   onStarted: () => void;
+  crawlerControl?: React.ReactNode;
 }) {
-  const [query, setQuery] = useState("");
+  const {
+    query,
+    setQuery,
+    searchState,
+    setSearchState,
+    selected,
+    setSelected,
+    scope,
+    setScope,
+    pageLimit,
+    setPageLimit,
+    searchLimit,
+    setSearchLimit,
+    showAdvanced,
+    setShowAdvanced,
+    serperTotalForQuery,
+    setSerperTotalForQuery,
+    flush: flushSearch,
+  } = useScrapeStore();
+
   const [searching, setSearching] = useState(false);
   const [loadingMore, setLoadingMore] = useState(false);
   const [gatheringAll, setGatheringAll] = useState(false);
   const [gatherProgress, setGatherProgress] = useState(0);
-  const [searchState, setSearchState] = useState<SearchState | null>(null);
-  const [selected, setSelected] = useState<Record<string, boolean>>({});
-  const [scope, setScope] = useState<CrawlScope>("domain");
-  const [pageLimit, setPageLimit] = useState(25);
-  const [manualUrl, setManualUrl] = useState("");
+  const [manualUrl, setManualUrl] = useState('');
   const [starting, setStarting] = useState(false);
   /** When true: only the exact custom URLs are scraped — no deep crawl/pagination */
   const [specificUrls, setSpecificUrls] = useState(false);
-  const [serperConfigured, setSerperConfigured] = useState<boolean | null>(
-    null,
-  );
-  const [firecrawlConfigured, setFirecrawlConfigured] = useState<
-    boolean | null
-  >(null);
+  const [serperConfigured, setSerperConfigured] = useState<boolean | null>(null);
+  const [firecrawlConfigured, setFirecrawlConfigured] = useState<boolean | null>(null);
   const [confirmOpen, setConfirmOpen] = useState(false);
+  const { data: configData } = useSWR('/api/config', fetcher);
+  const activeProvider = configData?.config?.webSearchProvider || 'tavily';
   const [fetchingCount, setFetchingCount] = useState(false);
-  const [serperTotalForQuery, setSerperTotalForQuery] = useState<{
-    query: string;
-    total: number;
-    totalPages: number;
-  } | null>(null);
   const [cachedQueries, setCachedQueries] = useState<string[]>([]);
   const [showDropdown, setShowDropdown] = useState(false);
-  const [searchLimit, setSearchLimit] = useState(15);
   const inputRef = useRef<HTMLInputElement>(null);
   const dropdownRef = useRef<HTMLDivElement>(null);
 
@@ -139,13 +135,13 @@ export function ScrapePanel({
         setShowDropdown(false);
       }
     }
-    document.addEventListener("mousedown", handleClickOutside);
-    return () => document.removeEventListener("mousedown", handleClickOutside);
+    document.addEventListener('mousedown', handleClickOutside);
+    return () => document.removeEventListener('mousedown', handleClickOutside);
   }, []);
 
   useEffect(() => {
     try {
-      const saved = localStorage.getItem("scrape_recent_queries");
+      const saved = localStorage.getItem('scrape_recent_queries');
       if (saved) setCachedQueries(JSON.parse(saved));
     } catch {
       // ignore
@@ -156,12 +152,9 @@ export function ScrapePanel({
     const trimmed = q.trim();
     if (!trimmed) return;
     setCachedQueries((prev) => {
-      const newCache = [
-        trimmed,
-        ...prev.filter((item) => item !== trimmed),
-      ].slice(0, 10);
+      const newCache = [trimmed, ...prev.filter((item) => item !== trimmed)].slice(0, 10);
       try {
-        localStorage.setItem("scrape_recent_queries", JSON.stringify(newCache));
+        localStorage.setItem('scrape_recent_queries', JSON.stringify(newCache));
       } catch {
         // ignore
       }
@@ -174,7 +167,7 @@ export function ScrapePanel({
     setCachedQueries((prev) => {
       const newCache = prev.filter((item) => item !== q);
       try {
-        localStorage.setItem("scrape_recent_queries", JSON.stringify(newCache));
+        localStorage.setItem('scrape_recent_queries', JSON.stringify(newCache));
       } catch {
         // ignore
       }
@@ -182,35 +175,26 @@ export function ScrapePanel({
     });
   }
 
-  // Check providers on mount
   useEffect(() => {
-    // Check Serper
-    fetch("/api/search/google")
-      .then((r) => r.json())
-      .then((d) => setSerperConfigured(d.configured ?? false))
-      .catch(() => setSerperConfigured(false));
     // Check Firecrawl
-    fetch("/api/search")
+    fetch('/api/search')
       .then((r) => r.json())
       .then((d) => setFirecrawlConfigured(d.configured ?? false))
       .catch(() => setFirecrawlConfigured(false));
   }, []);
 
-  const selectedUrls = useMemo(
-    () => Object.keys(selected).filter((u) => selected[u]),
-    [selected],
-  );
+  const selectedUrls = useMemo(() => Object.keys(selected).filter((u) => selected[u]), [selected]);
 
   /** Search using Firecrawl (preferred — no Serper credits used for search). */
   async function searchFirecrawl(q: string, isMulti: boolean) {
     try {
-      const res = await fetch("/api/search", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
+      const res = await fetch('/api/search', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({ query: q, limit: searchLimit }),
       });
       const data = await res.json();
-      if (!res.ok) throw new Error(data.error ?? "Search failed");
+      if (!res.ok) throw new Error(data.error ?? 'Search failed');
       const newItems = (data.results as SearchResultItem[]) ?? [];
 
       setSearchState((prev) => {
@@ -219,55 +203,69 @@ export function ScrapePanel({
         const fresh = newItems.filter((r) => !existingUrls.has(r.url));
         return {
           results: [...existingResults, ...fresh],
-          totalResults: isMulti
-            ? (prev?.totalResults ?? 0) + fresh.length
-            : fresh.length,
+          totalResults: isMulti ? (prev?.totalResults ?? 0) + fresh.length : fresh.length,
           totalResultsIsEstimate: false,
           currentPage: 1,
           totalPages: 1,
           hasMore: false,
-          query: isMulti ? "Multiple queries" : q,
-          searchProvider: "firecrawl",
+          query: isMulti ? 'Multiple queries' : q,
+          searchProvider: 'firecrawl',
         };
       });
 
       const newSelected = Object.fromEntries(newItems.map((r) => [r.url, true]));
       setSelected((prev) => ({ ...prev, ...newSelected }));
 
-      if (newItems.length === 0 && !isMulti)
-        toast.message("No results — try different keywords.");
+      if (newItems.length === 0 && !isMulti) toast.message('No results. Try different keywords.');
 
-      // If Serper is also configured, fetch the total count in background
-      if (serperConfigured && !isMulti) {
-        fetchSerperTotalCount(q);
+      if (activeProvider && !isMulti) {
+        fetchProviderTotalCount(q, activeProvider);
       }
     } catch (err) {
       toast.error(formatErrorMessage(err));
     }
   }
 
-  /** Search using Serper (fallback if Firecrawl not available). */
-  async function searchSerper(
+  /** Search using generic provider. */
+  async function searchGeneric(
     q: string,
+    provider: string,
     page: number,
     isMulti: boolean,
     appendPagination = false,
   ) {
-    if (!appendPagination) {
-      // It's the initial search, searching flag is handled in runSearch
-    } else {
+    if (appendPagination) {
       setLoadingMore(true);
     }
 
+    const endpoint =
+      provider === 'serper' || provider === 'google'
+        ? '/api/search/google'
+        : provider === 'brave'
+        ? '/api/search/brave'
+        : provider === 'bing'
+        ? '/api/search/bing'
+        : provider === 'tavily'
+        ? '/api/search/tavily'
+        : provider === 'exa'
+        ? '/api/search/exa'
+        : null;
+
+    if (!endpoint) {
+      toast.error('Invalid search provider');
+      if (appendPagination) setLoadingMore(false);
+      return;
+    }
+
     try {
-      const res = await fetch("/api/search/google", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
+      const res = await fetch(endpoint, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({ query: q, page }),
       });
       const data = await res.json();
 
-      if (!res.ok) throw new Error(data.error ?? "Search failed");
+      if (!res.ok) throw new Error(data.error ?? 'Search failed');
 
       const newItems: SearchResultItem[] = (data.items ?? []).map(
         (item: { url: string; title: string; description?: string }) => ({
@@ -289,7 +287,7 @@ export function ScrapePanel({
           totalPages: data.totalPages ?? 1,
           hasMore: data.hasMore ?? false,
           query: data.query ?? q,
-          searchProvider: "serper",
+          searchProvider: provider as any,
         };
       });
 
@@ -297,7 +295,7 @@ export function ScrapePanel({
       setSelected((prev) => ({ ...prev, ...newSelected }));
 
       if (!isMulti) {
-        // Update Serper total
+        // Update total
         setSerperTotalForQuery({
           query: q,
           total: data.totalResults ?? 0,
@@ -305,23 +303,39 @@ export function ScrapePanel({
         });
       }
 
-      if (page === 1 && (data.totalResults ?? 0) === 0 && !isMulti) {
-        toast.message("No results — try different keywords.");
+      if (page === 1 && newItems.length === 0 && !isMulti) {
+        toast.message('No results. Try different keywords.');
       }
     } catch (err) {
+      if (page === 1) throw err;
       toast.error(formatErrorMessage(err));
     } finally {
       if (appendPagination) setLoadingMore(false);
     }
   }
 
-  /** Fetch total count from Serper without using it for results (saves on Firecrawl credits) */
-  async function fetchSerperTotalCount(q: string) {
+  /** Fetch total count from provider without using it for results (saves on Firecrawl credits) */
+  async function fetchProviderTotalCount(q: string, provider: string) {
+    const endpoint =
+      provider === 'serper' || provider === 'google'
+        ? '/api/search/google'
+        : provider === 'brave'
+        ? '/api/search/brave'
+        : provider === 'bing'
+        ? '/api/search/bing'
+        : provider === 'tavily'
+        ? '/api/search/tavily'
+        : provider === 'exa'
+        ? '/api/search/exa'
+        : null;
+
+    if (!endpoint) return;
+
     setFetchingCount(true);
     try {
-      const res = await fetch("/api/search/google", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
+      const res = await fetch(endpoint, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({ query: q, page: 1 }),
       });
       const data = await res.json();
@@ -342,16 +356,15 @@ export function ScrapePanel({
   async function runSearch() {
     setShowDropdown(false);
     if (!query.trim()) {
-      toast.error("Enter one or more keywords to search.");
+      toast.error('Enter one or more keywords to search.');
       return;
     }
 
-    // We do NOT clear selected URLs here so users can keep them across searches
     setSerperTotalForQuery(null);
     saveQuery(query);
 
     const queries = query
-      .split(",")
+      .split(',')
       .map((q) => q.trim())
       .filter(Boolean);
     if (queries.length === 0) return;
@@ -360,22 +373,17 @@ export function ScrapePanel({
     // Removed setSearchState(null) so we append instead of clear
 
     try {
-      if (firecrawlConfigured) {
-        await Promise.all(
-          queries.map((q) => searchFirecrawl(q, queries.length > 1)),
-        );
-      } else if (serperConfigured) {
-        await Promise.all(
-          queries.map((q) => searchSerper(q, 1, queries.length > 1, false)),
-        );
+      if (activeProvider === 'local' || !activeProvider) {
+        await Promise.all(queries.map((q) => searchFirecrawl(q, queries.length > 1)));
       } else {
-        toast.error(
-          formatErrorMessage(
-            new Error(
-              "No search provider available. Configure Firecrawl or add SERPER_API_KEY.",
-            ),
-          ),
-        );
+        try {
+          await Promise.all(
+            queries.map((q) => searchGeneric(q, activeProvider, 1, queries.length > 1, false)),
+          );
+        } catch (err) {
+          toast.error(`${activeProvider} search failed. Falling back to local crawler...`);
+          await Promise.all(queries.map((q) => searchFirecrawl(q, queries.length > 1)));
+        }
       }
     } finally {
       setSearching(false);
@@ -384,9 +392,10 @@ export function ScrapePanel({
 
   async function loadNextPage() {
     if (!searchState?.hasMore || loadingMore) return;
-    if (searchState.searchProvider === "serper") {
-      await searchSerper(
+    if (searchState.searchProvider !== 'firecrawl') {
+      await searchGeneric(
         searchState.query,
+        searchState.searchProvider,
         searchState.currentPage + 1,
         false,
         true,
@@ -394,25 +403,40 @@ export function ScrapePanel({
     }
   }
 
-  /** Automatically paginate through ALL available Serper results. */
+  /** Automatically paginate through ALL available provider results. */
   async function gatherAll() {
-    if (!searchState || searchState.searchProvider !== "serper") return;
+    if (!searchState || searchState.searchProvider === 'firecrawl') return;
     setGatheringAll(true);
     setGatherProgress(searchState.currentPage);
 
     let page = searchState.currentPage + 1;
     const total = searchState.totalPages;
+    const provider = searchState.searchProvider;
+
+    const endpoint =
+      provider === 'serper' || provider === 'google'
+        ? '/api/search/google'
+        : provider === 'brave'
+        ? '/api/search/brave'
+        : provider === 'bing'
+        ? '/api/search/bing'
+        : provider === 'tavily'
+        ? '/api/search/tavily'
+        : provider === 'exa'
+        ? '/api/search/exa'
+        : null;
 
     try {
       while (page <= total) {
         setGatherProgress(page);
-        const res = await fetch("/api/search/google", {
-          method: "POST",
-          headers: { "Content-Type": "application/json" },
+        if (!endpoint) break;
+        const res = await fetch(endpoint, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify({ query: query.trim(), page }),
         });
         const data = await res.json();
-        if (!res.ok) throw new Error(data.error ?? "Search failed");
+        if (!res.ok) throw new Error(data.error ?? 'Search failed');
 
         const newItems: SearchResultItem[] = (data.items ?? []).map(
           (item: { url: string; title: string; description?: string }) => ({
@@ -429,8 +453,7 @@ export function ScrapePanel({
           return {
             ...prev,
             results: [...prev.results, ...fresh],
-            totalResultsIsEstimate:
-              data.totalResultsIsEstimate ?? prev.totalResultsIsEstimate,
+            totalResultsIsEstimate: data.totalResultsIsEstimate ?? prev.totalResultsIsEstimate,
             currentPage: data.currentPage ?? page,
             totalPages: data.totalPages ?? prev.totalPages,
             hasMore: data.hasMore ?? false,
@@ -444,7 +467,7 @@ export function ScrapePanel({
         page++;
         await new Promise((r) => setTimeout(r, 300));
       }
-      toast.success("All available results gathered!");
+      toast.success('All available results gathered!');
     } catch (err) {
       toast.error(formatErrorMessage(err));
     } finally {
@@ -460,7 +483,7 @@ export function ScrapePanel({
       .filter((u) => /^https?:\/\//i.test(u));
 
     if (urls.length === 0) {
-      toast.error("Enter at least one full URL starting with http(s)://");
+      toast.error('Enter at least one full URL starting with http(s)://');
       return;
     }
 
@@ -487,14 +510,14 @@ export function ScrapePanel({
             currentPage: 1,
             totalPages: 1,
             hasMore: false,
-            query: "",
-            searchProvider: "firecrawl",
+            query: '',
+            searchProvider: 'firecrawl',
           };
     });
 
     const newSelected = Object.fromEntries(urls.map((u) => [u, true]));
     setSelected((prev) => ({ ...prev, ...newSelected }));
-    setManualUrl("");
+    setManualUrl('');
   }
 
   const handleFileUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
@@ -504,17 +527,17 @@ export function ScrapePanel({
     reader.onload = (e) => {
       const text = e.target?.result as string;
       if (text) {
-        setManualUrl((prev) => prev + (prev ? "\n" : "") + text);
+        setManualUrl((prev) => prev + (prev ? '\n' : '') + text);
       }
     };
     reader.readAsText(file);
-    e.target.value = ""; // reset input
+    e.target.value = ''; // reset input
   };
 
   /** Open confirm modal before starting ETL */
   function handleStartClick() {
     if (selectedUrls.length === 0) {
-      toast.error("Select at least one URL to scrape.");
+      toast.error('Select at least one URL to scrape.');
       return;
     }
     setConfirmOpen(true);
@@ -523,12 +546,11 @@ export function ScrapePanel({
   async function startJob() {
     setStarting(true);
     setConfirmOpen(false);
-    // When "Specific URLs" is on, force page-level scrape (no crawl)
-    const effectiveScope: CrawlScope = specificUrls ? "page" : scope;
+    const effectiveScope: CrawlScope = specificUrls ? 'page' : scope;
     try {
-      const res = await fetch("/api/jobs", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
+      const res = await fetch('/api/jobs', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({
           keywords: query || selectedUrls[0],
           pageLimit: specificUrls ? 1 : pageLimit,
@@ -536,7 +558,7 @@ export function ScrapePanel({
         }),
       });
       const data = await res.json();
-      if (!res.ok) throw new Error(data.error ?? "Could not start job");
+      if (!res.ok) throw new Error(data.error ?? 'Could not start job');
 
       const { totalPages, estimatedSeconds } = estimateEtlDuration(
         selectedUrls.length,
@@ -544,10 +566,14 @@ export function ScrapePanel({
         specificUrls ? 1 : pageLimit,
       );
 
-      toast.success("ETL job started — running in background", {
+      toast.success('Scraping started, running in background', {
         description: specificUrls
-          ? `Scraping ${selectedUrls.length} specific URL(s) only — no deep crawl. You can navigate away — progress is tracked automatically.`
-          : `${selectedUrls.length} ${effectiveScope === "domain" ? "domain(s)" : "page(s)"} · ~${totalPages} pages · ETA ${formatDuration(estimatedSeconds)}. You can navigate away — progress is tracked automatically.`,
+          ? `Scraping ${selectedUrls.length} specific URL(s) only, no deep crawl. You can navigate away. Progress is tracked automatically.`
+          : `${selectedUrls.length} ${
+              effectiveScope === 'domain' ? 'domain(s)' : 'page(s)'
+            } · ~${totalPages} pages · ETA ${formatDuration(
+              estimatedSeconds,
+            )}. You can navigate away. Progress is tracked automatically.`,
         duration: 8000,
       });
 
@@ -562,12 +588,10 @@ export function ScrapePanel({
 
   const results = searchState?.results ?? [];
   const gatherAllAvailable =
-    searchState?.searchProvider === "serper" &&
-    searchState.hasMore &&
-    !gatheringAll;
+    searchState?.searchProvider !== 'firecrawl' && searchState?.hasMore && !gatheringAll;
 
   // Estimate for confirmation modal
-  const effectiveScopeForEstimate: CrawlScope = specificUrls ? "page" : scope;
+  const effectiveScopeForEstimate: CrawlScope = specificUrls ? 'page' : scope;
   const estimate = useMemo(
     () =>
       estimateEtlDuration(
@@ -578,7 +602,7 @@ export function ScrapePanel({
     [selectedUrls.length, effectiveScopeForEstimate, specificUrls, pageLimit],
   );
 
-  const [inputMode, setInputMode] = useState<"search" | "url">("search");
+  const [inputMode, setInputMode] = useState<'search' | 'url'>('search');
 
   return (
     <div className="space-y-4">
@@ -586,22 +610,23 @@ export function ScrapePanel({
       <div className="space-y-3">
         {/* Mode toggle + input */}
         <div className="flex items-center gap-2">
+          {crawlerControl}
           <Tabs
             value={inputMode}
-            onValueChange={(v) => setInputMode(v as "search" | "url")}
+            onValueChange={(v) => setInputMode(v as 'search' | 'url')}
             className="shrink-0"
           >
-            <TabsList className="inline-flex h-9 items-center justify-center rounded-lg bg-muted/40 border border-border p-0.5 text-muted-foreground">
+            <TabsList className="inline-flex bg-white/70 h-9 items-center justify-center rounded-lg  border border-border p-0.5 text-muted-foreground">
               <TabsTrigger
                 value="search"
-                className="inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-xs font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm hover:text-foreground h-full"
+                className="inline-flex items-center h-9 justify-center whitespace-nowrap rounded-md px-3 py-1 text-xs font-medium transition-all focus-visible:outline-none data-[state=active]:bg-white data-[state=active]:text-foreground data-[state=active]: hover:text-foreground"
               >
                 <Search className="size-3.5 mr-1.5" />
                 Search
               </TabsTrigger>
               <TabsTrigger
                 value="url"
-                className="inline-flex items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-xs font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]:shadow-sm hover:text-foreground h-full"
+                className="inline-flex h-9 items-center justify-center whitespace-nowrap rounded-md px-3 py-1 text-xs font-medium transition-all focus-visible:outline-none data-[state=active]:bg-background data-[state=active]:text-foreground data-[state=active]: hover:text-foreground"
               >
                 <Globe className="size-3.5 mr-1.5" />
                 Direct URL
@@ -611,30 +636,48 @@ export function ScrapePanel({
 
           {/* Input field */}
           <div className="relative flex-1">
-            {inputMode === "search" ? (
+            {inputMode === 'search' ? (
               <>
+                <Search className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground z-10 pointer-events-none" />
                 <Input
                   ref={inputRef}
                   id="kw"
+                  autoComplete="off"
                   placeholder='Search keywords, e.g. "RAG pipelines"'
                   value={query}
                   disabled={disabled}
+                  className="pl-9 pr-22"
                   onChange={(e) => {
                     setQuery(e.target.value);
                     setShowDropdown(true);
                   }}
                   onFocus={() => setShowDropdown(true)}
                   onKeyDown={(e) => {
-                    if (e.key === "Enter") {
+                    if (e.key === 'Enter') {
                       setShowDropdown(false);
                       runSearch();
                     }
                   }}
                 />
+                {query.length > 0 && (
+                  <Button
+                    size="sm"
+                    variant="secondary"
+                    className="absolute right-1 top-1/2 -translate-y-1/2 h-8"
+                    onClick={() => {
+                      setShowDropdown(false);
+                      runSearch();
+                    }}
+                    disabled={disabled || searching}
+                  >
+                    {searching ? <Loader2 className="size-3.5 animate-spin mr-1" /> : null}
+                    Search
+                  </Button>
+                )}
                 {showDropdown && cachedQueries.length > 0 && (
                   <div
                     ref={dropdownRef}
-                    className="absolute z-10 top-full left-0 mt-1 w-full rounded-md border border-border bg-popover shadow-md overflow-hidden"
+                    className="absolute z-10 top-full left-0 mt-1 w-full rounded-md border border-border bg-popover  overflow-hidden"
                   >
                     <div className="py-1">
                       {cachedQueries.map((cq) => (
@@ -663,18 +706,33 @@ export function ScrapePanel({
                 )}
               </>
             ) : (
-              <Input
-                placeholder="Paste URLs (comma or space separated)"
-                value={manualUrl}
-                disabled={disabled}
-                onChange={(e) => setManualUrl(e.target.value)}
-                onKeyDown={(e) => e.key === "Enter" && addManual()}
-              />
+              <>
+                <Globe className="absolute left-3 top-1/2 -translate-y-1/2 size-4 text-muted-foreground z-10 pointer-events-none" />
+                <Input
+                  placeholder="Paste URLs (comma or space separated)"
+                  value={manualUrl}
+                  disabled={disabled}
+                  className="pl-9 pr-16"
+                  onChange={(e) => setManualUrl(e.target.value)}
+                  onKeyDown={(e) => e.key === 'Enter' && addManual()}
+                />
+                {manualUrl.length > 0 && (
+                  <Button
+                    size="sm"
+                    variant="secondary"
+                    className="absolute right-1 top-1/2 -translate-y-1/2 h-8"
+                    onClick={addManual}
+                    disabled={disabled}
+                  >
+                    Add
+                  </Button>
+                )}
+              </>
             )}
           </div>
 
           {/* Search limit (only in search mode) */}
-          {inputMode === "search" && (
+          {inputMode === 'search' && (
             <Input
               type="number"
               min={1}
@@ -687,181 +745,168 @@ export function ScrapePanel({
             />
           )}
 
-          {/* Action button */}
-          {inputMode === "search" ? (
-            <Button
-              onClick={() => {
-                setShowDropdown(false);
-                runSearch();
-              }}
-              disabled={disabled || searching}
-              className="shrink-0"
-            >
-              {searching ? (
-                <Loader2 className="size-4 animate-spin" />
-              ) : (
-                <Search className="size-4" />
-              )}
-              Search
-            </Button>
-          ) : (
-            <Button
-              variant="outline"
-              onClick={addManual}
-              disabled={disabled}
-              className="shrink-0"
-            >
-              <Plus className="size-4" />
-              Add
-            </Button>
-          )}
+          {/* Start Scraping Button (always visible) */}
+          <Button
+            onClick={handleStartClick}
+            disabled={disabled || starting || selectedUrls.length === 0}
+            className="shrink-0"
+          >
+            {starting ? (
+              <Loader2 className="size-4 animate-spin mr-1" />
+            ) : (
+              <Rocket className="size-4 mr-2" />
+            )}
+            Start Scraping
+            {selectedUrls.length > 0 && (
+              <Badge variant="secondary" className="ml-1">
+                {selectedUrls.length}
+              </Badge>
+            )}
+          </Button>
         </div>
       </div>
 
       {/* No provider configured hint */}
-      {firecrawlConfigured === false && serperConfigured === false && (
+      {firecrawlConfigured === false && !activeProvider && (
         <div className="flex items-start gap-2 rounded-md border border-border bg-muted/30 px-3 py-2.5 text-xs text-muted-foreground">
           <Info className="mt-0.5 size-3.5 shrink-0" />
           <span>
-            <strong className="text-foreground">
-              No search provider configured.
-            </strong>{" "}
-            Set up Firecrawl (recommended) for search and scraping, or add{" "}
-            <code className="rounded bg-muted px-1 font-mono">
-              SERPER_API_KEY
-            </code>{" "}
-            to your{" "}
-            <code className="rounded bg-muted px-1 font-mono">.env</code> for
-            Google search.
+            <strong className="text-foreground">No search provider configured.</strong> Set up
+            Firecrawl (recommended) for search and scraping, or configure a Web Search provider in
+            Settings.
           </span>
         </div>
       )}
 
-      {/* Scope + launch — compact single row */}
-      <div className="flex items-center gap-3 flex-wrap">
-        <Select
-          defaultValue={scope}
-          value={scope}
-          onValueChange={(v) => setScope(v as CrawlScope)}
-          disabled={specificUrls || disabled}
+      {/* Minimal Action Row */}
+      <div className="flex items-center justify-between mt-2">
+        <button
+          type="button"
+          onClick={() => setShowAdvanced(!showAdvanced)}
+          className="flex items-center gap-1.5 text-xs text-muted-foreground hover:text-foreground transition-colors"
         >
-          <SelectTrigger className="w-auto min-w-[180px] flex-1 bg-white h-9 text-xs">
-            <SelectValue />
-          </SelectTrigger>
-          <SelectContent>
-            <SelectItem value="page">Scrape pages only</SelectItem>
-            <SelectItem value="domain">Crawl whole domain</SelectItem>
-          </SelectContent>
-        </Select>
-
-        {scope === "domain" && !specificUrls && (
-          <div className="flex items-center gap-1.5">
-            <Label
-              htmlFor="limit"
-              className="text-xs text-muted-foreground whitespace-nowrap"
-            >
-              Max pages
-            </Label>
-            <Input
-              id="limit"
-              type="number"
-              min={1}
-              max={500}
-              value={pageLimit}
-              disabled={specificUrls || disabled}
-              onChange={(e) => setPageLimit(Number(e.target.value) || 1)}
-              className="w-20 h-9 tabular-nums"
-            />
-          </div>
+          {showAdvanced ? (
+            <ChevronDown className="size-3.5" />
+          ) : (
+            <ChevronsDown className="size-3.5" />
+          )}
+          {showAdvanced ? 'Hide advanced settings' : 'Advanced settings'}
+        </button>
+
+        {/* Estimation inline */}
+        {selectedUrls.length > 0 && (
+          <span className="text-xs text-muted-foreground flex items-center gap-1.5 ml-auto">
+            <Clock className="size-3" />
+            {specificUrls
+              ? `${selectedUrls.length} URL${selectedUrls.length !== 1 ? 's' : ''}`
+              : `~${estimate.totalPages.toLocaleString()} pages · ${formatDuration(
+                  estimate.estimatedSeconds,
+                )}`}
+          </span>
         )}
+      </div>
 
-        <div className="flex items-center gap-1.5">
-          <Switch
-            id="specific-urls-inline"
-            checked={specificUrls}
-            onCheckedChange={setSpecificUrls}
-            disabled={disabled}
-            size="sm"
-          />
-          <Label
-            htmlFor="specific-urls-inline"
-            className="text-xs font-medium cursor-pointer flex items-center gap-1"
-          >
-            Exact URLs
-            <TooltipProvider delay={0}>
-              <Tooltip>
-                <TooltipTrigger
-                  type="button"
-                  className="text-muted-foreground hover:text-foreground"
-                  onClick={(e) => e.preventDefault()}
-                >
-                  <Info className="size-3.5" />
-                </TooltipTrigger>
-                <TooltipContent>
-                  <p className="max-w-[200px] text-center">
-                    Scrape exact URLs only — no deep crawl or pagination.
-                  </p>
-                </TooltipContent>
-              </Tooltip>
-            </TooltipProvider>
-          </Label>
-        </div>
+      {/* Advanced Settings Collapsible */}
+      {showAdvanced && (
+        <div className="flex flex-wrap items-center gap-4 rounded-lg border border-border bg-muted/30 p-3 mt-2 animate-in slide-in-from-top-2 fade-in duration-200">
+          <div className="flex items-center gap-2">
+            <Label className="text-xs text-muted-foreground">Scope</Label>
+            <Select
+              defaultValue={scope}
+              value={scope}
+              onValueChange={(v) => setScope(v as CrawlScope)}
+              disabled={specificUrls || disabled}
+            >
+              <SelectTrigger className="w-40 bg-white h-8 text-xs">
+                <SelectValue />
+              </SelectTrigger>
+              <SelectContent>
+                <SelectItem value="page">Scrape pages only</SelectItem>
+                <SelectItem value="domain">Crawl whole domain</SelectItem>
+              </SelectContent>
+            </Select>
+          </div>
 
-        <div className="flex items-center gap-2 ml-auto">
-          {/* ETL estimation inline */}
-          {selectedUrls.length > 0 && (
-            <span className="text-xs text-muted-foreground flex items-center gap-1.5">
-              <Clock className="size-3" />
-              {specificUrls
-                ? `${selectedUrls.length} URL${selectedUrls.length !== 1 ? "s" : ""}`
-                : `~${estimate.totalPages.toLocaleString()} pages · ${formatDuration(estimate.estimatedSeconds)}`}
-            </span>
+          {scope === 'domain' && !specificUrls && (
+            <div className="flex items-center gap-2">
+              <Label htmlFor="limit" className="text-xs text-muted-foreground">
+                Max pages
+              </Label>
+              <Input
+                id="limit"
+                type="number"
+                min={1}
+                max={500}
+                value={pageLimit}
+                disabled={specificUrls || disabled}
+                onChange={(e) => setPageLimit(Number(e.target.value) || 1)}
+                className="w-20 h-8 tabular-nums text-xs"
+              />
+            </div>
           )}
 
-          <Button
-            onClick={handleStartClick}
-            disabled={disabled || starting || selectedUrls.length === 0}
-            className="shrink-0"
-          >
-            {starting ? (
-              <Loader2 className="size-4 animate-spin" />
-            ) : (
-              <Rocket className="size-4" />
-            )}
-            Start ETL
-            {selectedUrls.length > 0 && (
-              <Badge variant="secondary" className="ml-1">
-                {selectedUrls.length}
-              </Badge>
-            )}
-          </Button>
+          <div className="flex items-center gap-2">
+            <Switch
+              id="specific-urls-inline"
+              checked={specificUrls}
+              onCheckedChange={setSpecificUrls}
+              disabled={disabled}
+              size="sm"
+            />
+            <Label
+              htmlFor="specific-urls-inline"
+              className="text-xs font-medium cursor-pointer flex items-center gap-1"
+            >
+              Exact URLs only
+              <TooltipProvider delay={0}>
+                <Tooltip>
+                  <TooltipTrigger
+                    type="button"
+                    className="text-muted-foreground hover:text-foreground"
+                    onClick={(e) => e.preventDefault()}
+                  >
+                    <Info className="size-3" />
+                  </TooltipTrigger>
+                  <TooltipContent>
+                    <p className="max-w-50 text-center">
+                      Scrape exact URLs only — no deep crawl or pagination.
+                    </p>
+                  </TooltipContent>
+                </Tooltip>
+              </TooltipProvider>
+            </Label>
+          </div>
         </div>
-      </div>
+      )}
 
       {/* Results list */}
       {results.length > 0 && (
         <div className="rounded-lg border border-border">
           <div className="flex items-center justify-between border-b border-border px-3 py-2">
             <span className="text-xs font-medium text-muted-foreground">
-              {results.length} source{results.length === 1 ? "" : "s"} ·{" "}
-              {selectedUrls.length} selected
+              {results.length} source{results.length === 1 ? '' : 's'} · {selectedUrls.length}{' '}
+              selected
             </span>
-            <button
-              type="button"
-              className="text-xs text-primary hover:underline"
-              onClick={() => {
-                const all = selectedUrls.length !== results.length;
-                setSelected(
-                  all
-                    ? Object.fromEntries(results.map((r) => [r.url, true]))
-                    : {},
-                );
-              }}
-            >
-              {selectedUrls.length === results.length
-                ? "Clear all"
-                : "Select all"}
-            </button>
+            <div className="flex items-center gap-4">
+              <button
+                type="button"
+                className="text-xs text-primary hover:underline"
+                onClick={() => {
+                  const all = selectedUrls.length !== results.length;
+                  setSelected(all ? Object.fromEntries(results.map((r) => [r.url, true])) : {});
+                }}
+              >
+                {selectedUrls.length === results.length ? 'Deselect all' : 'Select all'}
+              </button>
+              <button
+                type="button"
+                className="flex items-center gap-1 text-xs text-destructive hover:underline"
+                onClick={flushSearch}
+              >
+                <Trash2 className="size-3.5" />
+                Clear
+              </button>
+            </div>
           </div>
           <ul className="max-h-72 divide-y divide-border overflow-y-auto">
             {results.map((r) => {
@@ -870,21 +915,18 @@ export function ScrapePanel({
                 <li key={r.url}>
                   <label
                     className={cn(
-                      "flex cursor-pointer items-start gap-3 px-3 py-2.5 transition-colors hover:bg-muted/50",
-                      checked && "bg-accent/50",
+                      'flex cursor-pointer items-start gap-3 px-3 py-2.5 transition-colors hover:bg-muted/50',
+                      'bg-white',
+                      checked && 'bg-accent/50',
                     )}
                   >
                     <Checkbox
                       checked={checked}
-                      onCheckedChange={(v) =>
-                        setSelected((prev) => ({ ...prev, [r.url]: !!v }))
-                      }
+                      onCheckedChange={(v) => setSelected((prev) => ({ ...prev, [r.url]: !!v }))}
                       className="mt-0.5"
                     />
                     <span className="min-w-0 flex-1">
-                      <span className="block truncate text-sm font-medium">
-                        {r.title}
-                      </span>
+                      <span className="block truncate text-sm font-medium">{r.title}</span>
                       <span className="flex items-center gap-1 truncate text-xs text-muted-foreground">
                         <Globe className="size-3 shrink-0" />
                         {domainOf(r.url)}
@@ -902,7 +944,7 @@ export function ScrapePanel({
           </ul>
 
           {/* Pagination footer */}
-          {searchState?.searchProvider === "serper" &&
+          {searchState?.searchProvider === 'serper' &&
             (searchState.hasMore || loadingMore || gatheringAll) && (
               <div className="flex items-center justify-between border-t border-border px-3 py-2">
                 <span className="text-xs text-muted-foreground">
@@ -919,9 +961,7 @@ export function ScrapePanel({
                       onClick={gatherAll}
                     >
                       <ChevronsDown className="size-3.5" />
-                      Gather all (
-                      {searchState.totalPages - searchState.currentPage} more
-                      pages)
+                      Gather all ({searchState.totalPages - searchState.currentPage} more pages)
                     </Button>
                   )}
                   <Button
@@ -929,9 +969,7 @@ export function ScrapePanel({
                     variant="outline"
                     className="h-7 gap-1 text-xs"
                     onClick={loadNextPage}
-                    disabled={
-                      !searchState?.hasMore || loadingMore || gatheringAll
-                    }
+                    disabled={!searchState?.hasMore || loadingMore || gatheringAll}
                   >
                     {loadingMore ? (
                       <Loader2 className="size-3.5 animate-spin" />
@@ -950,13 +988,21 @@ export function ScrapePanel({
       <GenericAlert
         open={confirmOpen}
         onOpenChange={setConfirmOpen}
-        title="Start ETL Job?"
+        title="Start Scraping?"
         description={
           specificUrls
-            ? `This will scrape ${selectedUrls.length} specific URL${selectedUrls.length !== 1 ? "s" : ""} exactly as provided — no deep crawl or pagination.\n\nThe job will run in the background. You can navigate away and check progress anytime.`
-            : `This will scrape ${selectedUrls.length} ${scope === "domain" ? "domain(s)" : "page(s)"}${scope === "domain" ? ` with up to ${pageLimit} pages each` : ""}.\n\n• Total pages: ~${estimate.totalPages.toLocaleString()}\n• Estimated time: ${formatDuration(estimate.estimatedSeconds)}\n\nThe job will run in the background. You can navigate away and check progress anytime.`
+            ? `This will scrape ${selectedUrls.length} specific URL${
+                selectedUrls.length !== 1 ? 's' : ''
+              } exactly as provided, no deep crawl or pagination.\n\nThe job will run in the background. You can navigate away and check progress anytime.`
+            : `This will scrape ${selectedUrls.length} ${
+                scope === 'domain' ? 'domain(s)' : 'page(s)'
+              }${
+                scope === 'domain' ? ` with up to ${pageLimit} pages each` : ''
+              }.\n\n• Total pages: ~${estimate.totalPages.toLocaleString()}\n• Estimated time: ${formatDuration(
+                estimate.estimatedSeconds,
+              )}\n\nThe job will run in the background. You can navigate away and check progress anytime.`
         }
-        actionText="Start ETL Job"
+        actionText="Start Scraping"
         onAction={startJob}
       />
     </div>