larkup 0.1.36 → 0.1.38

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # larkup
 
+## 0.1.38
+
+### Patch Changes
+
+- Fix 10 production issues: SSE controller-closed error, cleaner system prompt, smarter KB search, one-shot web search, corpus refresh after scrape, minimal KB loading UI, settings sidebar navigation, force-delete processing media, playlist confirmation, image preview in chat, marketplace install progress toast.
+
+## 0.1.37
+
+### Patch Changes
+
+- c1b12cb: Restore reliable no-Docker local crawler search with a resilient public-search fallback, make current-event chat questions use enabled web search, speed up first-party tool installation, and improve media URL progress handling. Simplify settings and prompt UX, and make failed-crawl URL copying reliable.
+- Updated dependencies [c1b12cb]
+  - @larkup/scraper@0.1.27
+  - @larkup/marketplace@0.1.10
+  - @larkup/tool-video-audio@0.2.3
+  - @larkup/tool-doc-editor@0.2.5
+
 ## 0.1.36
 
 ### Patch Changes

package/app/api/chat/route.ts

@@ -64,28 +64,24 @@ function createChatModel(
 
 const SMART_RETRIEVAL_POLICY = `
 
-CONVERSATION, RETRIEVAL, AND MEDIA POLICY:
-
-TOOL PRIORITY (strict order):
-1. searchKnowledgeBase — ALWAYS try this FIRST when the question may relate to the user's indexed data. This includes any question mentioning "I", "my", "our", "the document", "the file", "the diagram", "the image", "show me", or referencing specific topics the user has likely indexed.
-2. Web search (if enabled) — Use ONLY if searchKnowledgeBase returned zero relevant hits AND the question is factual and benefits from web results.
-3. Other tools (tabular queries, analysis, etc.) — Use when the question specifically requires them.
-4. General knowledge — Answer from your training data ONLY for general knowledge questions clearly unrelated to the user's data.
-5. If no tool yields relevant results AND you cannot answer reliably from general knowledge, respond honestly: "I could not find relevant information in your knowledge base or available sources."
-
-RETRIEVAL RULES:
-- Default to a short, direct answer. One to three sentences is usually enough; use structure only when the task is genuinely complex.
-- Do not search for greetings, casual conversation, writing help, brainstorming, explanations you can answer reliably from general knowledge, or questions unrelated to the user's indexed data.
-- Use searchKnowledgeBase only when the answer depends on the user's indexed/private content, an exact source-specific fact, or a topic that plausibly refers to something the user indexed.
-- Treat questions about the user or their materials as knowledge-base questions: phrases such as "I", "my", "our", "you have", "the database", "the document", "the file", or "the diagram" require searchKnowledgeBase before answering whenever indexed content may contain the answer.
-- Search at most once per user request. Never repeat a search when a prior tool result in this conversation already contains enough evidence. If the results do not answer the question, say so plainly instead of making more searches or guessing.
-
-IMAGE AND MEDIA RULES:
-- NEVER output raw markdown image syntax like ![alt](url). The chat UI blocks these and shows "[Image unavailable]". Instead, always use the presentMedia tool to display images, videos, or audio from indexed content.
-- When search results contain images[].imageUrl (an image extracted from a PDF), use presentMedia with that exact imageUrl. When they contain a mediaAssetId, use presentMedia with the exact assetId. This is the ONLY way indexed images can appear in the conversation.
-- For a follow-up such as "show me that part", "play the clip", "show the image", or "let me hear it", reuse the mediaAssetId and timestamps already present in the earlier search result and call presentMedia directly. Do not search again just to rediscover the same media.
-- Use presentMedia to embed exactly one best video, image, or audio citation when the user asks to see/hear it or when one compact preview materially supports the answer. Do not present every search result.
-- Keep the written answer minimal around a media citation. Do not print internal media reference markers.
+TOOL RULES:
+1. searchKnowledgeBase → Use FIRST when the question is about the user's indexed data (documents, files, images, videos, diagrams, anything they uploaded).
+2. webSearch → Use ONLY after searchKnowledgeBase returned nothing relevant AND the question needs current/public facts.
+3. Other tools → Use when the question specifically requires them.
+4. General knowledge → Answer directly for common knowledge unrelated to the user's data.
+5. If nothing helps, say so honestly.
+
+DO:
+- Keep answers short and direct. 1–3 sentences unless the task is complex.
+- Search at most ONCE per tool per turn. Never repeat the same search.
+- Use presentMedia to show images/videos/audio from indexed content.
+- For follow-ups like "show me that", reuse the mediaAssetId from earlier results — do not search again.
+
+DO NOT:
+- Search for greetings, casual chat, writing help, or general knowledge you already know.
+- Output markdown images like ![alt](url) — the UI blocks them. Always use presentMedia instead.
+- Present every search result. Pick the single best one.
+- Guess or fabricate when results are insufficient.
 `;
 
 function latestUserText(messages: UIMessage[]): string {
@@ -101,8 +97,29 @@ function latestUserText(messages: UIMessage[]): string {
   return '';
 }
 
+/** Only force KB search when the user explicitly refers to their own data.
+ * Matches "my document", "our files", "the uploaded image" etc. but NOT
+ * generic uses like "I want to know", "can you tell me", "I think". */
 function requiresKnowledgeBaseSearch(text: string): boolean {
-  return /\b(my|our|we|i|database|db|document|file|diagram|image|picture|upload(?:ed)?|knowledge base|corpus)\b|\bwhat is my\b/i.test(
+  // Skip generic conversational patterns
+  if (/^(hi|hello|hey|thanks|thank you|ok|sure|yes|no|please|help)\b/i.test(text.trim())) {
+    return false;
+  }
+  // Match possessive references to user data
+  return (
+    /\b(my|our)\s+(document|file|data|image|picture|diagram|video|audio|upload|corpus|knowledge|database|db|pdf|report|spreadsheet|presentation)/i.test(
+      text,
+    ) ||
+    /\b(the|this|that)\s+(document|file|diagram|image|picture|upload|pdf|report)/i.test(text) ||
+    /\b(uploaded?|indexed|scraped|knowledge base|corpus)\b/i.test(text) ||
+    /\bshow\s+me\b/i.test(text)
+  );
+}
+
+/** Questions whose answer is inherently time-sensitive should not rely on the
+ * model's training snapshot when Web Search is available. */
+function requiresCurrentWebSearch(text: string): boolean {
+  return /\b(who won|winner|final score|match result|latest|current|today|yesterday|breaking news|weather|stock price|exchange rate|election result|schedule)\b/i.test(
     text,
   );
 }
@@ -364,10 +381,13 @@ ${fieldLines}`;
   }
 
   const systemPrompt =
-    (config.systemPrompt || DEFAULT_SYSTEM_PROMPT) +
+    DEFAULT_SYSTEM_PROMPT +
+    (config.systemPrompt && config.systemPrompt !== DEFAULT_SYSTEM_PROMPT
+      ? `\n\nUSER INSTRUCTIONS:\n${config.systemPrompt}`
+      : '') +
     SMART_RETRIEVAL_POLICY +
     (config.webSearchEnabled
-      ? '\nWEB SEARCH IS ENABLED: For a public, current, or external factual question, you MUST call webSearch after any required knowledge-base search. Do not tell the user to check the web yourself when this tool is available.\n'
+      ? '\nWeb search is available. Use it for current events or public facts not in the knowledge base.\n'
       : '') +
     tabularContext +
     docContext;
@@ -375,6 +395,10 @@ ${fieldLines}`;
   const forceKnowledgeBaseSearch =
     requiresKnowledgeBaseSearch(latestUserText(messagesToProcess)) &&
     Boolean(tools.searchKnowledgeBase);
+  const forceWebSearch =
+    config.webSearchEnabled === true &&
+    requiresCurrentWebSearch(latestUserText(messagesToProcess)) &&
+    Boolean(tools.webSearch);
 
   // Debug: log payload sizes to console in development
   if (process.env.NODE_ENV === 'development') {
@@ -386,19 +410,24 @@ ${fieldLines}`;
     system: systemPrompt,
     messages: await convertToModelMessages(safeMessages, { tools }),
     maxOutputTokens: 4096,
-    // Five steps covers the only useful multi-step case (retrieve, optionally
-    // present one media item, then answer) without allowing a tool loop to
-    // burn the context window and end with no response.
-    stopWhen: stepCountIs(5),
+    // Three steps: retrieve → optionally present media → answer.
+    stopWhen: stepCountIs(3),
     toolChoice: 'auto',
     prepareStep: ({ stepNumber }) => {
+      // Step 0: force KB search if needed
       if (forceKnowledgeBaseSearch && stepNumber === 0) {
         return { toolChoice: { type: 'tool', toolName: 'searchKnowledgeBase' } };
       }
-      // A retrieval is deliberately one-shot per turn. This is applied at the
-      // runtime level rather than relying only on the model prompt.
-      if (forceKnowledgeBaseSearch) {
-        return { activeTools: Object.keys(tools).filter((name) => name !== 'searchKnowledgeBase') };
+      // Step 1 (or 0 if no KB): force web search if needed
+      if (forceWebSearch && (!forceKnowledgeBaseSearch || stepNumber === 1)) {
+        return { toolChoice: { type: 'tool', toolName: 'webSearch' } };
+      }
+      // After forced tools, remove them so the model cannot loop on them.
+      const usedTools: string[] = [];
+      if (forceKnowledgeBaseSearch) usedTools.push('searchKnowledgeBase');
+      if (forceWebSearch) usedTools.push('webSearch');
+      if (usedTools.length > 0) {
+        return { activeTools: Object.keys(tools).filter((name) => !usedTools.includes(name)) };
       }
       return undefined;
     },

package/app/api/media/process/route.ts

@@ -165,6 +165,12 @@ async function enqueueMediaProcessing(
               const entries = await tool.importMediaUrl(currentAsset.originalUrl, {
                 outputDir: tmpDir,
                 playlistMax: 10, // max items
+                onProgress: ({ percent, message }: { percent?: number; message: string }) =>
+                  reportStage('download', {
+                    status: 'running',
+                    percent,
+                    message,
+                  }),
               });
 
               if (!entries || entries.length === 0) {

package/app/api/media/route.ts

@@ -177,6 +177,7 @@ async function removeMedia(req: Request) {
   const url = new URL(req.url);
   const id = url.searchParams.get('id');
   const ids = url.searchParams.get('ids');
+  const force = url.searchParams.get('force') === 'true';
 
   const storage = createStorageProvider();
 
@@ -184,7 +185,7 @@ async function removeMedia(req: Request) {
     const { getMediaAsset } = await import('@larkup/core/media-store');
     const asset = await getMediaAsset(id);
     if (asset) {
-      if (isMediaProcessing(asset)) {
+      if (!force && isMediaProcessing(asset)) {
         return NextResponse.json(
           { error: 'Wait for the media indexing job to finish before deleting it.' },
           { status: 409 },
@@ -201,7 +202,7 @@ async function removeMedia(req: Request) {
     const idList = ids.split(',');
     const assets = await readMediaAssets();
     const selectedAssets = assets.filter((asset) => idList.includes(asset.id));
-    if (selectedAssets.some(isMediaProcessing)) {
+    if (!force && selectedAssets.some(isMediaProcessing)) {
       return NextResponse.json(
         { error: 'Wait for all selected media indexing jobs to finish before deleting them.' },
         { status: 409 },

package/app/api/media/stream/route.ts

@@ -50,14 +50,29 @@ export async function GET(req: Request) {
 
   let interval: ReturnType<typeof setInterval> | undefined;
   let keepAlive: ReturnType<typeof setInterval> | undefined;
+  let closed = false;
+
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
       let lastPayload = '';
       let checking = false;
       let storageSnapshot = { usedBytes: 0, fileCount: 0 };
       let storageCheckedAt = 0;
+
+      const safeEnqueue = (data: Uint8Array) => {
+        if (closed) return;
+        try {
+          controller.enqueue(data);
+        } catch {
+          // Controller already closed — client disconnected.
+          closed = true;
+          if (interval) clearInterval(interval);
+          if (keepAlive) clearInterval(keepAlive);
+        }
+      };
+
       const publish = async () => {
-        if (checking) return;
+        if (checking || closed) return;
         checking = true;
         try {
           if (Date.now() - storageCheckedAt > 30_000) {
@@ -81,7 +96,7 @@ export async function GET(req: Request) {
               0,
               ...nextSnapshot.assets.map((asset) => asset.processingRevision ?? 0),
             );
-            controller.enqueue(
+            safeEnqueue(
               encoder.encode(`id: ${revision}\nevent: media-update\ndata: ${payload}\n\n`),
             );
           }
@@ -94,9 +109,10 @@ export async function GET(req: Request) {
 
       await publish();
       interval = setInterval(() => void publish(), 1_000);
-      keepAlive = setInterval(() => controller.enqueue(encoder.encode(': keep-alive\n\n')), 15_000);
+      keepAlive = setInterval(() => safeEnqueue(encoder.encode(': keep-alive\n\n')), 15_000);
     },
     cancel() {
+      closed = true;
       if (interval) clearInterval(interval);
       if (keepAlive) clearInterval(keepAlive);
     },

package/app/settings/page.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { useState, useEffect, Suspense } from 'react';
+import { useEffect, Suspense } from 'react';
 import { useSearchParams, useRouter, usePathname } from 'next/navigation';
 import { SettingsLayout, type SettingsSection } from '@/components/settings/settings-layout';
 import { GeneralSection } from '@/components/settings/general-section';
@@ -20,38 +20,29 @@ function SettingsContent() {
   const router = useRouter();
   const pathname = usePathname();
 
-  const getInitialSection = (): SettingsSection => {
+  const resolveSection = (): SettingsSection => {
     if (searchParams.has('ai-models')) return 'models';
     const section = searchParams.get('section') as SettingsSection;
     if (section) return section;
     return 'general';
   };
 
-  const [activeSection, setActiveSection] = useState<SettingsSection>(getInitialSection());
+  // Always derive the active section from the URL — this ensures navigating
+  // away and back never leaves the sidebar stuck on a stale section.
+  const activeSection = resolveSection();
 
   useEffect(() => {
-    const currentSection = searchParams.get('section');
-    const hasAiModels = searchParams.has('ai-models');
-
-    if (hasAiModels) {
+    // Normalize legacy `?ai-models` param to `?section=models`
+    if (searchParams.has('ai-models')) {
       const params = new URLSearchParams(searchParams.toString());
       params.delete('ai-models');
       params.set('section', 'models');
       router.replace(`${pathname}?${params.toString()}`);
-      if (activeSection !== 'models') {
-        setActiveSection('models');
-      }
-    } else if (currentSection && currentSection !== activeSection) {
-      setActiveSection(currentSection as SettingsSection);
     }
   }, [searchParams, pathname, router]);
 
   const handleSectionChange = (newSection: SettingsSection) => {
-    setActiveSection(newSection);
-    const params = new URLSearchParams(searchParams.toString());
-    params.delete('ai-models');
-    params.set('section', newSection);
-    router.replace(`${pathname}?${params.toString()}`);
+    router.replace(`${pathname}?section=${newSection}`, { scroll: false });
   };
 
   return (

package/components/chat/chat-workspace.tsx

@@ -530,12 +530,19 @@ function ChatWorkspaceInner() {
     const config = configData?.config;
     if (!config) return;
 
-    const isTavily = config.webSearchProvider === 'tavily' || !config.webSearchProvider;
-    const hasKey = isTavily ? !!config.tavilyApiKey : !!config.serperApiKey; // Server uses its own or serper, fallback to check serper or just assume server doesn't need one if it's configured. Let's just check tavily if tavily.
+    const provider = config.webSearchProvider || 'tavily';
+    const apiKeys: Record<string, string | undefined> = {
+      tavily: config.tavilyApiKey,
+      serper: config.serperApiKey,
+      google: config.serperApiKey,
+      brave: config.braveApiKey,
+      bing: config.bingApiKey,
+      exa: config.exaApiKey,
+    };
 
-    if (isTavily && !config.tavilyApiKey && !config.serperApiKey) {
+    if (provider !== 'local' && !apiKeys[provider]) {
       toast.error('Web Search API key is missing. Please configure it in Settings.');
-      router.push('/settings');
+      router.push('/settings?section=general');
       return;
     }
 

package/components/chat/message-item.tsx

@@ -382,7 +382,7 @@ function renderToolPart(
           <div key={index} className="mb-2 w-full">
             <div className="inline-flex items-center gap-2 px-2 py-1.5 text-xs text-muted-foreground bg-muted/30 rounded-md border border-border/50">
               <Globe className="h-3.5 w-3.5 shrink-0" />
-              <span className="truncate max-w-[250px] sm:max-w-[400px]">
+              <span className="truncate max-w-62.5 sm:max-w-100">
                 Searched web for "{input?.query}"
               </span>
               <span className="shrink-0 text-[10px] bg-secondary text-foreground px-1.5 py-0.5 rounded-full font-medium ml-1">
@@ -395,12 +395,24 @@ function renderToolPart(
 
       case 'analyzeImageDeeply': {
         if (output.error) return null;
+        const imageUrl = input?.imageUrl as string | undefined;
         return (
           <div key={index} className="mb-2 w-full">
             <div className="inline-flex items-center gap-2 px-2 py-1.5 text-xs text-muted-foreground bg-muted/30 rounded-md border border-border/50">
               <Sparkles className="h-3.5 w-3.5 shrink-0" />
-              <span className="truncate max-w-[250px] sm:max-w-[400px]">Analyzed image</span>
+              <span className="truncate max-w-62.5 sm:max-w-100">Analyzed image</span>
             </div>
+            {imageUrl ? (
+              <div className="mt-1.5 max-w-xs">
+                {/* eslint-disable-next-line @next/next/no-img-element */}
+                <img
+                  src={imageUrl}
+                  alt="Analyzed image"
+                  className="rounded-lg border border-border/50 object-cover max-h-48 w-auto"
+                  loading="lazy"
+                />
+              </div>
+            ) : null}
           </div>
         );
       }
@@ -433,7 +445,7 @@ function renderToolPart(
           <div key={index} className="mb-2 w-full">
             <div className="inline-flex items-center gap-2 px-2 py-1.5 text-xs text-muted-foreground bg-muted/30 rounded-md border border-border/50">
               <CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
-              <span className="truncate max-w-[250px] sm:max-w-[400px]">Used {toolName}</span>
+              <span className="truncate max-w-62.5 sm:max-w-100">Used {toolName}</span>
             </div>
           </div>
         );
@@ -714,7 +726,7 @@ export function MessageItem({
                     <div className="size-7 shrink-0 rounded-md overflow-hidden bg-background border border-border/90 flex items-center justify-center p-1">
                       <img src={iconPath} alt={nameStr} className="w-full h-full object-contain" />
                     </div>
-                    <div className="flex flex-col justify-center max-w-[150px]">
+                    <div className="flex flex-col justify-center max-w-37.5">
                       <span className="text-xs font-medium truncate text-foreground">
                         {nameStr}
                       </span>
@@ -932,10 +944,10 @@ function renderMarkdown(text: string): string {
   html = html.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (match, label, url) => {
     const lowerUrl = url.toLowerCase();
     if (lowerUrl.match(/\.(mp4|webm|ogg)$/) || lowerUrl.includes('video')) {
-      return `<video src="${url}" controls preload="metadata" class="max-w-[360px] w-full rounded-lg my-2 border border-border bg-card"></video>`;
+      return `<video src="${url}" controls preload="metadata" class="max-w-90 w-full rounded-lg my-2 border border-border bg-card"></video>`;
     }
     if (lowerUrl.match(/\.(mp3|wav|m4a|oga)$/) || lowerUrl.includes('audio')) {
-      return `<audio src="${url}" controls preload="metadata" class="max-w-[320px] w-full my-2"></audio>`;
+      return `<audio src="${url}" controls preload="metadata" class="max-w-80 w-full my-2"></audio>`;
     }
     return `<a href="${url}" target="_blank" rel="noreferrer" class="msg-link">${label}</a>`;
   });

package/components/chat/tools/knowledge-base-result.tsx

@@ -73,7 +73,7 @@ export function KnowledgeBaseResult({
           ) : (
             <Database className="h-3.5 w-3.5 shrink-0" />
           )}
-          <span className="truncate max-w-[250px] sm:max-w-[400px]">
+          <span className="truncate max-w-62.5 sm:max-w-100">
             {isRunning ? 'Checking knowledge base…' : 'Checked knowledge base'}
           </span>
           {!isRunning && hits.length > 0 ? (
@@ -92,9 +92,8 @@ export function KnowledgeBaseResult({
       {open && (isRunning || hits.length > 0) ? (
         <div className="mt-2 flex flex-col gap-px border border-border rounded-xl bg-transparent overflow-hidden">
           {isRunning && hits.length === 0 ? (
-            <div className="flex items-center gap-2 px-3.5 py-3 text-xs text-muted-foreground">
-              <Loader2 className="size-3.5 animate-spin" />
-              Looking through your Knowledge Base…
+            <div className="flex items-center gap-2 px-3.5 py-2.5">
+              <Loader2 className="size-3 animate-spin text-muted-foreground" />
             </div>
           ) : null}
           {hits.slice(0, 3).map((h, i) => (

package/components/data/data-workspace.tsx

@@ -172,11 +172,15 @@ export function DataWorkspace() {
       );
 
       if (justCompleted.length > 0) {
-        toast.success('Indexing completed.');
+        toast.success('Scraping completed.');
+        // Refresh the corpus immediately so new documents appear.
+        mutateDocuments();
+        // Documents may still be flushing — re-fetch after a short delay.
+        setTimeout(() => mutateDocuments(), 2500);
       }
     }
     prevJobsRef.current = jobs;
-  }, [jobs]);
+  }, [jobs, mutateDocuments]);
 
   const docsQuery = useSWR<DocsResponse>('/api/documents', fetcher, {
     refreshInterval: hasActive ? 5000 : 0,

package/components/data/jobs-panel.tsx

@@ -119,6 +119,27 @@ export function JobsPanel({ jobs, onChanged }: { jobs: CrawlJob[]; onChanged: ()
     setSelected(next);
   }
 
+  async function copyFailedUrls(urls: string) {
+    try {
+      if (navigator.clipboard?.writeText) {
+        await navigator.clipboard.writeText(urls);
+      } else {
+        const textarea = document.createElement('textarea');
+        textarea.value = urls;
+        textarea.style.position = 'fixed';
+        textarea.style.opacity = '0';
+        document.body.appendChild(textarea);
+        textarea.select();
+        const copied = document.execCommand('copy');
+        textarea.remove();
+        if (!copied) throw new Error('The browser denied clipboard access.');
+      }
+      toast.success('Copied failed URLs to clipboard');
+    } catch {
+      toast.error('Could not copy URLs. Select them manually and try again.');
+    }
+  }
+
   async function executeCancel(id: string) {
     setCancelling(true);
     try {
@@ -316,8 +337,7 @@ export function JobsPanel({ jobs, onChanged }: { jobs: CrawlJob[]; onChanged: ()
                               .filter((t) => t.status === 'failed')
                               .map((t) => t.url)
                               .join('\n');
-                            navigator.clipboard.writeText(failedUrls);
-                            toast.success('Copied failed URLs to clipboard');
+                            void copyFailedUrls(failedUrls);
                           }}
                           className="rounded p-1 text-muted-foreground hover:text-foreground transition-colors"
                           title="Copy failed URLs"