larkup 0.1.40 → 0.1.42

package/.next/standalone/apps/web/components/data/media-panel.tsx

@@ -731,13 +731,7 @@ function MediaContent({
       if (!res.ok) throw new Error(data.error ?? 'Media import failed');
       const ids = (data.assets as MediaAsset[]).map((asset) => asset.id);
       toast.success(`${mediaType === 'video' ? 'Video' : 'Audio'} import started`, {
-        description: `Expected time: ${
-          remoteDuration
-            ? `about ${remoteEstimate.processingMinutes} minute${
-                remoteEstimate.processingMinutes === 1 ? '' : 's'
-              }`
-            : 'a few minutes'
-        }. You can safely leave — we will keep indexing it in the background.`,
+        description: 'Queued for download and indexing. You can safely leave in the meantime.',
         duration: 8_000,
       });
       await processAssets(ids);
@@ -1385,7 +1379,7 @@ function MediaContent({
           <AlertDialogFooter>
             <AlertDialogCancel>Keep file</AlertDialogCancel>
             <AlertDialogAction
-              className="bg-destructive text-destructive-foreground hover:bg-destructive/90"
+              className="bg-destructive text-white hover:bg-destructive/90"
               onClick={() => {
                 if (assetToRemove) void handleDelete(assetToRemove.id);
                 setAssetToRemove(null);

package/.next/standalone/apps/web/components/data/scrape-panel.tsx

@@ -576,20 +576,12 @@ export function ScrapePanel({
       const data = await res.json();
       if (!res.ok) throw new Error(data.error ?? 'Could not start job');
 
-      const { totalPages, estimatedSeconds } = estimateEtlDuration(
-        selectedUrls.length,
-        effectiveScope,
-        specificUrls ? 1 : pageLimit,
-      );
-
-      toast.success('Scraping started, running in background', {
+      toast.success('Scrape queued', {
         description: specificUrls
-          ? `Scraping ${selectedUrls.length} specific URL(s) only, no deep crawl. You can navigate away. Progress is tracked automatically.`
+          ? `We will connect to ${selectedUrls.length} specific URL(s) and show real page progress as each response arrives. You can navigate away.`
           : `${selectedUrls.length} ${
               effectiveScope === 'domain' ? 'domain(s)' : 'page(s)'
-            } · ~${totalPages} pages · ETA ${formatDuration(
-              estimatedSeconds,
-            )}. You can navigate away. Progress is tracked automatically.`,
+            } selected. We will show real page progress as each response arrives; no artificial completion estimate. You can navigate away.`,
         duration: 8000,
       });
 

package/.next/standalone/apps/web/components/data/upload-panel.tsx

@@ -219,7 +219,10 @@ export function UploadPanel({ onAdded }: { onAdded: () => void }) {
             format: 'plain',
             rawContent: text,
             fileObject: file,
-            indexImages: !text || text.trim().length === 0,
+            // PDF diagrams are often embedded alongside ordinary text. Keep
+            // visual indexing on by default, with the existing per-file toggle
+            // available for users who only want text extraction.
+            indexImages: true,
           });
         } else {
           const content = await file.text();
@@ -572,7 +575,7 @@ export function UploadPanel({ onAdded }: { onAdded: () => void }) {
               </button>
             </div>
           </div>
-          <ul className="space-y-1.5 max-h-[350px] overflow-y-auto pr-1">
+          <ul className="space-y-1.5 max-h-87.5 overflow-y-auto pr-1">
             {staged.map((f) => (
               <li
                 key={f.id}
@@ -782,7 +785,7 @@ export function UploadPanel({ onAdded }: { onAdded: () => void }) {
                       <p className="text-[11px] text-muted-foreground mb-2 -mt-1">
                         Selected columns will be combined to form the searchable content.
                       </p>
-                      <div className="max-h-[120px] overflow-y-auto space-y-2 pr-2">
+                      <div className="max-h-30 overflow-y-auto space-y-2 pr-2">
                         {editingFile.keys?.map((k) => {
                           const isSelected = editingFile.contentKeys?.includes(k);
                           return (
@@ -871,7 +874,7 @@ export function UploadPanel({ onAdded }: { onAdded: () => void }) {
                       <p className="text-[11px] text-muted-foreground mb-2 -mt-1">
                         These columns will be stored as searchable metadata.
                       </p>
-                      <div className="max-h-[120px] overflow-y-auto space-y-2 pr-2">
+                      <div className="max-h-30 overflow-y-auto space-y-2 pr-2">
                         {editingFile.keys?.map((k) => {
                           const isSelected = editingFile.metadataKeys?.includes(k);
                           const disabled = k === editingFile.titleKey;

package/.next/standalone/apps/web/components/index/index-workspace.tsx

@@ -95,7 +95,7 @@ export function IndexWorkspace({
       setIsIndexing(true);
     } else if (isIndexing && status === 'completed') {
       setIsIndexing(false);
-      toast.success('Indexing complete — your data is ready to use.');
+      toast.success('Indexing complete. Your data is ready to use.');
       try {
         const context = new (window.AudioContext || (window as any).webkitAudioContext)();
         const oscillator = context.createOscillator();

package/.next/standalone/apps/web/lib/pdf-images.ts

@@ -3,7 +3,12 @@ import * as pdfjsLib from 'pdfjs-dist';
 export async function extractImagesFromPDF(
   file: File,
 ): Promise<{ base64: string; pageNumber: number; index: number }[]> {
-  pdfjsLib.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjsLib.version}/pdf.worker.mjs`;
+  // Keep extraction self-contained. The previous CDN worker could fail behind
+  // a firewall or offline, silently leaving PDFs with no indexed visuals.
+  pdfjsLib.GlobalWorkerOptions.workerSrc = new URL(
+    'pdfjs-dist/build/pdf.worker.min.mjs',
+    import.meta.url,
+  ).toString();
 
   const arrayBuffer = await file.arrayBuffer();
   const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(arrayBuffer) }).promise;
@@ -30,6 +35,7 @@ export async function extractImagesFromPDF(
     try {
       const page = await pdf.getPage(i);
       const opList = await page.getOperatorList();
+      const pageStartCount = images.length;
 
       const imageOps = [];
       for (let j = 0; j < opList.fnArray.length; j++) {
@@ -86,6 +92,29 @@ export async function extractImagesFromPDF(
           console.error(`Error extracting image ${objId} on page ${i}`, err);
         }
       }
+
+      // Diagrams and slides in PDFs are commonly vector artwork, not embedded
+      // bitmap objects. When image indexing is explicitly enabled, render such
+      // a page as one bounded visual so its labels and structure remain
+      // available to the vision model and to chat previews.
+      if (images.length === pageStartCount) {
+        try {
+          const viewport = page.getViewport({ scale: 1 });
+          const maxDimension = 1_600;
+          const scale = Math.min(1, maxDimension / Math.max(viewport.width, viewport.height));
+          const renderedViewport = page.getViewport({ scale: Math.max(scale, 0.1) });
+          const canvas = document.createElement('canvas');
+          canvas.width = Math.max(1, Math.round(renderedViewport.width));
+          canvas.height = Math.max(1, Math.round(renderedViewport.height));
+          const ctx = canvas.getContext('2d');
+          if (ctx) {
+            await page.render({ canvas, viewport: renderedViewport }).promise;
+            images.push({ base64: toDataUrl(canvas), pageNumber: i, index: imageIndex++ });
+          }
+        } catch (err) {
+          console.error(`Error rendering PDF page ${i} for visual indexing`, err);
+        }
+      }
     } catch (err) {
       console.error(`Error processing page ${i}`, err);
     }

package/.next/standalone/apps/web/lib/retrieval-routing.ts

@@ -0,0 +1,72 @@
+/**
+ * Keeps retrieval routing deterministic before the chat model is asked to
+ * decide whether a returned source is actually useful.
+ */
+export function requiresKnowledgeBaseSearch(text: string): boolean {
+  const normalized = text.trim();
+  if (/^(hi|hello|hey|thanks|thank you|ok|sure|yes|no|please|help)\b/i.test(normalized)) {
+    return false;
+  }
+
+  return (
+    /\b(my|our)\s+(favo(?:u)?rite|preference|choice|answer|name|result|score|winner)\b/i.test(
+      normalized,
+    ) ||
+    /\b(?:what|which|who|where|when|how)\b[\s\S]{0,80}\b(my|our)\b/i.test(normalized) ||
+    /\b(?:this|that|the)\s+(?:match|video|episode|recording|diagram|chart|image|page|section)\b/i.test(
+      normalized,
+    ) ||
+    /\b(?:shown|listed|named|counted|under)\b[\s\S]{0,80}\b(?:image|diagram|chart|pdf|document|page|section|resources)\b/i.test(
+      normalized,
+    ) ||
+    /\b(my|our)\s+(document|file|data|image|picture|diagram|video|audio|upload|corpus|knowledge|database|db|pdf|report|spreadsheet|presentation)\b/i.test(
+      normalized,
+    ) ||
+    /\b(?:search|check|query)\s+(?:the\s+)?(?:knowledge base|corpus|database|db)\b/i.test(
+      normalized,
+    ) ||
+    /\b(the|this|that)\s+(document|file|diagram|image|picture|upload|pdf|report)\b/i.test(
+      normalized,
+    ) ||
+    /\b(uploaded?|indexed|scraped|knowledge base|corpus)\b/i.test(normalized) ||
+    /\bshow\s+me\b/i.test(normalized)
+  );
+}
+
+/** Current/public questions should begin on the web unless they explicitly
+ * refer to the user's indexed data, which the caller gives precedence to. */
+export function requiresCurrentWebSearch(text: string): boolean {
+  return /\b(?:search (?:the )?web|search online|look (?:it )?up|internet|latest|current|today|yesterday|breaking news|weather|stock price|exchange rate|election result|schedule|who won|winner|match result|final score)\b/i.test(
+    text,
+  );
+}
+
+/** Only the first retrieval is forced. The other source remains available for
+ * exactly one recovery attempt; subsequent steps are reserved for analysis and
+ * the final answer. */
+export function retrievalToolsForStep(options: {
+  stepNumber: number;
+  forceKnowledgeBaseSearch: boolean;
+  forceWebSearch: boolean;
+  toolNames: string[];
+}): { toolChoice?: { type: 'tool'; toolName: string }; activeTools?: string[] } | undefined {
+  const { stepNumber, forceKnowledgeBaseSearch, forceWebSearch, toolNames } = options;
+  const without = (...blocked: string[]) => toolNames.filter((name) => !blocked.includes(name));
+
+  if (stepNumber === 0 && forceKnowledgeBaseSearch) {
+    return { toolChoice: { type: 'tool', toolName: 'searchKnowledgeBase' } };
+  }
+  if (stepNumber === 0 && forceWebSearch) {
+    return { toolChoice: { type: 'tool', toolName: 'webSearch' } };
+  }
+  if (stepNumber === 0) {
+    return { activeTools: without('webSearch', 'searchKnowledgeBase') };
+  }
+  if (stepNumber === 1 && forceKnowledgeBaseSearch) {
+    return { activeTools: without('searchKnowledgeBase') };
+  }
+  if (stepNumber === 1 && forceWebSearch) {
+    return { activeTools: without('webSearch') };
+  }
+  return { activeTools: without('webSearch', 'searchKnowledgeBase') };
+}

package/.next/standalone/apps/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "larkup",
-  "version": "0.1.40",
+  "version": "0.1.42",
   "type": "module",
   "bin": {
     "larkup": "./bin/larkup.js"

package/.next/standalone/apps/web/server.js

@@ -15,7 +15,7 @@ const currentPort = parseInt(process.env.PORT, 10) || 3000
 const hostname = process.env.HOSTNAME || '0.0.0.0'
 
 let keepAliveTimeout = parseInt(process.env.KEEP_ALIVE_TIMEOUT, 10)
-const nextConfig = {"env":{"NEXT_PUBLIC_APP_VERSION":"0.1.40"},"webpack":null,"typescript":{"ignoreBuildErrors":true},"typedRoutes":false,"distDir":"./.next","cleanDistDir":true,"assetPrefix":"","cacheMaxMemorySize":52428800,"configOrigin":"next.config.mjs","useFileSystemPublicRoutes":true,"generateEtags":true,"pageExtensions":["tsx","ts","jsx","js"],"poweredByHeader":true,"compress":true,"images":{"deviceSizes":[640,750,828,1080,1200,1920,2048,3840],"imageSizes":[32,48,64,96,128,256,384],"path":"/_next/image","loader":"default","loaderFile":"","domains":[],"disableStaticImages":false,"minimumCacheTTL":14400,"formats":["image/webp"],"maximumRedirects":3,"maximumResponseBody":50000000,"dangerouslyAllowLocalIP":false,"dangerouslyAllowSVG":false,"contentSecurityPolicy":"script-src 'none'; frame-src 'none'; sandbox;","contentDispositionType":"attachment","localPatterns":[{"pathname":"**","search":""}],"remotePatterns":[],"qualities":[75],"unoptimized":true,"customCacheHandler":false},"devIndicators":{"position":"bottom-left"},"onDemandEntries":{"maxInactiveAge":60000,"pagesBufferLength":5},"basePath":"","sassOptions":{},"trailingSlash":false,"i18n":null,"productionBrowserSourceMaps":false,"excludeDefaultMomentLocales":true,"reactProductionProfiling":false,"reactStrictMode":null,"reactMaxHeadersLength":6000,"httpAgentOptions":{"keepAlive":true},"logging":{"serverFunctions":true,"browserToTerminal":"warn"},"compiler":{},"expireTime":31536000,"staticPageGenerationTimeout":60,"output":"standalone","modularizeImports":{"@mui/icons-material":{"transform":"@mui/icons-material/{{member}}"},"lodash":{"transform":"lodash/{{member}}"}},"outputFileTracingRoot":"/Users/abdelrahmanabounida/Desktop/my-ai-startup/larkup/larkup","cacheComponents":false,"cacheLife":{"default":{"stale":300,"revalidate":900,"expire":4294967294},"seconds":{"stale":30,"revalidate":1,"expire":60},"minutes":{"stale":300,"revalidate":60,"expire":3600},"hours":{"stale":300,"revalidate":3600,"expire":86400},"days":{"stale":300,"revalidate":86400,"expire":604800},"weeks":{"stale":300,"revalidate":604800,"expire":2592000},"max":{"stale":300,"revalidate":2592000,"expire":31536000}},"cacheHandlers":{},"experimental":{"appNewScrollHandler":false,"useSkewCookie":false,"cssChunking":true,"multiZoneDraftMode":false,"appNavFailHandling":false,"prerenderEarlyExit":true,"serverMinification":true,"linkNoTouchStart":false,"caseSensitiveRoutes":false,"cachedNavigations":false,"partialFallbacks":false,"dynamicOnHover":false,"varyParams":false,"prefetchInlining":false,"preloadEntriesOnStart":true,"clientRouterFilter":true,"clientRouterFilterRedirects":false,"fetchCacheKeyPrefix":"","proxyPrefetch":"flexible","optimisticClientCache":true,"manualClientBasePath":false,"cpus":9,"memoryBasedWorkersCount":false,"imgOptConcurrency":null,"imgOptTimeoutInSeconds":7,"imgOptMaxInputPixels":268402689,"imgOptSequentialRead":null,"isrFlushToDisk":true,"workerThreads":false,"optimizeCss":false,"nextScriptWorkers":false,"scrollRestoration":false,"externalDir":false,"disableOptimizedLoading":false,"gzipSize":true,"craCompat":false,"esmExternals":true,"fullySpecified":false,"swcTraceProfiling":false,"forceSwcTransforms":false,"largePageDataBytes":128000,"typedEnv":false,"parallelServerCompiles":false,"parallelServerBuildTraces":false,"ppr":false,"authInterrupts":false,"webpackMemoryOptimizations":false,"optimizeServerReact":true,"strictRouteTypes":false,"useTypeScriptCli":false,"viewTransition":false,"removeUncaughtErrorAndRejectionListeners":false,"validateRSCRequestHeaders":false,"staleTimes":{"dynamic":0,"static":300},"reactDebugChannel":true,"serverComponentsHmrCache":true,"staticGenerationMaxConcurrency":8,"staticGenerationMinPagesPerWorker":25,"transitionIndicator":false,"gestureTransition":false,"inlineCss":false,"useCache":false,"globalNotFound":false,"browserDebugInfoInTerminal":"warn","lockDistDir":true,"proxyClientMaxBodySize":10485760,"hideLogsAfterAbort":false,"mcpServer":true,"turbopackFileSystemCacheForDev":true,"turbopackFileSystemCacheForBuild":false,"turbopackInferModuleSideEffects":true,"turbopackPluginRuntimeStrategy":"childProcesses","optimizePackageImports":["lucide-react","date-fns","lodash-es","ramda","antd","react-bootstrap","ahooks","@ant-design/icons","@headlessui/react","@headlessui-float/react","@heroicons/react/20/solid","@heroicons/react/24/solid","@heroicons/react/24/outline","@visx/visx","@tremor/react","rxjs","@mui/material","@mui/icons-material","recharts","react-use","effect","@effect/schema","@effect/platform","@effect/platform-node","@effect/platform-browser","@effect/platform-bun","@effect/sql","@effect/sql-mssql","@effect/sql-mysql2","@effect/sql-pg","@effect/sql-sqlite-node","@effect/sql-sqlite-bun","@effect/sql-sqlite-wasm","@effect/sql-sqlite-react-native","@effect/rpc","@effect/rpc-http","@effect/typeclass","@effect/experimental","@effect/opentelemetry","@material-ui/core","@material-ui/icons","@tabler/icons-react","mui-core","react-icons/ai","react-icons/bi","react-icons/bs","react-icons/cg","react-icons/ci","react-icons/di","react-icons/fa","react-icons/fa6","react-icons/fc","react-icons/fi","react-icons/gi","react-icons/go","react-icons/gr","react-icons/hi","react-icons/hi2","react-icons/im","react-icons/io","react-icons/io5","react-icons/lia","react-icons/lib","react-icons/lu","react-icons/md","react-icons/pi","react-icons/ri","react-icons/rx","react-icons/si","react-icons/sl","react-icons/tb","react-icons/tfi","react-icons/ti","react-icons/vsc","react-icons/wi"],"trustHostHeader":false,"isExperimentalCompile":false},"htmlLimitedBots":"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight","bundlePagesRouterDependencies":false,"configFileName":"next.config.mjs","transpilePackages":["@larkup/core","@larkup/vector-stores","@larkup/scraper","@larkup/marketplace"],"serverExternalPackages":["@lancedb/lancedb","chromadb","dockerode","@larkup/sandbox"],"turbopack":{"root":"/Users/abdelrahmanabounida/Desktop/my-ai-startup/larkup/larkup"},"distDirRoot":".next","_originalRewrites":{"beforeFiles":[],"afterFiles":[{"source":"/uploads/:path*","destination":"/api/uploads/:path*"}],"fallback":[]}}
+const nextConfig = {"env":{"NEXT_PUBLIC_APP_VERSION":"0.1.42"},"webpack":null,"typescript":{"ignoreBuildErrors":true},"typedRoutes":false,"distDir":"./.next","cleanDistDir":true,"assetPrefix":"","cacheMaxMemorySize":52428800,"configOrigin":"next.config.mjs","useFileSystemPublicRoutes":true,"generateEtags":true,"pageExtensions":["tsx","ts","jsx","js"],"poweredByHeader":true,"compress":true,"images":{"deviceSizes":[640,750,828,1080,1200,1920,2048,3840],"imageSizes":[32,48,64,96,128,256,384],"path":"/_next/image","loader":"default","loaderFile":"","domains":[],"disableStaticImages":false,"minimumCacheTTL":14400,"formats":["image/webp"],"maximumRedirects":3,"maximumResponseBody":50000000,"dangerouslyAllowLocalIP":false,"dangerouslyAllowSVG":false,"contentSecurityPolicy":"script-src 'none'; frame-src 'none'; sandbox;","contentDispositionType":"attachment","localPatterns":[{"pathname":"**","search":""}],"remotePatterns":[],"qualities":[75],"unoptimized":true,"customCacheHandler":false},"devIndicators":{"position":"bottom-left"},"onDemandEntries":{"maxInactiveAge":60000,"pagesBufferLength":5},"basePath":"","sassOptions":{},"trailingSlash":false,"i18n":null,"productionBrowserSourceMaps":false,"excludeDefaultMomentLocales":true,"reactProductionProfiling":false,"reactStrictMode":null,"reactMaxHeadersLength":6000,"httpAgentOptions":{"keepAlive":true},"logging":{"serverFunctions":true,"browserToTerminal":"warn"},"compiler":{},"expireTime":31536000,"staticPageGenerationTimeout":60,"output":"standalone","modularizeImports":{"@mui/icons-material":{"transform":"@mui/icons-material/{{member}}"},"lodash":{"transform":"lodash/{{member}}"}},"outputFileTracingRoot":"/Users/abdelrahmanabounida/Desktop/my-ai-startup/larkup/larkup","cacheComponents":false,"cacheLife":{"default":{"stale":300,"revalidate":900,"expire":4294967294},"seconds":{"stale":30,"revalidate":1,"expire":60},"minutes":{"stale":300,"revalidate":60,"expire":3600},"hours":{"stale":300,"revalidate":3600,"expire":86400},"days":{"stale":300,"revalidate":86400,"expire":604800},"weeks":{"stale":300,"revalidate":604800,"expire":2592000},"max":{"stale":300,"revalidate":2592000,"expire":31536000}},"cacheHandlers":{},"experimental":{"appNewScrollHandler":false,"useSkewCookie":false,"cssChunking":true,"multiZoneDraftMode":false,"appNavFailHandling":false,"prerenderEarlyExit":true,"serverMinification":true,"linkNoTouchStart":false,"caseSensitiveRoutes":false,"cachedNavigations":false,"partialFallbacks":false,"dynamicOnHover":false,"varyParams":false,"prefetchInlining":false,"preloadEntriesOnStart":true,"clientRouterFilter":true,"clientRouterFilterRedirects":false,"fetchCacheKeyPrefix":"","proxyPrefetch":"flexible","optimisticClientCache":true,"manualClientBasePath":false,"cpus":9,"memoryBasedWorkersCount":false,"imgOptConcurrency":null,"imgOptTimeoutInSeconds":7,"imgOptMaxInputPixels":268402689,"imgOptSequentialRead":null,"isrFlushToDisk":true,"workerThreads":false,"optimizeCss":false,"nextScriptWorkers":false,"scrollRestoration":false,"externalDir":false,"disableOptimizedLoading":false,"gzipSize":true,"craCompat":false,"esmExternals":true,"fullySpecified":false,"swcTraceProfiling":false,"forceSwcTransforms":false,"largePageDataBytes":128000,"typedEnv":false,"parallelServerCompiles":false,"parallelServerBuildTraces":false,"ppr":false,"authInterrupts":false,"webpackMemoryOptimizations":false,"optimizeServerReact":true,"strictRouteTypes":false,"useTypeScriptCli":false,"viewTransition":false,"removeUncaughtErrorAndRejectionListeners":false,"validateRSCRequestHeaders":false,"staleTimes":{"dynamic":0,"static":300},"reactDebugChannel":true,"serverComponentsHmrCache":true,"staticGenerationMaxConcurrency":8,"staticGenerationMinPagesPerWorker":25,"transitionIndicator":false,"gestureTransition":false,"inlineCss":false,"useCache":false,"globalNotFound":false,"browserDebugInfoInTerminal":"warn","lockDistDir":true,"proxyClientMaxBodySize":10485760,"hideLogsAfterAbort":false,"mcpServer":true,"turbopackFileSystemCacheForDev":true,"turbopackFileSystemCacheForBuild":false,"turbopackInferModuleSideEffects":true,"turbopackPluginRuntimeStrategy":"childProcesses","optimizePackageImports":["lucide-react","date-fns","lodash-es","ramda","antd","react-bootstrap","ahooks","@ant-design/icons","@headlessui/react","@headlessui-float/react","@heroicons/react/20/solid","@heroicons/react/24/solid","@heroicons/react/24/outline","@visx/visx","@tremor/react","rxjs","@mui/material","@mui/icons-material","recharts","react-use","effect","@effect/schema","@effect/platform","@effect/platform-node","@effect/platform-browser","@effect/platform-bun","@effect/sql","@effect/sql-mssql","@effect/sql-mysql2","@effect/sql-pg","@effect/sql-sqlite-node","@effect/sql-sqlite-bun","@effect/sql-sqlite-wasm","@effect/sql-sqlite-react-native","@effect/rpc","@effect/rpc-http","@effect/typeclass","@effect/experimental","@effect/opentelemetry","@material-ui/core","@material-ui/icons","@tabler/icons-react","mui-core","react-icons/ai","react-icons/bi","react-icons/bs","react-icons/cg","react-icons/ci","react-icons/di","react-icons/fa","react-icons/fa6","react-icons/fc","react-icons/fi","react-icons/gi","react-icons/go","react-icons/gr","react-icons/hi","react-icons/hi2","react-icons/im","react-icons/io","react-icons/io5","react-icons/lia","react-icons/lib","react-icons/lu","react-icons/md","react-icons/pi","react-icons/ri","react-icons/rx","react-icons/si","react-icons/sl","react-icons/tb","react-icons/tfi","react-icons/ti","react-icons/vsc","react-icons/wi"],"trustHostHeader":false,"isExperimentalCompile":false},"htmlLimitedBots":"[\\w-]+-Google|Google-[\\w-]+|Chrome-Lighthouse|Slurp|DuckDuckBot|baiduspider|yandex|sogou|bitlybot|tumblr|vkShare|quora link preview|redditbot|ia_archiver|Bingbot|BingPreview|applebot|facebookexternalhit|facebookcatalog|Twitterbot|LinkedInBot|Slackbot|Discordbot|WhatsApp|SkypeUriPreview|Yeti|googleweblight","bundlePagesRouterDependencies":false,"configFileName":"next.config.mjs","transpilePackages":["@larkup/core","@larkup/vector-stores","@larkup/scraper","@larkup/marketplace"],"serverExternalPackages":["@lancedb/lancedb","chromadb","dockerode","@larkup/sandbox"],"turbopack":{"root":"/Users/abdelrahmanabounida/Desktop/my-ai-startup/larkup/larkup"},"distDirRoot":".next","_originalRewrites":{"beforeFiles":[],"afterFiles":[{"source":"/uploads/:path*","destination":"/api/uploads/:path*"}],"fallback":[]}}
 
 process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(nextConfig)
 

package/.next/standalone/packages/tools/doc-editor/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @larkup/tool-doc-editor
 
+## 0.2.7
+
+### Patch Changes
+
+- Updated dependencies
+  - @larkup/core@0.2.6
+
 ## 0.2.6
 
 ### Patch Changes

package/.next/standalone/packages/tools/doc-editor/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/tool-doc-editor",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "publishConfig": {
     "access": "public"
   },

package/.next/standalone/packages/tools/video-audio/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @larkup/tool-video-audio
 
+## 0.2.4
+
+### Patch Changes
+
+- 0ff7dec: Stabilize native crawl state, prioritise local knowledge retrieval, and improve media-import progress and source-download recovery.
+
 ## 0.2.3
 
 ### Patch Changes

package/.next/standalone/packages/tools/video-audio/dist/url-importer.js

@@ -57,7 +57,7 @@ export async function importMediaUrl(url, options) {
     if (isYouTube(parsed)) {
         const template = path.join(options.outputDir, '%(title).120B [%(id)s].%(ext)s');
         const print = '{"path":%(filepath)j,"title":%(title)j,"originalUrl":%(webpage_url)j,"ext":%(ext)j}';
-        const output = await runYtDlp([
+        const commonArgs = [
             '--no-playlist',
             '--playlist-end',
             String(options.playlistMax ?? 10),
@@ -67,8 +67,6 @@ export async function importMediaUrl(url, options) {
             '2',
             '--fragment-retries',
             '2',
-            '--concurrent-fragments',
-            '4',
             '--format',
             'best[height<=360]/bestvideo[height<=360]+bestaudio/best',
             '--merge-output-format',
@@ -77,8 +75,32 @@ export async function importMediaUrl(url, options) {
             template,
             '--print',
             `after_move:${print}`,
-            url,
-        ], options.onProgress);
+        ];
+        let output;
+        try {
+            output = await runYtDlp([...commonArgs, '--concurrent-fragments', '4', url], options.onProgress);
+        }
+        catch (error) {
+            if (!isVideoHostForbidden(error))
+                throw error;
+            // Video hosts occasionally reject a freshly-issued media URL. Re-extract
+            // it with a conservative one-fragment retry before failing the asset.
+            options.onProgress?.({ message: 'Source rejected the download; refreshing the video link…' });
+            try {
+                output = await runYtDlp([
+                    ...commonArgs,
+                    '--force-ipv4',
+                    '--extractor-retries',
+                    '3',
+                    '--concurrent-fragments',
+                    '1',
+                    url,
+                ], options.onProgress);
+            }
+            catch (retryError) {
+                throw friendlyVideoDownloadError(retryError);
+            }
+        }
         const imported = output
             .split('\n')
             .filter(Boolean)
@@ -135,6 +157,16 @@ export async function importMediaUrl(url, options) {
         },
     ];
 }
+function isVideoHostForbidden(error) {
+    return error instanceof Error && /\b(?:403|forbidden)\b/i.test(error.message);
+}
+/** Keep upstream yt-dlp output out of the end-user status while retaining a useful cause. */
+function friendlyVideoDownloadError(error) {
+    if (isVideoHostForbidden(error)) {
+        return new Error('The video host refused this download (HTTP 403) after a safe retry. This is a temporary source restriction, not a problem with your existing index. Try again later, or upload the original video file directly.');
+    }
+    return error instanceof Error ? error : new Error('Video download failed. Please try again.');
+}
 /**
  * Read the video's own manual or automatic captions through yt-dlp. This is
  * both faster and usually more accurate than transcribing compressed audio,

package/.next/standalone/packages/tools/video-audio/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larkup/tool-video-audio",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "publishConfig": {
     "access": "public"
   },

package/.next/standalone/packages/tools/video-audio/src/url-importer.ts

@@ -99,31 +99,53 @@ export async function importMediaUrl(
     const template = path.join(options.outputDir, '%(title).120B [%(id)s].%(ext)s');
     const print =
       '{"path":%(filepath)j,"title":%(title)j,"originalUrl":%(webpage_url)j,"ext":%(ext)j}';
-    const output = await runYtDlp(
-      [
-        '--no-playlist',
-        '--playlist-end',
-        String(options.playlistMax ?? 10),
-        '--socket-timeout',
-        '20',
-        '--retries',
-        '2',
-        '--fragment-retries',
-        '2',
-        '--concurrent-fragments',
-        '4',
-        '--format',
-        'best[height<=360]/bestvideo[height<=360]+bestaudio/best',
-        '--merge-output-format',
-        'mp4',
-        '-o',
-        template,
-        '--print',
-        `after_move:${print}`,
-        url,
-      ],
-      options.onProgress,
-    );
+    const commonArgs = [
+      '--no-playlist',
+      '--playlist-end',
+      String(options.playlistMax ?? 10),
+      '--socket-timeout',
+      '20',
+      '--retries',
+      '2',
+      '--fragment-retries',
+      '2',
+      '--format',
+      'best[height<=360]/bestvideo[height<=360]+bestaudio/best',
+      '--merge-output-format',
+      'mp4',
+      '-o',
+      template,
+      '--print',
+      `after_move:${print}`,
+    ];
+    let output: string;
+    try {
+      output = await runYtDlp(
+        [...commonArgs, '--concurrent-fragments', '4', url],
+        options.onProgress,
+      );
+    } catch (error) {
+      if (!isVideoHostForbidden(error)) throw error;
+      // Video hosts occasionally reject a freshly-issued media URL. Re-extract
+      // it with a conservative one-fragment retry before failing the asset.
+      options.onProgress?.({ message: 'Source rejected the download; refreshing the video link…' });
+      try {
+        output = await runYtDlp(
+          [
+            ...commonArgs,
+            '--force-ipv4',
+            '--extractor-retries',
+            '3',
+            '--concurrent-fragments',
+            '1',
+            url,
+          ],
+          options.onProgress,
+        );
+      } catch (retryError) {
+        throw friendlyVideoDownloadError(retryError);
+      }
+    }
     const imported = output
       .split('\n')
       .filter(Boolean)
@@ -186,6 +208,20 @@ export async function importMediaUrl(
   ];
 }
 
+function isVideoHostForbidden(error: unknown): boolean {
+  return error instanceof Error && /\b(?:403|forbidden)\b/i.test(error.message);
+}
+
+/** Keep upstream yt-dlp output out of the end-user status while retaining a useful cause. */
+function friendlyVideoDownloadError(error: unknown): Error {
+  if (isVideoHostForbidden(error)) {
+    return new Error(
+      'The video host refused this download (HTTP 403) after a safe retry. This is a temporary source restriction, not a problem with your existing index. Try again later, or upload the original video file directly.',
+    );
+  }
+  return error instanceof Error ? error : new Error('Video download failed. Please try again.');
+}
+
 /**
  * Read the video's own manual or automatic captions through yt-dlp. This is
  * both faster and usually more accurate than transcribing compressed audio,

package/.next/static/chunks/1_knu6c2q82r_.js

@@ -0,0 +1 @@
+(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,349871,t=>{t.q("/_next/static/media/pdf.worker.min.43wid0v0nfx-f.mjs")},385290,t=>{"use strict";var e=t.i(410386);async function a(a){e.GlobalWorkerOptions.workerSrc=new t.U(t.r(349871)).toString();let r=await a.arrayBuffer(),i=await e.getDocument({data:new Uint8Array(r)}).promise,n=i.numPages,h=[],o=0,g=t=>{let e=Math.min(1,1600/Math.max(t.width,t.height));if(1===e)return t.toDataURL("image/jpeg",.86);let a=document.createElement("canvas");return a.width=Math.max(1,Math.round(t.width*e)),a.height=Math.max(1,Math.round(t.height*e)),a.getContext("2d")?.drawImage(t,0,0,a.width,a.height),a.toDataURL("image/jpeg",.86)};for(let t=1;t<=n;t++)try{let a=await i.getPage(t),r=await a.getOperatorList(),n=h.length,m=[];for(let t=0;t<r.fnArray.length;t++)(r.fnArray[t]===e.OPS.paintImageXObject||r.fnArray[t]===e.OPS.paintInlineImageXObject)&&m.push(r.argsArray[t][0]);for(let e of m)try{let r=await new Promise(t=>{try{a.objs.get(e,e=>t(e))}catch(e){t(null)}});if(r&&r.bitmap){let e=document.createElement("canvas");e.width=r.bitmap.width||r.width,e.height=r.bitmap.height||r.height;let a=e.getContext("2d");a&&(a.drawImage(r.bitmap,0,0),h.push({base64:g(e),pageNumber:t,index:o++}))}else if(r&&r.data&&r.width&&r.height){let e=document.createElement("canvas");e.width=r.width,e.height=r.height;let a=e.getContext("2d");if(a){let i=new ImageData(new Uint8ClampedArray(r.data),r.width,r.height);a.putImageData(i,0,0),h.push({base64:g(e),pageNumber:t,index:o++})}}}catch(a){console.error(`Error extracting image ${e} on page ${t}`,a)}if(h.length===n)try{let e=a.getViewport({scale:1}),r=Math.min(1,1600/Math.max(e.width,e.height)),i=a.getViewport({scale:Math.max(r,.1)}),n=document.createElement("canvas");n.width=Math.max(1,Math.round(i.width)),n.height=Math.max(1,Math.round(i.height)),n.getContext("2d")&&(await a.render({canvas:n,viewport:i}).promise,h.push({base64:g(n),pageNumber:t,index:o++}))}catch(e){console.error(`Error rendering PDF page ${t} for visual indexing`,e)}}catch(e){console.error(`Error processing page ${t}`,e)}return h}t.s(["extractImagesFromPDF",0,a])}]);