larkup 0.1.34 → 0.1.35

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # larkup
 
+## 0.1.35
+
+### Patch Changes
+
+- 5ba20af: Make the local crawler work without Docker, make Marketplace installs opt-in and removable, and bound RAG retrieval tool loops and context growth.
+- Updated dependencies [5ba20af]
+  - @larkup/scraper@0.1.26
+  - @larkup/marketplace@0.1.9
+  - @larkup/tool-doc-editor@0.2.5
+  - @larkup/tool-video-audio@0.2.2
+
 ## 0.1.34
 
 ### Patch Changes

package/app/api/chat/route.ts

@@ -78,7 +78,7 @@ RETRIEVAL RULES:
 - Do not search for greetings, casual conversation, writing help, brainstorming, explanations you can answer reliably from general knowledge, or questions unrelated to the user's indexed data.
 - Use searchKnowledgeBase only when the answer depends on the user's indexed/private content, an exact source-specific fact, or a topic that plausibly refers to something the user indexed.
 - Treat questions about the user or their materials as knowledge-base questions: phrases such as "I", "my", "our", "you have", "the database", "the document", "the file", or "the diagram" require searchKnowledgeBase before answering whenever indexed content may contain the answer.
-- Search at most once per user request unless the first search is clearly irrelevant. Never repeat a search when a prior tool result in this conversation already contains enough evidence.
+- Search at most once per user request. Never repeat a search when a prior tool result in this conversation already contains enough evidence. If the results do not answer the question, say so plainly instead of making more searches or guessing.
 
 IMAGE AND MEDIA RULES:
 - NEVER output raw markdown image syntax like ![alt](url). The chat UI blocks these and shows "[Image unavailable]". Instead, always use the presentMedia tool to display images, videos, or audio from indexed content.
@@ -178,6 +178,26 @@ export async function POST(req: Request) {
         }
       }
       if (typeof resultObj !== 'object' || resultObj === null) return result;
+      // Keep previous retrieval evidence available for follow-up questions,
+      // while preventing several full five-chunk search responses from taking
+      // over the model context on a long conversation.
+      if (toolName === 'searchKnowledgeBase' && Array.isArray(resultObj.hits)) {
+        const compact = {
+          ...resultObj,
+          hits: resultObj.hits.map((hit: any) => ({
+            documentId: hit.documentId,
+            title: hit.title,
+            url: hit.url,
+            score: hit.score,
+            text: typeof hit.text === 'string' ? hit.text.slice(0, 1_000) : hit.text,
+            images: hit.images,
+            metadata: hit.metadata,
+            timelineContext: hit.timelineContext,
+            endingContext: hit.endingContext,
+          })),
+        };
+        return isString ? JSON.stringify(compact) : compact;
+      }
       const isDocTool =
         toolName &&
         ['fillDocumentForm', 'editDocument', 'requestDocumentSignature'].includes(toolName);
@@ -365,11 +385,23 @@ ${fieldLines}`;
     model,
     system: systemPrompt,
     messages: await convertToModelMessages(safeMessages, { tools }),
-    maxOutputTokens: 8192,
-    stopWhen: stepCountIs(8),
-    toolChoice: forceKnowledgeBaseSearch
-      ? { type: 'tool', toolName: 'searchKnowledgeBase' }
-      : 'auto',
+    maxOutputTokens: 4096,
+    // Five steps covers the only useful multi-step case (retrieve, optionally
+    // present one media item, then answer) without allowing a tool loop to
+    // burn the context window and end with no response.
+    stopWhen: stepCountIs(5),
+    toolChoice: 'auto',
+    prepareStep: ({ stepNumber }) => {
+      if (forceKnowledgeBaseSearch && stepNumber === 0) {
+        return { toolChoice: { type: 'tool', toolName: 'searchKnowledgeBase' } };
+      }
+      // A retrieval is deliberately one-shot per turn. This is applied at the
+      // runtime level rather than relying only on the model prompt.
+      if (forceKnowledgeBaseSearch) {
+        return { activeTools: Object.keys(tools).filter((name) => name !== 'searchKnowledgeBase') };
+      }
+      return undefined;
+    },
     onFinish: async ({ usage, response }) => {
       const { trackUsageEvent, estimateCost } = await import('@larkup/core/analytics-store');
       const u = usage as any;

package/components/data/firecrawl-notice.tsx

@@ -33,7 +33,7 @@ const fetcher = (url: string) => fetch(url).then((r) => r.json());
  * Web-crawler launcher notice.
  *
  * Adapts to the runtime environment:
- * - **Web / Desktop**: lets the user spin up a self-hosted crawler via Docker.
+ * - **Web / Desktop**: starts Larkup's built-in crawler (no Docker required).
  * - **Docker**: detects a sibling crawler service on the Docker network.
  *
  * The API key (cloud) option is managed in Settings — this component only
@@ -59,7 +59,7 @@ export function FirecrawlNotice({
   const runtimeEnv = data?.runtimeEnv ?? 'web';
   const running = state?.running ?? false;
   const dockerReady = docker?.compose ?? false;
-  const hasError = !cloudConfigured && (!dockerReady || !!state?.lastError);
+  const hasError = !cloudConfigured && (!running || !!state?.lastError);
 
   useEffect(() => {
     if (!isLoading) {
@@ -76,7 +76,7 @@ export function FirecrawlNotice({
           description:
             runtimeEnv === 'docker'
               ? 'Checking if the crawler service is available on the network.'
-              : 'Pulling images and starting containers. This can take a minute on first run.',
+              : 'Starting Larkup’s built-in crawler. No Docker setup is required.',
         },
       );
     }
@@ -100,8 +100,8 @@ export function FirecrawlNotice({
               'Start Larkup with the optional crawler profile in docker-compose, or add a Firecrawl API key in Settings.',
           });
         } else {
-          toast.warning('Web crawler is starting', {
-            description: 'It may take a moment to become available.',
+          toast.success('Web crawler is ready', {
+            description: 'You can now scrape websites without Docker.',
           });
         }
       } else {

package/components/data/media-panel.tsx

@@ -1634,9 +1634,9 @@ function ActiveIndexingList({
                   {asset.processingMessage || 'Preparing media indexing...'}
                 </p>
               </div>
-              {/* {asset.processingSteps?.length ? (
+              {asset.processingSteps?.length ? (
                 <PipelineSteps steps={asset.processingSteps} />
-              ) : null} */}
+              ) : null}
             </div>
           );
         })}

package/components/ui/skeleton.tsx

@@ -1,13 +1,14 @@
-import { cn } from "@/lib/utils"
+import { type HTMLAttributes } from 'react';
+import { cn } from '@/lib/utils';
 
-function Skeleton({ className, ...props }: React.ComponentProps<"div">) {
+function Skeleton({ className, ...props }: HTMLAttributes<HTMLDivElement>) {
   return (
     <div
       data-slot="skeleton"
-      className={cn("animate-pulse rounded-md bg-muted", className)}
+      className={cn('animate-pulse rounded-md bg-muted', className)}
       {...props}
     />
-  )
+  );
 }
 
-export { Skeleton }
+export { Skeleton };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "larkup",
-  "version": "0.1.34",
+  "version": "0.1.35",
   "type": "module",
   "bin": {
     "larkup": "./bin/larkup.js"
@@ -53,13 +53,13 @@
     "xlsx": "^0.18.5",
     "zod": "^4.4.3",
     "zustand": "^5.0.14",
-    "@larkup/tool-doc-editor": "0.2.5",
     "@larkup/core": "0.2.4",
-    "@larkup/vector-stores": "0.1.23",
+    "@larkup/sandbox": "0.1.2",
     "@larkup/tool-video-audio": "0.2.2",
-    "@larkup/marketplace": "0.1.8",
-    "@larkup/scraper": "0.1.25",
-    "@larkup/sandbox": "0.1.2"
+    "@larkup/scraper": "0.1.26",
+    "@larkup/marketplace": "0.1.9",
+    "@larkup/tool-doc-editor": "0.2.5",
+    "@larkup/vector-stores": "0.1.23"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.2.0",

package/.next/server/chunks/[root-of-the-server]__067hiof._.js

@@ -1,61 +0,0 @@
-module.exports=[367165,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(638725),n=e.i(192441);async function i(){let e=await (0,n.getDataDir)();if(!e)return a.DEFAULT_CONFIG;try{let n=await t.promises.readFile(r.default.join(e,"config.json"),"utf8"),i=JSON.parse(n),o=i.customEmbeddings,s=i.embeddingModelId;if(i.customEmbedding&&!o?.length){let e=i.customEmbedding;o=[e],"custom"===s&&(s=`custom:${e.modelName}`)}let l={...a.DEFAULT_CONFIG,...i,embeddingModelId:s??a.DEFAULT_CONFIG.embeddingModelId,customEmbeddings:o,chunking:{...a.DEFAULT_CONFIG.chunking,...i.chunking},storeConfig:{...a.DEFAULT_CONFIG.storeConfig,...i.storeConfig}};return delete l.customEmbedding,l}catch{return a.DEFAULT_CONFIG}}async function o(e){let a=await (0,n.requireDataDir)(),i={...e,updatedAt:new Date().toISOString()};return await t.promises.writeFile(r.default.join(a,"config.json"),JSON.stringify(i,null,2),"utf8"),i}e.s(["readConfig",0,i,"writeConfig",0,o])},874533,(e,t,r)=>{t.exports=e.x("node:child_process",()=>require("node:child_process"))},812057,(e,t,r)=>{t.exports=e.x("node:util",()=>require("node:util"))},700259,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(761095),n=e.i(666680),i=e.i(874533),o=e.i(812057),s=e.i(367165);let l=(0,o.promisify)(i.exec);async function c(e,t){return l(e,{timeout:t,env:{...process.env,PATH:`${process.env.PATH||""}:/usr/local/bin:/opt/homebrew/bin:/bin:/usr/bin`}})}let d=r.default.join(process.cwd(),".larkup"),u=r.default.join(d,"firecrawl-local.json"),p=r.default.join(d,"firecrawl","docker-compose.yml"),h="ragtoolkit-firecrawl";async function w(e){return new Promise((t,r)=>{let n=a.default.createServer();n.listen(e,()=>{let e=n.address().port;n.close(()=>t(e))}),n.on("error",a=>{"EADDRINUSE"===a.code?w(0===e?0:e+1).then(t,r):r(a)})})}async function m(){try{let e=r.default.join(process.cwd(),"proxies.txt"),a=(await t.promises.readFile(e,"utf8")).split("\n").map(e=>e.trim()).filter(e=>e.length>0);if(a.length>0){let e=a[Math.floor(Math.random()*a.length)].split(":");if(e.length>=4)return{server:`http://${e[0]}:${e[1]}`,username:e[2],password:e[3]}}}catch(e){}let e=await (0,s.readConfig)();return e.useScraperProxy&&e.scraperProxyServer?{server:e.scraperProxyServer,username:e.scraperProxyUsername,password:e.scraperProxyPassword}:null}let f={running:!1,endpoint:"http://localhost:3002",apiKey:"",port:3002,project:h};async function g(){try{let e=await t.promises.readFile(u,"utf8");return{...f,...JSON.parse(e)}}catch{return f}}async function y(e){return await t.promises.mkdir(d,{recursive:!0}),await t.promises.writeFile(u,JSON.stringify(e,null,2),"utf8"),e}async function E(){try{await c("docker --version")}catch(e){return console.error("DOCKER ERR:",e),{docker:!1,compose:null}}try{return await c("docker compose version"),{docker:!0,compose:"docker compose"}}catch(e){console.error("DOCKER COMPOSE ERR:",e)}try{return await c("docker-compose version"),{docker:!0,compose:"docker-compose"}}catch{return{docker:!0,compose:null}}}async function R(){let{docker:e,compose:t}=await E();return e?t?{docker:!0,compose:!0,message:"Docker is ready."}:{docker:!0,compose:!1,message:"Docker is installed but the Compose plugin was not found. Install `docker compose` to launch Firecrawl locally."}:{docker:!1,compose:!1,message:"Docker isn't available on this machine. Install Docker Desktop (or the Docker engine) and make sure it's running."}}function v(e,t,r){let a=r?[`PROXY_SERVER: "${r.server}"`,...r.username&&r.password?[`PROXY_USERNAME: "${r.username}"`,`PROXY_PASSWORD: "${r.password}"`]:[]]:[],n=['PORT: "3000"',...a].map(e=>`      ${e}`).join("\n"),i=['PORT: "3002"',...a,'HOST: "0.0.0.0"','REDIS_URL: "redis://redis:6379"','REDIS_RATE_LIMIT_URL: "redis://redis:6379"','PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape"','USE_DB_AUTHENTICATION: "false"',`TEST_API_KEY: "${e}"`,`BULL_AUTH_KEY: "${e}"`,'DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_RABBITMQ_URL: "amqp://user:password@rabbitmq:5672"'].map(e=>`      ${e}`).join("\n");return`# Auto-generated by RAG Toolkit. Do not edit by hand.
-services:
-  db:
-    image: ghcr.io/firecrawl/nuq-postgres:latest
-    environment:
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U user -d postgres"]
-      interval: 5s
-      timeout: 5s
-      retries: 12
-    restart: unless-stopped
-
-  rabbitmq:
-    image: rabbitmq:3-management-alpine
-    environment:
-      RABBITMQ_DEFAULT_USER: user
-      RABBITMQ_DEFAULT_PASS: password
-    healthcheck:
-      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
-      interval: 5s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    restart: unless-stopped
-
-  redis:
-    image: redis:alpine
-    command: redis-server --save "" --appendonly no
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 5s
-      timeout: 3s
-      retries: 12
-    restart: unless-stopped
-
-  playwright-service:
-    image: ghcr.io/firecrawl/playwright-service:latest
-    environment:
-${n}
-    restart: unless-stopped
-
-  api:
-    image: ghcr.io/firecrawl/firecrawl:latest
-    depends_on:
-      db:
-        condition: service_healthy
-      rabbitmq:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "${t}:3002"
-    environment:
-${i}
-    restart: unless-stopped
-`}async function S(){let e=await R();if(!e.compose)return y({...await g(),running:!1,lastError:e.message});let{compose:a}=await E(),i=await g(),o=i.apiKey||`fc-local-${(0,n.randomUUID)()}`,s=i.port||3002,l=await m();await t.promises.mkdir(r.default.dirname(p),{recursive:!0}),await t.promises.writeFile(p,v(o,s,l),"utf8");try{await c(`${a} -p ${h} -f "${p}" up -d`,18e4)}catch(r){let e=r instanceof Error?r.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return y({...i,apiKey:o,port:s,running:!1,lastError:e});try{s=await w(s+1),await t.promises.writeFile(p,v(o,s,l),"utf8"),await c(`${a} -p ${h} -f "${p}" up -d`,18e4)}catch(e){return y({...i,apiKey:o,port:s,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let d=(0,t.existsSync)("/.dockerenv"),u=`http://${d?"host.docker.internal":"localhost"}:${s}`,f=await k(u,9e4);return y({running:f,endpoint:u,apiKey:o,port:s,project:h,startedAt:new Date().toISOString(),lastError:f?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function A(){let{compose:e}=await E(),t=await g();if(e)try{await c(`${e} -p ${h} -f "${p}" down`,6e4)}catch{}return y({...t,running:!1,startedAt:void 0})}async function C(){let e=await g();if(!e.startedAt)return e;let t=await b(e.endpoint);return t!==e.running?y({...e,running:t}):e}async function b(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function k(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(await b(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}let T="http://firecrawl-api:3002";async function _(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,t.existsSync)("/.dockerenv")?{available:await b(T),endpoint:T}:{available:!1,endpoint:""}}async function D(){let{available:e}=await _();return e?y({running:!0,endpoint:T,apiKey:`fc-docker-${(0,n.randomUUID)()}`,port:3002,project:"docker-sibling",startedAt:new Date().toISOString()}):y({...f,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,R,"checkDockerSibling",0,_,"connectDockerSibling",0,D,"readLocalState",0,g,"refreshLocalStatus",0,C,"startLocal",0,S,"stopLocal",0,A])},304368,e=>{"use strict";var t=e.i(367165),r=e.i(700259);class a extends Error{status;constructor(e,t){super(e),this.status=t,this.name="FirecrawlError"}}async function n(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();if(e.running&&e.apiKey)return{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new a("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let n=e.firecrawlApiKey;if(n)return{base:"https://api.firecrawl.dev/v1",key:n,mode:"cloud"};throw new a("The cloud web crawler needs an API key. Add it in Settings.",401)}async function i(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function o(e,t){let{base:r,key:i}=await n(),o=await fetch(`${r}${e}`,{...t,headers:{Authorization:`Bearer ${i}`,"Content-Type":"application/json",...t?.headers??{}},cache:"no-store"}),s=await o.text(),c=s?l(s):{};if(!o.ok)throw new a(c?.error||`Firecrawl request failed (${o.status})`,o.status);return c}async function s(e){let{key:t}=await n(),r=await fetch(e,{headers:{Authorization:`Bearer ${t}`},cache:"no-store"}),i=await r.text(),o=i?l(i):{};if(!r.ok)throw new a(o?.error||`Firecrawl request failed (${r.status})`,r.status);return o}function l(e){try{return JSON.parse(e)}catch{return{}}}async function c(e,t=10){return((await o("/search",{method:"POST",body:JSON.stringify({query:e,limit:t})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let d={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function u(e){let t=(await o("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...d})})).data??{};return{url:t.metadata?.sourceURL||e,title:t.metadata?.title||t.metadata?.ogTitle||e,markdown:t.markdown??""}}async function p(e,t){let r=await o("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:t,allowExternalLinks:!1,scrapeOptions:{...d,formats:["markdown"],onlyMainContent:!0}})});if(!r.id)throw new a("Firecrawl did not return a crawl id");return r.id}async function h(e,t=!1){let r=t?await s(e):await o(`/crawl/${e}`,{method:"GET"});return{state:r.status??"scraping",total:r.total??0,completed:r.completed??0,next:r.next,pages:(r.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function w(e){try{await o(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,a,"cancelCrawl",0,w,"getCrawlStatus",0,h,"isFirecrawlConfigured",0,i,"scrapePage",0,u,"searchWeb",0,c,"startCrawl",0,p])},426237,e=>{"use strict";var t=e.i(766576),r=e.i(304368);async function a(){return t.NextResponse.json({configured:await (0,r.isFirecrawlConfigured)()})}async function n(e){try{let{query:a,limit:n}=await e.json();if(!a||!a.trim())return t.NextResponse.json({error:"A search query is required."},{status:400});let i=await (0,r.searchWeb)(a.trim(),Math.min(n??10,25));return t.NextResponse.json({results:i})}catch(n){let e=n instanceof r.FirecrawlError?n.status??500:500,a=n instanceof Error?n.message:"Search failed.";return t.NextResponse.json({error:a},{status:e})}}e.s(["GET",0,a,"POST",0,n,"dynamic",0,"force-dynamic","runtime",0,"nodejs"])},695936,e=>{"use strict";var t=e.i(239441),r=e.i(449055),a=e.i(391328),n=e.i(6352),i=e.i(746789),o=e.i(423437),s=e.i(178820),l=e.i(569191),c=e.i(576251),d=e.i(199962),u=e.i(547545),p=e.i(664046),h=e.i(235691),w=e.i(298907),m=e.i(110763),f=e.i(193695);e.i(673243);var g=e.i(236167),y=e.i(426237);let E=new t.AppRouteRouteModule({definition:{kind:r.RouteKind.APP_ROUTE,page:"/api/search/route",pathname:"/api/search",filename:"route",bundlePath:""},distDir:".next",relativeProjectDir:"",resolvedPagePath:"[project]/apps/web/app/api/search/route.ts",nextConfigOutput:"standalone",userland:y,...{}}),{workAsyncStorage:R,workUnitAsyncStorage:v,serverHooks:S}=E;async function A(e,t,a){a.requestMeta&&(0,n.setRequestMeta)(e,a.requestMeta),E.isDev&&(0,n.addRequestMeta)(e,"devRequestTimingInternalsEnd",process.hrtime.bigint());let y="/api/search/route";y=y.replace(/\/index$/,"")||"/";let R=await E.prepare(e,t,{srcPage:y,multiZoneDraftMode:!1});if(!R)return t.statusCode=400,t.end("Bad Request"),null==a.waitUntil||a.waitUntil.call(a,Promise.resolve()),null;let{buildId:v,deploymentId:S,params:A,nextConfig:C,parsedUrl:b,isDraftMode:k,prerenderManifest:T,routerServerContext:_,isOnDemandRevalidate:D,revalidateOnlyGenerated:O,resolvedPathname:P,clientReferenceManifest:x,serverActionsManifest:U}=R,N=(0,s.normalizeAppPath)(y),I=!!(T.dynamicRoutes[N]||T.routes[P]),$=async()=>((null==_?void 0:_.render404)?await _.render404(e,t,b,!1):t.end("This page could not be found"),null);if(I&&!k){let e=!!T.routes[P],t=T.dynamicRoutes[N];if(t&&!1===t.fallback&&!e){if(C.adapterPath)return await $();throw new f.NoFallbackError}}let F=null;!I||E.isDev||k||(F="/index"===(F=P)?"/":F);let L=!0===E.isDev||!I,q=I&&!L;U&&x&&(0,o.setManifestsSingleton)({page:y,clientReferenceManifest:x,serverActionsManifest:U});let M=e.method||"GET",j=(0,i.getTracer)(),H=j.getActiveScopeSpan(),K=!!(null==_?void 0:_.isWrappedByNextServer),B=!!(0,n.getRequestMeta)(e,"minimalMode"),G=(0,n.getRequestMeta)(e,"incrementalCache")||await E.getIncrementalCache(e,C,T,B);null==G||G.resetRequestCache(),globalThis.__incrementalCache=G;let W={params:A,previewProps:T.preview,renderOpts:{experimental:{authInterrupts:!!C.experimental.authInterrupts},cacheComponents:!!C.cacheComponents,supportsDynamicResponse:L,incrementalCache:G,cacheLifeProfiles:C.cacheLife,waitUntil:a.waitUntil,onClose:e=>{t.on("close",e)},onAfterTaskError:void 0,onInstrumentationRequestError:(t,r,a,n)=>E.onRequestError(e,t,a,n,_)},sharedContext:{buildId:v,deploymentId:S}},J=new l.NodeNextRequest(e),V=new l.NodeNextResponse(t),X=c.NextRequestAdapter.fromNodeNextRequest(J,(0,c.signalFromNodeResponse)(t));try{let n,o=async e=>E.handle(X,W).finally(()=>{if(!e)return;e.setAttributes({"http.status_code":t.statusCode,"next.rsc":!1});let r=j.getRootSpanAttributes();if(!r)return;if(r.get("next.span_type")!==d.BaseServerSpan.handleRequest)return void console.warn(`Unexpected root span type '${r.get("next.span_type")}'. Please report this Next.js issue https://github.com/vercel/next.js`);let a=r.get("next.route");if(a){let t=`${M} ${a}`;e.setAttributes({"next.route":a,"http.route":a,"next.span_name":t}),e.updateName(t),n&&n!==e&&(n.setAttribute("http.route",a),n.updateName(t))}else e.updateName(`${M} ${y}`)}),s=async n=>{var i,s;let l=async({previousCacheEntry:r})=>{try{if(!B&&D&&O&&!r)return t.statusCode=404,t.setHeader("x-nextjs-cache","REVALIDATED"),t.end("This page could not be found"),null;let i=await o(n);e.fetchMetrics=W.renderOpts.fetchMetrics;let s=W.renderOpts.pendingWaitUntil;s&&a.waitUntil&&(a.waitUntil(s),s=void 0);let l=W.renderOpts.collectedTags;if(!I)return await (0,p.sendResponse)(J,V,i,W.renderOpts.pendingWaitUntil),null;{let e=await i.blob(),t=(0,h.toNodeOutgoingHttpHeaders)(i.headers);l&&(t[m.NEXT_CACHE_TAGS_HEADER]=l),!t["content-type"]&&e.type&&(t["content-type"]=e.type);let r=void 0!==W.renderOpts.collectedRevalidate&&!(W.renderOpts.collectedRevalidate>=m.INFINITE_CACHE)&&W.renderOpts.collectedRevalidate,a=void 0===W.renderOpts.collectedExpire||W.renderOpts.collectedExpire>=m.INFINITE_CACHE?void 0:W.renderOpts.collectedExpire;return{value:{kind:g.CachedRouteKind.APP_ROUTE,status:i.status,body:Buffer.from(await e.arrayBuffer()),headers:t},cacheControl:{revalidate:r,expire:a}}}}catch(t){throw(null==r?void 0:r.isStale)&&await E.onRequestError(e,t,{routerKind:"App Router",routePath:y,routeType:"route",revalidateReason:(0,u.getRevalidateReason)({isStaticGeneration:q,isOnDemandRevalidate:D})},!1,_),t}},c=await E.handleResponse({req:e,nextConfig:C,cacheKey:F,routeKind:r.RouteKind.APP_ROUTE,isFallback:!1,prerenderManifest:T,isRoutePPREnabled:!1,isOnDemandRevalidate:D,revalidateOnlyGenerated:O,responseGenerator:l,waitUntil:a.waitUntil,isMinimalMode:B});if(!I)return null;if((null==c||null==(i=c.value)?void 0:i.kind)!==g.CachedRouteKind.APP_ROUTE)throw Object.defineProperty(Error(`Invariant: app-route received invalid cache entry ${null==c||null==(s=c.value)?void 0:s.kind}`),"__NEXT_ERROR_CODE",{value:"E701",enumerable:!1,configurable:!0});B||t.setHeader("x-nextjs-cache",D?"REVALIDATED":c.isMiss?"MISS":c.isStale?"STALE":"HIT"),k&&t.setHeader("Cache-Control","private, no-cache, no-store, max-age=0, must-revalidate");let d=(0,h.fromNodeOutgoingHttpHeaders)(c.value.headers);return B&&I||d.delete(m.NEXT_CACHE_TAGS_HEADER),!c.cacheControl||t.getHeader("Cache-Control")||d.get("Cache-Control")||d.set("Cache-Control",(0,w.getCacheControlHeader)(c.cacheControl)),await (0,p.sendResponse)(J,V,new Response(c.value.body,{headers:d,status:c.value.status||200})),null};K&&H?await s(H):(n=j.getActiveScopeSpan(),await j.withPropagatedContext(e.headers,()=>j.trace(d.BaseServerSpan.handleRequest,{spanName:`${M} ${y}`,kind:i.SpanKind.SERVER,attributes:{"http.method":M,"http.target":e.url}},s),void 0,!K))}catch(t){if(t instanceof f.NoFallbackError||await E.onRequestError(e,t,{routerKind:"App Router",routePath:N,routeType:"route",revalidateReason:(0,u.getRevalidateReason)({isStaticGeneration:q,isOnDemandRevalidate:D})},!1,_),I)throw t;return await (0,p.sendResponse)(J,V,new Response(null,{status:500})),null}}e.s(["handler",0,A,"patchFetch",0,function(){return(0,a.patchFetch)({workAsyncStorage:R,workUnitAsyncStorage:v})},"routeModule",0,E,"serverHooks",0,S,"workAsyncStorage",0,R,"workUnitAsyncStorage",0,v])}];
-
-//# sourceMappingURL=%5Broot-of-the-server%5D__067hiof._.js.map

package/.next/server/chunks/[root-of-the-server]__0a_0ywc._.js

@@ -1,61 +0,0 @@
-module.exports=[367165,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(638725),n=e.i(192441);async function i(){let e=await (0,n.getDataDir)();if(!e)return a.DEFAULT_CONFIG;try{let n=await t.promises.readFile(r.default.join(e,"config.json"),"utf8"),i=JSON.parse(n),o=i.customEmbeddings,s=i.embeddingModelId;if(i.customEmbedding&&!o?.length){let e=i.customEmbedding;o=[e],"custom"===s&&(s=`custom:${e.modelName}`)}let l={...a.DEFAULT_CONFIG,...i,embeddingModelId:s??a.DEFAULT_CONFIG.embeddingModelId,customEmbeddings:o,chunking:{...a.DEFAULT_CONFIG.chunking,...i.chunking},storeConfig:{...a.DEFAULT_CONFIG.storeConfig,...i.storeConfig}};return delete l.customEmbedding,l}catch{return a.DEFAULT_CONFIG}}async function o(e){let a=await (0,n.requireDataDir)(),i={...e,updatedAt:new Date().toISOString()};return await t.promises.writeFile(r.default.join(a,"config.json"),JSON.stringify(i,null,2),"utf8"),i}e.s(["readConfig",0,i,"writeConfig",0,o])},874533,(e,t,r)=>{t.exports=e.x("node:child_process",()=>require("node:child_process"))},812057,(e,t,r)=>{t.exports=e.x("node:util",()=>require("node:util"))},705252,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(666680),n=e.i(192441);let i=Promise.resolve();function o(e){let t=i.then(e,e);return i=t.catch(()=>{}),t}async function s(e){let t=e?await (0,n.requireDataDir)():await (0,n.getDataDir)();return t?r.default.join(t,"documents.json"):null}async function l(){let e=await s(!1);if(!e)return[];try{let r=await t.promises.readFile(e,"utf8"),a=JSON.parse(r);if(!Array.isArray(a))throw Error(`Invalid document store at ${e}: expected a JSON array.`);return a}catch(e){if(e?.code==="ENOENT")return[];throw e}}async function c(e){let n=await s(!0);if(!n)return;let i=r.default.join(r.default.dirname(n),`.${r.default.basename(n)}.${process.pid}.${(0,a.randomUUID)()}.tmp`);try{await t.promises.writeFile(i,JSON.stringify(e,null,2),"utf8"),await t.promises.rename(i,n)}catch(e){throw await t.promises.unlink(i).catch(()=>{}),e}}function u(e){let t=e.content.trim();return{id:e.id??(0,a.randomUUID)(),title:e.title.trim()||"Untitled",url:e.url,source:e.source,content:t,charCount:t.length,jobId:e.jobId,metadata:e.metadata,status:"unindexed",createdAt:new Date().toISOString()}}async function d(){let e=await l(),t=e.reduce((e,t)=>e+t.charCount,0);return{docCount:e.length,charCount:t,bySource:e.reduce((e,t)=>(e[t.source]=(e[t.source]??0)+1,e),{})}}e.s(["addCrawledDocuments",0,function(e,t){return o(async()=>{let r=await l(),a=new Set(r.filter(e=>e.url).map(e=>e.url)),n=0;for(let i of t)!(i.url&&a.has(i.url))&&i.content.trim()&&(r.push(u({...i,jobId:e})),i.url&&a.add(i.url),n++);return n>0&&await c(r),n})},"addDocument",0,function(e){return o(async()=>{let t=await l(),r=u(e);return t.push(r),await c(t),r})},"addDocuments",0,function(e){return o(async()=>{if(0===e.length)return[];let t=await l(),r=e.map(u);return t.push(...r),await c(t),r})},"clearDocuments",0,function(){return o(async()=>{await c([])})},"corpusStats",0,d,"deleteDocument",0,function(e){return o(async()=>{let t=await l();await c(t.filter(t=>t.id!==e))})},"deleteDocuments",0,function(e){return o(async()=>{let t=await l(),r=new Set(e);await c(t.filter(e=>!r.has(e.id)))})},"readDocuments",0,l,"replaceDocuments",0,function(e){return o(async()=>{let t=new Date().toISOString();await c(e.map(e=>({...e,status:"unindexed",createdAt:e.createdAt||t})))})},"updateDocument",0,function(e,t){return o(async()=>{let r=await l(),a=r.findIndex(t=>t.id===e);if(a<0)return;let n=r[a],i=void 0!==t.content?t.content.trim():n.content,o={...n,title:void 0!==t.title?t.title.trim()||"Untitled":n.title,content:i,charCount:i.length,url:void 0!==t.url?t.url||void 0:n.url,metadata:void 0!==t.metadata?t.metadata:n.metadata,status:"unindexed"};return r[a]=o,await c(r),o})},"updateDocumentsStatus",0,function(e,t){return o(async()=>{let r=await l(),a=new Set(e),n=!1;for(let e of r)a.has(e.id)&&(e.status=t,n=!0);n&&await c(r)})}])},700259,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(761095),n=e.i(666680),i=e.i(874533),o=e.i(812057),s=e.i(367165);let l=(0,o.promisify)(i.exec);async function c(e,t){return l(e,{timeout:t,env:{...process.env,PATH:`${process.env.PATH||""}:/usr/local/bin:/opt/homebrew/bin:/bin:/usr/bin`}})}let u=r.default.join(process.cwd(),".larkup"),d=r.default.join(u,"firecrawl-local.json"),p=r.default.join(u,"firecrawl","docker-compose.yml"),w="ragtoolkit-firecrawl";async function f(e){return new Promise((t,r)=>{let n=a.default.createServer();n.listen(e,()=>{let e=n.address().port;n.close(()=>t(e))}),n.on("error",a=>{"EADDRINUSE"===a.code?f(0===e?0:e+1).then(t,r):r(a)})})}async function m(){try{let e=r.default.join(process.cwd(),"proxies.txt"),a=(await t.promises.readFile(e,"utf8")).split("\n").map(e=>e.trim()).filter(e=>e.length>0);if(a.length>0){let e=a[Math.floor(Math.random()*a.length)].split(":");if(e.length>=4)return{server:`http://${e[0]}:${e[1]}`,username:e[2],password:e[3]}}}catch(e){}let e=await (0,s.readConfig)();return e.useScraperProxy&&e.scraperProxyServer?{server:e.scraperProxyServer,username:e.scraperProxyUsername,password:e.scraperProxyPassword}:null}let h={running:!1,endpoint:"http://localhost:3002",apiKey:"",port:3002,project:w};async function g(){try{let e=await t.promises.readFile(d,"utf8");return{...h,...JSON.parse(e)}}catch{return h}}async function y(e){return await t.promises.mkdir(u,{recursive:!0}),await t.promises.writeFile(d,JSON.stringify(e,null,2),"utf8"),e}async function v(){try{await c("docker --version")}catch(e){return console.error("DOCKER ERR:",e),{docker:!1,compose:null}}try{return await c("docker compose version"),{docker:!0,compose:"docker compose"}}catch(e){console.error("DOCKER COMPOSE ERR:",e)}try{return await c("docker-compose version"),{docker:!0,compose:"docker-compose"}}catch{return{docker:!0,compose:null}}}async function E(){let{docker:e,compose:t}=await v();return e?t?{docker:!0,compose:!0,message:"Docker is ready."}:{docker:!0,compose:!1,message:"Docker is installed but the Compose plugin was not found. Install `docker compose` to launch Firecrawl locally."}:{docker:!1,compose:!1,message:"Docker isn't available on this machine. Install Docker Desktop (or the Docker engine) and make sure it's running."}}function S(e,t,r){let a=r?[`PROXY_SERVER: "${r.server}"`,...r.username&&r.password?[`PROXY_USERNAME: "${r.username}"`,`PROXY_PASSWORD: "${r.password}"`]:[]]:[],n=['PORT: "3000"',...a].map(e=>`      ${e}`).join("\n"),i=['PORT: "3002"',...a,'HOST: "0.0.0.0"','REDIS_URL: "redis://redis:6379"','REDIS_RATE_LIMIT_URL: "redis://redis:6379"','PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape"','USE_DB_AUTHENTICATION: "false"',`TEST_API_KEY: "${e}"`,`BULL_AUTH_KEY: "${e}"`,'DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_RABBITMQ_URL: "amqp://user:password@rabbitmq:5672"'].map(e=>`      ${e}`).join("\n");return`# Auto-generated by RAG Toolkit. Do not edit by hand.
-services:
-  db:
-    image: ghcr.io/firecrawl/nuq-postgres:latest
-    environment:
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U user -d postgres"]
-      interval: 5s
-      timeout: 5s
-      retries: 12
-    restart: unless-stopped
-
-  rabbitmq:
-    image: rabbitmq:3-management-alpine
-    environment:
-      RABBITMQ_DEFAULT_USER: user
-      RABBITMQ_DEFAULT_PASS: password
-    healthcheck:
-      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
-      interval: 5s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    restart: unless-stopped
-
-  redis:
-    image: redis:alpine
-    command: redis-server --save "" --appendonly no
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 5s
-      timeout: 3s
-      retries: 12
-    restart: unless-stopped
-
-  playwright-service:
-    image: ghcr.io/firecrawl/playwright-service:latest
-    environment:
-${n}
-    restart: unless-stopped
-
-  api:
-    image: ghcr.io/firecrawl/firecrawl:latest
-    depends_on:
-      db:
-        condition: service_healthy
-      rabbitmq:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "${t}:3002"
-    environment:
-${i}
-    restart: unless-stopped
-`}async function R(){let e=await E();if(!e.compose)return y({...await g(),running:!1,lastError:e.message});let{compose:a}=await v(),i=await g(),o=i.apiKey||`fc-local-${(0,n.randomUUID)()}`,s=i.port||3002,l=await m();await t.promises.mkdir(r.default.dirname(p),{recursive:!0}),await t.promises.writeFile(p,S(o,s,l),"utf8");try{await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(r){let e=r instanceof Error?r.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return y({...i,apiKey:o,port:s,running:!1,lastError:e});try{s=await f(s+1),await t.promises.writeFile(p,S(o,s,l),"utf8"),await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(e){return y({...i,apiKey:o,port:s,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let u=(0,t.existsSync)("/.dockerenv"),d=`http://${u?"host.docker.internal":"localhost"}:${s}`,h=await A(d,9e4);return y({running:h,endpoint:d,apiKey:o,port:s,project:w,startedAt:new Date().toISOString(),lastError:h?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function b(){let{compose:e}=await v(),t=await g();if(e)try{await c(`${e} -p ${w} -f "${p}" down`,6e4)}catch{}return y({...t,running:!1,startedAt:void 0})}async function C(){let e=await g();if(!e.startedAt)return e;let t=await D(e.endpoint);return t!==e.running?y({...e,running:t}):e}async function D(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function A(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(await D(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}let k="http://firecrawl-api:3002";async function T(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,t.existsSync)("/.dockerenv")?{available:await D(k),endpoint:k}:{available:!1,endpoint:""}}async function O(){let{available:e}=await T();return e?y({running:!0,endpoint:k,apiKey:`fc-docker-${(0,n.randomUUID)()}`,port:3002,project:"docker-sibling",startedAt:new Date().toISOString()}):y({...h,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,E,"checkDockerSibling",0,T,"connectDockerSibling",0,O,"readLocalState",0,g,"refreshLocalStatus",0,C,"startLocal",0,R,"stopLocal",0,b])},304368,e=>{"use strict";var t=e.i(367165),r=e.i(700259);class a extends Error{status;constructor(e,t){super(e),this.status=t,this.name="FirecrawlError"}}async function n(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();if(e.running&&e.apiKey)return{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new a("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let n=e.firecrawlApiKey;if(n)return{base:"https://api.firecrawl.dev/v1",key:n,mode:"cloud"};throw new a("The cloud web crawler needs an API key. Add it in Settings.",401)}async function i(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function o(e,t){let{base:r,key:i}=await n(),o=await fetch(`${r}${e}`,{...t,headers:{Authorization:`Bearer ${i}`,"Content-Type":"application/json",...t?.headers??{}},cache:"no-store"}),s=await o.text(),c=s?l(s):{};if(!o.ok)throw new a(c?.error||`Firecrawl request failed (${o.status})`,o.status);return c}async function s(e){let{key:t}=await n(),r=await fetch(e,{headers:{Authorization:`Bearer ${t}`},cache:"no-store"}),i=await r.text(),o=i?l(i):{};if(!r.ok)throw new a(o?.error||`Firecrawl request failed (${r.status})`,r.status);return o}function l(e){try{return JSON.parse(e)}catch{return{}}}async function c(e,t=10){return((await o("/search",{method:"POST",body:JSON.stringify({query:e,limit:t})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let u={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function d(e){let t=(await o("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...u})})).data??{};return{url:t.metadata?.sourceURL||e,title:t.metadata?.title||t.metadata?.ogTitle||e,markdown:t.markdown??""}}async function p(e,t){let r=await o("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:t,allowExternalLinks:!1,scrapeOptions:{...u,formats:["markdown"],onlyMainContent:!0}})});if(!r.id)throw new a("Firecrawl did not return a crawl id");return r.id}async function w(e,t=!1){let r=t?await s(e):await o(`/crawl/${e}`,{method:"GET"});return{state:r.status??"scraping",total:r.total??0,completed:r.completed??0,next:r.next,pages:(r.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function f(e){try{await o(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,a,"cancelCrawl",0,f,"getCrawlStatus",0,w,"isFirecrawlConfigured",0,i,"scrapePage",0,d,"searchWeb",0,c,"startCrawl",0,p])},798898,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(192441);let n=Promise.resolve();function i(e){let t=n.then(e,e);return n=t.catch(()=>{}),t}async function o(e){let t=e?await (0,a.requireDataDir)():await (0,a.getDataDir)();return t?r.default.join(t,"jobs.json"):null}async function s(){let e=await o(!1);if(!e)return[];try{let r=await t.promises.readFile(e,"utf8");return JSON.parse(r)}catch{return[]}}async function l(e){let r=await o(!0);r&&await t.promises.writeFile(r,JSON.stringify(e,null,2),"utf8")}async function c(e){return(await s()).find(t=>t.id===e)}e.s(["deleteJob",0,function(e){return i(async()=>{let t=await s();await l(t.filter(t=>t.id!==e))})},"getJob",0,c,"readJobs",0,s,"saveJob",0,function(e){return i(async()=>{let t=await s(),r=t.findIndex(t=>t.id===e.id),a={...e,updatedAt:new Date().toISOString()};return r>=0?t[r]=a:t.unshift(a),await l(t),a})},"updateJob",0,function(e,t){return i(async()=>{let r=await s(),a=r.findIndex(t=>t.id===e);if(a<0)return;let n={...t(r[a]),updatedAt:new Date().toISOString()};return r[a]=n,await l(r),n})}])},962044,e=>{"use strict";var t=e.i(239441),r=e.i(449055),a=e.i(391328),n=e.i(6352),i=e.i(746789),o=e.i(423437),s=e.i(178820),l=e.i(569191),c=e.i(576251),u=e.i(199962),d=e.i(547545),p=e.i(664046),w=e.i(235691),f=e.i(298907),m=e.i(110763),h=e.i(193695);e.i(673243);var g=e.i(236167),y=e.i(766576),v=e.i(798898),E=e.i(705252),S=e.i(304368);let R=new Set;function b(e){if(!e)return!1;let t=e.toLowerCase();return t.includes("verification successful. waiting for")||t.includes("please verify you are a human")||t.includes("checking if the site connection is secure")||t.includes("just a moment...")||t.includes("enable javascript and cookies to continue")}async function C(e,t){try{let r=await (0,S.scrapePage)(t.url);if(b(r.markdown))throw Error("Scraping blocked by anti-bot protection (e.g. Cloudflare)");return{added:await (0,E.addCrawledDocuments)(e,[{title:r.title,url:r.url,content:r.markdown,source:"scrape"}]),target:{...t,status:"completed",pagesCrawled:+!!r.markdown}}}catch(e){return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"scrape failed"}}}}async function D(e,t,r){try{if(!t.firecrawlId){let e=await (0,S.startCrawl)(t.url,r);return{added:0,target:{...t,firecrawlId:e,status:"running"}}}let a=t.firecrawlId,n=!1,i=0,o=0,s=t.pagesCrawled,l="running",c=[];for(;i<5;){let e=await (0,S.getCrawlStatus)(a,n);for(let t of(s=e.completed||s,e.pages))b(t.markdown)||c.push({title:t.title,url:t.url,content:t.markdown,source:"scrape"});if(i++,"completed"===e.state?l="completed":"failed"===e.state?l="failed":"cancelled"===e.state&&(l="cancelled"),e.next)a=e.next,n=!0;else break}return c.length&&(o=await (0,E.addCrawledDocuments)(e,c)),{added:o,target:{...t,status:l,pagesCrawled:s}}}catch(e){return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"crawl failed"}}}}async function A(e){let t=await (0,v.getJob)(e);if(t){if(["completed","failed","cancelled"].includes(t.status)||R.has(e))return t;R.add(e);try{let r=await Promise.all(t.targets.map(r=>"completed"===r.status||"failed"===r.status?Promise.resolve({target:r,added:0}):"page"===r.scope?C(e,r):D(e,r,t.pageLimit))),a=r.map(e=>e.target),n=a.reduce((e,t)=>e+t.pagesCrawled,0);return await (0,v.updateJob)(e,e=>({...e,targets:a,pagesCrawled:n,docCount:e.docCount+r.reduce((e,t)=>e+t.added,0),status:a.every(e=>"completed"===e.status)?"completed":a.every(e=>"failed"===e.status)?"failed":a.every(e=>"completed"===e.status||"failed"===e.status)&&a.some(e=>"completed"===e.status)?"completed":"running"}))}finally{R.delete(e)}}}async function k(e){let t=await (0,v.getJob)(e);if(t)return await Promise.all(t.targets.filter(e=>e.firecrawlId&&"running"===e.status).map(e=>(0,S.cancelCrawl)(e.firecrawlId))),(0,v.updateJob)(e,e=>({...e,status:"cancelled",targets:e.targets.map(e=>"running"===e.status||"queued"===e.status?{...e,status:"cancelled"}:e)}))}async function T(e,{params:t}){let{id:r}=await t,a=await A(r);return a?y.NextResponse.json({job:a}):y.NextResponse.json({error:"Job not found."},{status:404})}async function O(e,{params:t}){let{id:r}=await t;if(new URL(e.url).searchParams.get("remove"))return await (0,v.deleteJob)(r),y.NextResponse.json({ok:!0});if(!await (0,v.getJob)(r))return y.NextResponse.json({error:"Job not found."},{status:404});let a=await k(r);return y.NextResponse.json({job:a})}e.s(["DELETE",0,O,"GET",0,T,"dynamic",0,"force-dynamic","runtime",0,"nodejs"],934385);var I=e.i(934385);let x=new t.AppRouteRouteModule({definition:{kind:r.RouteKind.APP_ROUTE,page:"/api/jobs/[id]/route",pathname:"/api/jobs/[id]",filename:"route",bundlePath:""},distDir:".next",relativeProjectDir:"",resolvedPagePath:"[project]/apps/web/app/api/jobs/[id]/route.ts",nextConfigOutput:"standalone",userland:I,...{}}),{workAsyncStorage:_,workUnitAsyncStorage:P,serverHooks:U}=x;async function N(e,t,a){a.requestMeta&&(0,n.setRequestMeta)(e,a.requestMeta),x.isDev&&(0,n.addRequestMeta)(e,"devRequestTimingInternalsEnd",process.hrtime.bigint());let y="/api/jobs/[id]/route";y=y.replace(/\/index$/,"")||"/";let v=await x.prepare(e,t,{srcPage:y,multiZoneDraftMode:!1});if(!v)return t.statusCode=400,t.end("Bad Request"),null==a.waitUntil||a.waitUntil.call(a,Promise.resolve()),null;let{buildId:E,deploymentId:S,params:R,nextConfig:b,parsedUrl:C,isDraftMode:D,prerenderManifest:A,routerServerContext:k,isOnDemandRevalidate:T,revalidateOnlyGenerated:O,resolvedPathname:I,clientReferenceManifest:_,serverActionsManifest:P}=v,U=(0,s.normalizeAppPath)(y),N=!!(A.dynamicRoutes[U]||A.routes[I]),$=async()=>((null==k?void 0:k.render404)?await k.render404(e,t,C,!1):t.end("This page could not be found"),null);if(N&&!D){let e=!!A.routes[I],t=A.dynamicRoutes[U];if(t&&!1===t.fallback&&!e){if(b.adapterPath)return await $();throw new h.NoFallbackError}}let L=null;!N||x.isDev||D||(L="/index"===(L=I)?"/":L);let j=!0===x.isDev||!N,F=N&&!j;P&&_&&(0,o.setManifestsSingleton)({page:y,clientReferenceManifest:_,serverActionsManifest:P});let q=e.method||"GET",M=(0,i.getTracer)(),J=M.getActiveScopeSpan(),H=!!(null==k?void 0:k.isWrappedByNextServer),K=!!(0,n.getRequestMeta)(e,"minimalMode"),B=(0,n.getRequestMeta)(e,"incrementalCache")||await x.getIncrementalCache(e,b,A,K);null==B||B.resetRequestCache(),globalThis.__incrementalCache=B;let G={params:R,previewProps:A.preview,renderOpts:{experimental:{authInterrupts:!!b.experimental.authInterrupts},cacheComponents:!!b.cacheComponents,supportsDynamicResponse:j,incrementalCache:B,cacheLifeProfiles:b.cacheLife,waitUntil:a.waitUntil,onClose:e=>{t.on("close",e)},onAfterTaskError:void 0,onInstrumentationRequestError:(t,r,a,n)=>x.onRequestError(e,t,a,n,k)},sharedContext:{buildId:E,deploymentId:S}},W=new l.NodeNextRequest(e),V=new l.NodeNextResponse(t),X=c.NextRequestAdapter.fromNodeNextRequest(W,(0,c.signalFromNodeResponse)(t));try{let n,o=async e=>x.handle(X,G).finally(()=>{if(!e)return;e.setAttributes({"http.status_code":t.statusCode,"next.rsc":!1});let r=M.getRootSpanAttributes();if(!r)return;if(r.get("next.span_type")!==u.BaseServerSpan.handleRequest)return void console.warn(`Unexpected root span type '${r.get("next.span_type")}'. Please report this Next.js issue https://github.com/vercel/next.js`);let a=r.get("next.route");if(a){let t=`${q} ${a}`;e.setAttributes({"next.route":a,"http.route":a,"next.span_name":t}),e.updateName(t),n&&n!==e&&(n.setAttribute("http.route",a),n.updateName(t))}else e.updateName(`${q} ${y}`)}),s=async n=>{var i,s;let l=async({previousCacheEntry:r})=>{try{if(!K&&T&&O&&!r)return t.statusCode=404,t.setHeader("x-nextjs-cache","REVALIDATED"),t.end("This page could not be found"),null;let i=await o(n);e.fetchMetrics=G.renderOpts.fetchMetrics;let s=G.renderOpts.pendingWaitUntil;s&&a.waitUntil&&(a.waitUntil(s),s=void 0);let l=G.renderOpts.collectedTags;if(!N)return await (0,p.sendResponse)(W,V,i,G.renderOpts.pendingWaitUntil),null;{let e=await i.blob(),t=(0,w.toNodeOutgoingHttpHeaders)(i.headers);l&&(t[m.NEXT_CACHE_TAGS_HEADER]=l),!t["content-type"]&&e.type&&(t["content-type"]=e.type);let r=void 0!==G.renderOpts.collectedRevalidate&&!(G.renderOpts.collectedRevalidate>=m.INFINITE_CACHE)&&G.renderOpts.collectedRevalidate,a=void 0===G.renderOpts.collectedExpire||G.renderOpts.collectedExpire>=m.INFINITE_CACHE?void 0:G.renderOpts.collectedExpire;return{value:{kind:g.CachedRouteKind.APP_ROUTE,status:i.status,body:Buffer.from(await e.arrayBuffer()),headers:t},cacheControl:{revalidate:r,expire:a}}}}catch(t){throw(null==r?void 0:r.isStale)&&await x.onRequestError(e,t,{routerKind:"App Router",routePath:y,routeType:"route",revalidateReason:(0,d.getRevalidateReason)({isStaticGeneration:F,isOnDemandRevalidate:T})},!1,k),t}},c=await x.handleResponse({req:e,nextConfig:b,cacheKey:L,routeKind:r.RouteKind.APP_ROUTE,isFallback:!1,prerenderManifest:A,isRoutePPREnabled:!1,isOnDemandRevalidate:T,revalidateOnlyGenerated:O,responseGenerator:l,waitUntil:a.waitUntil,isMinimalMode:K});if(!N)return null;if((null==c||null==(i=c.value)?void 0:i.kind)!==g.CachedRouteKind.APP_ROUTE)throw Object.defineProperty(Error(`Invariant: app-route received invalid cache entry ${null==c||null==(s=c.value)?void 0:s.kind}`),"__NEXT_ERROR_CODE",{value:"E701",enumerable:!1,configurable:!0});K||t.setHeader("x-nextjs-cache",T?"REVALIDATED":c.isMiss?"MISS":c.isStale?"STALE":"HIT"),D&&t.setHeader("Cache-Control","private, no-cache, no-store, max-age=0, must-revalidate");let u=(0,w.fromNodeOutgoingHttpHeaders)(c.value.headers);return K&&N||u.delete(m.NEXT_CACHE_TAGS_HEADER),!c.cacheControl||t.getHeader("Cache-Control")||u.get("Cache-Control")||u.set("Cache-Control",(0,f.getCacheControlHeader)(c.cacheControl)),await (0,p.sendResponse)(W,V,new Response(c.value.body,{headers:u,status:c.value.status||200})),null};H&&J?await s(J):(n=M.getActiveScopeSpan(),await M.withPropagatedContext(e.headers,()=>M.trace(u.BaseServerSpan.handleRequest,{spanName:`${q} ${y}`,kind:i.SpanKind.SERVER,attributes:{"http.method":q,"http.target":e.url}},s),void 0,!H))}catch(t){if(t instanceof h.NoFallbackError||await x.onRequestError(e,t,{routerKind:"App Router",routePath:U,routeType:"route",revalidateReason:(0,d.getRevalidateReason)({isStaticGeneration:F,isOnDemandRevalidate:T})},!1,k),N)throw t;return await (0,p.sendResponse)(W,V,new Response(null,{status:500})),null}}e.s(["handler",0,N,"patchFetch",0,function(){return(0,a.patchFetch)({workAsyncStorage:_,workUnitAsyncStorage:P})},"routeModule",0,x,"serverHooks",0,U,"workAsyncStorage",0,_,"workUnitAsyncStorage",0,P],962044)}];
-
-//# sourceMappingURL=%5Broot-of-the-server%5D__0a_0ywc._.js.map

package/.next/server/chunks/[root-of-the-server]__0u17zp8._.js

@@ -1,61 +0,0 @@
-module.exports=[367165,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(638725),n=e.i(192441);async function i(){let e=await (0,n.getDataDir)();if(!e)return a.DEFAULT_CONFIG;try{let n=await t.promises.readFile(r.default.join(e,"config.json"),"utf8"),i=JSON.parse(n),o=i.customEmbeddings,s=i.embeddingModelId;if(i.customEmbedding&&!o?.length){let e=i.customEmbedding;o=[e],"custom"===s&&(s=`custom:${e.modelName}`)}let l={...a.DEFAULT_CONFIG,...i,embeddingModelId:s??a.DEFAULT_CONFIG.embeddingModelId,customEmbeddings:o,chunking:{...a.DEFAULT_CONFIG.chunking,...i.chunking},storeConfig:{...a.DEFAULT_CONFIG.storeConfig,...i.storeConfig}};return delete l.customEmbedding,l}catch{return a.DEFAULT_CONFIG}}async function o(e){let a=await (0,n.requireDataDir)(),i={...e,updatedAt:new Date().toISOString()};return await t.promises.writeFile(r.default.join(a,"config.json"),JSON.stringify(i,null,2),"utf8"),i}e.s(["readConfig",0,i,"writeConfig",0,o])},874533,(e,t,r)=>{t.exports=e.x("node:child_process",()=>require("node:child_process"))},812057,(e,t,r)=>{t.exports=e.x("node:util",()=>require("node:util"))},705252,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(666680),n=e.i(192441);let i=Promise.resolve();function o(e){let t=i.then(e,e);return i=t.catch(()=>{}),t}async function s(e){let t=e?await (0,n.requireDataDir)():await (0,n.getDataDir)();return t?r.default.join(t,"documents.json"):null}async function l(){let e=await s(!1);if(!e)return[];try{let r=await t.promises.readFile(e,"utf8"),a=JSON.parse(r);if(!Array.isArray(a))throw Error(`Invalid document store at ${e}: expected a JSON array.`);return a}catch(e){if(e?.code==="ENOENT")return[];throw e}}async function c(e){let n=await s(!0);if(!n)return;let i=r.default.join(r.default.dirname(n),`.${r.default.basename(n)}.${process.pid}.${(0,a.randomUUID)()}.tmp`);try{await t.promises.writeFile(i,JSON.stringify(e,null,2),"utf8"),await t.promises.rename(i,n)}catch(e){throw await t.promises.unlink(i).catch(()=>{}),e}}function u(e){let t=e.content.trim();return{id:e.id??(0,a.randomUUID)(),title:e.title.trim()||"Untitled",url:e.url,source:e.source,content:t,charCount:t.length,jobId:e.jobId,metadata:e.metadata,status:"unindexed",createdAt:new Date().toISOString()}}async function d(){let e=await l(),t=e.reduce((e,t)=>e+t.charCount,0);return{docCount:e.length,charCount:t,bySource:e.reduce((e,t)=>(e[t.source]=(e[t.source]??0)+1,e),{})}}e.s(["addCrawledDocuments",0,function(e,t){return o(async()=>{let r=await l(),a=new Set(r.filter(e=>e.url).map(e=>e.url)),n=0;for(let i of t)!(i.url&&a.has(i.url))&&i.content.trim()&&(r.push(u({...i,jobId:e})),i.url&&a.add(i.url),n++);return n>0&&await c(r),n})},"addDocument",0,function(e){return o(async()=>{let t=await l(),r=u(e);return t.push(r),await c(t),r})},"addDocuments",0,function(e){return o(async()=>{if(0===e.length)return[];let t=await l(),r=e.map(u);return t.push(...r),await c(t),r})},"clearDocuments",0,function(){return o(async()=>{await c([])})},"corpusStats",0,d,"deleteDocument",0,function(e){return o(async()=>{let t=await l();await c(t.filter(t=>t.id!==e))})},"deleteDocuments",0,function(e){return o(async()=>{let t=await l(),r=new Set(e);await c(t.filter(e=>!r.has(e.id)))})},"readDocuments",0,l,"replaceDocuments",0,function(e){return o(async()=>{let t=new Date().toISOString();await c(e.map(e=>({...e,status:"unindexed",createdAt:e.createdAt||t})))})},"updateDocument",0,function(e,t){return o(async()=>{let r=await l(),a=r.findIndex(t=>t.id===e);if(a<0)return;let n=r[a],i=void 0!==t.content?t.content.trim():n.content,o={...n,title:void 0!==t.title?t.title.trim()||"Untitled":n.title,content:i,charCount:i.length,url:void 0!==t.url?t.url||void 0:n.url,metadata:void 0!==t.metadata?t.metadata:n.metadata,status:"unindexed"};return r[a]=o,await c(r),o})},"updateDocumentsStatus",0,function(e,t){return o(async()=>{let r=await l(),a=new Set(e),n=!1;for(let e of r)a.has(e.id)&&(e.status=t,n=!0);n&&await c(r)})}])},700259,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(761095),n=e.i(666680),i=e.i(874533),o=e.i(812057),s=e.i(367165);let l=(0,o.promisify)(i.exec);async function c(e,t){return l(e,{timeout:t,env:{...process.env,PATH:`${process.env.PATH||""}:/usr/local/bin:/opt/homebrew/bin:/bin:/usr/bin`}})}let u=r.default.join(process.cwd(),".larkup"),d=r.default.join(u,"firecrawl-local.json"),p=r.default.join(u,"firecrawl","docker-compose.yml"),w="ragtoolkit-firecrawl";async function m(e){return new Promise((t,r)=>{let n=a.default.createServer();n.listen(e,()=>{let e=n.address().port;n.close(()=>t(e))}),n.on("error",a=>{"EADDRINUSE"===a.code?m(0===e?0:e+1).then(t,r):r(a)})})}async function f(){try{let e=r.default.join(process.cwd(),"proxies.txt"),a=(await t.promises.readFile(e,"utf8")).split("\n").map(e=>e.trim()).filter(e=>e.length>0);if(a.length>0){let e=a[Math.floor(Math.random()*a.length)].split(":");if(e.length>=4)return{server:`http://${e[0]}:${e[1]}`,username:e[2],password:e[3]}}}catch(e){}let e=await (0,s.readConfig)();return e.useScraperProxy&&e.scraperProxyServer?{server:e.scraperProxyServer,username:e.scraperProxyUsername,password:e.scraperProxyPassword}:null}let h={running:!1,endpoint:"http://localhost:3002",apiKey:"",port:3002,project:w};async function g(){try{let e=await t.promises.readFile(d,"utf8");return{...h,...JSON.parse(e)}}catch{return h}}async function y(e){return await t.promises.mkdir(u,{recursive:!0}),await t.promises.writeFile(d,JSON.stringify(e,null,2),"utf8"),e}async function v(){try{await c("docker --version")}catch(e){return console.error("DOCKER ERR:",e),{docker:!1,compose:null}}try{return await c("docker compose version"),{docker:!0,compose:"docker compose"}}catch(e){console.error("DOCKER COMPOSE ERR:",e)}try{return await c("docker-compose version"),{docker:!0,compose:"docker-compose"}}catch{return{docker:!0,compose:null}}}async function S(){let{docker:e,compose:t}=await v();return e?t?{docker:!0,compose:!0,message:"Docker is ready."}:{docker:!0,compose:!1,message:"Docker is installed but the Compose plugin was not found. Install `docker compose` to launch Firecrawl locally."}:{docker:!1,compose:!1,message:"Docker isn't available on this machine. Install Docker Desktop (or the Docker engine) and make sure it's running."}}function R(e,t,r){let a=r?[`PROXY_SERVER: "${r.server}"`,...r.username&&r.password?[`PROXY_USERNAME: "${r.username}"`,`PROXY_PASSWORD: "${r.password}"`]:[]]:[],n=['PORT: "3000"',...a].map(e=>`      ${e}`).join("\n"),i=['PORT: "3002"',...a,'HOST: "0.0.0.0"','REDIS_URL: "redis://redis:6379"','REDIS_RATE_LIMIT_URL: "redis://redis:6379"','PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape"','USE_DB_AUTHENTICATION: "false"',`TEST_API_KEY: "${e}"`,`BULL_AUTH_KEY: "${e}"`,'DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_RABBITMQ_URL: "amqp://user:password@rabbitmq:5672"'].map(e=>`      ${e}`).join("\n");return`# Auto-generated by RAG Toolkit. Do not edit by hand.
-services:
-  db:
-    image: ghcr.io/firecrawl/nuq-postgres:latest
-    environment:
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U user -d postgres"]
-      interval: 5s
-      timeout: 5s
-      retries: 12
-    restart: unless-stopped
-
-  rabbitmq:
-    image: rabbitmq:3-management-alpine
-    environment:
-      RABBITMQ_DEFAULT_USER: user
-      RABBITMQ_DEFAULT_PASS: password
-    healthcheck:
-      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
-      interval: 5s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    restart: unless-stopped
-
-  redis:
-    image: redis:alpine
-    command: redis-server --save "" --appendonly no
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 5s
-      timeout: 3s
-      retries: 12
-    restart: unless-stopped
-
-  playwright-service:
-    image: ghcr.io/firecrawl/playwright-service:latest
-    environment:
-${n}
-    restart: unless-stopped
-
-  api:
-    image: ghcr.io/firecrawl/firecrawl:latest
-    depends_on:
-      db:
-        condition: service_healthy
-      rabbitmq:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "${t}:3002"
-    environment:
-${i}
-    restart: unless-stopped
-`}async function E(){let e=await S();if(!e.compose)return y({...await g(),running:!1,lastError:e.message});let{compose:a}=await v(),i=await g(),o=i.apiKey||`fc-local-${(0,n.randomUUID)()}`,s=i.port||3002,l=await f();await t.promises.mkdir(r.default.dirname(p),{recursive:!0}),await t.promises.writeFile(p,R(o,s,l),"utf8");try{await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(r){let e=r instanceof Error?r.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return y({...i,apiKey:o,port:s,running:!1,lastError:e});try{s=await m(s+1),await t.promises.writeFile(p,R(o,s,l),"utf8"),await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(e){return y({...i,apiKey:o,port:s,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let u=(0,t.existsSync)("/.dockerenv"),d=`http://${u?"host.docker.internal":"localhost"}:${s}`,h=await D(d,9e4);return y({running:h,endpoint:d,apiKey:o,port:s,project:w,startedAt:new Date().toISOString(),lastError:h?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function b(){let{compose:e}=await v(),t=await g();if(e)try{await c(`${e} -p ${w} -f "${p}" down`,6e4)}catch{}return y({...t,running:!1,startedAt:void 0})}async function C(){let e=await g();if(!e.startedAt)return e;let t=await A(e.endpoint);return t!==e.running?y({...e,running:t}):e}async function A(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function D(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(await A(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}let k="http://firecrawl-api:3002";async function O(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,t.existsSync)("/.dockerenv")?{available:await A(k),endpoint:k}:{available:!1,endpoint:""}}async function T(){let{available:e}=await O();return e?y({running:!0,endpoint:k,apiKey:`fc-docker-${(0,n.randomUUID)()}`,port:3002,project:"docker-sibling",startedAt:new Date().toISOString()}):y({...h,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,S,"checkDockerSibling",0,O,"connectDockerSibling",0,T,"readLocalState",0,g,"refreshLocalStatus",0,C,"startLocal",0,E,"stopLocal",0,b])},304368,e=>{"use strict";var t=e.i(367165),r=e.i(700259);class a extends Error{status;constructor(e,t){super(e),this.status=t,this.name="FirecrawlError"}}async function n(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();if(e.running&&e.apiKey)return{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new a("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let n=e.firecrawlApiKey;if(n)return{base:"https://api.firecrawl.dev/v1",key:n,mode:"cloud"};throw new a("The cloud web crawler needs an API key. Add it in Settings.",401)}async function i(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function o(e,t){let{base:r,key:i}=await n(),o=await fetch(`${r}${e}`,{...t,headers:{Authorization:`Bearer ${i}`,"Content-Type":"application/json",...t?.headers??{}},cache:"no-store"}),s=await o.text(),c=s?l(s):{};if(!o.ok)throw new a(c?.error||`Firecrawl request failed (${o.status})`,o.status);return c}async function s(e){let{key:t}=await n(),r=await fetch(e,{headers:{Authorization:`Bearer ${t}`},cache:"no-store"}),i=await r.text(),o=i?l(i):{};if(!r.ok)throw new a(o?.error||`Firecrawl request failed (${r.status})`,r.status);return o}function l(e){try{return JSON.parse(e)}catch{return{}}}async function c(e,t=10){return((await o("/search",{method:"POST",body:JSON.stringify({query:e,limit:t})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let u={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function d(e){let t=(await o("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...u})})).data??{};return{url:t.metadata?.sourceURL||e,title:t.metadata?.title||t.metadata?.ogTitle||e,markdown:t.markdown??""}}async function p(e,t){let r=await o("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:t,allowExternalLinks:!1,scrapeOptions:{...u,formats:["markdown"],onlyMainContent:!0}})});if(!r.id)throw new a("Firecrawl did not return a crawl id");return r.id}async function w(e,t=!1){let r=t?await s(e):await o(`/crawl/${e}`,{method:"GET"});return{state:r.status??"scraping",total:r.total??0,completed:r.completed??0,next:r.next,pages:(r.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function m(e){try{await o(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,a,"cancelCrawl",0,m,"getCrawlStatus",0,w,"isFirecrawlConfigured",0,i,"scrapePage",0,d,"searchWeb",0,c,"startCrawl",0,p])},798898,e=>{"use strict";var t=e.i(902157),r=e.i(750227),a=e.i(192441);let n=Promise.resolve();function i(e){let t=n.then(e,e);return n=t.catch(()=>{}),t}async function o(e){let t=e?await (0,a.requireDataDir)():await (0,a.getDataDir)();return t?r.default.join(t,"jobs.json"):null}async function s(){let e=await o(!1);if(!e)return[];try{let r=await t.promises.readFile(e,"utf8");return JSON.parse(r)}catch{return[]}}async function l(e){let r=await o(!0);r&&await t.promises.writeFile(r,JSON.stringify(e,null,2),"utf8")}async function c(e){return(await s()).find(t=>t.id===e)}e.s(["deleteJob",0,function(e){return i(async()=>{let t=await s();await l(t.filter(t=>t.id!==e))})},"getJob",0,c,"readJobs",0,s,"saveJob",0,function(e){return i(async()=>{let t=await s(),r=t.findIndex(t=>t.id===e.id),a={...e,updatedAt:new Date().toISOString()};return r>=0?t[r]=a:t.unshift(a),await l(t),a})},"updateJob",0,function(e,t){return i(async()=>{let r=await s(),a=r.findIndex(t=>t.id===e);if(a<0)return;let n={...t(r[a]),updatedAt:new Date().toISOString()};return r[a]=n,await l(r),n})}])},491382,e=>{"use strict";var t=e.i(239441),r=e.i(449055),a=e.i(391328),n=e.i(6352),i=e.i(746789),o=e.i(423437),s=e.i(178820),l=e.i(569191),c=e.i(576251),u=e.i(199962),d=e.i(547545),p=e.i(664046),w=e.i(235691),m=e.i(298907),f=e.i(110763),h=e.i(193695);e.i(673243);var g=e.i(236167),y=e.i(766576),v=e.i(666680),S=e.i(798898),R=e.i(304368),E=e.i(705252);async function b(){let e=await (0,S.readJobs)();return y.NextResponse.json({jobs:e,configured:await (0,R.isFirecrawlConfigured)()})}async function C(e){let t;if(!await (0,R.isFirecrawlConfigured)())return y.NextResponse.json({error:"No web crawler is available. Choose and configure a crawler provider in Settings."},{status:401});try{t=await e.json()}catch{return y.NextResponse.json({error:"Invalid request body."},{status:400})}let r=new Set((await (0,E.readDocuments)()).filter(e=>e.url).map(e=>e.url)),a=new Map,n=new Map;(t.targets??[]).forEach(e=>{if(e.url&&/^https?:\/\//i.test(e.url))try{let t=new URL(e.url).origin;r.has(t)||r.has(t+"/")||a.set(t,{url:t,scope:"domain"===e.scope?"domain":"page",status:"queued",pagesCrawled:0}),e.url===t||e.url===t+"/"||r.has(e.url)||n.set(e.url,{url:e.url,scope:"domain"===e.scope?"domain":"page",status:"queued",pagesCrawled:0})}catch{r.has(e.url)||n.set(e.url,{url:e.url,scope:"domain"===e.scope?"domain":"page",status:"queued",pagesCrawled:0})}});let i=[...Array.from(a.values()),...Array.from(n.values())];if(0===i.length)return y.NextResponse.json({error:"Select at least one valid new URL to scrape."},{status:400});let o={id:(0,v.randomUUID)(),keywords:t.keywords?.trim()||i[0].url,targets:i,status:"queued",pageLimit:Math.min(Math.max(t.pageLimit??2e3,1),2e3),pagesCrawled:0,docCount:0,createdAt:new Date().toISOString(),updatedAt:new Date().toISOString()};return await (0,S.saveJob)(o),y.NextResponse.json({job:o},{status:201})}e.s(["GET",0,b,"POST",0,C,"dynamic",0,"force-dynamic","runtime",0,"nodejs"],889376);var A=e.i(889376);let D=new t.AppRouteRouteModule({definition:{kind:r.RouteKind.APP_ROUTE,page:"/api/jobs/route",pathname:"/api/jobs",filename:"route",bundlePath:""},distDir:".next",relativeProjectDir:"",resolvedPagePath:"[project]/apps/web/app/api/jobs/route.ts",nextConfigOutput:"standalone",userland:A,...{}}),{workAsyncStorage:k,workUnitAsyncStorage:O,serverHooks:T}=D;async function I(e,t,a){a.requestMeta&&(0,n.setRequestMeta)(e,a.requestMeta),D.isDev&&(0,n.addRequestMeta)(e,"devRequestTimingInternalsEnd",process.hrtime.bigint());let y="/api/jobs/route";y=y.replace(/\/index$/,"")||"/";let v=await D.prepare(e,t,{srcPage:y,multiZoneDraftMode:!1});if(!v)return t.statusCode=400,t.end("Bad Request"),null==a.waitUntil||a.waitUntil.call(a,Promise.resolve()),null;let{buildId:S,deploymentId:R,params:E,nextConfig:b,parsedUrl:C,isDraftMode:A,prerenderManifest:k,routerServerContext:O,isOnDemandRevalidate:T,revalidateOnlyGenerated:I,resolvedPathname:U,clientReferenceManifest:_,serverActionsManifest:x}=v,N=(0,s.normalizeAppPath)(y),P=!!(k.dynamicRoutes[N]||k.routes[U]),$=async()=>((null==O?void 0:O.render404)?await O.render404(e,t,C,!1):t.end("This page could not be found"),null);if(P&&!A){let e=!!k.routes[U],t=k.dynamicRoutes[N];if(t&&!1===t.fallback&&!e){if(b.adapterPath)return await $();throw new h.NoFallbackError}}let F=null;!P||D.isDev||A||(F="/index"===(F=U)?"/":F);let L=!0===D.isDev||!P,j=P&&!L;x&&_&&(0,o.setManifestsSingleton)({page:y,clientReferenceManifest:_,serverActionsManifest:x});let q=e.method||"GET",M=(0,i.getTracer)(),H=M.getActiveScopeSpan(),K=!!(null==O?void 0:O.isWrappedByNextServer),B=!!(0,n.getRequestMeta)(e,"minimalMode"),J=(0,n.getRequestMeta)(e,"incrementalCache")||await D.getIncrementalCache(e,b,k,B);null==J||J.resetRequestCache(),globalThis.__incrementalCache=J;let G={params:E,previewProps:k.preview,renderOpts:{experimental:{authInterrupts:!!b.experimental.authInterrupts},cacheComponents:!!b.cacheComponents,supportsDynamicResponse:L,incrementalCache:J,cacheLifeProfiles:b.cacheLife,waitUntil:a.waitUntil,onClose:e=>{t.on("close",e)},onAfterTaskError:void 0,onInstrumentationRequestError:(t,r,a,n)=>D.onRequestError(e,t,a,n,O)},sharedContext:{buildId:S,deploymentId:R}},W=new l.NodeNextRequest(e),V=new l.NodeNextResponse(t),X=c.NextRequestAdapter.fromNodeNextRequest(W,(0,c.signalFromNodeResponse)(t));try{let n,o=async e=>D.handle(X,G).finally(()=>{if(!e)return;e.setAttributes({"http.status_code":t.statusCode,"next.rsc":!1});let r=M.getRootSpanAttributes();if(!r)return;if(r.get("next.span_type")!==u.BaseServerSpan.handleRequest)return void console.warn(`Unexpected root span type '${r.get("next.span_type")}'. Please report this Next.js issue https://github.com/vercel/next.js`);let a=r.get("next.route");if(a){let t=`${q} ${a}`;e.setAttributes({"next.route":a,"http.route":a,"next.span_name":t}),e.updateName(t),n&&n!==e&&(n.setAttribute("http.route",a),n.updateName(t))}else e.updateName(`${q} ${y}`)}),s=async n=>{var i,s;let l=async({previousCacheEntry:r})=>{try{if(!B&&T&&I&&!r)return t.statusCode=404,t.setHeader("x-nextjs-cache","REVALIDATED"),t.end("This page could not be found"),null;let i=await o(n);e.fetchMetrics=G.renderOpts.fetchMetrics;let s=G.renderOpts.pendingWaitUntil;s&&a.waitUntil&&(a.waitUntil(s),s=void 0);let l=G.renderOpts.collectedTags;if(!P)return await (0,p.sendResponse)(W,V,i,G.renderOpts.pendingWaitUntil),null;{let e=await i.blob(),t=(0,w.toNodeOutgoingHttpHeaders)(i.headers);l&&(t[f.NEXT_CACHE_TAGS_HEADER]=l),!t["content-type"]&&e.type&&(t["content-type"]=e.type);let r=void 0!==G.renderOpts.collectedRevalidate&&!(G.renderOpts.collectedRevalidate>=f.INFINITE_CACHE)&&G.renderOpts.collectedRevalidate,a=void 0===G.renderOpts.collectedExpire||G.renderOpts.collectedExpire>=f.INFINITE_CACHE?void 0:G.renderOpts.collectedExpire;return{value:{kind:g.CachedRouteKind.APP_ROUTE,status:i.status,body:Buffer.from(await e.arrayBuffer()),headers:t},cacheControl:{revalidate:r,expire:a}}}}catch(t){throw(null==r?void 0:r.isStale)&&await D.onRequestError(e,t,{routerKind:"App Router",routePath:y,routeType:"route",revalidateReason:(0,d.getRevalidateReason)({isStaticGeneration:j,isOnDemandRevalidate:T})},!1,O),t}},c=await D.handleResponse({req:e,nextConfig:b,cacheKey:F,routeKind:r.RouteKind.APP_ROUTE,isFallback:!1,prerenderManifest:k,isRoutePPREnabled:!1,isOnDemandRevalidate:T,revalidateOnlyGenerated:I,responseGenerator:l,waitUntil:a.waitUntil,isMinimalMode:B});if(!P)return null;if((null==c||null==(i=c.value)?void 0:i.kind)!==g.CachedRouteKind.APP_ROUTE)throw Object.defineProperty(Error(`Invariant: app-route received invalid cache entry ${null==c||null==(s=c.value)?void 0:s.kind}`),"__NEXT_ERROR_CODE",{value:"E701",enumerable:!1,configurable:!0});B||t.setHeader("x-nextjs-cache",T?"REVALIDATED":c.isMiss?"MISS":c.isStale?"STALE":"HIT"),A&&t.setHeader("Cache-Control","private, no-cache, no-store, max-age=0, must-revalidate");let u=(0,w.fromNodeOutgoingHttpHeaders)(c.value.headers);return B&&P||u.delete(f.NEXT_CACHE_TAGS_HEADER),!c.cacheControl||t.getHeader("Cache-Control")||u.get("Cache-Control")||u.set("Cache-Control",(0,m.getCacheControlHeader)(c.cacheControl)),await (0,p.sendResponse)(W,V,new Response(c.value.body,{headers:u,status:c.value.status||200})),null};K&&H?await s(H):(n=M.getActiveScopeSpan(),await M.withPropagatedContext(e.headers,()=>M.trace(u.BaseServerSpan.handleRequest,{spanName:`${q} ${y}`,kind:i.SpanKind.SERVER,attributes:{"http.method":q,"http.target":e.url}},s),void 0,!K))}catch(t){if(t instanceof h.NoFallbackError||await D.onRequestError(e,t,{routerKind:"App Router",routePath:N,routeType:"route",revalidateReason:(0,d.getRevalidateReason)({isStaticGeneration:j,isOnDemandRevalidate:T})},!1,O),P)throw t;return await (0,p.sendResponse)(W,V,new Response(null,{status:500})),null}}e.s(["handler",0,I,"patchFetch",0,function(){return(0,a.patchFetch)({workAsyncStorage:k,workUnitAsyncStorage:O})},"routeModule",0,D,"serverHooks",0,T,"workAsyncStorage",0,k,"workUnitAsyncStorage",0,O],491382)}];
-
-//# sourceMappingURL=%5Broot-of-the-server%5D__0u17zp8._.js.map

package/.next/server/chunks/[root-of-the-server]__1vsvck0._.js

@@ -1,61 +0,0 @@
-module.exports=[324725,(e,r,t)=>{r.exports=e.x("next/dist/server/app-render/after-task-async-storage.external.js",()=>require("next/dist/server/app-render/after-task-async-storage.external.js"))},700259,e=>{"use strict";var r=e.i(902157),t=e.i(750227),a=e.i(761095),n=e.i(666680),i=e.i(874533),s=e.i(812057),o=e.i(367165);let c=(0,s.promisify)(i.exec);async function l(e,r){return c(e,{timeout:r,env:{...process.env,PATH:`${process.env.PATH||""}:/usr/local/bin:/opt/homebrew/bin:/bin:/usr/bin`}})}let d=t.default.join(process.cwd(),".larkup"),u=t.default.join(d,"firecrawl-local.json"),p=t.default.join(d,"firecrawl","docker-compose.yml"),w="ragtoolkit-firecrawl";async function m(e){return new Promise((r,t)=>{let n=a.default.createServer();n.listen(e,()=>{let e=n.address().port;n.close(()=>r(e))}),n.on("error",a=>{"EADDRINUSE"===a.code?m(0===e?0:e+1).then(r,t):t(a)})})}async function f(){try{let e=t.default.join(process.cwd(),"proxies.txt"),a=(await r.promises.readFile(e,"utf8")).split("\n").map(e=>e.trim()).filter(e=>e.length>0);if(a.length>0){let e=a[Math.floor(Math.random()*a.length)].split(":");if(e.length>=4)return{server:`http://${e[0]}:${e[1]}`,username:e[2],password:e[3]}}}catch(e){}let e=await (0,o.readConfig)();return e.useScraperProxy&&e.scraperProxyServer?{server:e.scraperProxyServer,username:e.scraperProxyUsername,password:e.scraperProxyPassword}:null}let h={running:!1,endpoint:"http://localhost:3002",apiKey:"",port:3002,project:w};async function y(){try{let e=await r.promises.readFile(u,"utf8");return{...h,...JSON.parse(e)}}catch{return h}}async function g(e){return await r.promises.mkdir(d,{recursive:!0}),await r.promises.writeFile(u,JSON.stringify(e,null,2),"utf8"),e}async function S(){try{await l("docker --version")}catch(e){return console.error("DOCKER ERR:",e),{docker:!1,compose:null}}try{return await l("docker compose version"),{docker:!0,compose:"docker compose"}}catch(e){console.error("DOCKER COMPOSE ERR:",e)}try{return await l("docker-compose version"),{docker:!0,compose:"docker-compose"}}catch{return{docker:!0,compose:null}}}async function k(){let{docker:e,compose:r}=await S();return e?r?{docker:!0,compose:!0,message:"Docker is ready."}:{docker:!0,compose:!1,message:"Docker is installed but the Compose plugin was not found. Install `docker compose` to launch Firecrawl locally."}:{docker:!1,compose:!1,message:"Docker isn't available on this machine. Install Docker Desktop (or the Docker engine) and make sure it's running."}}function v(e,r,t){let a=t?[`PROXY_SERVER: "${t.server}"`,...t.username&&t.password?[`PROXY_USERNAME: "${t.username}"`,`PROXY_PASSWORD: "${t.password}"`]:[]]:[],n=['PORT: "3000"',...a].map(e=>`      ${e}`).join("\n"),i=['PORT: "3002"',...a,'HOST: "0.0.0.0"','REDIS_URL: "redis://redis:6379"','REDIS_RATE_LIMIT_URL: "redis://redis:6379"','PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000/scrape"','USE_DB_AUTHENTICATION: "false"',`TEST_API_KEY: "${e}"`,`BULL_AUTH_KEY: "${e}"`,'DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_DATABASE_URL: "postgresql://user:password@db:5432/postgres"','NUQ_RABBITMQ_URL: "amqp://user:password@rabbitmq:5672"'].map(e=>`      ${e}`).join("\n");return`# Auto-generated by RAG Toolkit. Do not edit by hand.
-services:
-  db:
-    image: ghcr.io/firecrawl/nuq-postgres:latest
-    environment:
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: password
-      POSTGRES_DB: postgres
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U user -d postgres"]
-      interval: 5s
-      timeout: 5s
-      retries: 12
-    restart: unless-stopped
-
-  rabbitmq:
-    image: rabbitmq:3-management-alpine
-    environment:
-      RABBITMQ_DEFAULT_USER: user
-      RABBITMQ_DEFAULT_PASS: password
-    healthcheck:
-      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
-      interval: 5s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    restart: unless-stopped
-
-  redis:
-    image: redis:alpine
-    command: redis-server --save "" --appendonly no
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 5s
-      timeout: 3s
-      retries: 12
-    restart: unless-stopped
-
-  playwright-service:
-    image: ghcr.io/firecrawl/playwright-service:latest
-    environment:
-${n}
-    restart: unless-stopped
-
-  api:
-    image: ghcr.io/firecrawl/firecrawl:latest
-    depends_on:
-      db:
-        condition: service_healthy
-      rabbitmq:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "${r}:3002"
-    environment:
-${i}
-    restart: unless-stopped
-`}async function E(){let e=await k();if(!e.compose)return g({...await y(),running:!1,lastError:e.message});let{compose:a}=await S(),i=await y(),s=i.apiKey||`fc-local-${(0,n.randomUUID)()}`,o=i.port||3002,c=await f();await r.promises.mkdir(t.default.dirname(p),{recursive:!0}),await r.promises.writeFile(p,v(s,o,c),"utf8");try{await l(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(t){let e=t instanceof Error?t.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return g({...i,apiKey:s,port:o,running:!1,lastError:e});try{o=await m(o+1),await r.promises.writeFile(p,v(s,o,c),"utf8"),await l(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(e){return g({...i,apiKey:s,port:o,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let d=(0,r.existsSync)("/.dockerenv"),u=`http://${d?"host.docker.internal":"localhost"}:${o}`,h=await T(u,9e4);return g({running:h,endpoint:u,apiKey:s,port:o,project:w,startedAt:new Date().toISOString(),lastError:h?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function b(){let{compose:e}=await S(),r=await y();if(e)try{await l(`${e} -p ${w} -f "${p}" down`,6e4)}catch{}return g({...r,running:!1,startedAt:void 0})}async function R(){let e=await y();if(!e.startedAt)return e;let r=await A(e.endpoint);return r!==e.running?g({...e,running:r}):e}async function A(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function T(e,r){let t=Date.now()+r;for(;Date.now()<t;){if(await A(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}let D="http://firecrawl-api:3002";async function _(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,r.existsSync)("/.dockerenv")?{available:await A(D),endpoint:D}:{available:!1,endpoint:""}}async function P(){let{available:e}=await _();return e?g({running:!0,endpoint:D,apiKey:`fc-docker-${(0,n.randomUUID)()}`,port:3002,project:"docker-sibling",startedAt:new Date().toISOString()}):g({...h,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,k,"checkDockerSibling",0,_,"connectDockerSibling",0,P,"readLocalState",0,y,"refreshLocalStatus",0,R,"startLocal",0,E,"stopLocal",0,b])},304368,e=>{"use strict";var r=e.i(367165),t=e.i(700259);class a extends Error{status;constructor(e,r){super(e),this.status=r,this.name="FirecrawlError"}}async function n(){let e=await (0,r.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,t.readLocalState)();if(e.running&&e.apiKey)return{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new a("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let n=e.firecrawlApiKey;if(n)return{base:"https://api.firecrawl.dev/v1",key:n,mode:"cloud"};throw new a("The cloud web crawler needs an API key. Add it in Settings.",401)}async function i(){let e=await (0,r.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,t.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function s(e,r){let{base:t,key:i}=await n(),s=await fetch(`${t}${e}`,{...r,headers:{Authorization:`Bearer ${i}`,"Content-Type":"application/json",...r?.headers??{}},cache:"no-store"}),o=await s.text(),l=o?c(o):{};if(!s.ok)throw new a(l?.error||`Firecrawl request failed (${s.status})`,s.status);return l}async function o(e){let{key:r}=await n(),t=await fetch(e,{headers:{Authorization:`Bearer ${r}`},cache:"no-store"}),i=await t.text(),s=i?c(i):{};if(!t.ok)throw new a(s?.error||`Firecrawl request failed (${t.status})`,t.status);return s}function c(e){try{return JSON.parse(e)}catch{return{}}}async function l(e,r=10){return((await s("/search",{method:"POST",body:JSON.stringify({query:e,limit:r})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let d={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function u(e){let r=(await s("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...d})})).data??{};return{url:r.metadata?.sourceURL||e,title:r.metadata?.title||r.metadata?.ogTitle||e,markdown:r.markdown??""}}async function p(e,r){let t=await s("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:r,allowExternalLinks:!1,scrapeOptions:{...d,formats:["markdown"],onlyMainContent:!0}})});if(!t.id)throw new a("Firecrawl did not return a crawl id");return t.id}async function w(e,r=!1){let t=r?await o(e):await s(`/crawl/${e}`,{method:"GET"});return{state:t.status??"scraping",total:t.total??0,completed:t.completed??0,next:t.next,pages:(t.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function m(e){try{await s(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,a,"cancelCrawl",0,m,"getCrawlStatus",0,w,"isFirecrawlConfigured",0,i,"scrapePage",0,u,"searchWeb",0,l,"startCrawl",0,p])},426237,e=>{"use strict";var r=e.i(766576),t=e.i(304368);async function a(){return r.NextResponse.json({configured:await (0,t.isFirecrawlConfigured)()})}async function n(e){try{let{query:a,limit:n}=await e.json();if(!a||!a.trim())return r.NextResponse.json({error:"A search query is required."},{status:400});let i=await (0,t.searchWeb)(a.trim(),Math.min(n??10,25));return r.NextResponse.json({results:i})}catch(n){let e=n instanceof t.FirecrawlError?n.status??500:500,a=n instanceof Error?n.message:"Search failed.";return r.NextResponse.json({error:a},{status:e})}}e.s(["GET",0,a,"POST",0,n,"dynamic",0,"force-dynamic","runtime",0,"nodejs"])}];
-
-//# sourceMappingURL=%5Broot-of-the-server%5D__1vsvck0._.js.map