larkup 0.1.39 → 0.1.41

package/.next/server/chunks/[root-of-the-server]__0j2l54r._.js

@@ -1,3 +1,3 @@
-module.exports=[367165,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(638725),n=e.i(192441);async function i(){let e=await (0,n.getDataDir)();if(!e)return r.DEFAULT_CONFIG;try{let n=await t.promises.readFile(a.default.join(e,"config.json"),"utf8"),i=JSON.parse(n),o=i.customEmbeddings,s=i.embeddingModelId;if(i.customEmbedding&&!o?.length){let e=i.customEmbedding;o=[e],"custom"===s&&(s=`custom:${e.modelName}`)}let u={...r.DEFAULT_CONFIG,...i,embeddingModelId:s??r.DEFAULT_CONFIG.embeddingModelId,customEmbeddings:o,chunking:{...r.DEFAULT_CONFIG.chunking,...i.chunking},storeConfig:{...r.DEFAULT_CONFIG.storeConfig,...i.storeConfig}};return delete u.customEmbedding,u}catch{return r.DEFAULT_CONFIG}}async function o(e){let r=await (0,n.requireDataDir)(),i={...e,updatedAt:new Date().toISOString()};return await t.promises.writeFile(a.default.join(r,"config.json"),JSON.stringify(i,null,2),"utf8"),i}e.s(["readConfig",0,i,"writeConfig",0,o])},874533,(e,t,a)=>{t.exports=e.x("node:child_process",()=>require("node:child_process"))},812057,(e,t,a)=>{t.exports=e.x("node:util",()=>require("node:util"))},705252,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(666680),n=e.i(192441);let i=Promise.resolve();function o(e){let t=i.then(e,e);return i=t.catch(()=>{}),t}async function s(e){let t=e?await (0,n.requireDataDir)():await (0,n.getDataDir)();return t?a.default.join(t,"documents.json"):null}async function u(){let e=await s(!1);if(!e)return[];try{let a=await t.promises.readFile(e,"utf8"),r=JSON.parse(a);if(!Array.isArray(r))throw Error(`Invalid document store at ${e}: expected a JSON array.`);return r}catch(e){if(e?.code==="ENOENT")return[];throw e}}async function l(e){let n=await s(!0);if(!n)return;let i=a.default.join(a.default.dirname(n),`.${a.default.basename(n)}.${process.pid}.${(0,r.randomUUID)()}.tmp`);try{await t.promises.writeFile(i,JSON.stringify(e,null,2),"utf8"),await t.promises.rename(i,n)}catch(e){throw await t.promises.unlink(i).catch(()=>{}),e}}function d(e){let t=e.content.trim();return{id:e.id??(0,r.randomUUID)(),title:e.title.trim()||"Untitled",url:e.url,source:e.source,content:t,charCount:t.length,jobId:e.jobId,metadata:e.metadata,status:"unindexed",createdAt:new Date().toISOString()}}async function c(){let e=await u(),t=e.reduce((e,t)=>e+t.charCount,0);return{docCount:e.length,charCount:t,bySource:e.reduce((e,t)=>(e[t.source]=(e[t.source]??0)+1,e),{})}}e.s(["addCrawledDocuments",0,function(e,t){return o(async()=>{let a=await u(),r=new Set(a.filter(e=>e.url).map(e=>e.url)),n=0;for(let i of t)!(i.url&&r.has(i.url))&&i.content.trim()&&(a.push(d({...i,jobId:e})),i.url&&r.add(i.url),n++);return n>0&&await l(a),n})},"addDocument",0,function(e){return o(async()=>{let t=await u(),a=d(e);return t.push(a),await l(t),a})},"addDocuments",0,function(e){return o(async()=>{if(0===e.length)return[];let t=await u(),a=e.map(d);return t.push(...a),await l(t),a})},"clearDocuments",0,function(){return o(async()=>{await l([])})},"corpusStats",0,c,"deleteDocument",0,function(e){return o(async()=>{let t=await u();await l(t.filter(t=>t.id!==e))})},"deleteDocuments",0,function(e){return o(async()=>{let t=await u(),a=new Set(e);await l(t.filter(e=>!a.has(e.id)))})},"readDocuments",0,u,"replaceDocuments",0,function(e){return o(async()=>{let t=new Date().toISOString();await l(e.map(e=>({...e,status:"unindexed",createdAt:e.createdAt||t})))})},"updateDocument",0,function(e,t){return o(async()=>{let a=await u(),r=a.findIndex(t=>t.id===e);if(r<0)return;let n=a[r],i=void 0!==t.content?t.content.trim():n.content,o={...n,title:void 0!==t.title?t.title.trim()||"Untitled":n.title,content:i,charCount:i.length,url:void 0!==t.url?t.url||void 0:n.url,metadata:void 0!==t.metadata?t.metadata:n.metadata,status:"unindexed"};return a[r]=o,await l(a),o})},"updateDocumentsStatus",0,function(e,t){return o(async()=>{let a=await u(),r=new Set(e),n=!1;for(let e of a)r.has(e.id)&&(e.status=t,n=!0);n&&await l(a)})}])},798898,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(192441);let n=Promise.resolve();function i(e){let t=n.then(e,e);return n=t.catch(()=>{}),t}async function o(e){let t=e?await (0,r.requireDataDir)():await (0,r.getDataDir)();return t?a.default.join(t,"jobs.json"):null}async function s(){let e=await o(!1);if(!e)return[];try{let a=await t.promises.readFile(e,"utf8");return JSON.parse(a)}catch{return[]}}async function u(e){let a=await o(!0);a&&await t.promises.writeFile(a,JSON.stringify(e,null,2),"utf8")}async function l(e){return(await s()).find(t=>t.id===e)}e.s(["deleteJob",0,function(e){return i(async()=>{let t=await s();await u(t.filter(t=>t.id!==e))})},"getJob",0,l,"readJobs",0,s,"saveJob",0,function(e){return i(async()=>{let t=await s(),a=t.findIndex(t=>t.id===e.id),r={...e,updatedAt:new Date().toISOString()};return a>=0?t[a]=r:t.unshift(r),await u(t),r})},"updateJob",0,function(e,t){return i(async()=>{let a=await s(),r=a.findIndex(t=>t.id===e);if(r<0)return;let n={...t(a[r]),updatedAt:new Date().toISOString()};return a[r]=n,await u(a),n})}])},962044,e=>{"use strict";var t=e.i(239441),a=e.i(449055),r=e.i(391328),n=e.i(6352),i=e.i(746789),o=e.i(423437),s=e.i(178820),u=e.i(569191),l=e.i(576251),d=e.i(199962),c=e.i(547545),p=e.i(664046),f=e.i(235691),w=e.i(298907),m=e.i(110763),g=e.i(193695);e.i(673243);var h=e.i(236167),y=e.i(766576),v=e.i(798898),C=e.i(705252),R=e.i(304368);let E=new Set;function b(e){if(!e)return!1;let t=e.toLowerCase();return t.includes("verification successful. waiting for")||t.includes("please verify you are a human")||t.includes("checking if the site connection is secure")||t.includes("just a moment...")||t.includes("enable javascript and cookies to continue")}async function D(e,t){try{let a=await (0,R.scrapePage)(t.url);if(b(a.markdown))throw Error("Scraping blocked by anti-bot protection (e.g. Cloudflare)");return{added:await (0,C.addCrawledDocuments)(e,[{title:a.title,url:a.url,content:a.markdown,source:"scrape"}]),target:{...t,status:"completed",pagesCrawled:+!!a.markdown}}}catch(e){return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"scrape failed"}}}}async function x(e,t,a){try{if(!t.firecrawlId){let e=await (0,R.startCrawl)(t.url,a);return{added:0,target:{...t,firecrawlId:e,status:"running"}}}let r=t.firecrawlId,n=!1,i=0,o=0,s=t.pagesCrawled,u="running",l=[];for(;i<5;){let e=await (0,R.getCrawlStatus)(r,n);for(let t of(s=e.completed||s,e.pages))b(t.markdown)||l.push({title:t.title,url:t.url,content:t.markdown,source:"scrape"});if(i++,"completed"===e.state?u="completed":"failed"===e.state?u="failed":"cancelled"===e.state&&(u="cancelled"),e.next)r=e.next,n=!0;else break}return l.length&&(o=await (0,C.addCrawledDocuments)(e,l)),{added:o,target:{...t,status:u,pagesCrawled:s}}}catch(e){return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"crawl failed"}}}}async function S(e){let t=await (0,v.getJob)(e);if(t){if(["completed","failed","cancelled"].includes(t.status)||E.has(e))return t;E.add(e);try{let a=await Promise.all(t.targets.map(a=>"completed"===a.status||"failed"===a.status?Promise.resolve({target:a,added:0}):"page"===a.scope?D(e,a):x(e,a,t.pageLimit))),r=a.map(e=>e.target),n=r.reduce((e,t)=>e+t.pagesCrawled,0);return await (0,v.updateJob)(e,e=>({...e,targets:r,pagesCrawled:n,docCount:e.docCount+a.reduce((e,t)=>e+t.added,0),status:r.every(e=>"completed"===e.status)?"completed":r.every(e=>"failed"===e.status)?"failed":r.every(e=>"completed"===e.status||"failed"===e.status)&&r.some(e=>"completed"===e.status)?"completed":"running"}))}finally{E.delete(e)}}}async function N(e){let t=await (0,v.getJob)(e);if(t)return await Promise.all(t.targets.filter(e=>e.firecrawlId&&"running"===e.status).map(e=>(0,R.cancelCrawl)(e.firecrawlId))),(0,v.updateJob)(e,e=>({...e,status:"cancelled",targets:e.targets.map(e=>"running"===e.status||"queued"===e.status?{...e,status:"cancelled"}:e)}))}async function A(e,{params:t}){let{id:a}=await t,r=await S(a);return r?y.NextResponse.json({job:r}):y.NextResponse.json({error:"Job not found."},{status:404})}async function I(e,{params:t}){let{id:a}=await t;if(new URL(e.url).searchParams.get("remove"))return await (0,v.deleteJob)(a),y.NextResponse.json({ok:!0});if(!await (0,v.getJob)(a))return y.NextResponse.json({error:"Job not found."},{status:404});let r=await N(a);return y.NextResponse.json({job:r})}e.s(["DELETE",0,I,"GET",0,A,"dynamic",0,"force-dynamic","runtime",0,"nodejs"],934385);var O=e.i(934385);let T=new t.AppRouteRouteModule({definition:{kind:a.RouteKind.APP_ROUTE,page:"/api/jobs/[id]/route",pathname:"/api/jobs/[id]",filename:"route",bundlePath:""},distDir:".next",relativeProjectDir:"",resolvedPagePath:"[project]/apps/web/app/api/jobs/[id]/route.ts",nextConfigOutput:"standalone",userland:O,...{}}),{workAsyncStorage:j,workUnitAsyncStorage:P,serverHooks:_}=T;async function U(e,t,r){r.requestMeta&&(0,n.setRequestMeta)(e,r.requestMeta),T.isDev&&(0,n.addRequestMeta)(e,"devRequestTimingInternalsEnd",process.hrtime.bigint());let y="/api/jobs/[id]/route";y=y.replace(/\/index$/,"")||"/";let v=await T.prepare(e,t,{srcPage:y,multiZoneDraftMode:!1});if(!v)return t.statusCode=400,t.end("Bad Request"),null==r.waitUntil||r.waitUntil.call(r,Promise.resolve()),null;let{buildId:C,deploymentId:R,params:E,nextConfig:b,parsedUrl:D,isDraftMode:x,prerenderManifest:S,routerServerContext:N,isOnDemandRevalidate:A,revalidateOnlyGenerated:I,resolvedPathname:O,clientReferenceManifest:j,serverActionsManifest:P}=v,_=(0,s.normalizeAppPath)(y),U=!!(S.dynamicRoutes[_]||S.routes[O]),k=async()=>((null==N?void 0:N.render404)?await N.render404(e,t,D,!1):t.end("This page could not be found"),null);if(U&&!x){let e=!!S.routes[O],t=S.dynamicRoutes[_];if(t&&!1===t.fallback&&!e){if(b.adapterPath)return await k();throw new g.NoFallbackError}}let F=null;!U||T.isDev||x||(F="/index"===(F=O)?"/":F);let q=!0===T.isDev||!U,J=U&&!q;P&&j&&(0,o.setManifestsSingleton)({page:y,clientReferenceManifest:j,serverActionsManifest:P});let M=e.method||"GET",H=(0,i.getTracer)(),L=H.getActiveScopeSpan(),$=!!(null==N?void 0:N.isWrappedByNextServer),G=!!(0,n.getRequestMeta)(e,"minimalMode"),K=(0,n.getRequestMeta)(e,"incrementalCache")||await T.getIncrementalCache(e,b,S,G);null==K||K.resetRequestCache(),globalThis.__incrementalCache=K;let B={params:E,previewProps:S.preview,renderOpts:{experimental:{authInterrupts:!!b.experimental.authInterrupts},cacheComponents:!!b.cacheComponents,supportsDynamicResponse:q,incrementalCache:K,cacheLifeProfiles:b.cacheLife,waitUntil:r.waitUntil,onClose:e=>{t.on("close",e)},onAfterTaskError:void 0,onInstrumentationRequestError:(t,a,r,n)=>T.onRequestError(e,t,r,n,N)},sharedContext:{buildId:C,deploymentId:R}},V=new u.NodeNextRequest(e),W=new u.NodeNextResponse(t),X=l.NextRequestAdapter.fromNodeNextRequest(V,(0,l.signalFromNodeResponse)(t));try{let n,o=async e=>T.handle(X,B).finally(()=>{if(!e)return;e.setAttributes({"http.status_code":t.statusCode,"next.rsc":!1});let a=H.getRootSpanAttributes();if(!a)return;if(a.get("next.span_type")!==d.BaseServerSpan.handleRequest)return void console.warn(`Unexpected root span type '${a.get("next.span_type")}'. Please report this Next.js issue https://github.com/vercel/next.js`);let r=a.get("next.route");if(r){let t=`${M} ${r}`;e.setAttributes({"next.route":r,"http.route":r,"next.span_name":t}),e.updateName(t),n&&n!==e&&(n.setAttribute("http.route",r),n.updateName(t))}else e.updateName(`${M} ${y}`)}),s=async n=>{var i,s;let u=async({previousCacheEntry:a})=>{try{if(!G&&A&&I&&!a)return t.statusCode=404,t.setHeader("x-nextjs-cache","REVALIDATED"),t.end("This page could not be found"),null;let i=await o(n);e.fetchMetrics=B.renderOpts.fetchMetrics;let s=B.renderOpts.pendingWaitUntil;s&&r.waitUntil&&(r.waitUntil(s),s=void 0);let u=B.renderOpts.collectedTags;if(!U)return await (0,p.sendResponse)(V,W,i,B.renderOpts.pendingWaitUntil),null;{let e=await i.blob(),t=(0,f.toNodeOutgoingHttpHeaders)(i.headers);u&&(t[m.NEXT_CACHE_TAGS_HEADER]=u),!t["content-type"]&&e.type&&(t["content-type"]=e.type);let a=void 0!==B.renderOpts.collectedRevalidate&&!(B.renderOpts.collectedRevalidate>=m.INFINITE_CACHE)&&B.renderOpts.collectedRevalidate,r=void 0===B.renderOpts.collectedExpire||B.renderOpts.collectedExpire>=m.INFINITE_CACHE?void 0:B.renderOpts.collectedExpire;return{value:{kind:h.CachedRouteKind.APP_ROUTE,status:i.status,body:Buffer.from(await e.arrayBuffer()),headers:t},cacheControl:{revalidate:a,expire:r}}}}catch(t){throw(null==a?void 0:a.isStale)&&await T.onRequestError(e,t,{routerKind:"App Router",routePath:y,routeType:"route",revalidateReason:(0,c.getRevalidateReason)({isStaticGeneration:J,isOnDemandRevalidate:A})},!1,N),t}},l=await T.handleResponse({req:e,nextConfig:b,cacheKey:F,routeKind:a.RouteKind.APP_ROUTE,isFallback:!1,prerenderManifest:S,isRoutePPREnabled:!1,isOnDemandRevalidate:A,revalidateOnlyGenerated:I,responseGenerator:u,waitUntil:r.waitUntil,isMinimalMode:G});if(!U)return null;if((null==l||null==(i=l.value)?void 0:i.kind)!==h.CachedRouteKind.APP_ROUTE)throw Object.defineProperty(Error(`Invariant: app-route received invalid cache entry ${null==l||null==(s=l.value)?void 0:s.kind}`),"__NEXT_ERROR_CODE",{value:"E701",enumerable:!1,configurable:!0});G||t.setHeader("x-nextjs-cache",A?"REVALIDATED":l.isMiss?"MISS":l.isStale?"STALE":"HIT"),x&&t.setHeader("Cache-Control","private, no-cache, no-store, max-age=0, must-revalidate");let d=(0,f.fromNodeOutgoingHttpHeaders)(l.value.headers);return G&&U||d.delete(m.NEXT_CACHE_TAGS_HEADER),!l.cacheControl||t.getHeader("Cache-Control")||d.get("Cache-Control")||d.set("Cache-Control",(0,w.getCacheControlHeader)(l.cacheControl)),await (0,p.sendResponse)(V,W,new Response(l.value.body,{headers:d,status:l.value.status||200})),null};$&&L?await s(L):(n=H.getActiveScopeSpan(),await H.withPropagatedContext(e.headers,()=>H.trace(d.BaseServerSpan.handleRequest,{spanName:`${M} ${y}`,kind:i.SpanKind.SERVER,attributes:{"http.method":M,"http.target":e.url}},s),void 0,!$))}catch(t){if(t instanceof g.NoFallbackError||await T.onRequestError(e,t,{routerKind:"App Router",routePath:_,routeType:"route",revalidateReason:(0,c.getRevalidateReason)({isStaticGeneration:J,isOnDemandRevalidate:A})},!1,N),U)throw t;return await (0,p.sendResponse)(V,W,new Response(null,{status:500})),null}}e.s(["handler",0,U,"patchFetch",0,function(){return(0,r.patchFetch)({workAsyncStorage:j,workUnitAsyncStorage:P})},"routeModule",0,T,"serverHooks",0,_,"workAsyncStorage",0,j,"workUnitAsyncStorage",0,P],962044)}];
+module.exports=[367165,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(638725),n=e.i(192441);async function i(){let e=await (0,n.getDataDir)();if(!e)return r.DEFAULT_CONFIG;try{let n=await t.promises.readFile(a.default.join(e,"config.json"),"utf8"),i=JSON.parse(n),o=i.customEmbeddings,s=i.embeddingModelId;if(i.customEmbedding&&!o?.length){let e=i.customEmbedding;o=[e],"custom"===s&&(s=`custom:${e.modelName}`)}let u={...r.DEFAULT_CONFIG,...i,embeddingModelId:s??r.DEFAULT_CONFIG.embeddingModelId,customEmbeddings:o,chunking:{...r.DEFAULT_CONFIG.chunking,...i.chunking},storeConfig:{...r.DEFAULT_CONFIG.storeConfig,...i.storeConfig}};return delete u.customEmbedding,u}catch{return r.DEFAULT_CONFIG}}async function o(e){let r=await (0,n.requireDataDir)(),i={...e,updatedAt:new Date().toISOString()};return await t.promises.writeFile(a.default.join(r,"config.json"),JSON.stringify(i,null,2),"utf8"),i}e.s(["readConfig",0,i,"writeConfig",0,o])},874533,(e,t,a)=>{t.exports=e.x("node:child_process",()=>require("node:child_process"))},812057,(e,t,a)=>{t.exports=e.x("node:util",()=>require("node:util"))},705252,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(666680),n=e.i(192441);let i=Promise.resolve();function o(e){let t=i.then(e,e);return i=t.catch(()=>{}),t}async function s(e){let t=e?await (0,n.requireDataDir)():await (0,n.getDataDir)();return t?a.default.join(t,"documents.json"):null}async function u(){let e=await s(!1);if(!e)return[];try{let a=await t.promises.readFile(e,"utf8"),r=JSON.parse(a);if(!Array.isArray(r))throw Error(`Invalid document store at ${e}: expected a JSON array.`);return r}catch(e){if(e?.code==="ENOENT")return[];throw e}}async function l(e){let n=await s(!0);if(!n)return;let i=a.default.join(a.default.dirname(n),`.${a.default.basename(n)}.${process.pid}.${(0,r.randomUUID)()}.tmp`);try{await t.promises.writeFile(i,JSON.stringify(e,null,2),"utf8"),await t.promises.rename(i,n)}catch(e){throw await t.promises.unlink(i).catch(()=>{}),e}}function d(e){let t=e.content.trim();return{id:e.id??(0,r.randomUUID)(),title:e.title.trim()||"Untitled",url:e.url,source:e.source,content:t,charCount:t.length,jobId:e.jobId,metadata:e.metadata,status:"unindexed",createdAt:new Date().toISOString()}}async function c(){let e=await u(),t=e.reduce((e,t)=>e+t.charCount,0);return{docCount:e.length,charCount:t,bySource:e.reduce((e,t)=>(e[t.source]=(e[t.source]??0)+1,e),{})}}e.s(["addCrawledDocuments",0,function(e,t){return o(async()=>{let a=await u(),r=new Set(a.filter(e=>e.url).map(e=>e.url)),n=0;for(let i of t)!(i.url&&r.has(i.url))&&i.content.trim()&&(a.push(d({...i,jobId:e})),i.url&&r.add(i.url),n++);return n>0&&await l(a),n})},"addDocument",0,function(e){return o(async()=>{let t=await u(),a=d(e);return t.push(a),await l(t),a})},"addDocuments",0,function(e){return o(async()=>{if(0===e.length)return[];let t=await u(),a=e.map(d);return t.push(...a),await l(t),a})},"clearDocuments",0,function(){return o(async()=>{await l([])})},"corpusStats",0,c,"deleteDocument",0,function(e){return o(async()=>{let t=await u();await l(t.filter(t=>t.id!==e))})},"deleteDocuments",0,function(e){return o(async()=>{let t=await u(),a=new Set(e);await l(t.filter(e=>!a.has(e.id)))})},"readDocuments",0,u,"replaceDocuments",0,function(e){return o(async()=>{let t=new Date().toISOString();await l(e.map(e=>({...e,status:"unindexed",createdAt:e.createdAt||t})))})},"updateDocument",0,function(e,t){return o(async()=>{let a=await u(),r=a.findIndex(t=>t.id===e);if(r<0)return;let n=a[r],i=void 0!==t.content?t.content.trim():n.content,o={...n,title:void 0!==t.title?t.title.trim()||"Untitled":n.title,content:i,charCount:i.length,url:void 0!==t.url?t.url||void 0:n.url,metadata:void 0!==t.metadata?t.metadata:n.metadata,status:"unindexed"};return a[r]=o,await l(a),o})},"updateDocumentsStatus",0,function(e,t){return o(async()=>{let a=await u(),r=new Set(e),n=!1;for(let e of a)r.has(e.id)&&(e.status=t,n=!0);n&&await l(a)})}])},798898,e=>{"use strict";var t=e.i(902157),a=e.i(750227),r=e.i(192441);let n=Promise.resolve();function i(e){let t=n.then(e,e);return n=t.catch(()=>{}),t}async function o(e){let t=e?await (0,r.requireDataDir)():await (0,r.getDataDir)();return t?a.default.join(t,"jobs.json"):null}async function s(){let e=await o(!1);if(!e)return[];try{let a=await t.promises.readFile(e,"utf8");return JSON.parse(a)}catch{return[]}}async function u(e){let a=await o(!0);a&&await t.promises.writeFile(a,JSON.stringify(e,null,2),"utf8")}async function l(e){return(await s()).find(t=>t.id===e)}e.s(["deleteJob",0,function(e){return i(async()=>{let t=await s();await u(t.filter(t=>t.id!==e))})},"getJob",0,l,"readJobs",0,s,"saveJob",0,function(e){return i(async()=>{let t=await s(),a=t.findIndex(t=>t.id===e.id),r={...e,updatedAt:new Date().toISOString()};return a>=0?t[a]=r:t.unshift(r),await u(t),r})},"updateJob",0,function(e,t){return i(async()=>{let a=await s(),r=a.findIndex(t=>t.id===e);if(r<0)return;let n={...t(a[r]),updatedAt:new Date().toISOString()};return a[r]=n,await u(a),n})}])},962044,e=>{"use strict";var t=e.i(239441),a=e.i(449055),r=e.i(391328),n=e.i(6352),i=e.i(746789),o=e.i(423437),s=e.i(178820),u=e.i(569191),l=e.i(576251),d=e.i(199962),c=e.i(547545),p=e.i(664046),f=e.i(235691),w=e.i(298907),m=e.i(110763),g=e.i(193695);e.i(673243);var h=e.i(236167),y=e.i(766576),v=e.i(798898),C=e.i(705252),R=e.i(304368);let E=new Set;function b(e){if(!e)return!1;let t=e.toLowerCase();return t.includes("verification successful. waiting for")||t.includes("please verify you are a human")||t.includes("checking if the site connection is secure")||t.includes("just a moment...")||t.includes("enable javascript and cookies to continue")}async function D(e,t){try{let a=await (0,R.scrapePage)(t.url);if(b(a.markdown))throw Error("Scraping blocked by anti-bot protection (e.g. Cloudflare)");return{added:await (0,C.addCrawledDocuments)(e,[{title:a.title,url:a.url,content:a.markdown,source:"scrape"}]),target:{...t,status:"completed",pagesCrawled:+!!a.markdown}}}catch(e){return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"scrape failed"}}}}async function x(e,t,a){try{if(!t.firecrawlId){let e=await (0,R.startCrawl)(t.url,a);return{added:0,target:{...t,firecrawlId:e,status:"running"}}}let r=t.firecrawlId,n=!1,i=0,o=0,s=t.pagesCrawled,u="running",l=[];for(;i<5;){let e=await (0,R.getCrawlStatus)(r,n);for(let t of(s=e.completed||s,e.pages))b(t.markdown)||l.push({title:t.title,url:t.url,content:t.markdown,source:"scrape"});if(i++,"completed"===e.state?u="completed":"failed"===e.state?u="failed":"cancelled"===e.state&&(u="cancelled"),e.next)r=e.next,n=!0;else break}return l.length&&(o=await (0,C.addCrawledDocuments)(e,l)),{added:o,target:{...t,status:u,pagesCrawled:s}}}catch(e){if(t.firecrawlId?.startsWith("native-")&&e instanceof Error&&"Native crawl was not found."===e.message)return{added:0,target:{...t,firecrawlId:void 0,status:"running",error:"Restarting the local crawl…"}};return{added:0,target:{...t,status:"failed",error:e instanceof Error?e.message:"crawl failed"}}}}async function S(e){let t=await (0,v.getJob)(e);if(t){if(["completed","failed","cancelled"].includes(t.status)||E.has(e))return t;E.add(e);try{let a=await Promise.all(t.targets.map(a=>"completed"===a.status||"failed"===a.status?Promise.resolve({target:a,added:0}):"page"===a.scope?D(e,a):x(e,a,t.pageLimit))),r=a.map(e=>e.target),n=r.reduce((e,t)=>e+t.pagesCrawled,0);return await (0,v.updateJob)(e,e=>({...e,targets:r,pagesCrawled:n,docCount:e.docCount+a.reduce((e,t)=>e+t.added,0),status:r.every(e=>"completed"===e.status)?"completed":r.every(e=>"failed"===e.status)?"failed":r.every(e=>"completed"===e.status||"failed"===e.status)&&r.some(e=>"completed"===e.status)?"completed":"running"}))}finally{E.delete(e)}}}async function N(e){let t=await (0,v.getJob)(e);if(t)return await Promise.all(t.targets.filter(e=>e.firecrawlId&&"running"===e.status).map(e=>(0,R.cancelCrawl)(e.firecrawlId))),(0,v.updateJob)(e,e=>({...e,status:"cancelled",targets:e.targets.map(e=>"running"===e.status||"queued"===e.status?{...e,status:"cancelled"}:e)}))}async function A(e,{params:t}){let{id:a}=await t,r=await S(a);return r?y.NextResponse.json({job:r}):y.NextResponse.json({error:"Job not found."},{status:404})}async function I(e,{params:t}){let{id:a}=await t;if(new URL(e.url).searchParams.get("remove"))return await (0,v.deleteJob)(a),y.NextResponse.json({ok:!0});if(!await (0,v.getJob)(a))return y.NextResponse.json({error:"Job not found."},{status:404});let r=await N(a);return y.NextResponse.json({job:r})}e.s(["DELETE",0,I,"GET",0,A,"dynamic",0,"force-dynamic","runtime",0,"nodejs"],934385);var O=e.i(934385);let T=new t.AppRouteRouteModule({definition:{kind:a.RouteKind.APP_ROUTE,page:"/api/jobs/[id]/route",pathname:"/api/jobs/[id]",filename:"route",bundlePath:""},distDir:".next",relativeProjectDir:"",resolvedPagePath:"[project]/apps/web/app/api/jobs/[id]/route.ts",nextConfigOutput:"standalone",userland:O,...{}}),{workAsyncStorage:j,workUnitAsyncStorage:P,serverHooks:_}=T;async function U(e,t,r){r.requestMeta&&(0,n.setRequestMeta)(e,r.requestMeta),T.isDev&&(0,n.addRequestMeta)(e,"devRequestTimingInternalsEnd",process.hrtime.bigint());let y="/api/jobs/[id]/route";y=y.replace(/\/index$/,"")||"/";let v=await T.prepare(e,t,{srcPage:y,multiZoneDraftMode:!1});if(!v)return t.statusCode=400,t.end("Bad Request"),null==r.waitUntil||r.waitUntil.call(r,Promise.resolve()),null;let{buildId:C,deploymentId:R,params:E,nextConfig:b,parsedUrl:D,isDraftMode:x,prerenderManifest:S,routerServerContext:N,isOnDemandRevalidate:A,revalidateOnlyGenerated:I,resolvedPathname:O,clientReferenceManifest:j,serverActionsManifest:P}=v,_=(0,s.normalizeAppPath)(y),U=!!(S.dynamicRoutes[_]||S.routes[O]),k=async()=>((null==N?void 0:N.render404)?await N.render404(e,t,D,!1):t.end("This page could not be found"),null);if(U&&!x){let e=!!S.routes[O],t=S.dynamicRoutes[_];if(t&&!1===t.fallback&&!e){if(b.adapterPath)return await k();throw new g.NoFallbackError}}let F=null;!U||T.isDev||x||(F="/index"===(F=O)?"/":F);let q=!0===T.isDev||!U,J=U&&!q;P&&j&&(0,o.setManifestsSingleton)({page:y,clientReferenceManifest:j,serverActionsManifest:P});let M=e.method||"GET",H=(0,i.getTracer)(),L=H.getActiveScopeSpan(),$=!!(null==N?void 0:N.isWrappedByNextServer),G=!!(0,n.getRequestMeta)(e,"minimalMode"),K=(0,n.getRequestMeta)(e,"incrementalCache")||await T.getIncrementalCache(e,b,S,G);null==K||K.resetRequestCache(),globalThis.__incrementalCache=K;let B={params:E,previewProps:S.preview,renderOpts:{experimental:{authInterrupts:!!b.experimental.authInterrupts},cacheComponents:!!b.cacheComponents,supportsDynamicResponse:q,incrementalCache:K,cacheLifeProfiles:b.cacheLife,waitUntil:r.waitUntil,onClose:e=>{t.on("close",e)},onAfterTaskError:void 0,onInstrumentationRequestError:(t,a,r,n)=>T.onRequestError(e,t,r,n,N)},sharedContext:{buildId:C,deploymentId:R}},W=new u.NodeNextRequest(e),V=new u.NodeNextResponse(t),X=l.NextRequestAdapter.fromNodeNextRequest(W,(0,l.signalFromNodeResponse)(t));try{let n,o=async e=>T.handle(X,B).finally(()=>{if(!e)return;e.setAttributes({"http.status_code":t.statusCode,"next.rsc":!1});let a=H.getRootSpanAttributes();if(!a)return;if(a.get("next.span_type")!==d.BaseServerSpan.handleRequest)return void console.warn(`Unexpected root span type '${a.get("next.span_type")}'. Please report this Next.js issue https://github.com/vercel/next.js`);let r=a.get("next.route");if(r){let t=`${M} ${r}`;e.setAttributes({"next.route":r,"http.route":r,"next.span_name":t}),e.updateName(t),n&&n!==e&&(n.setAttribute("http.route",r),n.updateName(t))}else e.updateName(`${M} ${y}`)}),s=async n=>{var i,s;let u=async({previousCacheEntry:a})=>{try{if(!G&&A&&I&&!a)return t.statusCode=404,t.setHeader("x-nextjs-cache","REVALIDATED"),t.end("This page could not be found"),null;let i=await o(n);e.fetchMetrics=B.renderOpts.fetchMetrics;let s=B.renderOpts.pendingWaitUntil;s&&r.waitUntil&&(r.waitUntil(s),s=void 0);let u=B.renderOpts.collectedTags;if(!U)return await (0,p.sendResponse)(W,V,i,B.renderOpts.pendingWaitUntil),null;{let e=await i.blob(),t=(0,f.toNodeOutgoingHttpHeaders)(i.headers);u&&(t[m.NEXT_CACHE_TAGS_HEADER]=u),!t["content-type"]&&e.type&&(t["content-type"]=e.type);let a=void 0!==B.renderOpts.collectedRevalidate&&!(B.renderOpts.collectedRevalidate>=m.INFINITE_CACHE)&&B.renderOpts.collectedRevalidate,r=void 0===B.renderOpts.collectedExpire||B.renderOpts.collectedExpire>=m.INFINITE_CACHE?void 0:B.renderOpts.collectedExpire;return{value:{kind:h.CachedRouteKind.APP_ROUTE,status:i.status,body:Buffer.from(await e.arrayBuffer()),headers:t},cacheControl:{revalidate:a,expire:r}}}}catch(t){throw(null==a?void 0:a.isStale)&&await T.onRequestError(e,t,{routerKind:"App Router",routePath:y,routeType:"route",revalidateReason:(0,c.getRevalidateReason)({isStaticGeneration:J,isOnDemandRevalidate:A})},!1,N),t}},l=await T.handleResponse({req:e,nextConfig:b,cacheKey:F,routeKind:a.RouteKind.APP_ROUTE,isFallback:!1,prerenderManifest:S,isRoutePPREnabled:!1,isOnDemandRevalidate:A,revalidateOnlyGenerated:I,responseGenerator:u,waitUntil:r.waitUntil,isMinimalMode:G});if(!U)return null;if((null==l||null==(i=l.value)?void 0:i.kind)!==h.CachedRouteKind.APP_ROUTE)throw Object.defineProperty(Error(`Invariant: app-route received invalid cache entry ${null==l||null==(s=l.value)?void 0:s.kind}`),"__NEXT_ERROR_CODE",{value:"E701",enumerable:!1,configurable:!0});G||t.setHeader("x-nextjs-cache",A?"REVALIDATED":l.isMiss?"MISS":l.isStale?"STALE":"HIT"),x&&t.setHeader("Cache-Control","private, no-cache, no-store, max-age=0, must-revalidate");let d=(0,f.fromNodeOutgoingHttpHeaders)(l.value.headers);return G&&U||d.delete(m.NEXT_CACHE_TAGS_HEADER),!l.cacheControl||t.getHeader("Cache-Control")||d.get("Cache-Control")||d.set("Cache-Control",(0,w.getCacheControlHeader)(l.cacheControl)),await (0,p.sendResponse)(W,V,new Response(l.value.body,{headers:d,status:l.value.status||200})),null};$&&L?await s(L):(n=H.getActiveScopeSpan(),await H.withPropagatedContext(e.headers,()=>H.trace(d.BaseServerSpan.handleRequest,{spanName:`${M} ${y}`,kind:i.SpanKind.SERVER,attributes:{"http.method":M,"http.target":e.url}},s),void 0,!$))}catch(t){if(t instanceof g.NoFallbackError||await T.onRequestError(e,t,{routerKind:"App Router",routePath:_,routeType:"route",revalidateReason:(0,c.getRevalidateReason)({isStaticGeneration:J,isOnDemandRevalidate:A})},!1,N),U)throw t;return await (0,p.sendResponse)(W,V,new Response(null,{status:500})),null}}e.s(["handler",0,U,"patchFetch",0,function(){return(0,r.patchFetch)({workAsyncStorage:j,workUnitAsyncStorage:P})},"routeModule",0,T,"serverHooks",0,_,"workAsyncStorage",0,j,"workUnitAsyncStorage",0,P],962044)}];
 
 //# sourceMappingURL=%5Broot-of-the-server%5D__0j2l54r._.js.map

package/.next/server/chunks/packages_scraper_src_0kkcv_v._.js

@@ -56,6 +56,6 @@ ${i}
     environment:
 ${n}
     restart: unless-stopped
-`}async function b(){if(!$())return y({running:!0,endpoint:"native://larkup-crawler",apiKey:"native",port:0,project:"native",mode:"native",startedAt:new Date().toISOString(),lastError:void 0});let e=await S();if(!e.compose)return y({...await g(),running:!1,lastError:e.message});let{compose:a}=await v(),n=await g(),o=n.apiKey||`fc-local-${(0,i.randomUUID)()}`,s=n.port||3002,l=await h();await t.promises.mkdir(r.default.dirname(p),{recursive:!0}),await t.promises.writeFile(p,k(o,s,l),"utf8");try{await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(r){let e=r instanceof Error?r.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return y({...n,apiKey:o,port:s,running:!1,lastError:e});try{s=await m(s+1),await t.promises.writeFile(p,k(o,s,l),"utf8"),await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(e){return y({...n,apiKey:o,port:s,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let u=(0,t.existsSync)("/.dockerenv"),d=`http://${u?"host.docker.internal":"localhost"}:${s}`,f=await U(d,9e4);return y({running:f,endpoint:d,apiKey:o,port:s,project:w,mode:"firecrawl",startedAt:new Date().toISOString(),lastError:f?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function E(){let e=await g();if("native"===e.mode)return y({...e,running:!1,startedAt:void 0});let{compose:t}=await v();if(t)try{await c(`${t} -p ${w} -f "${p}" down`,6e4)}catch{}return y({...e,running:!1,startedAt:void 0})}async function A(){let e=await g();if("native"===e.mode||!e.startedAt)return e;let t=await R(e.endpoint);return t!==e.running?y({...e,running:t}):e}async function R(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function U(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(await R(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}function $(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,t.existsSync)("/.dockerenv")}let T="http://firecrawl-api:3002";async function x(){return $()?{available:await R(T),endpoint:T}:{available:!1,endpoint:""}}async function D(){let{available:e}=await x();return e?y({running:!0,endpoint:T,apiKey:`fc-docker-${(0,i.randomUUID)()}`,port:3002,project:"docker-sibling",mode:"firecrawl",startedAt:new Date().toISOString()}):y({...f,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,S,"checkDockerSibling",0,x,"connectDockerSibling",0,D,"readLocalState",0,g,"refreshLocalStatus",0,A,"startLocal",0,b,"stopLocal",0,E])},304368,e=>{"use strict";var t=e.i(367165),r=e.i(700259),a=e.i(902157),i=e.i(750227),n=e.i(666680);let o=i.default.join(process.cwd(),".larkup"),s=i.default.join(o,"native-crawls.json"),l="LarkupCrawler/1.0 (+https://larkup.de)";async function c(){try{return JSON.parse(await a.promises.readFile(s,"utf8"))}catch{return{}}}async function u(e){await a.promises.mkdir(o,{recursive:!0}),await a.promises.writeFile(s,JSON.stringify(e,null,2),"utf8")}function d(e){return e.replace(/&nbsp;/gi," ").replace(/&amp;/gi,"&").replace(/&lt;/gi,"<").replace(/&gt;/gi,">").replace(/&quot;/gi,'"').replace(/&#39;/gi,"'").replace(/&#(x[0-9a-f]+|\d+);/gi,(e,t)=>{let r=String(t).toLowerCase().startsWith("x")?parseInt(String(t).slice(1),16):parseInt(String(t),10);return Number.isFinite(r)?String.fromCodePoint(r):""})}function p(e){return d(e.replace(/<[^>]+>/g," ")).replace(/\s+/g," ").trim()}function w(e){try{let t=new URL(d(e));return f(t.searchParams.get("uddg")||t.toString())}catch{return null}}async function m(e,t=10){let r=Math.max(1,Math.min(t,25)),a={"User-Agent":l,Accept:"text/html,application/xhtml+xml","Accept-Language":"en-US,en;q=0.9"};try{let t=await fetch(`https://search.brave.com/search?q=${encodeURIComponent(e)}&source=web`,{headers:a,redirect:"follow",signal:AbortSignal.timeout(2e4)});if(t.ok){let e=function(e,t){let r=[],a=new Set;for(let i of e.matchAll(/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bl1\b[^"']*["'][^>]*>([\s\S]*?)<\/a>/gi)){let e=w(i[1]);if(!e||a.has(e))continue;a.add(e);let n=i[2].match(/\btitle=["']([^"']+)["']/i);if(r.push({url:e,title:p(n?.[1]||i[2])||e}),r.length>=t)break}return r}(await t.text(),r);if(e.length)return e}}catch{}let i=await fetch(`https://www.bing.com/search?format=rss&q=${encodeURIComponent(e)}`,{headers:{...a,Accept:"application/rss+xml,application/xml,text/xml"},signal:AbortSignal.timeout(2e4)});if(!i.ok)throw Error(`Built-in web search returned ${i.status}.`);let n=function(e,t){let r=[],a=new Set;for(let i of e.matchAll(/<item>([\s\S]*?)<\/item>/gi)){let e=w(h(i[1],"link")??"");if(!(!e||a.has(e))&&(a.add(e),r.push({url:e,title:p(h(i[1],"title")??e),description:p(h(i[1],"description")??"")||void 0}),r.length>=t))break}return r}(await i.text(),r);if(!n.length)throw Error("Built-in web search returned no usable results.");return n}function h(e,t){return e.match(RegExp(`<${t}>([\\s\\S]*?)<\\/${t}>`,"i"))?.[1]}function f(e){try{let t=new URL(e);if(!/^https?:$/.test(t.protocol))return null;return t.hash="",t.toString()}catch{return null}}async function g(e){let t,r=await fetch(e,{headers:{"User-Agent":l,Accept:"text/html,application/xhtml+xml"},redirect:"follow",signal:AbortSignal.timeout(2e4)});if(!r.ok)throw Error(`Website returned ${r.status} ${r.statusText}`);let a=r.headers.get("content-type")??"";if(!/text\/html|application\/xhtml\+xml/i.test(a))throw Error(`Unsupported page type: ${a||"unknown"}`);let i=(await r.text()).slice(0,2e6),n=f(r.url)??e,o=d(i.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi," ").replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi," ").replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi," ").replace(/<(br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section|\/article)\b[^>]*>/gi,"\n").replace(/<[^>]+>/g," ")).replace(/\r/g,"").replace(/[ \t]+/g," ").replace(/\n\s*\n\s*\n+/g,"\n\n").replace(/\s+([.,;:!?])/g,"$1").trim();if(!o)throw Error("The page did not contain readable HTML text.");return{url:n,title:(t=i.match(/<title[^>]*>([\s\S]*?)<\/title>/i))&&d(t[1].replace(/<[^>]+>/g," ").trim())||n,markdown:o,links:function(e,t,r){let a=new Set;for(let i of e.matchAll(/<a\s[^>]*?href\s*=\s*["']([^"'#]+)["']/gi)){let e=f(new URL(i[1],t).toString());e&&new URL(e).origin===r&&a.add(e)}return[...a]}(i,n,new URL(n).origin)}}async function y(e,t){let r=f(e);if(!r)throw Error("A valid http or https URL is required.");let a=`native-${(0,n.randomUUID)()}`,i=await c();return i[a]={id:a,origin:new URL(r).origin,limit:Math.max(1,t),queue:[r],visited:[],pages:[],delivered:0},await u(i),a}async function v(e){let t=await c(),r=t[e];if(!r)throw Error("Native crawl was not found.");if(r.cancelled)return{state:"cancelled",total:r.limit,completed:r.pages.length,pages:[]};for(let e=0;e<2&&r.queue.length&&r.pages.length<r.limit;e++){let e=r.queue.shift();if(!r.visited.includes(e)){r.visited.push(e);try{let t=await g(e);for(let e of(r.pages.push(t),t.links??[]))new URL(e).origin===r.origin&&(r.visited.includes(e)||r.queue.includes(e)||r.queue.push(e))}catch{}}}let a=r.pages.slice(r.delivered).map(({links:e,...t})=>t);r.delivered=r.pages.length;let i=!r.queue.length||r.pages.length>=r.limit;return await u(t),{state:i?"completed":"scraping",total:r.limit,completed:r.pages.length,pages:a}}async function S(e){let t=await c();t[e]&&(t[e].cancelled=!0,await u(t))}class k extends Error{status;constructor(e,t){super(e),this.status=t,this.name="FirecrawlError"}}async function b(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();if(e.running&&e.apiKey)return"native"===e.mode||e.endpoint.startsWith("native://")?{base:e.endpoint,key:e.apiKey,mode:"native"}:{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new k("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let a=e.firecrawlApiKey;if(a)return{base:"https://api.firecrawl.dev/v1",key:a,mode:"cloud"};throw new k("The cloud web crawler needs an API key. Add it in Settings.",401)}async function E(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function A(e,t){let{base:r,key:a}=await b(),i=await fetch(`${r}${e}`,{...t,headers:{Authorization:`Bearer ${a}`,"Content-Type":"application/json",...t?.headers??{}},cache:"no-store"}),n=await i.text(),o=n?U(n):{};if(!i.ok)throw new k(o?.error||`Firecrawl request failed (${i.status})`,i.status);return o}async function R(e){let{key:t}=await b(),r=await fetch(e,{headers:{Authorization:`Bearer ${t}`},cache:"no-store"}),a=await r.text(),i=a?U(a):{};if(!r.ok)throw new k(i?.error||`Firecrawl request failed (${r.status})`,r.status);return i}function U(e){try{return JSON.parse(e)}catch{return{}}}async function $(e,t=10){return"native"===(await b()).mode?m(e,t):((await A("/search",{method:"POST",body:JSON.stringify({query:e,limit:t})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let T={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function x(e){if("native"===(await b()).mode){let t=await g(e);return{url:t.url,title:t.title,markdown:t.markdown}}let t=(await A("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...T})})).data??{};return{url:t.metadata?.sourceURL||e,title:t.metadata?.title||t.metadata?.ogTitle||e,markdown:t.markdown??""}}async function D(e,t){if("native"===(await b()).mode)return y(e,t);let r=await A("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:t,allowExternalLinks:!1,scrapeOptions:{...T,formats:["markdown"],onlyMainContent:!0}})});if(!r.id)throw new k("Firecrawl did not return a crawl id");return r.id}async function L(e,t=!1){if(!t&&e.startsWith("native-"))return v(e);let r=t?await R(e):await A(`/crawl/${e}`,{method:"GET"});return{state:r.status??"scraping",total:r.total??0,completed:r.completed??0,next:r.next,pages:(r.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function C(e){if(e.startsWith("native-"))return void await S(e);try{await A(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,k,"cancelCrawl",0,C,"getCrawlStatus",0,L,"isFirecrawlConfigured",0,E,"scrapePage",0,x,"searchWeb",0,$,"startCrawl",0,D],304368)}];
+`}async function b(){if(!$())return y({running:!0,endpoint:"native://larkup-crawler",apiKey:"native",port:0,project:"native",mode:"native",startedAt:new Date().toISOString(),lastError:void 0});let e=await S();if(!e.compose)return y({...await g(),running:!1,lastError:e.message});let{compose:a}=await v(),n=await g(),o=n.apiKey||`fc-local-${(0,i.randomUUID)()}`,s=n.port||3002,l=await h();await t.promises.mkdir(r.default.dirname(p),{recursive:!0}),await t.promises.writeFile(p,k(o,s,l),"utf8");try{await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(r){let e=r instanceof Error?r.message:"docker compose failed";if(!(e.includes("address already in use")||e.includes("Ports are not available")))return y({...n,apiKey:o,port:s,running:!1,lastError:e});try{s=await m(s+1),await t.promises.writeFile(p,k(o,s,l),"utf8"),await c(`${a} -p ${w} -f "${p}" up -d`,18e4)}catch(e){return y({...n,apiKey:o,port:s,running:!1,lastError:e instanceof Error?e.message:"docker compose retry failed"})}}let u=(0,t.existsSync)("/.dockerenv"),d=`http://${u?"host.docker.internal":"localhost"}:${s}`,f=await U(d,9e4);return y({running:f,endpoint:d,apiKey:o,port:s,project:w,mode:"firecrawl",startedAt:new Date().toISOString(),lastError:f?void 0:"Containers started but the API did not become healthy in time. It may still be warming up — try refreshing in a moment."})}async function E(){let e=await g();if("native"===e.mode)return y({...e,running:!1,startedAt:void 0});let{compose:t}=await v();if(t)try{await c(`${t} -p ${w} -f "${p}" down`,6e4)}catch{}return y({...e,running:!1,startedAt:void 0})}async function A(){let e=await g();if("native"===e.mode||!e.startedAt)return e;let t=await R(e.endpoint);return t!==e.running?y({...e,running:t}):e}async function R(e){try{return(await fetch(e,{method:"GET",signal:AbortSignal.timeout(4e3)})).status>0}catch{return!1}}async function U(e,t){let r=Date.now()+t;for(;Date.now()<r;){if(await R(e))return!0;await new Promise(e=>setTimeout(e,3e3))}return!1}function $(){return"true"===process.env.DOCKER_ENV||"1"===process.env.DOCKER_BUILD||(0,t.existsSync)("/.dockerenv")}let T="http://firecrawl-api:3002";async function x(){return $()?{available:await R(T),endpoint:T}:{available:!1,endpoint:""}}async function D(){let{available:e}=await x();return e?y({running:!0,endpoint:T,apiKey:`fc-docker-${(0,i.randomUUID)()}`,port:3002,project:"docker-sibling",mode:"firecrawl",startedAt:new Date().toISOString()}):y({...f,running:!1,lastError:"The web crawler service is not available in this Docker deployment."})}e.s(["checkDocker",0,S,"checkDockerSibling",0,x,"connectDockerSibling",0,D,"readLocalState",0,g,"refreshLocalStatus",0,A,"startLocal",0,b,"stopLocal",0,E])},304368,e=>{"use strict";var t=e.i(367165),r=e.i(700259),a=e.i(902157),i=e.i(750227),n=e.i(666680);let o=i.default.join(process.cwd(),".larkup"),s=i.default.join(o,"native-crawls"),l=i.default.join(o,"native-crawls.json"),c="LarkupCrawler/1.0 (+https://larkup.de)";async function u(){try{return JSON.parse(await a.promises.readFile(l,"utf8"))}catch{return{}}}function d(e){return i.default.join(s,`${e}.json`)}async function p(e){try{return JSON.parse(await a.promises.readFile(d(e),"utf8"))}catch{return(await u())[e]}}async function w(e){await a.promises.mkdir(s,{recursive:!0}),await a.promises.writeFile(d(e.id),JSON.stringify(e,null,2),"utf8")}function m(e){return e.replace(/&nbsp;/gi," ").replace(/&amp;/gi,"&").replace(/&lt;/gi,"<").replace(/&gt;/gi,">").replace(/&quot;/gi,'"').replace(/&#39;/gi,"'").replace(/&#(x[0-9a-f]+|\d+);/gi,(e,t)=>{let r=String(t).toLowerCase().startsWith("x")?parseInt(String(t).slice(1),16):parseInt(String(t),10);return Number.isFinite(r)?String.fromCodePoint(r):""})}function h(e){return m(e.replace(/<[^>]+>/g," ")).replace(/\s+/g," ").trim()}function f(e){try{let t=new URL(m(e));return v(t.searchParams.get("uddg")||t.toString())}catch{return null}}async function g(e,t=10){let r=Math.max(1,Math.min(t,25)),a={"User-Agent":c,Accept:"text/html,application/xhtml+xml","Accept-Language":"en-US,en;q=0.9"};try{let t=await fetch(`https://search.brave.com/search?q=${encodeURIComponent(e)}&source=web`,{headers:a,redirect:"follow",signal:AbortSignal.timeout(2e4)});if(t.ok){let e=function(e,t){let r=[],a=new Set;for(let i of e.matchAll(/<a\b[^>]*href=["']([^"']+)["'][^>]*class=["'][^"']*\bl1\b[^"']*["'][^>]*>([\s\S]*?)<\/a>/gi)){let e=f(i[1]);if(!e||a.has(e))continue;a.add(e);let n=i[2].match(/\btitle=["']([^"']+)["']/i);if(r.push({url:e,title:h(n?.[1]||i[2])||e}),r.length>=t)break}return r}(await t.text(),r);if(e.length)return e}}catch{}let i=await fetch(`https://www.bing.com/search?format=rss&q=${encodeURIComponent(e)}`,{headers:{...a,Accept:"application/rss+xml,application/xml,text/xml"},signal:AbortSignal.timeout(2e4)});if(!i.ok)throw Error(`Built-in web search returned ${i.status}.`);let n=function(e,t){let r=[],a=new Set;for(let i of e.matchAll(/<item>([\s\S]*?)<\/item>/gi)){let e=f(y(i[1],"link")??"");if(!(!e||a.has(e))&&(a.add(e),r.push({url:e,title:h(y(i[1],"title")??e),description:h(y(i[1],"description")??"")||void 0}),r.length>=t))break}return r}(await i.text(),r);if(!n.length)throw Error("Built-in web search returned no usable results.");return n}function y(e,t){return e.match(RegExp(`<${t}>([\\s\\S]*?)<\\/${t}>`,"i"))?.[1]}function v(e){try{let t=new URL(e);if(!/^https?:$/.test(t.protocol))return null;return t.hash="",t.toString()}catch{return null}}async function S(e){let t,r=await fetch(e,{headers:{"User-Agent":c,Accept:"text/html,application/xhtml+xml"},redirect:"follow",signal:AbortSignal.timeout(2e4)});if(!r.ok)throw Error(`Website returned ${r.status} ${r.statusText}`);let a=r.headers.get("content-type")??"";if(!/text\/html|application\/xhtml\+xml/i.test(a))throw Error(`Unsupported page type: ${a||"unknown"}`);let i=(await r.text()).slice(0,2e6),n=v(r.url)??e,o=m(i.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi," ").replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi," ").replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi," ").replace(/<(br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section|\/article)\b[^>]*>/gi,"\n").replace(/<[^>]+>/g," ")).replace(/\r/g,"").replace(/[ \t]+/g," ").replace(/\n\s*\n\s*\n+/g,"\n\n").replace(/\s+([.,;:!?])/g,"$1").trim();if(!o)throw Error("The page did not contain readable HTML text.");return{url:n,title:(t=i.match(/<title[^>]*>([\s\S]*?)<\/title>/i))&&m(t[1].replace(/<[^>]+>/g," ").trim())||n,markdown:o,links:function(e,t,r){let a=new Set;for(let i of e.matchAll(/<a\s[^>]*?href\s*=\s*["']([^"'#]+)["']/gi)){let e=v(new URL(i[1],t).toString());e&&new URL(e).origin===r&&a.add(e)}return[...a]}(i,n,new URL(n).origin)}}async function k(e,t){let r=v(e);if(!r)throw Error("A valid http or https URL is required.");let a=`native-${(0,n.randomUUID)()}`;return await w({id:a,origin:new URL(r).origin,limit:Math.max(1,t),queue:[r],visited:[],pages:[],delivered:0}),a}async function b(e){let t=await p(e);if(!t)throw Error("Native crawl was not found.");if(t.cancelled)return{state:"cancelled",total:t.limit,completed:t.pages.length,pages:[]};for(let e=0;e<2&&t.queue.length&&t.pages.length<t.limit;e++){let e=t.queue.shift();if(!t.visited.includes(e)){t.visited.push(e);try{let r=await S(e);for(let e of(t.pages.push(r),r.links??[]))new URL(e).origin===t.origin&&(t.visited.includes(e)||t.queue.includes(e)||t.queue.push(e))}catch{}}}let r=t.pages.slice(t.delivered).map(({links:e,...t})=>t);t.delivered=t.pages.length;let a=!t.queue.length||t.pages.length>=t.limit;return await w(t),{state:a?"completed":"scraping",total:t.limit,completed:t.pages.length,pages:r}}async function E(e){let t=await p(e);t&&(t.cancelled=!0,await w(t))}class A extends Error{status;constructor(e,t){super(e),this.status=t,this.name="FirecrawlError"}}async function R(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();if(e.running&&e.apiKey)return"native"===e.mode||e.endpoint.startsWith("native://")?{base:e.endpoint,key:e.apiKey,mode:"native"}:{base:`${e.endpoint}/v1`,key:e.apiKey,mode:"local"};throw new A("The selected web crawler is not running. Choose a crawler provider in Settings.",401)}let a=e.firecrawlApiKey;if(a)return{base:"https://api.firecrawl.dev/v1",key:a,mode:"cloud"};throw new A("The cloud web crawler needs an API key. Add it in Settings.",401)}async function U(){let e=await (0,t.readConfig)();if("local"===(e.webCrawlerProvider||"local")){let e=await (0,r.readLocalState)();return!!(e.running&&e.apiKey)}return!!e.firecrawlApiKey}async function $(e,t){let{base:r,key:a}=await R(),i=await fetch(`${r}${e}`,{...t,headers:{Authorization:`Bearer ${a}`,"Content-Type":"application/json",...t?.headers??{}},cache:"no-store"}),n=await i.text(),o=n?x(n):{};if(!i.ok)throw new A(o?.error||`Firecrawl request failed (${i.status})`,i.status);return o}async function T(e){let{key:t}=await R(),r=await fetch(e,{headers:{Authorization:`Bearer ${t}`},cache:"no-store"}),a=await r.text(),i=a?x(a):{};if(!r.ok)throw new A(i?.error||`Firecrawl request failed (${r.status})`,r.status);return i}function x(e){try{return JSON.parse(e)}catch{return{}}}async function D(e,t=10){return"native"===(await R()).mode?g(e,t):((await $("/search",{method:"POST",body:JSON.stringify({query:e,limit:t})})).data??[]).map(e=>({url:e.url,title:e.title||e.url,description:e.description}))}let L={waitFor:1e4,onlyMainContent:!1,headers:{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36","Accept-Language":"en-US,en;q=0.9","Sec-Ch-Ua":'"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"Windows"'}};async function C(e){if("native"===(await R()).mode){let t=await S(e);return{url:t.url,title:t.title,markdown:t.markdown}}let t=(await $("/scrape",{method:"POST",body:JSON.stringify({url:e,formats:["markdown"],...L})})).data??{};return{url:t.metadata?.sourceURL||e,title:t.metadata?.title||t.metadata?.ogTitle||e,markdown:t.markdown??""}}async function P(e,t){if("native"===(await R()).mode)return k(e,t);let r=await $("/crawl",{method:"POST",body:JSON.stringify({url:e,limit:t,allowExternalLinks:!1,scrapeOptions:{...L,formats:["markdown"],onlyMainContent:!0}})});if(!r.id)throw new A("Firecrawl did not return a crawl id");return r.id}async function _(e,t=!1){if(!t&&e.startsWith("native-"))return b(e);let r=t?await T(e):await $(`/crawl/${e}`,{method:"GET"});return{state:r.status??"scraping",total:r.total??0,completed:r.completed??0,next:r.next,pages:(r.data??[]).map(e=>({url:e.metadata?.sourceURL||e.metadata?.url||"",title:e.metadata?.title||e.metadata?.sourceURL||"Untitled",markdown:e.markdown??""}))}}async function O(e){if(e.startsWith("native-"))return void await E(e);try{await $(`/crawl/${e}`,{method:"DELETE"})}catch{}}e.s(["FirecrawlError",0,A,"cancelCrawl",0,O,"getCrawlStatus",0,_,"isFirecrawlConfigured",0,U,"scrapePage",0,C,"searchWeb",0,D,"startCrawl",0,P],304368)}];
 
 //# sourceMappingURL=packages_scraper_src_0kkcv_v._.js.map

package/.next/server/chunks/ssr/{[root-of-the-server]__0m21fli._.js → [root-of-the-server]__0pgotbk._.js}

@@ -1,3 +1,3 @@
-module.exports=[918622,(a,b,c)=>{b.exports=a.x("next/dist/compiled/next-server/app-page-turbo.runtime.prod.js",()=>require("next/dist/compiled/next-server/app-page-turbo.runtime.prod.js"))},556704,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/work-async-storage.external.js",()=>require("next/dist/server/app-render/work-async-storage.external.js"))},832319,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/work-unit-async-storage.external.js",()=>require("next/dist/server/app-render/work-unit-async-storage.external.js"))},324725,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/after-task-async-storage.external.js",()=>require("next/dist/server/app-render/after-task-async-storage.external.js"))},120635,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/action-async-storage.external.js",()=>require("next/dist/server/app-render/action-async-storage.external.js"))},43285,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/dynamic-access-async-storage.external.js",()=>require("next/dist/server/app-render/dynamic-access-async-storage.external.js"))},440463,(a,b,c)=>{"use strict";b.exports=a.r(918622)},887556,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].React},360809,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactJsxRuntime},199793,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactDOM},104842,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.AppRouterContext},527797,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.HooksClientContext},641048,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.ServerInsertedHtml},495580,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactServerDOMTurbopackClient},816995,a=>{"use strict";let b=(0,a.i(326729).default)("chevron-left",[["path",{d:"m15 18-6-6 6-6",key:"1wnfg3"}]]);a.s(["ChevronLeft",0,b],816995)},426490,a=>{"use strict";let b=(0,a.i(326729).default)("rotate-ccw",[["path",{d:"M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8",key:"1357e3"}],["path",{d:"M3 3v5h5",key:"1xhq8a"}]]);a.s(["RotateCcw",0,b],426490)},566005,a=>{"use strict";let b=(0,a.i(326729).default)("settings",[["path",{d:"M9.671 4.136a2.34 2.34 0 0 1 4.659 0 2.34 2.34 0 0 0 3.319 1.915 2.34 2.34 0 0 1 2.33 4.033 2.34 2.34 0 0 0 0 3.831 2.34 2.34 0 0 1-2.33 4.033 2.34 2.34 0 0 0-3.319 1.915 2.34 2.34 0 0 1-4.659 0 2.34 2.34 0 0 0-3.32-1.915 2.34 2.34 0 0 1-2.33-4.033 2.34 2.34 0 0 0 0-3.831A2.34 2.34 0 0 1 6.35 6.051a2.34 2.34 0 0 0 3.319-1.915",key:"1i5ecw"}],["circle",{cx:"12",cy:"12",r:"3",key:"1v7zrd"}]]);a.s(["Settings",0,b],566005)},216615,a=>{"use strict";let b=(0,a.i(326729).default)("arrow-up",[["path",{d:"m5 12 7-7 7 7",key:"hav0vg"}],["path",{d:"M12 19V5",key:"x0mq9r"}]]);a.s(["ArrowUp",0,b],216615)},914053,a=>{a.v(b=>Promise.all(["server/chunks/ssr/node_modules__pnpm_0xqqdqu._.js"].map(b=>a.l(b))).then(()=>b(960091)))},69832,a=>{a.v(b=>Promise.all(["server/chunks/ssr/node_modules__pnpm_19epw40._.js"].map(b=>a.l(b))).then(()=>b(858630)))}];
+module.exports=[918622,(a,b,c)=>{b.exports=a.x("next/dist/compiled/next-server/app-page-turbo.runtime.prod.js",()=>require("next/dist/compiled/next-server/app-page-turbo.runtime.prod.js"))},556704,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/work-async-storage.external.js",()=>require("next/dist/server/app-render/work-async-storage.external.js"))},832319,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/work-unit-async-storage.external.js",()=>require("next/dist/server/app-render/work-unit-async-storage.external.js"))},324725,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/after-task-async-storage.external.js",()=>require("next/dist/server/app-render/after-task-async-storage.external.js"))},120635,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/action-async-storage.external.js",()=>require("next/dist/server/app-render/action-async-storage.external.js"))},43285,(a,b,c)=>{b.exports=a.x("next/dist/server/app-render/dynamic-access-async-storage.external.js",()=>require("next/dist/server/app-render/dynamic-access-async-storage.external.js"))},440463,(a,b,c)=>{"use strict";b.exports=a.r(918622)},887556,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].React},360809,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactJsxRuntime},199793,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactDOM},104842,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.AppRouterContext},527797,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.HooksClientContext},641048,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored.contexts.ServerInsertedHtml},495580,(a,b,c)=>{"use strict";b.exports=a.r(440463).vendored["react-ssr"].ReactServerDOMTurbopackClient},816995,a=>{"use strict";let b=(0,a.i(326729).default)("chevron-left",[["path",{d:"m15 18-6-6 6-6",key:"1wnfg3"}]]);a.s(["ChevronLeft",0,b],816995)},566005,a=>{"use strict";let b=(0,a.i(326729).default)("settings",[["path",{d:"M9.671 4.136a2.34 2.34 0 0 1 4.659 0 2.34 2.34 0 0 0 3.319 1.915 2.34 2.34 0 0 1 2.33 4.033 2.34 2.34 0 0 0 0 3.831 2.34 2.34 0 0 1-2.33 4.033 2.34 2.34 0 0 0-3.319 1.915 2.34 2.34 0 0 1-4.659 0 2.34 2.34 0 0 0-3.32-1.915 2.34 2.34 0 0 1-2.33-4.033 2.34 2.34 0 0 0 0-3.831A2.34 2.34 0 0 1 6.35 6.051a2.34 2.34 0 0 0 3.319-1.915",key:"1i5ecw"}],["circle",{cx:"12",cy:"12",r:"3",key:"1v7zrd"}]]);a.s(["Settings",0,b],566005)},426490,a=>{"use strict";let b=(0,a.i(326729).default)("rotate-ccw",[["path",{d:"M3 12a9 9 0 1 0 9-9 9.75 9.75 0 0 0-6.74 2.74L3 8",key:"1357e3"}],["path",{d:"M3 3v5h5",key:"1xhq8a"}]]);a.s(["RotateCcw",0,b],426490)},216615,a=>{"use strict";let b=(0,a.i(326729).default)("arrow-up",[["path",{d:"m5 12 7-7 7 7",key:"hav0vg"}],["path",{d:"M12 19V5",key:"x0mq9r"}]]);a.s(["ArrowUp",0,b],216615)},914053,a=>{a.v(b=>Promise.all(["server/chunks/ssr/node_modules__pnpm_0xqqdqu._.js"].map(b=>a.l(b))).then(()=>b(960091)))},69832,a=>{a.v(b=>Promise.all(["server/chunks/ssr/node_modules__pnpm_19epw40._.js"].map(b=>a.l(b))).then(()=>b(858630)))}];
 
-//# sourceMappingURL=%5Broot-of-the-server%5D__0m21fli._.js.map
+//# sourceMappingURL=%5Broot-of-the-server%5D__0pgotbk._.js.map

package/.next/server/chunks/ssr/[root-of-the-server]__1k1mxo2._.js

@@ -0,0 +1,3 @@
+module.exports=[688947,(a,b,c)=>{b.exports=a.x("stream",()=>require("stream"))},683389,a=>{a.v(b=>Promise.all(["server/chunks/ssr/apps_web_lib_pdf-images_ts_13-l5wa._.js","server/chunks/ssr/0uph_pdfjs-dist_build_pdf_mjs_0pn2q_y._.js"].map(b=>a.l(b))).then(()=>b(802183)))}];
+
+//# sourceMappingURL=%5Broot-of-the-server%5D__1k1mxo2._.js.map