@redocly/realm 0.135.1 → 0.135.2

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @redocly/realm
 
+## 0.135.2
+
+### Patch Changes
+
+- c6f35a51db4: Hardened access control of `llms.txt` on the homepage Markdown page.
+- c6f35a51db4: Improved response time on sites with many pages that use the `llms.txt` feature.
+
 ## 0.135.1
 
 ### Patch Changes

package/dist/server/plugins/search/ai-indexer/prepare-semantic-documents.js

@@ -1,3 +1,3 @@
-import p from"node:path";import{existsSync as X}from"node:fs";import{writeFile as D,access as N,rm as k,constants as B}from"fs/promises";import{REDOCLY_ROUTE_RBAC as _,REDOCLY_TEAMS_RBAC as h}from"@redocly/config";import U from"picomatch";import{combineUrls as j}from"@redocly/theme/core/utils";import{DEFAULT_LOCALE_PLACEHOLDER as E}from"../../../../constants/common.js";import{AI_INDEX_EXPORT_FOLDER as G}from"../../../constants/plugins/search.js";import{LLMS_TXT_FILE_NAME as H}from"../../../constants/common.js";import{envConfig as C}from"../../../config/env-config.js";import{logger as a}from"../../../tools/notifiers/logger.js";import{shaHexShort as Y}from"../../../utils/crypto/sha-hex-short.js";import{promiseMapLimit as P}from"../../../utils/async/promise-map-limit.js";import{getLlmsTxtMdPathBySlug as K}from"../../../utils/llmstxt/get-llms-txt-md-path-by-slug.js";import{buildAgentFeedbackLlmsFooter as J}from"../../../utils/llmstxt/agent-feedback-llms-footer.js";import{validateLLMsTxtConfig as V,generateLLMsTxt as W}from"../llmstxt/index.js";import{ensureDir as F}from"../../../utils/fs.js";import{isResourcePubliclyAccessible as q,extractTeamsFromScopeItems as z,getRbacTeamsListForResource as Q}from"../../../utils/rbac.js";const w=20,S="llms.txt:";async function Dt(t,i,{embeddingsEnabled:o,llmstxtEnabled:n}){const c=p.join(t.outdir,H);if(!n){try{await N(c,B.W_OK),await k(c)}catch{}if(!o)return}o&&a.info("Preparing semantic documents..."),n&&a.info(`${S} Generating llms.txt files...`);const T=a.startTiming(),l=a.startTiming(),m=t.getGlobalConfig("seo"),f=m?.llmstxt,d=C.REDOCLY_EXP_LLMSTXT_AGENT_FEEDBACK_ENABLED??!1,M=p.resolve(t.outdir,G),u=t.getConfig(),O=u.access?.rbac??{},b=t.getAllRoutes(),y=u?.l10n?.defaultLocale||u?.i18n?.defaultLocale||E,I=(f?.excludeFiles||[]).map(e=>U(e)),R=[];if(await P(b,w,async e=>{if(e.excludeFromSearch)return;const v=await Z(e,t),x=await tt(e,v,i,t);if(x){if(n)for(const s of await x.getLLMsTxts()){if(I.some($=>$(s.fsPath)))continue;const g=K(s.slug),r=p.join(t.outdir,g);F(r);const L=d?J(m,s.slug):void 0,A=L?`
+import L from"node:path";import{existsSync as N}from"node:fs";import{writeFile as R,access as X,rm as B,constants as k}from"fs/promises";import{REDOCLY_ROUTE_RBAC as _,REDOCLY_TEAMS_RBAC as h}from"@redocly/config";import j from"picomatch";import{DEFAULT_LOCALE_PLACEHOLDER as E}from"../../../../constants/common.js";import{AI_INDEX_EXPORT_FOLDER as G}from"../../../constants/plugins/search.js";import{LLMS_TXT_FILE_NAME as Y}from"../../../constants/common.js";import{envConfig as C}from"../../../config/env-config.js";import{logger as r}from"../../../tools/notifiers/logger.js";import{shaHexShort as H}from"../../../utils/crypto/sha-hex-short.js";import{promiseMapLimit as P}from"../../../utils/async/promise-map-limit.js";import{getLlmsTxtMdPathBySlug as K}from"../../../utils/llmstxt/get-llms-txt-md-path-by-slug.js";import{buildAgentFeedbackLlmsFooter as U}from"../../../utils/llmstxt/agent-feedback-llms-footer.js";import{validateLLMsTxtConfig as J,generateLLMsTxt as V}from"../llmstxt/index.js";import{ensureDir as F}from"../../../utils/fs.js";import{isResourcePubliclyAccessible as W,extractTeamsFromScopeItems as q,getRbacTeamsListForResource as z}from"../../../utils/rbac.js";const w=20,S="llms.txt:";async function At(t,i,{embeddingsEnabled:o,llmstxtEnabled:s}){const c=L.join(t.outdir,Y);if(!s){try{await X(c,k.W_OK),await B(c)}catch{}if(!o)return}o&&r.info("Preparing semantic documents..."),s&&r.info(`${S} Generating llms.txt files...`);const p=r.startTiming(),l=r.startTiming(),m=t.getGlobalConfig("seo"),f=m?.llmstxt,T=C.REDOCLY_EXP_LLMSTXT_AGENT_FEEDBACK_ENABLED??!1,M=L.resolve(t.outdir,G),u=t.getConfig(),O=u.access?.rbac??{},y=t.getAllRoutes(),b=u?.l10n?.defaultLocale||u?.i18n?.defaultLocale||E,I=(f?.excludeFiles||[]).map(e=>j(e)),d=[];if(await P(y,w,async e=>{if(e.excludeFromSearch)return;const $=await Q(e,t),x=await Z(e,$,i,t);if(x){if(s)for(const n of await x.getLLMsTxts()){if(I.some(v=>v(n.fsPath)))continue;const A=K(n.slug),a=L.join(t.outdir,A);F(a);const g=T?U(m,n.slug):void 0,D=g?`
 
-${L}`:"";await D(r,`${s.content}${A}`),t.setResourceResponseHeaders(g,[{name:"X-Robots-Tag",value:"noindex, follow"},...u.seo?.siteUrl?[{name:"Link",value:`<${j(u.seo.siteUrl,s.slug)}>; rel="canonical"`}]:[]]),s.includeInLLMsTxt&&q(e,u)&&R.push(s)}if(o&&C.isBuildMode){const s=await x.getSearchDocuments();if(!s.length)return;const g=et(e,O),r=e.versions?.find(({active:A})=>A),L=r&&{folder:r.folderId,label:r.label,default:r.default};await ot(s,e.fsPath,g,M,y,L)}}}),o&&a.infoTime(l,"Semantic search documents prepared"),R.length)try{V(f);const e=await W(R,f,{title:m?.title,description:m?.description},i);F(c),await D(c,e),a.infoTime(T,`${S} files generated`)}catch(e){a.error(`${S} Failed to generate llms.txt file. ${e.message}`)}}async function Z(t,i){return t.getStaticData?t.getStaticData(t,i):{}}async function tt(t,i,o,n){if(t.getAiDocumentsStore)return t.getAiDocumentsStore(t,{...i,[h]:t[h],[_]:t[_]},o,n)}function et(t,i){const o=z(t?.[h]);return o?.length?o:Q(t,i)}async function ot(t,i,o,n,c,T){await P(t,w,async l=>{const m=`${Y(i+l.content)}.json`,f=F(p.join(n,m)),d=l.locale===E?c:l.locale;X(f)||await D(f,JSON.stringify({...l,rbacTeams:o||[],version:T,locale:d}),"utf-8")})}export{Dt as prepareSemanticDocuments};
+${g}`:"";await R(a,`${n.content}${D}`),n.includeInLLMsTxt&&W(e,u)&&d.push(n)}if(o&&C.isBuildMode){const n=await x.getSearchDocuments();if(!n.length)return;const A=tt(e,O),a=e.versions?.find(({active:D})=>D),g=a&&{folder:a.folderId,label:a.label,default:a.default};await et(n,e.fsPath,A,M,b,g)}}}),o&&r.infoTime(l,"Semantic search documents prepared"),d.length)try{J(f);const e=await V(d,f,{title:m?.title,description:m?.description},i);F(c),await R(c,e),r.infoTime(p,`${S} files generated`)}catch(e){r.error(`${S} Failed to generate llms.txt file. ${e.message}`)}}async function Q(t,i){return t.getStaticData?t.getStaticData(t,i):{}}async function Z(t,i,o,s){if(t.getAiDocumentsStore)return t.getAiDocumentsStore(t,{...i,[h]:t[h],[_]:t[_]},o,s)}function tt(t,i){const o=q(t?.[h]);return o?.length?o:z(t,i)}async function et(t,i,o,s,c,p){await P(t,w,async l=>{const m=`${H(i+l.content)}.json`,f=F(L.join(s,m)),T=l.locale===E?c:l.locale;N(f)||await R(f,JSON.stringify({...l,rbacTeams:o||[],version:p,locale:T}),"utf-8")})}export{At as prepareSemanticDocuments};

package/dist/server/web-server/routes/dynamic-route.js

@@ -1 +1 @@
-import m from"path";import{REDOCLY_ROUTE_RBAC as U,REDOCLY_TEAMS_RBAC as b}from"@redocly/config";import{withoutPathPrefix as j,withPathPrefix as V}from"@redocly/theme/core/utils";import{CACHE_CONTROL_NO_STORE_HEADER_VALUE as M,CACHE_CONTROL_PUBLIC_HTML as W,DEFAULT_IMMUTABLE_CACHE_MAX_AGE as Z,X_REDOCLY_CACHE_CONTROL_HEADER as J,X_REDOCLY_CACHE_CONTROL_PUBLIC_HTML as K}from"../../constants/common.js";import{removeTrailingSlash as Q}from"../../../utils/url/remove-trailing-slash.js";import{findInIterable as x}from"../../../utils/collection/find-in-iterable.js";import{sanitizeRedirectPathname as ee}from"../../../utils/url/sanitize-redirect-pathname.js";import{envConfig as te}from"../../config/env-config.js";import{sanitizePath as ne}from"../../../utils/path/sanitize-path.js";import{normalizeRouteSlug as oe}from"../../../utils/path/normalize-route-slug.js";import{isPathInFolder as re}from"../../../utils/path/is-path-in-folder.js";import{getLlmsTxtMdPathBySlug as ae}from"../../utils/llmstxt/get-llms-txt-md-path-by-slug.js";import{removeLeadingSlash as D}from"../../../utils/url/remove-leading-slash.js";import{processRedirects as ie}from"./helpers/process-redirects.js";import{renderPage as se,getServerProps as ce}from"../../ssr/index.js";import{canAccessAsset as le,canAccessResource as H}from"../../utils/rbac.js";import{handleErrorPageRender as q,handleUnauthorized as v,handleUnauthorizedAsset as B}from"../utils.js";import{DEFAULT_MAX_AGE_FOR_MIME_TYPE as me,MIME_TYPES as ue}from"../mime-types.js";import{fileExistsAsync as F}from"../../utils/index.js";import{isAiAgentRequest as de}from"../../utils/ai-agent-detection.js";import{getRedirectRoute as ge}from"../utils/legacy-openapi-redirects.js";import{getContentTypeHeaderValue as fe}from"../utils/content-type.js";import{telemetry as pe}from"../../../cli/telemetry/index.js";import{telemetry as he}from"../../telemetry/index.js";function Xe(e,z,Y){return async n=>{const R=n.get("logger"),a=n.req,i=new URL(a.url),r=j(ne(decodeURIComponent(i.pathname))),A=m.parse(r).ext===".md",E=de({accept:a.header("accept"),signatureAgent:a.header("signature-agent"),signature:a.header("signature"),signatureInput:a.header("signature-input"),userAgent:a.header("user-agent")}),l=oe(r),o=(a.method==="GET"||a.method==="HEAD")&&!A?e.getRouteBySlug(l,{followRedirect:!1})||x(e.routesBySlug?.values(),t=>t.hasClientRoutes&&(r===t.slug||r.startsWith(t.slug+"/"))):void 0,L=e.getRedirect(l);if(L){const t=ie({redirect:L,reqUrlSearch:i.search});return he.sendRedirectMessage([{object:"redirect",from:l,templateId:t.type.toString()}]),n.newResponse(null,t.type,{Location:t.location})}const f=te.isProductionEnv?301:302;if(o?.metadata?.type==="openapi"){const t=ge(i.pathname);if(t)return R.info("Legacy OpenAPI docs redirect from "+i.pathname),n.newResponse(null,f,{Location:encodeURI(t)});if(i.pathname.match(/[A-Z]/))return R.warn("Redirect to lowercase route to avoid 404 error"),n.newResponse(null,f,{Location:encodeURI(i.pathname.toLowerCase())})}if(r.endsWith("/")&&r!=="/"){const t=ee(new URL(l||"/",n.req.url).pathname);return n.newResponse(null,f,{Location:encodeURI(V((t==="/"?"/":Q(t))+i.search))})}const u=o&&E?ae(o.slug):void 0,_=u?await F(m.resolve(e.outdir,D(u))):!1,{isAuthenticated:d,teams:p,claims:{email:h}}=n.get("auth");if(o&&!H(o,{isAuthenticated:d,email:h,teams:p},e.config.access?.rbac,e.config.access?.requiresLogin))return d?q(n,e,{slug:o.slug,[b]:o[b],[U]:o[U]},403):v(n,e,o.slug);if(o&&(!E||!_)){const t=await z(o),c=await ce(o,n,t,e),{html:X,statusCode:y,error:k}=await se(o,c,n,e,pe),O=e.config.access?.rbac,I=!e.config.access?.requiresLogin&&(!O||!Object.keys(O).length),S=y>=400||!!k||!!e.compilationErrors?.length,$=I&&!S;let g;return S?g=M:I?g=W:g="private",n.html(X,y,{"Cache-Control":g,...$&&{[J]:K}})}const w=u&&_?u:r,G=D(w),s=m.resolve(e.outdir,G);if(!re(s,e.outdir))return B(n);if(A){const t=r==="index.html.md"?"/":r.replace(/\.md$/,""),c=e.getRouteBySlug(t,{followRedirect:!1});if(c&&!H(c,{isAuthenticated:d,email:h,teams:p},e.config.access?.rbac,e.config.access?.requiresLogin))return v(n,e,r)}const P=e.getGlobalConfig("access");if(!le(w,P?.rbac||{},P?.requiresLogin||!1,e.getGlobalConfig("directoryPaths"),{isAuthenticated:d,email:h,teams:p}))return B(n);const T=ue[m.extname(s)]||"text/plain",C=s.match(/assets\/.*\.[a-f0-9]{8,}\..+/)||s.match(/runtime\/chunks\/.*/)?Z:me[T],N=C?{"Cache-Control":`public, max-age=${C}, immutable`,Expires:new Date(Date.now()+C*1e3).toUTCString()}:{"Cache-Control":M};if(await F(s)){const t=a.query("download")!=null,c=await Y(s);return n.newResponse(c,200,{"Content-Type":fe(T),"Access-Control-Allow-Origin":"*",...N,...t&&{"Content-Disposition":`attachment; filename="${m.basename(s)}"`}})}else return q(n,e,{slug:l},404)}}export{Xe as dynamicRouteHandler};
+import m from"path";import{REDOCLY_ROUTE_RBAC as M,REDOCLY_TEAMS_RBAC as S}from"@redocly/config";import{combineUrls as V,withoutPathPrefix as W,withPathPrefix as Z}from"@redocly/theme/core/utils";import{CACHE_CONTROL_NO_STORE_HEADER_VALUE as D,CACHE_CONTROL_PUBLIC_HTML as J,DEFAULT_IMMUTABLE_CACHE_MAX_AGE as K,X_REDOCLY_CACHE_CONTROL_HEADER as Q,X_REDOCLY_CACHE_CONTROL_PUBLIC_HTML as x}from"../../constants/common.js";import{removeTrailingSlash as ee}from"../../../utils/url/remove-trailing-slash.js";import{findInIterable as te}from"../../../utils/collection/find-in-iterable.js";import{sanitizeRedirectPathname as ne}from"../../../utils/url/sanitize-redirect-pathname.js";import{envConfig as oe}from"../../config/env-config.js";import{sanitizePath as re}from"../../../utils/path/sanitize-path.js";import{normalizeRouteSlug as ie}from"../../../utils/path/normalize-route-slug.js";import{isPathInFolder as ae}from"../../../utils/path/is-path-in-folder.js";import{getLlmsTxtMdPathBySlug as se}from"../../utils/llmstxt/get-llms-txt-md-path-by-slug.js";import{removeLeadingSlash as H}from"../../../utils/url/remove-leading-slash.js";import{processRedirects as ce}from"./helpers/process-redirects.js";import{renderPage as le,getServerProps as me}from"../../ssr/index.js";import{canAccessAsset as ue,canAccessResource as q}from"../../utils/rbac.js";import{handleErrorPageRender as v,handleUnauthorized as B,handleUnauthorizedAsset as F}from"../utils.js";import{DEFAULT_MAX_AGE_FOR_MIME_TYPE as de,MIME_TYPES as ge}from"../mime-types.js";import{fileExistsAsync as z}from"../../utils/index.js";import{isAiAgentRequest as fe}from"../../utils/ai-agent-detection.js";import{getRedirectRoute as pe}from"../utils/legacy-openapi-redirects.js";import{getContentTypeHeaderValue as he}from"../utils/content-type.js";import{telemetry as Re}from"../../../cli/telemetry/index.js";import{telemetry as Ce}from"../../telemetry/index.js";function $e(e,Y,G){return async n=>{const A=n.get("logger"),i=n.req,a=new URL(i.url),r=W(re(decodeURIComponent(a.pathname))),E=m.parse(r).ext===".md",L=fe({accept:i.header("accept"),signatureAgent:i.header("signature-agent"),signature:i.header("signature"),signatureInput:i.header("signature-input"),userAgent:i.header("user-agent")}),c=ie(r),o=(i.method==="GET"||i.method==="HEAD")&&!E?e.getRouteBySlug(c,{followRedirect:!1})||te(e.routesBySlug?.values(),t=>t.hasClientRoutes&&(r===t.slug||r.startsWith(t.slug+"/"))):void 0,_=e.getRedirect(c);if(_){const t=ce({redirect:_,reqUrlSearch:a.search});return Ce.sendRedirectMessage([{object:"redirect",from:c,templateId:t.type.toString()}]),n.newResponse(null,t.type,{Location:t.location})}const f=oe.isProductionEnv?301:302;if(o?.metadata?.type==="openapi"){const t=pe(a.pathname);if(t)return A.info("Legacy OpenAPI docs redirect from "+a.pathname),n.newResponse(null,f,{Location:encodeURI(t)});if(a.pathname.match(/[A-Z]/))return A.warn("Redirect to lowercase route to avoid 404 error"),n.newResponse(null,f,{Location:encodeURI(a.pathname.toLowerCase())})}if(r.endsWith("/")&&r!=="/"){const t=ne(new URL(c||"/",n.req.url).pathname);return n.newResponse(null,f,{Location:encodeURI(Z((t==="/"?"/":ee(t))+a.search))})}const u=o&&L?se(o.slug):void 0,w=u?await z(m.resolve(e.outdir,H(u))):!1,{isAuthenticated:d,teams:p,claims:{email:h}}=n.get("auth");if(o&&!q(o,{isAuthenticated:d,email:h,teams:p},e.config.access?.rbac,e.config.access?.requiresLogin))return d?v(n,e,{slug:o.slug,[S]:o[S],[M]:o[M]},403):B(n,e,o.slug);if(o&&(!L||!w)){const t=await Y(o),C=await me(o,n,t,e),{html:k,statusCode:y,error:$}=await le(o,C,n,e,Re),b=e.config.access?.rbac,O=!e.config.access?.requiresLogin&&(!b||!Object.keys(b).length),I=y>=400||!!$||!!e.compilationErrors?.length,j=O&&!I;let g;return I?g=D:O?g=J:g="private",n.html(k,y,{"Cache-Control":g,...j&&{[Q]:x}})}const P=u&&w?u:r,N=H(P),s=m.resolve(e.outdir,N);if(!ae(s,e.outdir))return F(n);let l;if(E){const t=r==="/index.html.md"?"/":r.replace(/\.md$/,"");if(l=e.getRouteBySlug(t,{followRedirect:!1}),l&&!q(l,{isAuthenticated:d,email:h,teams:p},e.config.access?.rbac,e.config.access?.requiresLogin))return B(n,e,r)}const T=e.getGlobalConfig("access");if(!ue(P,T?.rbac||{},T?.requiresLogin||!1,e.getGlobalConfig("directoryPaths"),{isAuthenticated:d,email:h,teams:p}))return F(n);const U=ge[m.extname(s)]||"text/plain",R=s.match(/assets\/.*\.[a-f0-9]{8,}\..+/)||s.match(/runtime\/chunks\/.*/)?K:de[U],X=R?{"Cache-Control":`public, max-age=${R}, immutable`,Expires:new Date(Date.now()+R*1e3).toUTCString()}:{"Cache-Control":D};if(await z(s)){const t=i.query("download")!=null,C=await G(s);return n.newResponse(C,200,{"Content-Type":he(U),"Access-Control-Allow-Origin":"*",...X,...l&&{"X-Robots-Tag":"noindex, follow",...e.config.seo?.siteUrl&&{Link:`<${V(e.config.seo.siteUrl,l.slug)}>; rel="canonical"`}},...t&&{"Content-Disposition":`attachment; filename="${m.basename(s)}"`}})}else return v(n,e,{slug:c},404)}}export{$e as dynamicRouteHandler};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redocly/realm",
-  "version": "0.135.1",
+  "version": "0.135.2",
   "description": "",
   "type": "module",
   "bin": {
@@ -93,14 +93,14 @@
     "xpath": "0.0.34",
     "yaml-ast-parser": "0.0.43",
     "zod": "^3.25.76",
-    "@redocly/config": "0.50.1",
     "@redocly/asyncapi-docs": "1.12.1",
-    "@redocly/openapi-docs": "3.23.1",
+    "@redocly/portal-legacy-ui": "0.18.0",
     "@redocly/graphql-docs": "1.12.1",
     "@redocly/realm-asyncapi-sdk": "0.13.0",
     "@redocly/theme": "0.67.0",
     "@redocly/portal-plugin-mock-server": "0.20.1",
-    "@redocly/portal-legacy-ui": "0.18.0"
+    "@redocly/config": "0.50.1",
+    "@redocly/openapi-docs": "3.23.1"
   },
   "peerDependencies": {
     "react": "^19.2.4",