ghost 6.55.0 → 6.57.0

package/core/server/services/url/lazy-url-service.js

@@ -3,21 +3,21 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LazyUrlService = void 0;
 const debug = require('@tryghost/debug')('services:url:lazy');
 const errors = require('@tryghost/errors');
+const logging = require('@tryghost/logging');
 const localUtils = require('../../../shared/url-utils').default;
 const { matchPermalink, toLookupParams } = require('./permalink-matcher');
 const { buildFilter, filterMatches, routerTypeOf } = require('./router-filter');
+const { fetchRoutableResources: defaultFetchRoutableResources } = require('./routable-resources');
 const urlConfig = require('./config');
-// Per-resource-type base filters, mirroring eager's resource-fetch filters
-// (`modelOptions.filter` in services/url/config.js). Deliberately duplicated
-// rather than imported: lazy is meant to replace eager, at which point eager's
-// config goes away and this becomes the single source. `fields` lists the
-// record columns each filter reads; a resource that reaches URL generation
-// must carry them (a thin one is rejected — see _assertBaseFieldsPresent).
+// Per-resource-type gate on which rows get a URL at all — the forward-lookup
+// counterpart of the fetch filters in routable-resources.js. `fields` lists
+// the record columns each filter reads; a resource that reaches URL generation
+// must carry them (see _missingBaseFields).
 //
 // Authors are intentionally absent: users.visibility is schema-pinned to
-// 'public' (isIn: [['public']]), so eager's visibility:public author filter
-// never excludes anyone — every author is routable, and serialized authors
-// drop visibility anyway (#10438).
+// 'public' (isIn: [['public']]), so the visibility:public author filter never
+// excludes anyone — every author is routable, and serialized authors drop
+// visibility anyway (#10438).
 const BASE_FILTERS = {
     posts: { filter: 'status:published+type:post', fields: ['status', 'type'] },
     pages: { filter: 'status:published+type:page', fields: ['status', 'type'] },
@@ -33,19 +33,18 @@ function buildBaseFilters() {
     }
     return baseFilters;
 }
-// Columns eager drops from its in-memory URL cache (the `exclude` lists in
-// services/url/config.js). Eager evaluates a router's collection filter against
-// that reduced cached record, so an excluded column reads as absent — NQL then
-// treats it as null. Lazy loads full records and would see the real value, so
-// to preserve eager's behaviour it strips these columns before evaluating
-// router filters (and neither requires nor force-loads them). Keyed by
-// resourceType ('posts'/'pages'/'tags'). Read from eager's config directly so
-// the two can't drift; when eager is retired this derives from whatever
-// replaces it. The base filter is unaffected — it runs against the full record.
+// Columns a router filter must not see (the `exclude` lists in
+// services/url/config.js), keyed by resourceType. Ghost previously answered
+// URLs from a precomputed cache that dropped these columns, so a filter
+// referencing one matched it as absent — NQL reads absent as null. This
+// service loads full records and would see the real value, so it strips them
+// before evaluating router filters (and neither requires nor force-loads
+// them), keeping routes.yaml files that rely on that behaviour working. The
+// base filter above is unaffected — it runs against the full record.
 function buildExcludedFilterFields() {
     const excluded = new Map();
     for (const entry of urlConfig) {
-        const exclude = entry?.modelOptions?.exclude;
+        const exclude = entry?.exclude;
         if (Array.isArray(exclude)) {
             excluded.set(entry.type, new Set(exclude));
         }
@@ -53,10 +52,18 @@ function buildExcludedFilterFields() {
     return excluded;
 }
 const EMPTY_FIELD_SET = new Set();
+// 1, 10, 100, ... — exact for integers, unlike a log10 comparison.
+function isPowerOfTen(n) {
+    let power = 1;
+    while (power < n) {
+        power *= 10;
+    }
+    return power === n;
+}
 // Relation roots are loaded via getRequiredRelations (as withRelated), not as
 // scalar columns; `page`/`type` are the router-type discriminator, always set
 // on the resource. Everything else a router filter references is a scalar
-// own-column the resource must carry to be routed like eager.
+// own-column the resource must carry to be routed.
 const FILTER_NON_SCALAR_FIELDS = new Set([
     'tag', 'tags', 'author', 'authors', 'primary_tag', 'primary_author', 'page', 'type'
 ]);
@@ -85,20 +92,27 @@ function filterScalarFields(filter) {
 }
 const ROUTER_TYPE_TO_DB_TYPE = { posts: 'post', pages: 'page' };
 /**
- * On-demand replacement for the eager UrlService: computes URLs and ownership
- * per call from the registered router configs instead of precomputing a full
- * map at boot. Forward lookups are pure; resolveUrl is the only DB-touching
- * path, and only through the injected findResource hook.
+ * Ghost's URL service: computes URLs and ownership per call from the
+ * registered router configs. Forward lookups are pure; resolveUrl and
+ * getRoutableResources are the only DB-touching paths, and only through the
+ * injected findResource / fetchRoutableResources hooks.
  */
 class LazyUrlService {
     urlUtils;
     findResource;
+    fetchRoutableResources;
     // Router configs in registration order, which is also their priority.
     routerConfigs;
     requiredRelations;
     baseFilters;
     excludedFilterFields;
-    constructor({ urlUtils = localUtils, findResource }) {
+    // True once routers have been registered. Cleared by reset() so the
+    // route-reload window re-gates the maintenance middleware.
+    routersReady;
+    // How many times each thin-resource cause has been seen — see
+    // _degradeThinResource.
+    reportedThinResources;
+    constructor({ urlUtils = localUtils, findResource, fetchRoutableResources = defaultFetchRoutableResources }) {
         if (typeof findResource !== 'function') {
             throw new errors.IncorrectUsageError({
                 message: 'LazyUrlService requires a findResource function'
@@ -106,10 +120,13 @@ class LazyUrlService {
         }
         this.urlUtils = urlUtils;
         this.findResource = findResource;
+        this.fetchRoutableResources = fetchRoutableResources;
         this.routerConfigs = [];
         this.requiredRelations = null;
         this.baseFilters = buildBaseFilters();
         this.excludedFilterFields = buildExcludedFilterFields();
+        this.routersReady = false;
+        this.reportedThinResources = new Map();
     }
     onRouterAddedType(identifier, filter, resourceType, permalink) {
         debug('onRouterAddedType', identifier, resourceType, permalink, filter);
@@ -121,6 +138,7 @@ class LazyUrlService {
             compiledFilter: buildFilter(filter)
         });
         this.requiredRelations = null;
+        this.routersReady = true;
     }
     onRouterUpdated() {
         // Defensive: a router update could change a filter the cache derived
@@ -130,6 +148,8 @@ class LazyUrlService {
     reset() {
         this.routerConfigs = [];
         this.requiredRelations = null;
+        this.routersReady = false;
+        this.reportedThinResources.clear();
     }
     getRequiredRelations() {
         if (this.requiredRelations !== null) {
@@ -157,10 +177,10 @@ class LazyUrlService {
         this.requiredRelations = [...required];
         return [...this.requiredRelations];
     }
-    // Columns a resource of this type must carry for the lazy service to build
+    // Columns a resource of this type must carry for the service to build
     // its URL: its base-filter columns plus the scalar columns its routers'
     // permalinks substitute and filters read. Relations are covered separately
-    // by getRequiredRelations; eager needs none of this (it looks URLs up by id).
+    // by getRequiredRelations.
     getRequiredFields(routerType) {
         const fields = new Set();
         const excluded = this._excludedFilterFieldsFor(routerType);
@@ -192,9 +212,10 @@ class LazyUrlService {
                     fields.add(computed);
                 }
             }
-            // Skip columns eager drops from its cache: it never force-loads them
-            // and evaluates their filters against an absent value, so lazy must
-            // not require them either (see buildExcludedFilterFields).
+            // Skip the columns a router filter must not see: their filters are
+            // evaluated against an absent value, so requiring them would
+            // force-load a column that can't affect the result (see
+            // buildExcludedFilterFields).
             filterScalarFields(config.filter).forEach((field) => {
                 if (!excluded.has(field)) {
                     fields.add(field);
@@ -204,40 +225,62 @@ class LazyUrlService {
         return [...fields];
     }
     hasFinished() {
-        return true;
+        return this.routersReady;
+    }
+    /**
+     * All routable rows of a type. The columns URL computation needs come
+     * from the active routing config, so the rows are never thin for it;
+     * callers name any extra columns they want back.
+     */
+    async getRoutableResources(type, { columns = [] } = {}) {
+        return this.fetchRoutableResources(type, {
+            columns,
+            requiredFields: this.getRequiredFields(type),
+            requiredRelations: this.getRequiredRelations()
+        });
     }
+    /**
+     * A thin resource — one missing a column its base filter or a router
+     * filter reads — is a caller bug: we cannot tell whether it routes. It is
+     * reported and degraded to /404/ rather than thrown, because 500ing a page
+     * that does route is worse than a 404 on one that may not.
+     */
     getUrlForResource(resource, options = {}) {
         const routerType = routerTypeOf(resource);
         if (!routerType) {
-            return this._formatNotFound(options);
+            return this.notFoundUrl(options);
         }
         const record = this._recordForFilter(resource);
-        // The router (collection) filter is evaluated against eager's reduced
-        // column set; the base filter keeps the full record (it reads status /
-        // type / visibility, which eager applies at query time, not in cache).
+        // The router (collection) filter is evaluated against the reduced
+        // column set (see buildExcludedFilterFields); the base filter keeps the
+        // full record, since it reads status / type / visibility.
         const filterRecord = this._recordForRouterFilter(record, routerType);
-        // Eager only builds URLs for resources that pass the per-type base
-        // filter (e.g. visibility:public tags, status:published posts), so a
-        // resource failing it has no URL there and must 404 here too. Checked
-        // only when a router for the type exists, since otherwise the resource
-        // 404s regardless.
+        // Only resources that pass the per-type base filter (visibility:public
+        // tags, status:published posts) get a URL. Checked only when a router
+        // for the type exists, since otherwise the resource 404s regardless.
         if (this._hasRouterForType(routerType)) {
-            this._assertBaseFieldsPresent(routerType, resource);
+            const missing = this._missingBaseFields(routerType, resource);
+            if (missing) {
+                return this._degradeThinResource(resource, missing, options);
+            }
             if (!this._baseFilterMatches(routerType, record)) {
-                return this._formatNotFound(options);
+                return this.notFoundUrl(options);
             }
         }
         for (const config of this.routerConfigs) {
             if (config.resourceType !== routerType) {
                 continue;
             }
-            this._assertNotThin(config, resource, routerType);
+            const missing = this._missingRouterFields(config, resource, routerType);
+            if (missing) {
+                return this._degradeThinResource(resource, missing, options);
+            }
             if (filterMatches(config.compiledFilter, filterRecord)) {
                 const path = this.urlUtils.replacePermalink(config.permalink, resource);
                 return this._formatPath(path, options);
             }
         }
-        return this._formatNotFound(options);
+        return this.notFoundUrl(options);
     }
     ownsResource(routerIdentifier, resource) {
         if (!resource) {
@@ -247,15 +290,15 @@ class LazyUrlService {
         if (!routerType) {
             return false;
         }
-        // A resource failing its base filter is not in eager's map, so no
-        // router owns it. Mirrors the base-filter gate in getUrlForResource.
+        // A resource failing its base filter has no URL, so no router owns it.
+        // Mirrors the base-filter gate in getUrlForResource.
         const record = this._recordForFilter(resource);
         if (!this._baseFilterMatches(routerType, record)) {
             return false;
         }
         const filterRecord = this._recordForRouterFilter(record, routerType);
         // Ownership is exclusive: only the first matching router of the type
-        // owns the resource, matching eager's reservation.
+        // owns the resource.
         const owner = this.routerConfigs.find(c => c.resourceType === routerType && filterMatches(c.compiledFilter, filterRecord));
         return !!owner && owner.identifier === routerIdentifier;
     }
@@ -299,11 +342,11 @@ class LazyUrlService {
         }
         return null;
     }
-    // Eager only resolves a URL that equals a resource's generated (canonical)
-    // URL, so we regenerate the record's URL for this permalink and confirm the
-    // captured params match it. Without this, derived/relation segments the
-    // query can't filter on (year/month, primary_tag) would resolve any slug,
-    // 200-ing a URL the eager service 404s.
+    // Only a URL that equals the resource's own generated (canonical) URL
+    // resolves, so we regenerate the record's URL for this permalink and
+    // confirm the captured params match it. Without this, derived/relation
+    // segments the query can't filter on (year/month, primary_tag) would
+    // resolve any slug, 200-ing a URL that has no page.
     _matchesCanonicalUrl(config, params, resource) {
         const canonicalPath = this.urlUtils.replacePermalink(config.permalink, resource);
         const canonicalParams = matchPermalink(config.permalink, canonicalPath);
@@ -314,6 +357,50 @@ class LazyUrlService {
         const canonical = canonicalParams;
         return Object.keys(captured).every(key => canonical[key] === captured[key]);
     }
+    // Reports which columns were missing alongside what the caller handed over
+    // — the resource's shape, and what the active routing config needs — so
+    // the fetch that under-fetched can be found from the log line alone.
+    //
+    // Reported on the first occurrence of each distinct cause, then at each
+    // order of magnitude. A caller that under-fetches does so for every row it
+    // serializes, so a browse over a large collection would otherwise emit one
+    // error per row — but reporting only once would hide that the problem is
+    // still happening, and how widely. `occurrences` carries that instead.
+    // Cleared on reset(), since a new routing config can make a previously
+    // thin resource fine (or newly broken).
+    _degradeThinResource(resource, thin, options) {
+        const producedBy = options.serializerContext;
+        // The producer is part of the key: two endpoints under-fetching the
+        // same way are two bugs, and the endpoint set is small enough that
+        // this cannot flood.
+        const key = [
+            thin.resourceType,
+            thin.routerIdentifier ?? '',
+            thin.missing.join(','),
+            producedBy ? `${producedBy.apiType}:${producedBy.docName}:${producedBy.method}` : ''
+        ].join('|');
+        const occurrences = (this.reportedThinResources.get(key) ?? 0) + 1;
+        this.reportedThinResources.set(key, occurrences);
+        if (!isPowerOfTen(occurrences)) {
+            return this.notFoundUrl(options);
+        }
+        logging.error(new errors.InternalServerError({
+            message: 'URL service could not build a URL, degraded to /404/',
+            code: 'LAZY_URL_RESOLUTION_ERROR',
+            errorDetails: {
+                method: 'getUrlForResource',
+                type: resource.type,
+                id: resource.id,
+                status: resource.status,
+                resourceKeys: Object.keys(resource),
+                requiredRelations: this.getRequiredRelations(),
+                occurrences,
+                ...(producedBy ? { serializer: producedBy } : {}),
+                ...thin
+            }
+        }));
+        return this.notFoundUrl(options);
+    }
     _hasRouterForType(routerType) {
         return this.routerConfigs.some(config => config.resourceType === routerType);
     }
@@ -326,32 +413,23 @@ class LazyUrlService {
     }
     // A resource that reaches URL generation must carry the columns its base
     // filter reads (status for posts/pages, visibility for tags) — production
-    // callers always do (full models, or forced by the serializers). Without
-    // them the filter can't be evaluated and we'd silently 404 a URL eager would
-    // have produced, so refuse loudly instead of guessing.
-    _assertBaseFieldsPresent(routerType, resource) {
+    // callers always do, via full models or the serializers' force-load.
+    // Without them the filter can't be evaluated at all, so we cannot tell a
+    // genuine miss from a caller that under-fetched.
+    _missingBaseFields(routerType, resource) {
         const base = this.baseFilters.get(routerType);
         if (!base) {
-            return;
+            return null;
         }
         const r = resource;
         const missing = base.fields.filter(field => r[field] === undefined);
         if (missing.length === 0) {
-            return;
+            return null;
         }
-        throw new errors.InternalServerError({
-            message: 'Thin resource passed to LazyUrlService.getUrlForResource',
-            code: 'LAZY_URL_THIN_RESOURCE',
-            errorDetails: {
-                resourceType: routerType,
-                resourceId: resource.id,
-                baseFilter: base.filter,
-                missing
-            }
-        });
+        return { resourceType: routerType, baseFilter: base.filter, missing };
     }
     // Normalizes the plural router type to the singular DB value for filter
-    // evaluation only, so page: filters match as in the eager generator.
+    // evaluation only, so `page:` filters in routes.yaml match.
     _recordForFilter(resource) {
         const record = resource;
         const dbType = ROUTER_TYPE_TO_DB_TYPE[record.type];
@@ -360,8 +438,7 @@ class LazyUrlService {
     _excludedFilterFieldsFor(routerType) {
         return this.excludedFilterFields.get(routerType) ?? EMPTY_FIELD_SET;
     }
-    // Strips the columns eager drops from its cache so a router/collection
-    // filter is evaluated against the same reduced shape eager uses — an
+    // Strips the columns a router/collection filter must not see, so an
     // excluded column reads as absent (→ null in NQL) rather than its real
     // value. Only affects filters that reference an excluded column; every
     // other filter sees an identical record.
@@ -387,8 +464,16 @@ class LazyUrlService {
         }
         return path;
     }
-    // Mirrors the eager miss path: the /404/ fallback carries no subdirectory.
-    _formatNotFound(options) {
+    /**
+     * The /404/ a resource gets when nothing routes it, formatted for the given
+     * options. Deliberately omits the argument `_formatPath` passes.
+     *
+     * The subdirectory is not lost by that omission: `createUrl` takes it from
+     * its own base whenever the url is relative, so a subdirectory install gets
+     * `/blog/404/` from here either way. The third argument `_formatPath`
+     * passes is `trailingSlash`, and `/404/` already ends in one.
+     */
+    notFoundUrl(options = {}) {
         if (options.absolute) {
             return this.urlUtils.createUrl('/404/', options.absolute);
         }
@@ -398,13 +483,13 @@ class LazyUrlService {
         return '/404/';
     }
     // A filtered router that references a relation the resource doesn't carry
-    // can't be evaluated: lazy would 404 here while eager (full data in memory)
-    // returns a URL. Callers must hand the service fully-inflated resources, so
-    // a thin one is a programmer error we refuse loudly rather than mask as a
-    // silent /404/.
-    _assertNotThin(config, resource, routerType) {
+    // can't be evaluated, so the resource would 404 for a reason that has
+    // nothing to do with the routing config. Callers must hand over
+    // fully-inflated resources; a thin one is reported rather than silently
+    // 404'd (see getUrlForResource).
+    _missingRouterFields(config, resource, routerType) {
         if (!config.filter) {
-            return;
+            return null;
         }
         const r = resource;
         const missing = [];
@@ -420,8 +505,8 @@ class LazyUrlService {
         if (/\bprimary_author\b/.test(config.filter) && r.primary_author === undefined) {
             missing.push('primary_author');
         }
-        // Columns eager drops from its cache are never required — its filters
-        // match them as absent, so a resource lacking one is not thin here.
+        // Excluded columns are never required — their filters match them as
+        // absent, so a resource lacking one is not thin here.
         const excluded = this._excludedFilterFieldsFor(routerType);
         for (const field of filterScalarFields(config.filter)) {
             if (!excluded.has(field) && r[field] === undefined) {
@@ -429,19 +514,14 @@ class LazyUrlService {
             }
         }
         if (missing.length === 0) {
-            return;
-        }
-        throw new errors.InternalServerError({
-            message: 'Thin resource passed to LazyUrlService.getUrlForResource',
-            code: 'LAZY_URL_THIN_RESOURCE',
-            errorDetails: {
-                resourceType: routerType,
-                resourceId: resource.id,
-                routerIdentifier: config.identifier,
-                filter: config.filter,
-                missing
-            }
-        });
+            return null;
+        }
+        return {
+            resourceType: routerType,
+            routerIdentifier: config.identifier,
+            filter: config.filter,
+            missing
+        };
     }
 }
 exports.LazyUrlService = LazyUrlService;