apostrophe 4.27.1 → 4.28.0

package/modules/@apostrophecms/url/index.js

@@ -7,10 +7,145 @@ const _ = require('lodash');
 const qs = require('qs');
 
 module.exports = {
-  options: { alias: 'url' },
+
+  options: {
+    alias: 'url',
+    static: false
+  },
+
+  restApiRoutes(self) {
+    return {
+      // GET /api/v1/@apostrophecms/url
+      //
+      // Returns the result of `getAllUrlMetadata` — an object
+      // with `pages` and `attachments` properties.
+      // See the `getAllUrlMetadata` method for full documentation.
+      async getAll(req) {
+        if (!self.isExternalFront(req)) {
+          throw self.apos.error('forbidden');
+        }
+        if (!self.options.static) {
+          throw self.apos.error('invalid',
+            'The @apostrophecms/url module must be configured with the "static: true" option to use this API. ' +
+            'Without it, URL metadata for filters and pagination cannot be fully enumerated for a static build.'
+          );
+        }
+
+        // Parse and sanitize attachment options from the query string.
+        const launder = self.apos.launder;
+        const wantAttachments = launder.boolean(req.query.attachments);
+        const splitSizes = (val) => {
+          const list = launder.string(val)
+            .split(',').map(s => s.trim()).filter(Boolean);
+          return list.length ? list : undefined;
+        };
+        const attachments = wantAttachments
+          ? {
+            sizes: splitSizes(req.query.attachmentSizes),
+            skipSizes: splitSizes(req.query.attachmentSkipSizes),
+            scope: launder.select(
+              req.query.attachmentScope,
+              [ 'used', 'all' ],
+              'used'
+            )
+          }
+          : false;
+
+        return self.getAllUrlMetadata(req, {
+          attachments
+        });
+      }
+    };
+  },
+
+  handlers(self) {
+    return {
+      '@apostrophecms/page:beforeSend': {
+        addStaticUrlsFlag(req) {
+          req.data.staticUrls = !!self.options.static;
+        }
+      }
+    };
+  },
+
   methods(self) {
     return {
 
+      // Returns `true` if the given `req` represents a static build
+      // request. This is the single source of truth — modules should
+      // use this method rather than inspecting `req` properties
+      // directly.
+      //
+      // Static build requests are those made by an external frontend
+      // (e.g. Astro) that opted in to static-build URL handling via
+      // the `x-apos-static-base-url: 1` header. The Express
+      // middleware sets `req.aposStaticBuild` and (when configured)
+      // `req.staticBaseUrl` in response.
+      isStaticBuild(req) {
+        return !!req.aposStaticBuild;
+      },
+
+      // Returns `true` if the given `req` originates from an
+      // external frontend integration (e.g. Astro, Next.js).
+      // This is the single source of truth — modules should use
+      // this method rather than inspecting `req.aposExternalFront`
+      // directly.
+      isExternalFront(req) {
+        return !!req.aposExternalFront;
+      },
+
+      // Returns the effective base URL for the given request.
+      //
+      // Resolution order:
+      //  1. If a hostname is configured for the active locale,
+      //     `<protocol>://<hostname>` is returned (locale-specific
+      //     host always wins, prefix is never appended).
+      //  2. If the request is a static build (`isStaticBuild(req)`),
+      //     `req.staticBaseUrl` + prefix is returned (or the empty
+      //     string when none is configured).
+      //  3. Otherwise, `apos.baseUrl` + prefix is returned (or the
+      //     empty string).
+      //
+      // ### `options.strict`
+      //
+      // When `true`, guarantees a non-empty return value:
+      //  - In a static build where `staticBaseUrl` is empty,
+      //    falls back to `apos.baseUrl`.
+      //  - In a non-static context where `apos.baseUrl` is empty,
+      //    still returns the empty string (nothing more to fall
+      //    back to).
+      //
+      // Use `strict: true` when an absolute URL is required (e.g.
+      // sitemap `<loc>` values).
+      //
+      // ### `options.prefix`
+      //
+      // When `true` (the default), the global `apos.prefix` is
+      // appended to the returned URL (e.g. `/blog`).  The prefix
+      // is **not** appended when a locale-specific hostname is
+      // used — that hostname already represents the full origin.
+      //
+      // Pass `prefix: false` to obtain only the origin / base URL
+      // without the prefix.  This is the legacy behavior of
+      // `apos.page.getBaseUrl(req)` before the delegation to this
+      // method.
+      getBaseUrl(req, { strict = false, prefix = true } = {}) {
+        const hostname = self.apos.i18n.locales?.[req.locale]?.hostname;
+        if (hostname) {
+          // Locale hostnames are fully qualified origins;
+          // the global prefix does not apply.
+          return `${req.protocol}://${hostname}`;
+        }
+        const aposPrefix = prefix ? (self.apos.prefix || '') : '';
+        if (self.isStaticBuild(req)) {
+          const staticUrl = req.staticBaseUrl || '';
+          if (staticUrl || !strict) {
+            return staticUrl + aposPrefix;
+          }
+        }
+        return (self.apos.baseUrl || '') + aposPrefix;
+      },
+
       // Build filter URLs. `data` is an object whose properties
       // become new query parameters. These parameters override any
       // existing parameters of the same name in the URL. If you
@@ -210,6 +345,289 @@ module.exports = {
         } else {
           return restoreHash(base);
         }
+      },
+
+      // Generate a list of all URLs reachable with the given
+      // req object. Used internally to implement static site
+      // generation and sitemaps. Usually called in a loop,
+      // once for each locale.
+      //
+      // ## Returned shape
+      //
+      // The return value is always:
+      //
+      // ```js
+      // {
+      //   pages: [ ...page metadata entries ],
+      //   attachments: { // null when not requested
+      //     uploadsUrl: '/uploads',
+      //     results: [
+      //       { _id: 'abc', urls: [{ size?, path }] },
+      //       ...
+      //     ]
+      //   }
+      // }
+      // ```
+      //
+      // ## Page metadata entries (`pages`)
+      //
+      // Each entry in the `pages` array may contain the
+      // following properties:
+      //
+      // ### `url` (string, always present)
+      // The URL path for this entry — a purely relative path
+      // without origin or prefix (e.g. `/articles` or
+      // `/articles/category/tech`, never
+      // `https://example.com/my-repo/articles`).
+      //
+      // For document entries, the framework strips the base
+      // URL (origin + prefix) automatically after collection.
+      // For **literal content** entries added by event
+      // handlers, the URL **must** be provided as a relative,
+      // prefix-free path (e.g. `/robots.txt`, not
+      // `/my-repo/robots.txt`). The consumer (e.g. Astro
+      // integration) is responsible for prepending the prefix
+      // when fetching from the backend.
+      //
+      // ### `i18nId` (string, always present)
+      // A stable identifier that is consistent across localized
+      // versions of the same logical URL. Used by external
+      // frontends (e.g. Astro) to correlate URLs across locales.
+      // For the primary view of a document this equals `aposDocId`.
+      // For derived URLs (pagination, filter combinations) it is
+      // built by appending suffixes to the base doc's `aposDocId`,
+      // e.g. `myDocId.category.tech.1` or `myDocId.2`.
+      //
+      // ### `type` (string, present for document entries)
+      // The Apostrophe doc `type` name (e.g. `'article'`,
+      // `'@apostrophecms/home-page'`). Absent on non-document
+      // entries such as literal content URLs.
+      //
+      // ### `aposDocId` (string, present for document entries)
+      // The locale-independent document ID. Absent on
+      // non-document entries.
+      //
+      // ### `_id` (string, present for document entries)
+      // The full locale-qualified MongoDB `_id` of the document
+      // (e.g. `'xyz:en:published'`). Absent on non-document
+      // entries.
+      //
+      // ### `contentType` (string, present for literal content entries only)
+      // A MIME type such as `'text/css'` or `'text/plain'`.
+      // When present, this signals that the URL returns non-HTML
+      // content that should be proxied literally by the consumer
+      // (e.g. an Astro static build). The consumer should fetch
+      // the `url` and write the response body to disk with the
+      // given content type instead of rendering it as a page.
+      // When absent, the URL is an ordinary HTML page.
+      //
+      // Document entries (pages, pieces, etc.) should NEVER set
+      // `contentType`. Consumers such as the sitemap module and
+      // Astro use its absence to identify renderable HTML pages
+      // vs literal assets.
+      //
+      // Literal content entries should NOT include `changefreq`
+      // or `priority` — those are only meaningful for document
+      // entries in sitemaps.
+      //
+      // ### `sitemap` (boolean, optional, default `true`)
+      // When explicitly set to `false`, the entry is excluded
+      // from sitemap generation but still included in static
+      // builds. Useful for URLs that must exist in the build
+      // (e.g. paginated filter pages, CSS files) but should
+      // not appear in `sitemap.xml`. If omitted, the entry is
+      // included in sitemaps.
+      //
+      // ### `changefreq` (string, optional, document entries only)
+      // Sitemap hint (e.g. `'daily'`). Included for legacy
+      // sitemap compatibility. Google explicitly ignores this.
+      // Must NOT be set on literal content entries.
+      //
+      // ### `priority` (number, optional, document entries only)
+      // Sitemap priority hint (e.g. `1.0`). Included for legacy
+      // sitemap compatibility. Google explicitly ignores this.
+      // Must NOT be set on literal content entries.
+      //
+      // ## Literal content entries
+      //
+      // Some entries represent non-HTML content that should be
+      // served literally with a specific MIME type, such as
+      // CSS stylesheets, `robots.txt`, `llms.txt`, etc. These
+      // entries include a `contentType` property (e.g.
+      // `text/css`, `text/plain`). Consumers of this API
+      // (e.g. an Astro static build) should fetch the `url`
+      // and serve the response body with the specified content
+      // type rather than rendering it as an HTML page.
+      //
+      // ## Extension points
+      //
+      // This method emits the
+      // `@apostrophecms/url:getAllUrlMetadata` event, so
+      // that handlers in any module can add URLs to the
+      // results. The default implementation already calls
+      // `getAllUrlMetadata` on every doc type manager that
+      // has at least one doc in the database, so listening
+      // for the event is only for edge cases that can't be
+      // covered by extending `getAllUrlMetadata` or
+      // `getUrlMetadata` on such a manager.
+      //
+      // Handlers should respect `excludeTypes`.
+      //
+      // **Important:** handlers that push literal content
+      // entries must provide a relative, prefix-free `url`
+      // path (e.g. `/robots.txt`). The base URL stripping
+      // that runs after collection only applies to document
+      // entries whose `url` starts with the effective base
+      // URL — it will not strip a prefix that was manually
+      // added by a handler. Providing a relative path
+      // ensures correct behaviour regardless of whether a
+      // prefix is configured.
+      //
+      // ## Attachment metadata (`attachments`)
+      //
+      // When `options.attachments` is a truthy object, attachment
+      // metadata is collected after URL enumeration and returned
+      // alongside the pages.  The option accepts:
+      //
+      // - `scope` (`'used'` | `'all'`): `'used'` (default) limits
+      //   to attachments referenced by documents present in the
+      //   results.  `'all'` returns every non-archived attachment.
+      // - `sizes` (string[]): explicit image sizes to include.
+      // - `skipSizes` (string[]): image sizes to exclude.
+      //
+      // `attachments.uploadsUrl` is the uploadfs base URL prefix
+      // (e.g. `/uploads` or `https://cdn.example.com`).
+      //
+      // Each entry in `attachments.results` contains:
+      // - `_id` (string): the attachment record ID.
+      // - `urls` (array): `{ size, path }` objects where `path`
+      //   is the uploadfs-relative file path.
+      //
+      // After attachment metadata is collected, the
+      // `@apostrophecms/url:getAllAttachmentMetadata` event is
+      // emitted. Handlers receive `(req, results, options)` where
+      // `results` is the attachments results array and `options`
+      // includes `{ sizes, skipSizes, scope, uploadsUrl }`. This
+      // is an escape hatch for edge cases where a module needs to
+      // contribute additional attachment entries or modify the
+      // results programmatically.
+      //
+      async getAllUrlMetadata(req, { excludeTypes = [], attachments = false } = {}) {
+        // Ensure global doc is available for event handlers
+        // that may need it (e.g. @apostrophecms/styles)
+        await self.apos.global.addGlobalToData(req);
+        const results = [];
+        const allAttachmentDocIds = new Set();
+        const collectDocIds = !!attachments && attachments.scope !== 'all';
+        const types = await self.apos.doc.db.distinct('type');
+        for (const type of types) {
+          if (!excludeTypes.includes(type)) {
+            const manager = self.apos.doc.getManager(type);
+            if (!manager?.getAllUrlMetadata) {
+              continue;
+            }
+            const {
+              metadata,
+              attachmentDocIds
+            } = await manager
+              .getAllUrlMetadata(req, { attachments: collectDocIds });
+            for (const entry of metadata) {
+              results.push(entry);
+            }
+            for (const id of attachmentDocIds) {
+              allAttachmentDocIds.add(id);
+            }
+          }
+        }
+        await self.emit('getAllUrlMetadata', req, results, { excludeTypes });
+
+        const response = {
+          pages: results,
+          attachments: null
+        };
+
+        if (attachments) {
+          const {
+            sizes, skipSizes, scope
+          } = attachments;
+
+          const docIds = collectDocIds
+            ? [ ...allAttachmentDocIds ]
+            : undefined;
+
+          response.attachments = {
+            uploadsUrl: self.apos.attachment.uploadfs.getUrl(),
+            results: await self.apos.attachment.getStaticMetadata({
+              docIds,
+              sizes,
+              skipSizes
+            })
+          };
+          await self.emit(
+            'getAllAttachmentMetadata',
+            req,
+            response.attachments.results,
+            {
+              sizes,
+              skipSizes,
+              scope,
+              uploadsUrl: response.attachments.uploadsUrl
+            }
+          );
+        }
+
+        // Strip the base URL (origin + prefix) that `_url` values
+        // were built with, producing purely relative, prefix-free
+        // paths (e.g. `/about`, `/fr/articles/page/2`).
+        const effectiveBaseUrl = self.getBaseUrl(req);
+        if (effectiveBaseUrl) {
+          for (const entry of response.pages) {
+            if (entry.url?.startsWith(effectiveBaseUrl)) {
+              entry.url = entry.url.slice(effectiveBaseUrl.length) || '/';
+            }
+          }
+        }
+
+        // Strip the backend origin from uploadsUrl so that the
+        // consumer receives a relative, prefix-qualified path
+        // (e.g. `/uploads` or `/cms/uploads`).
+        // CDN URLs (different origin) and already-relative URLs
+        // are left untouched.
+        if (response.attachments) {
+          const baseUrl = self.apos.baseUrl || '';
+          if (baseUrl && response.attachments.uploadsUrl.startsWith(baseUrl)) {
+            response.attachments.uploadsUrl =
+              response.attachments.uploadsUrl.slice(baseUrl.length);
+          }
+        }
+
+        return response;
+      },
+      // Returns a string suitable to append to the original page URL when we're
+      // specifying a particular filter and a page number. Pages start with 1
+      getChoiceFilter(name, value, page) {
+        if (value === null) {
+          return '';
+        }
+        name = encodeURIComponent(name);
+        value = encodeURIComponent(value);
+        if (self.options.static) {
+          return `/${name}/${value}${page > 1 ? `/page/${page}` : ''}`;
+        } else {
+          return `?${name}=${value}${page > 1 ? `&page=${page}` : ''}`;
+        }
+      },
+      // Returns a string suitable to append to the original page URL when all we're
+      // adding is a page number. Pages start with 1
+      getPageFilter(page) {
+        if (page <= 1) {
+          return '';
+        }
+        if (self.options.static) {
+          return `/page/${page}`;
+        } else {
+          return `?page=${page}`;
+        }
       }
     };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apostrophe",
-  "version": "4.27.1",
+  "version": "4.28.0",
   "description": "The Apostrophe Content Management System.",
   "main": "index.js",
   "repository": {
@@ -110,7 +110,7 @@
     "tiny-emitter": "^2.1.0",
     "tough-cookie": "^4.0.0",
     "underscore.string": "^3.3.4",
-    "uploadfs": "^1.25.1",
+    "uploadfs": "^1.26.1",
     "void-elements": "^3.1.0",
     "vue": "^3.5.20",
     "vue-advanced-cropper": "^2.8.8",
@@ -119,12 +119,12 @@
     "webpack": "^5.72.0",
     "webpack-merge": "^5.7.3",
     "xregexp": "^2.0.0",
-    "@apostrophecms/emulate-mongo-3-driver": "^1.0.6",
-    "express-cache-on-demand": "^1.0.4",
-    "postcss-viewport-to-container-toggle": "^2.2.0",
     "boring": "^1.1.1",
+    "postcss-viewport-to-container-toggle": "^2.3.0",
+    "sanitize-html": "^2.17.2",
+    "@apostrophecms/emulate-mongo-3-driver": "^1.0.6",
     "broadband": "^1.1.0",
-    "sanitize-html": "^2.17.0"
+    "express-cache-on-demand": "^1.0.4"
   },
   "devDependencies": {
     "eslint": "^9.39.1",

package/test/add-missing-schema-fields-project/node_modules/.package-lock.json

@@ -0,0 +1,131 @@
+{
+  "name": "add-missing-schema-fields-project",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "../..": {
+      "version": "4.27.0",
+      "license": "MIT",
+      "dependencies": {
+        "@apostrophecms/emulate-mongo-3-driver": "workspace:^",
+        "@apostrophecms/vue-material-design-icons": "^1.0.0",
+        "@ctrl/tinycolor": "^4.1.0",
+        "@floating-ui/dom": "^1.5.3",
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/semantic-conventions": "^1.0.1",
+        "@paralleldrive/cuid2": "^2.2.2",
+        "@tiptap/extension-color": "^2.4.0",
+        "@tiptap/extension-floating-menu": "^2.0.3",
+        "@tiptap/extension-highlight": "^2.0.3",
+        "@tiptap/extension-link": "^2.0.3",
+        "@tiptap/extension-placeholder": "^2.0.3",
+        "@tiptap/extension-subscript": "^2.0.3",
+        "@tiptap/extension-superscript": "^2.0.3",
+        "@tiptap/extension-table": "^2.0.3",
+        "@tiptap/extension-table-cell": "^2.0.3",
+        "@tiptap/extension-table-header": "^2.0.3",
+        "@tiptap/extension-table-row": "^2.0.3",
+        "@tiptap/extension-text-align": "^2.0.3",
+        "@tiptap/extension-text-style": "^2.0.3",
+        "@tiptap/extension-underline": "^2.0.3",
+        "@tiptap/starter-kit": "^2.0.3",
+        "@tiptap/vue-3": "^2.0.3",
+        "@vue/compiler-sfc": "^3.3.8",
+        "autoprefixer": "^10.4.1",
+        "bluebird": "^3.7.2",
+        "body-parser": "^1.18.2",
+        "boring": "workspace:^",
+        "broadband": "workspace:^",
+        "cheerio": "^1.0.0-rc.10",
+        "chokidar": "^3.5.2",
+        "common-tags": "^1.8.0",
+        "concat-with-sourcemaps": "^1.1.0",
+        "connect-mongo": "^5.1.0",
+        "cookie-parser": "^1.4.5",
+        "cors": "^2.8.5",
+        "css-loader": "^5.2.4",
+        "cssnano": "^7.1.1",
+        "csv-parse": "^5.6.0",
+        "dayjs": "^1.9.8",
+        "dompurify": "^3.2.5",
+        "encodeurl": "^2.0.0",
+        "express": "^4.16.4",
+        "express-bearer-token": "^3.0.0",
+        "express-cache-on-demand": "workspace:^",
+        "express-session": "^1.18.2",
+        "fs-extra": "^7.0.1",
+        "glob": "^10.4.5",
+        "he": "^1.2.0",
+        "html-to-text": "^9.0.5",
+        "i18next": "^20.3.2",
+        "i18next-http-middleware": "^3.1.5",
+        "import-fresh": "^3.3.0",
+        "is-wsl": "^2.2.0",
+        "jsdom": "^24.1.0",
+        "klona": "^2.0.4",
+        "launder": "^1.4.0",
+        "lodash": "^4.17.21",
+        "mini-css-extract-plugin": "^1.6.0",
+        "minimatch": "^3.0.4",
+        "mkdirp": "^0.5.5",
+        "multer": "^2.0.2",
+        "node-fetch": "^2.6.1",
+        "nodemailer": "^7.0.10",
+        "nunjucks": "^3.2.1",
+        "oembetter": "^1.1.3",
+        "parseurl": "^1.3.3",
+        "passport": "^0.6.0",
+        "passport-local": "^1.0.0",
+        "path-to-regexp": "^1.8.0",
+        "performance-now": "^2.1.0",
+        "pinia": "^2.1.7",
+        "postcss": "^8.4.47",
+        "postcss-html": "^1.3.0",
+        "postcss-loader": "^8.1.1",
+        "postcss-scss": "^4.0.3",
+        "postcss-viewport-to-container-toggle": "workspace:^",
+        "prompts": "^2.4.1",
+        "qs": "^6.10.1",
+        "regexp-quote": "0.0.0",
+        "resolve": "^1.19.0",
+        "resolve-from": "^5.0.0",
+        "sanitize-html": "workspace:^",
+        "sass": "^1.80.3",
+        "sass-loader": "^16.0.0",
+        "server-destroy": "^1.0.1",
+        "sluggo": "^1.0.0",
+        "sortablejs": "^1.15.0",
+        "sortablejs-vue3": "^1.2.11",
+        "tiny-emitter": "^2.1.0",
+        "tough-cookie": "^4.0.0",
+        "underscore.string": "^3.3.4",
+        "uploadfs": "^1.25.1",
+        "void-elements": "^3.1.0",
+        "vue": "^3.5.20",
+        "vue-advanced-cropper": "^2.8.8",
+        "vue-loader": "^17.1.0",
+        "vue-style-loader": "^4.1.3",
+        "webpack": "^5.72.0",
+        "webpack-merge": "^5.7.3",
+        "xregexp": "^2.0.0"
+      },
+      "devDependencies": {
+        "eslint": "^9.39.1",
+        "eslint-config-apostrophe": "workspace:^",
+        "form-data": "^4.0.4",
+        "mocha": "^11.7.1",
+        "nyc": "^17.1.0",
+        "stylelint": "^16.5.0",
+        "stylelint-config-apostrophe": "workspace:^"
+      },
+      "engines": {
+        "node": ">=16.0.0"
+      }
+    },
+    "node_modules/apostrophe": {
+      "resolved": "../..",
+      "link": true
+    }
+  }
+}

package/test/external-front.js

@@ -11,6 +11,7 @@ describe('External Front', function() {
   this.timeout(t.timeout);
 
   after(function() {
+    delete process.env.APOS_EXTERNAL_FRONT_KEY;
     return t.destroy(apos);
   });