mikser-io 9.0.5 → 9.2.0

package/docs/plugins.md

@@ -681,6 +681,7 @@ api: {
   base: '/api',                    // mount prefix; default '/api'
   pageSize: 10,                    // global default; per-endpoint override below
   renderTimeout: 30_000,           // global default; per-endpoint override below
+  bodyLimit: '2mb',                // global default; per-endpoint override below
   endpoints: {
     public: {
       // No token → publicly readable. The query function scopes what
@@ -706,11 +707,23 @@ api: {
       token: process.env.API_RENDER_TOKEN,
       operations: ['render'],
       renderTimeout: 60_000,
+      bodyLimit: '8mb',            // override the global bodyLimit
     },
   },
 }
 ```
 
+**`bodyLimit`** is worth setting deliberately on a `render` endpoint. The request
+body carries the whole entity, so its size is the size of everything the layout
+needs — a mail template handed a customer's recent history is easily several
+hundred kilobytes, which is not large but is well past Express's 100kb default.
+
+The failure mode argues for getting this right rather than discovering it: the
+body is rejected while the stream is being read, so the render never runs, the
+rejection is not logged as a render error, and the caller receives an HTML error
+page from a JSON API. It reads as "the thing I was rendering for is broken",
+not as "the request was too big".
+
 **Routes per endpoint:**
 
 | Method | Path | Operation | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io",
-  "version": "9.0.5",
+  "version": "9.2.0",
   "description": "<p align=\"center\">   <img src=\"mikser-lockup-stacked.svg\" alt=\"mikser\" width=\"198\" /> </p>",
   "main": "index.js",
   "exports": {

package/src/catalog.js

@@ -567,19 +567,38 @@ async function expandAndProject(entity, expand) {
     return { ...result, meta: projectMeta(result.meta) }
 }
 
+// Combine a caller's filter with an endpoint scope. A sift-shaped scope
+// becomes part of the query so sift-to-sql can push it down; a function scope
+// returns the filter untouched and is applied by the caller after the fetch.
+export function scopedFilter(filter, scope) {
+    if (!scope || typeof scope === 'function') return filter
+    const hasFilter = filter && Object.keys(filter).length > 0
+    return hasFilter ? { $and: [filter, scope] } : scope
+}
+
 export async function queryEntities({
     filter, sort, fields, skip, limit, expand, scope,
 } = {}) {
     const effectiveLimit = Math.min(100, Math.max(1, limit ?? 25))
     const effectiveSkip = Math.max(0, skip ?? 0)
 
-    recordQuery(filter)
+    // A sift-shaped `scope` is merged into the filter so it reaches
+    // findEntities, where sift-to-sql pushes what it can into the WHERE
+    // clause. A function `scope` cannot be translated and stays a post-fetch
+    // predicate — which means the query materializes every row the caller's
+    // own filter matched, including the ones the endpoint would then reject.
+    // For an endpoint whose filter is broad (or absent) that is the entire
+    // catalog, per request, and `limit` does not help: it is applied after
+    // this. Prefer the object form.
+    const scopePredicate = typeof scope === 'function' ? scope : null
+    const effectiveFilter = scopedFilter(filter, scope)
+
+    recordQuery(effectiveFilter)
 
     // Materialize via findEntities (which handles sqlite + shim
-    // dispatch + js fallback). scope is a JS predicate applied
-    // post-fetch.
-    let all = await findEntities(filter)
-    if (scope) all = all.filter(scope)
+    // dispatch + js fallback).
+    let all = await findEntities(effectiveFilter)
+    if (scopePredicate) all = all.filter(scopePredicate)
 
     const total = all.length
 

package/src/plugins/api.js

@@ -359,6 +359,23 @@ export function api(options = {}) {
         const base = apiBase   // alias so existing local references still work
         const globalPageSize = options.pageSize ?? 10
         const globalRenderTimeout = options.renderTimeout ?? 30_000
+        // express.json() defaults to 100kb, which is a sensible ceiling for a
+        // REST body and much too small for `render`: the caller posts an entity
+        // whose meta carries everything the layout needs, and a mail template
+        // given a list of a customer's recent bookings runs 150–400kb. That
+        // exceeds the default without being remotely large.
+        //
+        // The failure is quiet at the wrong end. The renderer never sees the
+        // request — raw-body rejects it while reading the stream — so nothing
+        // appears in mikser's log, and the caller gets an HTML error page for a
+        // JSON API. On gpoint that was 168 renders lost in under two hours,
+        // including the customer booking confirmations, and it looked from the
+        // outside like the mail was broken rather than the render.
+        //
+        // 2mb, and configurable per endpoint like every other limit here. Not
+        // unbounded: a token-gated endpoint is still a body an attacker can
+        // choose the size of.
+        const globalBodyLimit = options.bodyLimit ?? '2mb'
 
         // Preview workflow (render → cache → URL) lives in its own
         // plugin (src/plugins/preview.js) as of v7.3.0. The api plugin
@@ -377,7 +394,7 @@ export function api(options = {}) {
         //   api.endpoints.admin   { token: '...', operations: ['list','update','delete','render'] }
         for (const [name, ep] of Object.entries(endpoints)) {
             const router = express.Router()
-            router.use(express.json())
+            router.use(express.json({ limit: ep.bodyLimit ?? globalBodyLimit }))
 
             // Operations default to the safer shape when no token is set
             // (read-only) and full access when token-gated. `subscribe`
@@ -390,7 +407,14 @@ export function api(options = {}) {
                 : ['list']
             const allowedOps = new Set(ep.operations ?? defaultOps)
 
-            const query = typeof ep.query === 'function' ? ep.query : null
+            // The endpoint's scope. A sift filter is the form to prefer —
+            // queryEntities merges it into the WHERE clause, so the endpoint
+            // never materializes rows it would only reject. A function still
+            // works and is applied post-fetch, which costs every row the
+            // caller's filter matched.
+            const query = (typeof ep.query === 'function' || (ep.query && typeof ep.query === 'object'))
+                ? ep.query
+                : null
             const pageSize = ep.pageSize ?? globalPageSize
             const renderTimeout = ep.renderTimeout ?? globalRenderTimeout
 

package/src/render.js

@@ -376,7 +376,10 @@ export function useRenderer(runtime, { defaultTimeout = 30_000 } = {}) {
      * - `catalog: true` (default) — keep the entity in the catalog after
      *   the render. Pass `catalog: false` to prune the catalog row;
      *   useful for on-demand renders where the metadata row would just
-     *   accumulate.
+     *   accumulate. Requires `save: false` — the prune goes through the
+     *   journal, and a DELETE takes the manifest's file cleanup with it,
+     *   so it is only safe for a render that wrote nothing. With
+     *   `save: true` the row is kept and a warning is logged.
      * - `save: true` (default) — write the rendered output to disk at
      *   `<outputFolder>/<entity.destination>`. Pass `save: false` to
      *   skip the final disk write; the bytes still come back via
@@ -439,18 +442,24 @@ export function useRenderer(runtime, { defaultTimeout = 30_000 } = {}) {
         })
 
         if (catalog === false) {
-            // Explicit opt-out: prune the catalog row so it doesn't
-            // accumulate. The rendered output file stays on disk — the
-            // bytes are the work product. We deliberately bypass the
-            // journal/DELETE path here (which would also unlink the file
-            // via the manifest module's cleanup) and splice the entity
-            // out of the in-memory catalog directly. Strict equality so
-            // ambiguous inputs (null, "false", 0) fall through to the
-            // default of keeping the row.
-            const entities = runtime.catalog?.data?.entities
-            if (entities) {
-                const idx = entities.findIndex(e => e.id === result.entity.id)
-                if (idx >= 0) entities.splice(idx, 1)
+            // Prune the row through the journal, so the DELETE lands in
+            // sqlite at onPersist alongside the CREATE that put it there.
+            // Strict equality — null / "false" / 0 keep the row.
+            //
+            // Only when `save` is also false. A DELETE carries the
+            // manifest's file cleanup with it, which unlinks the render's
+            // output; that is correct for an entity that produced no file
+            // and wrong for one that did. `catalog: false, save: true`
+            // therefore keeps its row, and says so rather than dropping
+            // the output on the floor.
+            if (save === false) {
+                await runtime.delete(result.entity)
+            } else {
+                useLogger()?.warn(
+                    'render: catalog:false ignored for %s — it needs save:false, ' +
+                    'because pruning the row also unlinks the rendered output',
+                    result.entity.id,
+                )
             }
         }