mikser-io-layouts 2.10.0 → 2.11.1

package/lib/mcp.js

@@ -77,7 +77,7 @@ export function registerMcpTools({ runtime, useLogger, findEntity, findEntities,
 const ENGINE_KEYS = new Set([
     'layout', 'layouts', 'task', 'destination', 'format', 'template', 'href',
     'lang', 'url', 'pages', 'parent', 'plugins', 'presets', 'refs', 'date',
-    'draft', 'stamp', 'postprocessor',
+    'draft', 'stamp', 'postprocessor', 'schema',
 ])
 
 // A document's meta in the same vocabulary a layout contract uses: dotted
@@ -188,6 +188,9 @@ export function registerCheckTool({ runtime, findEntity, findEntities, useDataba
                 }
                 return
             case 'array':
+                // The element path is NOT added as a key of its own. Requiring
+                // `tags[]` would claim the array must be non-empty, which no
+                // schema said — an empty list is a valid `z.array()`.
                 schemaKeys(def.element, `${prefix}[]`, required, optional, depth + 1)
                 return
             default:
@@ -202,6 +205,55 @@ export function registerCheckTool({ runtime, findEntity, findEntities, useDataba
         return [...req, ...opt]
     }
 
+    // How this record compares with its own kind.
+    //
+    // A layout contract answers "what does this template read". It cannot answer
+    // "is this record complete AS A RECORD" — a catalog entry missing a field
+    // its siblings all carry renders perfectly, reads nothing that is absent,
+    // and passes every other check here. That is among the commonest content
+    // defects and nothing else in this tool can see it.
+    //
+    // Deliberately weak evidence, and reported as such: records differ for good
+    // reasons, and a document is not wrong for lacking what its neighbours have.
+    // It is a prompt to look.
+    //
+    // Needs a real peer group. With one sibling "most of them" means nothing, so
+    // below MIN_PEERS the comparison is not made rather than made badly.
+    const MIN_PEERS = 3
+    const PEER_SHARE = 0.7
+
+    const declaredTypeOf = (entity) => {
+        const key = runtime.options.schemasKey ?? runtime.options.schemaKey ?? 'meta.schema'
+        return key.split('.').reduce((o, k) => o?.[k], entity) ?? entity?.meta?.schema ?? null
+    }
+
+    const peerGapsFor = async (entity) => {
+        const type = declaredTypeOf(entity)
+        if (!type) return null
+        const key = runtime.options.schemasKey ?? runtime.options.schemaKey ?? 'meta.schema'
+        const siblings = ((await findEntities({ [key]: type })) ?? [])
+            .filter(e => e?.id && e.id !== entity.id && e.collection === entity.collection)
+        if (siblings.length < MIN_PEERS) return null
+
+        const counts = new Map()
+        for (const peer of siblings) {
+            for (const k of flattenMeta(peer.meta ?? {}, '', new Set())) {
+                counts.set(k, (counts.get(k) ?? 0) + 1)
+            }
+        }
+        const mine = flattenMeta(entity.meta ?? {}, '', new Set())
+        const threshold = Math.ceil(siblings.length * PEER_SHARE)
+        const gaps = [...counts]
+            .filter(([k, c]) => c >= threshold && !mine.has(k))
+            .filter(([k]) => !ENGINE_KEYS.has(k.split('.')[0].replace(/\[\]$/, '')))
+            // A key whose PARENT this document also lacks is one finding, not
+            // several: report `card`, not `card.order` and `card.wide` under it.
+            .filter(([k]) => { const parent = k.replace(/\.[^.]+$|\[\]$/, ''); return parent === k || mine.has(parent) })
+            .map(([key, count]) => ({ key, siblings: count, of: siblings.length }))
+            .sort((a, b) => b.siblings - a.siblings || a.key.localeCompare(b.key))
+        return { type, peers: siblings.length, gaps }
+    }
+
     // The schema declared for this entity, if the schemas plugin is loaded and
     // the entity names one. `schemaKey` is the plugin's own config — commonly
     // `meta.layout`, so a layout and its schema share a name.
@@ -394,38 +446,78 @@ export function registerCheckTool({ runtime, findEntity, findEntities, useDataba
                     // A real contract, from the other side: the keys renders
                     // actually took off this entity, held to the same standard
                     // as a page — same comparison, same lists.
-                    const cmp = compare(entity, new Set(read.keys), new Set())
+                    // A declared schema outranks consumer reads here for the
+                    // same reason it does on a page: it says what the record
+                    // must contain, where consumer reads only say what someone
+                    // happened to look at. Without this a data document was the
+                    // one kind that could not benefit from its own schema.
+                    // Two questions, two comparisons — as on a page.
+                    //
+                    // `unused` and `untraceable` ask "does anything READ this",
+                    // which only consumption answers. Running them against the
+                    // schema instead reported keys as unread that the recorded
+                    // reads plainly contain: a schema does not enumerate array
+                    // elements, so every `x[]` in the document looked untouched.
+                    const byReads = compare(entity, new Set(read.keys), new Set())
+                    const declared = schemaFor(entity)
+                    let cmp = byReads
+                    if (declared) {
+                        const req = new Set(); const opt = new Set()
+                        schemaKeys(declared.schema, '', req, opt)
+                        const bySchema = compare(entity, new Set([...req, ...opt]), opt)
+                        // `missing` from the declaration, the rest from reality.
+                        cmp = { ...byReads, missing: bySchema.missing,
+                                missingOptional: bySchema.missingOptional }
+                    }
+                    // Matters most here: a data document is otherwise only
+                    // checked against whatever its consumers happen to read, so
+                    // a field every sibling carries and nobody reads yet is
+                    // invisible.
+                    const peers = await peerGapsFor(entity)
                     return ok({
                         entity: id, kind: 'data',
                         verdict: 'not a page — other entities consume it',
                         consumedBy: consumers,
                         checked: read.keys.length > 0,
                         reliable: read.keys.length > 0,
+                        missingFrom: declared ? 'schema' : 'consumers',
                         missing: cmp.missing,
+                        missingOptional: cmp.missingOptional,
                         likelyTypos: cmp.likelyTypos,
                         unused: cmp.unused,
                         untraceable: cmp.untraceable,
+                        ...(peers?.gaps.length
+                            ? { peerGaps: peers.gaps, peerGroup: { type: peers.type, peers: peers.peers } }
+                            : {}),
                         consumedKeys: read.keys,
                         counts: { consumedKeys: read.keys.length, provided: cmp.provided.size,
                                   missing: cmp.missing.length, unused: cmp.unused.length },
                         notes: [
-                            `This entity never renders. ${consumers.length} render(s) pulled it in through a catalog query, so it is data, not a page — building will not make it appear.`,
-                            read.keys.length
-                                ? 'consumedKeys is what those renders actually READ off this entity. That is its contract: a key here with no value is a hole in the pages listed above, not in this file.'
-                                : 'No key-level reads are recorded yet, so only the consumer list is known. Build once with a current mikser and the fields each render takes off this entity get recorded.',
-                            'untraceable means an ancestor WAS read but the members could not be followed: the value was handed to a template and provenance does not survive that hop. Treat them as consumed.',
-                            'unused here means no render took the key. It may still be served to an API client or another process, which this tool cannot see — mikser is not always a static site generator.',
+                            `Never renders — ${consumers.length} render(s) query it. Building will not make it appear, and a key it lacks shows up as a hole in those pages.`,
+                            'missing: check missingFrom — "schema" is declared, "consumers" is only what someone happened to read off it.',
+                            'Look, do not panic: consumedKeys (what renders actually took), peerGaps (most records of this type carry it), unused (may still be served to an API client), untraceable (an ancestor was read but its members could not be followed). None of these is a defect on its own.',
                         ],
                     })
                 }
+                // Nothing reads it, so nothing can say what it needs — except
+                // its own kind. This is the one check available here, and the
+                // only place in the tool where peer comparison is not merely
+                // corroboration.
+                const orphanPeers = await peerGapsFor(entity)
                 return ok({
                     entity: id, kind: 'unreferenced',
                     verdict: 'no contract could be derived',
                     checked: false,
+                    ...(orphanPeers?.gaps.length
+                        ? { peerGaps: orphanPeers.gaps,
+                            peerGroup: { type: orphanPeers.type, peers: orphanPeers.peers } }
+                        : {}),
                     notes: [
-                        'This entity has never rendered, declares no meta.layout, and no render in the manifest queried it. Nothing in this project reads it.',
-                        'That is not an error. The catalog is readable over the API, so another process or an external consumer may depend on it, and this tool cannot see them.',
-                        'If it was MEANT to be a page, it matched no layout — check the layouts match patterns or set meta.layout.',
+                        'Never renders, declares no layout, and no render queried it. Not an error: the catalog is readable over the API, so a consumer outside this project may depend on it.',
+                        ...(orphanPeers?.gaps.length
+                            ? ['peerGaps is the only check possible here — nothing reads this entity, so only the other records of its type say anything about it.']
+                            : []),
+                        'If it was meant to be a page it matched no layout: check the layouts match patterns, or set meta.layout.',
                     ],
                 })
             }
@@ -455,6 +547,7 @@ export function registerCheckTool({ runtime, findEntity, findEntities, useDataba
             // required-versus-optional outright, needs no template parsing, and
             // is engine-agnostic by construction — where one exists, nothing
             // read out of a template should be able to call a document broken.
+            const peerCheck = await peerGapsFor(entity)
             const declaredSchema = schemaFor(entity)
             let missing = inferred.missing
             let missingOptional = inferred.missingOptional
@@ -509,22 +602,23 @@ export function registerCheckTool({ runtime, findEntity, findEntities, useDataba
                 ...(drift ? { drift } : {}),
                 unused,
                 untraceable,
+                ...(peerCheck?.gaps.length
+                    ? { peerGaps: peerCheck.gaps, peerGroup: { type: peerCheck.type, peers: peerCheck.peers } }
+                    : {}),
                 unresolvedSections,
                 counts: { consumed: consumed.size, provided: provided.size, missing: missing.length,
                           missingOptional: missingOptional.length, unused: unused.length },
                 contract: { complete: contract.complete, incomplete: contract.incomplete ?? [],
                             scopedToPartials: usedPartials?.length ?? null },
                 observedReads: observed.length,
+                // One line per field, saying what to DO with it. These are
+                // read on every call, so anything that is not actionable is
+                // cost without value.
                 notes: [
-                    'missing is the strong signal, but read missingFrom first. "schema" means a zod schema DECLARES the key required — authoritative, and it needed no template parsing. "inferred" means it was read out of the templates, which models one engine\'s semantics and has been wrong: a guard it cannot see makes a fine document look broken. Treat an inferred missing as strong evidence, not proof.',
-                    'drift appears only with a schema, and is where the two sources disagree: readButNotDeclared is a key the layouts consume that nothing declares (a schema gap, or a typo in a template), declaredButNotRead is a declared key no layout reads (dead, or consumed over the API).',
-                    'likelyTypos pairs an unused key against a wanted one within two edits. A real typo produces both, which is why the pair is worth more than either list.',
-                    'missingOptional is NOT a defect. The layout guards these behind a condition, so it was written to work without them — they are listed only so a deliberate omission can be told from a forgotten one.',
-                    'unresolvedSections named a section that resolved to no template under THIS layout. Weak evidence: the section may render under a different layout, its template may not be written yet, or the data may be consumed outside the SSG entirely — the catalog is readable over the API. It never fails a check.',
-                    'untraceable is NOT a list of problems. An ancestor of each key WAS read, but the value was handed on — a sidecar returning it to a template — and provenance does not survive that hop, so the members could not be followed. Treat them as consumed unless you have another reason to doubt it.',
-                    'unused is weak evidence too, and weakest of all where mikser is not a static site generator. A key may be consumed by a DIFFERENT layout, or by an API client reading the catalog over HTTP, or by another process entirely — none of which this tool can see. Engine-owned keys (href, lang, layout, task, ...) are excluded outright.',
-                    'The contract is scoped to the partials this entity ACTUALLY rendered with, not to every branch the layout could take. Unscoped, a page that uses two of twenty sections is checked against all twenty.',
-                    'reliable is false unless the contract was fully read AND this entity has rendered. An absence is only evidence when it is true.',
+                    'missing: the layout needs it and this document has no value for it. Check missingFrom — "schema" is declared and authoritative, "inferred" is read from templates and is strong evidence rather than proof.',
+                    'likelyTypos: a key you wrote paired with one the layout wanted, within two edits. A typo produces both, so the pair is worth more than either list.',
+                    'Look, do not panic: missingOptional (guarded by the layout, safe to omit), unused (may be read by another layout or an API client), untraceable (an ancestor was read but its members could not be followed), unresolvedSections (matched no template here), peerGaps (most records of this type carry it). None of these is a defect on its own.',
+                    'reliable: false means the contract had gaps or this entity has not rendered. An absence is only evidence when it is true.',
                 ],
             })
         },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mikser-io-layouts",
-  "version": "2.10.0",
+  "version": "2.11.1",
   "description": "SSG-flavor task-production policy for mikser-io. Multi-match layout assignment, per-layout destinations, postprocess chain attachment, layout inspection.",
   "main": "index.js",
   "type": "module",

package/test/check-entity.test.js

@@ -198,7 +198,8 @@ describe('mikser_check_entity: a page', () => {
         const r = await check('/documents/extra.md')
         assert.ok(r.unused.includes('card'), `unused: ${r.unused.join(', ')}`)
         assert.deepEqual(r.missing, [], 'an unused key is not a missing one')
-        assert.ok(r.notes.some(n => /DIFFERENT layout/.test(n)))
+        assert.ok(r.notes.some(n => /another layout or an API client/.test(n)),
+            'the caveat has to travel with the finding')
     })
 
     it('never reports engine-owned keys as unused', async () => {
@@ -214,7 +215,8 @@ describe('mikser_check_entity: a page', () => {
         const r = await check('/documents/ghost.md')
         assert.deepEqual(r.unresolvedSections, ['nosuchsection'])
         assert.deepEqual(r.missing, [], 'an unresolved section is not a missing key')
-        assert.ok(r.notes.some(n => /unresolvedSections/.test(n) && /never fails/.test(n)))
+        assert.ok(r.notes.some(n => /unresolvedSections/.test(n) && /defect/.test(n)),
+            'it must be named as something to look at, not a failure')
     })
 
     it('uses the layout the entity actually rendered with', async () => {
@@ -233,7 +235,7 @@ describe('mikser_check_entity: not a page', () => {
         assert.equal(r.kind, 'data')
         assert.deepEqual(r.consumedBy, ['/documents/good.md'])
         assert.ok(!JSON.stringify(r.notes).includes('build once'))
-        assert.ok(r.notes.some(n => /hole in the pages listed above/.test(n)),
+        assert.ok(r.notes.some(n => /hole in those pages/.test(n)),
             'it has to say where a mistake would actually show up')
     })
 
@@ -260,7 +262,8 @@ describe('mikser_check_entity: not a page', () => {
         const group = r.untraceable.find(g => g.under === 'enquiry')
         assert.ok(group, `expected them collapsed under enquiry, got ${JSON.stringify(r.untraceable)}`)
         assert.ok(group.keys.includes('enquiry.fields[].label'))
-        assert.ok(r.notes.some(n => /Treat them as consumed/.test(n)))
+        assert.ok(r.notes.some(n => /untraceable/.test(n) && /could not be followed/.test(n)),
+            'the reason it cannot be judged has to travel with it')
     })
 
     it('still reports a key nothing read at all', async () => {
@@ -384,7 +387,90 @@ describe('mikser_check_entity: with a declared schema', () => {
     it('labels an inferred contract as inferred when there is no schema', async () => {
         const r = await check('/documents/good.md')
         assert.equal(r.missingFrom, 'inferred')
-        assert.ok(r.notes.some(n => /has been wrong/.test(n)),
+        assert.ok(r.notes.some(n => /strong evidence rather than proof/.test(n)),
             'the weaker source has to say that it is weaker')
     })
 })
+
+// Is this record complete AS A RECORD?
+//
+// A layout contract cannot answer that. A catalog entry missing a field its
+// siblings all carry renders perfectly, reads nothing that is absent, and passes
+// every other check here — and a derived schema will not catch it either, since
+// a key one sibling lacks becomes `.optional()`. Comparing a record with its own
+// kind is the only thing that sees it.
+//
+// Weak evidence on purpose: records differ for good reasons.
+describe('mikser_check_entity: peer comparison', () => {
+    let peerCheck
+
+    const entry = (n, extra = {}) => ({
+        id: `/documents/cat-${n}.md`, name: `cat-${n}`, collection: 'documents',
+        meta: { schema: 'catalog', title: `T${n}`, href: `/c/${n}`, summary: 'S', ...extra },
+    })
+
+    before(async () => {
+        // Nine complete siblings, one missing `summary`, and one carrying a key
+        // only it has.
+        const entities = [
+            ...Array.from({ length: 9 }, (_, i) => entry(i)),
+            (() => { const e = entry('gap'); delete e.meta.summary; return e })(),
+            entry('extra', { oddball: true }),
+        ]
+        const findEntity = async ({ id }) => entities.find(e => e.id === id) ?? null
+        const findEntities = async (q) => entities.filter(e =>
+            Object.entries(q ?? {}).every(([k, v]) => k.split('.').reduce((o, x) => o?.[x], e) === v))
+        const runtime = { options: {}, renderers: new Map() }
+        runtime.options.layouts = { inspect: async () => ({ references: {} }) }
+        const registry = new Map()
+        runtime.options.mcp = { simpleTool: (n, _d, _s, h) => registry.set(n, h) }
+        registerCheckTool({
+            runtime, findEntity, findEntities,
+            // No snapshots: these never render, so they are data documents.
+            useDatabase: () => ({ handle: { prepare: () => ({ all: () => [], get: () => undefined }) } }),
+            collection: 'layouts', logger: { debug() {} },
+        })
+        peerCheck = async (id) =>
+            JSON.parse((await registry.get('mikser_check_entity')({ id })).content[0].text)
+    })
+
+    it('reports a key most siblings carry and this record lacks', async () => {
+        const r = await peerCheck('/documents/cat-gap.md')
+        const gap = r.peerGaps?.find(g => g.key === 'summary')
+        assert.ok(gap, `no peer gap found: ${JSON.stringify(r.peerGaps)}`)
+        assert.equal(gap.siblings, 10, 'ten of the ten siblings have it')
+        assert.equal(gap.of, 10)
+    })
+
+    it('names the group, so the count means something', async () => {
+        const r = await peerCheck('/documents/cat-gap.md')
+        assert.equal(r.peerGroup.type, 'catalog')
+        assert.equal(r.peerGroup.peers, 10)
+    })
+
+    it('says nothing about a complete record', async () => {
+        // No false positives, or nobody reads the list.
+        const r = await peerCheck('/documents/cat-0.md')
+        assert.equal(r.peerGaps, undefined)
+    })
+
+    it('does not report a key only ONE sibling has', async () => {
+        // `oddball` exists on exactly one record. One record is not a pattern.
+        const r = await peerCheck('/documents/cat-0.md')
+        assert.ok(!(r.peerGaps ?? []).some(g => g.key === 'oddball'))
+    })
+
+    it('never fails the check — it is a prompt, not a defect', async () => {
+        const r = await peerCheck('/documents/cat-gap.md')
+        assert.notEqual(r.kind, undefined)
+        assert.ok(r.notes.some(n => /peerGaps/.test(n) && /defect|only check possible/.test(n)),
+            'the weakness has to travel with the finding')
+    })
+
+    it('excludes engine-owned keys from the comparison', async () => {
+        const r = await peerCheck('/documents/cat-gap.md')
+        for (const k of ['schema', 'href', 'lang']) {
+            assert.ok(!(r.peerGaps ?? []).some(g => g.key === k), `${k} leaked into peerGaps`)
+        }
+    })
+})