@pikku/inspector 0.12.50 → 0.12.52

package/src/utils/permissions-shorthand.test.ts

@@ -0,0 +1,79 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import * as ts from 'typescript'
+import { extractPermissionPikkuNames } from './permissions.js'
+
+/**
+ * A checker stub. With no symbols to resolve, every reference falls back to a
+ * deterministic `__temp_` id — which is fine, because the question here is
+ * whether a declared permission is *seen*, not what it resolves to. A dropped
+ * property yields nothing at all, and that is the bug being pinned.
+ */
+const checker = {
+  getSymbolAtLocation: () => undefined,
+  getAliasedSymbol: () => undefined,
+  getShorthandAssignmentValueSymbol: () => undefined,
+} as unknown as ts.TypeChecker
+
+/** Parses `permissions: <expr>` and counts the references the inspector records. */
+const countFor = (permissionsExpression: string): number => {
+  const file = ts.createSourceFile(
+    'fn.ts',
+    `const config = { permissions: ${permissionsExpression} }`,
+    ts.ScriptTarget.Latest,
+    true
+  )
+  let node: ts.Expression | undefined
+  const visit = (n: ts.Node) => {
+    if (
+      ts.isPropertyAssignment(n) &&
+      ts.isIdentifier(n.name) &&
+      n.name.text === 'permissions'
+    ) {
+      node = n.initializer
+    }
+    ts.forEachChild(n, visit)
+  }
+  visit(file)
+  assert.ok(node, 'failed to find the permissions node')
+  return extractPermissionPikkuNames(node, checker, '/root').length
+}
+
+describe('extractPermissionPikkuNames', () => {
+  test('records a permission declared with object shorthand', () => {
+    // `permissions: { canAdminOrg }` is enforced identically to the longhand
+    // form at runtime — verifyPermissions has a non-array branch — but was
+    // dropped here, so meta reported a guarded function as carrying no
+    // permissions at all. An audit reading meta saw an open door where one
+    // was shut.
+    assert.equal(countFor('{ canAdminOrg }'), 1)
+  })
+
+  test('records the longhand form', () => {
+    assert.equal(countFor('{ org: canAdminOrg }'), 1)
+  })
+
+  test('records shorthand and longhand together', () => {
+    assert.equal(countFor('{ canAdminOrg, billing: canBill }'), 2)
+  })
+
+  test('records several shorthands', () => {
+    assert.equal(countFor('{ canAdminOrg, canBill }'), 2)
+  })
+
+  test('records the array form', () => {
+    assert.equal(countFor('[canAdminOrg, canBill]'), 2)
+  })
+
+  test('records a group whose value is an array', () => {
+    assert.equal(countFor('{ org: [canAdminOrg, canBill] }'), 2)
+  })
+
+  test('records a call expression, shorthand-adjacent', () => {
+    assert.equal(countFor('{ quota: hasEmailQuota(100) }'), 1)
+  })
+
+  test('records nothing for an empty object', () => {
+    assert.equal(countFor('{}'), 0)
+  })
+})

package/src/utils/permissions.ts

@@ -51,6 +51,13 @@ export function extractPermissionPikkuNames(
     for (const prop of node.properties) {
       if (ts.isPropertyAssignment(prop)) {
         extractFromElement(prop.initializer)
+      } else if (ts.isShorthandPropertyAssignment(prop)) {
+        // `permissions: { canAdminOrg }`. The runtime resolves this exactly
+        // like the longhand form — `verifyPermissions` has a non-array branch —
+        // so dropping it here reported a guarded function as having no
+        // permissions at all, which is the most dangerous direction to be wrong
+        // in: an audit reading meta sees an open door where one is shut.
+        extractFromElement(prop.name)
       }
     }
   }

package/src/utils/post-process.test.ts

@@ -10,6 +10,7 @@ import {
   validateVariableOverrides,
   validateRemoteAddonDependencies,
   validateRemoteAddonAuth,
+  validateAgentToolReferences,
 } from './post-process.js'
 import { ErrorCode } from '../error-codes.js'
 import type { InspectorState, InspectorLogger } from '../types.js'
@@ -580,3 +581,135 @@ describe('validateRemoteAddonAuth (bound credential/secret must exist)', () => {
     assert.equal(criticals[0]!.code, ErrorCode.REMOTE_ADDON_AUTH_UNRESOLVED)
   })
 })
+
+const makeAgentState = (
+  tools: string[],
+  {
+    functions = {},
+    addonFunctions = {},
+    addons = [],
+    internalMeta = {},
+  }: {
+    functions?: Record<string, any>
+    addonFunctions?: Record<string, Record<string, any>>
+    addons?: string[]
+    internalMeta?: Record<string, string>
+  } = {}
+): Omit<InspectorState, 'typesLookup'> =>
+  ({
+    agents: {
+      agentsMeta: {
+        triage: { tools, sourceFile: 'src/triage.agent.ts' },
+      },
+    },
+    rpc: {
+      internalMeta,
+      wireAddonDeclarations: new Map(
+        addons.map((name) => [name, { package: `@addon/${name}` }])
+      ),
+    },
+    functions: { meta: functions },
+    addonFunctions,
+  }) as unknown as Omit<InspectorState, 'typesLookup'>
+
+describe('validateAgentToolReferences (ref() resolved at build time)', () => {
+  test('a local tool that does not exist is an error', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['doesNotExist'], {
+        functions: { realTool: { description: 'A real one' } },
+      })
+    )
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.AGENT_TOOL_NOT_FOUND)
+  })
+
+  test('an addon namespace that is not wired is an error, and lists the ones that are', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['nope:thing'], { addons: ['todos'] })
+    )
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.AGENT_TOOL_UNKNOWN_NAMESPACE)
+    assert.match(criticals[0]!.message, /Wired namespaces: todos/)
+  })
+
+  test('a wired addon that does not export the function is an error', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['todos:noSuchFn'], {
+        addons: ['todos'],
+        addonFunctions: { todos: { addTodo: { description: 'Adds a todo' } } },
+      })
+    )
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.AGENT_TOOL_NOT_FOUND)
+  })
+
+  test('an addon that has not been built yet is not treated as a missing function', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['todos:addTodo'], { addons: ['todos'] })
+    )
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a workflow reference is resolved elsewhere and left alone', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['workflow:onboarding']),
+      { strictMeta: true }
+    )
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a missing description is only an error under strictMeta', () => {
+    const state = makeAgentState(['bareTool'], {
+      functions: { bareTool: {} },
+    })
+
+    const lenient = makeCriticalLogger()
+    validateAgentToolReferences(lenient.logger, state)
+    assert.deepEqual(lenient.criticals, [])
+
+    const strict = makeCriticalLogger()
+    validateAgentToolReferences(strict.logger, state, { strictMeta: true })
+    assert.equal(strict.criticals.length, 1)
+    assert.equal(
+      strict.criticals[0]!.code,
+      ErrorCode.AGENT_TOOL_MISSING_DESCRIPTION
+    )
+  })
+
+  test('a title does not satisfy strictMeta, and the message says so', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['titledTool'], {
+        functions: { titledTool: { title: 'Titled Tool' } },
+      }),
+      { strictMeta: true }
+    )
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.AGENT_TOOL_MISSING_DESCRIPTION)
+    assert.match(criticals[0]!.message, /a 'title' does not count/)
+  })
+
+  test('an addon function is checked against the addon’s own meta', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentToolReferences(
+      logger,
+      makeAgentState(['todos:addTodo'], {
+        addons: ['todos'],
+        addonFunctions: { todos: { addTodo: { description: 'Adds a todo' } } },
+      }),
+      { strictMeta: true }
+    )
+    assert.deepEqual(criticals, [])
+  })
+})

package/src/utils/post-process.ts

@@ -10,10 +10,14 @@ import type {
   MiddlewareMetadata,
   PermissionMetadata,
 } from '@pikku/core'
+import type ts from 'typescript'
 import { existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { extractTypeKeys } from './type-utils.js'
 import { ErrorCode } from '../error-codes.js'
+import { isSecretBrokerFunction } from './secret-brokers.js'
+import { findSecretAliasServices } from './secret-alias-services.js'
+import { relative } from 'node:path'
 import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
 import { flattenScopeDefinitions } from '@pikku/core/scope'
 import type { WorkflowStepMeta } from '@pikku/core/workflow'
@@ -611,6 +615,9 @@ const serializeGroupMap = (
       count: meta.count,
       instanceIds: meta.instanceIds,
       isFactory: meta.isFactory,
+      ...(meta.additionalRegistrations && {
+        additionalRegistrations: meta.additionalRegistrations,
+      }),
     }
   }
   return result
@@ -745,8 +752,11 @@ export function validateScenarioServices(
  *
  * - the step target must be a static string literal (PKU678) — otherwise it
  *   can't be bundled, typed or drawn;
- * - a `browser: true` step must be given an actor (PKU677), since the browser
- *   context signs in as that persona.
+ * - a step declaring a `browser` binding must be given an actor (PKU677), since
+ *   the browser context signs in as that persona. It is checked statically even
+ *   though the binding only runs on a browser run: a step that can never be
+ *   driven by a human is a latent failure, not a passing default run.
+ * - a scenario written as a step ladder must assert something (PKU680).
  *
  * It also marks the step's function as used, so `scenario run` boots the
  * singletons the step actually needs.
@@ -755,9 +765,12 @@ export function validateScenarioSteps(
   logger: InspectorLogger,
   state: InspectorState | Omit<InspectorState, 'typesLookup'>
 ): void {
+  const phasesSeen = new Set<string>()
+
   const visit = (workflowName: string, steps: WorkflowStepMeta[]): void => {
     for (const step of steps) {
       if (step.type === 'scenarioStep') {
+        phasesSeen.add(step.phase)
         if (step.stepFunc === DYNAMIC_SCENARIO_STEP_TARGET) {
           logger.critical(
             ErrorCode.SCENARIO_STEP_TARGET_NOT_STATIC,
@@ -769,10 +782,13 @@ export function validateScenarioSteps(
         }
         state.serviceAggregation.usedFunctions.add(step.stepFunc)
         const stepMeta = state.functions.meta[step.stepFunc]
-        if (stepMeta?.scenarioStepBrowser && !step.actor) {
+        if (
+          stepMeta?.scenarioStepSurfaces?.includes('browser') &&
+          !step.actor
+        ) {
           logger.critical(
             ErrorCode.SCENARIO_BROWSER_STEP_NEEDS_ACTOR,
-            `Scenario '${workflowName}' calls browser step '${step.stepFunc}' without an actor. ` +
+            `Scenario '${workflowName}' calls step '${step.stepFunc}', which declares a browser binding, without an actor. ` +
               `Pass { actor: actors.<name> } so the browser context signs in as that persona.`
           )
         }
@@ -795,7 +811,20 @@ export function validateScenarioSteps(
   }
 
   for (const [workflowName, meta] of Object.entries(state.workflows.meta)) {
+    phasesSeen.clear()
     visit(workflowName, meta.steps ?? [])
+    // A scenario written as a step ladder but with no `then` proves only that
+    // nothing threw. It is also invisible to witness coverage — it contributes
+    // 0/0, so it can never lower the number — which makes an assertion-free
+    // flow the cheapest way to make a suite look better covered than it is.
+    if (phasesSeen.size > 0 && !phasesSeen.has('then')) {
+      logger.critical(
+        ErrorCode.SCENARIO_HAS_NO_ASSERTION,
+        `Scenario '${workflowName}' has ${[...phasesSeen].join('/')} steps but never asserts. ` +
+          `Add a scenario.then(...) naming what the actor should now see — a flow with no 'then' ` +
+          `only proves nothing threw, and contributes nothing to witness coverage.`
+      )
+    }
   }
 }
 
@@ -863,13 +892,18 @@ export function validateSchemaWiringSeparation(
     wiringFiles.add(file)
   }
 
-  // Middleware wirings (addHTTPMiddleware calls)
-  for (const meta of state.http.routeMiddleware.values()) {
-    wiringFiles.add(meta.sourceFile)
-  }
-  for (const meta of state.middleware.tagMiddleware.values()) {
-    wiringFiles.add(meta.sourceFile)
+  // Middleware wirings (addHTTPMiddleware calls). A group can be registered
+  // from more than one file, and every one of them is a wiring file.
+  const addGroupFiles = (groups: Map<string, MiddlewareGroupMeta>) => {
+    for (const meta of groups.values()) {
+      wiringFiles.add(meta.sourceFile)
+      for (const registration of meta.additionalRegistrations ?? []) {
+        wiringFiles.add(registration.sourceFile)
+      }
+    }
   }
+  addGroupFiles(state.http.routeMiddleware)
+  addGroupFiles(state.middleware.tagMiddleware)
 
   // Check for overlap
   for (const file of schemaFiles) {
@@ -888,6 +922,101 @@ export function validateSchemaWiringSeparation(
   }
 }
 
+/** Marks the functions that keep the full `SecretService`. */
+export function computeSecretBrokers(state: InspectorState): void {
+  for (const [id, meta] of Object.entries(state.functions.meta)) {
+    if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
+      meta.secretBroker = true
+    }
+  }
+}
+
+/**
+ * Refuses a singleton service that is a `SecretService` under another name.
+ *
+ * `secrets` is omitted from every function-facing services type, but
+ * `pikkuServices(async ({ secrets }) => ({ cfg: secrets }))` re-exposes it under
+ * a name the omit does not cover. A name-based rule misses this; the type does
+ * not.
+ */
+export function validateNoSecretAliasServices(
+  logger: InspectorLogger,
+  checker: ts.TypeChecker,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!('typesLookup' in state)) {
+    return
+  }
+  const singletonServicesTypes = state.typesLookup.get('SingletonServices')
+  if (!singletonServicesTypes?.length) {
+    return
+  }
+  const aliases = findSecretAliasServices(singletonServicesTypes[0]!, checker)
+  if (aliases.length === 0) {
+    return
+  }
+
+  const report = (kind: string, id: string, services: string[]) => {
+    const used = services.filter((service) => aliases.includes(service))
+    if (used.length === 0) {
+      return
+    }
+    logger.critical(
+      ErrorCode.SECRET_SERVICE_ALIASED,
+      `${kind} '${id}' receives ${used.map((s) => `'${s}'`).join(', ')}, which ${used.length === 1 ? 'is' : 'are'} a SecretService under another name. ` +
+        `SecretService is confined to pikkuServices, pikkuWireServices, addon service factories and middleware — give a service the secret value when you construct it and expose only what the function needs.`
+    )
+  }
+
+  for (const [id, meta] of Object.entries(state.functions.meta)) {
+    if (meta.secretBroker) {
+      continue
+    }
+    report('Function', id, meta.services?.services ?? [])
+  }
+  for (const [id, def] of Object.entries(state.permissions.definitions)) {
+    report('Permission', id, def.services?.services ?? [])
+  }
+}
+
+/**
+ * Reports secret reads the catalogue does not account for.
+ *
+ * `getSecret('X')` with no matching `defineSecret` is a value that will be missing
+ * at first request rather than at deploy; a non-literal key is a read the
+ * manifest cannot cover, so a per-unit scope cannot be narrowed around it.
+ */
+export function validateSecretUsage(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  const declared = new Set(state.secrets.definitions.map((d) => d.secretId))
+  for (const definition of state.secrets.definitions) {
+    if (definition.oauth2?.tokenSecretId) {
+      declared.add(definition.oauth2.tokenSecretId)
+    }
+  }
+
+  for (const [file, usage] of state.secrets.usage) {
+    const relativeFile = relative(state.rootDir, file)
+    const undeclared = usage.keys.filter((key) => !declared.has(key))
+    if (undeclared.length > 0) {
+      logger.diagnostic({
+        severity: 'warn',
+        code: ErrorCode.SECRET_NOT_DECLARED,
+        message: `${relativeFile} reads ${undeclared.map((k) => `'${k}'`).join(', ')}, which no defineSecret declares. Declare it so it is validated at deploy time and shown to whoever has to provision it.`,
+      })
+    }
+    if (usage.dynamic.length > 0) {
+      logger.diagnostic({
+        severity: 'warn',
+        code: ErrorCode.SECRET_KEY_NOT_STATIC,
+        message: `${relativeFile} reads a secret with a non-literal key (${usage.dynamic.join(', ')}). The deployment cannot narrow its secret scope around a key it cannot see.`,
+      })
+    }
+  }
+}
+
 export function computeDiagnostics(state: InspectorState): void {
   const diagnostics: InspectorDiagnostic[] = []
 
@@ -969,7 +1098,7 @@ export function computeDiagnostics(state: InspectorState): void {
 
 /**
  * Validates that every scope referenced by a function is declared via
- * `wireScope`. Runs after all visitors, so declaration order does not matter.
+ * `defineScope`. Runs after all visitors, so declaration order does not matter.
  *
  * A `*` suffix is a wildcard requirement: `admin:*` requires the `admin` scope
  * to be declared, and grants its whole subtree.
@@ -1001,7 +1130,176 @@ export function validateScopeReferences(
         const available = Array.from(declared)
         logger.critical(
           ErrorCode.INVALID_VALUE,
-          `Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with wireScope. Available scopes: ${available.join(', ') || 'none'}`
+          `Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Resolves every `tools:` entry on every agent and reports the ones that do not
+ * land on a real function, or that land on one with nothing to tell the model.
+ *
+ * `ref('todos:listTodos')` is a bare string to the inspector — `ref` is not
+ * resolved, only unwrapped — so until now a reference to a function that never
+ * existed generated cleanly and failed at agent-run time. Addon namespaces come
+ * from `wireAddon`, and each addon's generated metadata is read off disk, which
+ * is the only view of it this project has.
+ *
+ * The description check is separate and opt-in (`--strict-meta`), because the
+ * description is what the model is told the tool does: without one, the tool is
+ * offered to it under its own name (`ai-agent-prepare.ts`), which is a silent
+ * quality loss rather than a failure. A title deliberately does not satisfy it —
+ * a title labels a tool in a UI, it does not tell a model when to reach for it.
+ */
+export function validateAgentToolReferences(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>,
+  options: InspectorOptions = {}
+): void {
+  const agents = Object.entries(state.agents.agentsMeta)
+  if (agents.length === 0) return
+
+  const { wireAddonDeclarations } = state.rpc
+
+  for (const [agentKey, agent] of agents) {
+    const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
+
+    for (const tool of agent.tools ?? []) {
+      // A workflow reference is resolved against the workflow map, not here.
+      if (tool.startsWith('workflow:')) continue
+
+      const separator = tool.indexOf(':')
+      let meta: { description?: string; title?: string } | undefined
+
+      if (separator === -1) {
+        const funcId = state.rpc.internalMeta[tool] ?? tool
+        meta = state.functions.meta[funcId]
+        if (!meta) {
+          logger.critical(
+            ErrorCode.AGENT_TOOL_NOT_FOUND,
+            `AI agent '${agentKey}'${where} references tool '${tool}', which is not a function in this project.`
+          )
+          continue
+        }
+      } else {
+        const namespace = tool.slice(0, separator)
+        const funcName = tool.slice(separator + 1)
+        const addon = wireAddonDeclarations?.get(namespace)
+        if (!addon) {
+          const known = Array.from(wireAddonDeclarations?.keys() ?? [])
+          logger.critical(
+            ErrorCode.AGENT_TOOL_UNKNOWN_NAMESPACE,
+            `AI agent '${agentKey}'${where} references tool '${tool}', but no addon is wired under the namespace '${namespace}'. ` +
+              `Wired namespaces: ${known.join(', ') || 'none'}.`
+          )
+          continue
+        }
+
+        // An addon that has not been built yet contributed no metadata, which
+        // is not the same as one whose function is missing.
+        const addonMeta = state.addonFunctions[namespace]
+        if (!addonMeta) continue
+
+        meta = addonMeta[funcName]
+        if (!meta) {
+          logger.critical(
+            ErrorCode.AGENT_TOOL_NOT_FOUND,
+            `AI agent '${agentKey}'${where} references tool '${tool}', but addon '${namespace}' ('${addon.package}') exposes no function '${funcName}'.`
+          )
+          continue
+        }
+      }
+
+      if (options.strictMeta && !meta.description) {
+        logger.critical(
+          ErrorCode.AGENT_TOOL_MISSING_DESCRIPTION,
+          `AI agent '${agentKey}'${where} uses tool '${tool}', which has no description. ` +
+            `An agent tool's description is what the model is told it does; without one it is offered the tool under its own name. ` +
+            `Add a 'description' to '${tool}'${meta.title ? " (a 'title' does not count — it labels the tool, it does not explain it)" : ''}.`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Validates that every scope granted by a system role is declared via
+ * `defineScope`. Runs after all visitors, so declaration order does not matter.
+ *
+ * The same rule functions get, for a sharper reason: a role granting an
+ * undeclared scope is not merely useless, it is *invisibly* useless. No
+ * function checks the scope, so the role appears to work — every call a
+ * persona holding it makes succeeds or fails for unrelated reasons, and the
+ * authorization boundary it was written to describe is never exercised.
+ *
+ * A `*` suffix grants a subtree; the node it hangs off must exist.
+ */
+export function validateSystemRoleScopes(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!state.systemRoles.definitions.length) {
+    return
+  }
+
+  const declared = new Set(
+    flattenScopeDefinitions(state.scopes.definitions).map((s) => s.id)
+  )
+
+  for (const role of state.systemRoles.definitions) {
+    for (const scope of role.scopes) {
+      if (scope === '*') {
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `System role '${role.name}' grants the bare wildcard scope '*'. Grant the roots it should cover instead — '*' hides what the role actually confers.`
+        )
+        continue
+      }
+
+      const declaredForm = scope.endsWith(':*') ? scope.slice(0, -2) : scope
+      if (!declared.has(declaredForm)) {
+        const available = Array.from(declared)
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `System role '${role.name}' grants scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Validates that every role a persona names is declared with
+ * `defineSystemRole`. Runs after all visitors, so declaration order does not
+ * matter.
+ *
+ * Custom roles are refused rather than tolerated. A role composed in the
+ * console does not exist at build time and can be deleted after it, so a
+ * persona pinned to one silently stops testing what it claims to — and the
+ * symptom is a scatter of 403s that read like authorization findings rather
+ * than like a deleted role.
+ */
+export function validatePersonaRoles(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!state.personas.definitions.length) {
+    return
+  }
+
+  const declared = new Set(state.systemRoles.definitions.map((r) => r.name))
+
+  for (const persona of state.personas.definitions) {
+    for (const role of persona.roles) {
+      if (!declared.has(role)) {
+        const available = Array.from(declared).sort()
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `Persona '${persona.id}' holds role '${role}', which is not declared with defineSystemRole. ` +
+            `A persona may only name a system role — a role composed in the console can be deleted, and the persona would go on claiming to test it. ` +
+            `Declared roles: ${available.join(', ') || 'none'}`
         )
       }
     }