@pikku/inspector 0.12.56 → 0.12.59

package/src/add/add-scenario-step.test.ts

@@ -5,7 +5,8 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { inspect } from '../inspector.js'
 import type { InspectorLogger } from '../types.js'
-import type { ScenarioStepMeta, WorkflowStepMeta } from '@pikku/core/workflow'
+import type { WorkflowStepMeta } from '@pikku/core/workflow'
+import type { ScenarioStepMeta } from '@pikku/core/scenario'
 
 function makeLogger(
   criticals: Array<{ code: string; message: string }>
@@ -243,6 +244,25 @@ describe('pikkuScenarioStep', () => {
     }
   })
 
+  test('a graded agent run is a witness too', async () => {
+    // An agent's answer cannot be matched against a fixed string, so a scenario
+    // that grades one is asserting — even though `expectScore`, like the other
+    // helpers, is an inline step that reaches PKU680 as nothing at all.
+    const { criticals, cleanup } = await run([
+      "const answer = await scenario.when('asks for a summary', 'runAssistant', { prompt: 'hi' }, { actor: actors.shopper })",
+      "await scenario.expectScore('answered briefly', answer.runId, 'brevity', { atLeast: 0.8 })",
+    ])
+    try {
+      assert.equal(
+        criticals.filter((c) => c.code === 'PKU680').length,
+        0,
+        `got: ${JSON.stringify(criticals)}`
+      )
+    } finally {
+      await cleanup()
+    }
+  })
+
   test('an expectation parked in an uncalled helper is still no assertion', async () => {
     const { criticals, cleanup } = await run([
       'const neverCalled = async () => {',

package/src/add/add-wire-addon.test.ts

@@ -61,9 +61,83 @@ describe('addWireAddon', () => {
       secretOverrides: undefined,
       variableOverrides: undefined,
       credentialOverrides: undefined,
+      secretGrants: undefined,
+      credentialGrants: undefined,
+      globalSecrets: undefined,
+      globalCredentials: undefined,
     })
   })
 
+  test('captures the secrets and credentials the host lends an addon', () => {
+    const declarations = inspect(`
+      wireAddon({
+        name: 'graph',
+        package: '@pikku/addon-graph',
+        secretGrants: ['STRIPE_KEY', 'GITHUB_TOKEN'],
+        credentialGrants: ['slack'],
+      })
+    `)
+
+    assert.deepEqual(declarations.get('graph').secretGrants, [
+      'STRIPE_KEY',
+      'GITHUB_TOKEN',
+    ])
+    assert.deepEqual(declarations.get('graph').credentialGrants, ['slack'])
+  })
+
+  test('drops a grant list that is not statically knowable', () => {
+    const declarations = inspect(`
+      wireAddon({
+        name: 'graph',
+        package: '@pikku/addon-graph',
+        secretGrants: someRuntimeList,
+      })
+    `)
+
+    assert.equal(declarations.get('graph').secretGrants, undefined)
+  })
+
+  test('captures the reason an addon was exempted from credential scoping', () => {
+    const declarations = inspect(`
+      wireAddon({
+        name: 'console',
+        package: '@pikku/addon-console',
+        globalCredentials: 'links credentials an operator names at runtime',
+      })
+    `)
+
+    assert.equal(
+      declarations.get('console').globalCredentials,
+      'links credentials an operator names at runtime'
+    )
+  })
+
+  test('captures the reason an addon was exempted from secret scoping', () => {
+    const declarations = inspect(`
+      wireAddon({
+        name: 'console',
+        package: '@pikku/addon-console',
+        globalSecrets: 'administers secrets an operator names at runtime',
+      })
+    `)
+
+    assert.equal(
+      declarations.get('console').globalSecrets,
+      'administers secrets an operator names at runtime'
+    )
+  })
+
+  test('a globalSecrets reason that is not a literal still reports the grant', () => {
+    // Unlike `scopes`, dropping this would under-report: the grant is real at
+    // runtime whatever the reason evaluates to. The source text is recorded so
+    // the manifest names the addon and points at where the reason comes from.
+    const declarations = inspect(`
+      wireAddon({ name: 'console', package: '@x/y', globalSecrets: REASON })
+    `)
+
+    assert.equal(declarations.get('console').globalSecrets, 'REASON')
+  })
+
   test('leaves the gates undefined when none are declared', () => {
     const declarations = inspect(`
       wireAddon({ name: 'console', package: '@pikku/addon-console' })

package/src/add/add-wire-addon.ts

@@ -60,6 +60,10 @@ export function addWireAddon(
   let secretOverrides: Record<string, string> | undefined
   let variableOverrides: Record<string, string> | undefined
   let credentialOverrides: Record<string, string> | undefined
+  let secretGrants: string[] | undefined
+  let credentialGrants: string[] | undefined
+  let globalSecrets: string | undefined
+  let globalCredentials: string | undefined
 
   for (const prop of firstArg.properties) {
     if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue
@@ -102,6 +106,20 @@ export function addWireAddon(
       ts.isObjectLiteralExpression(prop.initializer)
     ) {
       credentialOverrides = parseStringRecord(prop.initializer)
+    } else if (key === 'secretGrants') {
+      secretGrants = parseStringArray(prop.initializer)
+    } else if (key === 'credentialGrants') {
+      credentialGrants = parseStringArray(prop.initializer)
+    } else if (key === 'globalSecrets') {
+      // The grant is real at runtime whatever the reason evaluates to, so a
+      // non-literal is recorded as its source text rather than dropped.
+      globalSecrets = ts.isStringLiteral(prop.initializer)
+        ? prop.initializer.text
+        : prop.initializer.getText()
+    } else if (key === 'globalCredentials') {
+      globalCredentials = ts.isStringLiteral(prop.initializer)
+        ? prop.initializer.text
+        : prop.initializer.getText()
     }
   }
 
@@ -118,6 +136,10 @@ export function addWireAddon(
     secretOverrides,
     variableOverrides,
     credentialOverrides,
+    secretGrants,
+    credentialGrants,
+    globalSecrets,
+    globalCredentials,
   })
   state.rpc.usedAddons.add(name)
   state.rpc.wireAddonFiles.add(node.getSourceFile().fileName)

package/src/add/scenario-instrumentation.ts

@@ -13,12 +13,18 @@
  * `pikku dev` registers the implementations itself. Ignoring by name (rather
  * than deleting the scaffold and requiring every project to regenerate) is what
  * keeps an existing project's checked-in `scenarios.gen.ts` from deploying.
+ *
+ * `pikkuScenarioGradeRun` was never scaffolded — it arrived after the split —
+ * but it belongs here for the same reason: grading a run on demand, ignoring the
+ * sample rate that governs live traffic, is a thing you do to a development
+ * server, and the name is reserved so an application cannot claim it.
  */
 export const SCENARIO_INSTRUMENTATION_FUNCTIONS = [
   'pikkuScenarioTakeLiveCoverage',
   'pikkuScenarioResetLiveCoverage',
   'pikkuScenarioResetStubs',
   'pikkuScenarioGetStubCalls',
+  'pikkuScenarioGradeRun',
 ] as const
 
 export type ScenarioInstrumentationFunction =

package/src/error-codes.ts

@@ -91,6 +91,9 @@ export enum ErrorCode {
   AGENT_TOOL_NOT_FOUND = 'PKU153',
   AGENT_TOOL_MISSING_DESCRIPTION = 'PKU154',
 
+  // Agent scorer reference errors
+  AGENT_SCORER_NOT_FOUND = 'PKU155',
+
   // File structure errors
   SCHEMA_AND_WIRING_COLOCATED = 'PKU490',
 
@@ -102,6 +105,7 @@ export enum ErrorCode {
   // Dependency integrity errors
   DUPLICATE_CORE_VERSION = 'PKU717',
   CORE_VERSION_SKEW = 'PKU718',
+  SPLIT_TYPE_IDENTITY = 'PKU719',
 
   // Data classification errors
   PII_IN_OUTPUT = 'PKU910',

package/src/inspector.ts

@@ -16,6 +16,7 @@ import {
   stampAuthHandlerServices,
   validateAgentModels,
   validateAgentToolReferences,
+  validateAgentScorerReferences,
   validateSecretOverrides,
   validateVariableOverrides,
   validateCredentialOverrides,
@@ -28,7 +29,6 @@ import {
   computeMiddlewareGroupsMeta,
   computePermissionsGroupsMeta,
   computeRequiredSchemas,
-  computeSecretBrokers,
   validateNoSecretAliasServices,
   validateSecretUsage,
   computeDiagnostics,
@@ -160,6 +160,10 @@ export function getInitialInspectorState(rootDir: string): InspectorState {
       agentsMeta: {},
       files: new Map(),
     },
+    scorers: {
+      scorersMeta: {},
+      files: new Map(),
+    },
     cli: {
       meta: {
         programs: {},
@@ -484,7 +488,6 @@ export const inspect = async (
     computeResolvedIOTypes(state)
     computeMiddlewareGroupsMeta(state)
     computePermissionsGroupsMeta(state)
-    computeSecretBrokers(state)
     validateNoSecretAliasServices(logger, checker, state)
     validateSecretUsage(logger, state)
     computeDiagnostics(state)
@@ -511,8 +514,9 @@ export const inspect = async (
       )
     }
 
-    validateAgentModels(logger, state)
+    validateAgentModels(logger, state, options.modelAliases)
     validateAgentToolReferences(logger, state, options)
+    validateAgentScorerReferences(logger, state)
     validateSecretOverrides(logger, state)
     validateVariableOverrides(logger, state)
     validateCredentialOverrides(logger, state)

package/src/types.ts

@@ -13,6 +13,7 @@ import type {
   MCPPromptMeta,
 } from '@pikku/core/mcp'
 import type { AIAgentMeta } from '@pikku/core/ai-agent'
+import type { ScorerMeta } from '@pikku/core/ai-scorer'
 import type { CLIMeta } from '@pikku/core/cli'
 import type { CLICommandMeta } from '@pikku/core/cli'
 import type { NodesMeta } from '@pikku/core/node'
@@ -298,6 +299,8 @@ export type InspectorOptions = Partial<{
    * tool's own name, so this is a quality gate rather than a correctness one.
    */
   strictMeta: boolean
+  /** The `models` table from pikku.config.json, alias -> `provider/model`. */
+  modelAliases: Record<string, string>
   sourceFile: ts.SourceFile
   /**
    * The project's tsconfig, read for its path mappings only. The inspector's
@@ -585,6 +588,20 @@ export interface InspectorState {
         secretOverrides?: Record<string, string>
         variableOverrides?: Record<string, string>
         credentialOverrides?: Record<string, string>
+        /** Secrets the app lends this instance, named as the addon reads them. */
+        secretGrants?: string[]
+        /** Credentials the app lends this instance, named as the addon reads them. */
+        credentialGrants?: string[]
+        /**
+         * The app's stated reason for handing this instance the whole
+         * `SecretService` instead of one scoped to its declared secrets.
+         */
+        globalSecrets?: string
+        /**
+         * The app's stated reason for handing this instance the whole
+         * `CredentialService` instead of one scoped to its declared credentials.
+         */
+        globalCredentials?: string
       }
     >
     wireAddonFiles: Set<string>
@@ -599,6 +616,10 @@ export interface InspectorState {
     agentsMeta: AIAgentMeta
     files: Map<string, { path: string; exportedName: string }>
   }
+  scorers: {
+    scorersMeta: ScorerMeta
+    files: Map<string, { path: string; exportedName: string }>
+  }
   cli: {
     meta: CLIMeta
     files: Set<string>

package/src/utils/filter-inspector-state.ts

@@ -419,6 +419,14 @@ export function filterInspectorState(
       agentsMeta: JSON.parse(JSON.stringify(state.agents?.agentsMeta ?? {})),
       files: new Map(),
     },
+    // Scorers carry through every filter. They are named by the agents that
+    // survive it, and a deployment that quietly stopped grading would look
+    // exactly like one whose scorers all returned nothing.
+    scorers: {
+      ...state.scorers,
+      scorersMeta: JSON.parse(JSON.stringify(state.scorers?.scorersMeta ?? {})),
+      files: new Map(state.scorers?.files ?? []),
+    },
     rpc: {
       ...state.rpc,
       internalMeta: { ...state.rpc.internalMeta }, // Clone to avoid mutating original

package/src/utils/post-process.test.ts

@@ -11,6 +11,7 @@ import {
   validateRemoteAddonDependencies,
   validateRemoteAddonAuth,
   validateAgentToolReferences,
+  validateAgentModels,
 } from './post-process.js'
 import { ErrorCode } from '../error-codes.js'
 import type { InspectorState, InspectorLogger } from '../types.js'
@@ -58,6 +59,33 @@ const makeOverrideState = (
     },
   }) as unknown as Omit<InspectorState, 'typesLookup'>
 
+const makeGrantState = (
+  kind: 'secrets' | 'credentials',
+  declaredNames: string[],
+  decl: {
+    secretGrants?: string[]
+    credentialGrants?: string[]
+    secretOverrides?: Record<string, string>
+    credentialOverrides?: Record<string, string>
+  }
+): Omit<InspectorState, 'typesLookup'> =>
+  ({
+    rpc: {
+      wireAddonDeclarations: new Map([
+        ['slack-marketing', { package: '@addon/slack', ...decl }],
+      ]),
+    },
+    secrets: {
+      definitions:
+        kind === 'secrets' ? declaredNames.map((name) => ({ name })) : [],
+    },
+    credentials: {
+      definitions:
+        kind === 'credentials' ? declaredNames.map((name) => ({ name })) : [],
+    },
+    variables: { definitions: [] },
+  }) as unknown as Omit<InspectorState, 'typesLookup'>
+
 function makeState(
   overrides: {
     usedFunctions?: string[]
@@ -413,6 +441,46 @@ describe('override validation resolves the override target (value), not the logi
     validateVariableOverrides(logger, state)
     assert.deepEqual(criticals, [])
   })
+
+  test('validateSecretOverrides accepts a grant naming a declared secret', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    const state = makeGrantState('secrets', ['STRIPE_KEY'], {
+      secretGrants: ['STRIPE_KEY'],
+    })
+    validateSecretOverrides(logger, state)
+    assert.deepEqual(criticals, [])
+  })
+
+  test('validateSecretOverrides flags a grant naming no declared secret', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    const state = makeGrantState('secrets', ['STRIPE_KEY'], {
+      secretGrants: ['STIRPE_KEY'],
+    })
+    validateSecretOverrides(logger, state)
+    assert.equal(criticals.length, 1)
+    assert.match(criticals[0]!.message, /STIRPE_KEY/)
+    assert.match(criticals[0]!.message, /STRIPE_KEY/)
+  })
+
+  test('validateSecretOverrides resolves a grant through its override before looking it up', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    const state = makeGrantState('secrets', ['PROD_EMAIL_KEY'], {
+      secretGrants: ['MAILGUN_KEY'],
+      secretOverrides: { MAILGUN_KEY: 'PROD_EMAIL_KEY' },
+    })
+    validateSecretOverrides(logger, state)
+    assert.deepEqual(criticals, [])
+  })
+
+  test('validateCredentialOverrides flags a grant naming no declared credential', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    const state = makeGrantState('credentials', ['marketing_cred'], {
+      credentialGrants: ['ghost_cred'],
+    })
+    validateCredentialOverrides(logger, state)
+    assert.equal(criticals.length, 1)
+    assert.match(criticals[0]!.message, /ghost_cred/)
+  })
 })
 
 const makeRemoteState = (
@@ -713,3 +781,73 @@ describe('validateAgentToolReferences (ref() resolved at build time)', () => {
     assert.deepEqual(criticals, [])
   })
 })
+
+const makeModelState = (
+  models: Record<string, string>
+): Omit<InspectorState, 'typesLookup'> =>
+  ({
+    agents: {
+      agentsMeta: Object.fromEntries(
+        Object.entries(models).map(([name, model]) => [
+          name,
+          { name, model, sourceFile: `src/${name}.agent.ts` },
+        ])
+      ),
+    },
+  }) as unknown as Omit<InspectorState, 'typesLookup'>
+
+describe('validateAgentModels (aliases resolve against pikku.config.json)', () => {
+  test('a provider-qualified model needs no alias', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'openai/gpt-5-mini' }))
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a missing model is still reported', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: '' }))
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.MISSING_MODEL)
+  })
+
+  test('a bare name that the models table defines is accepted', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'cheap' }), {
+      cheap: 'openai/gpt-5-mini',
+    })
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a bare name with no alias is an error naming the ones that exist', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'exspensive' }), {
+      cheap: 'openai/gpt-5-mini',
+      expensive: 'openai/gpt-5',
+    })
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+    assert.match(criticals[0]!.message, /cheap, expensive/)
+  })
+
+  test('with no models table a bare name still fails, as it always has', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'gpt-5-mini' }))
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+  })
+
+  for (const inherited of ['toString', 'constructor', '__proto__']) {
+    test(`a bare '${inherited}' is not an alias just because Object has one`, () => {
+      const { logger, criticals } = makeCriticalLogger()
+      validateAgentModels(logger, makeModelState({ triage: inherited }), {
+        cheap: 'openai/gpt-5-mini',
+      })
+      assert.equal(
+        criticals.length,
+        1,
+        'an inherited property name must not read as a configured alias'
+      )
+      assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+    })
+  }
+})

package/src/utils/post-process.ts

@@ -15,7 +15,6 @@ import { existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { extractTypeKeys } from './type-utils.js'
 import { ErrorCode } from '../error-codes.js'
-import { isSecretBrokerFunction } from './secret-brokers.js'
 import { findSecretAliasServices } from './secret-alias-services.js'
 import { relative } from 'node:path'
 import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
@@ -384,10 +383,8 @@ export function validateSecretOverrides(
   )
 
   for (const [namespace, addonDecl] of wireAddonDeclarations.entries()) {
-    if (!addonDecl.secretOverrides) continue
-
     for (const [logicalName, resolvedName] of Object.entries(
-      addonDecl.secretOverrides
+      addonDecl.secretOverrides ?? {}
     )) {
       if (!secretIds.has(resolvedName)) {
         const availableSecrets = Array.from(secretIds)
@@ -397,6 +394,20 @@ export function validateSecretOverrides(
         )
       }
     }
+
+    // A grant names the secret as the addon reads it, so it resolves through
+    // the override map before it can be looked up in the project.
+    for (const logicalName of addonDecl.secretGrants ?? []) {
+      const resolvedName =
+        addonDecl.secretOverrides?.[logicalName] ?? logicalName
+      if (!secretIds.has(resolvedName)) {
+        const availableSecrets = Array.from(secretIds)
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `Secret grant '${logicalName}' in addon '${namespace}' (${addonDecl.package}) targets a secret that does not exist. Available secrets: ${availableSecrets.join(', ') || 'none'}`
+        )
+      }
+    }
   }
 }
 
@@ -412,10 +423,8 @@ export function validateCredentialOverrides(
   )
 
   for (const [namespace, addonDecl] of wireAddonDeclarations.entries()) {
-    if (!addonDecl.credentialOverrides) continue
-
     for (const [logicalName, resolvedName] of Object.entries(
-      addonDecl.credentialOverrides
+      addonDecl.credentialOverrides ?? {}
     )) {
       if (!credentialNames.has(resolvedName)) {
         const availableCredentials = Array.from(credentialNames)
@@ -425,6 +434,18 @@ export function validateCredentialOverrides(
         )
       }
     }
+
+    for (const logicalName of addonDecl.credentialGrants ?? []) {
+      const resolvedName =
+        addonDecl.credentialOverrides?.[logicalName] ?? logicalName
+      if (!credentialNames.has(resolvedName)) {
+        const availableCredentials = Array.from(credentialNames)
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `Credential grant '${logicalName}' in addon '${namespace}' (${addonDecl.package}) targets a credential that does not exist. Available credentials: ${availableCredentials.join(', ') || 'none'}`
+        )
+      }
+    }
   }
 }
 
@@ -746,9 +767,14 @@ export function validateSchemaReferences(
   })
 }
 
+/**
+ * A model is either concrete (`provider/model`) or an alias, which must appear
+ * in the `models` table from pikku.config.json.
+ */
 export function validateAgentModels(
   logger: InspectorLogger,
-  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>,
+  modelAliases: Record<string, string> = {}
 ): void {
   for (const [, meta] of Object.entries(state.agents.agentsMeta)) {
     const model = meta.model
@@ -759,10 +785,17 @@ export function validateAgentModels(
       )
       continue
     }
-    if (!model.includes('/')) {
+    // `hasOwn`, not a truthy lookup: every object literal inherits `toString`,
+    // `constructor` and `__proto__`, so a bare model named after one of them
+    // would read as a configured alias and pass a check nothing else repeats.
+    if (!model.includes('/') && !Object.hasOwn(modelAliases, model)) {
+      const known = Object.keys(modelAliases).sort()
       logger.critical(
         ErrorCode.INVALID_MODEL,
-        `AI agent '${meta.name}' uses model '${model}', which must be provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4').`
+        `AI agent '${meta.name}' uses model '${model}', which is neither provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4') nor an alias in the "models" table of pikku.config.json. ` +
+          (known.length
+            ? `Known aliases: ${known.join(', ')}.`
+            : `No "models" aliases are configured.`)
       )
     }
   }
@@ -982,15 +1015,6 @@ export function validateSchemaWiringSeparation(
   }
 }
 
-/** Marks the functions that keep the full `SecretService`. */
-export function computeSecretBrokers(state: InspectorState): void {
-  for (const [id, meta] of Object.entries(state.functions.meta)) {
-    if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
-      meta.secretBroker = true
-    }
-  }
-}
-
 /**
  * Refuses a singleton service that is a `SecretService` under another name.
  *
@@ -1029,9 +1053,6 @@ export function validateNoSecretAliasServices(
   }
 
   for (const [id, meta] of Object.entries(state.functions.meta)) {
-    if (meta.secretBroker) {
-      continue
-    }
     report('Function', id, meta.services?.services ?? [])
   }
   for (const [id, def] of Object.entries(state.permissions.definitions)) {
@@ -1284,6 +1305,32 @@ export function validateAgentToolReferences(
   }
 }
 
+/**
+ * Validates that every scorer an agent asks to be graded by exists.
+ *
+ * The runtime warns and skips an unresolvable name rather than failing a run
+ * that has already answered the user — which means a typo would otherwise cost
+ * nothing visible and simply grade nothing, forever.
+ */
+export function validateAgentScorerReferences(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  const known = Object.keys(state.scorers.scorersMeta)
+
+  for (const [agentKey, agent] of Object.entries(state.agents.agentsMeta)) {
+    const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
+    for (const scorer of agent.scorers ?? []) {
+      if (known.includes(scorer)) continue
+      logger.critical(
+        ErrorCode.AGENT_SCORER_NOT_FOUND,
+        `AI agent '${agentKey}'${where} asks to be graded by scorer '${scorer}', which is not declared in this project. ` +
+          `Declared scorers: ${known.join(', ') || 'none'}.`
+      )
+    }
+  }
+}
+
 /**
  * Validates that every scope granted by a system role is declared via
  * `defineScope`. Runs after all visitors, so declaration order does not matter.

package/src/utils/serialize-inspector-state.ts

@@ -170,6 +170,10 @@ export interface SerializableInspectorState {
           secretOverrides?: Record<string, string>
           variableOverrides?: Record<string, string>
           credentialOverrides?: Record<string, string>
+          secretGrants?: string[]
+          credentialGrants?: string[]
+          globalSecrets?: string
+          globalCredentials?: string
         },
       ]
     >
@@ -185,6 +189,10 @@ export interface SerializableInspectorState {
     agentsMeta: InspectorState['agents']['agentsMeta']
     files: [string, { path: string; exportedName: string }][]
   }
+  scorers: {
+    scorersMeta: InspectorState['scorers']['scorersMeta']
+    files: [string, { path: string; exportedName: string }][]
+  }
   cli: {
     meta: InspectorState['cli']['meta']
     files: string[]
@@ -409,6 +417,10 @@ export function serializeInspectorState(
       agentsMeta: state.agents?.agentsMeta ?? {},
       files: Array.from(state.agents?.files?.entries() ?? []),
     },
+    scorers: {
+      scorersMeta: state.scorers?.scorersMeta ?? {},
+      files: Array.from(state.scorers?.files?.entries() ?? []),
+    },
     cli: {
       meta: state.cli.meta,
       files: Array.from(state.cli.files),
@@ -613,6 +625,10 @@ export function deserializeInspectorState(
       agentsMeta: data.agents?.agentsMeta || {},
       files: new Map(data.agents?.files || []),
     },
+    scorers: {
+      scorersMeta: data.scorers?.scorersMeta || {},
+      files: new Map(data.scorers?.files || []),
+    },
     cli: {
       meta: data.cli.meta,
       files: new Set(data.cli.files),