@pikku/inspector 0.12.49 → 0.12.52

package/src/utils/post-process.ts

@@ -10,10 +10,14 @@ import type {
   MiddlewareMetadata,
   PermissionMetadata,
 } from '@pikku/core'
+import type ts from 'typescript'
 import { existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { extractTypeKeys } from './type-utils.js'
 import { ErrorCode } from '../error-codes.js'
+import { isSecretBrokerFunction } from './secret-brokers.js'
+import { findSecretAliasServices } from './secret-alias-services.js'
+import { relative } from 'node:path'
 import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
 import { flattenScopeDefinitions } from '@pikku/core/scope'
 import type { WorkflowStepMeta } from '@pikku/core/workflow'
@@ -611,6 +615,9 @@ const serializeGroupMap = (
       count: meta.count,
       instanceIds: meta.instanceIds,
       isFactory: meta.isFactory,
+      ...(meta.additionalRegistrations && {
+        additionalRegistrations: meta.additionalRegistrations,
+      }),
     }
   }
   return result
@@ -745,8 +752,11 @@ export function validateScenarioServices(
  *
  * - the step target must be a static string literal (PKU678) — otherwise it
  *   can't be bundled, typed or drawn;
- * - a `browser: true` step must be given an actor (PKU677), since the browser
- *   context signs in as that persona.
+ * - a step declaring a `browser` binding must be given an actor (PKU677), since
+ *   the browser context signs in as that persona. It is checked statically even
+ *   though the binding only runs on a browser run: a step that can never be
+ *   driven by a human is a latent failure, not a passing default run.
+ * - a scenario written as a step ladder must assert something (PKU680).
  *
  * It also marks the step's function as used, so `scenario run` boots the
  * singletons the step actually needs.
@@ -755,9 +765,12 @@ export function validateScenarioSteps(
   logger: InspectorLogger,
   state: InspectorState | Omit<InspectorState, 'typesLookup'>
 ): void {
+  const phasesSeen = new Set<string>()
+
   const visit = (workflowName: string, steps: WorkflowStepMeta[]): void => {
     for (const step of steps) {
       if (step.type === 'scenarioStep') {
+        phasesSeen.add(step.phase)
         if (step.stepFunc === DYNAMIC_SCENARIO_STEP_TARGET) {
           logger.critical(
             ErrorCode.SCENARIO_STEP_TARGET_NOT_STATIC,
@@ -769,10 +782,13 @@ export function validateScenarioSteps(
         }
         state.serviceAggregation.usedFunctions.add(step.stepFunc)
         const stepMeta = state.functions.meta[step.stepFunc]
-        if (stepMeta?.scenarioStepBrowser && !step.actor) {
+        if (
+          stepMeta?.scenarioStepSurfaces?.includes('browser') &&
+          !step.actor
+        ) {
           logger.critical(
             ErrorCode.SCENARIO_BROWSER_STEP_NEEDS_ACTOR,
-            `Scenario '${workflowName}' calls browser step '${step.stepFunc}' without an actor. ` +
+            `Scenario '${workflowName}' calls step '${step.stepFunc}', which declares a browser binding, without an actor. ` +
               `Pass { actor: actors.<name> } so the browser context signs in as that persona.`
           )
         }
@@ -795,7 +811,20 @@ export function validateScenarioSteps(
   }
 
   for (const [workflowName, meta] of Object.entries(state.workflows.meta)) {
+    phasesSeen.clear()
     visit(workflowName, meta.steps ?? [])
+    // A scenario written as a step ladder but with no `then` proves only that
+    // nothing threw. It is also invisible to witness coverage — it contributes
+    // 0/0, so it can never lower the number — which makes an assertion-free
+    // flow the cheapest way to make a suite look better covered than it is.
+    if (phasesSeen.size > 0 && !phasesSeen.has('then')) {
+      logger.critical(
+        ErrorCode.SCENARIO_HAS_NO_ASSERTION,
+        `Scenario '${workflowName}' has ${[...phasesSeen].join('/')} steps but never asserts. ` +
+          `Add a scenario.then(...) naming what the actor should now see — a flow with no 'then' ` +
+          `only proves nothing threw, and contributes nothing to witness coverage.`
+      )
+    }
   }
 }
 
@@ -863,13 +892,18 @@ export function validateSchemaWiringSeparation(
     wiringFiles.add(file)
   }
 
-  // Middleware wirings (addHTTPMiddleware calls)
-  for (const meta of state.http.routeMiddleware.values()) {
-    wiringFiles.add(meta.sourceFile)
-  }
-  for (const meta of state.middleware.tagMiddleware.values()) {
-    wiringFiles.add(meta.sourceFile)
+  // Middleware wirings (addHTTPMiddleware calls). A group can be registered
+  // from more than one file, and every one of them is a wiring file.
+  const addGroupFiles = (groups: Map<string, MiddlewareGroupMeta>) => {
+    for (const meta of groups.values()) {
+      wiringFiles.add(meta.sourceFile)
+      for (const registration of meta.additionalRegistrations ?? []) {
+        wiringFiles.add(registration.sourceFile)
+      }
+    }
   }
+  addGroupFiles(state.http.routeMiddleware)
+  addGroupFiles(state.middleware.tagMiddleware)
 
   // Check for overlap
   for (const file of schemaFiles) {
@@ -888,6 +922,101 @@ export function validateSchemaWiringSeparation(
   }
 }
 
+/** Marks the functions that keep the full `SecretService`. */
+export function computeSecretBrokers(state: InspectorState): void {
+  for (const [id, meta] of Object.entries(state.functions.meta)) {
+    if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
+      meta.secretBroker = true
+    }
+  }
+}
+
+/**
+ * Refuses a singleton service that is a `SecretService` under another name.
+ *
+ * `secrets` is omitted from every function-facing services type, but
+ * `pikkuServices(async ({ secrets }) => ({ cfg: secrets }))` re-exposes it under
+ * a name the omit does not cover. A name-based rule misses this; the type does
+ * not.
+ */
+export function validateNoSecretAliasServices(
+  logger: InspectorLogger,
+  checker: ts.TypeChecker,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!('typesLookup' in state)) {
+    return
+  }
+  const singletonServicesTypes = state.typesLookup.get('SingletonServices')
+  if (!singletonServicesTypes?.length) {
+    return
+  }
+  const aliases = findSecretAliasServices(singletonServicesTypes[0]!, checker)
+  if (aliases.length === 0) {
+    return
+  }
+
+  const report = (kind: string, id: string, services: string[]) => {
+    const used = services.filter((service) => aliases.includes(service))
+    if (used.length === 0) {
+      return
+    }
+    logger.critical(
+      ErrorCode.SECRET_SERVICE_ALIASED,
+      `${kind} '${id}' receives ${used.map((s) => `'${s}'`).join(', ')}, which ${used.length === 1 ? 'is' : 'are'} a SecretService under another name. ` +
+        `SecretService is confined to pikkuServices, pikkuWireServices, addon service factories and middleware — give a service the secret value when you construct it and expose only what the function needs.`
+    )
+  }
+
+  for (const [id, meta] of Object.entries(state.functions.meta)) {
+    if (meta.secretBroker) {
+      continue
+    }
+    report('Function', id, meta.services?.services ?? [])
+  }
+  for (const [id, def] of Object.entries(state.permissions.definitions)) {
+    report('Permission', id, def.services?.services ?? [])
+  }
+}
+
+/**
+ * Reports secret reads the catalogue does not account for.
+ *
+ * `getSecret('X')` with no matching `defineSecret` is a value that will be missing
+ * at first request rather than at deploy; a non-literal key is a read the
+ * manifest cannot cover, so a per-unit scope cannot be narrowed around it.
+ */
+export function validateSecretUsage(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  const declared = new Set(state.secrets.definitions.map((d) => d.secretId))
+  for (const definition of state.secrets.definitions) {
+    if (definition.oauth2?.tokenSecretId) {
+      declared.add(definition.oauth2.tokenSecretId)
+    }
+  }
+
+  for (const [file, usage] of state.secrets.usage) {
+    const relativeFile = relative(state.rootDir, file)
+    const undeclared = usage.keys.filter((key) => !declared.has(key))
+    if (undeclared.length > 0) {
+      logger.diagnostic({
+        severity: 'warn',
+        code: ErrorCode.SECRET_NOT_DECLARED,
+        message: `${relativeFile} reads ${undeclared.map((k) => `'${k}'`).join(', ')}, which no defineSecret declares. Declare it so it is validated at deploy time and shown to whoever has to provision it.`,
+      })
+    }
+    if (usage.dynamic.length > 0) {
+      logger.diagnostic({
+        severity: 'warn',
+        code: ErrorCode.SECRET_KEY_NOT_STATIC,
+        message: `${relativeFile} reads a secret with a non-literal key (${usage.dynamic.join(', ')}). The deployment cannot narrow its secret scope around a key it cannot see.`,
+      })
+    }
+  }
+}
+
 export function computeDiagnostics(state: InspectorState): void {
   const diagnostics: InspectorDiagnostic[] = []
 
@@ -969,7 +1098,7 @@ export function computeDiagnostics(state: InspectorState): void {
 
 /**
  * Validates that every scope referenced by a function is declared via
- * `wireScope`. Runs after all visitors, so declaration order does not matter.
+ * `defineScope`. Runs after all visitors, so declaration order does not matter.
  *
  * A `*` suffix is a wildcard requirement: `admin:*` requires the `admin` scope
  * to be declared, and grants its whole subtree.
@@ -1001,7 +1130,176 @@ export function validateScopeReferences(
         const available = Array.from(declared)
         logger.critical(
           ErrorCode.INVALID_VALUE,
-          `Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with wireScope. Available scopes: ${available.join(', ') || 'none'}`
+          `Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Resolves every `tools:` entry on every agent and reports the ones that do not
+ * land on a real function, or that land on one with nothing to tell the model.
+ *
+ * `ref('todos:listTodos')` is a bare string to the inspector — `ref` is not
+ * resolved, only unwrapped — so until now a reference to a function that never
+ * existed generated cleanly and failed at agent-run time. Addon namespaces come
+ * from `wireAddon`, and each addon's generated metadata is read off disk, which
+ * is the only view of it this project has.
+ *
+ * The description check is separate and opt-in (`--strict-meta`), because the
+ * description is what the model is told the tool does: without one, the tool is
+ * offered to it under its own name (`ai-agent-prepare.ts`), which is a silent
+ * quality loss rather than a failure. A title deliberately does not satisfy it —
+ * a title labels a tool in a UI, it does not tell a model when to reach for it.
+ */
+export function validateAgentToolReferences(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>,
+  options: InspectorOptions = {}
+): void {
+  const agents = Object.entries(state.agents.agentsMeta)
+  if (agents.length === 0) return
+
+  const { wireAddonDeclarations } = state.rpc
+
+  for (const [agentKey, agent] of agents) {
+    const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
+
+    for (const tool of agent.tools ?? []) {
+      // A workflow reference is resolved against the workflow map, not here.
+      if (tool.startsWith('workflow:')) continue
+
+      const separator = tool.indexOf(':')
+      let meta: { description?: string; title?: string } | undefined
+
+      if (separator === -1) {
+        const funcId = state.rpc.internalMeta[tool] ?? tool
+        meta = state.functions.meta[funcId]
+        if (!meta) {
+          logger.critical(
+            ErrorCode.AGENT_TOOL_NOT_FOUND,
+            `AI agent '${agentKey}'${where} references tool '${tool}', which is not a function in this project.`
+          )
+          continue
+        }
+      } else {
+        const namespace = tool.slice(0, separator)
+        const funcName = tool.slice(separator + 1)
+        const addon = wireAddonDeclarations?.get(namespace)
+        if (!addon) {
+          const known = Array.from(wireAddonDeclarations?.keys() ?? [])
+          logger.critical(
+            ErrorCode.AGENT_TOOL_UNKNOWN_NAMESPACE,
+            `AI agent '${agentKey}'${where} references tool '${tool}', but no addon is wired under the namespace '${namespace}'. ` +
+              `Wired namespaces: ${known.join(', ') || 'none'}.`
+          )
+          continue
+        }
+
+        // An addon that has not been built yet contributed no metadata, which
+        // is not the same as one whose function is missing.
+        const addonMeta = state.addonFunctions[namespace]
+        if (!addonMeta) continue
+
+        meta = addonMeta[funcName]
+        if (!meta) {
+          logger.critical(
+            ErrorCode.AGENT_TOOL_NOT_FOUND,
+            `AI agent '${agentKey}'${where} references tool '${tool}', but addon '${namespace}' ('${addon.package}') exposes no function '${funcName}'.`
+          )
+          continue
+        }
+      }
+
+      if (options.strictMeta && !meta.description) {
+        logger.critical(
+          ErrorCode.AGENT_TOOL_MISSING_DESCRIPTION,
+          `AI agent '${agentKey}'${where} uses tool '${tool}', which has no description. ` +
+            `An agent tool's description is what the model is told it does; without one it is offered the tool under its own name. ` +
+            `Add a 'description' to '${tool}'${meta.title ? " (a 'title' does not count — it labels the tool, it does not explain it)" : ''}.`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Validates that every scope granted by a system role is declared via
+ * `defineScope`. Runs after all visitors, so declaration order does not matter.
+ *
+ * The same rule functions get, for a sharper reason: a role granting an
+ * undeclared scope is not merely useless, it is *invisibly* useless. No
+ * function checks the scope, so the role appears to work — every call a
+ * persona holding it makes succeeds or fails for unrelated reasons, and the
+ * authorization boundary it was written to describe is never exercised.
+ *
+ * A `*` suffix grants a subtree; the node it hangs off must exist.
+ */
+export function validateSystemRoleScopes(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!state.systemRoles.definitions.length) {
+    return
+  }
+
+  const declared = new Set(
+    flattenScopeDefinitions(state.scopes.definitions).map((s) => s.id)
+  )
+
+  for (const role of state.systemRoles.definitions) {
+    for (const scope of role.scopes) {
+      if (scope === '*') {
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `System role '${role.name}' grants the bare wildcard scope '*'. Grant the roots it should cover instead — '*' hides what the role actually confers.`
+        )
+        continue
+      }
+
+      const declaredForm = scope.endsWith(':*') ? scope.slice(0, -2) : scope
+      if (!declared.has(declaredForm)) {
+        const available = Array.from(declared)
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `System role '${role.name}' grants scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
+        )
+      }
+    }
+  }
+}
+
+/**
+ * Validates that every role a persona names is declared with
+ * `defineSystemRole`. Runs after all visitors, so declaration order does not
+ * matter.
+ *
+ * Custom roles are refused rather than tolerated. A role composed in the
+ * console does not exist at build time and can be deleted after it, so a
+ * persona pinned to one silently stops testing what it claims to — and the
+ * symptom is a scatter of 403s that read like authorization findings rather
+ * than like a deleted role.
+ */
+export function validatePersonaRoles(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  if (!state.personas.definitions.length) {
+    return
+  }
+
+  const declared = new Set(state.systemRoles.definitions.map((r) => r.name))
+
+  for (const persona of state.personas.definitions) {
+    for (const role of persona.roles) {
+      if (!declared.has(role)) {
+        const available = Array.from(declared).sort()
+        logger.critical(
+          ErrorCode.INVALID_VALUE,
+          `Persona '${persona.id}' holds role '${role}', which is not declared with defineSystemRole. ` +
+            `A persona may only name a system role — a role composed in the console can be deleted, and the persona would go on claiming to test it. ` +
+            `Declared roles: ${available.join(', ') || 'none'}`
         )
       }
     }

package/src/utils/schema-cache.test.ts

@@ -0,0 +1,124 @@
+import { test, describe } from 'node:test'
+import { strict as assert } from 'node:assert'
+import {
+  mkdtempSync,
+  rmSync,
+  statSync,
+  utimesSync,
+  writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+  readDiskTSSchemas,
+  writeDiskTSSchemas,
+  type SchemaDep,
+} from './schema-generator.js'
+import type { InspectorLogger } from '../types.js'
+
+const silentLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as unknown as InspectorLogger
+
+const KEY = 'test-key'
+const SCHEMAS = { Thing: { type: 'object' } }
+
+/** A cache directory and one source file the cached schemas claim to derive from. */
+function makeCache(sourceContent: string): {
+  cacheDir: string
+  source: string
+  deps: SchemaDep[]
+  cleanup: () => void
+} {
+  const dir = mkdtempSync(join(tmpdir(), 'pikku-schema-cache-'))
+  const source = join(dir, 'thing.ts')
+  writeFileSync(source, sourceContent)
+  const stat = statSync(source)
+  return {
+    cacheDir: dir,
+    source,
+    deps: [{ path: source, mtimeMs: stat.mtimeMs, size: stat.size }],
+    cleanup: () => rmSync(dir, { recursive: true, force: true }),
+  }
+}
+
+describe('TS schema disk cache', () => {
+  test('serves the cached schemas while their sources are untouched', () => {
+    const { cacheDir, deps, cleanup } = makeCache('export interface Thing {}')
+    try {
+      writeDiskTSSchemas(silentLogger, cacheDir, KEY, SCHEMAS, deps)
+      const hit = readDiskTSSchemas(silentLogger, cacheDir, KEY)
+      assert.deepEqual(hit?.schemas, SCHEMAS)
+    } finally {
+      cleanup()
+    }
+  })
+
+  test('discards the cache when a type it was built from changes', () => {
+    const { cacheDir, source, deps, cleanup } = makeCache(
+      'export interface Thing {}'
+    )
+    try {
+      writeDiskTSSchemas(silentLogger, cacheDir, KEY, SCHEMAS, deps)
+      // The shape changes but the name does not, so the cache key — which hashes
+      // only the synthesized custom-types source — is identical. This is the case
+      // that used to serve a stale schema, and it survived `rm -rf .pikku`
+      // because the cache lives under node_modules/.cache.
+      writeFileSync(source, 'export interface Thing { added: string }')
+      assert.equal(readDiskTSSchemas(silentLogger, cacheDir, KEY), null)
+    } finally {
+      cleanup()
+    }
+  })
+
+  test('discards the cache when a source is rewritten to the same size', () => {
+    const { cacheDir, source, deps, cleanup } = makeCache('export type A = 1')
+    try {
+      writeDiskTSSchemas(silentLogger, cacheDir, KEY, SCHEMAS, deps)
+      // Same byte count, different type: size alone would call this unchanged, so
+      // the mtime half of the fingerprint is what has to catch it.
+      writeFileSync(source, 'export type A = 2')
+      utimesSync(source, new Date(), new Date(Date.now() + 5000))
+      assert.equal(readDiskTSSchemas(silentLogger, cacheDir, KEY), null)
+    } finally {
+      cleanup()
+    }
+  })
+
+  test('discards the cache when a source it was built from is deleted', () => {
+    const { cacheDir, source, deps, cleanup } = makeCache('export type A = 1')
+    try {
+      writeDiskTSSchemas(silentLogger, cacheDir, KEY, SCHEMAS, deps)
+      rmSync(source)
+      assert.equal(readDiskTSSchemas(silentLogger, cacheDir, KEY), null)
+    } finally {
+      cleanup()
+    }
+  })
+
+  test('discards a cache written before deps were tracked', () => {
+    const { cacheDir, cleanup } = makeCache('export type A = 1')
+    try {
+      writeFileSync(
+        join(cacheDir, 'ts-schemas.json'),
+        JSON.stringify({ key: KEY, schemas: SCHEMAS })
+      )
+      assert.equal(readDiskTSSchemas(silentLogger, cacheDir, KEY), null)
+    } finally {
+      cleanup()
+    }
+  })
+
+  test('still misses on a different key', () => {
+    const { cacheDir, deps, cleanup } = makeCache('export type A = 1')
+    try {
+      writeDiskTSSchemas(silentLogger, cacheDir, KEY, SCHEMAS, deps)
+      assert.equal(readDiskTSSchemas(silentLogger, cacheDir, 'other'), null)
+    } finally {
+      cleanup()
+    }
+  })
+})

package/src/utils/schema-generator.test.ts

@@ -5,7 +5,7 @@ import { tmpdir } from 'os'
 import { join } from 'path'
 
 import { getInitialInspectorState } from '../inspector.js'
-import { generateAllSchemas } from './schema-generator.js'
+import { generateAllSchemas, schemaRuntimeFile } from './schema-generator.js'
 
 const debugLines: string[] = []
 const logger = {
@@ -162,3 +162,64 @@ describe('generateAllSchemas', () => {
     )
   })
 })
+
+/**
+ * A schema imported from a built workspace package resolves to that package's
+ * declaration file, and a `.d.ts` has no runtime exports at all — every one of
+ * them is a type. Importing it to read the zod object yields an empty module and
+ * every schema in it is reported missing, so the emitted JS beside it is what has
+ * to be imported.
+ */
+describe('schemaRuntimeFile', () => {
+  test('a source file the developer wrote is imported as it is', () => {
+    const written = join(projectDir, 'src', 'types.ts')
+    assert.equal(schemaRuntimeFile(written), written)
+  })
+
+  test('a declaration file resolves to the JS emitted beside it', () => {
+    writeFileSync(
+      join(projectDir, 'src', 'reindex.d.ts'),
+      'export declare const a: number\n'
+    )
+    writeFileSync(join(projectDir, 'src', 'reindex.js'), 'export const a = 1\n')
+    assert.equal(
+      schemaRuntimeFile(join(projectDir, 'src', 'reindex.d.ts')),
+      join(projectDir, 'src', 'reindex.js')
+    )
+  })
+
+  test('.d.mts resolves to the ESM file beside it', () => {
+    writeFileSync(
+      join(projectDir, 'src', 'esm.d.mts'),
+      'export declare const a: number\n'
+    )
+    writeFileSync(join(projectDir, 'src', 'esm.mjs'), 'export const a = 1\n')
+    assert.equal(
+      schemaRuntimeFile(join(projectDir, 'src', 'esm.d.mts')),
+      join(projectDir, 'src', 'esm.mjs')
+    )
+  })
+
+  test('.d.cts resolves to the CommonJS file beside it', () => {
+    // Its own test rather than a second assertion above: the two extensions are
+    // separate branches, and one covering both is one that passes while half of
+    // it is broken.
+    writeFileSync(
+      join(projectDir, 'src', 'commonjs.d.cts'),
+      'export declare const a: number\n'
+    )
+    writeFileSync(join(projectDir, 'src', 'commonjs.cjs'), 'exports.a = 1\n')
+    assert.equal(
+      schemaRuntimeFile(join(projectDir, 'src', 'commonjs.d.cts')),
+      join(projectDir, 'src', 'commonjs.cjs')
+    )
+  })
+
+  test('a declaration file with no emitted JS keeps its own path', () => {
+    // So the "could not find exported schema" error still names a file the
+    // developer can open, rather than one that was never written.
+    const orphan = join(projectDir, 'src', 'types-only.d.ts')
+    writeFileSync(orphan, 'export declare const a: number\n')
+    assert.equal(schemaRuntimeFile(orphan), orphan)
+  })
+})