@pikku/inspector 0.12.56 → 0.12.58

package/src/utils/post-process.test.ts

@@ -11,6 +11,7 @@ import {
   validateRemoteAddonDependencies,
   validateRemoteAddonAuth,
   validateAgentToolReferences,
+  validateAgentModels,
 } from './post-process.js'
 import { ErrorCode } from '../error-codes.js'
 import type { InspectorState, InspectorLogger } from '../types.js'
@@ -713,3 +714,73 @@ describe('validateAgentToolReferences (ref() resolved at build time)', () => {
     assert.deepEqual(criticals, [])
   })
 })
+
+const makeModelState = (
+  models: Record<string, string>
+): Omit<InspectorState, 'typesLookup'> =>
+  ({
+    agents: {
+      agentsMeta: Object.fromEntries(
+        Object.entries(models).map(([name, model]) => [
+          name,
+          { name, model, sourceFile: `src/${name}.agent.ts` },
+        ])
+      ),
+    },
+  }) as unknown as Omit<InspectorState, 'typesLookup'>
+
+describe('validateAgentModels (aliases resolve against pikku.config.json)', () => {
+  test('a provider-qualified model needs no alias', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'openai/gpt-5-mini' }))
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a missing model is still reported', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: '' }))
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.MISSING_MODEL)
+  })
+
+  test('a bare name that the models table defines is accepted', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'cheap' }), {
+      cheap: 'openai/gpt-5-mini',
+    })
+    assert.deepEqual(criticals, [])
+  })
+
+  test('a bare name with no alias is an error naming the ones that exist', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'exspensive' }), {
+      cheap: 'openai/gpt-5-mini',
+      expensive: 'openai/gpt-5',
+    })
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+    assert.match(criticals[0]!.message, /cheap, expensive/)
+  })
+
+  test('with no models table a bare name still fails, as it always has', () => {
+    const { logger, criticals } = makeCriticalLogger()
+    validateAgentModels(logger, makeModelState({ triage: 'gpt-5-mini' }))
+    assert.equal(criticals.length, 1)
+    assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+  })
+
+  for (const inherited of ['toString', 'constructor', '__proto__']) {
+    test(`a bare '${inherited}' is not an alias just because Object has one`, () => {
+      const { logger, criticals } = makeCriticalLogger()
+      validateAgentModels(logger, makeModelState({ triage: inherited }), {
+        cheap: 'openai/gpt-5-mini',
+      })
+      assert.equal(
+        criticals.length,
+        1,
+        'an inherited property name must not read as a configured alias'
+      )
+      assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
+    })
+  }
+})

package/src/utils/post-process.ts

@@ -15,7 +15,6 @@ import { existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { extractTypeKeys } from './type-utils.js'
 import { ErrorCode } from '../error-codes.js'
-import { isSecretBrokerFunction } from './secret-brokers.js'
 import { findSecretAliasServices } from './secret-alias-services.js'
 import { relative } from 'node:path'
 import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
@@ -746,9 +745,14 @@ export function validateSchemaReferences(
   })
 }
 
+/**
+ * A model is either concrete (`provider/model`) or an alias, which must appear
+ * in the `models` table from pikku.config.json.
+ */
 export function validateAgentModels(
   logger: InspectorLogger,
-  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>,
+  modelAliases: Record<string, string> = {}
 ): void {
   for (const [, meta] of Object.entries(state.agents.agentsMeta)) {
     const model = meta.model
@@ -759,10 +763,17 @@ export function validateAgentModels(
       )
       continue
     }
-    if (!model.includes('/')) {
+    // `hasOwn`, not a truthy lookup: every object literal inherits `toString`,
+    // `constructor` and `__proto__`, so a bare model named after one of them
+    // would read as a configured alias and pass a check nothing else repeats.
+    if (!model.includes('/') && !Object.hasOwn(modelAliases, model)) {
+      const known = Object.keys(modelAliases).sort()
       logger.critical(
         ErrorCode.INVALID_MODEL,
-        `AI agent '${meta.name}' uses model '${model}', which must be provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4').`
+        `AI agent '${meta.name}' uses model '${model}', which is neither provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4') nor an alias in the "models" table of pikku.config.json. ` +
+          (known.length
+            ? `Known aliases: ${known.join(', ')}.`
+            : `No "models" aliases are configured.`)
       )
     }
   }
@@ -982,15 +993,6 @@ export function validateSchemaWiringSeparation(
   }
 }
 
-/** Marks the functions that keep the full `SecretService`. */
-export function computeSecretBrokers(state: InspectorState): void {
-  for (const [id, meta] of Object.entries(state.functions.meta)) {
-    if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
-      meta.secretBroker = true
-    }
-  }
-}
-
 /**
  * Refuses a singleton service that is a `SecretService` under another name.
  *
@@ -1029,9 +1031,6 @@ export function validateNoSecretAliasServices(
   }
 
   for (const [id, meta] of Object.entries(state.functions.meta)) {
-    if (meta.secretBroker) {
-      continue
-    }
     report('Function', id, meta.services?.services ?? [])
   }
   for (const [id, def] of Object.entries(state.permissions.definitions)) {
@@ -1284,6 +1283,32 @@ export function validateAgentToolReferences(
   }
 }
 
+/**
+ * Validates that every scorer an agent asks to be graded by exists.
+ *
+ * The runtime warns and skips an unresolvable name rather than failing a run
+ * that has already answered the user — which means a typo would otherwise cost
+ * nothing visible and simply grade nothing, forever.
+ */
+export function validateAgentScorerReferences(
+  logger: InspectorLogger,
+  state: InspectorState | Omit<InspectorState, 'typesLookup'>
+): void {
+  const known = Object.keys(state.scorers.scorersMeta)
+
+  for (const [agentKey, agent] of Object.entries(state.agents.agentsMeta)) {
+    const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
+    for (const scorer of agent.scorers ?? []) {
+      if (known.includes(scorer)) continue
+      logger.critical(
+        ErrorCode.AGENT_SCORER_NOT_FOUND,
+        `AI agent '${agentKey}'${where} asks to be graded by scorer '${scorer}', which is not declared in this project. ` +
+          `Declared scorers: ${known.join(', ') || 'none'}.`
+      )
+    }
+  }
+}
+
 /**
  * Validates that every scope granted by a system role is declared via
  * `defineScope`. Runs after all visitors, so declaration order does not matter.

package/src/utils/serialize-inspector-state.ts

@@ -170,6 +170,8 @@ export interface SerializableInspectorState {
           secretOverrides?: Record<string, string>
           variableOverrides?: Record<string, string>
           credentialOverrides?: Record<string, string>
+          globalSecrets?: string
+          globalCredentials?: string
         },
       ]
     >
@@ -185,6 +187,10 @@ export interface SerializableInspectorState {
     agentsMeta: InspectorState['agents']['agentsMeta']
     files: [string, { path: string; exportedName: string }][]
   }
+  scorers: {
+    scorersMeta: InspectorState['scorers']['scorersMeta']
+    files: [string, { path: string; exportedName: string }][]
+  }
   cli: {
     meta: InspectorState['cli']['meta']
     files: string[]
@@ -409,6 +415,10 @@ export function serializeInspectorState(
       agentsMeta: state.agents?.agentsMeta ?? {},
       files: Array.from(state.agents?.files?.entries() ?? []),
     },
+    scorers: {
+      scorersMeta: state.scorers?.scorersMeta ?? {},
+      files: Array.from(state.scorers?.files?.entries() ?? []),
+    },
     cli: {
       meta: state.cli.meta,
       files: Array.from(state.cli.files),
@@ -613,6 +623,10 @@ export function deserializeInspectorState(
       agentsMeta: data.agents?.agentsMeta || {},
       files: new Map(data.agents?.files || []),
     },
+    scorers: {
+      scorersMeta: data.scorers?.scorersMeta || {},
+      files: new Map(data.scorers?.files || []),
+    },
     cli: {
       meta: data.cli.meta,
       files: new Set(data.cli.files),

package/src/utils/workflow/dsl/extract-approval-policy.test.ts

@@ -0,0 +1,68 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+import * as ts from 'typescript'
+import { extractDSLWorkflow } from './extract-dsl-workflow.js'
+import type { ApprovalStepMeta } from '@pikku/core/workflow'
+
+const checker = {
+  getSymbolAtLocation: () => undefined,
+  getAliasedSymbol: () => undefined,
+  getShorthandAssignmentValueSymbol: () => undefined,
+  getTypeAtLocation: () => undefined,
+} as unknown as ts.TypeChecker
+
+/** Extracts the single approval step from a workflow body. */
+const approvalStepFor = (options: string): ApprovalStepMeta => {
+  const file = ts.createSourceFile(
+    'wf.ts',
+    `const wf = pikkuWorkflowFunc({
+      func: async (services, data, { workflow }) => {
+        await workflow.approval('Release funds', ${options})
+      },
+    })`,
+    ts.ScriptTarget.Latest,
+    true
+  )
+
+  let node: ts.Node | undefined
+  const visit = (n: ts.Node) => {
+    if (ts.isCallExpression(n) && n.expression.getText() === 'pikkuWorkflowFunc') {
+      node = n
+    }
+    ts.forEachChild(n, visit)
+  }
+  visit(file)
+  assert.ok(node, 'failed to find the workflow definition')
+
+  const result = extractDSLWorkflow(node, checker)
+  assert.equal(result.status, 'ok', JSON.stringify(result))
+  const step = result.steps?.find(
+    (s): s is ApprovalStepMeta => s.type === 'approval'
+  )
+  assert.ok(step, 'no approval step was extracted')
+  return step
+}
+
+describe('an approval gate carries its policy into the graph', () => {
+  test('approvers and approverScope survive extraction', () => {
+    const step = approvalStepFor(
+      `{ schema: DecisionSchema, approvers: 'not-initiator', approverScope: 'payments:approve' }`
+    )
+    assert.equal(step.approvers, 'not-initiator')
+    assert.equal(step.approverScope, 'payments:approve')
+  })
+
+  test('expiry still survives alongside them', () => {
+    const step = approvalStepFor(
+      `{ schema: DecisionSchema, expiry: '3d', approvers: 'owner' }`
+    )
+    assert.equal(step.expiry, '3d')
+    assert.equal(step.approvers, 'owner')
+  })
+
+  test('a gate declaring no policy records none', () => {
+    const step = approvalStepFor(`{ schema: DecisionSchema }`)
+    assert.equal(step.approvers, undefined)
+    assert.equal(step.approverScope, undefined)
+  })
+})

package/src/utils/workflow/dsl/extract-dsl-workflow.ts

@@ -1051,14 +1051,15 @@ function extractApprovalStep(
       step.outputVar = outputVar
     }
     // The `schema` option is a runtime value validated inside the workflow body,
-    // so it is deliberately not serialized here — only `expiry`, which the graph
-    // and planned-step ladder need in order to describe the gate.
+    // so it is deliberately not serialized here — only the literal options the
+    // graph and planned-step ladder need in order to describe the gate.
     const options = args[1]
     if (options && ts.isObjectLiteralExpression(options)) {
       for (const prop of options.properties) {
+        if (!ts.isPropertyAssignment(prop)) continue
+        const name = prop.name.getText()
         if (
-          ts.isPropertyAssignment(prop) &&
-          prop.name.getText() === 'expiry' &&
+          name === 'expiry' &&
           (ts.isStringLiteral(prop.initializer) ||
             ts.isNumericLiteral(prop.initializer))
         ) {
@@ -1066,6 +1067,13 @@ function extractApprovalStep(
             ? Number(prop.initializer.text)
             : prop.initializer.text
         }
+        if (name === 'approvers' && ts.isStringLiteral(prop.initializer)) {
+          step.approvers = prop.initializer
+            .text as ApprovalStepMeta['approvers']
+        }
+        if (name === 'approverScope' && ts.isStringLiteral(prop.initializer)) {
+          step.approverScope = prop.initializer.text
+        }
       }
     }
     return step

package/src/utils/workflow/dsl/patterns.ts

@@ -77,20 +77,23 @@ export function getScenarioStepPhase(
 /**
  * The expectation helpers that assert without being a `then`.
  *
- * `expectService` and `expectError` are inline steps rather than scenario
- * steps, so they carry no phase and never reach the step meta — but each one is
- * a witness: a recorded service call, or a refusal. PKU680 has to see them, or
- * a scenario whose only assertion is one of these reads as asserting nothing.
+ * `expectService`, `expectError` and `expectScore` are inline steps rather than
+ * scenario steps, so they carry no phase and never reach the step meta — but
+ * each one is a witness: a recorded service call, a refusal, or a graded run.
+ * PKU680 has to see them, or a scenario whose only assertion is one of these
+ * reads as asserting nothing.
  */
 const SCENARIO_EXPECTATION_CALLS = [
   'expectService',
   'expectError',
   'expectEventually',
+  'expectScore',
 ] as const
 
 /**
  * Check if a call expression is one of the scenario expectation helpers —
- * `scenario.expectService()`, `.expectError()` or `.expectEventually()`.
+ * `scenario.expectService()`, `.expectError()`, `.expectEventually()` or
+ * `.expectScore()`.
  */
 export function isScenarioExpectationCall(node: ts.CallExpression): boolean {
   return (

package/src/visit.ts

@@ -34,6 +34,7 @@ import { addVariable } from './add/add-variable.js'
 import { addWorkflowGraph } from './add/add-workflow-graph.js'
 import { addFeature } from './add/add-feature.js'
 import { addAIAgent } from './add/add-ai-agent.js'
+import { addAIScorer } from './add/add-ai-scorer.js'
 import { addApprovalDescription } from './add/add-approval-description.js'
 
 export const visitSetup = (
@@ -172,6 +173,7 @@ export const visitRoutes = (
   addWorkflowGraph(logger, node, checker, state, nextOptions)
   addFeature(logger, node, checker, state, nextOptions)
   addAIAgent(logger, node, checker, state, nextOptions)
+  addAIScorer(logger, node, checker, state, nextOptions)
 
   ts.forEachChild(node, (child) =>
     visitRoutes(logger, checker, child, state, nextOptions)