@pikku/inspector 0.12.55 → 0.12.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +168 -0
  2. package/dist/add/add-ai-agent.js +25 -0
  3. package/dist/add/add-ai-scorer.d.ts +2 -0
  4. package/dist/add/add-ai-scorer.js +61 -0
  5. package/dist/add/add-wire-addon.js +16 -0
  6. package/dist/add/scenario-instrumentation.d.ts +6 -1
  7. package/dist/add/scenario-instrumentation.js +6 -0
  8. package/dist/error-codes.d.ts +2 -0
  9. package/dist/error-codes.js +3 -0
  10. package/dist/inspector.js +13 -4
  11. package/dist/types.d.ts +20 -0
  12. package/dist/utils/filter-inspector-state.js +8 -0
  13. package/dist/utils/post-process.d.ts +28 -3
  14. package/dist/utils/post-process.js +85 -15
  15. package/dist/utils/serialize-inspector-state.d.ts +9 -0
  16. package/dist/utils/serialize-inspector-state.js +8 -0
  17. package/dist/utils/workflow/dsl/extract-dsl-workflow.js +13 -4
  18. package/dist/utils/workflow/dsl/patterns.d.ts +2 -1
  19. package/dist/utils/workflow/dsl/patterns.js +8 -5
  20. package/dist/utils/workflow/graph/serialize-workflow-graph.js +2 -10
  21. package/dist/visit.js +2 -0
  22. package/package.json +3 -3
  23. package/src/add/add-ai-agent.ts +35 -0
  24. package/src/add/add-ai-scorer.test.ts +197 -0
  25. package/src/add/add-ai-scorer.ts +83 -0
  26. package/src/add/add-scenario-step.test.ts +19 -0
  27. package/src/add/add-wire-addon.test.ts +43 -0
  28. package/src/add/add-wire-addon.ts +14 -0
  29. package/src/add/scenario-instrumentation.ts +6 -0
  30. package/src/error-codes.ts +4 -0
  31. package/src/inspector.ts +15 -4
  32. package/src/types.ts +17 -0
  33. package/src/utils/filter-inspector-state.ts +8 -0
  34. package/src/utils/post-process.test.ts +71 -0
  35. package/src/utils/post-process.ts +98 -16
  36. package/src/utils/serialize-inspector-state.ts +14 -0
  37. package/src/utils/workflow/dsl/extract-approval-policy.test.ts +68 -0
  38. package/src/utils/workflow/dsl/extract-dsl-workflow.ts +12 -4
  39. package/src/utils/workflow/dsl/patterns.ts +8 -5
  40. package/src/utils/workflow/graph/serialize-workflow-graph.ts +2 -15
  41. package/src/visit.ts +2 -0
  42. package/tsconfig.tsbuildinfo +1 -1
  43. package/src/utils/secret-brokers.test.ts +0 -53
  44. package/src/utils/secret-brokers.ts +0 -11
@@ -243,6 +243,25 @@ describe('pikkuScenarioStep', () => {
243
243
  }
244
244
  })
245
245
 
246
+ test('a graded agent run is a witness too', async () => {
247
+ // An agent's answer cannot be matched against a fixed string, so a scenario
248
+ // that grades one is asserting — even though `expectScore`, like the other
249
+ // helpers, is an inline step that reaches PKU680 as nothing at all.
250
+ const { criticals, cleanup } = await run([
251
+ "const answer = await scenario.when('asks for a summary', 'runAssistant', { prompt: 'hi' }, { actor: actors.shopper })",
252
+ "await scenario.expectScore('answered briefly', answer.runId, 'brevity', { atLeast: 0.8 })",
253
+ ])
254
+ try {
255
+ assert.equal(
256
+ criticals.filter((c) => c.code === 'PKU680').length,
257
+ 0,
258
+ `got: ${JSON.stringify(criticals)}`
259
+ )
260
+ } finally {
261
+ await cleanup()
262
+ }
263
+ })
264
+
246
265
  test('an expectation parked in an uncalled helper is still no assertion', async () => {
247
266
  const { criticals, cleanup } = await run([
248
267
  'const neverCalled = async () => {',
@@ -61,9 +61,52 @@ describe('addWireAddon', () => {
61
61
  secretOverrides: undefined,
62
62
  variableOverrides: undefined,
63
63
  credentialOverrides: undefined,
64
+ globalSecrets: undefined,
65
+ globalCredentials: undefined,
64
66
  })
65
67
  })
66
68
 
69
+ test('captures the reason an addon was exempted from credential scoping', () => {
70
+ const declarations = inspect(`
71
+ wireAddon({
72
+ name: 'console',
73
+ package: '@pikku/addon-console',
74
+ globalCredentials: 'links credentials an operator names at runtime',
75
+ })
76
+ `)
77
+
78
+ assert.equal(
79
+ declarations.get('console').globalCredentials,
80
+ 'links credentials an operator names at runtime'
81
+ )
82
+ })
83
+
84
+ test('captures the reason an addon was exempted from secret scoping', () => {
85
+ const declarations = inspect(`
86
+ wireAddon({
87
+ name: 'console',
88
+ package: '@pikku/addon-console',
89
+ globalSecrets: 'administers secrets an operator names at runtime',
90
+ })
91
+ `)
92
+
93
+ assert.equal(
94
+ declarations.get('console').globalSecrets,
95
+ 'administers secrets an operator names at runtime'
96
+ )
97
+ })
98
+
99
+ test('a globalSecrets reason that is not a literal still reports the grant', () => {
100
+ // Unlike `scopes`, dropping this would under-report: the grant is real at
101
+ // runtime whatever the reason evaluates to. The source text is recorded so
102
+ // the manifest names the addon and points at where the reason comes from.
103
+ const declarations = inspect(`
104
+ wireAddon({ name: 'console', package: '@x/y', globalSecrets: REASON })
105
+ `)
106
+
107
+ assert.equal(declarations.get('console').globalSecrets, 'REASON')
108
+ })
109
+
67
110
  test('leaves the gates undefined when none are declared', () => {
68
111
  const declarations = inspect(`
69
112
  wireAddon({ name: 'console', package: '@pikku/addon-console' })
@@ -60,6 +60,8 @@ export function addWireAddon(
60
60
  let secretOverrides: Record<string, string> | undefined
61
61
  let variableOverrides: Record<string, string> | undefined
62
62
  let credentialOverrides: Record<string, string> | undefined
63
+ let globalSecrets: string | undefined
64
+ let globalCredentials: string | undefined
63
65
 
64
66
  for (const prop of firstArg.properties) {
65
67
  if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue
@@ -102,6 +104,16 @@ export function addWireAddon(
102
104
  ts.isObjectLiteralExpression(prop.initializer)
103
105
  ) {
104
106
  credentialOverrides = parseStringRecord(prop.initializer)
107
+ } else if (key === 'globalSecrets') {
108
+ // The grant is real at runtime whatever the reason evaluates to, so a
109
+ // non-literal is recorded as its source text rather than dropped.
110
+ globalSecrets = ts.isStringLiteral(prop.initializer)
111
+ ? prop.initializer.text
112
+ : prop.initializer.getText()
113
+ } else if (key === 'globalCredentials') {
114
+ globalCredentials = ts.isStringLiteral(prop.initializer)
115
+ ? prop.initializer.text
116
+ : prop.initializer.getText()
105
117
  }
106
118
  }
107
119
 
@@ -118,6 +130,8 @@ export function addWireAddon(
118
130
  secretOverrides,
119
131
  variableOverrides,
120
132
  credentialOverrides,
133
+ globalSecrets,
134
+ globalCredentials,
121
135
  })
122
136
  state.rpc.usedAddons.add(name)
123
137
  state.rpc.wireAddonFiles.add(node.getSourceFile().fileName)
@@ -13,12 +13,18 @@
13
13
  * `pikku dev` registers the implementations itself. Ignoring by name (rather
14
14
  * than deleting the scaffold and requiring every project to regenerate) is what
15
15
  * keeps an existing project's checked-in `scenarios.gen.ts` from deploying.
16
+ *
17
+ * `pikkuScenarioGradeRun` was never scaffolded — it arrived after the split —
18
+ * but it belongs here for the same reason: grading a run on demand, ignoring the
19
+ * sample rate that governs live traffic, is a thing you do to a development
20
+ * server, and the name is reserved so an application cannot claim it.
16
21
  */
17
22
  export const SCENARIO_INSTRUMENTATION_FUNCTIONS = [
18
23
  'pikkuScenarioTakeLiveCoverage',
19
24
  'pikkuScenarioResetLiveCoverage',
20
25
  'pikkuScenarioResetStubs',
21
26
  'pikkuScenarioGetStubCalls',
27
+ 'pikkuScenarioGradeRun',
22
28
  ] as const
23
29
 
24
30
  export type ScenarioInstrumentationFunction =
@@ -45,6 +45,7 @@ export enum ErrorCode {
45
45
 
46
46
  // Configuration errors
47
47
  SCHEMA_GENERATION_ERROR = 'PKU456',
48
+ SCHEMA_REFERENCE_UNRESOLVED = 'PKU463',
48
49
  INLINE_SCHEMA = 'PKU489',
49
50
 
50
51
  // Function errors
@@ -90,6 +91,9 @@ export enum ErrorCode {
90
91
  AGENT_TOOL_NOT_FOUND = 'PKU153',
91
92
  AGENT_TOOL_MISSING_DESCRIPTION = 'PKU154',
92
93
 
94
+ // Agent scorer reference errors
95
+ AGENT_SCORER_NOT_FOUND = 'PKU155',
96
+
93
97
  // File structure errors
94
98
  SCHEMA_AND_WIRING_COLOCATED = 'PKU490',
95
99
 
package/src/inspector.ts CHANGED
@@ -16,6 +16,7 @@ import {
16
16
  stampAuthHandlerServices,
17
17
  validateAgentModels,
18
18
  validateAgentToolReferences,
19
+ validateAgentScorerReferences,
19
20
  validateSecretOverrides,
20
21
  validateVariableOverrides,
21
22
  validateCredentialOverrides,
@@ -28,10 +29,10 @@ import {
28
29
  computeMiddlewareGroupsMeta,
29
30
  computePermissionsGroupsMeta,
30
31
  computeRequiredSchemas,
31
- computeSecretBrokers,
32
32
  validateNoSecretAliasServices,
33
33
  validateSecretUsage,
34
34
  computeDiagnostics,
35
+ validateSchemaReferences,
35
36
  validateSchemaWiringSeparation,
36
37
  validateScenarioServices,
37
38
  validateScenarioSteps,
@@ -41,7 +42,7 @@ import { validateExposedFunctionsGated } from './utils/validate-exposed-function
41
42
  import { validateTagsResolveToMiddleware } from './utils/validate-tags-resolve-to-middleware.js'
42
43
  import { annotateHttpRouteAuth } from './utils/annotate-http-route-auth.js'
43
44
  import { generateOpenAPISpec } from './utils/serialize-openapi-json.js'
44
- import { pikkuState } from '@pikku/core/internal'
45
+ import { pikkuState } from '@pikku/core/ecosystem'
45
46
  import { resolveLatestVersions } from './utils/resolve-versions.js'
46
47
  import { finalizeWorkflows } from './utils/workflow/graph/finalize-workflows.js'
47
48
  import {
@@ -159,6 +160,10 @@ export function getInitialInspectorState(rootDir: string): InspectorState {
159
160
  agentsMeta: {},
160
161
  files: new Map(),
161
162
  },
163
+ scorers: {
164
+ scorersMeta: {},
165
+ files: new Map(),
166
+ },
162
167
  cli: {
163
168
  meta: {
164
169
  programs: {},
@@ -444,6 +449,12 @@ export const inspect = async (
444
449
  // Re-load addon schemas (generateAllSchemas replaces state.schemas)
445
450
  await loadAddonSchemas(logger, state)
446
451
 
452
+ // After the addon merge, not before: an addon supplies its own schemas here,
453
+ // and checking earlier would report every one of them as unresolved.
454
+ if (options.schemaConfig) {
455
+ validateSchemaReferences(logger, state)
456
+ }
457
+
447
458
  state.manifest.initial = options.manifest ?? null
448
459
  const contracts = extractContractsFromMeta(state.functions.meta)
449
460
  const baseManifest = state.manifest.initial ?? createEmptyManifest()
@@ -477,7 +488,6 @@ export const inspect = async (
477
488
  computeResolvedIOTypes(state)
478
489
  computeMiddlewareGroupsMeta(state)
479
490
  computePermissionsGroupsMeta(state)
480
- computeSecretBrokers(state)
481
491
  validateNoSecretAliasServices(logger, checker, state)
482
492
  validateSecretUsage(logger, state)
483
493
  computeDiagnostics(state)
@@ -504,8 +514,9 @@ export const inspect = async (
504
514
  )
505
515
  }
506
516
 
507
- validateAgentModels(logger, state)
517
+ validateAgentModels(logger, state, options.modelAliases)
508
518
  validateAgentToolReferences(logger, state, options)
519
+ validateAgentScorerReferences(logger, state)
509
520
  validateSecretOverrides(logger, state)
510
521
  validateVariableOverrides(logger, state)
511
522
  validateCredentialOverrides(logger, state)
package/src/types.ts CHANGED
@@ -13,6 +13,7 @@ import type {
13
13
  MCPPromptMeta,
14
14
  } from '@pikku/core/mcp'
15
15
  import type { AIAgentMeta } from '@pikku/core/ai-agent'
16
+ import type { ScorerMeta } from '@pikku/core/ai-scorer'
16
17
  import type { CLIMeta } from '@pikku/core/cli'
17
18
  import type { CLICommandMeta } from '@pikku/core/cli'
18
19
  import type { NodesMeta } from '@pikku/core/node'
@@ -298,6 +299,8 @@ export type InspectorOptions = Partial<{
298
299
  * tool's own name, so this is a quality gate rather than a correctness one.
299
300
  */
300
301
  strictMeta: boolean
302
+ /** The `models` table from pikku.config.json, alias -> `provider/model`. */
303
+ modelAliases: Record<string, string>
301
304
  sourceFile: ts.SourceFile
302
305
  /**
303
306
  * The project's tsconfig, read for its path mappings only. The inspector's
@@ -585,6 +588,16 @@ export interface InspectorState {
585
588
  secretOverrides?: Record<string, string>
586
589
  variableOverrides?: Record<string, string>
587
590
  credentialOverrides?: Record<string, string>
591
+ /**
592
+ * The app's stated reason for handing this instance the whole
593
+ * `SecretService` instead of one scoped to its declared secrets.
594
+ */
595
+ globalSecrets?: string
596
+ /**
597
+ * The app's stated reason for handing this instance the whole
598
+ * `CredentialService` instead of one scoped to its declared credentials.
599
+ */
600
+ globalCredentials?: string
588
601
  }
589
602
  >
590
603
  wireAddonFiles: Set<string>
@@ -599,6 +612,10 @@ export interface InspectorState {
599
612
  agentsMeta: AIAgentMeta
600
613
  files: Map<string, { path: string; exportedName: string }>
601
614
  }
615
+ scorers: {
616
+ scorersMeta: ScorerMeta
617
+ files: Map<string, { path: string; exportedName: string }>
618
+ }
602
619
  cli: {
603
620
  meta: CLIMeta
604
621
  files: Set<string>
@@ -419,6 +419,14 @@ export function filterInspectorState(
419
419
  agentsMeta: JSON.parse(JSON.stringify(state.agents?.agentsMeta ?? {})),
420
420
  files: new Map(),
421
421
  },
422
+ // Scorers carry through every filter. They are named by the agents that
423
+ // survive it, and a deployment that quietly stopped grading would look
424
+ // exactly like one whose scorers all returned nothing.
425
+ scorers: {
426
+ ...state.scorers,
427
+ scorersMeta: JSON.parse(JSON.stringify(state.scorers?.scorersMeta ?? {})),
428
+ files: new Map(state.scorers?.files ?? []),
429
+ },
422
430
  rpc: {
423
431
  ...state.rpc,
424
432
  internalMeta: { ...state.rpc.internalMeta }, // Clone to avoid mutating original
@@ -11,6 +11,7 @@ import {
11
11
  validateRemoteAddonDependencies,
12
12
  validateRemoteAddonAuth,
13
13
  validateAgentToolReferences,
14
+ validateAgentModels,
14
15
  } from './post-process.js'
15
16
  import { ErrorCode } from '../error-codes.js'
16
17
  import type { InspectorState, InspectorLogger } from '../types.js'
@@ -713,3 +714,73 @@ describe('validateAgentToolReferences (ref() resolved at build time)', () => {
713
714
  assert.deepEqual(criticals, [])
714
715
  })
715
716
  })
717
+
718
+ const makeModelState = (
719
+ models: Record<string, string>
720
+ ): Omit<InspectorState, 'typesLookup'> =>
721
+ ({
722
+ agents: {
723
+ agentsMeta: Object.fromEntries(
724
+ Object.entries(models).map(([name, model]) => [
725
+ name,
726
+ { name, model, sourceFile: `src/${name}.agent.ts` },
727
+ ])
728
+ ),
729
+ },
730
+ }) as unknown as Omit<InspectorState, 'typesLookup'>
731
+
732
+ describe('validateAgentModels (aliases resolve against pikku.config.json)', () => {
733
+ test('a provider-qualified model needs no alias', () => {
734
+ const { logger, criticals } = makeCriticalLogger()
735
+ validateAgentModels(logger, makeModelState({ triage: 'openai/gpt-5-mini' }))
736
+ assert.deepEqual(criticals, [])
737
+ })
738
+
739
+ test('a missing model is still reported', () => {
740
+ const { logger, criticals } = makeCriticalLogger()
741
+ validateAgentModels(logger, makeModelState({ triage: '' }))
742
+ assert.equal(criticals.length, 1)
743
+ assert.equal(criticals[0]!.code, ErrorCode.MISSING_MODEL)
744
+ })
745
+
746
+ test('a bare name that the models table defines is accepted', () => {
747
+ const { logger, criticals } = makeCriticalLogger()
748
+ validateAgentModels(logger, makeModelState({ triage: 'cheap' }), {
749
+ cheap: 'openai/gpt-5-mini',
750
+ })
751
+ assert.deepEqual(criticals, [])
752
+ })
753
+
754
+ test('a bare name with no alias is an error naming the ones that exist', () => {
755
+ const { logger, criticals } = makeCriticalLogger()
756
+ validateAgentModels(logger, makeModelState({ triage: 'exspensive' }), {
757
+ cheap: 'openai/gpt-5-mini',
758
+ expensive: 'openai/gpt-5',
759
+ })
760
+ assert.equal(criticals.length, 1)
761
+ assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
762
+ assert.match(criticals[0]!.message, /cheap, expensive/)
763
+ })
764
+
765
+ test('with no models table a bare name still fails, as it always has', () => {
766
+ const { logger, criticals } = makeCriticalLogger()
767
+ validateAgentModels(logger, makeModelState({ triage: 'gpt-5-mini' }))
768
+ assert.equal(criticals.length, 1)
769
+ assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
770
+ })
771
+
772
+ for (const inherited of ['toString', 'constructor', '__proto__']) {
773
+ test(`a bare '${inherited}' is not an alias just because Object has one`, () => {
774
+ const { logger, criticals } = makeCriticalLogger()
775
+ validateAgentModels(logger, makeModelState({ triage: inherited }), {
776
+ cheap: 'openai/gpt-5-mini',
777
+ })
778
+ assert.equal(
779
+ criticals.length,
780
+ 1,
781
+ 'an inherited property name must not read as a configured alias'
782
+ )
783
+ assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
784
+ })
785
+ }
786
+ })
@@ -15,7 +15,6 @@ import { existsSync, readFileSync } from 'node:fs'
15
15
  import { join } from 'node:path'
16
16
  import { extractTypeKeys } from './type-utils.js'
17
17
  import { ErrorCode } from '../error-codes.js'
18
- import { isSecretBrokerFunction } from './secret-brokers.js'
19
18
  import { findSecretAliasServices } from './secret-alias-services.js'
20
19
  import { relative } from 'node:path'
21
20
  import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
@@ -689,9 +688,71 @@ export function computeRequiredSchemas(
689
688
  ])
690
689
  }
691
690
 
691
+ /**
692
+ * Every schema a function's contract names has to exist once codegen is done.
693
+ *
694
+ * The way it stops existing is quiet. `generateCustomTypes` hands the schema
695
+ * generator a virtual source file that *imports* each named contract type, so a
696
+ * type that is declared but never exported resolves to nothing and yields no
697
+ * schema — while the function's meta still carries the name. Nothing downstream
698
+ * reads that name until the first call, which fails with a runtime
699
+ * `MissingSchemaError` in a deployment rather than in the build that caused it.
700
+ *
701
+ * Reported rather than critical: the reference is only *guaranteed* to break
702
+ * the functions that are actually invoked, so this blocks CI through
703
+ * `--fail-on-error` without failing a dev server over a function nobody calls.
704
+ */
705
+ export function validateSchemaReferences(
706
+ logger: InspectorLogger,
707
+ state: InspectorState
708
+ ): void {
709
+ const { functions, schemas } = state
710
+ const available = new Set(Object.keys(schemas ?? {}))
711
+ const unresolved: string[] = []
712
+
713
+ const resolve = (typeName: string): string => {
714
+ try {
715
+ return functions.typesMap.getUniqueName(typeName)
716
+ } catch {
717
+ return typeName
718
+ }
719
+ }
720
+
721
+ for (const [funcName, meta] of Object.entries(functions.meta)) {
722
+ for (const [role, declared] of [
723
+ ['input', meta.inputs?.[0]],
724
+ ['output', meta.outputs?.[0]],
725
+ ] as const) {
726
+ if (!declared || PRIMITIVE_TYPES.has(declared)) continue
727
+ const resolved = resolve(declared)
728
+ if (PRIMITIVE_TYPES.has(resolved) || available.has(resolved)) continue
729
+ unresolved.push(`${funcName}.${role} → '${resolved}'`)
730
+ }
731
+ }
732
+
733
+ if (unresolved.length === 0) return
734
+
735
+ logger.diagnostic({
736
+ severity: 'error',
737
+ code: ErrorCode.SCHEMA_REFERENCE_UNRESOLVED,
738
+ message:
739
+ `${unresolved.length} function contract${unresolved.length === 1 ? '' : 's'} ` +
740
+ `name${unresolved.length === 1 ? 's' : ''} a schema that was never generated, so ` +
741
+ `calling ${unresolved.length === 1 ? 'it' : 'them'} fails at runtime with ` +
742
+ `MissingSchemaError: ${unresolved.join(', ')}. ` +
743
+ `The usual cause is a named type used as the function's input/output that ` +
744
+ `is declared but not exported — export it, or inline the type at the call site.`,
745
+ })
746
+ }
747
+
748
+ /**
749
+ * A model is either concrete (`provider/model`) or an alias, which must appear
750
+ * in the `models` table from pikku.config.json.
751
+ */
692
752
  export function validateAgentModels(
693
753
  logger: InspectorLogger,
694
- state: InspectorState | Omit<InspectorState, 'typesLookup'>
754
+ state: InspectorState | Omit<InspectorState, 'typesLookup'>,
755
+ modelAliases: Record<string, string> = {}
695
756
  ): void {
696
757
  for (const [, meta] of Object.entries(state.agents.agentsMeta)) {
697
758
  const model = meta.model
@@ -702,10 +763,17 @@ export function validateAgentModels(
702
763
  )
703
764
  continue
704
765
  }
705
- if (!model.includes('/')) {
766
+ // `hasOwn`, not a truthy lookup: every object literal inherits `toString`,
767
+ // `constructor` and `__proto__`, so a bare model named after one of them
768
+ // would read as a configured alias and pass a check nothing else repeats.
769
+ if (!model.includes('/') && !Object.hasOwn(modelAliases, model)) {
770
+ const known = Object.keys(modelAliases).sort()
706
771
  logger.critical(
707
772
  ErrorCode.INVALID_MODEL,
708
- `AI agent '${meta.name}' uses model '${model}', which must be provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4').`
773
+ `AI agent '${meta.name}' uses model '${model}', which is neither provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4') nor an alias in the "models" table of pikku.config.json. ` +
774
+ (known.length
775
+ ? `Known aliases: ${known.join(', ')}.`
776
+ : `No "models" aliases are configured.`)
709
777
  )
710
778
  }
711
779
  }
@@ -925,15 +993,6 @@ export function validateSchemaWiringSeparation(
925
993
  }
926
994
  }
927
995
 
928
- /** Marks the functions that keep the full `SecretService`. */
929
- export function computeSecretBrokers(state: InspectorState): void {
930
- for (const [id, meta] of Object.entries(state.functions.meta)) {
931
- if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
932
- meta.secretBroker = true
933
- }
934
- }
935
- }
936
-
937
996
  /**
938
997
  * Refuses a singleton service that is a `SecretService` under another name.
939
998
  *
@@ -972,9 +1031,6 @@ export function validateNoSecretAliasServices(
972
1031
  }
973
1032
 
974
1033
  for (const [id, meta] of Object.entries(state.functions.meta)) {
975
- if (meta.secretBroker) {
976
- continue
977
- }
978
1034
  report('Function', id, meta.services?.services ?? [])
979
1035
  }
980
1036
  for (const [id, def] of Object.entries(state.permissions.definitions)) {
@@ -1227,6 +1283,32 @@ export function validateAgentToolReferences(
1227
1283
  }
1228
1284
  }
1229
1285
 
1286
+ /**
1287
+ * Validates that every scorer an agent asks to be graded by exists.
1288
+ *
1289
+ * The runtime warns and skips an unresolvable name rather than failing a run
1290
+ * that has already answered the user — which means a typo would otherwise cost
1291
+ * nothing visible and simply grade nothing, forever.
1292
+ */
1293
+ export function validateAgentScorerReferences(
1294
+ logger: InspectorLogger,
1295
+ state: InspectorState | Omit<InspectorState, 'typesLookup'>
1296
+ ): void {
1297
+ const known = Object.keys(state.scorers.scorersMeta)
1298
+
1299
+ for (const [agentKey, agent] of Object.entries(state.agents.agentsMeta)) {
1300
+ const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
1301
+ for (const scorer of agent.scorers ?? []) {
1302
+ if (known.includes(scorer)) continue
1303
+ logger.critical(
1304
+ ErrorCode.AGENT_SCORER_NOT_FOUND,
1305
+ `AI agent '${agentKey}'${where} asks to be graded by scorer '${scorer}', which is not declared in this project. ` +
1306
+ `Declared scorers: ${known.join(', ') || 'none'}.`
1307
+ )
1308
+ }
1309
+ }
1310
+ }
1311
+
1230
1312
  /**
1231
1313
  * Validates that every scope granted by a system role is declared via
1232
1314
  * `defineScope`. Runs after all visitors, so declaration order does not matter.
@@ -170,6 +170,8 @@ export interface SerializableInspectorState {
170
170
  secretOverrides?: Record<string, string>
171
171
  variableOverrides?: Record<string, string>
172
172
  credentialOverrides?: Record<string, string>
173
+ globalSecrets?: string
174
+ globalCredentials?: string
173
175
  },
174
176
  ]
175
177
  >
@@ -185,6 +187,10 @@ export interface SerializableInspectorState {
185
187
  agentsMeta: InspectorState['agents']['agentsMeta']
186
188
  files: [string, { path: string; exportedName: string }][]
187
189
  }
190
+ scorers: {
191
+ scorersMeta: InspectorState['scorers']['scorersMeta']
192
+ files: [string, { path: string; exportedName: string }][]
193
+ }
188
194
  cli: {
189
195
  meta: InspectorState['cli']['meta']
190
196
  files: string[]
@@ -409,6 +415,10 @@ export function serializeInspectorState(
409
415
  agentsMeta: state.agents?.agentsMeta ?? {},
410
416
  files: Array.from(state.agents?.files?.entries() ?? []),
411
417
  },
418
+ scorers: {
419
+ scorersMeta: state.scorers?.scorersMeta ?? {},
420
+ files: Array.from(state.scorers?.files?.entries() ?? []),
421
+ },
412
422
  cli: {
413
423
  meta: state.cli.meta,
414
424
  files: Array.from(state.cli.files),
@@ -613,6 +623,10 @@ export function deserializeInspectorState(
613
623
  agentsMeta: data.agents?.agentsMeta || {},
614
624
  files: new Map(data.agents?.files || []),
615
625
  },
626
+ scorers: {
627
+ scorersMeta: data.scorers?.scorersMeta || {},
628
+ files: new Map(data.scorers?.files || []),
629
+ },
616
630
  cli: {
617
631
  meta: data.cli.meta,
618
632
  files: new Set(data.cli.files),
@@ -0,0 +1,68 @@
1
+ import { describe, test } from 'node:test'
2
+ import assert from 'node:assert/strict'
3
+ import * as ts from 'typescript'
4
+ import { extractDSLWorkflow } from './extract-dsl-workflow.js'
5
+ import type { ApprovalStepMeta } from '@pikku/core/workflow'
6
+
7
+ const checker = {
8
+ getSymbolAtLocation: () => undefined,
9
+ getAliasedSymbol: () => undefined,
10
+ getShorthandAssignmentValueSymbol: () => undefined,
11
+ getTypeAtLocation: () => undefined,
12
+ } as unknown as ts.TypeChecker
13
+
14
+ /** Extracts the single approval step from a workflow body. */
15
+ const approvalStepFor = (options: string): ApprovalStepMeta => {
16
+ const file = ts.createSourceFile(
17
+ 'wf.ts',
18
+ `const wf = pikkuWorkflowFunc({
19
+ func: async (services, data, { workflow }) => {
20
+ await workflow.approval('Release funds', ${options})
21
+ },
22
+ })`,
23
+ ts.ScriptTarget.Latest,
24
+ true
25
+ )
26
+
27
+ let node: ts.Node | undefined
28
+ const visit = (n: ts.Node) => {
29
+ if (ts.isCallExpression(n) && n.expression.getText() === 'pikkuWorkflowFunc') {
30
+ node = n
31
+ }
32
+ ts.forEachChild(n, visit)
33
+ }
34
+ visit(file)
35
+ assert.ok(node, 'failed to find the workflow definition')
36
+
37
+ const result = extractDSLWorkflow(node, checker)
38
+ assert.equal(result.status, 'ok', JSON.stringify(result))
39
+ const step = result.steps?.find(
40
+ (s): s is ApprovalStepMeta => s.type === 'approval'
41
+ )
42
+ assert.ok(step, 'no approval step was extracted')
43
+ return step
44
+ }
45
+
46
+ describe('an approval gate carries its policy into the graph', () => {
47
+ test('approvers and approverScope survive extraction', () => {
48
+ const step = approvalStepFor(
49
+ `{ schema: DecisionSchema, approvers: 'not-initiator', approverScope: 'payments:approve' }`
50
+ )
51
+ assert.equal(step.approvers, 'not-initiator')
52
+ assert.equal(step.approverScope, 'payments:approve')
53
+ })
54
+
55
+ test('expiry still survives alongside them', () => {
56
+ const step = approvalStepFor(
57
+ `{ schema: DecisionSchema, expiry: '3d', approvers: 'owner' }`
58
+ )
59
+ assert.equal(step.expiry, '3d')
60
+ assert.equal(step.approvers, 'owner')
61
+ })
62
+
63
+ test('a gate declaring no policy records none', () => {
64
+ const step = approvalStepFor(`{ schema: DecisionSchema }`)
65
+ assert.equal(step.approvers, undefined)
66
+ assert.equal(step.approverScope, undefined)
67
+ })
68
+ })
@@ -1051,14 +1051,15 @@ function extractApprovalStep(
1051
1051
  step.outputVar = outputVar
1052
1052
  }
1053
1053
  // The `schema` option is a runtime value validated inside the workflow body,
1054
- // so it is deliberately not serialized here — only `expiry`, which the graph
1055
- // and planned-step ladder need in order to describe the gate.
1054
+ // so it is deliberately not serialized here — only the literal options the
1055
+ // graph and planned-step ladder need in order to describe the gate.
1056
1056
  const options = args[1]
1057
1057
  if (options && ts.isObjectLiteralExpression(options)) {
1058
1058
  for (const prop of options.properties) {
1059
+ if (!ts.isPropertyAssignment(prop)) continue
1060
+ const name = prop.name.getText()
1059
1061
  if (
1060
- ts.isPropertyAssignment(prop) &&
1061
- prop.name.getText() === 'expiry' &&
1062
+ name === 'expiry' &&
1062
1063
  (ts.isStringLiteral(prop.initializer) ||
1063
1064
  ts.isNumericLiteral(prop.initializer))
1064
1065
  ) {
@@ -1066,6 +1067,13 @@ function extractApprovalStep(
1066
1067
  ? Number(prop.initializer.text)
1067
1068
  : prop.initializer.text
1068
1069
  }
1070
+ if (name === 'approvers' && ts.isStringLiteral(prop.initializer)) {
1071
+ step.approvers = prop.initializer
1072
+ .text as ApprovalStepMeta['approvers']
1073
+ }
1074
+ if (name === 'approverScope' && ts.isStringLiteral(prop.initializer)) {
1075
+ step.approverScope = prop.initializer.text
1076
+ }
1069
1077
  }
1070
1078
  }
1071
1079
  return step