@pikku/inspector 0.12.56 → 0.12.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +130 -0
  2. package/dist/add/add-ai-agent.js +25 -0
  3. package/dist/add/add-ai-scorer.d.ts +2 -0
  4. package/dist/add/add-ai-scorer.js +61 -0
  5. package/dist/add/add-wire-addon.js +16 -0
  6. package/dist/add/scenario-instrumentation.d.ts +6 -1
  7. package/dist/add/scenario-instrumentation.js +6 -0
  8. package/dist/error-codes.d.ts +1 -0
  9. package/dist/error-codes.js +2 -0
  10. package/dist/inspector.js +7 -3
  11. package/dist/types.d.ts +20 -0
  12. package/dist/utils/filter-inspector-state.js +8 -0
  13. package/dist/utils/post-process.d.ts +13 -3
  14. package/dist/utils/post-process.js +33 -15
  15. package/dist/utils/serialize-inspector-state.d.ts +9 -0
  16. package/dist/utils/serialize-inspector-state.js +8 -0
  17. package/dist/utils/workflow/dsl/extract-dsl-workflow.js +13 -4
  18. package/dist/utils/workflow/dsl/patterns.d.ts +2 -1
  19. package/dist/utils/workflow/dsl/patterns.js +8 -5
  20. package/dist/visit.js +2 -0
  21. package/package.json +3 -3
  22. package/src/add/add-ai-agent.ts +35 -0
  23. package/src/add/add-ai-scorer.test.ts +197 -0
  24. package/src/add/add-ai-scorer.ts +83 -0
  25. package/src/add/add-scenario-step.test.ts +19 -0
  26. package/src/add/add-wire-addon.test.ts +43 -0
  27. package/src/add/add-wire-addon.ts +14 -0
  28. package/src/add/scenario-instrumentation.ts +6 -0
  29. package/src/error-codes.ts +3 -0
  30. package/src/inspector.ts +7 -3
  31. package/src/types.ts +17 -0
  32. package/src/utils/filter-inspector-state.ts +8 -0
  33. package/src/utils/post-process.test.ts +71 -0
  34. package/src/utils/post-process.ts +41 -16
  35. package/src/utils/serialize-inspector-state.ts +14 -0
  36. package/src/utils/workflow/dsl/extract-approval-policy.test.ts +68 -0
  37. package/src/utils/workflow/dsl/extract-dsl-workflow.ts +12 -4
  38. package/src/utils/workflow/dsl/patterns.ts +8 -5
  39. package/src/visit.ts +2 -0
  40. package/tsconfig.tsbuildinfo +1 -1
  41. package/src/utils/secret-brokers.test.ts +0 -53
  42. package/src/utils/secret-brokers.ts +0 -11
@@ -57,19 +57,22 @@ export function getScenarioStepPhase(node) {
57
57
  /**
58
58
  * The expectation helpers that assert without being a `then`.
59
59
  *
60
- * `expectService` and `expectError` are inline steps rather than scenario
61
- * steps, so they carry no phase and never reach the step meta — but each one is
62
- * a witness: a recorded service call, or a refusal. PKU680 has to see them, or
63
- * a scenario whose only assertion is one of these reads as asserting nothing.
60
+ * `expectService`, `expectError` and `expectScore` are inline steps rather than
61
+ * scenario steps, so they carry no phase and never reach the step meta — but
62
+ * each one is a witness: a recorded service call, a refusal, or a graded run.
63
+ * PKU680 has to see them, or a scenario whose only assertion is one of these
64
+ * reads as asserting nothing.
64
65
  */
65
66
  const SCENARIO_EXPECTATION_CALLS = [
66
67
  'expectService',
67
68
  'expectError',
68
69
  'expectEventually',
70
+ 'expectScore',
69
71
  ];
70
72
  /**
71
73
  * Check if a call expression is one of the scenario expectation helpers —
72
- * `scenario.expectService()`, `.expectError()` or `.expectEventually()`.
74
+ * `scenario.expectService()`, `.expectError()`, `.expectEventually()` or
75
+ * `.expectScore()`.
73
76
  */
74
77
  export function isScenarioExpectationCall(node) {
75
78
  return (ts.isPropertyAccessExpression(node.expression) &&
package/dist/visit.js CHANGED
@@ -29,6 +29,7 @@ import { addVariable } from './add/add-variable.js';
29
29
  import { addWorkflowGraph } from './add/add-workflow-graph.js';
30
30
  import { addFeature } from './add/add-feature.js';
31
31
  import { addAIAgent } from './add/add-ai-agent.js';
32
+ import { addAIScorer } from './add/add-ai-scorer.js';
32
33
  import { addApprovalDescription } from './add/add-approval-description.js';
33
34
  export const visitSetup = (logger, checker, node, state, options) => {
34
35
  addFileExtendsCoreType(node, checker, state.singletonServicesTypeImportMap, 'CoreSingletonServices', state);
@@ -88,5 +89,6 @@ export const visitRoutes = (logger, checker, node, state, options) => {
88
89
  addWorkflowGraph(logger, node, checker, state, nextOptions);
89
90
  addFeature(logger, node, checker, state, nextOptions);
90
91
  addAIAgent(logger, node, checker, state, nextOptions);
92
+ addAIScorer(logger, node, checker, state, nextOptions);
91
93
  ts.forEachChild(node, (child) => visitRoutes(logger, checker, child, state, nextOptions));
92
94
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pikku/inspector",
3
- "version": "0.12.56",
3
+ "version": "0.12.58",
4
4
  "description": "TypeScript inspector for Pikku — discovers functions and wirings to drive code generation",
5
5
  "author": "yasser.fadl@gmail.com",
6
6
  "license": "BUSL-1.1",
@@ -45,10 +45,10 @@
45
45
  "zod-to-ts": "^2.1.0"
46
46
  },
47
47
  "peerDependencies": {
48
- "@pikku/core": "^0.12.80"
48
+ "@pikku/core": "^0.12.82"
49
49
  },
50
50
  "devDependencies": {
51
- "@pikku/core": "^0.12.80",
51
+ "@pikku/core": "^0.12.82",
52
52
  "@types/node": "^24.11.0"
53
53
  },
54
54
  "engines": {
@@ -330,6 +330,39 @@ function resolveIdentifierToAgentName(
330
330
  return null
331
331
  }
332
332
 
333
+ /**
334
+ * The scorers an agent asks to be graded by, which are plain names rather than
335
+ * references: a scorer is resolved out of the registry at grading time, not
336
+ * imported by the agent, so nothing here has a declaration to chase.
337
+ */
338
+ function resolveScorerNames(
339
+ obj: ts.ObjectLiteralExpression,
340
+ agentName: string,
341
+ logger: InspectorLogger
342
+ ): string[] | null {
343
+ const property = obj.properties.find(
344
+ (p) =>
345
+ ts.isPropertyAssignment(p) &&
346
+ ts.isIdentifier(p.name) &&
347
+ p.name.text === 'scorers'
348
+ )
349
+ if (!property || !ts.isPropertyAssignment(property)) return null
350
+ if (!ts.isArrayLiteralExpression(property.initializer)) return null
351
+
352
+ const names: string[] = []
353
+ for (const element of property.initializer.elements) {
354
+ if (!ts.isStringLiteral(element)) {
355
+ logger.critical(
356
+ ErrorCode.INVALID_VALUE,
357
+ `AI agent '${agentName}' lists a scorer that is not an inline string: \`${element.getText()}\`. A scorer is named, not imported, so the name has to be readable from source.`
358
+ )
359
+ continue
360
+ }
361
+ names.push(element.text)
362
+ }
363
+ return names.length > 0 ? names : null
364
+ }
365
+
333
366
  export const addAIAgent: AddWiring = (
334
367
  logger,
335
368
  node,
@@ -377,6 +410,7 @@ export const addAIAgent: AddWiring = (
377
410
  | number
378
411
  | null
379
412
  const toolChoiceValue = getPropertyValue(obj, 'toolChoice') as string | null
413
+ const scorersValue = resolveScorerNames(obj, nameValue || '', logger)
380
414
  const toolsValue = resolveToolReferences(
381
415
  obj,
382
416
  checker,
@@ -582,6 +616,7 @@ export const addAIAgent: AddWiring = (
582
616
  ...(toolsValue !== null && { tools: toolsValue }),
583
617
  ...(agentsValue !== null && { agents: agentsValue }),
584
618
  ...(workflowsValue !== null && { workflows: workflowsValue }),
619
+ ...(scorersValue !== null && { scorers: scorersValue }),
585
620
  tags,
586
621
  inputSchema,
587
622
  outputSchema,
@@ -0,0 +1,197 @@
1
+ import { strict as assert } from 'assert'
2
+ import { describe, test } from 'node:test'
3
+ import { mkdtemp, rm, writeFile } from 'node:fs/promises'
4
+ import { tmpdir } from 'node:os'
5
+ import { join } from 'node:path'
6
+ import { inspect } from '../inspector.js'
7
+ import { ErrorCode } from '../error-codes.js'
8
+ import type { InspectorLogger } from '../types.js'
9
+
10
+ const makeLogger = (criticals: Array<{ code: ErrorCode; message: string }>) =>
11
+ ({
12
+ debug: () => {},
13
+ info: () => {},
14
+ warn: () => {},
15
+ error: () => {},
16
+ diagnostic: ({ code, message }) => {
17
+ criticals.push({ code, message })
18
+ },
19
+ critical: (code: ErrorCode, message: string) => {
20
+ criticals.push({ code, message })
21
+ },
22
+ hasCriticalErrors: () => criticals.length > 0,
23
+ }) satisfies InspectorLogger
24
+
25
+ const inspectSources = async (sources: Record<string, string>) => {
26
+ const rootDir = await mkdtemp(join(tmpdir(), 'pikku-add-ai-scorer-'))
27
+ const files: Record<string, string> = {}
28
+ for (const [name, source] of Object.entries(sources)) {
29
+ files[name] = join(rootDir, name)
30
+ await writeFile(files[name]!, source)
31
+ }
32
+ const criticals: Array<{ code: ErrorCode; message: string }> = []
33
+ try {
34
+ const state = await inspect(makeLogger(criticals), Object.values(files), {
35
+ rootDir,
36
+ })
37
+ return { state, criticals }
38
+ } finally {
39
+ await rm(rootDir, { recursive: true, force: true })
40
+ }
41
+ }
42
+
43
+ const inspectSource = (source: string) =>
44
+ inspectSources({ 'grading.scorer.ts': source })
45
+
46
+ describe('addAIScorer', () => {
47
+ test('reads the lane off which constructor was called, not off a field', async () => {
48
+ const { state, criticals } = await inspectSource(
49
+ [
50
+ `export const brevity = pikkuAIScorer({`,
51
+ ` name: 'brevity',`,
52
+ ` description: 'Shorter answers score higher',`,
53
+ ` score: () => ({ score: 1 }),`,
54
+ `})`,
55
+ `export const helpfulness = pikkuAIJudge({`,
56
+ ` name: 'helpfulness',`,
57
+ ` description: 'Does the answer help',`,
58
+ ` model: 'claude-opus-5',`,
59
+ ` goal: 'Grade helpfulness.',`,
60
+ `})`,
61
+ ].join('\n')
62
+ )
63
+
64
+ assert.deepEqual(criticals, [])
65
+ assert.equal(state.scorers.scorersMeta['brevity']?.lane, 'fast')
66
+ assert.equal(state.scorers.scorersMeta['helpfulness']?.lane, 'slow')
67
+ })
68
+
69
+ test('defaults an unstated sample rate to grading every run', async () => {
70
+ const { state } = await inspectSource(
71
+ [
72
+ `export const brevity = pikkuAIScorer({`,
73
+ ` name: 'brevity',`,
74
+ ` description: 'Shorter answers score higher',`,
75
+ ` score: () => ({ score: 1 }),`,
76
+ `})`,
77
+ ].join('\n')
78
+ )
79
+
80
+ assert.equal(state.scorers.scorersMeta['brevity']?.sampleRate, 1)
81
+ assert.equal(state.scorers.scorersMeta['brevity']?.requiresReference, false)
82
+ })
83
+
84
+ test('carries the sample rate and the reference requirement into the meta', async () => {
85
+ const { state } = await inspectSource(
86
+ [
87
+ `export const correctness = pikkuAIJudge({`,
88
+ ` name: 'correctness',`,
89
+ ` description: 'Is the answer right',`,
90
+ ` model: 'claude-opus-5',`,
91
+ ` goal: 'Grade correctness.',`,
92
+ ` sampleRate: 0.1,`,
93
+ ` requiresReference: true,`,
94
+ `})`,
95
+ ].join('\n')
96
+ )
97
+
98
+ assert.equal(state.scorers.scorersMeta['correctness']?.sampleRate, 0.1)
99
+ assert.equal(
100
+ state.scorers.scorersMeta['correctness']?.requiresReference,
101
+ true
102
+ )
103
+ })
104
+
105
+ test('rejects a sample rate that is not a fraction', async () => {
106
+ const { state, criticals } = await inspectSource(
107
+ [
108
+ `export const brevity = pikkuAIScorer({`,
109
+ ` name: 'brevity',`,
110
+ ` description: 'Shorter answers score higher',`,
111
+ ` sampleRate: 50,`,
112
+ ` score: () => ({ score: 1 }),`,
113
+ `})`,
114
+ ].join('\n')
115
+ )
116
+
117
+ assert.equal(criticals[0]?.code, ErrorCode.INVALID_VALUE)
118
+ assert.match(criticals[0]!.message, /between 0 and 1/)
119
+ assert.deepEqual(state.scorers.scorersMeta, {})
120
+ })
121
+
122
+ test('a scorer with no description is refused, since the meta is what names it', async () => {
123
+ const { criticals } = await inspectSource(
124
+ [
125
+ `export const brevity = pikkuAIScorer({`,
126
+ ` name: 'brevity',`,
127
+ ` score: () => ({ score: 1 }),`,
128
+ `})`,
129
+ ].join('\n')
130
+ )
131
+
132
+ assert.equal(criticals[0]?.code, ErrorCode.MISSING_DESCRIPTION)
133
+ })
134
+
135
+ test('records where a scorer was declared, so codegen can import it back', async () => {
136
+ const { state } = await inspectSource(
137
+ [
138
+ `export const brevity = pikkuAIScorer({`,
139
+ ` name: 'brevity',`,
140
+ ` description: 'Shorter answers score higher',`,
141
+ ` score: () => ({ score: 1 }),`,
142
+ `})`,
143
+ ].join('\n')
144
+ )
145
+
146
+ assert.equal(state.scorers.files.get('brevity')?.exportedName, 'brevity')
147
+ assert.match(
148
+ state.scorers.files.get('brevity')?.path ?? '',
149
+ /grading\.scorer\.ts$/
150
+ )
151
+ })
152
+ })
153
+
154
+ describe('an agent asking to be graded', () => {
155
+ const scorerFile = [
156
+ `export const brevity = pikkuAIScorer({`,
157
+ ` name: 'brevity',`,
158
+ ` description: 'Shorter answers score higher',`,
159
+ ` score: () => ({ score: 1 }),`,
160
+ `})`,
161
+ ].join('\n')
162
+
163
+ const agent = (scorers: string) =>
164
+ [
165
+ `export const assistant = pikkuAIAgent({`,
166
+ ` name: 'assistant',`,
167
+ ` description: 'Answers questions',`,
168
+ ` goal: 'Answer the question.',`,
169
+ ` model: 'anthropic/claude-opus-5',`,
170
+ ` scorers: [${scorers}],`,
171
+ `})`,
172
+ ].join('\n')
173
+
174
+ test('carries the named scorers onto the agent meta', async () => {
175
+ const { state, criticals } = await inspectSources({
176
+ 'grading.scorer.ts': scorerFile,
177
+ 'assistant.agent.ts': agent(`'brevity'`),
178
+ })
179
+
180
+ assert.deepEqual(criticals, [])
181
+ assert.deepEqual(state.agents.agentsMeta['assistant']?.scorers, ['brevity'])
182
+ })
183
+
184
+ test('naming a scorer that does not exist fails the build rather than grading nothing', async () => {
185
+ const { criticals } = await inspectSources({
186
+ 'grading.scorer.ts': scorerFile,
187
+ 'assistant.agent.ts': agent(`'brevty'`),
188
+ })
189
+
190
+ const finding = criticals.find(
191
+ (critical) => critical.code === ErrorCode.AGENT_SCORER_NOT_FOUND
192
+ )
193
+ assert.ok(finding, `expected a scorer-not-found critical`)
194
+ assert.match(finding!.message, /brevty/)
195
+ assert.match(finding!.message, /Declared scorers: brevity/)
196
+ })
197
+ })
@@ -0,0 +1,83 @@
1
+ import * as ts from 'typescript'
2
+ import { getPropertyValue } from '../utils/get-property-value.js'
3
+ import type { AddWiring } from '../types.js'
4
+ import { extractFunctionName } from '../utils/extract-function-name.js'
5
+ import { ErrorCode } from '../error-codes.js'
6
+
7
+ /**
8
+ * `pikkuAIScorer` grades in code and `pikkuAIJudge` grades with a model, which
9
+ * is the whole difference between the two lanes — so the lane is read off which
10
+ * constructor was called rather than from a field an author could contradict.
11
+ */
12
+ const LANES: Record<string, 'fast' | 'slow'> = {
13
+ pikkuAIScorer: 'fast',
14
+ pikkuAIJudge: 'slow',
15
+ }
16
+
17
+ export const addAIScorer: AddWiring = (logger, node, checker, state) => {
18
+ if (!ts.isCallExpression(node)) return
19
+
20
+ const expression = node.expression
21
+ if (!ts.isIdentifier(expression)) return
22
+
23
+ const lane = LANES[expression.text]
24
+ if (!lane) return
25
+
26
+ const firstArg = node.arguments[0]
27
+ if (!firstArg || !ts.isObjectLiteralExpression(firstArg)) return
28
+
29
+ const obj = firstArg
30
+ const { exportedName } = extractFunctionName(node, checker, state.rootDir)
31
+
32
+ const nameValue = getPropertyValue(obj, 'name') as string | null
33
+ if (!nameValue) {
34
+ logger.critical(
35
+ ErrorCode.MISSING_NAME,
36
+ "AI scorer is missing the required 'name' property."
37
+ )
38
+ return
39
+ }
40
+
41
+ const description = getPropertyValue(obj, 'description') as string | null
42
+ if (!description) {
43
+ logger.critical(
44
+ ErrorCode.MISSING_DESCRIPTION,
45
+ `AI scorer '${nameValue}' is missing the required 'description' property.`
46
+ )
47
+ return
48
+ }
49
+
50
+ const sampleRateValue = getPropertyValue(obj, 'sampleRate') as number | null
51
+ if (
52
+ sampleRateValue !== null &&
53
+ (sampleRateValue < 0 || sampleRateValue > 1)
54
+ ) {
55
+ logger.critical(
56
+ ErrorCode.INVALID_VALUE,
57
+ `AI scorer '${nameValue}' has a sampleRate of ${sampleRateValue} — it is the fraction of live runs to grade, so it must be between 0 and 1.`
58
+ )
59
+ return
60
+ }
61
+
62
+ const requiresReference =
63
+ (getPropertyValue(obj, 'requiresReference') as boolean | null) ?? false
64
+
65
+ const scorerKey = exportedName || nameValue
66
+
67
+ if (exportedName) {
68
+ state.scorers.files.set(scorerKey, {
69
+ path: node.getSourceFile().fileName,
70
+ exportedName,
71
+ })
72
+ }
73
+
74
+ state.scorers.scorersMeta[scorerKey] = {
75
+ name: nameValue,
76
+ description,
77
+ lane,
78
+ sampleRate: sampleRateValue ?? 1,
79
+ requiresReference,
80
+ sourceFile: node.getSourceFile().fileName,
81
+ exportedName: exportedName || undefined,
82
+ }
83
+ }
@@ -243,6 +243,25 @@ describe('pikkuScenarioStep', () => {
243
243
  }
244
244
  })
245
245
 
246
+ test('a graded agent run is a witness too', async () => {
247
+ // An agent's answer cannot be matched against a fixed string, so a scenario
248
+ // that grades one is asserting — even though `expectScore`, like the other
249
+ // helpers, is an inline step that reaches PKU680 as nothing at all.
250
+ const { criticals, cleanup } = await run([
251
+ "const answer = await scenario.when('asks for a summary', 'runAssistant', { prompt: 'hi' }, { actor: actors.shopper })",
252
+ "await scenario.expectScore('answered briefly', answer.runId, 'brevity', { atLeast: 0.8 })",
253
+ ])
254
+ try {
255
+ assert.equal(
256
+ criticals.filter((c) => c.code === 'PKU680').length,
257
+ 0,
258
+ `got: ${JSON.stringify(criticals)}`
259
+ )
260
+ } finally {
261
+ await cleanup()
262
+ }
263
+ })
264
+
246
265
  test('an expectation parked in an uncalled helper is still no assertion', async () => {
247
266
  const { criticals, cleanup } = await run([
248
267
  'const neverCalled = async () => {',
@@ -61,9 +61,52 @@ describe('addWireAddon', () => {
61
61
  secretOverrides: undefined,
62
62
  variableOverrides: undefined,
63
63
  credentialOverrides: undefined,
64
+ globalSecrets: undefined,
65
+ globalCredentials: undefined,
64
66
  })
65
67
  })
66
68
 
69
+ test('captures the reason an addon was exempted from credential scoping', () => {
70
+ const declarations = inspect(`
71
+ wireAddon({
72
+ name: 'console',
73
+ package: '@pikku/addon-console',
74
+ globalCredentials: 'links credentials an operator names at runtime',
75
+ })
76
+ `)
77
+
78
+ assert.equal(
79
+ declarations.get('console').globalCredentials,
80
+ 'links credentials an operator names at runtime'
81
+ )
82
+ })
83
+
84
+ test('captures the reason an addon was exempted from secret scoping', () => {
85
+ const declarations = inspect(`
86
+ wireAddon({
87
+ name: 'console',
88
+ package: '@pikku/addon-console',
89
+ globalSecrets: 'administers secrets an operator names at runtime',
90
+ })
91
+ `)
92
+
93
+ assert.equal(
94
+ declarations.get('console').globalSecrets,
95
+ 'administers secrets an operator names at runtime'
96
+ )
97
+ })
98
+
99
+ test('a globalSecrets reason that is not a literal still reports the grant', () => {
100
+ // Unlike `scopes`, dropping this would under-report: the grant is real at
101
+ // runtime whatever the reason evaluates to. The source text is recorded so
102
+ // the manifest names the addon and points at where the reason comes from.
103
+ const declarations = inspect(`
104
+ wireAddon({ name: 'console', package: '@x/y', globalSecrets: REASON })
105
+ `)
106
+
107
+ assert.equal(declarations.get('console').globalSecrets, 'REASON')
108
+ })
109
+
67
110
  test('leaves the gates undefined when none are declared', () => {
68
111
  const declarations = inspect(`
69
112
  wireAddon({ name: 'console', package: '@pikku/addon-console' })
@@ -60,6 +60,8 @@ export function addWireAddon(
60
60
  let secretOverrides: Record<string, string> | undefined
61
61
  let variableOverrides: Record<string, string> | undefined
62
62
  let credentialOverrides: Record<string, string> | undefined
63
+ let globalSecrets: string | undefined
64
+ let globalCredentials: string | undefined
63
65
 
64
66
  for (const prop of firstArg.properties) {
65
67
  if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue
@@ -102,6 +104,16 @@ export function addWireAddon(
102
104
  ts.isObjectLiteralExpression(prop.initializer)
103
105
  ) {
104
106
  credentialOverrides = parseStringRecord(prop.initializer)
107
+ } else if (key === 'globalSecrets') {
108
+ // The grant is real at runtime whatever the reason evaluates to, so a
109
+ // non-literal is recorded as its source text rather than dropped.
110
+ globalSecrets = ts.isStringLiteral(prop.initializer)
111
+ ? prop.initializer.text
112
+ : prop.initializer.getText()
113
+ } else if (key === 'globalCredentials') {
114
+ globalCredentials = ts.isStringLiteral(prop.initializer)
115
+ ? prop.initializer.text
116
+ : prop.initializer.getText()
105
117
  }
106
118
  }
107
119
 
@@ -118,6 +130,8 @@ export function addWireAddon(
118
130
  secretOverrides,
119
131
  variableOverrides,
120
132
  credentialOverrides,
133
+ globalSecrets,
134
+ globalCredentials,
121
135
  })
122
136
  state.rpc.usedAddons.add(name)
123
137
  state.rpc.wireAddonFiles.add(node.getSourceFile().fileName)
@@ -13,12 +13,18 @@
13
13
  * `pikku dev` registers the implementations itself. Ignoring by name (rather
14
14
  * than deleting the scaffold and requiring every project to regenerate) is what
15
15
  * keeps an existing project's checked-in `scenarios.gen.ts` from deploying.
16
+ *
17
+ * `pikkuScenarioGradeRun` was never scaffolded — it arrived after the split —
18
+ * but it belongs here for the same reason: grading a run on demand, ignoring the
19
+ * sample rate that governs live traffic, is a thing you do to a development
20
+ * server, and the name is reserved so an application cannot claim it.
16
21
  */
17
22
  export const SCENARIO_INSTRUMENTATION_FUNCTIONS = [
18
23
  'pikkuScenarioTakeLiveCoverage',
19
24
  'pikkuScenarioResetLiveCoverage',
20
25
  'pikkuScenarioResetStubs',
21
26
  'pikkuScenarioGetStubCalls',
27
+ 'pikkuScenarioGradeRun',
22
28
  ] as const
23
29
 
24
30
  export type ScenarioInstrumentationFunction =
@@ -91,6 +91,9 @@ export enum ErrorCode {
91
91
  AGENT_TOOL_NOT_FOUND = 'PKU153',
92
92
  AGENT_TOOL_MISSING_DESCRIPTION = 'PKU154',
93
93
 
94
+ // Agent scorer reference errors
95
+ AGENT_SCORER_NOT_FOUND = 'PKU155',
96
+
94
97
  // File structure errors
95
98
  SCHEMA_AND_WIRING_COLOCATED = 'PKU490',
96
99
 
package/src/inspector.ts CHANGED
@@ -16,6 +16,7 @@ import {
16
16
  stampAuthHandlerServices,
17
17
  validateAgentModels,
18
18
  validateAgentToolReferences,
19
+ validateAgentScorerReferences,
19
20
  validateSecretOverrides,
20
21
  validateVariableOverrides,
21
22
  validateCredentialOverrides,
@@ -28,7 +29,6 @@ import {
28
29
  computeMiddlewareGroupsMeta,
29
30
  computePermissionsGroupsMeta,
30
31
  computeRequiredSchemas,
31
- computeSecretBrokers,
32
32
  validateNoSecretAliasServices,
33
33
  validateSecretUsage,
34
34
  computeDiagnostics,
@@ -160,6 +160,10 @@ export function getInitialInspectorState(rootDir: string): InspectorState {
160
160
  agentsMeta: {},
161
161
  files: new Map(),
162
162
  },
163
+ scorers: {
164
+ scorersMeta: {},
165
+ files: new Map(),
166
+ },
163
167
  cli: {
164
168
  meta: {
165
169
  programs: {},
@@ -484,7 +488,6 @@ export const inspect = async (
484
488
  computeResolvedIOTypes(state)
485
489
  computeMiddlewareGroupsMeta(state)
486
490
  computePermissionsGroupsMeta(state)
487
- computeSecretBrokers(state)
488
491
  validateNoSecretAliasServices(logger, checker, state)
489
492
  validateSecretUsage(logger, state)
490
493
  computeDiagnostics(state)
@@ -511,8 +514,9 @@ export const inspect = async (
511
514
  )
512
515
  }
513
516
 
514
- validateAgentModels(logger, state)
517
+ validateAgentModels(logger, state, options.modelAliases)
515
518
  validateAgentToolReferences(logger, state, options)
519
+ validateAgentScorerReferences(logger, state)
516
520
  validateSecretOverrides(logger, state)
517
521
  validateVariableOverrides(logger, state)
518
522
  validateCredentialOverrides(logger, state)
package/src/types.ts CHANGED
@@ -13,6 +13,7 @@ import type {
13
13
  MCPPromptMeta,
14
14
  } from '@pikku/core/mcp'
15
15
  import type { AIAgentMeta } from '@pikku/core/ai-agent'
16
+ import type { ScorerMeta } from '@pikku/core/ai-scorer'
16
17
  import type { CLIMeta } from '@pikku/core/cli'
17
18
  import type { CLICommandMeta } from '@pikku/core/cli'
18
19
  import type { NodesMeta } from '@pikku/core/node'
@@ -298,6 +299,8 @@ export type InspectorOptions = Partial<{
298
299
  * tool's own name, so this is a quality gate rather than a correctness one.
299
300
  */
300
301
  strictMeta: boolean
302
+ /** The `models` table from pikku.config.json, alias -> `provider/model`. */
303
+ modelAliases: Record<string, string>
301
304
  sourceFile: ts.SourceFile
302
305
  /**
303
306
  * The project's tsconfig, read for its path mappings only. The inspector's
@@ -585,6 +588,16 @@ export interface InspectorState {
585
588
  secretOverrides?: Record<string, string>
586
589
  variableOverrides?: Record<string, string>
587
590
  credentialOverrides?: Record<string, string>
591
+ /**
592
+ * The app's stated reason for handing this instance the whole
593
+ * `SecretService` instead of one scoped to its declared secrets.
594
+ */
595
+ globalSecrets?: string
596
+ /**
597
+ * The app's stated reason for handing this instance the whole
598
+ * `CredentialService` instead of one scoped to its declared credentials.
599
+ */
600
+ globalCredentials?: string
588
601
  }
589
602
  >
590
603
  wireAddonFiles: Set<string>
@@ -599,6 +612,10 @@ export interface InspectorState {
599
612
  agentsMeta: AIAgentMeta
600
613
  files: Map<string, { path: string; exportedName: string }>
601
614
  }
615
+ scorers: {
616
+ scorersMeta: ScorerMeta
617
+ files: Map<string, { path: string; exportedName: string }>
618
+ }
602
619
  cli: {
603
620
  meta: CLIMeta
604
621
  files: Set<string>
@@ -419,6 +419,14 @@ export function filterInspectorState(
419
419
  agentsMeta: JSON.parse(JSON.stringify(state.agents?.agentsMeta ?? {})),
420
420
  files: new Map(),
421
421
  },
422
+ // Scorers carry through every filter. They are named by the agents that
423
+ // survive it, and a deployment that quietly stopped grading would look
424
+ // exactly like one whose scorers all returned nothing.
425
+ scorers: {
426
+ ...state.scorers,
427
+ scorersMeta: JSON.parse(JSON.stringify(state.scorers?.scorersMeta ?? {})),
428
+ files: new Map(state.scorers?.files ?? []),
429
+ },
422
430
  rpc: {
423
431
  ...state.rpc,
424
432
  internalMeta: { ...state.rpc.internalMeta }, // Clone to avoid mutating original