@pikku/inspector 0.12.56 → 0.12.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -0
- package/dist/add/add-ai-agent.js +25 -0
- package/dist/add/add-ai-scorer.d.ts +2 -0
- package/dist/add/add-ai-scorer.js +61 -0
- package/dist/add/add-wire-addon.js +16 -0
- package/dist/add/scenario-instrumentation.d.ts +6 -1
- package/dist/add/scenario-instrumentation.js +6 -0
- package/dist/error-codes.d.ts +1 -0
- package/dist/error-codes.js +2 -0
- package/dist/inspector.js +7 -3
- package/dist/types.d.ts +20 -0
- package/dist/utils/filter-inspector-state.js +8 -0
- package/dist/utils/post-process.d.ts +13 -3
- package/dist/utils/post-process.js +33 -15
- package/dist/utils/serialize-inspector-state.d.ts +9 -0
- package/dist/utils/serialize-inspector-state.js +8 -0
- package/dist/utils/workflow/dsl/extract-dsl-workflow.js +13 -4
- package/dist/utils/workflow/dsl/patterns.d.ts +2 -1
- package/dist/utils/workflow/dsl/patterns.js +8 -5
- package/dist/visit.js +2 -0
- package/package.json +3 -3
- package/src/add/add-ai-agent.ts +35 -0
- package/src/add/add-ai-scorer.test.ts +197 -0
- package/src/add/add-ai-scorer.ts +83 -0
- package/src/add/add-scenario-step.test.ts +19 -0
- package/src/add/add-wire-addon.test.ts +43 -0
- package/src/add/add-wire-addon.ts +14 -0
- package/src/add/scenario-instrumentation.ts +6 -0
- package/src/error-codes.ts +3 -0
- package/src/inspector.ts +7 -3
- package/src/types.ts +17 -0
- package/src/utils/filter-inspector-state.ts +8 -0
- package/src/utils/post-process.test.ts +71 -0
- package/src/utils/post-process.ts +41 -16
- package/src/utils/serialize-inspector-state.ts +14 -0
- package/src/utils/workflow/dsl/extract-approval-policy.test.ts +68 -0
- package/src/utils/workflow/dsl/extract-dsl-workflow.ts +12 -4
- package/src/utils/workflow/dsl/patterns.ts +8 -5
- package/src/visit.ts +2 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/src/utils/secret-brokers.test.ts +0 -53
- package/src/utils/secret-brokers.ts +0 -11
|
@@ -57,19 +57,22 @@ export function getScenarioStepPhase(node) {
|
|
|
57
57
|
/**
|
|
58
58
|
* The expectation helpers that assert without being a `then`.
|
|
59
59
|
*
|
|
60
|
-
* `expectService` and `
|
|
61
|
-
* steps, so they carry no phase and never reach the step meta — but
|
|
62
|
-
* a witness: a recorded service call,
|
|
63
|
-
* a scenario whose only assertion is one of these
|
|
60
|
+
* `expectService`, `expectError` and `expectScore` are inline steps rather than
|
|
61
|
+
* scenario steps, so they carry no phase and never reach the step meta — but
|
|
62
|
+
* each one is a witness: a recorded service call, a refusal, or a graded run.
|
|
63
|
+
* PKU680 has to see them, or a scenario whose only assertion is one of these
|
|
64
|
+
* reads as asserting nothing.
|
|
64
65
|
*/
|
|
65
66
|
const SCENARIO_EXPECTATION_CALLS = [
|
|
66
67
|
'expectService',
|
|
67
68
|
'expectError',
|
|
68
69
|
'expectEventually',
|
|
70
|
+
'expectScore',
|
|
69
71
|
];
|
|
70
72
|
/**
|
|
71
73
|
* Check if a call expression is one of the scenario expectation helpers —
|
|
72
|
-
* `scenario.expectService()`, `.expectError()
|
|
74
|
+
* `scenario.expectService()`, `.expectError()`, `.expectEventually()` or
|
|
75
|
+
* `.expectScore()`.
|
|
73
76
|
*/
|
|
74
77
|
export function isScenarioExpectationCall(node) {
|
|
75
78
|
return (ts.isPropertyAccessExpression(node.expression) &&
|
package/dist/visit.js
CHANGED
|
@@ -29,6 +29,7 @@ import { addVariable } from './add/add-variable.js';
|
|
|
29
29
|
import { addWorkflowGraph } from './add/add-workflow-graph.js';
|
|
30
30
|
import { addFeature } from './add/add-feature.js';
|
|
31
31
|
import { addAIAgent } from './add/add-ai-agent.js';
|
|
32
|
+
import { addAIScorer } from './add/add-ai-scorer.js';
|
|
32
33
|
import { addApprovalDescription } from './add/add-approval-description.js';
|
|
33
34
|
export const visitSetup = (logger, checker, node, state, options) => {
|
|
34
35
|
addFileExtendsCoreType(node, checker, state.singletonServicesTypeImportMap, 'CoreSingletonServices', state);
|
|
@@ -88,5 +89,6 @@ export const visitRoutes = (logger, checker, node, state, options) => {
|
|
|
88
89
|
addWorkflowGraph(logger, node, checker, state, nextOptions);
|
|
89
90
|
addFeature(logger, node, checker, state, nextOptions);
|
|
90
91
|
addAIAgent(logger, node, checker, state, nextOptions);
|
|
92
|
+
addAIScorer(logger, node, checker, state, nextOptions);
|
|
91
93
|
ts.forEachChild(node, (child) => visitRoutes(logger, checker, child, state, nextOptions));
|
|
92
94
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/inspector",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.58",
|
|
4
4
|
"description": "TypeScript inspector for Pikku — discovers functions and wirings to drive code generation",
|
|
5
5
|
"author": "yasser.fadl@gmail.com",
|
|
6
6
|
"license": "BUSL-1.1",
|
|
@@ -45,10 +45,10 @@
|
|
|
45
45
|
"zod-to-ts": "^2.1.0"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
|
-
"@pikku/core": "^0.12.
|
|
48
|
+
"@pikku/core": "^0.12.82"
|
|
49
49
|
},
|
|
50
50
|
"devDependencies": {
|
|
51
|
-
"@pikku/core": "^0.12.
|
|
51
|
+
"@pikku/core": "^0.12.82",
|
|
52
52
|
"@types/node": "^24.11.0"
|
|
53
53
|
},
|
|
54
54
|
"engines": {
|
package/src/add/add-ai-agent.ts
CHANGED
|
@@ -330,6 +330,39 @@ function resolveIdentifierToAgentName(
|
|
|
330
330
|
return null
|
|
331
331
|
}
|
|
332
332
|
|
|
333
|
+
/**
|
|
334
|
+
* The scorers an agent asks to be graded by, which are plain names rather than
|
|
335
|
+
* references: a scorer is resolved out of the registry at grading time, not
|
|
336
|
+
* imported by the agent, so nothing here has a declaration to chase.
|
|
337
|
+
*/
|
|
338
|
+
function resolveScorerNames(
|
|
339
|
+
obj: ts.ObjectLiteralExpression,
|
|
340
|
+
agentName: string,
|
|
341
|
+
logger: InspectorLogger
|
|
342
|
+
): string[] | null {
|
|
343
|
+
const property = obj.properties.find(
|
|
344
|
+
(p) =>
|
|
345
|
+
ts.isPropertyAssignment(p) &&
|
|
346
|
+
ts.isIdentifier(p.name) &&
|
|
347
|
+
p.name.text === 'scorers'
|
|
348
|
+
)
|
|
349
|
+
if (!property || !ts.isPropertyAssignment(property)) return null
|
|
350
|
+
if (!ts.isArrayLiteralExpression(property.initializer)) return null
|
|
351
|
+
|
|
352
|
+
const names: string[] = []
|
|
353
|
+
for (const element of property.initializer.elements) {
|
|
354
|
+
if (!ts.isStringLiteral(element)) {
|
|
355
|
+
logger.critical(
|
|
356
|
+
ErrorCode.INVALID_VALUE,
|
|
357
|
+
`AI agent '${agentName}' lists a scorer that is not an inline string: \`${element.getText()}\`. A scorer is named, not imported, so the name has to be readable from source.`
|
|
358
|
+
)
|
|
359
|
+
continue
|
|
360
|
+
}
|
|
361
|
+
names.push(element.text)
|
|
362
|
+
}
|
|
363
|
+
return names.length > 0 ? names : null
|
|
364
|
+
}
|
|
365
|
+
|
|
333
366
|
export const addAIAgent: AddWiring = (
|
|
334
367
|
logger,
|
|
335
368
|
node,
|
|
@@ -377,6 +410,7 @@ export const addAIAgent: AddWiring = (
|
|
|
377
410
|
| number
|
|
378
411
|
| null
|
|
379
412
|
const toolChoiceValue = getPropertyValue(obj, 'toolChoice') as string | null
|
|
413
|
+
const scorersValue = resolveScorerNames(obj, nameValue || '', logger)
|
|
380
414
|
const toolsValue = resolveToolReferences(
|
|
381
415
|
obj,
|
|
382
416
|
checker,
|
|
@@ -582,6 +616,7 @@ export const addAIAgent: AddWiring = (
|
|
|
582
616
|
...(toolsValue !== null && { tools: toolsValue }),
|
|
583
617
|
...(agentsValue !== null && { agents: agentsValue }),
|
|
584
618
|
...(workflowsValue !== null && { workflows: workflowsValue }),
|
|
619
|
+
...(scorersValue !== null && { scorers: scorersValue }),
|
|
585
620
|
tags,
|
|
586
621
|
inputSchema,
|
|
587
622
|
outputSchema,
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { strict as assert } from 'assert'
|
|
2
|
+
import { describe, test } from 'node:test'
|
|
3
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { inspect } from '../inspector.js'
|
|
7
|
+
import { ErrorCode } from '../error-codes.js'
|
|
8
|
+
import type { InspectorLogger } from '../types.js'
|
|
9
|
+
|
|
10
|
+
const makeLogger = (criticals: Array<{ code: ErrorCode; message: string }>) =>
|
|
11
|
+
({
|
|
12
|
+
debug: () => {},
|
|
13
|
+
info: () => {},
|
|
14
|
+
warn: () => {},
|
|
15
|
+
error: () => {},
|
|
16
|
+
diagnostic: ({ code, message }) => {
|
|
17
|
+
criticals.push({ code, message })
|
|
18
|
+
},
|
|
19
|
+
critical: (code: ErrorCode, message: string) => {
|
|
20
|
+
criticals.push({ code, message })
|
|
21
|
+
},
|
|
22
|
+
hasCriticalErrors: () => criticals.length > 0,
|
|
23
|
+
}) satisfies InspectorLogger
|
|
24
|
+
|
|
25
|
+
const inspectSources = async (sources: Record<string, string>) => {
|
|
26
|
+
const rootDir = await mkdtemp(join(tmpdir(), 'pikku-add-ai-scorer-'))
|
|
27
|
+
const files: Record<string, string> = {}
|
|
28
|
+
for (const [name, source] of Object.entries(sources)) {
|
|
29
|
+
files[name] = join(rootDir, name)
|
|
30
|
+
await writeFile(files[name]!, source)
|
|
31
|
+
}
|
|
32
|
+
const criticals: Array<{ code: ErrorCode; message: string }> = []
|
|
33
|
+
try {
|
|
34
|
+
const state = await inspect(makeLogger(criticals), Object.values(files), {
|
|
35
|
+
rootDir,
|
|
36
|
+
})
|
|
37
|
+
return { state, criticals }
|
|
38
|
+
} finally {
|
|
39
|
+
await rm(rootDir, { recursive: true, force: true })
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const inspectSource = (source: string) =>
|
|
44
|
+
inspectSources({ 'grading.scorer.ts': source })
|
|
45
|
+
|
|
46
|
+
describe('addAIScorer', () => {
|
|
47
|
+
test('reads the lane off which constructor was called, not off a field', async () => {
|
|
48
|
+
const { state, criticals } = await inspectSource(
|
|
49
|
+
[
|
|
50
|
+
`export const brevity = pikkuAIScorer({`,
|
|
51
|
+
` name: 'brevity',`,
|
|
52
|
+
` description: 'Shorter answers score higher',`,
|
|
53
|
+
` score: () => ({ score: 1 }),`,
|
|
54
|
+
`})`,
|
|
55
|
+
`export const helpfulness = pikkuAIJudge({`,
|
|
56
|
+
` name: 'helpfulness',`,
|
|
57
|
+
` description: 'Does the answer help',`,
|
|
58
|
+
` model: 'claude-opus-5',`,
|
|
59
|
+
` goal: 'Grade helpfulness.',`,
|
|
60
|
+
`})`,
|
|
61
|
+
].join('\n')
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
assert.deepEqual(criticals, [])
|
|
65
|
+
assert.equal(state.scorers.scorersMeta['brevity']?.lane, 'fast')
|
|
66
|
+
assert.equal(state.scorers.scorersMeta['helpfulness']?.lane, 'slow')
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
test('defaults an unstated sample rate to grading every run', async () => {
|
|
70
|
+
const { state } = await inspectSource(
|
|
71
|
+
[
|
|
72
|
+
`export const brevity = pikkuAIScorer({`,
|
|
73
|
+
` name: 'brevity',`,
|
|
74
|
+
` description: 'Shorter answers score higher',`,
|
|
75
|
+
` score: () => ({ score: 1 }),`,
|
|
76
|
+
`})`,
|
|
77
|
+
].join('\n')
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
assert.equal(state.scorers.scorersMeta['brevity']?.sampleRate, 1)
|
|
81
|
+
assert.equal(state.scorers.scorersMeta['brevity']?.requiresReference, false)
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
test('carries the sample rate and the reference requirement into the meta', async () => {
|
|
85
|
+
const { state } = await inspectSource(
|
|
86
|
+
[
|
|
87
|
+
`export const correctness = pikkuAIJudge({`,
|
|
88
|
+
` name: 'correctness',`,
|
|
89
|
+
` description: 'Is the answer right',`,
|
|
90
|
+
` model: 'claude-opus-5',`,
|
|
91
|
+
` goal: 'Grade correctness.',`,
|
|
92
|
+
` sampleRate: 0.1,`,
|
|
93
|
+
` requiresReference: true,`,
|
|
94
|
+
`})`,
|
|
95
|
+
].join('\n')
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
assert.equal(state.scorers.scorersMeta['correctness']?.sampleRate, 0.1)
|
|
99
|
+
assert.equal(
|
|
100
|
+
state.scorers.scorersMeta['correctness']?.requiresReference,
|
|
101
|
+
true
|
|
102
|
+
)
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
test('rejects a sample rate that is not a fraction', async () => {
|
|
106
|
+
const { state, criticals } = await inspectSource(
|
|
107
|
+
[
|
|
108
|
+
`export const brevity = pikkuAIScorer({`,
|
|
109
|
+
` name: 'brevity',`,
|
|
110
|
+
` description: 'Shorter answers score higher',`,
|
|
111
|
+
` sampleRate: 50,`,
|
|
112
|
+
` score: () => ({ score: 1 }),`,
|
|
113
|
+
`})`,
|
|
114
|
+
].join('\n')
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
assert.equal(criticals[0]?.code, ErrorCode.INVALID_VALUE)
|
|
118
|
+
assert.match(criticals[0]!.message, /between 0 and 1/)
|
|
119
|
+
assert.deepEqual(state.scorers.scorersMeta, {})
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
test('a scorer with no description is refused, since the meta is what names it', async () => {
|
|
123
|
+
const { criticals } = await inspectSource(
|
|
124
|
+
[
|
|
125
|
+
`export const brevity = pikkuAIScorer({`,
|
|
126
|
+
` name: 'brevity',`,
|
|
127
|
+
` score: () => ({ score: 1 }),`,
|
|
128
|
+
`})`,
|
|
129
|
+
].join('\n')
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
assert.equal(criticals[0]?.code, ErrorCode.MISSING_DESCRIPTION)
|
|
133
|
+
})
|
|
134
|
+
|
|
135
|
+
test('records where a scorer was declared, so codegen can import it back', async () => {
|
|
136
|
+
const { state } = await inspectSource(
|
|
137
|
+
[
|
|
138
|
+
`export const brevity = pikkuAIScorer({`,
|
|
139
|
+
` name: 'brevity',`,
|
|
140
|
+
` description: 'Shorter answers score higher',`,
|
|
141
|
+
` score: () => ({ score: 1 }),`,
|
|
142
|
+
`})`,
|
|
143
|
+
].join('\n')
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
assert.equal(state.scorers.files.get('brevity')?.exportedName, 'brevity')
|
|
147
|
+
assert.match(
|
|
148
|
+
state.scorers.files.get('brevity')?.path ?? '',
|
|
149
|
+
/grading\.scorer\.ts$/
|
|
150
|
+
)
|
|
151
|
+
})
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
describe('an agent asking to be graded', () => {
|
|
155
|
+
const scorerFile = [
|
|
156
|
+
`export const brevity = pikkuAIScorer({`,
|
|
157
|
+
` name: 'brevity',`,
|
|
158
|
+
` description: 'Shorter answers score higher',`,
|
|
159
|
+
` score: () => ({ score: 1 }),`,
|
|
160
|
+
`})`,
|
|
161
|
+
].join('\n')
|
|
162
|
+
|
|
163
|
+
const agent = (scorers: string) =>
|
|
164
|
+
[
|
|
165
|
+
`export const assistant = pikkuAIAgent({`,
|
|
166
|
+
` name: 'assistant',`,
|
|
167
|
+
` description: 'Answers questions',`,
|
|
168
|
+
` goal: 'Answer the question.',`,
|
|
169
|
+
` model: 'anthropic/claude-opus-5',`,
|
|
170
|
+
` scorers: [${scorers}],`,
|
|
171
|
+
`})`,
|
|
172
|
+
].join('\n')
|
|
173
|
+
|
|
174
|
+
test('carries the named scorers onto the agent meta', async () => {
|
|
175
|
+
const { state, criticals } = await inspectSources({
|
|
176
|
+
'grading.scorer.ts': scorerFile,
|
|
177
|
+
'assistant.agent.ts': agent(`'brevity'`),
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
assert.deepEqual(criticals, [])
|
|
181
|
+
assert.deepEqual(state.agents.agentsMeta['assistant']?.scorers, ['brevity'])
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
test('naming a scorer that does not exist fails the build rather than grading nothing', async () => {
|
|
185
|
+
const { criticals } = await inspectSources({
|
|
186
|
+
'grading.scorer.ts': scorerFile,
|
|
187
|
+
'assistant.agent.ts': agent(`'brevty'`),
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
const finding = criticals.find(
|
|
191
|
+
(critical) => critical.code === ErrorCode.AGENT_SCORER_NOT_FOUND
|
|
192
|
+
)
|
|
193
|
+
assert.ok(finding, `expected a scorer-not-found critical`)
|
|
194
|
+
assert.match(finding!.message, /brevty/)
|
|
195
|
+
assert.match(finding!.message, /Declared scorers: brevity/)
|
|
196
|
+
})
|
|
197
|
+
})
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import * as ts from 'typescript'
|
|
2
|
+
import { getPropertyValue } from '../utils/get-property-value.js'
|
|
3
|
+
import type { AddWiring } from '../types.js'
|
|
4
|
+
import { extractFunctionName } from '../utils/extract-function-name.js'
|
|
5
|
+
import { ErrorCode } from '../error-codes.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `pikkuAIScorer` grades in code and `pikkuAIJudge` grades with a model, which
|
|
9
|
+
* is the whole difference between the two lanes — so the lane is read off which
|
|
10
|
+
* constructor was called rather than from a field an author could contradict.
|
|
11
|
+
*/
|
|
12
|
+
const LANES: Record<string, 'fast' | 'slow'> = {
|
|
13
|
+
pikkuAIScorer: 'fast',
|
|
14
|
+
pikkuAIJudge: 'slow',
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const addAIScorer: AddWiring = (logger, node, checker, state) => {
|
|
18
|
+
if (!ts.isCallExpression(node)) return
|
|
19
|
+
|
|
20
|
+
const expression = node.expression
|
|
21
|
+
if (!ts.isIdentifier(expression)) return
|
|
22
|
+
|
|
23
|
+
const lane = LANES[expression.text]
|
|
24
|
+
if (!lane) return
|
|
25
|
+
|
|
26
|
+
const firstArg = node.arguments[0]
|
|
27
|
+
if (!firstArg || !ts.isObjectLiteralExpression(firstArg)) return
|
|
28
|
+
|
|
29
|
+
const obj = firstArg
|
|
30
|
+
const { exportedName } = extractFunctionName(node, checker, state.rootDir)
|
|
31
|
+
|
|
32
|
+
const nameValue = getPropertyValue(obj, 'name') as string | null
|
|
33
|
+
if (!nameValue) {
|
|
34
|
+
logger.critical(
|
|
35
|
+
ErrorCode.MISSING_NAME,
|
|
36
|
+
"AI scorer is missing the required 'name' property."
|
|
37
|
+
)
|
|
38
|
+
return
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const description = getPropertyValue(obj, 'description') as string | null
|
|
42
|
+
if (!description) {
|
|
43
|
+
logger.critical(
|
|
44
|
+
ErrorCode.MISSING_DESCRIPTION,
|
|
45
|
+
`AI scorer '${nameValue}' is missing the required 'description' property.`
|
|
46
|
+
)
|
|
47
|
+
return
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const sampleRateValue = getPropertyValue(obj, 'sampleRate') as number | null
|
|
51
|
+
if (
|
|
52
|
+
sampleRateValue !== null &&
|
|
53
|
+
(sampleRateValue < 0 || sampleRateValue > 1)
|
|
54
|
+
) {
|
|
55
|
+
logger.critical(
|
|
56
|
+
ErrorCode.INVALID_VALUE,
|
|
57
|
+
`AI scorer '${nameValue}' has a sampleRate of ${sampleRateValue} — it is the fraction of live runs to grade, so it must be between 0 and 1.`
|
|
58
|
+
)
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const requiresReference =
|
|
63
|
+
(getPropertyValue(obj, 'requiresReference') as boolean | null) ?? false
|
|
64
|
+
|
|
65
|
+
const scorerKey = exportedName || nameValue
|
|
66
|
+
|
|
67
|
+
if (exportedName) {
|
|
68
|
+
state.scorers.files.set(scorerKey, {
|
|
69
|
+
path: node.getSourceFile().fileName,
|
|
70
|
+
exportedName,
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
state.scorers.scorersMeta[scorerKey] = {
|
|
75
|
+
name: nameValue,
|
|
76
|
+
description,
|
|
77
|
+
lane,
|
|
78
|
+
sampleRate: sampleRateValue ?? 1,
|
|
79
|
+
requiresReference,
|
|
80
|
+
sourceFile: node.getSourceFile().fileName,
|
|
81
|
+
exportedName: exportedName || undefined,
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -243,6 +243,25 @@ describe('pikkuScenarioStep', () => {
|
|
|
243
243
|
}
|
|
244
244
|
})
|
|
245
245
|
|
|
246
|
+
test('a graded agent run is a witness too', async () => {
|
|
247
|
+
// An agent's answer cannot be matched against a fixed string, so a scenario
|
|
248
|
+
// that grades one is asserting — even though `expectScore`, like the other
|
|
249
|
+
// helpers, is an inline step that reaches PKU680 as nothing at all.
|
|
250
|
+
const { criticals, cleanup } = await run([
|
|
251
|
+
"const answer = await scenario.when('asks for a summary', 'runAssistant', { prompt: 'hi' }, { actor: actors.shopper })",
|
|
252
|
+
"await scenario.expectScore('answered briefly', answer.runId, 'brevity', { atLeast: 0.8 })",
|
|
253
|
+
])
|
|
254
|
+
try {
|
|
255
|
+
assert.equal(
|
|
256
|
+
criticals.filter((c) => c.code === 'PKU680').length,
|
|
257
|
+
0,
|
|
258
|
+
`got: ${JSON.stringify(criticals)}`
|
|
259
|
+
)
|
|
260
|
+
} finally {
|
|
261
|
+
await cleanup()
|
|
262
|
+
}
|
|
263
|
+
})
|
|
264
|
+
|
|
246
265
|
test('an expectation parked in an uncalled helper is still no assertion', async () => {
|
|
247
266
|
const { criticals, cleanup } = await run([
|
|
248
267
|
'const neverCalled = async () => {',
|
|
@@ -61,9 +61,52 @@ describe('addWireAddon', () => {
|
|
|
61
61
|
secretOverrides: undefined,
|
|
62
62
|
variableOverrides: undefined,
|
|
63
63
|
credentialOverrides: undefined,
|
|
64
|
+
globalSecrets: undefined,
|
|
65
|
+
globalCredentials: undefined,
|
|
64
66
|
})
|
|
65
67
|
})
|
|
66
68
|
|
|
69
|
+
test('captures the reason an addon was exempted from credential scoping', () => {
|
|
70
|
+
const declarations = inspect(`
|
|
71
|
+
wireAddon({
|
|
72
|
+
name: 'console',
|
|
73
|
+
package: '@pikku/addon-console',
|
|
74
|
+
globalCredentials: 'links credentials an operator names at runtime',
|
|
75
|
+
})
|
|
76
|
+
`)
|
|
77
|
+
|
|
78
|
+
assert.equal(
|
|
79
|
+
declarations.get('console').globalCredentials,
|
|
80
|
+
'links credentials an operator names at runtime'
|
|
81
|
+
)
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
test('captures the reason an addon was exempted from secret scoping', () => {
|
|
85
|
+
const declarations = inspect(`
|
|
86
|
+
wireAddon({
|
|
87
|
+
name: 'console',
|
|
88
|
+
package: '@pikku/addon-console',
|
|
89
|
+
globalSecrets: 'administers secrets an operator names at runtime',
|
|
90
|
+
})
|
|
91
|
+
`)
|
|
92
|
+
|
|
93
|
+
assert.equal(
|
|
94
|
+
declarations.get('console').globalSecrets,
|
|
95
|
+
'administers secrets an operator names at runtime'
|
|
96
|
+
)
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
test('a globalSecrets reason that is not a literal still reports the grant', () => {
|
|
100
|
+
// Unlike `scopes`, dropping this would under-report: the grant is real at
|
|
101
|
+
// runtime whatever the reason evaluates to. The source text is recorded so
|
|
102
|
+
// the manifest names the addon and points at where the reason comes from.
|
|
103
|
+
const declarations = inspect(`
|
|
104
|
+
wireAddon({ name: 'console', package: '@x/y', globalSecrets: REASON })
|
|
105
|
+
`)
|
|
106
|
+
|
|
107
|
+
assert.equal(declarations.get('console').globalSecrets, 'REASON')
|
|
108
|
+
})
|
|
109
|
+
|
|
67
110
|
test('leaves the gates undefined when none are declared', () => {
|
|
68
111
|
const declarations = inspect(`
|
|
69
112
|
wireAddon({ name: 'console', package: '@pikku/addon-console' })
|
|
@@ -60,6 +60,8 @@ export function addWireAddon(
|
|
|
60
60
|
let secretOverrides: Record<string, string> | undefined
|
|
61
61
|
let variableOverrides: Record<string, string> | undefined
|
|
62
62
|
let credentialOverrides: Record<string, string> | undefined
|
|
63
|
+
let globalSecrets: string | undefined
|
|
64
|
+
let globalCredentials: string | undefined
|
|
63
65
|
|
|
64
66
|
for (const prop of firstArg.properties) {
|
|
65
67
|
if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue
|
|
@@ -102,6 +104,16 @@ export function addWireAddon(
|
|
|
102
104
|
ts.isObjectLiteralExpression(prop.initializer)
|
|
103
105
|
) {
|
|
104
106
|
credentialOverrides = parseStringRecord(prop.initializer)
|
|
107
|
+
} else if (key === 'globalSecrets') {
|
|
108
|
+
// The grant is real at runtime whatever the reason evaluates to, so a
|
|
109
|
+
// non-literal is recorded as its source text rather than dropped.
|
|
110
|
+
globalSecrets = ts.isStringLiteral(prop.initializer)
|
|
111
|
+
? prop.initializer.text
|
|
112
|
+
: prop.initializer.getText()
|
|
113
|
+
} else if (key === 'globalCredentials') {
|
|
114
|
+
globalCredentials = ts.isStringLiteral(prop.initializer)
|
|
115
|
+
? prop.initializer.text
|
|
116
|
+
: prop.initializer.getText()
|
|
105
117
|
}
|
|
106
118
|
}
|
|
107
119
|
|
|
@@ -118,6 +130,8 @@ export function addWireAddon(
|
|
|
118
130
|
secretOverrides,
|
|
119
131
|
variableOverrides,
|
|
120
132
|
credentialOverrides,
|
|
133
|
+
globalSecrets,
|
|
134
|
+
globalCredentials,
|
|
121
135
|
})
|
|
122
136
|
state.rpc.usedAddons.add(name)
|
|
123
137
|
state.rpc.wireAddonFiles.add(node.getSourceFile().fileName)
|
|
@@ -13,12 +13,18 @@
|
|
|
13
13
|
* `pikku dev` registers the implementations itself. Ignoring by name (rather
|
|
14
14
|
* than deleting the scaffold and requiring every project to regenerate) is what
|
|
15
15
|
* keeps an existing project's checked-in `scenarios.gen.ts` from deploying.
|
|
16
|
+
*
|
|
17
|
+
* `pikkuScenarioGradeRun` was never scaffolded — it arrived after the split —
|
|
18
|
+
* but it belongs here for the same reason: grading a run on demand, ignoring the
|
|
19
|
+
* sample rate that governs live traffic, is a thing you do to a development
|
|
20
|
+
* server, and the name is reserved so an application cannot claim it.
|
|
16
21
|
*/
|
|
17
22
|
export const SCENARIO_INSTRUMENTATION_FUNCTIONS = [
|
|
18
23
|
'pikkuScenarioTakeLiveCoverage',
|
|
19
24
|
'pikkuScenarioResetLiveCoverage',
|
|
20
25
|
'pikkuScenarioResetStubs',
|
|
21
26
|
'pikkuScenarioGetStubCalls',
|
|
27
|
+
'pikkuScenarioGradeRun',
|
|
22
28
|
] as const
|
|
23
29
|
|
|
24
30
|
export type ScenarioInstrumentationFunction =
|
package/src/error-codes.ts
CHANGED
|
@@ -91,6 +91,9 @@ export enum ErrorCode {
|
|
|
91
91
|
AGENT_TOOL_NOT_FOUND = 'PKU153',
|
|
92
92
|
AGENT_TOOL_MISSING_DESCRIPTION = 'PKU154',
|
|
93
93
|
|
|
94
|
+
// Agent scorer reference errors
|
|
95
|
+
AGENT_SCORER_NOT_FOUND = 'PKU155',
|
|
96
|
+
|
|
94
97
|
// File structure errors
|
|
95
98
|
SCHEMA_AND_WIRING_COLOCATED = 'PKU490',
|
|
96
99
|
|
package/src/inspector.ts
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
stampAuthHandlerServices,
|
|
17
17
|
validateAgentModels,
|
|
18
18
|
validateAgentToolReferences,
|
|
19
|
+
validateAgentScorerReferences,
|
|
19
20
|
validateSecretOverrides,
|
|
20
21
|
validateVariableOverrides,
|
|
21
22
|
validateCredentialOverrides,
|
|
@@ -28,7 +29,6 @@ import {
|
|
|
28
29
|
computeMiddlewareGroupsMeta,
|
|
29
30
|
computePermissionsGroupsMeta,
|
|
30
31
|
computeRequiredSchemas,
|
|
31
|
-
computeSecretBrokers,
|
|
32
32
|
validateNoSecretAliasServices,
|
|
33
33
|
validateSecretUsage,
|
|
34
34
|
computeDiagnostics,
|
|
@@ -160,6 +160,10 @@ export function getInitialInspectorState(rootDir: string): InspectorState {
|
|
|
160
160
|
agentsMeta: {},
|
|
161
161
|
files: new Map(),
|
|
162
162
|
},
|
|
163
|
+
scorers: {
|
|
164
|
+
scorersMeta: {},
|
|
165
|
+
files: new Map(),
|
|
166
|
+
},
|
|
163
167
|
cli: {
|
|
164
168
|
meta: {
|
|
165
169
|
programs: {},
|
|
@@ -484,7 +488,6 @@ export const inspect = async (
|
|
|
484
488
|
computeResolvedIOTypes(state)
|
|
485
489
|
computeMiddlewareGroupsMeta(state)
|
|
486
490
|
computePermissionsGroupsMeta(state)
|
|
487
|
-
computeSecretBrokers(state)
|
|
488
491
|
validateNoSecretAliasServices(logger, checker, state)
|
|
489
492
|
validateSecretUsage(logger, state)
|
|
490
493
|
computeDiagnostics(state)
|
|
@@ -511,8 +514,9 @@ export const inspect = async (
|
|
|
511
514
|
)
|
|
512
515
|
}
|
|
513
516
|
|
|
514
|
-
validateAgentModels(logger, state)
|
|
517
|
+
validateAgentModels(logger, state, options.modelAliases)
|
|
515
518
|
validateAgentToolReferences(logger, state, options)
|
|
519
|
+
validateAgentScorerReferences(logger, state)
|
|
516
520
|
validateSecretOverrides(logger, state)
|
|
517
521
|
validateVariableOverrides(logger, state)
|
|
518
522
|
validateCredentialOverrides(logger, state)
|
package/src/types.ts
CHANGED
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
MCPPromptMeta,
|
|
14
14
|
} from '@pikku/core/mcp'
|
|
15
15
|
import type { AIAgentMeta } from '@pikku/core/ai-agent'
|
|
16
|
+
import type { ScorerMeta } from '@pikku/core/ai-scorer'
|
|
16
17
|
import type { CLIMeta } from '@pikku/core/cli'
|
|
17
18
|
import type { CLICommandMeta } from '@pikku/core/cli'
|
|
18
19
|
import type { NodesMeta } from '@pikku/core/node'
|
|
@@ -298,6 +299,8 @@ export type InspectorOptions = Partial<{
|
|
|
298
299
|
* tool's own name, so this is a quality gate rather than a correctness one.
|
|
299
300
|
*/
|
|
300
301
|
strictMeta: boolean
|
|
302
|
+
/** The `models` table from pikku.config.json, alias -> `provider/model`. */
|
|
303
|
+
modelAliases: Record<string, string>
|
|
301
304
|
sourceFile: ts.SourceFile
|
|
302
305
|
/**
|
|
303
306
|
* The project's tsconfig, read for its path mappings only. The inspector's
|
|
@@ -585,6 +588,16 @@ export interface InspectorState {
|
|
|
585
588
|
secretOverrides?: Record<string, string>
|
|
586
589
|
variableOverrides?: Record<string, string>
|
|
587
590
|
credentialOverrides?: Record<string, string>
|
|
591
|
+
/**
|
|
592
|
+
* The app's stated reason for handing this instance the whole
|
|
593
|
+
* `SecretService` instead of one scoped to its declared secrets.
|
|
594
|
+
*/
|
|
595
|
+
globalSecrets?: string
|
|
596
|
+
/**
|
|
597
|
+
* The app's stated reason for handing this instance the whole
|
|
598
|
+
* `CredentialService` instead of one scoped to its declared credentials.
|
|
599
|
+
*/
|
|
600
|
+
globalCredentials?: string
|
|
588
601
|
}
|
|
589
602
|
>
|
|
590
603
|
wireAddonFiles: Set<string>
|
|
@@ -599,6 +612,10 @@ export interface InspectorState {
|
|
|
599
612
|
agentsMeta: AIAgentMeta
|
|
600
613
|
files: Map<string, { path: string; exportedName: string }>
|
|
601
614
|
}
|
|
615
|
+
scorers: {
|
|
616
|
+
scorersMeta: ScorerMeta
|
|
617
|
+
files: Map<string, { path: string; exportedName: string }>
|
|
618
|
+
}
|
|
602
619
|
cli: {
|
|
603
620
|
meta: CLIMeta
|
|
604
621
|
files: Set<string>
|
|
@@ -419,6 +419,14 @@ export function filterInspectorState(
|
|
|
419
419
|
agentsMeta: JSON.parse(JSON.stringify(state.agents?.agentsMeta ?? {})),
|
|
420
420
|
files: new Map(),
|
|
421
421
|
},
|
|
422
|
+
// Scorers carry through every filter. They are named by the agents that
|
|
423
|
+
// survive it, and a deployment that quietly stopped grading would look
|
|
424
|
+
// exactly like one whose scorers all returned nothing.
|
|
425
|
+
scorers: {
|
|
426
|
+
...state.scorers,
|
|
427
|
+
scorersMeta: JSON.parse(JSON.stringify(state.scorers?.scorersMeta ?? {})),
|
|
428
|
+
files: new Map(state.scorers?.files ?? []),
|
|
429
|
+
},
|
|
422
430
|
rpc: {
|
|
423
431
|
...state.rpc,
|
|
424
432
|
internalMeta: { ...state.rpc.internalMeta }, // Clone to avoid mutating original
|