@pikku/inspector 0.12.55 → 0.12.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +168 -0
- package/dist/add/add-ai-agent.js +25 -0
- package/dist/add/add-ai-scorer.d.ts +2 -0
- package/dist/add/add-ai-scorer.js +61 -0
- package/dist/add/add-wire-addon.js +16 -0
- package/dist/add/scenario-instrumentation.d.ts +6 -1
- package/dist/add/scenario-instrumentation.js +6 -0
- package/dist/error-codes.d.ts +2 -0
- package/dist/error-codes.js +3 -0
- package/dist/inspector.js +13 -4
- package/dist/types.d.ts +20 -0
- package/dist/utils/filter-inspector-state.js +8 -0
- package/dist/utils/post-process.d.ts +28 -3
- package/dist/utils/post-process.js +85 -15
- package/dist/utils/serialize-inspector-state.d.ts +9 -0
- package/dist/utils/serialize-inspector-state.js +8 -0
- package/dist/utils/workflow/dsl/extract-dsl-workflow.js +13 -4
- package/dist/utils/workflow/dsl/patterns.d.ts +2 -1
- package/dist/utils/workflow/dsl/patterns.js +8 -5
- package/dist/utils/workflow/graph/serialize-workflow-graph.js +2 -10
- package/dist/visit.js +2 -0
- package/package.json +3 -3
- package/src/add/add-ai-agent.ts +35 -0
- package/src/add/add-ai-scorer.test.ts +197 -0
- package/src/add/add-ai-scorer.ts +83 -0
- package/src/add/add-scenario-step.test.ts +19 -0
- package/src/add/add-wire-addon.test.ts +43 -0
- package/src/add/add-wire-addon.ts +14 -0
- package/src/add/scenario-instrumentation.ts +6 -0
- package/src/error-codes.ts +4 -0
- package/src/inspector.ts +15 -4
- package/src/types.ts +17 -0
- package/src/utils/filter-inspector-state.ts +8 -0
- package/src/utils/post-process.test.ts +71 -0
- package/src/utils/post-process.ts +98 -16
- package/src/utils/serialize-inspector-state.ts +14 -0
- package/src/utils/workflow/dsl/extract-approval-policy.test.ts +68 -0
- package/src/utils/workflow/dsl/extract-dsl-workflow.ts +12 -4
- package/src/utils/workflow/dsl/patterns.ts +8 -5
- package/src/utils/workflow/graph/serialize-workflow-graph.ts +2 -15
- package/src/visit.ts +2 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/src/utils/secret-brokers.test.ts +0 -53
- package/src/utils/secret-brokers.ts +0 -11
|
@@ -243,6 +243,25 @@ describe('pikkuScenarioStep', () => {
|
|
|
243
243
|
}
|
|
244
244
|
})
|
|
245
245
|
|
|
246
|
+
test('a graded agent run is a witness too', async () => {
|
|
247
|
+
// An agent's answer cannot be matched against a fixed string, so a scenario
|
|
248
|
+
// that grades one is asserting — even though `expectScore`, like the other
|
|
249
|
+
// helpers, is an inline step that reaches PKU680 as nothing at all.
|
|
250
|
+
const { criticals, cleanup } = await run([
|
|
251
|
+
"const answer = await scenario.when('asks for a summary', 'runAssistant', { prompt: 'hi' }, { actor: actors.shopper })",
|
|
252
|
+
"await scenario.expectScore('answered briefly', answer.runId, 'brevity', { atLeast: 0.8 })",
|
|
253
|
+
])
|
|
254
|
+
try {
|
|
255
|
+
assert.equal(
|
|
256
|
+
criticals.filter((c) => c.code === 'PKU680').length,
|
|
257
|
+
0,
|
|
258
|
+
`got: ${JSON.stringify(criticals)}`
|
|
259
|
+
)
|
|
260
|
+
} finally {
|
|
261
|
+
await cleanup()
|
|
262
|
+
}
|
|
263
|
+
})
|
|
264
|
+
|
|
246
265
|
test('an expectation parked in an uncalled helper is still no assertion', async () => {
|
|
247
266
|
const { criticals, cleanup } = await run([
|
|
248
267
|
'const neverCalled = async () => {',
|
|
@@ -61,9 +61,52 @@ describe('addWireAddon', () => {
|
|
|
61
61
|
secretOverrides: undefined,
|
|
62
62
|
variableOverrides: undefined,
|
|
63
63
|
credentialOverrides: undefined,
|
|
64
|
+
globalSecrets: undefined,
|
|
65
|
+
globalCredentials: undefined,
|
|
64
66
|
})
|
|
65
67
|
})
|
|
66
68
|
|
|
69
|
+
test('captures the reason an addon was exempted from credential scoping', () => {
|
|
70
|
+
const declarations = inspect(`
|
|
71
|
+
wireAddon({
|
|
72
|
+
name: 'console',
|
|
73
|
+
package: '@pikku/addon-console',
|
|
74
|
+
globalCredentials: 'links credentials an operator names at runtime',
|
|
75
|
+
})
|
|
76
|
+
`)
|
|
77
|
+
|
|
78
|
+
assert.equal(
|
|
79
|
+
declarations.get('console').globalCredentials,
|
|
80
|
+
'links credentials an operator names at runtime'
|
|
81
|
+
)
|
|
82
|
+
})
|
|
83
|
+
|
|
84
|
+
test('captures the reason an addon was exempted from secret scoping', () => {
|
|
85
|
+
const declarations = inspect(`
|
|
86
|
+
wireAddon({
|
|
87
|
+
name: 'console',
|
|
88
|
+
package: '@pikku/addon-console',
|
|
89
|
+
globalSecrets: 'administers secrets an operator names at runtime',
|
|
90
|
+
})
|
|
91
|
+
`)
|
|
92
|
+
|
|
93
|
+
assert.equal(
|
|
94
|
+
declarations.get('console').globalSecrets,
|
|
95
|
+
'administers secrets an operator names at runtime'
|
|
96
|
+
)
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
test('a globalSecrets reason that is not a literal still reports the grant', () => {
|
|
100
|
+
// Unlike `scopes`, dropping this would under-report: the grant is real at
|
|
101
|
+
// runtime whatever the reason evaluates to. The source text is recorded so
|
|
102
|
+
// the manifest names the addon and points at where the reason comes from.
|
|
103
|
+
const declarations = inspect(`
|
|
104
|
+
wireAddon({ name: 'console', package: '@x/y', globalSecrets: REASON })
|
|
105
|
+
`)
|
|
106
|
+
|
|
107
|
+
assert.equal(declarations.get('console').globalSecrets, 'REASON')
|
|
108
|
+
})
|
|
109
|
+
|
|
67
110
|
test('leaves the gates undefined when none are declared', () => {
|
|
68
111
|
const declarations = inspect(`
|
|
69
112
|
wireAddon({ name: 'console', package: '@pikku/addon-console' })
|
|
@@ -60,6 +60,8 @@ export function addWireAddon(
|
|
|
60
60
|
let secretOverrides: Record<string, string> | undefined
|
|
61
61
|
let variableOverrides: Record<string, string> | undefined
|
|
62
62
|
let credentialOverrides: Record<string, string> | undefined
|
|
63
|
+
let globalSecrets: string | undefined
|
|
64
|
+
let globalCredentials: string | undefined
|
|
63
65
|
|
|
64
66
|
for (const prop of firstArg.properties) {
|
|
65
67
|
if (!ts.isPropertyAssignment(prop) || !ts.isIdentifier(prop.name)) continue
|
|
@@ -102,6 +104,16 @@ export function addWireAddon(
|
|
|
102
104
|
ts.isObjectLiteralExpression(prop.initializer)
|
|
103
105
|
) {
|
|
104
106
|
credentialOverrides = parseStringRecord(prop.initializer)
|
|
107
|
+
} else if (key === 'globalSecrets') {
|
|
108
|
+
// The grant is real at runtime whatever the reason evaluates to, so a
|
|
109
|
+
// non-literal is recorded as its source text rather than dropped.
|
|
110
|
+
globalSecrets = ts.isStringLiteral(prop.initializer)
|
|
111
|
+
? prop.initializer.text
|
|
112
|
+
: prop.initializer.getText()
|
|
113
|
+
} else if (key === 'globalCredentials') {
|
|
114
|
+
globalCredentials = ts.isStringLiteral(prop.initializer)
|
|
115
|
+
? prop.initializer.text
|
|
116
|
+
: prop.initializer.getText()
|
|
105
117
|
}
|
|
106
118
|
}
|
|
107
119
|
|
|
@@ -118,6 +130,8 @@ export function addWireAddon(
|
|
|
118
130
|
secretOverrides,
|
|
119
131
|
variableOverrides,
|
|
120
132
|
credentialOverrides,
|
|
133
|
+
globalSecrets,
|
|
134
|
+
globalCredentials,
|
|
121
135
|
})
|
|
122
136
|
state.rpc.usedAddons.add(name)
|
|
123
137
|
state.rpc.wireAddonFiles.add(node.getSourceFile().fileName)
|
|
@@ -13,12 +13,18 @@
|
|
|
13
13
|
* `pikku dev` registers the implementations itself. Ignoring by name (rather
|
|
14
14
|
* than deleting the scaffold and requiring every project to regenerate) is what
|
|
15
15
|
* keeps an existing project's checked-in `scenarios.gen.ts` from deploying.
|
|
16
|
+
*
|
|
17
|
+
* `pikkuScenarioGradeRun` was never scaffolded — it arrived after the split —
|
|
18
|
+
* but it belongs here for the same reason: grading a run on demand, ignoring the
|
|
19
|
+
* sample rate that governs live traffic, is a thing you do to a development
|
|
20
|
+
* server, and the name is reserved so an application cannot claim it.
|
|
16
21
|
*/
|
|
17
22
|
export const SCENARIO_INSTRUMENTATION_FUNCTIONS = [
|
|
18
23
|
'pikkuScenarioTakeLiveCoverage',
|
|
19
24
|
'pikkuScenarioResetLiveCoverage',
|
|
20
25
|
'pikkuScenarioResetStubs',
|
|
21
26
|
'pikkuScenarioGetStubCalls',
|
|
27
|
+
'pikkuScenarioGradeRun',
|
|
22
28
|
] as const
|
|
23
29
|
|
|
24
30
|
export type ScenarioInstrumentationFunction =
|
package/src/error-codes.ts
CHANGED
|
@@ -45,6 +45,7 @@ export enum ErrorCode {
|
|
|
45
45
|
|
|
46
46
|
// Configuration errors
|
|
47
47
|
SCHEMA_GENERATION_ERROR = 'PKU456',
|
|
48
|
+
SCHEMA_REFERENCE_UNRESOLVED = 'PKU463',
|
|
48
49
|
INLINE_SCHEMA = 'PKU489',
|
|
49
50
|
|
|
50
51
|
// Function errors
|
|
@@ -90,6 +91,9 @@ export enum ErrorCode {
|
|
|
90
91
|
AGENT_TOOL_NOT_FOUND = 'PKU153',
|
|
91
92
|
AGENT_TOOL_MISSING_DESCRIPTION = 'PKU154',
|
|
92
93
|
|
|
94
|
+
// Agent scorer reference errors
|
|
95
|
+
AGENT_SCORER_NOT_FOUND = 'PKU155',
|
|
96
|
+
|
|
93
97
|
// File structure errors
|
|
94
98
|
SCHEMA_AND_WIRING_COLOCATED = 'PKU490',
|
|
95
99
|
|
package/src/inspector.ts
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
stampAuthHandlerServices,
|
|
17
17
|
validateAgentModels,
|
|
18
18
|
validateAgentToolReferences,
|
|
19
|
+
validateAgentScorerReferences,
|
|
19
20
|
validateSecretOverrides,
|
|
20
21
|
validateVariableOverrides,
|
|
21
22
|
validateCredentialOverrides,
|
|
@@ -28,10 +29,10 @@ import {
|
|
|
28
29
|
computeMiddlewareGroupsMeta,
|
|
29
30
|
computePermissionsGroupsMeta,
|
|
30
31
|
computeRequiredSchemas,
|
|
31
|
-
computeSecretBrokers,
|
|
32
32
|
validateNoSecretAliasServices,
|
|
33
33
|
validateSecretUsage,
|
|
34
34
|
computeDiagnostics,
|
|
35
|
+
validateSchemaReferences,
|
|
35
36
|
validateSchemaWiringSeparation,
|
|
36
37
|
validateScenarioServices,
|
|
37
38
|
validateScenarioSteps,
|
|
@@ -41,7 +42,7 @@ import { validateExposedFunctionsGated } from './utils/validate-exposed-function
|
|
|
41
42
|
import { validateTagsResolveToMiddleware } from './utils/validate-tags-resolve-to-middleware.js'
|
|
42
43
|
import { annotateHttpRouteAuth } from './utils/annotate-http-route-auth.js'
|
|
43
44
|
import { generateOpenAPISpec } from './utils/serialize-openapi-json.js'
|
|
44
|
-
import { pikkuState } from '@pikku/core/
|
|
45
|
+
import { pikkuState } from '@pikku/core/ecosystem'
|
|
45
46
|
import { resolveLatestVersions } from './utils/resolve-versions.js'
|
|
46
47
|
import { finalizeWorkflows } from './utils/workflow/graph/finalize-workflows.js'
|
|
47
48
|
import {
|
|
@@ -159,6 +160,10 @@ export function getInitialInspectorState(rootDir: string): InspectorState {
|
|
|
159
160
|
agentsMeta: {},
|
|
160
161
|
files: new Map(),
|
|
161
162
|
},
|
|
163
|
+
scorers: {
|
|
164
|
+
scorersMeta: {},
|
|
165
|
+
files: new Map(),
|
|
166
|
+
},
|
|
162
167
|
cli: {
|
|
163
168
|
meta: {
|
|
164
169
|
programs: {},
|
|
@@ -444,6 +449,12 @@ export const inspect = async (
|
|
|
444
449
|
// Re-load addon schemas (generateAllSchemas replaces state.schemas)
|
|
445
450
|
await loadAddonSchemas(logger, state)
|
|
446
451
|
|
|
452
|
+
// After the addon merge, not before: an addon supplies its own schemas here,
|
|
453
|
+
// and checking earlier would report every one of them as unresolved.
|
|
454
|
+
if (options.schemaConfig) {
|
|
455
|
+
validateSchemaReferences(logger, state)
|
|
456
|
+
}
|
|
457
|
+
|
|
447
458
|
state.manifest.initial = options.manifest ?? null
|
|
448
459
|
const contracts = extractContractsFromMeta(state.functions.meta)
|
|
449
460
|
const baseManifest = state.manifest.initial ?? createEmptyManifest()
|
|
@@ -477,7 +488,6 @@ export const inspect = async (
|
|
|
477
488
|
computeResolvedIOTypes(state)
|
|
478
489
|
computeMiddlewareGroupsMeta(state)
|
|
479
490
|
computePermissionsGroupsMeta(state)
|
|
480
|
-
computeSecretBrokers(state)
|
|
481
491
|
validateNoSecretAliasServices(logger, checker, state)
|
|
482
492
|
validateSecretUsage(logger, state)
|
|
483
493
|
computeDiagnostics(state)
|
|
@@ -504,8 +514,9 @@ export const inspect = async (
|
|
|
504
514
|
)
|
|
505
515
|
}
|
|
506
516
|
|
|
507
|
-
validateAgentModels(logger, state)
|
|
517
|
+
validateAgentModels(logger, state, options.modelAliases)
|
|
508
518
|
validateAgentToolReferences(logger, state, options)
|
|
519
|
+
validateAgentScorerReferences(logger, state)
|
|
509
520
|
validateSecretOverrides(logger, state)
|
|
510
521
|
validateVariableOverrides(logger, state)
|
|
511
522
|
validateCredentialOverrides(logger, state)
|
package/src/types.ts
CHANGED
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
MCPPromptMeta,
|
|
14
14
|
} from '@pikku/core/mcp'
|
|
15
15
|
import type { AIAgentMeta } from '@pikku/core/ai-agent'
|
|
16
|
+
import type { ScorerMeta } from '@pikku/core/ai-scorer'
|
|
16
17
|
import type { CLIMeta } from '@pikku/core/cli'
|
|
17
18
|
import type { CLICommandMeta } from '@pikku/core/cli'
|
|
18
19
|
import type { NodesMeta } from '@pikku/core/node'
|
|
@@ -298,6 +299,8 @@ export type InspectorOptions = Partial<{
|
|
|
298
299
|
* tool's own name, so this is a quality gate rather than a correctness one.
|
|
299
300
|
*/
|
|
300
301
|
strictMeta: boolean
|
|
302
|
+
/** The `models` table from pikku.config.json, alias -> `provider/model`. */
|
|
303
|
+
modelAliases: Record<string, string>
|
|
301
304
|
sourceFile: ts.SourceFile
|
|
302
305
|
/**
|
|
303
306
|
* The project's tsconfig, read for its path mappings only. The inspector's
|
|
@@ -585,6 +588,16 @@ export interface InspectorState {
|
|
|
585
588
|
secretOverrides?: Record<string, string>
|
|
586
589
|
variableOverrides?: Record<string, string>
|
|
587
590
|
credentialOverrides?: Record<string, string>
|
|
591
|
+
/**
|
|
592
|
+
* The app's stated reason for handing this instance the whole
|
|
593
|
+
* `SecretService` instead of one scoped to its declared secrets.
|
|
594
|
+
*/
|
|
595
|
+
globalSecrets?: string
|
|
596
|
+
/**
|
|
597
|
+
* The app's stated reason for handing this instance the whole
|
|
598
|
+
* `CredentialService` instead of one scoped to its declared credentials.
|
|
599
|
+
*/
|
|
600
|
+
globalCredentials?: string
|
|
588
601
|
}
|
|
589
602
|
>
|
|
590
603
|
wireAddonFiles: Set<string>
|
|
@@ -599,6 +612,10 @@ export interface InspectorState {
|
|
|
599
612
|
agentsMeta: AIAgentMeta
|
|
600
613
|
files: Map<string, { path: string; exportedName: string }>
|
|
601
614
|
}
|
|
615
|
+
scorers: {
|
|
616
|
+
scorersMeta: ScorerMeta
|
|
617
|
+
files: Map<string, { path: string; exportedName: string }>
|
|
618
|
+
}
|
|
602
619
|
cli: {
|
|
603
620
|
meta: CLIMeta
|
|
604
621
|
files: Set<string>
|
|
@@ -419,6 +419,14 @@ export function filterInspectorState(
|
|
|
419
419
|
agentsMeta: JSON.parse(JSON.stringify(state.agents?.agentsMeta ?? {})),
|
|
420
420
|
files: new Map(),
|
|
421
421
|
},
|
|
422
|
+
// Scorers carry through every filter. They are named by the agents that
|
|
423
|
+
// survive it, and a deployment that quietly stopped grading would look
|
|
424
|
+
// exactly like one whose scorers all returned nothing.
|
|
425
|
+
scorers: {
|
|
426
|
+
...state.scorers,
|
|
427
|
+
scorersMeta: JSON.parse(JSON.stringify(state.scorers?.scorersMeta ?? {})),
|
|
428
|
+
files: new Map(state.scorers?.files ?? []),
|
|
429
|
+
},
|
|
422
430
|
rpc: {
|
|
423
431
|
...state.rpc,
|
|
424
432
|
internalMeta: { ...state.rpc.internalMeta }, // Clone to avoid mutating original
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
validateRemoteAddonDependencies,
|
|
12
12
|
validateRemoteAddonAuth,
|
|
13
13
|
validateAgentToolReferences,
|
|
14
|
+
validateAgentModels,
|
|
14
15
|
} from './post-process.js'
|
|
15
16
|
import { ErrorCode } from '../error-codes.js'
|
|
16
17
|
import type { InspectorState, InspectorLogger } from '../types.js'
|
|
@@ -713,3 +714,73 @@ describe('validateAgentToolReferences (ref() resolved at build time)', () => {
|
|
|
713
714
|
assert.deepEqual(criticals, [])
|
|
714
715
|
})
|
|
715
716
|
})
|
|
717
|
+
|
|
718
|
+
const makeModelState = (
|
|
719
|
+
models: Record<string, string>
|
|
720
|
+
): Omit<InspectorState, 'typesLookup'> =>
|
|
721
|
+
({
|
|
722
|
+
agents: {
|
|
723
|
+
agentsMeta: Object.fromEntries(
|
|
724
|
+
Object.entries(models).map(([name, model]) => [
|
|
725
|
+
name,
|
|
726
|
+
{ name, model, sourceFile: `src/${name}.agent.ts` },
|
|
727
|
+
])
|
|
728
|
+
),
|
|
729
|
+
},
|
|
730
|
+
}) as unknown as Omit<InspectorState, 'typesLookup'>
|
|
731
|
+
|
|
732
|
+
describe('validateAgentModels (aliases resolve against pikku.config.json)', () => {
|
|
733
|
+
test('a provider-qualified model needs no alias', () => {
|
|
734
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
735
|
+
validateAgentModels(logger, makeModelState({ triage: 'openai/gpt-5-mini' }))
|
|
736
|
+
assert.deepEqual(criticals, [])
|
|
737
|
+
})
|
|
738
|
+
|
|
739
|
+
test('a missing model is still reported', () => {
|
|
740
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
741
|
+
validateAgentModels(logger, makeModelState({ triage: '' }))
|
|
742
|
+
assert.equal(criticals.length, 1)
|
|
743
|
+
assert.equal(criticals[0]!.code, ErrorCode.MISSING_MODEL)
|
|
744
|
+
})
|
|
745
|
+
|
|
746
|
+
test('a bare name that the models table defines is accepted', () => {
|
|
747
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
748
|
+
validateAgentModels(logger, makeModelState({ triage: 'cheap' }), {
|
|
749
|
+
cheap: 'openai/gpt-5-mini',
|
|
750
|
+
})
|
|
751
|
+
assert.deepEqual(criticals, [])
|
|
752
|
+
})
|
|
753
|
+
|
|
754
|
+
test('a bare name with no alias is an error naming the ones that exist', () => {
|
|
755
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
756
|
+
validateAgentModels(logger, makeModelState({ triage: 'exspensive' }), {
|
|
757
|
+
cheap: 'openai/gpt-5-mini',
|
|
758
|
+
expensive: 'openai/gpt-5',
|
|
759
|
+
})
|
|
760
|
+
assert.equal(criticals.length, 1)
|
|
761
|
+
assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
|
|
762
|
+
assert.match(criticals[0]!.message, /cheap, expensive/)
|
|
763
|
+
})
|
|
764
|
+
|
|
765
|
+
test('with no models table a bare name still fails, as it always has', () => {
|
|
766
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
767
|
+
validateAgentModels(logger, makeModelState({ triage: 'gpt-5-mini' }))
|
|
768
|
+
assert.equal(criticals.length, 1)
|
|
769
|
+
assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
|
|
770
|
+
})
|
|
771
|
+
|
|
772
|
+
for (const inherited of ['toString', 'constructor', '__proto__']) {
|
|
773
|
+
test(`a bare '${inherited}' is not an alias just because Object has one`, () => {
|
|
774
|
+
const { logger, criticals } = makeCriticalLogger()
|
|
775
|
+
validateAgentModels(logger, makeModelState({ triage: inherited }), {
|
|
776
|
+
cheap: 'openai/gpt-5-mini',
|
|
777
|
+
})
|
|
778
|
+
assert.equal(
|
|
779
|
+
criticals.length,
|
|
780
|
+
1,
|
|
781
|
+
'an inherited property name must not read as a configured alias'
|
|
782
|
+
)
|
|
783
|
+
assert.equal(criticals[0]!.code, ErrorCode.INVALID_MODEL)
|
|
784
|
+
})
|
|
785
|
+
}
|
|
786
|
+
})
|
|
@@ -15,7 +15,6 @@ import { existsSync, readFileSync } from 'node:fs'
|
|
|
15
15
|
import { join } from 'node:path'
|
|
16
16
|
import { extractTypeKeys } from './type-utils.js'
|
|
17
17
|
import { ErrorCode } from '../error-codes.js'
|
|
18
|
-
import { isSecretBrokerFunction } from './secret-brokers.js'
|
|
19
18
|
import { findSecretAliasServices } from './secret-alias-services.js'
|
|
20
19
|
import { relative } from 'node:path'
|
|
21
20
|
import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
|
|
@@ -689,9 +688,71 @@ export function computeRequiredSchemas(
|
|
|
689
688
|
])
|
|
690
689
|
}
|
|
691
690
|
|
|
691
|
+
/**
|
|
692
|
+
* Every schema a function's contract names has to exist once codegen is done.
|
|
693
|
+
*
|
|
694
|
+
* The way it stops existing is quiet. `generateCustomTypes` hands the schema
|
|
695
|
+
* generator a virtual source file that *imports* each named contract type, so a
|
|
696
|
+
* type that is declared but never exported resolves to nothing and yields no
|
|
697
|
+
* schema — while the function's meta still carries the name. Nothing downstream
|
|
698
|
+
* reads that name until the first call, which fails with a runtime
|
|
699
|
+
* `MissingSchemaError` in a deployment rather than in the build that caused it.
|
|
700
|
+
*
|
|
701
|
+
* Reported rather than critical: the reference is only *guaranteed* to break
|
|
702
|
+
* the functions that are actually invoked, so this blocks CI through
|
|
703
|
+
* `--fail-on-error` without failing a dev server over a function nobody calls.
|
|
704
|
+
*/
|
|
705
|
+
export function validateSchemaReferences(
|
|
706
|
+
logger: InspectorLogger,
|
|
707
|
+
state: InspectorState
|
|
708
|
+
): void {
|
|
709
|
+
const { functions, schemas } = state
|
|
710
|
+
const available = new Set(Object.keys(schemas ?? {}))
|
|
711
|
+
const unresolved: string[] = []
|
|
712
|
+
|
|
713
|
+
const resolve = (typeName: string): string => {
|
|
714
|
+
try {
|
|
715
|
+
return functions.typesMap.getUniqueName(typeName)
|
|
716
|
+
} catch {
|
|
717
|
+
return typeName
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
for (const [funcName, meta] of Object.entries(functions.meta)) {
|
|
722
|
+
for (const [role, declared] of [
|
|
723
|
+
['input', meta.inputs?.[0]],
|
|
724
|
+
['output', meta.outputs?.[0]],
|
|
725
|
+
] as const) {
|
|
726
|
+
if (!declared || PRIMITIVE_TYPES.has(declared)) continue
|
|
727
|
+
const resolved = resolve(declared)
|
|
728
|
+
if (PRIMITIVE_TYPES.has(resolved) || available.has(resolved)) continue
|
|
729
|
+
unresolved.push(`${funcName}.${role} → '${resolved}'`)
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
if (unresolved.length === 0) return
|
|
734
|
+
|
|
735
|
+
logger.diagnostic({
|
|
736
|
+
severity: 'error',
|
|
737
|
+
code: ErrorCode.SCHEMA_REFERENCE_UNRESOLVED,
|
|
738
|
+
message:
|
|
739
|
+
`${unresolved.length} function contract${unresolved.length === 1 ? '' : 's'} ` +
|
|
740
|
+
`name${unresolved.length === 1 ? 's' : ''} a schema that was never generated, so ` +
|
|
741
|
+
`calling ${unresolved.length === 1 ? 'it' : 'them'} fails at runtime with ` +
|
|
742
|
+
`MissingSchemaError: ${unresolved.join(', ')}. ` +
|
|
743
|
+
`The usual cause is a named type used as the function's input/output that ` +
|
|
744
|
+
`is declared but not exported — export it, or inline the type at the call site.`,
|
|
745
|
+
})
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* A model is either concrete (`provider/model`) or an alias, which must appear
|
|
750
|
+
* in the `models` table from pikku.config.json.
|
|
751
|
+
*/
|
|
692
752
|
export function validateAgentModels(
|
|
693
753
|
logger: InspectorLogger,
|
|
694
|
-
state: InspectorState | Omit<InspectorState, 'typesLookup'
|
|
754
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>,
|
|
755
|
+
modelAliases: Record<string, string> = {}
|
|
695
756
|
): void {
|
|
696
757
|
for (const [, meta] of Object.entries(state.agents.agentsMeta)) {
|
|
697
758
|
const model = meta.model
|
|
@@ -702,10 +763,17 @@ export function validateAgentModels(
|
|
|
702
763
|
)
|
|
703
764
|
continue
|
|
704
765
|
}
|
|
705
|
-
|
|
766
|
+
// `hasOwn`, not a truthy lookup: every object literal inherits `toString`,
|
|
767
|
+
// `constructor` and `__proto__`, so a bare model named after one of them
|
|
768
|
+
// would read as a configured alias and pass a check nothing else repeats.
|
|
769
|
+
if (!model.includes('/') && !Object.hasOwn(modelAliases, model)) {
|
|
770
|
+
const known = Object.keys(modelAliases).sort()
|
|
706
771
|
logger.critical(
|
|
707
772
|
ErrorCode.INVALID_MODEL,
|
|
708
|
-
`AI agent '${meta.name}' uses model '${model}', which
|
|
773
|
+
`AI agent '${meta.name}' uses model '${model}', which is neither provider-qualified as '<provider>/<model>' (e.g. 'openai/gpt-4') nor an alias in the "models" table of pikku.config.json. ` +
|
|
774
|
+
(known.length
|
|
775
|
+
? `Known aliases: ${known.join(', ')}.`
|
|
776
|
+
: `No "models" aliases are configured.`)
|
|
709
777
|
)
|
|
710
778
|
}
|
|
711
779
|
}
|
|
@@ -925,15 +993,6 @@ export function validateSchemaWiringSeparation(
|
|
|
925
993
|
}
|
|
926
994
|
}
|
|
927
995
|
|
|
928
|
-
/** Marks the functions that keep the full `SecretService`. */
|
|
929
|
-
export function computeSecretBrokers(state: InspectorState): void {
|
|
930
|
-
for (const [id, meta] of Object.entries(state.functions.meta)) {
|
|
931
|
-
if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
|
|
932
|
-
meta.secretBroker = true
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
}
|
|
936
|
-
|
|
937
996
|
/**
|
|
938
997
|
* Refuses a singleton service that is a `SecretService` under another name.
|
|
939
998
|
*
|
|
@@ -972,9 +1031,6 @@ export function validateNoSecretAliasServices(
|
|
|
972
1031
|
}
|
|
973
1032
|
|
|
974
1033
|
for (const [id, meta] of Object.entries(state.functions.meta)) {
|
|
975
|
-
if (meta.secretBroker) {
|
|
976
|
-
continue
|
|
977
|
-
}
|
|
978
1034
|
report('Function', id, meta.services?.services ?? [])
|
|
979
1035
|
}
|
|
980
1036
|
for (const [id, def] of Object.entries(state.permissions.definitions)) {
|
|
@@ -1227,6 +1283,32 @@ export function validateAgentToolReferences(
|
|
|
1227
1283
|
}
|
|
1228
1284
|
}
|
|
1229
1285
|
|
|
1286
|
+
/**
|
|
1287
|
+
* Validates that every scorer an agent asks to be graded by exists.
|
|
1288
|
+
*
|
|
1289
|
+
* The runtime warns and skips an unresolvable name rather than failing a run
|
|
1290
|
+
* that has already answered the user — which means a typo would otherwise cost
|
|
1291
|
+
* nothing visible and simply grade nothing, forever.
|
|
1292
|
+
*/
|
|
1293
|
+
export function validateAgentScorerReferences(
|
|
1294
|
+
logger: InspectorLogger,
|
|
1295
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
1296
|
+
): void {
|
|
1297
|
+
const known = Object.keys(state.scorers.scorersMeta)
|
|
1298
|
+
|
|
1299
|
+
for (const [agentKey, agent] of Object.entries(state.agents.agentsMeta)) {
|
|
1300
|
+
const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
|
|
1301
|
+
for (const scorer of agent.scorers ?? []) {
|
|
1302
|
+
if (known.includes(scorer)) continue
|
|
1303
|
+
logger.critical(
|
|
1304
|
+
ErrorCode.AGENT_SCORER_NOT_FOUND,
|
|
1305
|
+
`AI agent '${agentKey}'${where} asks to be graded by scorer '${scorer}', which is not declared in this project. ` +
|
|
1306
|
+
`Declared scorers: ${known.join(', ') || 'none'}.`
|
|
1307
|
+
)
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1230
1312
|
/**
|
|
1231
1313
|
* Validates that every scope granted by a system role is declared via
|
|
1232
1314
|
* `defineScope`. Runs after all visitors, so declaration order does not matter.
|
|
@@ -170,6 +170,8 @@ export interface SerializableInspectorState {
|
|
|
170
170
|
secretOverrides?: Record<string, string>
|
|
171
171
|
variableOverrides?: Record<string, string>
|
|
172
172
|
credentialOverrides?: Record<string, string>
|
|
173
|
+
globalSecrets?: string
|
|
174
|
+
globalCredentials?: string
|
|
173
175
|
},
|
|
174
176
|
]
|
|
175
177
|
>
|
|
@@ -185,6 +187,10 @@ export interface SerializableInspectorState {
|
|
|
185
187
|
agentsMeta: InspectorState['agents']['agentsMeta']
|
|
186
188
|
files: [string, { path: string; exportedName: string }][]
|
|
187
189
|
}
|
|
190
|
+
scorers: {
|
|
191
|
+
scorersMeta: InspectorState['scorers']['scorersMeta']
|
|
192
|
+
files: [string, { path: string; exportedName: string }][]
|
|
193
|
+
}
|
|
188
194
|
cli: {
|
|
189
195
|
meta: InspectorState['cli']['meta']
|
|
190
196
|
files: string[]
|
|
@@ -409,6 +415,10 @@ export function serializeInspectorState(
|
|
|
409
415
|
agentsMeta: state.agents?.agentsMeta ?? {},
|
|
410
416
|
files: Array.from(state.agents?.files?.entries() ?? []),
|
|
411
417
|
},
|
|
418
|
+
scorers: {
|
|
419
|
+
scorersMeta: state.scorers?.scorersMeta ?? {},
|
|
420
|
+
files: Array.from(state.scorers?.files?.entries() ?? []),
|
|
421
|
+
},
|
|
412
422
|
cli: {
|
|
413
423
|
meta: state.cli.meta,
|
|
414
424
|
files: Array.from(state.cli.files),
|
|
@@ -613,6 +623,10 @@ export function deserializeInspectorState(
|
|
|
613
623
|
agentsMeta: data.agents?.agentsMeta || {},
|
|
614
624
|
files: new Map(data.agents?.files || []),
|
|
615
625
|
},
|
|
626
|
+
scorers: {
|
|
627
|
+
scorersMeta: data.scorers?.scorersMeta || {},
|
|
628
|
+
files: new Map(data.scorers?.files || []),
|
|
629
|
+
},
|
|
616
630
|
cli: {
|
|
617
631
|
meta: data.cli.meta,
|
|
618
632
|
files: new Set(data.cli.files),
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { describe, test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import * as ts from 'typescript'
|
|
4
|
+
import { extractDSLWorkflow } from './extract-dsl-workflow.js'
|
|
5
|
+
import type { ApprovalStepMeta } from '@pikku/core/workflow'
|
|
6
|
+
|
|
7
|
+
const checker = {
|
|
8
|
+
getSymbolAtLocation: () => undefined,
|
|
9
|
+
getAliasedSymbol: () => undefined,
|
|
10
|
+
getShorthandAssignmentValueSymbol: () => undefined,
|
|
11
|
+
getTypeAtLocation: () => undefined,
|
|
12
|
+
} as unknown as ts.TypeChecker
|
|
13
|
+
|
|
14
|
+
/** Extracts the single approval step from a workflow body. */
|
|
15
|
+
const approvalStepFor = (options: string): ApprovalStepMeta => {
|
|
16
|
+
const file = ts.createSourceFile(
|
|
17
|
+
'wf.ts',
|
|
18
|
+
`const wf = pikkuWorkflowFunc({
|
|
19
|
+
func: async (services, data, { workflow }) => {
|
|
20
|
+
await workflow.approval('Release funds', ${options})
|
|
21
|
+
},
|
|
22
|
+
})`,
|
|
23
|
+
ts.ScriptTarget.Latest,
|
|
24
|
+
true
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
let node: ts.Node | undefined
|
|
28
|
+
const visit = (n: ts.Node) => {
|
|
29
|
+
if (ts.isCallExpression(n) && n.expression.getText() === 'pikkuWorkflowFunc') {
|
|
30
|
+
node = n
|
|
31
|
+
}
|
|
32
|
+
ts.forEachChild(n, visit)
|
|
33
|
+
}
|
|
34
|
+
visit(file)
|
|
35
|
+
assert.ok(node, 'failed to find the workflow definition')
|
|
36
|
+
|
|
37
|
+
const result = extractDSLWorkflow(node, checker)
|
|
38
|
+
assert.equal(result.status, 'ok', JSON.stringify(result))
|
|
39
|
+
const step = result.steps?.find(
|
|
40
|
+
(s): s is ApprovalStepMeta => s.type === 'approval'
|
|
41
|
+
)
|
|
42
|
+
assert.ok(step, 'no approval step was extracted')
|
|
43
|
+
return step
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
describe('an approval gate carries its policy into the graph', () => {
|
|
47
|
+
test('approvers and approverScope survive extraction', () => {
|
|
48
|
+
const step = approvalStepFor(
|
|
49
|
+
`{ schema: DecisionSchema, approvers: 'not-initiator', approverScope: 'payments:approve' }`
|
|
50
|
+
)
|
|
51
|
+
assert.equal(step.approvers, 'not-initiator')
|
|
52
|
+
assert.equal(step.approverScope, 'payments:approve')
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
test('expiry still survives alongside them', () => {
|
|
56
|
+
const step = approvalStepFor(
|
|
57
|
+
`{ schema: DecisionSchema, expiry: '3d', approvers: 'owner' }`
|
|
58
|
+
)
|
|
59
|
+
assert.equal(step.expiry, '3d')
|
|
60
|
+
assert.equal(step.approvers, 'owner')
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('a gate declaring no policy records none', () => {
|
|
64
|
+
const step = approvalStepFor(`{ schema: DecisionSchema }`)
|
|
65
|
+
assert.equal(step.approvers, undefined)
|
|
66
|
+
assert.equal(step.approverScope, undefined)
|
|
67
|
+
})
|
|
68
|
+
})
|
|
@@ -1051,14 +1051,15 @@ function extractApprovalStep(
|
|
|
1051
1051
|
step.outputVar = outputVar
|
|
1052
1052
|
}
|
|
1053
1053
|
// The `schema` option is a runtime value validated inside the workflow body,
|
|
1054
|
-
// so it is deliberately not serialized here — only
|
|
1055
|
-
// and planned-step ladder need in order to describe the gate.
|
|
1054
|
+
// so it is deliberately not serialized here — only the literal options the
|
|
1055
|
+
// graph and planned-step ladder need in order to describe the gate.
|
|
1056
1056
|
const options = args[1]
|
|
1057
1057
|
if (options && ts.isObjectLiteralExpression(options)) {
|
|
1058
1058
|
for (const prop of options.properties) {
|
|
1059
|
+
if (!ts.isPropertyAssignment(prop)) continue
|
|
1060
|
+
const name = prop.name.getText()
|
|
1059
1061
|
if (
|
|
1060
|
-
|
|
1061
|
-
prop.name.getText() === 'expiry' &&
|
|
1062
|
+
name === 'expiry' &&
|
|
1062
1063
|
(ts.isStringLiteral(prop.initializer) ||
|
|
1063
1064
|
ts.isNumericLiteral(prop.initializer))
|
|
1064
1065
|
) {
|
|
@@ -1066,6 +1067,13 @@ function extractApprovalStep(
|
|
|
1066
1067
|
? Number(prop.initializer.text)
|
|
1067
1068
|
: prop.initializer.text
|
|
1068
1069
|
}
|
|
1070
|
+
if (name === 'approvers' && ts.isStringLiteral(prop.initializer)) {
|
|
1071
|
+
step.approvers = prop.initializer
|
|
1072
|
+
.text as ApprovalStepMeta['approvers']
|
|
1073
|
+
}
|
|
1074
|
+
if (name === 'approverScope' && ts.isStringLiteral(prop.initializer)) {
|
|
1075
|
+
step.approverScope = prop.initializer.text
|
|
1076
|
+
}
|
|
1069
1077
|
}
|
|
1070
1078
|
}
|
|
1071
1079
|
return step
|