@pikku/inspector 0.12.50 → 0.12.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +743 -0
- package/OPTIMIZATION-PLAN.md +9 -3
- package/dist/add/add-auth.d.ts +1 -1
- package/dist/add/add-auth.js +1 -1
- package/dist/add/add-cli.js +7 -0
- package/dist/add/add-credential.js +1 -1
- package/dist/add/add-functions.js +199 -8
- package/dist/add/add-http-route.js +16 -6
- package/dist/add/add-middleware.js +46 -3
- package/dist/add/add-personas.d.ts +13 -0
- package/dist/add/add-personas.js +216 -0
- package/dist/add/add-scope.js +1 -1
- package/dist/add/add-secret.js +1 -1
- package/dist/add/add-system-role.d.ts +2 -0
- package/dist/add/add-system-role.js +102 -0
- package/dist/add/add-variable.js +1 -1
- package/dist/add/add-wire-addon.js +31 -0
- package/dist/add/add-workflow.js +12 -1
- package/dist/error-codes.d.ts +17 -1
- package/dist/error-codes.js +23 -0
- package/dist/index.d.ts +1 -1
- package/dist/inspector.js +33 -1
- package/dist/types.d.ts +55 -2
- package/dist/utils/annotate-http-route-auth.d.ts +22 -0
- package/dist/utils/annotate-http-route-auth.js +48 -0
- package/dist/utils/check-secret-sinks.d.ts +31 -0
- package/dist/utils/check-secret-sinks.js +117 -0
- package/dist/utils/extract-function-name.js +13 -4
- package/dist/utils/extract-function-node.js +18 -1
- package/dist/utils/extract-secret-usage.d.ts +9 -0
- package/dist/utils/extract-secret-usage.js +61 -0
- package/dist/utils/literal-properties.d.ts +22 -0
- package/dist/utils/literal-properties.js +69 -0
- package/dist/utils/load-addon-functions-meta.js +26 -1
- package/dist/utils/permissions.js +8 -0
- package/dist/utils/post-process.d.ts +68 -3
- package/dist/utils/post-process.js +246 -13
- package/dist/utils/secret-alias-services.d.ts +8 -0
- package/dist/utils/secret-alias-services.js +45 -0
- package/dist/utils/secret-brokers.d.ts +2 -0
- package/dist/utils/secret-brokers.js +9 -0
- package/dist/utils/serialize-inspector-state.d.ts +10 -0
- package/dist/utils/serialize-inspector-state.js +18 -0
- package/dist/utils/serialize-middleware-groups-meta.js +3 -0
- package/dist/utils/single-declaration.d.ts +32 -0
- package/dist/utils/single-declaration.js +43 -0
- package/dist/utils/validate-exposed-functions-gated.d.ts +14 -0
- package/dist/utils/validate-exposed-functions-gated.js +93 -0
- package/dist/utils/validate-tags-resolve-to-middleware.d.ts +20 -0
- package/dist/utils/validate-tags-resolve-to-middleware.js +41 -0
- package/dist/utils/workflow/dsl/extract-dsl-workflow.d.ts +6 -0
- package/dist/utils/workflow/dsl/extract-dsl-workflow.js +48 -1
- package/dist/utils/workflow/dsl/patterns.d.ts +9 -4
- package/dist/utils/workflow/dsl/patterns.js +26 -4
- package/dist/visit.js +4 -0
- package/package.json +3 -3
- package/src/add/add-auth.test.ts +5 -3
- package/src/add/add-auth.ts +1 -1
- package/src/add/add-cli.ts +8 -0
- package/src/add/add-credential.ts +1 -1
- package/src/add/add-functions.test.ts +263 -1
- package/src/add/add-functions.ts +273 -11
- package/src/add/add-http-route.ts +20 -7
- package/src/add/add-middleware.test.ts +196 -0
- package/src/add/add-middleware.ts +64 -4
- package/src/add/add-personas.test.ts +382 -0
- package/src/add/add-personas.ts +285 -0
- package/src/add/add-scenario-step.test.ts +87 -5
- package/src/add/add-scope.test.ts +103 -30
- package/src/add/add-scope.ts +1 -1
- package/src/add/add-secret.ts +1 -1
- package/src/add/add-subject-scenario-steps.test.ts +161 -0
- package/src/add/add-system-role.test.ts +297 -0
- package/src/add/add-system-role.ts +164 -0
- package/src/add/add-variable.ts +1 -1
- package/src/add/add-wire-addon.test.ts +111 -0
- package/src/add/add-wire-addon.ts +29 -0
- package/src/add/add-workflow.ts +21 -1
- package/src/add/addon-bans.test.ts +4 -4
- package/src/add/allow-escape-hatches.test.ts +144 -0
- package/src/add/audit-meta.test.ts +75 -0
- package/src/add/config-value-metadata.test.ts +11 -11
- package/src/add/secret-sink-check.test.ts +211 -0
- package/src/add/wire-name-literal.test.ts +2 -2
- package/src/error-codes.ts +26 -0
- package/src/index.ts +4 -1
- package/src/inspector.ts +39 -0
- package/src/types.ts +56 -2
- package/src/utils/annotate-http-route-auth.test.ts +163 -0
- package/src/utils/annotate-http-route-auth.ts +52 -0
- package/src/utils/check-secret-sinks.ts +145 -0
- package/src/utils/extract-function-name.ts +14 -4
- package/src/utils/extract-function-node.ts +18 -6
- package/src/utils/extract-secret-usage.test.ts +81 -0
- package/src/utils/extract-secret-usage.ts +73 -0
- package/src/utils/literal-properties.ts +97 -0
- package/src/utils/load-addon-functions-meta.ts +32 -2
- package/src/utils/permissions-shorthand.test.ts +79 -0
- package/src/utils/permissions.ts +7 -0
- package/src/utils/post-process.test.ts +133 -0
- package/src/utils/post-process.ts +313 -12
- package/src/utils/secret-alias-services.test.ts +146 -0
- package/src/utils/secret-alias-services.ts +58 -0
- package/src/utils/secret-brokers.test.ts +53 -0
- package/src/utils/secret-brokers.ts +11 -0
- package/src/utils/serialize-inspector-state.ts +28 -0
- package/src/utils/serialize-middleware-groups-meta.ts +3 -0
- package/src/utils/single-declaration.ts +57 -0
- package/src/utils/validate-exposed-functions-gated.test.ts +230 -0
- package/src/utils/validate-exposed-functions-gated.ts +103 -0
- package/src/utils/validate-tags-resolve-to-middleware.test.ts +105 -0
- package/src/utils/validate-tags-resolve-to-middleware.ts +48 -0
- package/src/utils/workflow/dsl/extract-dsl-workflow.ts +57 -0
- package/src/utils/workflow/dsl/patterns.ts +33 -4
- package/src/visit.ts +4 -0
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -10,10 +10,14 @@ import type {
|
|
|
10
10
|
MiddlewareMetadata,
|
|
11
11
|
PermissionMetadata,
|
|
12
12
|
} from '@pikku/core'
|
|
13
|
+
import type ts from 'typescript'
|
|
13
14
|
import { existsSync, readFileSync } from 'node:fs'
|
|
14
15
|
import { join } from 'node:path'
|
|
15
16
|
import { extractTypeKeys } from './type-utils.js'
|
|
16
17
|
import { ErrorCode } from '../error-codes.js'
|
|
18
|
+
import { isSecretBrokerFunction } from './secret-brokers.js'
|
|
19
|
+
import { findSecretAliasServices } from './secret-alias-services.js'
|
|
20
|
+
import { relative } from 'node:path'
|
|
17
21
|
import { AUTH_HANDLER_FUNC_ID } from '../add/add-auth.js'
|
|
18
22
|
import { flattenScopeDefinitions } from '@pikku/core/scope'
|
|
19
23
|
import type { WorkflowStepMeta } from '@pikku/core/workflow'
|
|
@@ -611,6 +615,9 @@ const serializeGroupMap = (
|
|
|
611
615
|
count: meta.count,
|
|
612
616
|
instanceIds: meta.instanceIds,
|
|
613
617
|
isFactory: meta.isFactory,
|
|
618
|
+
...(meta.additionalRegistrations && {
|
|
619
|
+
additionalRegistrations: meta.additionalRegistrations,
|
|
620
|
+
}),
|
|
614
621
|
}
|
|
615
622
|
}
|
|
616
623
|
return result
|
|
@@ -745,8 +752,11 @@ export function validateScenarioServices(
|
|
|
745
752
|
*
|
|
746
753
|
* - the step target must be a static string literal (PKU678) — otherwise it
|
|
747
754
|
* can't be bundled, typed or drawn;
|
|
748
|
-
* - a `browser
|
|
749
|
-
* context signs in as that persona.
|
|
755
|
+
* - a step declaring a `browser` binding must be given an actor (PKU677), since
|
|
756
|
+
* the browser context signs in as that persona. It is checked statically even
|
|
757
|
+
* though the binding only runs on a browser run: a step that can never be
|
|
758
|
+
* driven by a human is a latent failure, not a passing default run.
|
|
759
|
+
* - a scenario written as a step ladder must assert something (PKU680).
|
|
750
760
|
*
|
|
751
761
|
* It also marks the step's function as used, so `scenario run` boots the
|
|
752
762
|
* singletons the step actually needs.
|
|
@@ -755,9 +765,12 @@ export function validateScenarioSteps(
|
|
|
755
765
|
logger: InspectorLogger,
|
|
756
766
|
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
757
767
|
): void {
|
|
768
|
+
const phasesSeen = new Set<string>()
|
|
769
|
+
|
|
758
770
|
const visit = (workflowName: string, steps: WorkflowStepMeta[]): void => {
|
|
759
771
|
for (const step of steps) {
|
|
760
772
|
if (step.type === 'scenarioStep') {
|
|
773
|
+
phasesSeen.add(step.phase)
|
|
761
774
|
if (step.stepFunc === DYNAMIC_SCENARIO_STEP_TARGET) {
|
|
762
775
|
logger.critical(
|
|
763
776
|
ErrorCode.SCENARIO_STEP_TARGET_NOT_STATIC,
|
|
@@ -769,10 +782,13 @@ export function validateScenarioSteps(
|
|
|
769
782
|
}
|
|
770
783
|
state.serviceAggregation.usedFunctions.add(step.stepFunc)
|
|
771
784
|
const stepMeta = state.functions.meta[step.stepFunc]
|
|
772
|
-
if (
|
|
785
|
+
if (
|
|
786
|
+
stepMeta?.scenarioStepSurfaces?.includes('browser') &&
|
|
787
|
+
!step.actor
|
|
788
|
+
) {
|
|
773
789
|
logger.critical(
|
|
774
790
|
ErrorCode.SCENARIO_BROWSER_STEP_NEEDS_ACTOR,
|
|
775
|
-
`Scenario '${workflowName}' calls
|
|
791
|
+
`Scenario '${workflowName}' calls step '${step.stepFunc}', which declares a browser binding, without an actor. ` +
|
|
776
792
|
`Pass { actor: actors.<name> } so the browser context signs in as that persona.`
|
|
777
793
|
)
|
|
778
794
|
}
|
|
@@ -795,7 +811,23 @@ export function validateScenarioSteps(
|
|
|
795
811
|
}
|
|
796
812
|
|
|
797
813
|
for (const [workflowName, meta] of Object.entries(state.workflows.meta)) {
|
|
814
|
+
phasesSeen.clear()
|
|
798
815
|
visit(workflowName, meta.steps ?? [])
|
|
816
|
+
// A scenario written as a step ladder but with no `then` proves only that
|
|
817
|
+
// nothing threw. It is also invisible to witness coverage — it contributes
|
|
818
|
+
// 0/0, so it can never lower the number — which makes an assertion-free
|
|
819
|
+
// flow the cheapest way to make a suite look better covered than it is.
|
|
820
|
+
// `asserts` is the escape for a flow whose witness is an expectation helper
|
|
821
|
+
// instead: those are inline steps and leave no phase behind, but a recorded
|
|
822
|
+
// service call or a refusal is an assertion all the same.
|
|
823
|
+
if (phasesSeen.size > 0 && !phasesSeen.has('then') && !meta.asserts) {
|
|
824
|
+
logger.critical(
|
|
825
|
+
ErrorCode.SCENARIO_HAS_NO_ASSERTION,
|
|
826
|
+
`Scenario '${workflowName}' has ${[...phasesSeen].join('/')} steps but never asserts. ` +
|
|
827
|
+
`Add a scenario.then(...) naming what the actor should now see — a flow with no 'then' ` +
|
|
828
|
+
`only proves nothing threw, and contributes nothing to witness coverage.`
|
|
829
|
+
)
|
|
830
|
+
}
|
|
799
831
|
}
|
|
800
832
|
}
|
|
801
833
|
|
|
@@ -863,13 +895,18 @@ export function validateSchemaWiringSeparation(
|
|
|
863
895
|
wiringFiles.add(file)
|
|
864
896
|
}
|
|
865
897
|
|
|
866
|
-
// Middleware wirings (addHTTPMiddleware calls)
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
898
|
+
// Middleware wirings (addHTTPMiddleware calls). A group can be registered
|
|
899
|
+
// from more than one file, and every one of them is a wiring file.
|
|
900
|
+
const addGroupFiles = (groups: Map<string, MiddlewareGroupMeta>) => {
|
|
901
|
+
for (const meta of groups.values()) {
|
|
902
|
+
wiringFiles.add(meta.sourceFile)
|
|
903
|
+
for (const registration of meta.additionalRegistrations ?? []) {
|
|
904
|
+
wiringFiles.add(registration.sourceFile)
|
|
905
|
+
}
|
|
906
|
+
}
|
|
872
907
|
}
|
|
908
|
+
addGroupFiles(state.http.routeMiddleware)
|
|
909
|
+
addGroupFiles(state.middleware.tagMiddleware)
|
|
873
910
|
|
|
874
911
|
// Check for overlap
|
|
875
912
|
for (const file of schemaFiles) {
|
|
@@ -888,6 +925,101 @@ export function validateSchemaWiringSeparation(
|
|
|
888
925
|
}
|
|
889
926
|
}
|
|
890
927
|
|
|
928
|
+
/** Marks the functions that keep the full `SecretService`. */
|
|
929
|
+
export function computeSecretBrokers(state: InspectorState): void {
|
|
930
|
+
for (const [id, meta] of Object.entries(state.functions.meta)) {
|
|
931
|
+
if (isSecretBrokerFunction(meta.pikkuFuncId ?? id)) {
|
|
932
|
+
meta.secretBroker = true
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
/**
|
|
938
|
+
* Refuses a singleton service that is a `SecretService` under another name.
|
|
939
|
+
*
|
|
940
|
+
* `secrets` is omitted from every function-facing services type, but
|
|
941
|
+
* `pikkuServices(async ({ secrets }) => ({ cfg: secrets }))` re-exposes it under
|
|
942
|
+
* a name the omit does not cover. A name-based rule misses this; the type does
|
|
943
|
+
* not.
|
|
944
|
+
*/
|
|
945
|
+
export function validateNoSecretAliasServices(
|
|
946
|
+
logger: InspectorLogger,
|
|
947
|
+
checker: ts.TypeChecker,
|
|
948
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
949
|
+
): void {
|
|
950
|
+
if (!('typesLookup' in state)) {
|
|
951
|
+
return
|
|
952
|
+
}
|
|
953
|
+
const singletonServicesTypes = state.typesLookup.get('SingletonServices')
|
|
954
|
+
if (!singletonServicesTypes?.length) {
|
|
955
|
+
return
|
|
956
|
+
}
|
|
957
|
+
const aliases = findSecretAliasServices(singletonServicesTypes[0]!, checker)
|
|
958
|
+
if (aliases.length === 0) {
|
|
959
|
+
return
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
const report = (kind: string, id: string, services: string[]) => {
|
|
963
|
+
const used = services.filter((service) => aliases.includes(service))
|
|
964
|
+
if (used.length === 0) {
|
|
965
|
+
return
|
|
966
|
+
}
|
|
967
|
+
logger.critical(
|
|
968
|
+
ErrorCode.SECRET_SERVICE_ALIASED,
|
|
969
|
+
`${kind} '${id}' receives ${used.map((s) => `'${s}'`).join(', ')}, which ${used.length === 1 ? 'is' : 'are'} a SecretService under another name. ` +
|
|
970
|
+
`SecretService is confined to pikkuServices, pikkuWireServices, addon service factories and middleware — give a service the secret value when you construct it and expose only what the function needs.`
|
|
971
|
+
)
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
for (const [id, meta] of Object.entries(state.functions.meta)) {
|
|
975
|
+
if (meta.secretBroker) {
|
|
976
|
+
continue
|
|
977
|
+
}
|
|
978
|
+
report('Function', id, meta.services?.services ?? [])
|
|
979
|
+
}
|
|
980
|
+
for (const [id, def] of Object.entries(state.permissions.definitions)) {
|
|
981
|
+
report('Permission', id, def.services?.services ?? [])
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* Reports secret reads the catalogue does not account for.
|
|
987
|
+
*
|
|
988
|
+
* `getSecret('X')` with no matching `defineSecret` is a value that will be missing
|
|
989
|
+
* at first request rather than at deploy; a non-literal key is a read the
|
|
990
|
+
* manifest cannot cover, so a per-unit scope cannot be narrowed around it.
|
|
991
|
+
*/
|
|
992
|
+
export function validateSecretUsage(
|
|
993
|
+
logger: InspectorLogger,
|
|
994
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
995
|
+
): void {
|
|
996
|
+
const declared = new Set(state.secrets.definitions.map((d) => d.secretId))
|
|
997
|
+
for (const definition of state.secrets.definitions) {
|
|
998
|
+
if (definition.oauth2?.tokenSecretId) {
|
|
999
|
+
declared.add(definition.oauth2.tokenSecretId)
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
for (const [file, usage] of state.secrets.usage) {
|
|
1004
|
+
const relativeFile = relative(state.rootDir, file)
|
|
1005
|
+
const undeclared = usage.keys.filter((key) => !declared.has(key))
|
|
1006
|
+
if (undeclared.length > 0) {
|
|
1007
|
+
logger.diagnostic({
|
|
1008
|
+
severity: 'warn',
|
|
1009
|
+
code: ErrorCode.SECRET_NOT_DECLARED,
|
|
1010
|
+
message: `${relativeFile} reads ${undeclared.map((k) => `'${k}'`).join(', ')}, which no defineSecret declares. Declare it so it is validated at deploy time and shown to whoever has to provision it.`,
|
|
1011
|
+
})
|
|
1012
|
+
}
|
|
1013
|
+
if (usage.dynamic.length > 0) {
|
|
1014
|
+
logger.diagnostic({
|
|
1015
|
+
severity: 'warn',
|
|
1016
|
+
code: ErrorCode.SECRET_KEY_NOT_STATIC,
|
|
1017
|
+
message: `${relativeFile} reads a secret with a non-literal key (${usage.dynamic.join(', ')}). The deployment cannot narrow its secret scope around a key it cannot see.`,
|
|
1018
|
+
})
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
|
|
891
1023
|
export function computeDiagnostics(state: InspectorState): void {
|
|
892
1024
|
const diagnostics: InspectorDiagnostic[] = []
|
|
893
1025
|
|
|
@@ -969,7 +1101,7 @@ export function computeDiagnostics(state: InspectorState): void {
|
|
|
969
1101
|
|
|
970
1102
|
/**
|
|
971
1103
|
* Validates that every scope referenced by a function is declared via
|
|
972
|
-
* `
|
|
1104
|
+
* `defineScope`. Runs after all visitors, so declaration order does not matter.
|
|
973
1105
|
*
|
|
974
1106
|
* A `*` suffix is a wildcard requirement: `admin:*` requires the `admin` scope
|
|
975
1107
|
* to be declared, and grants its whole subtree.
|
|
@@ -1001,7 +1133,176 @@ export function validateScopeReferences(
|
|
|
1001
1133
|
const available = Array.from(declared)
|
|
1002
1134
|
logger.critical(
|
|
1003
1135
|
ErrorCode.INVALID_VALUE,
|
|
1004
|
-
`Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with
|
|
1136
|
+
`Function '${funcName}' requires scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
|
|
1137
|
+
)
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
/**
|
|
1144
|
+
* Resolves every `tools:` entry on every agent and reports the ones that do not
|
|
1145
|
+
* land on a real function, or that land on one with nothing to tell the model.
|
|
1146
|
+
*
|
|
1147
|
+
* `ref('todos:listTodos')` is a bare string to the inspector — `ref` is not
|
|
1148
|
+
* resolved, only unwrapped — so until now a reference to a function that never
|
|
1149
|
+
* existed generated cleanly and failed at agent-run time. Addon namespaces come
|
|
1150
|
+
* from `wireAddon`, and each addon's generated metadata is read off disk, which
|
|
1151
|
+
* is the only view of it this project has.
|
|
1152
|
+
*
|
|
1153
|
+
* The description check is separate and opt-in (`--strict-meta`), because the
|
|
1154
|
+
* description is what the model is told the tool does: without one, the tool is
|
|
1155
|
+
* offered to it under its own name (`ai-agent-prepare.ts`), which is a silent
|
|
1156
|
+
* quality loss rather than a failure. A title deliberately does not satisfy it —
|
|
1157
|
+
* a title labels a tool in a UI, it does not tell a model when to reach for it.
|
|
1158
|
+
*/
|
|
1159
|
+
export function validateAgentToolReferences(
|
|
1160
|
+
logger: InspectorLogger,
|
|
1161
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>,
|
|
1162
|
+
options: InspectorOptions = {}
|
|
1163
|
+
): void {
|
|
1164
|
+
const agents = Object.entries(state.agents.agentsMeta)
|
|
1165
|
+
if (agents.length === 0) return
|
|
1166
|
+
|
|
1167
|
+
const { wireAddonDeclarations } = state.rpc
|
|
1168
|
+
|
|
1169
|
+
for (const [agentKey, agent] of agents) {
|
|
1170
|
+
const where = agent.sourceFile ? ` (${agent.sourceFile})` : ''
|
|
1171
|
+
|
|
1172
|
+
for (const tool of agent.tools ?? []) {
|
|
1173
|
+
// A workflow reference is resolved against the workflow map, not here.
|
|
1174
|
+
if (tool.startsWith('workflow:')) continue
|
|
1175
|
+
|
|
1176
|
+
const separator = tool.indexOf(':')
|
|
1177
|
+
let meta: { description?: string; title?: string } | undefined
|
|
1178
|
+
|
|
1179
|
+
if (separator === -1) {
|
|
1180
|
+
const funcId = state.rpc.internalMeta[tool] ?? tool
|
|
1181
|
+
meta = state.functions.meta[funcId]
|
|
1182
|
+
if (!meta) {
|
|
1183
|
+
logger.critical(
|
|
1184
|
+
ErrorCode.AGENT_TOOL_NOT_FOUND,
|
|
1185
|
+
`AI agent '${agentKey}'${where} references tool '${tool}', which is not a function in this project.`
|
|
1186
|
+
)
|
|
1187
|
+
continue
|
|
1188
|
+
}
|
|
1189
|
+
} else {
|
|
1190
|
+
const namespace = tool.slice(0, separator)
|
|
1191
|
+
const funcName = tool.slice(separator + 1)
|
|
1192
|
+
const addon = wireAddonDeclarations?.get(namespace)
|
|
1193
|
+
if (!addon) {
|
|
1194
|
+
const known = Array.from(wireAddonDeclarations?.keys() ?? [])
|
|
1195
|
+
logger.critical(
|
|
1196
|
+
ErrorCode.AGENT_TOOL_UNKNOWN_NAMESPACE,
|
|
1197
|
+
`AI agent '${agentKey}'${where} references tool '${tool}', but no addon is wired under the namespace '${namespace}'. ` +
|
|
1198
|
+
`Wired namespaces: ${known.join(', ') || 'none'}.`
|
|
1199
|
+
)
|
|
1200
|
+
continue
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
// An addon that has not been built yet contributed no metadata, which
|
|
1204
|
+
// is not the same as one whose function is missing.
|
|
1205
|
+
const addonMeta = state.addonFunctions[namespace]
|
|
1206
|
+
if (!addonMeta) continue
|
|
1207
|
+
|
|
1208
|
+
meta = addonMeta[funcName]
|
|
1209
|
+
if (!meta) {
|
|
1210
|
+
logger.critical(
|
|
1211
|
+
ErrorCode.AGENT_TOOL_NOT_FOUND,
|
|
1212
|
+
`AI agent '${agentKey}'${where} references tool '${tool}', but addon '${namespace}' ('${addon.package}') exposes no function '${funcName}'.`
|
|
1213
|
+
)
|
|
1214
|
+
continue
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
if (options.strictMeta && !meta.description) {
|
|
1219
|
+
logger.critical(
|
|
1220
|
+
ErrorCode.AGENT_TOOL_MISSING_DESCRIPTION,
|
|
1221
|
+
`AI agent '${agentKey}'${where} uses tool '${tool}', which has no description. ` +
|
|
1222
|
+
`An agent tool's description is what the model is told it does; without one it is offered the tool under its own name. ` +
|
|
1223
|
+
`Add a 'description' to '${tool}'${meta.title ? " (a 'title' does not count — it labels the tool, it does not explain it)" : ''}.`
|
|
1224
|
+
)
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
/**
|
|
1231
|
+
* Validates that every scope granted by a system role is declared via
|
|
1232
|
+
* `defineScope`. Runs after all visitors, so declaration order does not matter.
|
|
1233
|
+
*
|
|
1234
|
+
* The same rule functions get, for a sharper reason: a role granting an
|
|
1235
|
+
* undeclared scope is not merely useless, it is *invisibly* useless. No
|
|
1236
|
+
* function checks the scope, so the role appears to work — every call a
|
|
1237
|
+
* persona holding it makes succeeds or fails for unrelated reasons, and the
|
|
1238
|
+
* authorization boundary it was written to describe is never exercised.
|
|
1239
|
+
*
|
|
1240
|
+
* A `*` suffix grants a subtree; the node it hangs off must exist.
|
|
1241
|
+
*/
|
|
1242
|
+
export function validateSystemRoleScopes(
|
|
1243
|
+
logger: InspectorLogger,
|
|
1244
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
1245
|
+
): void {
|
|
1246
|
+
if (!state.systemRoles.definitions.length) {
|
|
1247
|
+
return
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
const declared = new Set(
|
|
1251
|
+
flattenScopeDefinitions(state.scopes.definitions).map((s) => s.id)
|
|
1252
|
+
)
|
|
1253
|
+
|
|
1254
|
+
for (const role of state.systemRoles.definitions) {
|
|
1255
|
+
for (const scope of role.scopes) {
|
|
1256
|
+
if (scope === '*') {
|
|
1257
|
+
logger.critical(
|
|
1258
|
+
ErrorCode.INVALID_VALUE,
|
|
1259
|
+
`System role '${role.name}' grants the bare wildcard scope '*'. Grant the roots it should cover instead — '*' hides what the role actually confers.`
|
|
1260
|
+
)
|
|
1261
|
+
continue
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
const declaredForm = scope.endsWith(':*') ? scope.slice(0, -2) : scope
|
|
1265
|
+
if (!declared.has(declaredForm)) {
|
|
1266
|
+
const available = Array.from(declared)
|
|
1267
|
+
logger.critical(
|
|
1268
|
+
ErrorCode.INVALID_VALUE,
|
|
1269
|
+
`System role '${role.name}' grants scope '${scope}' which is not declared. Declare it with defineScope. Available scopes: ${available.join(', ') || 'none'}`
|
|
1270
|
+
)
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
/**
|
|
1277
|
+
* Validates that every role a persona names is declared with
|
|
1278
|
+
* `defineSystemRole`. Runs after all visitors, so declaration order does not
|
|
1279
|
+
* matter.
|
|
1280
|
+
*
|
|
1281
|
+
* Custom roles are refused rather than tolerated. A role composed in the
|
|
1282
|
+
* console does not exist at build time and can be deleted after it, so a
|
|
1283
|
+
* persona pinned to one silently stops testing what it claims to — and the
|
|
1284
|
+
* symptom is a scatter of 403s that read like authorization findings rather
|
|
1285
|
+
* than like a deleted role.
|
|
1286
|
+
*/
|
|
1287
|
+
export function validatePersonaRoles(
|
|
1288
|
+
logger: InspectorLogger,
|
|
1289
|
+
state: InspectorState | Omit<InspectorState, 'typesLookup'>
|
|
1290
|
+
): void {
|
|
1291
|
+
if (!state.personas.definitions.length) {
|
|
1292
|
+
return
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
const declared = new Set(state.systemRoles.definitions.map((r) => r.name))
|
|
1296
|
+
|
|
1297
|
+
for (const persona of state.personas.definitions) {
|
|
1298
|
+
for (const role of persona.roles) {
|
|
1299
|
+
if (!declared.has(role)) {
|
|
1300
|
+
const available = Array.from(declared).sort()
|
|
1301
|
+
logger.critical(
|
|
1302
|
+
ErrorCode.INVALID_VALUE,
|
|
1303
|
+
`Persona '${persona.id}' holds role '${role}', which is not declared with defineSystemRole. ` +
|
|
1304
|
+
`A persona may only name a system role — a role composed in the console can be deleted, and the persona would go on claiming to test it. ` +
|
|
1305
|
+
`Declared roles: ${available.join(', ') || 'none'}`
|
|
1005
1306
|
)
|
|
1006
1307
|
}
|
|
1007
1308
|
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { describe, test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import ts from 'typescript'
|
|
4
|
+
import {
|
|
5
|
+
findSecretAliasServices,
|
|
6
|
+
isSecretServiceType,
|
|
7
|
+
} from './secret-alias-services.js'
|
|
8
|
+
|
|
9
|
+
const SECRET_SERVICE = `
|
|
10
|
+
interface SecretService {
|
|
11
|
+
getSecret(key: string): Promise<string>
|
|
12
|
+
hasSecret(key: string): Promise<boolean>
|
|
13
|
+
setSecret(key: string, value: unknown): Promise<void>
|
|
14
|
+
deleteSecret(key: string): Promise<void>
|
|
15
|
+
getSecrets(keys: string[]): Promise<Record<string, unknown>>
|
|
16
|
+
}
|
|
17
|
+
`
|
|
18
|
+
|
|
19
|
+
const singletonServicesTypeOf = (source: string) => {
|
|
20
|
+
const fileName = '/services.ts'
|
|
21
|
+
const host: ts.CompilerHost = {
|
|
22
|
+
...ts.createCompilerHost({}),
|
|
23
|
+
getSourceFile: (name, languageVersion) =>
|
|
24
|
+
name === fileName
|
|
25
|
+
? ts.createSourceFile(name, source, languageVersion, true)
|
|
26
|
+
: undefined,
|
|
27
|
+
fileExists: (name) => name === fileName,
|
|
28
|
+
readFile: (name) => (name === fileName ? source : undefined),
|
|
29
|
+
}
|
|
30
|
+
const program = ts.createProgram([fileName], { noLib: true }, host)
|
|
31
|
+
const checker = program.getTypeChecker()
|
|
32
|
+
const sourceFile = program.getSourceFile(fileName)!
|
|
33
|
+
const declaration = sourceFile.statements.find(
|
|
34
|
+
(statement): statement is ts.InterfaceDeclaration =>
|
|
35
|
+
ts.isInterfaceDeclaration(statement) &&
|
|
36
|
+
statement.name.text === 'SingletonServices'
|
|
37
|
+
)!
|
|
38
|
+
return {
|
|
39
|
+
checker,
|
|
40
|
+
type: checker.getTypeAtLocation(declaration),
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
describe('findSecretAliasServices', () => {
|
|
45
|
+
test('finds a secret service exposed under another name', () => {
|
|
46
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
47
|
+
${SECRET_SERVICE}
|
|
48
|
+
interface SingletonServices {
|
|
49
|
+
secrets: SecretService
|
|
50
|
+
cfg: SecretService
|
|
51
|
+
}
|
|
52
|
+
`)
|
|
53
|
+
assert.deepEqual(findSecretAliasServices(type, checker), ['cfg'])
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
test('never reports `secrets` itself', () => {
|
|
57
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
58
|
+
${SECRET_SERVICE}
|
|
59
|
+
interface SingletonServices {
|
|
60
|
+
secrets: SecretService
|
|
61
|
+
}
|
|
62
|
+
`)
|
|
63
|
+
assert.deepEqual(findSecretAliasServices(type, checker), [])
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
test('leaves ordinary services alone', () => {
|
|
67
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
68
|
+
${SECRET_SERVICE}
|
|
69
|
+
interface Logger { info(message: string): void }
|
|
70
|
+
interface SingletonServices {
|
|
71
|
+
secrets: SecretService
|
|
72
|
+
logger: Logger
|
|
73
|
+
}
|
|
74
|
+
`)
|
|
75
|
+
assert.deepEqual(findSecretAliasServices(type, checker), [])
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
test('is not fooled by a service that only has getSecret', () => {
|
|
79
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
80
|
+
${SECRET_SERVICE}
|
|
81
|
+
interface Partialish { getSecret(key: string): Promise<string> }
|
|
82
|
+
interface SingletonServices {
|
|
83
|
+
secrets: SecretService
|
|
84
|
+
partialish: Partialish
|
|
85
|
+
}
|
|
86
|
+
`)
|
|
87
|
+
assert.deepEqual(findSecretAliasServices(type, checker), [])
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
test('sees through an optional service', () => {
|
|
91
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
92
|
+
${SECRET_SERVICE}
|
|
93
|
+
interface SingletonServices {
|
|
94
|
+
secrets: SecretService
|
|
95
|
+
cfg?: SecretService
|
|
96
|
+
}
|
|
97
|
+
`)
|
|
98
|
+
assert.deepEqual(findSecretAliasServices(type, checker), ['cfg'])
|
|
99
|
+
})
|
|
100
|
+
|
|
101
|
+
test('finds a subtype that adds methods', () => {
|
|
102
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
103
|
+
${SECRET_SERVICE}
|
|
104
|
+
interface Scoped extends SecretService { getAllStatus(): Promise<void> }
|
|
105
|
+
interface SingletonServices {
|
|
106
|
+
secrets: SecretService
|
|
107
|
+
scoped: Scoped
|
|
108
|
+
}
|
|
109
|
+
`)
|
|
110
|
+
assert.deepEqual(findSecretAliasServices(type, checker), ['scoped'])
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
test('reports every alias, sorted', () => {
|
|
114
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
115
|
+
${SECRET_SERVICE}
|
|
116
|
+
interface SingletonServices {
|
|
117
|
+
secrets: SecretService
|
|
118
|
+
zed: SecretService
|
|
119
|
+
alpha: SecretService
|
|
120
|
+
}
|
|
121
|
+
`)
|
|
122
|
+
assert.deepEqual(findSecretAliasServices(type, checker), ['alpha', 'zed'])
|
|
123
|
+
})
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
describe('isSecretServiceType', () => {
|
|
127
|
+
test('refuses a type whose members are not callable', () => {
|
|
128
|
+
const { type, checker } = singletonServicesTypeOf(`
|
|
129
|
+
${SECRET_SERVICE}
|
|
130
|
+
interface Lookalike {
|
|
131
|
+
getSecret: string
|
|
132
|
+
hasSecret: string
|
|
133
|
+
setSecret: string
|
|
134
|
+
deleteSecret: string
|
|
135
|
+
getSecrets: string
|
|
136
|
+
}
|
|
137
|
+
interface SingletonServices { lookalike: Lookalike }
|
|
138
|
+
`)
|
|
139
|
+
const property = type.getProperty('lookalike')!
|
|
140
|
+
const propertyType = checker.getTypeOfSymbolAtLocation(
|
|
141
|
+
property,
|
|
142
|
+
property.valueDeclaration!
|
|
143
|
+
)
|
|
144
|
+
assert.equal(isSecretServiceType(propertyType, checker), false)
|
|
145
|
+
})
|
|
146
|
+
})
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type ts from 'typescript'
|
|
2
|
+
|
|
3
|
+
const SECRET_SERVICE_METHODS = [
|
|
4
|
+
'getSecret',
|
|
5
|
+
'hasSecret',
|
|
6
|
+
'setSecret',
|
|
7
|
+
'deleteSecret',
|
|
8
|
+
'getSecrets',
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
/** Whether a type structurally satisfies `SecretService`. */
|
|
12
|
+
export const isSecretServiceType = (
|
|
13
|
+
type: ts.Type,
|
|
14
|
+
checker: ts.TypeChecker
|
|
15
|
+
): boolean => {
|
|
16
|
+
const nonNullable = checker.getNonNullableType(type)
|
|
17
|
+
return SECRET_SERVICE_METHODS.every((method) => {
|
|
18
|
+
const property = nonNullable.getProperty(method)
|
|
19
|
+
if (!property) {
|
|
20
|
+
return false
|
|
21
|
+
}
|
|
22
|
+
const declaration = property.valueDeclaration ?? property.declarations?.[0]
|
|
23
|
+
if (!declaration) {
|
|
24
|
+
return false
|
|
25
|
+
}
|
|
26
|
+
const propertyType = checker.getTypeOfSymbolAtLocation(
|
|
27
|
+
property,
|
|
28
|
+
declaration
|
|
29
|
+
)
|
|
30
|
+
return propertyType.getCallSignatures().length > 0
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Singleton service names, other than `secrets`, whose type is a
|
|
36
|
+
* `SecretService`.
|
|
37
|
+
*/
|
|
38
|
+
export const findSecretAliasServices = (
|
|
39
|
+
singletonServicesType: ts.Type,
|
|
40
|
+
checker: ts.TypeChecker
|
|
41
|
+
): string[] => {
|
|
42
|
+
const aliases: string[] = []
|
|
43
|
+
for (const property of singletonServicesType.getProperties()) {
|
|
44
|
+
const name = property.getName()
|
|
45
|
+
if (name === 'secrets') {
|
|
46
|
+
continue
|
|
47
|
+
}
|
|
48
|
+
const declaration = property.valueDeclaration ?? property.declarations?.[0]
|
|
49
|
+
if (!declaration) {
|
|
50
|
+
continue
|
|
51
|
+
}
|
|
52
|
+
const type = checker.getTypeOfSymbolAtLocation(property, declaration)
|
|
53
|
+
if (isSecretServiceType(type, checker)) {
|
|
54
|
+
aliases.push(name)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return aliases.sort()
|
|
58
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { describe, test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { isSecretBrokerFunction } from './secret-brokers.js'
|
|
4
|
+
import { computeSecretBrokers } from './post-process.js'
|
|
5
|
+
import type { InspectorState } from '../types.js'
|
|
6
|
+
|
|
7
|
+
describe('isSecretBrokerFunction', () => {
|
|
8
|
+
test('admits the console secret-admin functions', () => {
|
|
9
|
+
assert.ok(isSecretBrokerFunction('pikkuConsoleGetSecret'))
|
|
10
|
+
assert.ok(isSecretBrokerFunction('pikkuConsoleSetSecret'))
|
|
11
|
+
assert.ok(isSecretBrokerFunction('pikkuConsoleHasSecret'))
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
test('admits them when namespaced by an addon', () => {
|
|
15
|
+
assert.ok(isSecretBrokerFunction('pikku:pikkuConsoleGetSecret'))
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
test('admits nothing else', () => {
|
|
19
|
+
assert.equal(isSecretBrokerFunction('pikkuConsoleGetVariable'), false)
|
|
20
|
+
assert.equal(isSecretBrokerFunction('httpRequest'), false)
|
|
21
|
+
assert.equal(isSecretBrokerFunction('myFunction'), false)
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
test('is not fooled by a name that merely contains a broker name', () => {
|
|
25
|
+
assert.equal(isSecretBrokerFunction('notPikkuConsoleGetSecret'), false)
|
|
26
|
+
assert.equal(isSecretBrokerFunction('pikkuConsoleGetSecretly'), false)
|
|
27
|
+
})
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
describe('computeSecretBrokers', () => {
|
|
31
|
+
const stateWith = (meta: Record<string, any>) =>
|
|
32
|
+
({ functions: { meta } }) as unknown as InspectorState
|
|
33
|
+
|
|
34
|
+
test('flags a broker so the runner keeps its secrets', () => {
|
|
35
|
+
const state = stateWith({
|
|
36
|
+
pikkuConsoleGetSecret: { pikkuFuncId: 'pikkuConsoleGetSecret' },
|
|
37
|
+
})
|
|
38
|
+
computeSecretBrokers(state)
|
|
39
|
+
assert.equal(state.functions.meta.pikkuConsoleGetSecret!.secretBroker, true)
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
test('leaves an ordinary function unflagged', () => {
|
|
43
|
+
const state = stateWith({ createTodo: { pikkuFuncId: 'createTodo' } })
|
|
44
|
+
computeSecretBrokers(state)
|
|
45
|
+
assert.equal(state.functions.meta.createTodo!.secretBroker, undefined)
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
test('falls back to the meta key when there is no pikkuFuncId', () => {
|
|
49
|
+
const state = stateWith({ pikkuConsoleSetSecret: {} })
|
|
50
|
+
computeSecretBrokers(state)
|
|
51
|
+
assert.equal(state.functions.meta.pikkuConsoleSetSecret!.secretBroker, true)
|
|
52
|
+
})
|
|
53
|
+
})
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** The console's secret administration functions, and nothing else. */
|
|
2
|
+
const SECRET_BROKER_FUNCTIONS = new Set([
|
|
3
|
+
'pikkuConsoleSetSecret',
|
|
4
|
+
'pikkuConsoleGetSecret',
|
|
5
|
+
'pikkuConsoleHasSecret',
|
|
6
|
+
])
|
|
7
|
+
|
|
8
|
+
/** Whether this function keeps the full `SecretService`. */
|
|
9
|
+
export const isSecretBrokerFunction = (functionId: string): boolean =>
|
|
10
|
+
SECRET_BROKER_FUNCTIONS.has(functionId) ||
|
|
11
|
+
SECRET_BROKER_FUNCTIONS.has(functionId.split(':').pop() ?? '')
|