@pikku/core 0.12.82 → 0.12.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/README.md +1 -1
- package/dist/ecosystem/agent-scorer.d.ts +12 -0
- package/dist/ecosystem/agent-scorer.js +5 -0
- package/dist/ecosystem/agent.d.ts +18 -0
- package/dist/ecosystem/agent.js +6 -0
- package/dist/ecosystem/ai-agent.d.ts +18 -0
- package/dist/ecosystem/ai-agent.js +6 -0
- package/dist/ecosystem/ai-scorer.d.ts +12 -0
- package/dist/ecosystem/ai-scorer.js +5 -0
- package/dist/ecosystem/channel/local.d.ts +1 -0
- package/dist/ecosystem/channel/local.js +1 -0
- package/dist/ecosystem/channel.d.ts +12 -0
- package/dist/ecosystem/channel.js +4 -0
- package/dist/ecosystem/cli/channel.d.ts +16 -0
- package/dist/ecosystem/cli/channel.js +2 -0
- package/dist/ecosystem/cli.d.ts +9 -0
- package/dist/ecosystem/cli.js +2 -0
- package/dist/ecosystem/credential.d.ts +9 -0
- package/dist/ecosystem/credential.js +1 -0
- package/dist/ecosystem/dev.d.ts +8 -0
- package/dist/ecosystem/dev.js +1 -0
- package/dist/ecosystem/function.d.ts +13 -0
- package/dist/ecosystem/function.js +2 -0
- package/dist/ecosystem/gateway.d.ts +1 -0
- package/dist/ecosystem/gateway.js +1 -0
- package/dist/ecosystem/hmac.d.ts +1 -0
- package/dist/ecosystem/hmac.js +1 -0
- package/dist/ecosystem/http.d.ts +11 -0
- package/dist/ecosystem/http.js +3 -0
- package/dist/ecosystem/mcp.d.ts +3 -0
- package/dist/ecosystem/mcp.js +2 -0
- package/dist/ecosystem/middleware.d.ts +17 -0
- package/dist/ecosystem/middleware.js +6 -0
- package/dist/ecosystem/node-host-resolver.d.ts +1 -0
- package/dist/ecosystem/node-host-resolver.js +1 -0
- package/dist/ecosystem/node.d.ts +1 -0
- package/dist/ecosystem/node.js +1 -0
- package/dist/ecosystem/oauth2.d.ts +1 -0
- package/dist/ecosystem/oauth2.js +1 -0
- package/dist/ecosystem/persona.d.ts +15 -0
- package/dist/ecosystem/persona.js +4 -0
- package/dist/ecosystem/queue.d.ts +11 -0
- package/dist/ecosystem/queue.js +2 -0
- package/dist/ecosystem/remote.d.ts +8 -0
- package/dist/ecosystem/remote.js +1 -0
- package/dist/ecosystem/role.d.ts +3 -0
- package/dist/ecosystem/role.js +2 -0
- package/dist/ecosystem/rpc.d.ts +4 -0
- package/dist/ecosystem/rpc.js +2 -0
- package/dist/ecosystem/safe-fetch.d.ts +7 -0
- package/dist/ecosystem/safe-fetch.js +1 -0
- package/dist/ecosystem/scenario.d.ts +15 -0
- package/dist/ecosystem/scenario.js +4 -0
- package/dist/ecosystem/scheduler.d.ts +2 -0
- package/dist/ecosystem/scheduler.js +1 -0
- package/dist/ecosystem/schema.d.ts +1 -0
- package/dist/ecosystem/schema.js +1 -0
- package/dist/ecosystem/scope.d.ts +3 -0
- package/dist/ecosystem/scope.js +2 -0
- package/dist/ecosystem/secret.d.ts +9 -0
- package/dist/ecosystem/secret.js +1 -0
- package/dist/ecosystem/services/istanbul-coverage.d.ts +1 -0
- package/dist/ecosystem/services/istanbul-coverage.js +1 -0
- package/dist/ecosystem/services/local-content-request-handler.d.ts +9 -0
- package/dist/ecosystem/services/local-content-request-handler.js +1 -0
- package/dist/ecosystem/services/local-content.d.ts +1 -0
- package/dist/ecosystem/services/local-content.js +1 -0
- package/dist/ecosystem/services/v8-coverage.d.ts +1 -0
- package/dist/ecosystem/services/v8-coverage.js +1 -0
- package/dist/ecosystem/services.d.ts +28 -0
- package/dist/ecosystem/services.js +6 -0
- package/dist/ecosystem/testing.d.ts +7 -0
- package/dist/ecosystem/testing.js +1 -0
- package/dist/ecosystem/trigger.d.ts +1 -0
- package/dist/ecosystem/trigger.js +1 -0
- package/dist/ecosystem/types.d.ts +9 -0
- package/dist/ecosystem/types.js +5 -0
- package/dist/ecosystem/variable.d.ts +8 -0
- package/dist/ecosystem/variable.js +1 -0
- package/dist/ecosystem/virtual-user.d.ts +21 -0
- package/dist/ecosystem/virtual-user.js +5 -0
- package/dist/ecosystem/workflow.d.ts +18 -0
- package/dist/ecosystem/workflow.js +4 -0
- package/dist/index.d.ts +8 -8
- package/dist/index.js +3 -3
- package/dist/middleware/auth-apikey.d.ts +1 -1
- package/dist/middleware/auth-bearer.d.ts +1 -1
- package/dist/middleware/auth-cookie.d.ts +1 -1
- package/dist/middleware/cors.d.ts +1 -1
- package/dist/middleware/remote-auth.d.ts +1 -1
- package/dist/middleware/telemetry.d.ts +2 -2
- package/dist/services/agent-run-state-service.d.ts +29 -0
- package/dist/services/agent-run-state-service.js +1 -0
- package/dist/services/agent-runner-service.d.ts +220 -0
- package/dist/services/agent-runner-service.js +1 -0
- package/dist/services/agent-storage-service.d.ts +18 -0
- package/dist/services/agent-storage-service.js +1 -0
- package/dist/services/http-personas.d.ts +1 -1
- package/dist/services/http-personas.js +3 -3
- package/dist/services/in-memory-agent-run-state-service.d.ts +19 -0
- package/dist/services/in-memory-agent-run-state-service.js +54 -0
- package/dist/services/index.d.ts +9 -10
- package/dist/services/index.js +3 -3
- package/dist/services/meta-service.d.ts +5 -4
- package/dist/testing/service-tests/agent-run-service-tests.d.ts +1 -1
- package/dist/testing/service-tests/agent-run-service-tests.js +3 -3
- package/dist/testing/service-tests/agent-storage-service-tests.d.ts +3 -0
- package/dist/testing/service-tests/agent-storage-service-tests.js +302 -0
- package/dist/testing/service-tests.d.ts +4 -4
- package/dist/testing/service-tests.js +4 -4
- package/dist/types/core.types.d.ts +12 -11
- package/dist/types/core.types.js +1 -1
- package/dist/types/state.types.d.ts +11 -6
- package/dist/wirings/actor-flow/run-conversation.d.ts +2 -2
- package/dist/wirings/agent/agent-agui.d.ts +79 -0
- package/dist/wirings/agent/agent-agui.js +327 -0
- package/dist/wirings/agent/agent-finalize.d.ts +58 -0
- package/dist/wirings/agent/agent-finalize.js +138 -0
- package/dist/wirings/agent/agent-helpers.d.ts +35 -0
- package/dist/wirings/agent/agent-helpers.js +47 -0
- package/dist/wirings/agent/agent-interrupt.d.ts +153 -0
- package/dist/wirings/agent/agent-interrupt.js +257 -0
- package/dist/wirings/agent/agent-memory.d.ts +42 -0
- package/dist/wirings/agent/agent-memory.js +333 -0
- package/dist/wirings/agent/agent-model-config.d.ts +16 -0
- package/dist/wirings/agent/agent-model-config.js +51 -0
- package/dist/wirings/agent/agent-prepare.d.ts +117 -0
- package/dist/wirings/agent/agent-prepare.js +728 -0
- package/dist/wirings/agent/agent-registry.d.ts +16 -0
- package/dist/wirings/agent/agent-registry.js +51 -0
- package/dist/wirings/agent/agent-rpc.d.ts +15 -0
- package/dist/wirings/agent/agent-rpc.js +53 -0
- package/dist/wirings/agent/agent-runner.d.ts +7 -0
- package/dist/wirings/agent/agent-runner.js +580 -0
- package/dist/wirings/agent/agent-stream.d.ts +41 -0
- package/dist/wirings/agent/agent-stream.js +1029 -0
- package/dist/wirings/agent/agent-turn.d.ts +57 -0
- package/dist/wirings/agent/agent-turn.js +82 -0
- package/dist/wirings/agent/agent-utils.d.ts +1 -0
- package/dist/wirings/agent/agent-utils.js +1 -0
- package/dist/wirings/agent/agent.types.d.ts +625 -0
- package/dist/wirings/agent/agent.types.js +1 -0
- package/dist/wirings/agent/index.d.ts +12 -0
- package/dist/wirings/agent/index.js +10 -0
- package/dist/wirings/agent/voice-input.d.ts +69 -0
- package/dist/wirings/agent/voice-input.js +150 -0
- package/dist/wirings/agent/voice-output.d.ts +85 -0
- package/dist/wirings/agent/voice-output.js +196 -0
- package/dist/wirings/agent-scorer/agent-scorer-grade.d.ts +26 -0
- package/dist/wirings/agent-scorer/agent-scorer-grade.js +33 -0
- package/dist/wirings/agent-scorer/agent-scorer-judge.d.ts +17 -0
- package/dist/wirings/agent-scorer/agent-scorer-judge.js +92 -0
- package/dist/wirings/agent-scorer/agent-scorer-live.d.ts +15 -0
- package/dist/wirings/agent-scorer/agent-scorer-live.js +38 -0
- package/dist/wirings/agent-scorer/agent-scorer-registry.d.ts +18 -0
- package/dist/wirings/agent-scorer/agent-scorer-registry.js +46 -0
- package/dist/wirings/agent-scorer/agent-scorer-sampling.d.ts +8 -0
- package/dist/wirings/agent-scorer/agent-scorer-sampling.js +31 -0
- package/dist/wirings/agent-scorer/agent-scorer-snapshots.d.ts +10 -0
- package/dist/wirings/agent-scorer/agent-scorer-snapshots.js +40 -0
- package/dist/wirings/agent-scorer/agent-scorer-worker.d.ts +15 -0
- package/dist/wirings/agent-scorer/agent-scorer-worker.js +60 -0
- package/dist/wirings/agent-scorer/agent-scorer.d.ts +39 -0
- package/dist/wirings/agent-scorer/agent-scorer.js +40 -0
- package/dist/wirings/agent-scorer/agent-scorer.types.d.ts +90 -0
- package/dist/wirings/agent-scorer/agent-scorer.types.js +4 -0
- package/dist/wirings/agent-scorer/index.d.ts +6 -0
- package/dist/wirings/agent-scorer/index.js +5 -0
- package/dist/wirings/ai-agent/ai-agent-prepare.js +2 -0
- package/dist/wirings/ai-agent/voice-output.d.ts +1 -2
- package/dist/wirings/channel/channel-rpc.types.d.ts +2 -2
- package/dist/wirings/rpc/addon-runner.d.ts +4 -0
- package/dist/wirings/rpc/addon-runner.js +14 -2
- package/dist/wirings/rpc/rpc-runner.d.ts +1 -1
- package/dist/wirings/rpc/rpc-runner.js +4 -2
- package/dist/wirings/rpc/rpc-types.d.ts +5 -1
- package/dist/wirings/rpc/wire-addon.d.ts +12 -0
- package/dist/wirings/rpc/wire-addon.js +4 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.js +1 -1
- package/dist/wirings/virtual-user/virtual-user-agents.d.ts +1 -1
- package/dist/wirings/workflow/feature.d.ts +2 -1
- package/dist/wirings/workflow/index.d.ts +3 -14
- package/dist/wirings/workflow/index.js +0 -8
- package/dist/wirings/workflow/pikku-scenario-service.d.ts +10 -0
- package/dist/wirings/workflow/pikku-scenario-service.js +9 -0
- package/dist/wirings/workflow/run-timeline.d.ts +1 -2
- package/dist/wirings/workflow/scenario.types.d.ts +37 -0
- package/dist/wirings/workflow/scenario.types.js +1 -0
- package/dist/wirings/workflow/workflow.types.d.ts +2 -37
- package/knowledge/decisions/internals/{a-non-streaming-agent-run-registers-with-airunstate-too.md → a-non-streaming-agent-run-registers-with-agentrunstate-too.md} +6 -6
- package/knowledge/decisions/internals/a-resumed-agent-turn-is-as-interruptible-as-the-first.md +1 -1
- package/knowledge/decisions/internals/actor-flow-drives-the-target-through-a-transport-seam.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-agui-bridge-obeys-the-client-ordering-contract.md → agent-agui-bridge-obeys-the-client-ordering-contract.md} +5 -5
- package/knowledge/decisions/internals/{ai-agent-audio-chunks-carry-the-format-the-provider-returned.md → agent-audio-chunks-carry-the-format-the-provider-returned.md} +2 -2
- package/knowledge/decisions/internals/agent-context-waits-for-a-tool-result-still-being-written.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-credential-suspensions-hide-the-tool-result.md → agent-credential-suspensions-hide-the-tool-result.md} +2 -2
- package/knowledge/decisions/internals/{ai-agent-delegate-and-supervise-hide-different-text.md → agent-delegate-and-supervise-hide-different-text.md} +4 -4
- package/knowledge/decisions/internals/{ai-agent-llm-tool-arguments-have-nulls-stripped.md → agent-llm-tool-arguments-have-nulls-stripped.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-model-config-stays-a-single-resolution-seam.md → agent-model-config-stays-a-single-resolution-seam.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-onerror-hooks-cannot-change-the-failure.md → agent-onerror-hooks-cannot-change-the-failure.md} +6 -6
- package/knowledge/decisions/internals/{ai-agent-runner-methods-must-keep-their-receiver.md → agent-runner-methods-must-keep-their-receiver.md} +4 -4
- package/knowledge/decisions/internals/agent-speech-travels-as-a-custom-agui-event.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-stream-persistence-is-best-effort.md → agent-stream-persistence-is-best-effort.md} +4 -4
- package/knowledge/decisions/internals/{ai-agent-sub-agents-inherit-the-parent-context-block.md → agent-sub-agents-inherit-the-parent-context-block.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-tool-execute-failures-are-logged-unconditionally.md → agent-tool-execute-failures-are-logged-unconditionally.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-voice-input-transcribes-audio-parts-in-place.md → agent-voice-input-transcribes-audio-parts-in-place.md} +2 -2
- package/knowledge/decisions/internals/{ai-agent-working-memory-is-persisted-only-when-valid.md → agent-working-memory-is-persisted-only-when-valid.md} +2 -2
- package/knowledge/decisions/internals/an-agent-interrupt-is-not-a-failure.md +2 -2
- package/knowledge/decisions/internals/an-agent-run-owned-by-another-instance-says-so.md +3 -3
- package/knowledge/decisions/internals/an-agent-stream-send-must-return-the-inner-sends-promise.md +2 -2
- package/knowledge/decisions/internals/an-empty-text-part-is-omitted-from-an-agent-message.md +1 -1
- package/knowledge/decisions/internals/an-empty-transcript-is-not-recorded.md +1 -1
- package/knowledge/decisions/internals/speech-synthesis-picks-a-voice-per-sentence-and-warns-once.md +1 -1
- package/knowledge/decisions/internals/the-agent-done-event-goes-through-the-middleware-and-is-awaited.md +1 -1
- package/knowledge/decisions/internals/the-embedding-model-is-pinned-per-service-and-doc-query-embedding-is-split.md +1 -1
- package/knowledge/decisions/internals/the-per-invocation-rpc-view-is-a-class.md +5 -5
- package/knowledge/decisions/internals/the-transcript-event-is-sent-ahead-of-the-run.md +2 -2
- package/knowledge/decisions/internals/thread-history-records-the-transcript-not-the-audio.md +2 -2
- package/knowledge/decisions/internals/voice-output-speaks-unless-voice-input-explicitly-says-otherwise.md +1 -1
- package/knowledge/decisions/security/{ai-agent-approval-forwarding-requires-a-symbol-brand.md → agent-approval-forwarding-requires-a-symbol-brand.md} +3 -3
- package/knowledge/decisions/security/{ai-agent-credential-requests-are-symbol-branded.md → agent-credential-requests-are-symbol-branded.md} +4 -4
- package/knowledge/decisions/security/{ai-agent-gate-requires-a-session-only-when-auth-is-true.md → agent-gate-requires-a-session-only-when-auth-is-true.md} +3 -3
- package/knowledge/decisions/security/{ai-agent-ownership-failures-never-echo-the-resource.md → agent-ownership-failures-never-echo-the-resource.md} +2 -2
- package/knowledge/decisions/security/{ai-agent-resume-re-runs-the-authorization-gate.md → agent-resume-re-runs-the-authorization-gate.md} +4 -4
- package/knowledge/decisions/security/{ai-agent-sessionless-deployments-have-no-thread-ownership.md → agent-sessionless-deployments-have-no-thread-ownership.md} +5 -5
- package/knowledge/decisions/security/{ai-agent-thread-ownership-composes-the-session-principal.md → agent-thread-ownership-composes-the-session-principal.md} +2 -2
- package/knowledge/decisions/security/{ai-agent-tool-filtering-reads-the-live-function-config.md → agent-tool-filtering-reads-the-live-function-config.md} +2 -2
- package/knowledge/decisions/security/an-agent-approval-is-claimed-before-the-tool-runs.md +3 -3
- package/knowledge/decisions/security/an-empty-owners-constraint-matches-nothing.md +3 -3
- package/knowledge/decisions/security/console-addon-privileged-functions-gate-themselves.md +20 -13
- package/knowledge/decisions/security/index.md +1 -2
- package/package.json +4 -4
- package/scripts/generate-api-report.mts +32 -8
- package/src/agent-naming.test.ts +68 -0
- package/src/ecosystem/agent-scorer.ts +24 -0
- package/src/ecosystem/agent.ts +42 -0
- package/src/ecosystem/channel/local.ts +1 -0
- package/src/ecosystem/channel.ts +22 -0
- package/src/ecosystem/cli/channel.ts +25 -0
- package/src/ecosystem/cli.ts +21 -0
- package/src/ecosystem/credential.ts +10 -0
- package/src/ecosystem/dev.ts +12 -0
- package/src/ecosystem/function.ts +28 -0
- package/src/ecosystem/gateway.ts +4 -0
- package/src/ecosystem/hmac.ts +1 -0
- package/src/ecosystem/http.ts +22 -0
- package/src/ecosystem/mcp.ts +17 -0
- package/src/ecosystem/middleware.ts +27 -0
- package/src/ecosystem/node-host-resolver.ts +1 -0
- package/src/ecosystem/node.ts +5 -0
- package/src/ecosystem/oauth2.ts +4 -0
- package/src/ecosystem/persona.ts +30 -0
- package/src/ecosystem/queue.ts +23 -0
- package/src/ecosystem/remote.ts +9 -0
- package/src/ecosystem/role.ts +13 -0
- package/src/ecosystem/rpc.ts +7 -0
- package/src/ecosystem/safe-fetch.ts +8 -0
- package/src/ecosystem/scenario.ts +32 -0
- package/src/ecosystem/scheduler.ts +5 -0
- package/src/ecosystem/schema.ts +1 -0
- package/src/ecosystem/scope.ts +14 -0
- package/src/ecosystem/secret.ts +15 -0
- package/src/ecosystem/services/istanbul-coverage.ts +1 -0
- package/src/ecosystem/services/local-content-request-handler.ts +16 -0
- package/src/ecosystem/services/local-content.ts +1 -0
- package/src/ecosystem/services/v8-coverage.ts +1 -0
- package/src/ecosystem/services.ts +82 -0
- package/src/ecosystem/testing.ts +8 -0
- package/src/ecosystem/trigger.ts +7 -0
- package/src/ecosystem/types.ts +34 -0
- package/src/ecosystem/variable.ts +14 -0
- package/src/ecosystem/virtual-user.ts +48 -0
- package/src/ecosystem/workflow.ts +68 -0
- package/src/index.ts +8 -19
- package/src/pikku-state.ts +3 -3
- package/src/public-surface.json +183 -91
- package/src/public-surface.json.README +20 -2
- package/src/public-surface.test.ts +31 -3
- package/src/services/{ai-run-state-service.ts → agent-run-state-service.ts} +5 -5
- package/src/services/{ai-agent-runner-service.ts → agent-runner-service.ts} +18 -18
- package/src/services/{ai-storage-service.ts → agent-storage-service.ts} +7 -7
- package/src/services/http-personas-converse.test.ts +3 -3
- package/src/services/http-personas.ts +4 -4
- package/src/services/{in-memory-ai-run-state-service.ts → in-memory-agent-run-state-service.ts} +7 -7
- package/src/services/index.ts +13 -24
- package/src/services/meta-service.ts +5 -7
- package/src/testing/service-tests/agent-run-service-tests.ts +4 -4
- package/src/testing/service-tests/{ai-storage-service-tests.ts → agent-storage-service-tests.ts} +8 -8
- package/src/testing/service-tests.ts +10 -8
- package/src/types/core.types.ts +13 -13
- package/src/types/state.types.ts +11 -10
- package/src/wirings/actor-flow/run-conversation.test.ts +2 -2
- package/src/wirings/actor-flow/run-conversation.ts +7 -9
- package/src/wirings/{ai-agent/ai-agent-agui.test.ts → agent/agent-agui.test.ts} +126 -105
- package/src/wirings/{ai-agent/ai-agent-agui.ts → agent/agent-agui.ts} +7 -7
- package/src/wirings/{ai-agent/ai-agent-authorization.test.ts → agent/agent-authorization.test.ts} +6 -6
- package/src/wirings/{ai-agent/ai-agent-finalize.test.ts → agent/agent-finalize.test.ts} +17 -16
- package/src/wirings/{ai-agent/ai-agent-finalize.ts → agent/agent-finalize.ts} +22 -22
- package/src/wirings/{ai-agent/ai-agent-helpers.test.ts → agent/agent-helpers.test.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-helpers.ts → agent/agent-helpers.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-interrupt.test.ts → agent/agent-interrupt.test.ts} +67 -77
- package/src/wirings/{ai-agent/ai-agent-interrupt.ts → agent/agent-interrupt.ts} +5 -5
- package/src/wirings/{ai-agent/ai-agent-memory.test.ts → agent/agent-memory.test.ts} +12 -12
- package/src/wirings/{ai-agent/ai-agent-memory.ts → agent/agent-memory.ts} +36 -36
- package/src/wirings/{ai-agent/ai-agent-model-config.test.ts → agent/agent-model-config.test.ts} +2 -5
- package/src/wirings/{ai-agent/ai-agent-model-config.ts → agent/agent-model-config.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-prepare.test.ts → agent/agent-prepare.test.ts} +18 -18
- package/src/wirings/{ai-agent/ai-agent-prepare.ts → agent/agent-prepare.ts} +54 -51
- package/src/wirings/{ai-agent/ai-agent-registry.test.ts → agent/agent-registry.test.ts} +49 -43
- package/src/wirings/{ai-agent/ai-agent-registry.ts → agent/agent-registry.ts} +14 -14
- package/src/wirings/{ai-agent/ai-agent-resume-authorization.test.ts → agent/agent-resume-authorization.test.ts} +18 -18
- package/src/wirings/{ai-agent → agent}/agent-rpc.ts +16 -20
- package/src/wirings/{ai-agent/ai-agent-runner.test.ts → agent/agent-runner.test.ts} +105 -106
- package/src/wirings/{ai-agent/ai-agent-runner.ts → agent/agent-runner.ts} +90 -80
- package/src/wirings/{ai-agent/ai-agent-stream-output-hooks.test.ts → agent/agent-stream-output-hooks.test.ts} +25 -25
- package/src/wirings/{ai-agent/ai-agent-stream.test.ts → agent/agent-stream.test.ts} +147 -143
- package/src/wirings/{ai-agent/ai-agent-stream.ts → agent/agent-stream.ts} +147 -146
- package/src/wirings/{ai-agent/ai-agent-thread-ownership.test.ts → agent/agent-thread-ownership.test.ts} +24 -28
- package/src/wirings/{ai-agent/ai-agent-turn.test.ts → agent/agent-turn.test.ts} +3 -5
- package/src/wirings/{ai-agent/ai-agent-turn.ts → agent/agent-turn.ts} +7 -7
- package/src/wirings/{ai-agent/ai-agent.types.ts → agent/agent.types.ts} +51 -48
- package/src/wirings/agent/index.ts +56 -0
- package/src/wirings/{ai-agent → agent}/voice-input.test.ts +15 -15
- package/src/wirings/{ai-agent → agent}/voice-input.ts +11 -11
- package/src/wirings/{ai-agent → agent}/voice-output.test.ts +23 -23
- package/src/wirings/{ai-agent → agent}/voice-output.ts +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-grade.test.ts → agent-scorer/agent-scorer-grade.test.ts} +8 -8
- package/src/wirings/{ai-scorer/ai-scorer-grade.ts → agent-scorer/agent-scorer-grade.ts} +9 -9
- package/src/wirings/{ai-scorer/ai-scorer-judge.test.ts → agent-scorer/agent-scorer-judge.test.ts} +11 -14
- package/src/wirings/{ai-scorer/ai-scorer-judge.ts → agent-scorer/agent-scorer-judge.ts} +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-live.test.ts → agent-scorer/agent-scorer-live.test.ts} +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-live.ts → agent-scorer/agent-scorer-live.ts} +3 -3
- package/src/wirings/{ai-scorer/ai-scorer-registry.ts → agent-scorer/agent-scorer-registry.ts} +11 -8
- package/src/wirings/{ai-scorer/ai-scorer-sampling.test.ts → agent-scorer/agent-scorer-sampling.test.ts} +1 -1
- package/src/wirings/{ai-scorer/ai-scorer-snapshots.test.ts → agent-scorer/agent-scorer-snapshots.test.ts} +2 -2
- package/src/wirings/{ai-scorer/ai-scorer-snapshots.ts → agent-scorer/agent-scorer-snapshots.ts} +1 -1
- package/src/wirings/{ai-scorer/ai-scorer-worker.test.ts → agent-scorer/agent-scorer-worker.test.ts} +15 -15
- package/src/wirings/{ai-scorer/ai-scorer-worker.ts → agent-scorer/agent-scorer-worker.ts} +9 -7
- package/src/wirings/{ai-scorer/ai-scorer.ts → agent-scorer/agent-scorer.ts} +7 -7
- package/src/wirings/{ai-scorer/ai-scorer.types.ts → agent-scorer/agent-scorer.types.ts} +6 -6
- package/src/wirings/agent-scorer/index.ts +22 -0
- package/src/wirings/channel/channel-rpc.types.ts +2 -2
- package/src/wirings/rpc/addon-runner.ts +31 -3
- package/src/wirings/rpc/addon-secrets.test.ts +130 -0
- package/src/wirings/rpc/rpc-runner.ts +4 -2
- package/src/wirings/rpc/rpc-types.ts +5 -1
- package/src/wirings/rpc/wire-addon.ts +16 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.ts +1 -1
- package/src/wirings/virtual-user/run-virtual-user.test.ts +2 -2
- package/src/wirings/virtual-user/run-virtual-user.ts +3 -3
- package/src/wirings/virtual-user/virtual-user-agents.ts +1 -1
- package/src/wirings/workflow/feature.ts +2 -5
- package/src/wirings/workflow/index.ts +1 -49
- package/src/wirings/workflow/pikku-scenario-service.ts +21 -1
- package/src/wirings/workflow/run-timeline.ts +1 -1
- package/src/wirings/workflow/scenario.types.ts +63 -0
- package/src/wirings/workflow/workflow.types.ts +1 -54
- package/src/wirings-stay-decoupled.test.ts +3 -3
- package/tsconfig.tsbuildinfo +1 -1
- package/src/wirings/ai-agent/index.ts +0 -65
- package/src/wirings/ai-scorer/index.ts +0 -24
- /package/src/wirings/{ai-agent/ai-agent-utils.ts → agent/agent-utils.ts} +0 -0
- /package/src/wirings/{ai-scorer/ai-scorer-sampling.ts → agent-scorer/agent-scorer-sampling.ts} +0 -0
package/src/wirings/{ai-scorer/ai-scorer-live.test.ts → agent-scorer/agent-scorer-live.test.ts}
RENAMED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { beforeEach, describe, test } from 'node:test'
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
3
|
import { pikkuState, resetPikkuState } from '../../pikku-state.js'
|
|
4
|
-
import { scoreFinishedRun } from './
|
|
5
|
-
import {
|
|
6
|
-
import type {
|
|
4
|
+
import { scoreFinishedRun } from './agent-scorer-live.js'
|
|
5
|
+
import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
|
|
6
|
+
import type { PikkuAgentScorer, ScorerInput } from './agent-scorer.types.js'
|
|
7
7
|
|
|
8
|
-
const register = (agentName: string, scorers:
|
|
8
|
+
const register = (agentName: string, scorers: PikkuAgentScorer[]) => {
|
|
9
9
|
pikkuState(null, 'agent', 'agents').set(agentName, {
|
|
10
10
|
name: agentName,
|
|
11
11
|
scorers: scorers.map((scorer) => scorer.name),
|
|
@@ -49,7 +49,7 @@ const collectingServices = () => {
|
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
const alwaysOne =
|
|
52
|
+
const alwaysOne = pikkuAgentScorer({
|
|
53
53
|
name: 'brevity',
|
|
54
54
|
description: 'Shorter is better',
|
|
55
55
|
score: () => ({ score: 1 }),
|
|
@@ -61,7 +61,7 @@ describe('scoreFinishedRun', () => {
|
|
|
61
61
|
})
|
|
62
62
|
|
|
63
63
|
test('sends one message per scorer, each to its own lane', async () => {
|
|
64
|
-
const judge =
|
|
64
|
+
const judge = pikkuAgentJudge({
|
|
65
65
|
name: 'helpfulness',
|
|
66
66
|
description: 'Is the answer useful',
|
|
67
67
|
model: 'claude-opus-5',
|
|
@@ -75,14 +75,14 @@ describe('scoreFinishedRun', () => {
|
|
|
75
75
|
assert.deepEqual(
|
|
76
76
|
added.map((entry) => [entry.queueName, entry.data.scorerName]),
|
|
77
77
|
[
|
|
78
|
-
['
|
|
79
|
-
['
|
|
78
|
+
['agent-score-fast', 'brevity'],
|
|
79
|
+
['agent-score-slow', 'helpfulness'],
|
|
80
80
|
]
|
|
81
81
|
)
|
|
82
82
|
})
|
|
83
83
|
|
|
84
84
|
test('never grades a reference-based judge, because live traffic has no answer key', async () => {
|
|
85
|
-
const correctness =
|
|
85
|
+
const correctness = pikkuAgentJudge({
|
|
86
86
|
name: 'correctness',
|
|
87
87
|
description: 'Is the answer right',
|
|
88
88
|
model: 'claude-opus-5',
|
|
@@ -98,7 +98,7 @@ describe('scoreFinishedRun', () => {
|
|
|
98
98
|
})
|
|
99
99
|
|
|
100
100
|
test('honours the sample rate rather than grading every run', async () => {
|
|
101
|
-
const sampled =
|
|
101
|
+
const sampled = pikkuAgentScorer({
|
|
102
102
|
name: 'brevity',
|
|
103
103
|
description: 'Shorter is better',
|
|
104
104
|
sampleRate: 0,
|
|
@@ -138,7 +138,7 @@ describe('scoreFinishedRun', () => {
|
|
|
138
138
|
})
|
|
139
139
|
|
|
140
140
|
test('one scorer that cannot be enqueued does not stop the others', async () => {
|
|
141
|
-
const judge =
|
|
141
|
+
const judge = pikkuAgentJudge({
|
|
142
142
|
name: 'helpfulness',
|
|
143
143
|
description: 'Is the answer useful',
|
|
144
144
|
model: 'claude-opus-5',
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { CoreSingletonServices } from '../../types/core.types.js'
|
|
2
|
-
import { scorersForAgent } from './
|
|
3
|
-
import { isSampled } from './
|
|
4
|
-
import { SCORER_LANE_QUEUES, type ScorerInput } from './
|
|
2
|
+
import { scorersForAgent } from './agent-scorer-registry.js'
|
|
3
|
+
import { isSampled } from './agent-scorer-sampling.js'
|
|
4
|
+
import { SCORER_LANE_QUEUES, type ScorerInput } from './agent-scorer.types.js'
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Grade one finished run on live traffic.
|
package/src/wirings/{ai-scorer/ai-scorer-registry.ts → agent-scorer/agent-scorer-registry.ts}
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { pikkuState } from '../../pikku-state.js'
|
|
2
|
-
import type {
|
|
2
|
+
import type { PikkuAgentScorer } from './agent-scorer.types.js'
|
|
3
3
|
|
|
4
|
-
export const
|
|
4
|
+
export const addAgentScorer = (
|
|
5
5
|
scorerName: string,
|
|
6
|
-
scorer:
|
|
6
|
+
scorer: PikkuAgentScorer<any>,
|
|
7
7
|
packageName: string | null = null
|
|
8
8
|
) => {
|
|
9
9
|
const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta')
|
|
@@ -20,15 +20,18 @@ export const addAIScorer = (
|
|
|
20
20
|
scorers.set(scorerName, scorer)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
export const
|
|
23
|
+
export const getAgentScorers = () => pikkuState(null, 'agent', 'scorers')
|
|
24
24
|
|
|
25
|
-
export const
|
|
25
|
+
export const getAgentScorersMeta = () =>
|
|
26
|
+
pikkuState(null, 'agent', 'scorersMeta')
|
|
26
27
|
|
|
27
28
|
/**
|
|
28
29
|
* Resolve a scorer by name across every registered package, the way an agent's
|
|
29
30
|
* tools are resolved: a scorer declared in an addon is nameable by an app agent.
|
|
30
31
|
*/
|
|
31
|
-
export const
|
|
32
|
+
export const resolveAgentScorer = (
|
|
33
|
+
scorerName: string
|
|
34
|
+
): PikkuAgentScorer<any> => {
|
|
32
35
|
const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName)
|
|
33
36
|
if (!scorer) {
|
|
34
37
|
throw new Error(`AI scorer not found: ${scorerName}`)
|
|
@@ -45,10 +48,10 @@ export const resolveAIScorer = (scorerName: string): PikkuAIScorer<any> => {
|
|
|
45
48
|
export const scorersForAgent = (
|
|
46
49
|
agentName: string,
|
|
47
50
|
logger?: { warn: (message: string) => void }
|
|
48
|
-
):
|
|
51
|
+
): PikkuAgentScorer<any>[] => {
|
|
49
52
|
const agent = pikkuState(null, 'agent', 'agents').get(agentName)
|
|
50
53
|
const names = agent?.scorers ?? []
|
|
51
|
-
const scorers:
|
|
54
|
+
const scorers: PikkuAgentScorer<any>[] = []
|
|
52
55
|
for (const name of names) {
|
|
53
56
|
const scorer = pikkuState(null, 'agent', 'scorers').get(name)
|
|
54
57
|
if (!scorer) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, test } from 'node:test'
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
|
-
import { isSampled } from './
|
|
3
|
+
import { isSampled } from './agent-scorer-sampling.js'
|
|
4
4
|
|
|
5
5
|
describe('isSampled', () => {
|
|
6
6
|
test('grades every run at a rate of 1 and none at 0', () => {
|
|
@@ -6,8 +6,8 @@ import {
|
|
|
6
6
|
getScoreSnapshot,
|
|
7
7
|
recordScoreSnapshot,
|
|
8
8
|
resetScoreSnapshots,
|
|
9
|
-
} from './
|
|
10
|
-
import type { ScorerInput } from './
|
|
9
|
+
} from './agent-scorer-snapshots.js'
|
|
10
|
+
import type { ScorerInput } from './agent-scorer.types.js'
|
|
11
11
|
|
|
12
12
|
const run = (runId: string): ScorerInput => ({
|
|
13
13
|
runId,
|
package/src/wirings/{ai-scorer/ai-scorer-worker.test.ts → agent-scorer/agent-scorer-worker.test.ts}
RENAMED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { beforeEach, describe, test } from 'node:test'
|
|
2
2
|
import assert from 'node:assert/strict'
|
|
3
3
|
import { pikkuState, resetPikkuState } from '../../pikku-state.js'
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import type {
|
|
4
|
+
import { pikkuAgentScoreWorkerFunc } from './agent-scorer-worker.js'
|
|
5
|
+
import { pikkuAgentJudge, pikkuAgentScorer } from './agent-scorer.js'
|
|
6
|
+
import type { PikkuAgentScorer, ScoreJob } from './agent-scorer.types.js'
|
|
7
7
|
|
|
8
8
|
const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
|
|
9
9
|
scorerName: 'brevity',
|
|
@@ -16,28 +16,28 @@ const job = (overrides: Partial<ScoreJob> = {}): ScoreJob => ({
|
|
|
16
16
|
...overrides,
|
|
17
17
|
})
|
|
18
18
|
|
|
19
|
-
const install = (scorer:
|
|
19
|
+
const install = (scorer: PikkuAgentScorer, agentRunner?: unknown) => {
|
|
20
20
|
pikkuState(null, 'agent', 'scorers').set(scorer.name, scorer)
|
|
21
21
|
const saved: unknown[] = []
|
|
22
22
|
pikkuState(null, 'package', 'singletonServices', {
|
|
23
|
-
|
|
23
|
+
agentRunState: {
|
|
24
24
|
saveScore: async (score: unknown) => {
|
|
25
25
|
saved.push(score)
|
|
26
26
|
},
|
|
27
27
|
},
|
|
28
|
-
|
|
28
|
+
agentRunner,
|
|
29
29
|
} as never)
|
|
30
30
|
return saved
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
describe('
|
|
33
|
+
describe('pikkuAgentScoreWorkerFunc', () => {
|
|
34
34
|
beforeEach(() => {
|
|
35
35
|
resetPikkuState()
|
|
36
36
|
})
|
|
37
37
|
|
|
38
38
|
test('records what a heuristic scorer graded, against the run it graded', async () => {
|
|
39
39
|
const saved = install(
|
|
40
|
-
|
|
40
|
+
pikkuAgentScorer({
|
|
41
41
|
name: 'brevity',
|
|
42
42
|
description: 'Shorter is better',
|
|
43
43
|
score: (input) => ({
|
|
@@ -47,7 +47,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
47
47
|
})
|
|
48
48
|
)
|
|
49
49
|
|
|
50
|
-
await
|
|
50
|
+
await pikkuAgentScoreWorkerFunc({}, job())
|
|
51
51
|
|
|
52
52
|
assert.deepEqual(saved, [
|
|
53
53
|
{
|
|
@@ -61,7 +61,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
61
61
|
|
|
62
62
|
test('hands a judge to the model rather than calling a score function it has not got', async () => {
|
|
63
63
|
const saved = install(
|
|
64
|
-
|
|
64
|
+
pikkuAgentJudge({
|
|
65
65
|
name: 'helpfulness',
|
|
66
66
|
description: 'Is the answer useful',
|
|
67
67
|
model: 'claude-opus-5',
|
|
@@ -75,7 +75,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
75
75
|
}
|
|
76
76
|
)
|
|
77
77
|
|
|
78
|
-
await
|
|
78
|
+
await pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'helpfulness' }))
|
|
79
79
|
|
|
80
80
|
assert.deepEqual(saved, [
|
|
81
81
|
{
|
|
@@ -91,7 +91,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
91
91
|
test('fails the job when there is nowhere to record the grade, so it is retried rather than lost', async () => {
|
|
92
92
|
pikkuState(null, 'agent', 'scorers').set(
|
|
93
93
|
'brevity',
|
|
94
|
-
|
|
94
|
+
pikkuAgentScorer({
|
|
95
95
|
name: 'brevity',
|
|
96
96
|
description: 'Shorter is better',
|
|
97
97
|
score: () => ({ score: 1 }),
|
|
@@ -100,14 +100,14 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
100
100
|
pikkuState(null, 'package', 'singletonServices', {} as never)
|
|
101
101
|
|
|
102
102
|
await assert.rejects(
|
|
103
|
-
() =>
|
|
103
|
+
() => pikkuAgentScoreWorkerFunc({}, job()),
|
|
104
104
|
/AI run state service not initialized/
|
|
105
105
|
)
|
|
106
106
|
})
|
|
107
107
|
|
|
108
108
|
test('fails on a scorer name that resolves to nothing', async () => {
|
|
109
109
|
install(
|
|
110
|
-
|
|
110
|
+
pikkuAgentScorer({
|
|
111
111
|
name: 'brevity',
|
|
112
112
|
description: 'Shorter is better',
|
|
113
113
|
score: () => ({ score: 1 }),
|
|
@@ -115,7 +115,7 @@ describe('pikkuAIScoreWorkerFunc', () => {
|
|
|
115
115
|
)
|
|
116
116
|
|
|
117
117
|
await assert.rejects(
|
|
118
|
-
() =>
|
|
118
|
+
() => pikkuAgentScoreWorkerFunc({}, job({ scorerName: 'gone' })),
|
|
119
119
|
/AI scorer not found: gone/
|
|
120
120
|
)
|
|
121
121
|
})
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { pikkuState } from '../../pikku-state.js'
|
|
2
2
|
import { addFunction } from '../../function/function-runner.js'
|
|
3
3
|
import { wireQueueWorker } from '../queue/queue-runner.js'
|
|
4
|
-
import { gradeRun } from './
|
|
5
|
-
import { SCORER_LANE_QUEUES, type ScoreJob } from './
|
|
4
|
+
import { gradeRun } from './agent-scorer-grade.js'
|
|
5
|
+
import { SCORER_LANE_QUEUES, type ScoreJob } from './agent-scorer.types.js'
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* The one worker behind both lanes: it resolves the scorer by name and grades.
|
|
9
9
|
* The lane a job arrived on decides nothing except how long it may take.
|
|
10
10
|
*/
|
|
11
|
-
export async function
|
|
11
|
+
export async function pikkuAgentScoreWorkerFunc(
|
|
12
12
|
_services: Record<string, unknown>,
|
|
13
13
|
job: ScoreJob
|
|
14
14
|
): Promise<void> {
|
|
@@ -22,7 +22,7 @@ export async function pikkuAIScoreWorkerFunc(
|
|
|
22
22
|
await gradeRun(job, services, { persist: true })
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
const SCORE_WORKER_FUNC_ID = '
|
|
25
|
+
const SCORE_WORKER_FUNC_ID = 'pikkuAgentScoreWorker'
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* Bind the two lane queues to the shared worker.
|
|
@@ -32,7 +32,7 @@ const SCORE_WORKER_FUNC_ID = 'pikkuAIScoreWorker'
|
|
|
32
32
|
* for, so the metadata is synthesised here alongside the registration. This
|
|
33
33
|
* mirrors how workflows wire their own queues.
|
|
34
34
|
*/
|
|
35
|
-
export const
|
|
35
|
+
export const wireAgentScorerQueueWorkers = (): void => {
|
|
36
36
|
// No scorers means no lanes: a deployment that grades nothing should not be
|
|
37
37
|
// left holding two queues nothing ever writes to.
|
|
38
38
|
if (pikkuState(null, 'agent', 'scorers').size === 0) return
|
|
@@ -42,7 +42,9 @@ export const wireAIScorerQueueWorkers = (): void => {
|
|
|
42
42
|
const queueMeta = pikkuState(null, 'queue', 'meta')
|
|
43
43
|
|
|
44
44
|
if (!functions.has(SCORE_WORKER_FUNC_ID)) {
|
|
45
|
-
addFunction(SCORE_WORKER_FUNC_ID, {
|
|
45
|
+
addFunction(SCORE_WORKER_FUNC_ID, {
|
|
46
|
+
func: pikkuAgentScoreWorkerFunc,
|
|
47
|
+
} as never)
|
|
46
48
|
}
|
|
47
49
|
if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
|
|
48
50
|
functionsMeta[SCORE_WORKER_FUNC_ID] = {
|
|
@@ -63,7 +65,7 @@ export const wireAIScorerQueueWorkers = (): void => {
|
|
|
63
65
|
}
|
|
64
66
|
wireQueueWorker({
|
|
65
67
|
name: queueName,
|
|
66
|
-
func: { func:
|
|
68
|
+
func: { func: pikkuAgentScoreWorkerFunc },
|
|
67
69
|
} as never)
|
|
68
70
|
}
|
|
69
71
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type {
|
|
2
|
-
|
|
2
|
+
PikkuAgentScorer,
|
|
3
3
|
ScorerInput,
|
|
4
4
|
ScorerOutput,
|
|
5
|
-
} from './
|
|
5
|
+
} from './agent-scorer.types.js'
|
|
6
6
|
|
|
7
7
|
const assertSampleRate = (name: string, sampleRate: number | undefined) => {
|
|
8
8
|
if (sampleRate === undefined) return 1
|
|
@@ -18,7 +18,7 @@ const assertSampleRate = (name: string, sampleRate: number | undefined) => {
|
|
|
18
18
|
* A heuristic scorer: pure code over the finished run, no model call, so it
|
|
19
19
|
* grades on the fast lane.
|
|
20
20
|
*/
|
|
21
|
-
export const
|
|
21
|
+
export const pikkuAgentScorer = <Services = any>(config: {
|
|
22
22
|
name: string
|
|
23
23
|
description: string
|
|
24
24
|
/** 0..1 fraction of live runs to grade. Defaults to all of them. */
|
|
@@ -32,7 +32,7 @@ export const pikkuAIScorer = <Services = any>(config: {
|
|
|
32
32
|
input: ScorerInput,
|
|
33
33
|
services: Services
|
|
34
34
|
) => ScorerOutput | Promise<ScorerOutput>
|
|
35
|
-
}):
|
|
35
|
+
}): PikkuAgentScorer<Services> => ({
|
|
36
36
|
name: config.name,
|
|
37
37
|
description: config.description,
|
|
38
38
|
lane: 'fast',
|
|
@@ -45,11 +45,11 @@ export const pikkuAIScorer = <Services = any>(config: {
|
|
|
45
45
|
* An LLM-judge scorer: the runtime makes the model call and forces a structured
|
|
46
46
|
* `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
|
|
47
47
|
*
|
|
48
|
-
* The rubric field is `goal`, matching `
|
|
48
|
+
* The rubric field is `goal`, matching `pikkuAgent`'s prompt vocabulary — a
|
|
49
49
|
* judge is a degenerate agent, and should use the same word for the same thing.
|
|
50
50
|
* `prompt` is the escape hatch for non-standard framing.
|
|
51
51
|
*/
|
|
52
|
-
export const
|
|
52
|
+
export const pikkuAgentJudge = <Services = any>(config: {
|
|
53
53
|
name: string
|
|
54
54
|
description: string
|
|
55
55
|
/** 0..1 fraction of live runs to grade. Defaults to all of them. */
|
|
@@ -62,7 +62,7 @@ export const pikkuAIJudge = <Services = any>(config: {
|
|
|
62
62
|
model: string
|
|
63
63
|
goal: string
|
|
64
64
|
prompt?: (input: ScorerInput) => string
|
|
65
|
-
}):
|
|
65
|
+
}): PikkuAgentScorer<Services> => ({
|
|
66
66
|
name: config.name,
|
|
67
67
|
description: config.description,
|
|
68
68
|
lane: 'slow',
|
|
@@ -46,13 +46,13 @@ export interface ScorerOutput {
|
|
|
46
46
|
export type ScorerLane = 'fast' | 'slow'
|
|
47
47
|
|
|
48
48
|
export const SCORER_LANE_QUEUES: Record<ScorerLane, string> = {
|
|
49
|
-
fast: '
|
|
50
|
-
slow: '
|
|
49
|
+
fast: 'agent-score-fast',
|
|
50
|
+
slow: 'agent-score-slow',
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
/**
|
|
54
|
-
* A judge's model call. Present only on a scorer built with `
|
|
55
|
-
* `score` is present only on one built with `
|
|
54
|
+
* A judge's model call. Present only on a scorer built with `pikkuAgentJudge`;
|
|
55
|
+
* `score` is present only on one built with `pikkuAgentScorer`. Exactly one of the
|
|
56
56
|
* two is set, which is what the two constructors exist to guarantee.
|
|
57
57
|
*/
|
|
58
58
|
export type ScorerJudgeConfig = {
|
|
@@ -61,7 +61,7 @@ export type ScorerJudgeConfig = {
|
|
|
61
61
|
prompt?: (input: ScorerInput) => string
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
export type
|
|
64
|
+
export type PikkuAgentScorer<Services = any> = {
|
|
65
65
|
name: string
|
|
66
66
|
description: string
|
|
67
67
|
lane: ScorerLane
|
|
@@ -97,7 +97,7 @@ export type ScoreJob = ScorerInput & {
|
|
|
97
97
|
scorerName: string
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
export type
|
|
100
|
+
export type AgentRunScore = {
|
|
101
101
|
runId: string
|
|
102
102
|
scorerName: string
|
|
103
103
|
score: number
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export { pikkuAgentScorer, pikkuAgentJudge } from './agent-scorer.js'
|
|
2
|
+
export {
|
|
3
|
+
addAgentScorer,
|
|
4
|
+
getAgentScorers,
|
|
5
|
+
getAgentScorersMeta,
|
|
6
|
+
} from './agent-scorer-registry.js'
|
|
7
|
+
export { gradeRun } from './agent-scorer-grade.js'
|
|
8
|
+
export {
|
|
9
|
+
enableScoreSnapshots,
|
|
10
|
+
getScoreSnapshot,
|
|
11
|
+
} from './agent-scorer-snapshots.js'
|
|
12
|
+
export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js'
|
|
13
|
+
export type {
|
|
14
|
+
AgentRunScore,
|
|
15
|
+
PikkuAgentScorer,
|
|
16
|
+
ScoreJob,
|
|
17
|
+
ScorerInput,
|
|
18
|
+
ScorerJudgeConfig,
|
|
19
|
+
ScorerLane,
|
|
20
|
+
ScorerMeta,
|
|
21
|
+
ScorerOutput,
|
|
22
|
+
} from './agent-scorer.types.js'
|
|
@@ -92,7 +92,7 @@ export class ChannelRPCError extends Error {
|
|
|
92
92
|
* Whether a callable exposed to a remote decider needs a human to agree before
|
|
93
93
|
* it runs, and how to describe the invocation while asking.
|
|
94
94
|
*
|
|
95
|
-
* Shared with `
|
|
95
|
+
* Shared with `AgentToolDef`: both are an allowlist of named callables
|
|
96
96
|
* invoked by something other than the code that wrote them. The runtime around
|
|
97
97
|
* them is not shared — an agent suspends and resumes, a reverse call is a live
|
|
98
98
|
* await.
|
|
@@ -105,7 +105,7 @@ export interface ApprovalPolicy {
|
|
|
105
105
|
export type CapabilityHandler = (data: any) => Promise<unknown> | unknown
|
|
106
106
|
|
|
107
107
|
/**
|
|
108
|
-
* `needsApproval` is required here on purpose. On `
|
|
108
|
+
* `needsApproval` is required here on purpose. On `AgentToolDef` the same
|
|
109
109
|
* field is optional and absence means "do not ask"; here absence means the
|
|
110
110
|
* opposite, so it must not be expressible — a bare function is the
|
|
111
111
|
* unclassified form.
|
|
@@ -13,12 +13,32 @@ export type AddonInstance = {
|
|
|
13
13
|
secretOverrides?: Record<string, string>
|
|
14
14
|
variableOverrides?: Record<string, string>
|
|
15
15
|
credentialOverrides?: Record<string, string>
|
|
16
|
+
/** Set by the consuming app: secrets it lends this instance, as the addon names them. */
|
|
17
|
+
secretGrants?: string[]
|
|
18
|
+
/** Set by the consuming app: credentials it lends this instance, as the addon names them. */
|
|
19
|
+
credentialGrants?: string[]
|
|
16
20
|
/** Set by the consuming app to opt this instance out of secret scoping. */
|
|
17
21
|
globalSecrets?: string
|
|
18
22
|
/** Set by the consuming app to opt this instance out of credential scoping. */
|
|
19
23
|
globalCredentials?: string
|
|
20
24
|
}
|
|
21
25
|
|
|
26
|
+
/**
|
|
27
|
+
* What the addon declared, plus what the host lent it. Scoping runs outside the
|
|
28
|
+
* aliaser, so every name here is the one the addon reads — which is why an
|
|
29
|
+
* override's *key* grants, and its value does not.
|
|
30
|
+
*/
|
|
31
|
+
const allowedNames = (
|
|
32
|
+
declared: string[] | null | undefined,
|
|
33
|
+
grants: string[] | undefined,
|
|
34
|
+
overrides: Record<string, string> | undefined
|
|
35
|
+
): Set<string> =>
|
|
36
|
+
new Set([
|
|
37
|
+
...(declared ?? []),
|
|
38
|
+
...(grants ?? []),
|
|
39
|
+
...Object.keys(overrides ?? {}),
|
|
40
|
+
])
|
|
41
|
+
|
|
22
42
|
const aliasSecretService = (
|
|
23
43
|
secrets: SecretService,
|
|
24
44
|
overrides: Record<string, string>
|
|
@@ -142,7 +162,11 @@ export const getOrCreatePackageSingletonServices = async (
|
|
|
142
162
|
...existingServices,
|
|
143
163
|
secrets: new ScopedSecretService(
|
|
144
164
|
existingServices.secrets,
|
|
145
|
-
|
|
165
|
+
allowedNames(
|
|
166
|
+
pikkuState(packageName, 'package', 'declaredSecrets'),
|
|
167
|
+
addonInstance?.secretGrants,
|
|
168
|
+
addonInstance?.secretOverrides
|
|
169
|
+
)
|
|
146
170
|
),
|
|
147
171
|
}
|
|
148
172
|
}
|
|
@@ -151,10 +175,12 @@ export const getOrCreatePackageSingletonServices = async (
|
|
|
151
175
|
...existingServices,
|
|
152
176
|
credentialService: new ScopedCredentialService(
|
|
153
177
|
existingServices.credentialService,
|
|
154
|
-
|
|
178
|
+
allowedNames(
|
|
155
179
|
Object.keys(
|
|
156
180
|
pikkuState(packageName, 'package', 'credentialsMeta') ?? {}
|
|
157
|
-
)
|
|
181
|
+
),
|
|
182
|
+
addonInstance?.credentialGrants,
|
|
183
|
+
addonInstance?.credentialOverrides
|
|
158
184
|
)
|
|
159
185
|
),
|
|
160
186
|
}
|
|
@@ -202,6 +228,8 @@ export const addonInstanceForNamespace = (
|
|
|
202
228
|
secretOverrides: cfg.secretOverrides,
|
|
203
229
|
variableOverrides: cfg.variableOverrides,
|
|
204
230
|
credentialOverrides: cfg.credentialOverrides,
|
|
231
|
+
secretGrants: cfg.secretGrants,
|
|
232
|
+
credentialGrants: cfg.credentialGrants,
|
|
205
233
|
globalSecrets: cfg.globalSecrets,
|
|
206
234
|
globalCredentials: cfg.globalCredentials,
|
|
207
235
|
}
|
|
@@ -259,3 +259,133 @@ describe('addon credentials are scoped to what the addon declared', () => {
|
|
|
259
259
|
assert.deepEqual(await credentials.getAllUsers(), ['user-1'])
|
|
260
260
|
})
|
|
261
261
|
})
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* An addon whose secret names come off its input rather than its own source
|
|
265
|
+
* declares nothing, so the host lends it names it could not have derived.
|
|
266
|
+
*
|
|
267
|
+
* knowledge: decisions/security/decision-secretless-functions.md
|
|
268
|
+
*/
|
|
269
|
+
describe('the host grants an addon secrets it could not declare', () => {
|
|
270
|
+
beforeEach(() => {
|
|
271
|
+
resetPikkuState()
|
|
272
|
+
})
|
|
273
|
+
|
|
274
|
+
test('a granted secret is readable', async () => {
|
|
275
|
+
registerAddon([])
|
|
276
|
+
wireAddon({
|
|
277
|
+
name: 'example',
|
|
278
|
+
package: ADDON_PACKAGE,
|
|
279
|
+
secretGrants: ['STRIPE_KEY'],
|
|
280
|
+
})
|
|
281
|
+
|
|
282
|
+
const secrets = await secretsHandedToAddon('example')
|
|
283
|
+
assert.equal(await secrets.getSecret('STRIPE_KEY'), 'value-of-STRIPE_KEY')
|
|
284
|
+
})
|
|
285
|
+
|
|
286
|
+
test('a secret the host did not grant is still denied', async () => {
|
|
287
|
+
registerAddon([])
|
|
288
|
+
wireAddon({
|
|
289
|
+
name: 'example',
|
|
290
|
+
package: ADDON_PACKAGE,
|
|
291
|
+
secretGrants: ['STRIPE_KEY'],
|
|
292
|
+
})
|
|
293
|
+
|
|
294
|
+
const secrets = await secretsHandedToAddon('example')
|
|
295
|
+
await assert.rejects(() => secrets.getSecret('DATABASE_URL'), /denied/i)
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
test('a grant adds to what the addon declared rather than replacing it', async () => {
|
|
299
|
+
registerAddon(['ADDON_KEY'])
|
|
300
|
+
wireAddon({
|
|
301
|
+
name: 'example',
|
|
302
|
+
package: ADDON_PACKAGE,
|
|
303
|
+
secretGrants: ['STRIPE_KEY'],
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
const secrets = await secretsHandedToAddon('example')
|
|
307
|
+
assert.equal(await secrets.getSecret('ADDON_KEY'), 'value-of-ADDON_KEY')
|
|
308
|
+
assert.equal(await secrets.getSecret('STRIPE_KEY'), 'value-of-STRIPE_KEY')
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
test('an overridden secret is granted by the override itself', async () => {
|
|
312
|
+
registerAddon([])
|
|
313
|
+
wireAddon({
|
|
314
|
+
name: 'example',
|
|
315
|
+
package: ADDON_PACKAGE,
|
|
316
|
+
secretOverrides: { MAILGUN_KEY: 'PROD_EMAIL_KEY' },
|
|
317
|
+
})
|
|
318
|
+
|
|
319
|
+
const secrets = await secretsHandedToAddon('example')
|
|
320
|
+
assert.equal(
|
|
321
|
+
await secrets.getSecret('MAILGUN_KEY'),
|
|
322
|
+
'value-of-PROD_EMAIL_KEY'
|
|
323
|
+
)
|
|
324
|
+
})
|
|
325
|
+
|
|
326
|
+
test('a grant is checked before the override renames it', async () => {
|
|
327
|
+
registerAddon([])
|
|
328
|
+
wireAddon({
|
|
329
|
+
name: 'example',
|
|
330
|
+
package: ADDON_PACKAGE,
|
|
331
|
+
secretGrants: ['MAILGUN_KEY'],
|
|
332
|
+
secretOverrides: { MAILGUN_KEY: 'PROD_EMAIL_KEY' },
|
|
333
|
+
})
|
|
334
|
+
|
|
335
|
+
const secrets = await secretsHandedToAddon('example')
|
|
336
|
+
assert.equal(
|
|
337
|
+
await secrets.getSecret('MAILGUN_KEY'),
|
|
338
|
+
'value-of-PROD_EMAIL_KEY'
|
|
339
|
+
)
|
|
340
|
+
await assert.rejects(() => secrets.getSecret('PROD_EMAIL_KEY'), /denied/i)
|
|
341
|
+
})
|
|
342
|
+
|
|
343
|
+
test('a granted credential is readable', async () => {
|
|
344
|
+
registerAddon([], [])
|
|
345
|
+
wireAddon({
|
|
346
|
+
name: 'example',
|
|
347
|
+
package: ADDON_PACKAGE,
|
|
348
|
+
credentialGrants: ['stripe'],
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
const credentials = await credentialsHandedToAddon('example')
|
|
352
|
+
assert.equal(await credentials.get('stripe'), 'credential-of-stripe')
|
|
353
|
+
await assert.rejects(() => credentials.get('slack'), /denied/i)
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
test('a granted credential does not widen the rest of the service', async () => {
|
|
357
|
+
registerAddon([], [])
|
|
358
|
+
wireAddon({
|
|
359
|
+
name: 'example',
|
|
360
|
+
package: ADDON_PACKAGE,
|
|
361
|
+
credentialGrants: ['stripe'],
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
const credentials = await credentialsHandedToAddon('example')
|
|
365
|
+
await assert.rejects(() => credentials.getAllUsers(), /denied/i)
|
|
366
|
+
})
|
|
367
|
+
|
|
368
|
+
test('an addon without a singleton factory is granted too', async () => {
|
|
369
|
+
pikkuState(ADDON_PACKAGE, 'package', 'declaredSecrets', [])
|
|
370
|
+
wireAddon({
|
|
371
|
+
name: 'example',
|
|
372
|
+
package: ADDON_PACKAGE,
|
|
373
|
+
secretGrants: ['STRIPE_KEY'],
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
const services = await getOrCreatePackageSingletonServices(
|
|
377
|
+
ADDON_PACKAGE,
|
|
378
|
+
createParentServices(),
|
|
379
|
+
pikkuState(null, 'addons', 'packages').get('example') as never
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
assert.equal(
|
|
383
|
+
await services.secrets.getSecret('STRIPE_KEY'),
|
|
384
|
+
'value-of-STRIPE_KEY'
|
|
385
|
+
)
|
|
386
|
+
await assert.rejects(
|
|
387
|
+
() => services.secrets.getSecret('DATABASE_URL'),
|
|
388
|
+
/denied/i
|
|
389
|
+
)
|
|
390
|
+
})
|
|
391
|
+
})
|