@pikku/core 0.12.82 → 0.12.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/README.md +1 -1
- package/dist/ecosystem/agent-scorer.d.ts +12 -0
- package/dist/ecosystem/agent-scorer.js +5 -0
- package/dist/ecosystem/agent.d.ts +18 -0
- package/dist/ecosystem/agent.js +6 -0
- package/dist/ecosystem/ai-agent.d.ts +18 -0
- package/dist/ecosystem/ai-agent.js +6 -0
- package/dist/ecosystem/ai-scorer.d.ts +12 -0
- package/dist/ecosystem/ai-scorer.js +5 -0
- package/dist/ecosystem/channel/local.d.ts +1 -0
- package/dist/ecosystem/channel/local.js +1 -0
- package/dist/ecosystem/channel.d.ts +12 -0
- package/dist/ecosystem/channel.js +4 -0
- package/dist/ecosystem/cli/channel.d.ts +16 -0
- package/dist/ecosystem/cli/channel.js +2 -0
- package/dist/ecosystem/cli.d.ts +9 -0
- package/dist/ecosystem/cli.js +2 -0
- package/dist/ecosystem/credential.d.ts +9 -0
- package/dist/ecosystem/credential.js +1 -0
- package/dist/ecosystem/dev.d.ts +8 -0
- package/dist/ecosystem/dev.js +1 -0
- package/dist/ecosystem/function.d.ts +13 -0
- package/dist/ecosystem/function.js +2 -0
- package/dist/ecosystem/gateway.d.ts +1 -0
- package/dist/ecosystem/gateway.js +1 -0
- package/dist/ecosystem/hmac.d.ts +1 -0
- package/dist/ecosystem/hmac.js +1 -0
- package/dist/ecosystem/http.d.ts +11 -0
- package/dist/ecosystem/http.js +3 -0
- package/dist/ecosystem/mcp.d.ts +3 -0
- package/dist/ecosystem/mcp.js +2 -0
- package/dist/ecosystem/middleware.d.ts +17 -0
- package/dist/ecosystem/middleware.js +6 -0
- package/dist/ecosystem/node-host-resolver.d.ts +1 -0
- package/dist/ecosystem/node-host-resolver.js +1 -0
- package/dist/ecosystem/node.d.ts +1 -0
- package/dist/ecosystem/node.js +1 -0
- package/dist/ecosystem/oauth2.d.ts +1 -0
- package/dist/ecosystem/oauth2.js +1 -0
- package/dist/ecosystem/persona.d.ts +15 -0
- package/dist/ecosystem/persona.js +4 -0
- package/dist/ecosystem/queue.d.ts +11 -0
- package/dist/ecosystem/queue.js +2 -0
- package/dist/ecosystem/remote.d.ts +8 -0
- package/dist/ecosystem/remote.js +1 -0
- package/dist/ecosystem/role.d.ts +3 -0
- package/dist/ecosystem/role.js +2 -0
- package/dist/ecosystem/rpc.d.ts +4 -0
- package/dist/ecosystem/rpc.js +2 -0
- package/dist/ecosystem/safe-fetch.d.ts +7 -0
- package/dist/ecosystem/safe-fetch.js +1 -0
- package/dist/ecosystem/scenario.d.ts +15 -0
- package/dist/ecosystem/scenario.js +4 -0
- package/dist/ecosystem/scheduler.d.ts +2 -0
- package/dist/ecosystem/scheduler.js +1 -0
- package/dist/ecosystem/schema.d.ts +1 -0
- package/dist/ecosystem/schema.js +1 -0
- package/dist/ecosystem/scope.d.ts +3 -0
- package/dist/ecosystem/scope.js +2 -0
- package/dist/ecosystem/secret.d.ts +9 -0
- package/dist/ecosystem/secret.js +1 -0
- package/dist/ecosystem/services/istanbul-coverage.d.ts +1 -0
- package/dist/ecosystem/services/istanbul-coverage.js +1 -0
- package/dist/ecosystem/services/local-content-request-handler.d.ts +9 -0
- package/dist/ecosystem/services/local-content-request-handler.js +1 -0
- package/dist/ecosystem/services/local-content.d.ts +1 -0
- package/dist/ecosystem/services/local-content.js +1 -0
- package/dist/ecosystem/services/v8-coverage.d.ts +1 -0
- package/dist/ecosystem/services/v8-coverage.js +1 -0
- package/dist/ecosystem/services.d.ts +28 -0
- package/dist/ecosystem/services.js +6 -0
- package/dist/ecosystem/testing.d.ts +7 -0
- package/dist/ecosystem/testing.js +1 -0
- package/dist/ecosystem/trigger.d.ts +1 -0
- package/dist/ecosystem/trigger.js +1 -0
- package/dist/ecosystem/types.d.ts +9 -0
- package/dist/ecosystem/types.js +5 -0
- package/dist/ecosystem/variable.d.ts +8 -0
- package/dist/ecosystem/variable.js +1 -0
- package/dist/ecosystem/virtual-user.d.ts +21 -0
- package/dist/ecosystem/virtual-user.js +5 -0
- package/dist/ecosystem/workflow.d.ts +18 -0
- package/dist/ecosystem/workflow.js +4 -0
- package/dist/index.d.ts +8 -8
- package/dist/index.js +3 -3
- package/dist/middleware/auth-apikey.d.ts +1 -1
- package/dist/middleware/auth-bearer.d.ts +1 -1
- package/dist/middleware/auth-cookie.d.ts +1 -1
- package/dist/middleware/cors.d.ts +1 -1
- package/dist/middleware/remote-auth.d.ts +1 -1
- package/dist/middleware/telemetry.d.ts +2 -2
- package/dist/services/agent-run-state-service.d.ts +29 -0
- package/dist/services/agent-run-state-service.js +1 -0
- package/dist/services/agent-runner-service.d.ts +220 -0
- package/dist/services/agent-runner-service.js +1 -0
- package/dist/services/agent-storage-service.d.ts +18 -0
- package/dist/services/agent-storage-service.js +1 -0
- package/dist/services/http-personas.d.ts +1 -1
- package/dist/services/http-personas.js +3 -3
- package/dist/services/in-memory-agent-run-state-service.d.ts +19 -0
- package/dist/services/in-memory-agent-run-state-service.js +54 -0
- package/dist/services/index.d.ts +9 -10
- package/dist/services/index.js +3 -3
- package/dist/services/meta-service.d.ts +5 -4
- package/dist/testing/service-tests/agent-run-service-tests.d.ts +1 -1
- package/dist/testing/service-tests/agent-run-service-tests.js +3 -3
- package/dist/testing/service-tests/agent-storage-service-tests.d.ts +3 -0
- package/dist/testing/service-tests/agent-storage-service-tests.js +302 -0
- package/dist/testing/service-tests.d.ts +4 -4
- package/dist/testing/service-tests.js +4 -4
- package/dist/types/core.types.d.ts +12 -11
- package/dist/types/core.types.js +1 -1
- package/dist/types/state.types.d.ts +11 -6
- package/dist/wirings/actor-flow/run-conversation.d.ts +2 -2
- package/dist/wirings/agent/agent-agui.d.ts +79 -0
- package/dist/wirings/agent/agent-agui.js +327 -0
- package/dist/wirings/agent/agent-finalize.d.ts +58 -0
- package/dist/wirings/agent/agent-finalize.js +138 -0
- package/dist/wirings/agent/agent-helpers.d.ts +35 -0
- package/dist/wirings/agent/agent-helpers.js +47 -0
- package/dist/wirings/agent/agent-interrupt.d.ts +153 -0
- package/dist/wirings/agent/agent-interrupt.js +257 -0
- package/dist/wirings/agent/agent-memory.d.ts +42 -0
- package/dist/wirings/agent/agent-memory.js +333 -0
- package/dist/wirings/agent/agent-model-config.d.ts +16 -0
- package/dist/wirings/agent/agent-model-config.js +51 -0
- package/dist/wirings/agent/agent-prepare.d.ts +117 -0
- package/dist/wirings/agent/agent-prepare.js +728 -0
- package/dist/wirings/agent/agent-registry.d.ts +16 -0
- package/dist/wirings/agent/agent-registry.js +51 -0
- package/dist/wirings/agent/agent-rpc.d.ts +15 -0
- package/dist/wirings/agent/agent-rpc.js +53 -0
- package/dist/wirings/agent/agent-runner.d.ts +7 -0
- package/dist/wirings/agent/agent-runner.js +580 -0
- package/dist/wirings/agent/agent-stream.d.ts +41 -0
- package/dist/wirings/agent/agent-stream.js +1029 -0
- package/dist/wirings/agent/agent-turn.d.ts +57 -0
- package/dist/wirings/agent/agent-turn.js +82 -0
- package/dist/wirings/agent/agent-utils.d.ts +1 -0
- package/dist/wirings/agent/agent-utils.js +1 -0
- package/dist/wirings/agent/agent.types.d.ts +625 -0
- package/dist/wirings/agent/agent.types.js +1 -0
- package/dist/wirings/agent/index.d.ts +12 -0
- package/dist/wirings/agent/index.js +10 -0
- package/dist/wirings/agent/voice-input.d.ts +69 -0
- package/dist/wirings/agent/voice-input.js +150 -0
- package/dist/wirings/agent/voice-output.d.ts +85 -0
- package/dist/wirings/agent/voice-output.js +196 -0
- package/dist/wirings/agent-scorer/agent-scorer-grade.d.ts +26 -0
- package/dist/wirings/agent-scorer/agent-scorer-grade.js +33 -0
- package/dist/wirings/agent-scorer/agent-scorer-judge.d.ts +17 -0
- package/dist/wirings/agent-scorer/agent-scorer-judge.js +92 -0
- package/dist/wirings/agent-scorer/agent-scorer-live.d.ts +15 -0
- package/dist/wirings/agent-scorer/agent-scorer-live.js +38 -0
- package/dist/wirings/agent-scorer/agent-scorer-registry.d.ts +18 -0
- package/dist/wirings/agent-scorer/agent-scorer-registry.js +46 -0
- package/dist/wirings/agent-scorer/agent-scorer-sampling.d.ts +8 -0
- package/dist/wirings/agent-scorer/agent-scorer-sampling.js +31 -0
- package/dist/wirings/agent-scorer/agent-scorer-snapshots.d.ts +10 -0
- package/dist/wirings/agent-scorer/agent-scorer-snapshots.js +40 -0
- package/dist/wirings/agent-scorer/agent-scorer-worker.d.ts +15 -0
- package/dist/wirings/agent-scorer/agent-scorer-worker.js +60 -0
- package/dist/wirings/agent-scorer/agent-scorer.d.ts +39 -0
- package/dist/wirings/agent-scorer/agent-scorer.js +40 -0
- package/dist/wirings/agent-scorer/agent-scorer.types.d.ts +90 -0
- package/dist/wirings/agent-scorer/agent-scorer.types.js +4 -0
- package/dist/wirings/agent-scorer/index.d.ts +6 -0
- package/dist/wirings/agent-scorer/index.js +5 -0
- package/dist/wirings/ai-agent/ai-agent-prepare.js +2 -0
- package/dist/wirings/ai-agent/voice-output.d.ts +1 -2
- package/dist/wirings/channel/channel-rpc.types.d.ts +2 -2
- package/dist/wirings/rpc/addon-runner.d.ts +4 -0
- package/dist/wirings/rpc/addon-runner.js +14 -2
- package/dist/wirings/rpc/rpc-runner.d.ts +1 -1
- package/dist/wirings/rpc/rpc-runner.js +4 -2
- package/dist/wirings/rpc/rpc-types.d.ts +5 -1
- package/dist/wirings/rpc/wire-addon.d.ts +12 -0
- package/dist/wirings/rpc/wire-addon.js +4 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.js +1 -1
- package/dist/wirings/virtual-user/virtual-user-agents.d.ts +1 -1
- package/dist/wirings/workflow/feature.d.ts +2 -1
- package/dist/wirings/workflow/index.d.ts +3 -14
- package/dist/wirings/workflow/index.js +0 -8
- package/dist/wirings/workflow/pikku-scenario-service.d.ts +10 -0
- package/dist/wirings/workflow/pikku-scenario-service.js +9 -0
- package/dist/wirings/workflow/run-timeline.d.ts +1 -2
- package/dist/wirings/workflow/scenario.types.d.ts +37 -0
- package/dist/wirings/workflow/scenario.types.js +1 -0
- package/dist/wirings/workflow/workflow.types.d.ts +2 -37
- package/knowledge/decisions/internals/{a-non-streaming-agent-run-registers-with-airunstate-too.md → a-non-streaming-agent-run-registers-with-agentrunstate-too.md} +6 -6
- package/knowledge/decisions/internals/a-resumed-agent-turn-is-as-interruptible-as-the-first.md +1 -1
- package/knowledge/decisions/internals/actor-flow-drives-the-target-through-a-transport-seam.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-agui-bridge-obeys-the-client-ordering-contract.md → agent-agui-bridge-obeys-the-client-ordering-contract.md} +5 -5
- package/knowledge/decisions/internals/{ai-agent-audio-chunks-carry-the-format-the-provider-returned.md → agent-audio-chunks-carry-the-format-the-provider-returned.md} +2 -2
- package/knowledge/decisions/internals/agent-context-waits-for-a-tool-result-still-being-written.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-credential-suspensions-hide-the-tool-result.md → agent-credential-suspensions-hide-the-tool-result.md} +2 -2
- package/knowledge/decisions/internals/{ai-agent-delegate-and-supervise-hide-different-text.md → agent-delegate-and-supervise-hide-different-text.md} +4 -4
- package/knowledge/decisions/internals/{ai-agent-llm-tool-arguments-have-nulls-stripped.md → agent-llm-tool-arguments-have-nulls-stripped.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-model-config-stays-a-single-resolution-seam.md → agent-model-config-stays-a-single-resolution-seam.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-onerror-hooks-cannot-change-the-failure.md → agent-onerror-hooks-cannot-change-the-failure.md} +6 -6
- package/knowledge/decisions/internals/{ai-agent-runner-methods-must-keep-their-receiver.md → agent-runner-methods-must-keep-their-receiver.md} +4 -4
- package/knowledge/decisions/internals/agent-speech-travels-as-a-custom-agui-event.md +1 -1
- package/knowledge/decisions/internals/{ai-agent-stream-persistence-is-best-effort.md → agent-stream-persistence-is-best-effort.md} +4 -4
- package/knowledge/decisions/internals/{ai-agent-sub-agents-inherit-the-parent-context-block.md → agent-sub-agents-inherit-the-parent-context-block.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-tool-execute-failures-are-logged-unconditionally.md → agent-tool-execute-failures-are-logged-unconditionally.md} +3 -3
- package/knowledge/decisions/internals/{ai-agent-voice-input-transcribes-audio-parts-in-place.md → agent-voice-input-transcribes-audio-parts-in-place.md} +2 -2
- package/knowledge/decisions/internals/{ai-agent-working-memory-is-persisted-only-when-valid.md → agent-working-memory-is-persisted-only-when-valid.md} +2 -2
- package/knowledge/decisions/internals/an-agent-interrupt-is-not-a-failure.md +2 -2
- package/knowledge/decisions/internals/an-agent-run-owned-by-another-instance-says-so.md +3 -3
- package/knowledge/decisions/internals/an-agent-stream-send-must-return-the-inner-sends-promise.md +2 -2
- package/knowledge/decisions/internals/an-empty-text-part-is-omitted-from-an-agent-message.md +1 -1
- package/knowledge/decisions/internals/an-empty-transcript-is-not-recorded.md +1 -1
- package/knowledge/decisions/internals/speech-synthesis-picks-a-voice-per-sentence-and-warns-once.md +1 -1
- package/knowledge/decisions/internals/the-agent-done-event-goes-through-the-middleware-and-is-awaited.md +1 -1
- package/knowledge/decisions/internals/the-embedding-model-is-pinned-per-service-and-doc-query-embedding-is-split.md +1 -1
- package/knowledge/decisions/internals/the-per-invocation-rpc-view-is-a-class.md +5 -5
- package/knowledge/decisions/internals/the-transcript-event-is-sent-ahead-of-the-run.md +2 -2
- package/knowledge/decisions/internals/thread-history-records-the-transcript-not-the-audio.md +2 -2
- package/knowledge/decisions/internals/voice-output-speaks-unless-voice-input-explicitly-says-otherwise.md +1 -1
- package/knowledge/decisions/security/{ai-agent-approval-forwarding-requires-a-symbol-brand.md → agent-approval-forwarding-requires-a-symbol-brand.md} +3 -3
- package/knowledge/decisions/security/{ai-agent-credential-requests-are-symbol-branded.md → agent-credential-requests-are-symbol-branded.md} +4 -4
- package/knowledge/decisions/security/{ai-agent-gate-requires-a-session-only-when-auth-is-true.md → agent-gate-requires-a-session-only-when-auth-is-true.md} +3 -3
- package/knowledge/decisions/security/{ai-agent-ownership-failures-never-echo-the-resource.md → agent-ownership-failures-never-echo-the-resource.md} +2 -2
- package/knowledge/decisions/security/{ai-agent-resume-re-runs-the-authorization-gate.md → agent-resume-re-runs-the-authorization-gate.md} +4 -4
- package/knowledge/decisions/security/{ai-agent-sessionless-deployments-have-no-thread-ownership.md → agent-sessionless-deployments-have-no-thread-ownership.md} +5 -5
- package/knowledge/decisions/security/{ai-agent-thread-ownership-composes-the-session-principal.md → agent-thread-ownership-composes-the-session-principal.md} +2 -2
- package/knowledge/decisions/security/{ai-agent-tool-filtering-reads-the-live-function-config.md → agent-tool-filtering-reads-the-live-function-config.md} +2 -2
- package/knowledge/decisions/security/an-agent-approval-is-claimed-before-the-tool-runs.md +3 -3
- package/knowledge/decisions/security/an-empty-owners-constraint-matches-nothing.md +3 -3
- package/knowledge/decisions/security/console-addon-privileged-functions-gate-themselves.md +20 -13
- package/knowledge/decisions/security/index.md +1 -2
- package/package.json +4 -4
- package/scripts/generate-api-report.mts +32 -8
- package/src/agent-naming.test.ts +68 -0
- package/src/ecosystem/agent-scorer.ts +24 -0
- package/src/ecosystem/agent.ts +42 -0
- package/src/ecosystem/channel/local.ts +1 -0
- package/src/ecosystem/channel.ts +22 -0
- package/src/ecosystem/cli/channel.ts +25 -0
- package/src/ecosystem/cli.ts +21 -0
- package/src/ecosystem/credential.ts +10 -0
- package/src/ecosystem/dev.ts +12 -0
- package/src/ecosystem/function.ts +28 -0
- package/src/ecosystem/gateway.ts +4 -0
- package/src/ecosystem/hmac.ts +1 -0
- package/src/ecosystem/http.ts +22 -0
- package/src/ecosystem/mcp.ts +17 -0
- package/src/ecosystem/middleware.ts +27 -0
- package/src/ecosystem/node-host-resolver.ts +1 -0
- package/src/ecosystem/node.ts +5 -0
- package/src/ecosystem/oauth2.ts +4 -0
- package/src/ecosystem/persona.ts +30 -0
- package/src/ecosystem/queue.ts +23 -0
- package/src/ecosystem/remote.ts +9 -0
- package/src/ecosystem/role.ts +13 -0
- package/src/ecosystem/rpc.ts +7 -0
- package/src/ecosystem/safe-fetch.ts +8 -0
- package/src/ecosystem/scenario.ts +32 -0
- package/src/ecosystem/scheduler.ts +5 -0
- package/src/ecosystem/schema.ts +1 -0
- package/src/ecosystem/scope.ts +14 -0
- package/src/ecosystem/secret.ts +15 -0
- package/src/ecosystem/services/istanbul-coverage.ts +1 -0
- package/src/ecosystem/services/local-content-request-handler.ts +16 -0
- package/src/ecosystem/services/local-content.ts +1 -0
- package/src/ecosystem/services/v8-coverage.ts +1 -0
- package/src/ecosystem/services.ts +82 -0
- package/src/ecosystem/testing.ts +8 -0
- package/src/ecosystem/trigger.ts +7 -0
- package/src/ecosystem/types.ts +34 -0
- package/src/ecosystem/variable.ts +14 -0
- package/src/ecosystem/virtual-user.ts +48 -0
- package/src/ecosystem/workflow.ts +68 -0
- package/src/index.ts +8 -19
- package/src/pikku-state.ts +3 -3
- package/src/public-surface.json +183 -91
- package/src/public-surface.json.README +20 -2
- package/src/public-surface.test.ts +31 -3
- package/src/services/{ai-run-state-service.ts → agent-run-state-service.ts} +5 -5
- package/src/services/{ai-agent-runner-service.ts → agent-runner-service.ts} +18 -18
- package/src/services/{ai-storage-service.ts → agent-storage-service.ts} +7 -7
- package/src/services/http-personas-converse.test.ts +3 -3
- package/src/services/http-personas.ts +4 -4
- package/src/services/{in-memory-ai-run-state-service.ts → in-memory-agent-run-state-service.ts} +7 -7
- package/src/services/index.ts +13 -24
- package/src/services/meta-service.ts +5 -7
- package/src/testing/service-tests/agent-run-service-tests.ts +4 -4
- package/src/testing/service-tests/{ai-storage-service-tests.ts → agent-storage-service-tests.ts} +8 -8
- package/src/testing/service-tests.ts +10 -8
- package/src/types/core.types.ts +13 -13
- package/src/types/state.types.ts +11 -10
- package/src/wirings/actor-flow/run-conversation.test.ts +2 -2
- package/src/wirings/actor-flow/run-conversation.ts +7 -9
- package/src/wirings/{ai-agent/ai-agent-agui.test.ts → agent/agent-agui.test.ts} +126 -105
- package/src/wirings/{ai-agent/ai-agent-agui.ts → agent/agent-agui.ts} +7 -7
- package/src/wirings/{ai-agent/ai-agent-authorization.test.ts → agent/agent-authorization.test.ts} +6 -6
- package/src/wirings/{ai-agent/ai-agent-finalize.test.ts → agent/agent-finalize.test.ts} +17 -16
- package/src/wirings/{ai-agent/ai-agent-finalize.ts → agent/agent-finalize.ts} +22 -22
- package/src/wirings/{ai-agent/ai-agent-helpers.test.ts → agent/agent-helpers.test.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-helpers.ts → agent/agent-helpers.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-interrupt.test.ts → agent/agent-interrupt.test.ts} +67 -77
- package/src/wirings/{ai-agent/ai-agent-interrupt.ts → agent/agent-interrupt.ts} +5 -5
- package/src/wirings/{ai-agent/ai-agent-memory.test.ts → agent/agent-memory.test.ts} +12 -12
- package/src/wirings/{ai-agent/ai-agent-memory.ts → agent/agent-memory.ts} +36 -36
- package/src/wirings/{ai-agent/ai-agent-model-config.test.ts → agent/agent-model-config.test.ts} +2 -5
- package/src/wirings/{ai-agent/ai-agent-model-config.ts → agent/agent-model-config.ts} +1 -1
- package/src/wirings/{ai-agent/ai-agent-prepare.test.ts → agent/agent-prepare.test.ts} +18 -18
- package/src/wirings/{ai-agent/ai-agent-prepare.ts → agent/agent-prepare.ts} +54 -51
- package/src/wirings/{ai-agent/ai-agent-registry.test.ts → agent/agent-registry.test.ts} +49 -43
- package/src/wirings/{ai-agent/ai-agent-registry.ts → agent/agent-registry.ts} +14 -14
- package/src/wirings/{ai-agent/ai-agent-resume-authorization.test.ts → agent/agent-resume-authorization.test.ts} +18 -18
- package/src/wirings/{ai-agent → agent}/agent-rpc.ts +16 -20
- package/src/wirings/{ai-agent/ai-agent-runner.test.ts → agent/agent-runner.test.ts} +105 -106
- package/src/wirings/{ai-agent/ai-agent-runner.ts → agent/agent-runner.ts} +90 -80
- package/src/wirings/{ai-agent/ai-agent-stream-output-hooks.test.ts → agent/agent-stream-output-hooks.test.ts} +25 -25
- package/src/wirings/{ai-agent/ai-agent-stream.test.ts → agent/agent-stream.test.ts} +147 -143
- package/src/wirings/{ai-agent/ai-agent-stream.ts → agent/agent-stream.ts} +147 -146
- package/src/wirings/{ai-agent/ai-agent-thread-ownership.test.ts → agent/agent-thread-ownership.test.ts} +24 -28
- package/src/wirings/{ai-agent/ai-agent-turn.test.ts → agent/agent-turn.test.ts} +3 -5
- package/src/wirings/{ai-agent/ai-agent-turn.ts → agent/agent-turn.ts} +7 -7
- package/src/wirings/{ai-agent/ai-agent.types.ts → agent/agent.types.ts} +51 -48
- package/src/wirings/agent/index.ts +56 -0
- package/src/wirings/{ai-agent → agent}/voice-input.test.ts +15 -15
- package/src/wirings/{ai-agent → agent}/voice-input.ts +11 -11
- package/src/wirings/{ai-agent → agent}/voice-output.test.ts +23 -23
- package/src/wirings/{ai-agent → agent}/voice-output.ts +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-grade.test.ts → agent-scorer/agent-scorer-grade.test.ts} +8 -8
- package/src/wirings/{ai-scorer/ai-scorer-grade.ts → agent-scorer/agent-scorer-grade.ts} +9 -9
- package/src/wirings/{ai-scorer/ai-scorer-judge.test.ts → agent-scorer/agent-scorer-judge.test.ts} +11 -14
- package/src/wirings/{ai-scorer/ai-scorer-judge.ts → agent-scorer/agent-scorer-judge.ts} +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-live.test.ts → agent-scorer/agent-scorer-live.test.ts} +11 -11
- package/src/wirings/{ai-scorer/ai-scorer-live.ts → agent-scorer/agent-scorer-live.ts} +3 -3
- package/src/wirings/{ai-scorer/ai-scorer-registry.ts → agent-scorer/agent-scorer-registry.ts} +11 -8
- package/src/wirings/{ai-scorer/ai-scorer-sampling.test.ts → agent-scorer/agent-scorer-sampling.test.ts} +1 -1
- package/src/wirings/{ai-scorer/ai-scorer-snapshots.test.ts → agent-scorer/agent-scorer-snapshots.test.ts} +2 -2
- package/src/wirings/{ai-scorer/ai-scorer-snapshots.ts → agent-scorer/agent-scorer-snapshots.ts} +1 -1
- package/src/wirings/{ai-scorer/ai-scorer-worker.test.ts → agent-scorer/agent-scorer-worker.test.ts} +15 -15
- package/src/wirings/{ai-scorer/ai-scorer-worker.ts → agent-scorer/agent-scorer-worker.ts} +9 -7
- package/src/wirings/{ai-scorer/ai-scorer.ts → agent-scorer/agent-scorer.ts} +7 -7
- package/src/wirings/{ai-scorer/ai-scorer.types.ts → agent-scorer/agent-scorer.types.ts} +6 -6
- package/src/wirings/agent-scorer/index.ts +22 -0
- package/src/wirings/channel/channel-rpc.types.ts +2 -2
- package/src/wirings/rpc/addon-runner.ts +31 -3
- package/src/wirings/rpc/addon-secrets.test.ts +130 -0
- package/src/wirings/rpc/rpc-runner.ts +4 -2
- package/src/wirings/rpc/rpc-types.ts +5 -1
- package/src/wirings/rpc/wire-addon.ts +16 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.ts +1 -1
- package/src/wirings/virtual-user/run-virtual-user.test.ts +2 -2
- package/src/wirings/virtual-user/run-virtual-user.ts +3 -3
- package/src/wirings/virtual-user/virtual-user-agents.ts +1 -1
- package/src/wirings/workflow/feature.ts +2 -5
- package/src/wirings/workflow/index.ts +1 -49
- package/src/wirings/workflow/pikku-scenario-service.ts +21 -1
- package/src/wirings/workflow/run-timeline.ts +1 -1
- package/src/wirings/workflow/scenario.types.ts +63 -0
- package/src/wirings/workflow/workflow.types.ts +1 -54
- package/src/wirings-stay-decoupled.test.ts +3 -3
- package/tsconfig.tsbuildinfo +1 -1
- package/src/wirings/ai-agent/index.ts +0 -65
- package/src/wirings/ai-scorer/index.ts +0 -24
- /package/src/wirings/{ai-agent/ai-agent-utils.ts → agent/agent-utils.ts} +0 -0
- /package/src/wirings/{ai-scorer/ai-scorer-sampling.ts → agent-scorer/agent-scorer-sampling.ts} +0 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { CoreSingletonServices } from '../../types/core.types.js';
|
|
2
|
+
import { type ScorerInput } from './agent-scorer.types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Grade one finished run on live traffic.
|
|
5
|
+
*
|
|
6
|
+
* Called as the terminal step of `finalizeAgentRun`, so it is structurally last
|
|
7
|
+
* — a developer cannot register anything after it, and cannot reorder it. It is
|
|
8
|
+
* also strictly best-effort: the client already has its answer, so nothing here
|
|
9
|
+
* may fail the run.
|
|
10
|
+
*/
|
|
11
|
+
export declare const scoreFinishedRun: (run: ScorerInput, services: Pick<CoreSingletonServices, "logger"> & {
|
|
12
|
+
queueService?: {
|
|
13
|
+
add: (queueName: string, data: unknown) => Promise<any>;
|
|
14
|
+
};
|
|
15
|
+
}) => Promise<void>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { scorersForAgent } from './agent-scorer-registry.js';
|
|
2
|
+
import { isSampled } from './agent-scorer-sampling.js';
|
|
3
|
+
import { SCORER_LANE_QUEUES } from './agent-scorer.types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Grade one finished run on live traffic.
|
|
6
|
+
*
|
|
7
|
+
* Called as the terminal step of `finalizeAgentRun`, so it is structurally last
|
|
8
|
+
* — a developer cannot register anything after it, and cannot reorder it. It is
|
|
9
|
+
* also strictly best-effort: the client already has its answer, so nothing here
|
|
10
|
+
* may fail the run.
|
|
11
|
+
*/
|
|
12
|
+
export const scoreFinishedRun = async (run, services) => {
|
|
13
|
+
const scorers = scorersForAgent(run.agentName, services.logger).filter(
|
|
14
|
+
// A reference-based judge grades against an answer key, and live traffic
|
|
15
|
+
// has none.
|
|
16
|
+
(scorer) => !scorer.requiresReference);
|
|
17
|
+
if (scorers.length === 0)
|
|
18
|
+
return;
|
|
19
|
+
if (!services.queueService) {
|
|
20
|
+
services.logger?.warn(`[pikku] Agent '${run.agentName}' declares scorers but no queue service is registered — skipping live scoring`);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const sampled = scorers.filter((scorer) => isSampled(run.runId, scorer.name, scorer.sampleRate));
|
|
24
|
+
// One message per scorer, so each gets its own retry, isolation and lane.
|
|
25
|
+
await Promise.all(sampled.map(async (scorer) => {
|
|
26
|
+
try {
|
|
27
|
+
await services.queueService.add(SCORER_LANE_QUEUES[scorer.lane], {
|
|
28
|
+
...run,
|
|
29
|
+
scorerName: scorer.name,
|
|
30
|
+
// No reference: a live run has no answer key.
|
|
31
|
+
reference: undefined,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
services.logger?.error(`[pikku] Failed to enqueue the '${scorer.name}' grade of run ${run.runId}`, { error });
|
|
36
|
+
}
|
|
37
|
+
}));
|
|
38
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { PikkuAgentScorer } from './agent-scorer.types.js';
|
|
2
|
+
export declare const addAgentScorer: (scorerName: string, scorer: PikkuAgentScorer<any>, packageName?: string | null) => void;
|
|
3
|
+
export declare const getAgentScorers: () => Map<string, PikkuAgentScorer>;
|
|
4
|
+
export declare const getAgentScorersMeta: () => import("./agent-scorer.types.js").ScorerMeta;
|
|
5
|
+
/**
|
|
6
|
+
* Resolve a scorer by name across every registered package, the way an agent's
|
|
7
|
+
* tools are resolved: a scorer declared in an addon is nameable by an app agent.
|
|
8
|
+
*/
|
|
9
|
+
export declare const resolveAgentScorer: (scorerName: string) => PikkuAgentScorer<any>;
|
|
10
|
+
/**
|
|
11
|
+
* The scorers an agent asked to be graded by.
|
|
12
|
+
*
|
|
13
|
+
* A name that resolves to nothing is warned about rather than thrown: a missing
|
|
14
|
+
* scorer must not take down a run that has already answered the user.
|
|
15
|
+
*/
|
|
16
|
+
export declare const scorersForAgent: (agentName: string, logger?: {
|
|
17
|
+
warn: (message: string) => void;
|
|
18
|
+
}) => PikkuAgentScorer<any>[];
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { pikkuState } from '../../pikku-state.js';
|
|
2
|
+
export const addAgentScorer = (scorerName, scorer, packageName = null) => {
|
|
3
|
+
const scorersMeta = pikkuState(packageName, 'agent', 'scorersMeta');
|
|
4
|
+
if (!scorersMeta[scorerName]) {
|
|
5
|
+
console.warn(`[pikku] Skipping AI scorer '${scorerName}' — metadata not found. Scorers must be declared in a *.scorer.ts file.`);
|
|
6
|
+
return;
|
|
7
|
+
}
|
|
8
|
+
const scorers = pikkuState(packageName, 'agent', 'scorers');
|
|
9
|
+
if (scorers.has(scorerName)) {
|
|
10
|
+
throw new Error(`AI scorer already exists: ${scorerName}`);
|
|
11
|
+
}
|
|
12
|
+
scorers.set(scorerName, scorer);
|
|
13
|
+
};
|
|
14
|
+
export const getAgentScorers = () => pikkuState(null, 'agent', 'scorers');
|
|
15
|
+
export const getAgentScorersMeta = () => pikkuState(null, 'agent', 'scorersMeta');
|
|
16
|
+
/**
|
|
17
|
+
* Resolve a scorer by name across every registered package, the way an agent's
|
|
18
|
+
* tools are resolved: a scorer declared in an addon is nameable by an app agent.
|
|
19
|
+
*/
|
|
20
|
+
export const resolveAgentScorer = (scorerName) => {
|
|
21
|
+
const scorer = pikkuState(null, 'agent', 'scorers').get(scorerName);
|
|
22
|
+
if (!scorer) {
|
|
23
|
+
throw new Error(`AI scorer not found: ${scorerName}`);
|
|
24
|
+
}
|
|
25
|
+
return scorer;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* The scorers an agent asked to be graded by.
|
|
29
|
+
*
|
|
30
|
+
* A name that resolves to nothing is warned about rather than thrown: a missing
|
|
31
|
+
* scorer must not take down a run that has already answered the user.
|
|
32
|
+
*/
|
|
33
|
+
export const scorersForAgent = (agentName, logger) => {
|
|
34
|
+
const agent = pikkuState(null, 'agent', 'agents').get(agentName);
|
|
35
|
+
const names = agent?.scorers ?? [];
|
|
36
|
+
const scorers = [];
|
|
37
|
+
for (const name of names) {
|
|
38
|
+
const scorer = pikkuState(null, 'agent', 'scorers').get(name);
|
|
39
|
+
if (!scorer) {
|
|
40
|
+
logger?.warn(`[pikku] Agent '${agentName}' names scorer '${name}', which is not registered — skipping it`);
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
scorers.push(scorer);
|
|
44
|
+
}
|
|
45
|
+
return scorers;
|
|
46
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decide whether one run is in a scorer's sample.
|
|
3
|
+
*
|
|
4
|
+
* Deterministic on `(runId, scorerName)` rather than random, so re-running the
|
|
5
|
+
* decision — a retried job, a replayed run, a test — always lands the same way,
|
|
6
|
+
* and so two scorers at the same rate do not sample the same runs.
|
|
7
|
+
*/
|
|
8
|
+
export declare const isSampled: (runId: string, scorerName: string, sampleRate: number) => boolean;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decide whether one run is in a scorer's sample.
|
|
3
|
+
*
|
|
4
|
+
* Deterministic on `(runId, scorerName)` rather than random, so re-running the
|
|
5
|
+
* decision — a retried job, a replayed run, a test — always lands the same way,
|
|
6
|
+
* and so two scorers at the same rate do not sample the same runs.
|
|
7
|
+
*/
|
|
8
|
+
export const isSampled = (runId, scorerName, sampleRate) => {
|
|
9
|
+
if (sampleRate >= 1)
|
|
10
|
+
return true;
|
|
11
|
+
if (sampleRate <= 0)
|
|
12
|
+
return false;
|
|
13
|
+
const key = `${scorerName}:${runId}`;
|
|
14
|
+
// FNV-1a: no crypto dependency, and stable across processes and platforms,
|
|
15
|
+
// which a language-level string hash is not guaranteed to be.
|
|
16
|
+
let hash = 0x811c9dc5;
|
|
17
|
+
for (let i = 0; i < key.length; i++) {
|
|
18
|
+
hash ^= key.charCodeAt(i);
|
|
19
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
20
|
+
}
|
|
21
|
+
// FNV-1a alone leaves its high bits barely moved between keys that differ
|
|
22
|
+
// only in their last characters — sequential run ids being exactly that —
|
|
23
|
+
// which biases the fraction actually sampled well away from the rate asked
|
|
24
|
+
// for. MurmurHash3's finalizer spreads the low bits back over the whole word.
|
|
25
|
+
hash ^= hash >>> 16;
|
|
26
|
+
hash = Math.imul(hash, 0x85ebca6b) >>> 0;
|
|
27
|
+
hash ^= hash >>> 13;
|
|
28
|
+
hash = Math.imul(hash, 0xc2b2ae35) >>> 0;
|
|
29
|
+
hash ^= hash >>> 16;
|
|
30
|
+
return (hash >>> 0) / 0x100000000 < sampleRate;
|
|
31
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ScorerInput } from './agent-scorer.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Start retaining snapshots. Called where the scenario grading RPC is
|
|
4
|
+
* registered, so the buffer exists in exactly the processes that can read it —
|
|
5
|
+
* a development server — and never in a deployed bundle.
|
|
6
|
+
*/
|
|
7
|
+
export declare const enableScoreSnapshots: (maxRuns?: number) => void;
|
|
8
|
+
export declare const recordScoreSnapshot: (run: ScorerInput) => void;
|
|
9
|
+
export declare const getScoreSnapshot: (runId: string) => ScorerInput | undefined;
|
|
10
|
+
export declare const resetScoreSnapshots: () => void;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The last few finished runs, held so a scenario can grade one.
|
|
3
|
+
*
|
|
4
|
+
* A scenario cannot rebuild a `ScorerInput` from storage: the run record holds
|
|
5
|
+
* status and usage, and the prompt, answer and tool calls are spread across the
|
|
6
|
+
* thread's messages, where the boundary of a single run is not recoverable. So
|
|
7
|
+
* the runtime keeps the snapshot it already took at finalize — which is also
|
|
8
|
+
* what makes a scenario's grade comparable to a live one, since both grade the
|
|
9
|
+
* identical object, redactions included.
|
|
10
|
+
*
|
|
11
|
+
* Off unless something turns it on, and bounded when it is. A process that
|
|
12
|
+
* grades no scenarios holds nothing, and one that does cannot accumulate run
|
|
13
|
+
* content without limit.
|
|
14
|
+
*/
|
|
15
|
+
const snapshots = new Map();
|
|
16
|
+
let limit = 0;
|
|
17
|
+
/**
|
|
18
|
+
* Start retaining snapshots. Called where the scenario grading RPC is
|
|
19
|
+
* registered, so the buffer exists in exactly the processes that can read it —
|
|
20
|
+
* a development server — and never in a deployed bundle.
|
|
21
|
+
*/
|
|
22
|
+
export const enableScoreSnapshots = (maxRuns = 50) => {
|
|
23
|
+
limit = maxRuns;
|
|
24
|
+
};
|
|
25
|
+
export const recordScoreSnapshot = (run) => {
|
|
26
|
+
if (limit === 0)
|
|
27
|
+
return;
|
|
28
|
+
snapshots.set(run.runId, run);
|
|
29
|
+
while (snapshots.size > limit) {
|
|
30
|
+
const oldest = snapshots.keys().next().value;
|
|
31
|
+
if (oldest === undefined)
|
|
32
|
+
break;
|
|
33
|
+
snapshots.delete(oldest);
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
export const getScoreSnapshot = (runId) => snapshots.get(runId);
|
|
37
|
+
export const resetScoreSnapshots = () => {
|
|
38
|
+
snapshots.clear();
|
|
39
|
+
limit = 0;
|
|
40
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type ScoreJob } from './agent-scorer.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The one worker behind both lanes: it resolves the scorer by name and grades.
|
|
4
|
+
* The lane a job arrived on decides nothing except how long it may take.
|
|
5
|
+
*/
|
|
6
|
+
export declare function pikkuAgentScoreWorkerFunc(_services: Record<string, unknown>, job: ScoreJob): Promise<void>;
|
|
7
|
+
/**
|
|
8
|
+
* Bind the two lane queues to the shared worker.
|
|
9
|
+
*
|
|
10
|
+
* Registered programmatically rather than emitted into the user's bootstrap:
|
|
11
|
+
* `wireQueueWorker` warns and returns for a name codegen produced no metadata
|
|
12
|
+
* for, so the metadata is synthesised here alongside the registration. This
|
|
13
|
+
* mirrors how workflows wire their own queues.
|
|
14
|
+
*/
|
|
15
|
+
export declare const wireAgentScorerQueueWorkers: () => void;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { pikkuState } from '../../pikku-state.js';
|
|
2
|
+
import { addFunction } from '../../function/function-runner.js';
|
|
3
|
+
import { wireQueueWorker } from '../queue/queue-runner.js';
|
|
4
|
+
import { gradeRun } from './agent-scorer-grade.js';
|
|
5
|
+
import { SCORER_LANE_QUEUES } from './agent-scorer.types.js';
|
|
6
|
+
/**
|
|
7
|
+
* The one worker behind both lanes: it resolves the scorer by name and grades.
|
|
8
|
+
* The lane a job arrived on decides nothing except how long it may take.
|
|
9
|
+
*/
|
|
10
|
+
export async function pikkuAgentScoreWorkerFunc(_services, job) {
|
|
11
|
+
const services = pikkuState(null, 'package', 'singletonServices');
|
|
12
|
+
if (!services) {
|
|
13
|
+
throw new Error(`Singleton services not initialized: cannot run the '${job.scorerName}' grade of run ${job.runId}`);
|
|
14
|
+
}
|
|
15
|
+
await gradeRun(job, services, { persist: true });
|
|
16
|
+
}
|
|
17
|
+
const SCORE_WORKER_FUNC_ID = 'pikkuAgentScoreWorker';
|
|
18
|
+
/**
|
|
19
|
+
* Bind the two lane queues to the shared worker.
|
|
20
|
+
*
|
|
21
|
+
* Registered programmatically rather than emitted into the user's bootstrap:
|
|
22
|
+
* `wireQueueWorker` warns and returns for a name codegen produced no metadata
|
|
23
|
+
* for, so the metadata is synthesised here alongside the registration. This
|
|
24
|
+
* mirrors how workflows wire their own queues.
|
|
25
|
+
*/
|
|
26
|
+
export const wireAgentScorerQueueWorkers = () => {
|
|
27
|
+
// No scorers means no lanes: a deployment that grades nothing should not be
|
|
28
|
+
// left holding two queues nothing ever writes to.
|
|
29
|
+
if (pikkuState(null, 'agent', 'scorers').size === 0)
|
|
30
|
+
return;
|
|
31
|
+
const functions = pikkuState(null, 'function', 'functions');
|
|
32
|
+
const functionsMeta = pikkuState(null, 'function', 'meta');
|
|
33
|
+
const queueMeta = pikkuState(null, 'queue', 'meta');
|
|
34
|
+
if (!functions.has(SCORE_WORKER_FUNC_ID)) {
|
|
35
|
+
addFunction(SCORE_WORKER_FUNC_ID, {
|
|
36
|
+
func: pikkuAgentScoreWorkerFunc,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
if (!functionsMeta[SCORE_WORKER_FUNC_ID]) {
|
|
40
|
+
functionsMeta[SCORE_WORKER_FUNC_ID] = {
|
|
41
|
+
pikkuFuncId: SCORE_WORKER_FUNC_ID,
|
|
42
|
+
sessionless: true,
|
|
43
|
+
functionType: 'helper',
|
|
44
|
+
inputSchemaName: null,
|
|
45
|
+
outputSchemaName: null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
for (const queueName of Object.values(SCORER_LANE_QUEUES)) {
|
|
49
|
+
if (!queueMeta[queueName]) {
|
|
50
|
+
queueMeta[queueName] = {
|
|
51
|
+
pikkuFuncId: SCORE_WORKER_FUNC_ID,
|
|
52
|
+
name: queueName,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
wireQueueWorker({
|
|
56
|
+
name: queueName,
|
|
57
|
+
func: { func: pikkuAgentScoreWorkerFunc },
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { PikkuAgentScorer, ScorerInput, ScorerOutput } from './agent-scorer.types.js';
|
|
2
|
+
/**
|
|
3
|
+
* A heuristic scorer: pure code over the finished run, no model call, so it
|
|
4
|
+
* grades on the fast lane.
|
|
5
|
+
*/
|
|
6
|
+
export declare const pikkuAgentScorer: <Services = any>(config: {
|
|
7
|
+
name: string;
|
|
8
|
+
description: string;
|
|
9
|
+
/** 0..1 fraction of live runs to grade. Defaults to all of them. */
|
|
10
|
+
sampleRate?: number;
|
|
11
|
+
/**
|
|
12
|
+
* Grades against a known-correct answer. Such a scorer is test-only — live
|
|
13
|
+
* traffic has no answer key, so the runtime never samples it.
|
|
14
|
+
*/
|
|
15
|
+
requiresReference?: boolean;
|
|
16
|
+
score: (input: ScorerInput, services: Services) => ScorerOutput | Promise<ScorerOutput>;
|
|
17
|
+
}) => PikkuAgentScorer<Services>;
|
|
18
|
+
/**
|
|
19
|
+
* An LLM-judge scorer: the runtime makes the model call and forces a structured
|
|
20
|
+
* `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
|
|
21
|
+
*
|
|
22
|
+
* The rubric field is `goal`, matching `pikkuAgent`'s prompt vocabulary — a
|
|
23
|
+
* judge is a degenerate agent, and should use the same word for the same thing.
|
|
24
|
+
* `prompt` is the escape hatch for non-standard framing.
|
|
25
|
+
*/
|
|
26
|
+
export declare const pikkuAgentJudge: <Services = any>(config: {
|
|
27
|
+
name: string;
|
|
28
|
+
description: string;
|
|
29
|
+
/** 0..1 fraction of live runs to grade. Defaults to all of them. */
|
|
30
|
+
sampleRate?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Grades against a known-correct answer. Such a judge is test-only — live
|
|
33
|
+
* traffic has no answer key, so the runtime never samples it.
|
|
34
|
+
*/
|
|
35
|
+
requiresReference?: boolean;
|
|
36
|
+
model: string;
|
|
37
|
+
goal: string;
|
|
38
|
+
prompt?: (input: ScorerInput) => string;
|
|
39
|
+
}) => PikkuAgentScorer<Services>;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const assertSampleRate = (name, sampleRate) => {
|
|
2
|
+
if (sampleRate === undefined)
|
|
3
|
+
return 1;
|
|
4
|
+
if (sampleRate < 0 || sampleRate > 1) {
|
|
5
|
+
throw new Error(`Scorer '${name}' has a sampleRate of ${sampleRate} — it is a fraction of live runs to grade, so it must be between 0 and 1`);
|
|
6
|
+
}
|
|
7
|
+
return sampleRate;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* A heuristic scorer: pure code over the finished run, no model call, so it
|
|
11
|
+
* grades on the fast lane.
|
|
12
|
+
*/
|
|
13
|
+
export const pikkuAgentScorer = (config) => ({
|
|
14
|
+
name: config.name,
|
|
15
|
+
description: config.description,
|
|
16
|
+
lane: 'fast',
|
|
17
|
+
sampleRate: assertSampleRate(config.name, config.sampleRate),
|
|
18
|
+
requiresReference: config.requiresReference ?? false,
|
|
19
|
+
score: config.score,
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* An LLM-judge scorer: the runtime makes the model call and forces a structured
|
|
23
|
+
* `{ score, reason }`, so a judge is a rubric rather than a prompt to parse.
|
|
24
|
+
*
|
|
25
|
+
* The rubric field is `goal`, matching `pikkuAgent`'s prompt vocabulary — a
|
|
26
|
+
* judge is a degenerate agent, and should use the same word for the same thing.
|
|
27
|
+
* `prompt` is the escape hatch for non-standard framing.
|
|
28
|
+
*/
|
|
29
|
+
export const pikkuAgentJudge = (config) => ({
|
|
30
|
+
name: config.name,
|
|
31
|
+
description: config.description,
|
|
32
|
+
lane: 'slow',
|
|
33
|
+
sampleRate: assertSampleRate(config.name, config.sampleRate),
|
|
34
|
+
requiresReference: config.requiresReference ?? false,
|
|
35
|
+
judge: {
|
|
36
|
+
model: config.model,
|
|
37
|
+
goal: config.goal,
|
|
38
|
+
...(config.prompt ? { prompt: config.prompt } : {}),
|
|
39
|
+
},
|
|
40
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a scorer is shown about a finished run.
|
|
3
|
+
*
|
|
4
|
+
* One snapshot, taken after the output middleware has resolved, so a scorer and
|
|
5
|
+
* the persisted run record see the same thing — including the redactions.
|
|
6
|
+
*/
|
|
7
|
+
export interface ScorerInput {
|
|
8
|
+
runId: string;
|
|
9
|
+
agentName: string;
|
|
10
|
+
threadId?: string;
|
|
11
|
+
resourceId?: string;
|
|
12
|
+
/** The user prompt the run answered. */
|
|
13
|
+
input: string;
|
|
14
|
+
/** The agent's final text answer. */
|
|
15
|
+
output: string;
|
|
16
|
+
/**
|
|
17
|
+
* The known-correct answer. Supplied by a scenario for a reference-based
|
|
18
|
+
* judge; never set on live traffic, which has no answer key.
|
|
19
|
+
*/
|
|
20
|
+
reference?: string;
|
|
21
|
+
toolCalls: {
|
|
22
|
+
name: string;
|
|
23
|
+
args: unknown;
|
|
24
|
+
result?: unknown;
|
|
25
|
+
error?: string;
|
|
26
|
+
}[];
|
|
27
|
+
usage: {
|
|
28
|
+
inputTokens: number;
|
|
29
|
+
outputTokens: number;
|
|
30
|
+
model?: string;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export interface ScorerOutput {
|
|
34
|
+
/** 0..1, so grades are comparable across scorers. */
|
|
35
|
+
score: number;
|
|
36
|
+
reason?: string;
|
|
37
|
+
metadata?: Record<string, unknown>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The queue a scorer's jobs go to. Two lanes exist only so a flood of slow
|
|
41
|
+
* LLM-judge jobs cannot starve the cheap heuristic ones; within a lane it is
|
|
42
|
+
* plain FIFO.
|
|
43
|
+
*/
|
|
44
|
+
export type ScorerLane = 'fast' | 'slow';
|
|
45
|
+
export declare const SCORER_LANE_QUEUES: Record<ScorerLane, string>;
|
|
46
|
+
/**
|
|
47
|
+
* A judge's model call. Present only on a scorer built with `pikkuAgentJudge`;
|
|
48
|
+
* `score` is present only on one built with `pikkuAgentScorer`. Exactly one of the
|
|
49
|
+
* two is set, which is what the two constructors exist to guarantee.
|
|
50
|
+
*/
|
|
51
|
+
export type ScorerJudgeConfig = {
|
|
52
|
+
model: string;
|
|
53
|
+
goal: string;
|
|
54
|
+
prompt?: (input: ScorerInput) => string;
|
|
55
|
+
};
|
|
56
|
+
export type PikkuAgentScorer<Services = any> = {
|
|
57
|
+
name: string;
|
|
58
|
+
description: string;
|
|
59
|
+
lane: ScorerLane;
|
|
60
|
+
/** 0..1 fraction of live runs to grade. */
|
|
61
|
+
sampleRate: number;
|
|
62
|
+
/**
|
|
63
|
+
* Grades against a known-correct answer, so it is test-only: live traffic has
|
|
64
|
+
* no answer key and the runtime never samples it.
|
|
65
|
+
*/
|
|
66
|
+
requiresReference: boolean;
|
|
67
|
+
score?: (input: ScorerInput, services: Services) => ScorerOutput | Promise<ScorerOutput>;
|
|
68
|
+
judge?: ScorerJudgeConfig;
|
|
69
|
+
};
|
|
70
|
+
export type ScorerMeta = Record<string, {
|
|
71
|
+
name: string;
|
|
72
|
+
description: string;
|
|
73
|
+
lane: ScorerLane;
|
|
74
|
+
sampleRate: number;
|
|
75
|
+
requiresReference: boolean;
|
|
76
|
+
sourceFile?: string;
|
|
77
|
+
exportedName?: string;
|
|
78
|
+
}>;
|
|
79
|
+
/** A single scorer's job on a lane queue. */
|
|
80
|
+
export type ScoreJob = ScorerInput & {
|
|
81
|
+
scorerName: string;
|
|
82
|
+
};
|
|
83
|
+
export type AgentRunScore = {
|
|
84
|
+
runId: string;
|
|
85
|
+
scorerName: string;
|
|
86
|
+
score: number;
|
|
87
|
+
reason?: string;
|
|
88
|
+
metadata?: Record<string, unknown>;
|
|
89
|
+
createdAt: Date;
|
|
90
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { pikkuAgentScorer, pikkuAgentJudge } from './agent-scorer.js';
|
|
2
|
+
export { addAgentScorer, getAgentScorers, getAgentScorersMeta, } from './agent-scorer-registry.js';
|
|
3
|
+
export { gradeRun } from './agent-scorer-grade.js';
|
|
4
|
+
export { enableScoreSnapshots, getScoreSnapshot, } from './agent-scorer-snapshots.js';
|
|
5
|
+
export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js';
|
|
6
|
+
export type { AgentRunScore, PikkuAgentScorer, ScoreJob, ScorerInput, ScorerJudgeConfig, ScorerLane, ScorerMeta, ScorerOutput, } from './agent-scorer.types.js';
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { pikkuAgentScorer, pikkuAgentJudge } from './agent-scorer.js';
|
|
2
|
+
export { addAgentScorer, getAgentScorers, getAgentScorersMeta, } from './agent-scorer-registry.js';
|
|
3
|
+
export { gradeRun } from './agent-scorer-grade.js';
|
|
4
|
+
export { enableScoreSnapshots, getScoreSnapshot, } from './agent-scorer-snapshots.js';
|
|
5
|
+
export { wireAgentScorerQueueWorkers } from './agent-scorer-worker.js';
|
|
@@ -357,6 +357,8 @@ export async function buildToolDefs(params, agentSessionMap, resourceId, agentNa
|
|
|
357
357
|
secretOverrides: capturedAddonConfig?.secretOverrides,
|
|
358
358
|
variableOverrides: capturedAddonConfig?.variableOverrides,
|
|
359
359
|
credentialOverrides: capturedAddonConfig?.credentialOverrides,
|
|
360
|
+
secretGrants: capturedAddonConfig?.secretGrants,
|
|
361
|
+
credentialGrants: capturedAddonConfig?.credentialGrants,
|
|
360
362
|
globalSecrets: capturedAddonConfig?.globalSecrets,
|
|
361
363
|
globalCredentials: capturedAddonConfig?.globalCredentials,
|
|
362
364
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type VoiceOutputState = {
|
|
1
|
+
export type VoiceOutputState = {
|
|
2
2
|
textBuffer?: string;
|
|
3
3
|
/**
|
|
4
4
|
* The audio emitted so far, as a chain.
|
|
@@ -83,4 +83,3 @@ export declare const voiceOutput: (config?: {
|
|
|
83
83
|
webhook?: import("../../services/webhook-service.js").WebhookServiceConfig;
|
|
84
84
|
postgres?: import("../../types/core.types.js").PostgresConfig;
|
|
85
85
|
}>>;
|
|
86
|
-
export {};
|
|
@@ -51,7 +51,7 @@ export declare class ChannelRPCError extends Error {
|
|
|
51
51
|
* Whether a callable exposed to a remote decider needs a human to agree before
|
|
52
52
|
* it runs, and how to describe the invocation while asking.
|
|
53
53
|
*
|
|
54
|
-
* Shared with `
|
|
54
|
+
* Shared with `AgentToolDef`: both are an allowlist of named callables
|
|
55
55
|
* invoked by something other than the code that wrote them. The runtime around
|
|
56
56
|
* them is not shared — an agent suspends and resumes, a reverse call is a live
|
|
57
57
|
* await.
|
|
@@ -62,7 +62,7 @@ export interface ApprovalPolicy {
|
|
|
62
62
|
}
|
|
63
63
|
export type CapabilityHandler = (data: any) => Promise<unknown> | unknown;
|
|
64
64
|
/**
|
|
65
|
-
* `needsApproval` is required here on purpose. On `
|
|
65
|
+
* `needsApproval` is required here on purpose. On `AgentToolDef` the same
|
|
66
66
|
* field is optional and absence means "do not ask"; here absence means the
|
|
67
67
|
* opposite, so it must not be expressible — a bare function is the
|
|
68
68
|
* unclassified form.
|
|
@@ -4,6 +4,10 @@ export type AddonInstance = {
|
|
|
4
4
|
secretOverrides?: Record<string, string>;
|
|
5
5
|
variableOverrides?: Record<string, string>;
|
|
6
6
|
credentialOverrides?: Record<string, string>;
|
|
7
|
+
/** Set by the consuming app: secrets it lends this instance, as the addon names them. */
|
|
8
|
+
secretGrants?: string[];
|
|
9
|
+
/** Set by the consuming app: credentials it lends this instance, as the addon names them. */
|
|
10
|
+
credentialGrants?: string[];
|
|
7
11
|
/** Set by the consuming app to opt this instance out of secret scoping. */
|
|
8
12
|
globalSecrets?: string;
|
|
9
13
|
/** Set by the consuming app to opt this instance out of credential scoping. */
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import { pikkuState } from '../../pikku-state.js';
|
|
2
2
|
import { ScopedSecretService } from '../../services/scoped-secret-service.js';
|
|
3
3
|
import { ScopedCredentialService } from '../../services/scoped-credential-service.js';
|
|
4
|
+
/**
|
|
5
|
+
* What the addon declared, plus what the host lent it. Scoping runs outside the
|
|
6
|
+
* aliaser, so every name here is the one the addon reads — which is why an
|
|
7
|
+
* override's *key* grants, and its value does not.
|
|
8
|
+
*/
|
|
9
|
+
const allowedNames = (declared, grants, overrides) => new Set([
|
|
10
|
+
...(declared ?? []),
|
|
11
|
+
...(grants ?? []),
|
|
12
|
+
...Object.keys(overrides ?? {}),
|
|
13
|
+
]);
|
|
4
14
|
const aliasSecretService = (secrets, overrides) => {
|
|
5
15
|
const map = (key) => overrides[key] ?? key;
|
|
6
16
|
return {
|
|
@@ -97,13 +107,13 @@ export const getOrCreatePackageSingletonServices = async (packageName, parentSer
|
|
|
97
107
|
if (!addonInstance?.globalSecrets && existingServices.secrets) {
|
|
98
108
|
existingServices = {
|
|
99
109
|
...existingServices,
|
|
100
|
-
secrets: new ScopedSecretService(existingServices.secrets,
|
|
110
|
+
secrets: new ScopedSecretService(existingServices.secrets, allowedNames(pikkuState(packageName, 'package', 'declaredSecrets'), addonInstance?.secretGrants, addonInstance?.secretOverrides)),
|
|
101
111
|
};
|
|
102
112
|
}
|
|
103
113
|
if (!addonInstance?.globalCredentials && existingServices.credentialService) {
|
|
104
114
|
existingServices = {
|
|
105
115
|
...existingServices,
|
|
106
|
-
credentialService: new ScopedCredentialService(existingServices.credentialService,
|
|
116
|
+
credentialService: new ScopedCredentialService(existingServices.credentialService, allowedNames(Object.keys(pikkuState(packageName, 'package', 'credentialsMeta') ?? {}), addonInstance?.credentialGrants, addonInstance?.credentialOverrides)),
|
|
107
117
|
};
|
|
108
118
|
}
|
|
109
119
|
if (!factories || !factories.createSingletonServices) {
|
|
@@ -132,6 +142,8 @@ export const addonInstanceForNamespace = (namespace, expectedPackage) => {
|
|
|
132
142
|
secretOverrides: cfg.secretOverrides,
|
|
133
143
|
variableOverrides: cfg.variableOverrides,
|
|
134
144
|
credentialOverrides: cfg.credentialOverrides,
|
|
145
|
+
secretGrants: cfg.secretGrants,
|
|
146
|
+
credentialGrants: cfg.credentialGrants,
|
|
135
147
|
globalSecrets: cfg.globalSecrets,
|
|
136
148
|
globalCredentials: cfg.globalCredentials,
|
|
137
149
|
};
|
|
@@ -52,7 +52,7 @@ export declare class ContextAwareRPCService {
|
|
|
52
52
|
/**
|
|
53
53
|
* The agent facade, built on access.
|
|
54
54
|
*
|
|
55
|
-
* The implementation lives in `
|
|
55
|
+
* The implementation lives in `agent/agent-rpc.ts` so the agent surface is
|
|
56
56
|
* one file rather than a wing of this one; a getter rather than a field so a
|
|
57
57
|
* request that never touches an agent never builds it.
|
|
58
58
|
*/
|
|
@@ -4,7 +4,7 @@ import { pikkuState } from '../../pikku-state.js';
|
|
|
4
4
|
import { PikkuError, addError } from '../../errors/error-handler.js';
|
|
5
5
|
import { parseVersionedId } from '../../version.js';
|
|
6
6
|
import { resolveRemoteAddonToken } from './remote-addon-auth.js';
|
|
7
|
-
import { createAgentRPC } from '../
|
|
7
|
+
import { createAgentRPC } from '../agent/agent-rpc.js';
|
|
8
8
|
/**
|
|
9
9
|
* The session for a wire: read through `getSession` when a runner attached one,
|
|
10
10
|
* otherwise whatever was placed on the wire directly.
|
|
@@ -203,6 +203,8 @@ export class ContextAwareRPCService {
|
|
|
203
203
|
secretOverrides: resolved.addonConfig?.secretOverrides,
|
|
204
204
|
variableOverrides: resolved.addonConfig?.variableOverrides,
|
|
205
205
|
credentialOverrides: resolved.addonConfig?.credentialOverrides,
|
|
206
|
+
secretGrants: resolved.addonConfig?.secretGrants,
|
|
207
|
+
credentialGrants: resolved.addonConfig?.credentialGrants,
|
|
206
208
|
globalSecrets: resolved.addonConfig?.globalSecrets,
|
|
207
209
|
globalCredentials: resolved.addonConfig?.globalCredentials,
|
|
208
210
|
},
|
|
@@ -312,7 +314,7 @@ export class ContextAwareRPCService {
|
|
|
312
314
|
/**
|
|
313
315
|
* The agent facade, built on access.
|
|
314
316
|
*
|
|
315
|
-
* The implementation lives in `
|
|
317
|
+
* The implementation lives in `agent/agent-rpc.ts` so the agent surface is
|
|
316
318
|
* one file rather than a wing of this one; a getter rather than a field so a
|
|
317
319
|
* request that never touches an agent never builds it.
|
|
318
320
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { PikkuRawWire } from '../../types/core.types.js';
|
|
2
|
-
import type { AgentInterruptResult } from '../
|
|
2
|
+
import type { AgentInterruptResult } from '../agent/agent-interrupt.js';
|
|
3
3
|
export type PikkuRPC<Invoke extends (...args: any[]) => any = (...args: any[]) => any, Remote extends (...args: any[]) => any = (...args: any[]) => any, startWorkflow extends (...args: any[]) => any = (...args: any[]) => any, AgentRun extends (...args: any[]) => any = (...args: any[]) => any, AgentStream extends (...args: any[]) => any = (...args: any[]) => any> = {
|
|
4
4
|
depth: number;
|
|
5
5
|
global: boolean;
|
|
@@ -48,6 +48,10 @@ export interface ResolvedFunction {
|
|
|
48
48
|
secretOverrides?: Record<string, string>;
|
|
49
49
|
variableOverrides?: Record<string, string>;
|
|
50
50
|
credentialOverrides?: Record<string, string>;
|
|
51
|
+
/** Set by the consuming app: secrets it lends this instance, as the addon names them */
|
|
52
|
+
secretGrants?: string[];
|
|
53
|
+
/** Set by the consuming app: credentials it lends this instance, as the addon names them */
|
|
54
|
+
credentialGrants?: string[];
|
|
51
55
|
/** Set by the consuming app: hand this instance the unscoped `SecretService` */
|
|
52
56
|
globalSecrets?: string;
|
|
53
57
|
/** Set by the consuming app: hand this instance the unscoped `CredentialService` */
|