@theoremjs/agents 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1264 -2
- package/esm/_dnt.polyfills.d.ts +66 -0
- package/esm/_dnt.polyfills.js +51 -0
- package/esm/mod.d.ts +65 -0
- package/esm/mod.js +42 -0
- package/esm/package.json +3 -0
- package/esm/src/cli/bin.d.ts +7 -0
- package/esm/src/cli/bin.js +9 -0
- package/esm/src/cli/commands/bench.d.ts +6 -0
- package/esm/src/cli/commands/bench.js +458 -0
- package/esm/src/cli/commands/egress-compile.d.ts +17 -0
- package/esm/src/cli/commands/egress-compile.js +25 -0
- package/esm/src/cli/commands/eval.d.ts +37 -0
- package/esm/src/cli/commands/eval.js +190 -0
- package/esm/src/cli/commands/fuzz-canary.d.ts +15 -0
- package/esm/src/cli/commands/fuzz-canary.js +202 -0
- package/esm/src/cli/commands/fuzz-guardrails.d.ts +2 -0
- package/esm/src/cli/commands/fuzz-guardrails.js +5 -0
- package/esm/src/cli/commands/profile.d.ts +2 -0
- package/esm/src/cli/commands/profile.js +80 -0
- package/esm/src/cli/commands/run.d.ts +13 -0
- package/esm/src/cli/commands/run.js +45 -0
- package/esm/src/cli/commands/test.d.ts +25 -0
- package/esm/src/cli/commands/test.js +143 -0
- package/esm/src/cli/event-log.d.ts +15 -0
- package/esm/src/cli/event-log.js +136 -0
- package/esm/src/cli/index.d.ts +10 -0
- package/esm/src/cli/index.js +267 -0
- package/esm/src/cli/matrix/fixtures.d.ts +10 -0
- package/esm/src/cli/matrix/fixtures.js +72 -0
- package/esm/src/cli/matrix/synthesizer.d.ts +18 -0
- package/esm/src/cli/matrix/synthesizer.js +103 -0
- package/esm/src/evals/attachments.d.ts +4 -0
- package/esm/src/evals/attachments.js +53 -0
- package/esm/src/evals/breakdown.d.ts +28 -0
- package/esm/src/evals/breakdown.js +68 -0
- package/esm/src/evals/graders/answer.d.ts +13 -0
- package/esm/src/evals/graders/answer.js +133 -0
- package/esm/src/evals/graders/code.d.ts +64 -0
- package/esm/src/evals/graders/code.js +205 -0
- package/esm/src/evals/graders/judge.d.ts +20 -0
- package/esm/src/evals/graders/judge.js +365 -0
- package/esm/src/evals/graders/latency.d.ts +5 -0
- package/esm/src/evals/graders/latency.js +81 -0
- package/esm/src/evals/graders/live.d.ts +11 -0
- package/esm/src/evals/graders/live.js +40 -0
- package/esm/src/evals/graders/media.d.ts +18 -0
- package/esm/src/evals/graders/media.js +92 -0
- package/esm/src/evals/graders/shared.d.ts +11 -0
- package/esm/src/evals/graders/shared.js +62 -0
- package/esm/src/evals/graders/transcript.d.ts +17 -0
- package/esm/src/evals/graders/transcript.js +200 -0
- package/esm/src/evals/mod.d.ts +42 -0
- package/esm/src/evals/mod.js +30 -0
- package/esm/src/evals/record.d.ts +35 -0
- package/esm/src/evals/record.js +105 -0
- package/esm/src/evals/rubrics/catalog.d.ts +148 -0
- package/esm/src/evals/rubrics/catalog.js +846 -0
- package/esm/src/evals/rubrics/mod.d.ts +24 -0
- package/esm/src/evals/rubrics/mod.js +55 -0
- package/esm/src/evals/rubrics/mustache.d.ts +23 -0
- package/esm/src/evals/rubrics/mustache.js +154 -0
- package/esm/src/evals/rubrics/types.d.ts +54 -0
- package/esm/src/evals/rubrics/types.js +74 -0
- package/esm/src/evals/run.d.ts +118 -0
- package/esm/src/evals/run.js +452 -0
- package/esm/src/evals/suite.d.ts +19 -0
- package/esm/src/evals/suite.js +92 -0
- package/esm/src/evals/summary.d.ts +32 -0
- package/esm/src/evals/summary.js +30 -0
- package/esm/src/evals/trial.d.ts +27 -0
- package/esm/src/evals/trial.js +158 -0
- package/esm/src/evals/types.d.ts +260 -0
- package/esm/src/evals/types.js +126 -0
- package/esm/src/evals/verdict.d.ts +18 -0
- package/esm/src/evals/verdict.js +45 -0
- package/esm/src/guardrails/canary-gate.d.ts +21 -0
- package/esm/src/guardrails/canary-gate.js +40 -0
- package/esm/src/guardrails/canary.d.ts +112 -0
- package/esm/src/guardrails/canary.js +901 -0
- package/esm/src/guardrails/compile-egress.d.ts +11 -0
- package/esm/src/guardrails/compile-egress.js +11 -0
- package/esm/src/guardrails/corpus/canary-egress-attacks.d.ts +22 -0
- package/esm/src/guardrails/corpus/canary-egress-attacks.js +303 -0
- package/esm/src/guardrails/corpus/fuzz-inbound.d.ts +5 -0
- package/esm/src/guardrails/corpus/fuzz-inbound.js +202 -0
- package/esm/src/guardrails/corpus/inbound-payloads.d.ts +3 -0
- package/esm/src/guardrails/corpus/inbound-payloads.js +118 -0
- package/esm/src/guardrails/corpus/live-attacks.d.ts +13 -0
- package/esm/src/guardrails/corpus/live-attacks.js +224 -0
- package/esm/src/guardrails/corpus/mod.d.ts +7 -0
- package/esm/src/guardrails/corpus/mod.js +4 -0
- package/esm/src/guardrails/corpus/secrets.d.ts +12 -0
- package/esm/src/guardrails/corpus/secrets.js +16 -0
- package/esm/src/guardrails/corpus/strings.d.ts +23 -0
- package/esm/src/guardrails/corpus/strings.js +29 -0
- package/esm/src/guardrails/corpus/types.d.ts +32 -0
- package/esm/src/guardrails/corpus/types.js +1 -0
- package/esm/src/guardrails/egress-automata.d.ts +14 -0
- package/esm/src/guardrails/egress-automata.js +6059 -0
- package/esm/src/guardrails/egress-compiler.d.ts +33 -0
- package/esm/src/guardrails/egress-compiler.js +237 -0
- package/esm/src/guardrails/egress-patterns.d.ts +24 -0
- package/esm/src/guardrails/egress-patterns.js +32 -0
- package/esm/src/guardrails/egress-policy.d.ts +27 -0
- package/esm/src/guardrails/egress-policy.js +76 -0
- package/esm/src/guardrails/egress-rules.d.ts +54 -0
- package/esm/src/guardrails/egress-rules.js +49 -0
- package/esm/src/guardrails/egress-stream.d.ts +105 -0
- package/esm/src/guardrails/egress-stream.js +645 -0
- package/esm/src/guardrails/egress-urls.d.ts +83 -0
- package/esm/src/guardrails/egress-urls.js +917 -0
- package/esm/src/guardrails/egress.d.ts +106 -0
- package/esm/src/guardrails/egress.js +374 -0
- package/esm/src/guardrails/error.d.ts +26 -0
- package/esm/src/guardrails/error.js +91 -0
- package/esm/src/guardrails/event-schemas.d.ts +57 -0
- package/esm/src/guardrails/event-schemas.js +41 -0
- package/esm/src/guardrails/events.d.ts +9 -0
- package/esm/src/guardrails/events.js +42 -0
- package/esm/src/guardrails/hits.d.ts +8 -0
- package/esm/src/guardrails/hits.js +19 -0
- package/esm/src/guardrails/injection-patterns.d.ts +15 -0
- package/esm/src/guardrails/injection-patterns.js +94 -0
- package/esm/src/guardrails/injection.d.ts +14 -0
- package/esm/src/guardrails/injection.js +237 -0
- package/esm/src/guardrails/lexicon.d.ts +30 -0
- package/esm/src/guardrails/lexicon.js +478 -0
- package/esm/src/guardrails/live-outbound-gate.d.ts +54 -0
- package/esm/src/guardrails/live-outbound-gate.js +305 -0
- package/esm/src/guardrails/mod.d.ts +46 -0
- package/esm/src/guardrails/mod.js +29 -0
- package/esm/src/guardrails/network.d.ts +42 -0
- package/esm/src/guardrails/network.js +328 -0
- package/esm/src/guardrails/normalize.d.ts +6 -0
- package/esm/src/guardrails/normalize.js +132 -0
- package/esm/src/guardrails/policy.d.ts +19 -0
- package/esm/src/guardrails/policy.js +50 -0
- package/esm/src/guardrails/progressive-yield.d.ts +69 -0
- package/esm/src/guardrails/progressive-yield.js +237 -0
- package/esm/src/guardrails/prompt-echo.d.ts +38 -0
- package/esm/src/guardrails/prompt-echo.js +225 -0
- package/esm/src/guardrails/quota.d.ts +20 -0
- package/esm/src/guardrails/quota.js +80 -0
- package/esm/src/guardrails/rules.d.ts +56 -0
- package/esm/src/guardrails/rules.js +54 -0
- package/esm/src/guardrails/sanitize.d.ts +23 -0
- package/esm/src/guardrails/sanitize.js +184 -0
- package/esm/src/guardrails/sensitive.d.ts +29 -0
- package/esm/src/guardrails/sensitive.js +113 -0
- package/esm/src/guardrails/serialize.d.ts +14 -0
- package/esm/src/guardrails/serialize.js +37 -0
- package/esm/src/guardrails/testing.d.ts +9 -0
- package/esm/src/guardrails/testing.js +8 -0
- package/esm/src/guardrails/theorem-error.d.ts +18 -0
- package/esm/src/guardrails/theorem-error.js +53 -0
- package/esm/src/guardrails/thought-guard.d.ts +55 -0
- package/esm/src/guardrails/thought-guard.js +464 -0
- package/esm/src/guardrails/tool-directives.d.ts +19 -0
- package/esm/src/guardrails/tool-directives.js +109 -0
- package/esm/src/guardrails/tool-result.d.ts +61 -0
- package/esm/src/guardrails/tool-result.js +248 -0
- package/esm/src/guardrails/types.d.ts +242 -0
- package/esm/src/guardrails/types.js +54 -0
- package/esm/src/host/client-turn.d.ts +9 -0
- package/esm/src/host/client-turn.js +38 -0
- package/esm/src/host/mint-trace.d.ts +28 -0
- package/esm/src/host/mint-trace.js +59 -0
- package/esm/src/host/mod.d.ts +13 -0
- package/esm/src/host/mod.js +11 -0
- package/esm/src/host/readStreamingJsonStringField.d.ts +2 -0
- package/esm/src/host/readStreamingJsonStringField.js +68 -0
- package/esm/src/host/reply.d.ts +7 -0
- package/esm/src/host/reply.js +35 -0
- package/esm/src/interface/blocks.d.ts +8 -0
- package/esm/src/interface/blocks.js +213 -0
- package/esm/src/interface/composer-actions.d.ts +20 -0
- package/esm/src/interface/composer-actions.js +31 -0
- package/esm/src/interface/draft.d.ts +14 -0
- package/esm/src/interface/draft.js +28 -0
- package/esm/src/interface/from-profile.d.ts +12 -0
- package/esm/src/interface/from-profile.js +144 -0
- package/esm/src/interface/history.d.ts +40 -0
- package/esm/src/interface/history.js +269 -0
- package/esm/src/interface/inputs.d.ts +10 -0
- package/esm/src/interface/inputs.js +64 -0
- package/esm/src/interface/mod.d.ts +31 -0
- package/esm/src/interface/mod.js +21 -0
- package/esm/src/interface/models.d.ts +19 -0
- package/esm/src/interface/models.js +45 -0
- package/esm/src/interface/pending.d.ts +48 -0
- package/esm/src/interface/pending.js +146 -0
- package/esm/src/interface/profile-interface.d.ts +4 -0
- package/esm/src/interface/profile-interface.js +223 -0
- package/esm/src/interface/session.d.ts +64 -0
- package/esm/src/interface/session.js +121 -0
- package/esm/src/interface/tool-calls.d.ts +20 -0
- package/esm/src/interface/tool-calls.js +73 -0
- package/esm/src/interface/tool-invoke.d.ts +4 -0
- package/esm/src/interface/tool-invoke.js +20 -0
- package/esm/src/interface/tool-media.d.ts +9 -0
- package/esm/src/interface/tool-media.js +112 -0
- package/esm/src/interface/types.d.ts +213 -0
- package/esm/src/interface/types.js +4 -0
- package/esm/src/kernel/auth/credential-source.d.ts +23 -0
- package/esm/src/kernel/auth/credential-source.js +11 -0
- package/esm/src/kernel/auth/crypto.d.ts +51 -0
- package/esm/src/kernel/auth/crypto.js +161 -0
- package/esm/src/kernel/auth/mod.d.ts +5 -0
- package/esm/src/kernel/auth/mod.js +5 -0
- package/esm/src/kernel/auth/oauth.d.ts +17 -0
- package/esm/src/kernel/auth/oauth.js +438 -0
- package/esm/src/kernel/auth/scope-refusal.d.ts +9 -0
- package/esm/src/kernel/auth/scope-refusal.js +8 -0
- package/esm/src/kernel/auth/typed-secret.d.ts +13 -0
- package/esm/src/kernel/auth/typed-secret.js +23 -0
- package/esm/src/kernel/auth/types.d.ts +139 -0
- package/esm/src/kernel/auth/types.js +1 -0
- package/esm/src/kernel/default-scope.d.ts +34 -0
- package/esm/src/kernel/default-scope.js +72 -0
- package/esm/src/kernel/engine/compaction.d.ts +70 -0
- package/esm/src/kernel/engine/compaction.js +202 -0
- package/esm/src/kernel/engine/decision.d.ts +41 -0
- package/esm/src/kernel/engine/decision.js +412 -0
- package/esm/src/kernel/engine/hash.d.ts +6 -0
- package/esm/src/kernel/engine/hash.js +22 -0
- package/esm/src/kernel/engine/live-inbound.d.ts +9 -0
- package/esm/src/kernel/engine/live-inbound.js +19 -0
- package/esm/src/kernel/engine/live-ingress.d.ts +9 -0
- package/esm/src/kernel/engine/live-ingress.js +37 -0
- package/esm/src/kernel/engine/media-probe/audio.d.ts +7 -0
- package/esm/src/kernel/engine/media-probe/audio.js +342 -0
- package/esm/src/kernel/engine/media-probe/bytes.d.ts +6 -0
- package/esm/src/kernel/engine/media-probe/bytes.js +26 -0
- package/esm/src/kernel/engine/media-probe/image.d.ts +5 -0
- package/esm/src/kernel/engine/media-probe/image.js +171 -0
- package/esm/src/kernel/engine/media-probe/iso-bmff.d.ts +19 -0
- package/esm/src/kernel/engine/media-probe/iso-bmff.js +48 -0
- package/esm/src/kernel/engine/media-probe/matroska.d.ts +35 -0
- package/esm/src/kernel/engine/media-probe/matroska.js +226 -0
- package/esm/src/kernel/engine/media-probe/mp4.d.ts +22 -0
- package/esm/src/kernel/engine/media-probe/mp4.js +279 -0
- package/esm/src/kernel/engine/media-probe/opus.d.ts +4 -0
- package/esm/src/kernel/engine/media-probe/opus.js +23 -0
- package/esm/src/kernel/engine/media-probe/pdf.d.ts +5 -0
- package/esm/src/kernel/engine/media-probe/pdf.js +117 -0
- package/esm/src/kernel/engine/media-probe/video.d.ts +15 -0
- package/esm/src/kernel/engine/media-probe/video.js +49 -0
- package/esm/src/kernel/engine/record.d.ts +3 -0
- package/esm/src/kernel/engine/record.js +11 -0
- package/esm/src/kernel/engine/repair.d.ts +7 -0
- package/esm/src/kernel/engine/repair.js +36 -0
- package/esm/src/kernel/engine/runner/gates.d.ts +5 -0
- package/esm/src/kernel/engine/runner/gates.js +322 -0
- package/esm/src/kernel/engine/runner/mod.d.ts +15 -0
- package/esm/src/kernel/engine/runner/mod.js +574 -0
- package/esm/src/kernel/engine/runner/schema-validation.d.ts +14 -0
- package/esm/src/kernel/engine/runner/schema-validation.js +105 -0
- package/esm/src/kernel/engine/runner/stages.d.ts +28 -0
- package/esm/src/kernel/engine/runner/stages.js +48 -0
- package/esm/src/kernel/engine/runner/state.d.ts +76 -0
- package/esm/src/kernel/engine/runner/state.js +40 -0
- package/esm/src/kernel/engine/runner/steps.d.ts +14 -0
- package/esm/src/kernel/engine/runner/steps.js +360 -0
- package/esm/src/kernel/engine/runner/stream.d.ts +40 -0
- package/esm/src/kernel/engine/runner/stream.js +225 -0
- package/esm/src/kernel/engine/runner/usage.d.ts +45 -0
- package/esm/src/kernel/engine/runner/usage.js +150 -0
- package/esm/src/kernel/engine/session/mod.d.ts +31 -0
- package/esm/src/kernel/engine/session/mod.js +951 -0
- package/esm/src/kernel/engine/session/session-trace.d.ts +115 -0
- package/esm/src/kernel/engine/session/session-trace.js +489 -0
- package/esm/src/kernel/engine/token-estimate.d.ts +77 -0
- package/esm/src/kernel/engine/token-estimate.js +218 -0
- package/esm/src/kernel/engine/tool-trace.d.ts +58 -0
- package/esm/src/kernel/engine/tool-trace.js +131 -0
- package/esm/src/kernel/engine/tree.d.ts +2 -0
- package/esm/src/kernel/engine/tree.js +17 -0
- package/esm/src/kernel/engine/turn-trace.d.ts +118 -0
- package/esm/src/kernel/engine/turn-trace.js +863 -0
- package/esm/src/kernel/engine/usage.d.ts +30 -0
- package/esm/src/kernel/engine/usage.js +134 -0
- package/esm/src/kernel/interaction-parts.d.ts +10 -0
- package/esm/src/kernel/interaction-parts.js +22 -0
- package/esm/src/kernel/mod.d.ts +47 -0
- package/esm/src/kernel/mod.js +30 -0
- package/esm/src/kernel/profile-graph.d.ts +140 -0
- package/esm/src/kernel/profile-graph.js +151 -0
- package/esm/src/kernel/profile-presence.d.ts +9 -0
- package/esm/src/kernel/profile-presence.js +143 -0
- package/esm/src/kernel/profile-scope.d.ts +36 -0
- package/esm/src/kernel/profile-scope.js +262 -0
- package/esm/src/kernel/registry/attachments.d.ts +35 -0
- package/esm/src/kernel/registry/attachments.js +211 -0
- package/esm/src/kernel/registry/catalog.d.ts +19 -0
- package/esm/src/kernel/registry/catalog.js +78 -0
- package/esm/src/kernel/registry/ingress.d.ts +8 -0
- package/esm/src/kernel/registry/ingress.js +134 -0
- package/esm/src/kernel/registry/kernel-registry.d.ts +12 -0
- package/esm/src/kernel/registry/kernel-registry.js +8 -0
- package/esm/src/kernel/registry/profile-outputs.d.ts +3 -0
- package/esm/src/kernel/registry/profile-outputs.js +7 -0
- package/esm/src/kernel/registry/profiles.d.ts +92 -0
- package/esm/src/kernel/registry/profiles.js +698 -0
- package/esm/src/kernel/registry/provider-request.d.ts +6 -0
- package/esm/src/kernel/registry/provider-request.js +42 -0
- package/esm/src/kernel/registry/resolve.d.ts +14 -0
- package/esm/src/kernel/registry/resolve.js +237 -0
- package/esm/src/kernel/registry/schemas.d.ts +9 -0
- package/esm/src/kernel/registry/schemas.js +25 -0
- package/esm/src/kernel/registry/sole-model.d.ts +2 -0
- package/esm/src/kernel/registry/sole-model.js +4 -0
- package/esm/src/kernel/registry/system-prompt.d.ts +8 -0
- package/esm/src/kernel/registry/system-prompt.js +33 -0
- package/esm/src/kernel/registry/system-role.d.ts +3 -0
- package/esm/src/kernel/registry/system-role.js +9 -0
- package/esm/src/kernel/registry/vault.d.ts +13 -0
- package/esm/src/kernel/registry/vault.js +14 -0
- package/esm/src/kernel/schema.d.ts +172 -0
- package/esm/src/kernel/schema.js +653 -0
- package/esm/src/kernel/scope.d.ts +22 -0
- package/esm/src/kernel/scope.js +25 -0
- package/esm/src/kernel/stages.d.ts +141 -0
- package/esm/src/kernel/stages.js +388 -0
- package/esm/src/kernel/stop.d.ts +93 -0
- package/esm/src/kernel/stop.js +125 -0
- package/esm/src/kernel/system-parts.d.ts +26 -0
- package/esm/src/kernel/system-parts.js +98 -0
- package/esm/src/kernel/tools/activity-label.d.ts +16 -0
- package/esm/src/kernel/tools/activity-label.js +209 -0
- package/esm/src/kernel/tools/events.d.ts +51 -0
- package/esm/src/kernel/tools/events.js +149 -0
- package/esm/src/kernel/tools/execute.d.ts +64 -0
- package/esm/src/kernel/tools/execute.js +801 -0
- package/esm/src/kernel/tools/gate-answer.d.ts +50 -0
- package/esm/src/kernel/tools/gate-answer.js +54 -0
- package/esm/src/kernel/tools/harness.d.ts +16 -0
- package/esm/src/kernel/tools/harness.js +42 -0
- package/esm/src/kernel/tools/invoke.d.ts +10 -0
- package/esm/src/kernel/tools/invoke.js +129 -0
- package/esm/src/kernel/tools/mcp-sessions.d.ts +20 -0
- package/esm/src/kernel/tools/mcp-sessions.js +64 -0
- package/esm/src/kernel/tools/mod.d.ts +9 -0
- package/esm/src/kernel/tools/mod.js +6 -0
- package/esm/src/kernel/tools/model-text.d.ts +11 -0
- package/esm/src/kernel/tools/model-text.js +25 -0
- package/esm/src/kernel/tools/permission.d.ts +6 -0
- package/esm/src/kernel/tools/permission.js +38 -0
- package/esm/src/kernel/tools/project.d.ts +8 -0
- package/esm/src/kernel/tools/project.js +25 -0
- package/esm/src/kernel/tools/registry.d.ts +20 -0
- package/esm/src/kernel/tools/registry.js +84 -0
- package/esm/src/kernel/tools/remote.d.ts +110 -0
- package/esm/src/kernel/tools/remote.js +998 -0
- package/esm/src/kernel/tools/resolve.d.ts +28 -0
- package/esm/src/kernel/tools/resolve.js +259 -0
- package/esm/src/kernel/tools/schema.d.ts +10 -0
- package/esm/src/kernel/tools/schema.js +174 -0
- package/esm/src/kernel/tools/signed-in-fetch.d.ts +18 -0
- package/esm/src/kernel/tools/signed-in-fetch.js +45 -0
- package/esm/src/kernel/tools/stage-run.d.ts +93 -0
- package/esm/src/kernel/tools/stage-run.js +140 -0
- package/esm/src/kernel/tools/types.d.ts +287 -0
- package/esm/src/kernel/tools/types.js +1 -0
- package/esm/src/kernel/turn-events.d.ts +599 -0
- package/esm/src/kernel/turn-events.js +425 -0
- package/esm/src/kernel/types.d.ts +884 -0
- package/esm/src/kernel/types.js +1 -0
- package/esm/src/kernel/util/base64.d.ts +2 -0
- package/esm/src/kernel/util/base64.js +13 -0
- package/esm/src/kernel/util/exact-type.d.ts +31 -0
- package/esm/src/kernel/util/exact-type.js +1 -0
- package/esm/src/kernel/util/find-last.d.ts +3 -0
- package/esm/src/kernel/util/find-last.js +9 -0
- package/esm/src/kernel/util/mime.d.ts +2 -0
- package/esm/src/kernel/util/mime.js +4 -0
- package/esm/src/kernel/util/record.d.ts +1 -0
- package/esm/src/kernel/util/record.js +3 -0
- package/esm/src/observability/destinations.d.ts +9 -0
- package/esm/src/observability/destinations.js +38 -0
- package/esm/src/observability/jsonl.d.ts +18 -0
- package/esm/src/observability/jsonl.js +117 -0
- package/esm/src/observability/mod.d.ts +22 -0
- package/esm/src/observability/mod.js +16 -0
- package/esm/src/observability/openinference.d.ts +11 -0
- package/esm/src/observability/openinference.js +205 -0
- package/esm/src/observability/otlp.d.ts +68 -0
- package/esm/src/observability/otlp.js +98 -0
- package/esm/src/observability/phoenix.d.ts +29 -0
- package/esm/src/observability/phoenix.js +71 -0
- package/esm/src/observability/policy.d.ts +14 -0
- package/esm/src/observability/policy.js +69 -0
- package/esm/src/observability/resolve-policy.d.ts +8 -0
- package/esm/src/observability/resolve-policy.js +58 -0
- package/esm/src/observability/spans.d.ts +16 -0
- package/esm/src/observability/spans.js +56 -0
- package/esm/src/observability/trace-catalog.d.ts +111 -0
- package/esm/src/observability/trace-catalog.js +865 -0
- package/esm/src/observability/trace-record.d.ts +23 -0
- package/esm/src/observability/trace-record.js +223 -0
- package/esm/src/observability/trace-schema.d.ts +54 -0
- package/esm/src/observability/trace-schema.js +59 -0
- package/esm/src/observability/trace-sink.d.ts +13 -0
- package/esm/src/observability/trace-sink.js +3 -0
- package/esm/src/observability/trace-span.d.ts +92 -0
- package/esm/src/observability/trace-span.js +199 -0
- package/esm/src/observability/trace.d.ts +11 -0
- package/esm/src/observability/trace.js +35 -0
- package/esm/src/observability/types.d.ts +85 -0
- package/esm/src/observability/types.js +3 -0
- package/esm/src/presets/google/speech-voices.d.ts +8 -0
- package/esm/src/presets/google/speech-voices.js +38 -0
- package/esm/src/presets/google-limits.d.ts +12 -0
- package/esm/src/presets/google-limits.js +18 -0
- package/esm/src/presets/google.d.ts +104 -0
- package/esm/src/presets/google.js +188 -0
- package/esm/src/presets/mod.d.ts +10 -0
- package/esm/src/presets/mod.js +9 -0
- package/esm/src/presets/openrouter.d.ts +13 -0
- package/esm/src/presets/openrouter.js +13 -0
- package/esm/src/presets/typesafe.d.ts +2 -0
- package/esm/src/presets/typesafe.js +2 -0
- package/esm/src/providers/create-provider.d.ts +21 -0
- package/esm/src/providers/create-provider.js +90 -0
- package/esm/src/providers/decision/usage.d.ts +3 -0
- package/esm/src/providers/decision/usage.js +25 -0
- package/esm/src/providers/google/api-error.d.ts +8 -0
- package/esm/src/providers/google/api-error.js +123 -0
- package/esm/src/providers/google/grounding.d.ts +5 -0
- package/esm/src/providers/google/grounding.js +226 -0
- package/esm/src/providers/google/interactions/framing.d.ts +15 -0
- package/esm/src/providers/google/interactions/framing.js +263 -0
- package/esm/src/providers/google/interactions/mod.d.ts +2 -0
- package/esm/src/providers/google/interactions/mod.js +2 -0
- package/esm/src/providers/google/interactions/steps.d.ts +12 -0
- package/esm/src/providers/google/interactions/steps.js +200 -0
- package/esm/src/providers/google/interactions/stream.d.ts +24 -0
- package/esm/src/providers/google/interactions/stream.js +297 -0
- package/esm/src/providers/google/keys.d.ts +20 -0
- package/esm/src/providers/google/keys.js +87 -0
- package/esm/src/providers/google/live/framing.d.ts +51 -0
- package/esm/src/providers/google/live/framing.js +597 -0
- package/esm/src/providers/google/live/mod.d.ts +7 -0
- package/esm/src/providers/google/live/mod.js +7 -0
- package/esm/src/providers/google/live/openapi-schema.d.ts +1 -0
- package/esm/src/providers/google/live/openapi-schema.js +42 -0
- package/esm/src/providers/google/live/session.d.ts +14 -0
- package/esm/src/providers/google/live/session.js +150 -0
- package/esm/src/providers/google/live/stream.d.ts +51 -0
- package/esm/src/providers/google/live/stream.js +263 -0
- package/esm/src/providers/google/thinking.d.ts +3 -0
- package/esm/src/providers/google/thinking.js +8 -0
- package/esm/src/providers/google/urls.d.ts +6 -0
- package/esm/src/providers/google/urls.js +6 -0
- package/esm/src/providers/google/usage.d.ts +7 -0
- package/esm/src/providers/google/usage.js +49 -0
- package/esm/src/providers/local/local.d.ts +12 -0
- package/esm/src/providers/local/local.js +186 -0
- package/esm/src/providers/local/mod.d.ts +8 -0
- package/esm/src/providers/local/mod.js +8 -0
- package/esm/src/providers/mod.d.ts +11 -0
- package/esm/src/providers/mod.js +7 -0
- package/esm/src/providers/openrouter/cache-control.d.ts +5 -0
- package/esm/src/providers/openrouter/cache-control.js +4 -0
- package/esm/src/providers/openrouter/chat.d.ts +64 -0
- package/esm/src/providers/openrouter/chat.js +490 -0
- package/esm/src/providers/openrouter/image.d.ts +24 -0
- package/esm/src/providers/openrouter/image.js +187 -0
- package/esm/src/providers/openrouter/openai/compat.d.ts +19 -0
- package/esm/src/providers/openrouter/openai/compat.js +162 -0
- package/esm/src/providers/openrouter/openai/image-payload.d.ts +9 -0
- package/esm/src/providers/openrouter/openai/image-payload.js +115 -0
- package/esm/src/providers/openrouter/openai/sdk-messages.d.ts +10 -0
- package/esm/src/providers/openrouter/openai/sdk-messages.js +99 -0
- package/esm/src/providers/openrouter/openai/usage.d.ts +8 -0
- package/esm/src/providers/openrouter/openai/usage.js +56 -0
- package/esm/src/providers/openrouter/resolve-api-key.d.ts +3 -0
- package/esm/src/providers/openrouter/resolve-api-key.js +8 -0
- package/esm/src/providers/openrouter/speech.d.ts +18 -0
- package/esm/src/providers/openrouter/speech.js +109 -0
- package/esm/src/providers/probe.d.ts +5 -0
- package/esm/src/providers/probe.js +15 -0
- package/esm/src/providers/shared/builtin-wire.d.ts +4 -0
- package/esm/src/providers/shared/builtin-wire.js +9 -0
- package/esm/src/providers/shared/pcm.d.ts +14 -0
- package/esm/src/providers/shared/pcm.js +78 -0
- package/esm/src/providers/shared/response-identity.d.ts +11 -0
- package/esm/src/providers/shared/response-identity.js +12 -0
- package/esm/src/providers/shared/sse.d.ts +10 -0
- package/esm/src/providers/shared/sse.js +79 -0
- package/esm/src/providers/shared/structured-output.d.ts +4 -0
- package/esm/src/providers/shared/structured-output.js +11 -0
- package/esm/src/providers/shared/tool-args.d.ts +25 -0
- package/esm/src/providers/shared/tool-args.js +84 -0
- package/esm/src/providers/shared/upstream-tap.d.ts +8 -0
- package/esm/src/providers/shared/upstream-tap.js +85 -0
- package/esm/src/providers/shared/upstream-tape.d.ts +6 -0
- package/esm/src/providers/shared/upstream-tape.js +88 -0
- package/esm/src/providers/shared/vault.d.ts +13 -0
- package/esm/src/providers/shared/vault.js +41 -0
- package/esm/src/providers/types.d.ts +19 -0
- package/esm/src/providers/types.js +1 -0
- package/esm/src/surface/formats.d.ts +28 -0
- package/esm/src/surface/formats.js +139 -0
- package/esm/src/surface/mod.d.ts +14 -0
- package/esm/src/surface/mod.js +10 -0
- package/esm/src/surface/runtime.d.ts +40 -0
- package/esm/src/surface/runtime.js +439 -0
- package/esm/src/surface/tools.d.ts +15 -0
- package/esm/src/surface/tools.js +80 -0
- package/esm/src/surface/types.d.ts +104 -0
- package/esm/src/surface/types.js +4 -0
- package/package.json +108 -4
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* o200k is not every model's tokenizer, so text counts are an estimate for all families. Media is
|
|
3
|
+
* counted only by a rule the family's billed usage was measured to follow; a family without one
|
|
4
|
+
* reports media as `unknownMedia`, never a borrowed rate.
|
|
5
|
+
*
|
|
6
|
+
* Gemini 3 (`gemini-3*` flash / pro text models, direct or via OpenRouter).
|
|
7
|
+
* Measured 22/09/2026 against the usage Gemini bills — Interactions `usage`
|
|
8
|
+
* and `generateContent` `usageMetadata` agree exactly — on gemini-3.8-flash
|
|
9
|
+
* at the default media resolution (THEOREM never sets `media_resolution`).
|
|
10
|
+
* `countTokens` is not the oracle: it reports 560 per PDF page and 32 tokens
|
|
11
|
+
* per audio second, and neither is what is billed.
|
|
12
|
+
* - image: a patch grid inside a 1120-token budget that keeps the aspect
|
|
13
|
+
* ratio — `⌊√(1120·w/h)⌋ × ⌊√(1120·h/w)⌋` (1024×1024 → 1089, 1920×1080 →
|
|
14
|
+
* 1100, 4032×3024 HEIC → 1064). Needs the pixel size from the image header.
|
|
15
|
+
* - audio: `⌈seconds × 25⌉`, seconds being decoded samples ÷ rate
|
|
16
|
+
* (`audioSeconds`): 2.2 s → 56, 0.5 s → 13. Sample rate, channels and
|
|
17
|
+
* silence do not matter. Raw PCM: bare `audio/pcm` is 16 kHz mono 16-bit;
|
|
18
|
+
* `audio/L16` needs both `rate` and `channels`. `audio/pcm` with parameters,
|
|
19
|
+
* `audio/alaw`, and `audio/mulaw` are refused by Interactions — unknown.
|
|
20
|
+
* Mono `audio/L16` is unknown: Gemini converts it first and adds text tokens
|
|
21
|
+
* no rule reproduces (about 31–33, plus 2 per second, plus 1 audio token
|
|
22
|
+
* away from 16 kHz).
|
|
23
|
+
* - video: `frames × ⌊√(70·w/h)⌋ × ⌊√(70·h/w)⌋ + ⌈min(audio, frames) × 25⌉`,
|
|
24
|
+
* `frames` = the video track's seconds rounded half up (66 per frame at
|
|
25
|
+
* 16:9, 63 at 4:3; fps, codec and container do not matter). Under half a
|
|
26
|
+
* second rounds to no frames, which Gemini refuses — unknown. MP4 / MOV /
|
|
27
|
+
* 3GP and WebM only (`videoInfo`); other containers are unknown.
|
|
28
|
+
* - PDF: 520 tokens per page. Text on the page adds nothing.
|
|
29
|
+
* - text documents (plain, Markdown, CSV, JSON, HTML, CSS, XML, RTF,
|
|
30
|
+
* JavaScript, Python) as their UTF-8 text. `text/md` and
|
|
31
|
+
* `application/x-python` are unknown: Gemini converts them first, adding
|
|
32
|
+
* about 30 text tokens no rule reproduces.
|
|
33
|
+
* - Provider file references (`uri`): unknown — the bytes are not here.
|
|
34
|
+
*
|
|
35
|
+
* Known gap: ADTS AAC. Gemini estimates its length from the bitrate; the
|
|
36
|
+
* decoded length here runs 2–3 tokens over at 10 s.
|
|
37
|
+
*
|
|
38
|
+
* Not yet checked against billed usage: video in WebM / MOV / 3GP, anamorphic
|
|
39
|
+
* video, and Gemini 3 models other than gemini-3.8-flash. The full probe and
|
|
40
|
+
* open decisions: https://github.com/masudl-hub/theoremai/issues/18
|
|
41
|
+
*/
|
|
42
|
+
import { historyMessageParts } from '../interaction-parts.js';
|
|
43
|
+
import { MEDIA_INPUT_KINDS } from '../schema.js';
|
|
44
|
+
import { base64ToBytes } from '../util/base64.js';
|
|
45
|
+
import { mimeEssence } from '../util/mime.js';
|
|
46
|
+
import { audioSeconds } from './media-probe/audio.js';
|
|
47
|
+
import { imageSize } from './media-probe/image.js';
|
|
48
|
+
import { pdfPageCount } from './media-probe/pdf.js';
|
|
49
|
+
import { videoInfo } from './media-probe/video.js';
|
|
50
|
+
export const TOKEN_TEXT_ENCODING = 'o200k_base';
|
|
51
|
+
const GEMINI_3_TEXT_MODEL = /^gemini-3(?:\.\d+)?-(?:flash|pro)(?:-lite)?(?:-preview)?$/;
|
|
52
|
+
const OPENROUTER_GOOGLE_PREFIX = 'google/';
|
|
53
|
+
const GEMINI_3_IMAGE_BUDGET = 1120;
|
|
54
|
+
const GEMINI_3_VIDEO_FRAME_BUDGET = 70;
|
|
55
|
+
const GEMINI_3_AUDIO_PER_SECOND = 25;
|
|
56
|
+
const GEMINI_3_PDF_PER_PAGE = 520;
|
|
57
|
+
/** Bare `audio/pcm`: 16-bit mono at 16 kHz. */
|
|
58
|
+
const PCM_DEFAULT_RATE = 16_000;
|
|
59
|
+
const PCM_BYTES_PER_SAMPLE = 2;
|
|
60
|
+
/** Document types Gemini reads as their own text. */
|
|
61
|
+
const GEMINI_3_TEXT_DOCUMENTS = new Set([
|
|
62
|
+
'text/plain',
|
|
63
|
+
'text/csv',
|
|
64
|
+
'text/markdown',
|
|
65
|
+
'text/html',
|
|
66
|
+
'text/css',
|
|
67
|
+
'text/xml',
|
|
68
|
+
'text/rtf',
|
|
69
|
+
'text/javascript',
|
|
70
|
+
'application/x-javascript',
|
|
71
|
+
'text/x-python',
|
|
72
|
+
'application/json',
|
|
73
|
+
]);
|
|
74
|
+
/**
|
|
75
|
+
* Media family for a model binding — `undefined` when no verified rule covers
|
|
76
|
+
* it. OpenRouter uses the routed model's family (`google/gemini-3…`).
|
|
77
|
+
*/
|
|
78
|
+
export function mediaTokenFamily(binding) {
|
|
79
|
+
const modelId = binding.provider === 'google'
|
|
80
|
+
? binding.apiId
|
|
81
|
+
: binding.provider === 'openrouter' && binding.apiId.startsWith(OPENROUTER_GOOGLE_PREFIX)
|
|
82
|
+
? binding.apiId.slice(OPENROUTER_GOOGLE_PREFIX.length)
|
|
83
|
+
: undefined;
|
|
84
|
+
if (modelId === undefined)
|
|
85
|
+
return undefined;
|
|
86
|
+
return GEMINI_3_TEXT_MODEL.test(modelId) ? 'gemini-3' : undefined;
|
|
87
|
+
}
|
|
88
|
+
function mimeParam(mimeType, name) {
|
|
89
|
+
for (const piece of mimeType.split(';').slice(1)) {
|
|
90
|
+
const [key, value] = piece.split('=').map((s) => s.trim().toLowerCase());
|
|
91
|
+
if (key === name) {
|
|
92
|
+
const n = Number(value);
|
|
93
|
+
return Number.isInteger(n) && n > 0 ? n : undefined;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
98
|
+
function hasParams(mimeType) {
|
|
99
|
+
return mimeType.includes(';');
|
|
100
|
+
}
|
|
101
|
+
/** Seconds of raw 16-bit PCM, or `undefined` when Gemini refuses how it is declared. */
|
|
102
|
+
function pcmSeconds(mimeType, bytes) {
|
|
103
|
+
const essence = mimeEssence(mimeType);
|
|
104
|
+
let rate;
|
|
105
|
+
let channels;
|
|
106
|
+
if (essence === 'audio/pcm') {
|
|
107
|
+
if (hasParams(mimeType))
|
|
108
|
+
return undefined;
|
|
109
|
+
rate = PCM_DEFAULT_RATE;
|
|
110
|
+
channels = 1;
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
rate = mimeParam(mimeType, 'rate');
|
|
114
|
+
channels = mimeParam(mimeType, 'channels');
|
|
115
|
+
// Mono L16 is converted before it is counted.
|
|
116
|
+
if (channels === 1)
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
if (!rate || !channels)
|
|
120
|
+
return undefined;
|
|
121
|
+
return Math.floor(bytes.length / (PCM_BYTES_PER_SAMPLE * channels)) / rate;
|
|
122
|
+
}
|
|
123
|
+
function gemini3Audio(mimeType, bytes) {
|
|
124
|
+
const essence = mimeEssence(mimeType);
|
|
125
|
+
const seconds = essence === 'audio/pcm' || essence === 'audio/l16'
|
|
126
|
+
? pcmSeconds(mimeType, bytes)
|
|
127
|
+
: essence === 'audio/alaw' || essence === 'audio/mulaw'
|
|
128
|
+
? undefined
|
|
129
|
+
: audioSeconds(bytes);
|
|
130
|
+
return seconds === undefined ? undefined : Math.ceil(seconds * GEMINI_3_AUDIO_PER_SECOND);
|
|
131
|
+
}
|
|
132
|
+
function patchGrid(budget, w, h) {
|
|
133
|
+
return Math.floor(Math.sqrt((budget * w) / h)) * Math.floor(Math.sqrt((budget * h) / w));
|
|
134
|
+
}
|
|
135
|
+
function gemini3Video(bytes) {
|
|
136
|
+
const info = videoInfo(bytes);
|
|
137
|
+
if (!info || info.audioSeconds === undefined)
|
|
138
|
+
return undefined;
|
|
139
|
+
const frames = Math.floor(info.seconds + 0.5);
|
|
140
|
+
if (frames === 0)
|
|
141
|
+
return undefined;
|
|
142
|
+
return (frames * patchGrid(GEMINI_3_VIDEO_FRAME_BUDGET, info.width, info.height) +
|
|
143
|
+
Math.ceil(Math.min(info.audioSeconds, frames) * GEMINI_3_AUDIO_PER_SECOND));
|
|
144
|
+
}
|
|
145
|
+
function utf8(bytes) {
|
|
146
|
+
try {
|
|
147
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// Not UTF-8: what Gemini reads from it is unknown.
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
async function gemini3Media(payload, text) {
|
|
155
|
+
if (!('data' in payload))
|
|
156
|
+
return undefined;
|
|
157
|
+
const essence = mimeEssence(payload.mimeType);
|
|
158
|
+
const bytes = base64ToBytes(payload.data);
|
|
159
|
+
switch (MEDIA_INPUT_KINDS[essence]) {
|
|
160
|
+
case 'image': {
|
|
161
|
+
const size = imageSize(bytes);
|
|
162
|
+
return size ? patchGrid(GEMINI_3_IMAGE_BUDGET, size.width, size.height) : undefined;
|
|
163
|
+
}
|
|
164
|
+
case 'audio':
|
|
165
|
+
return gemini3Audio(payload.mimeType, bytes);
|
|
166
|
+
case 'video':
|
|
167
|
+
return gemini3Video(bytes);
|
|
168
|
+
case 'document': {
|
|
169
|
+
if (essence === 'application/pdf') {
|
|
170
|
+
const pages = await pdfPageCount(bytes);
|
|
171
|
+
return pages === undefined ? undefined : pages * GEMINI_3_PDF_PER_PAGE;
|
|
172
|
+
}
|
|
173
|
+
if (!GEMINI_3_TEXT_DOCUMENTS.has(essence))
|
|
174
|
+
return undefined;
|
|
175
|
+
const decoded = utf8(bytes);
|
|
176
|
+
return decoded === undefined ? undefined : text(decoded);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
let encodePromise = null;
|
|
182
|
+
function buildEstimator(encode) {
|
|
183
|
+
const text = (value) => (value ? encode(value).length : 0);
|
|
184
|
+
const media = async (payload, family) => family === 'gemini-3' ? await gemini3Media(payload, text) : undefined;
|
|
185
|
+
const parts = async (list, family) => {
|
|
186
|
+
const count = { tokens: 0, unknownMedia: 0 };
|
|
187
|
+
for (const part of list) {
|
|
188
|
+
if (part.type === 'text') {
|
|
189
|
+
count.tokens += text(part.text);
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const n = await media(part, family);
|
|
193
|
+
if (n === undefined)
|
|
194
|
+
count.unknownMedia += 1;
|
|
195
|
+
else
|
|
196
|
+
count.tokens += n;
|
|
197
|
+
}
|
|
198
|
+
return count;
|
|
199
|
+
};
|
|
200
|
+
const messages = async (list, family) => {
|
|
201
|
+
const count = { tokens: 0, unknownMedia: 0 };
|
|
202
|
+
for (const msg of list) {
|
|
203
|
+
const fromParts = await parts(historyMessageParts(msg), family);
|
|
204
|
+
count.tokens += fromParts.tokens;
|
|
205
|
+
count.unknownMedia += fromParts.unknownMedia;
|
|
206
|
+
for (const tc of msg.tool_calls ?? []) {
|
|
207
|
+
count.tokens += text(tc.function.name) + text(tc.function.arguments);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return count;
|
|
211
|
+
};
|
|
212
|
+
return { text, media, parts, messages };
|
|
213
|
+
}
|
|
214
|
+
// The ranks import lazily, so hosts that never estimate never pay for them.
|
|
215
|
+
export async function loadTokenEstimator() {
|
|
216
|
+
encodePromise ??= import('gpt-tokenizer/encoding/o200k_base').then((m) => m.encode);
|
|
217
|
+
return buildEstimator(await encodePromise);
|
|
218
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { GuardrailEvent, ToolOrigin } from '../../guardrails/types.js';
|
|
2
|
+
import { type SpanHandle, type TraceAttributes } from '../../observability/trace-span.js';
|
|
3
|
+
import type { ToolPermission } from '../schema.js';
|
|
4
|
+
import type { ToolFailure } from '../tools/types.js';
|
|
5
|
+
import type { InteractionPart, TurnEvent } from '../types.js';
|
|
6
|
+
import { type GuardrailCheck, type SentToolCall } from './turn-trace.js';
|
|
7
|
+
/** A check at the tool boundary: the call's arguments, the remote-content gate, the result, or the network target. */
|
|
8
|
+
type ToolCheck = Extract<GuardrailCheck, 'tool_arguments' | 'taint' | 'tool_result' | 'tool_failure' | 'network' | 'network_request'>;
|
|
9
|
+
/**
|
|
10
|
+
* Records one tool-boundary check on the tool's span with the time it took:
|
|
11
|
+
* its decision, or a pass (`allow`) when it let the call or result through.
|
|
12
|
+
* Trace only; the host still hears about decisions alone. An untraced call has no span.
|
|
13
|
+
*/
|
|
14
|
+
declare function recordToolCheck(span: SpanHandle | undefined, check: ToolCheck, durationMs: number, guardrail: GuardrailEvent | undefined): void;
|
|
15
|
+
/** How a tool call ended. Only `error` is a failure; the rest were stopped or finished. */
|
|
16
|
+
type ToolOutcome = 'ok' | 'error' | 'denied' | 'gated' | 'paused' | 'cancelled';
|
|
17
|
+
interface ToolCallStart {
|
|
18
|
+
name: string;
|
|
19
|
+
callId: string;
|
|
20
|
+
call: SentToolCall;
|
|
21
|
+
/** Absent when the tool is not registered. */
|
|
22
|
+
origin?: ToolOrigin;
|
|
23
|
+
permission?: ToolPermission;
|
|
24
|
+
/** The host's answer to an earlier gate, when this call resumes one. */
|
|
25
|
+
approved?: boolean;
|
|
26
|
+
/** The turn step whose model call asked for this tool. */
|
|
27
|
+
step?: number;
|
|
28
|
+
}
|
|
29
|
+
interface ToolCallEnd {
|
|
30
|
+
outcome: ToolOutcome;
|
|
31
|
+
/** What the model reads back: its text and any media. Absent when nothing is read back. */
|
|
32
|
+
result?: {
|
|
33
|
+
text: string;
|
|
34
|
+
parts?: readonly InteractionPart[];
|
|
35
|
+
};
|
|
36
|
+
/** The tool's raw output, present only when its body completed. */
|
|
37
|
+
data?: {
|
|
38
|
+
value: unknown;
|
|
39
|
+
};
|
|
40
|
+
/** The failure, on `error`: its kind (`error.type`) and the tool's code for it. */
|
|
41
|
+
failure?: Pick<ToolFailure, 'code' | 'kind'>;
|
|
42
|
+
/** Thrown out of the call (not an abort). */
|
|
43
|
+
thrown?: unknown;
|
|
44
|
+
}
|
|
45
|
+
interface ToolCallTrace {
|
|
46
|
+
span: SpanHandle;
|
|
47
|
+
observe: (event: TurnEvent) => void;
|
|
48
|
+
end: (end: ToolCallEnd) => void;
|
|
49
|
+
}
|
|
50
|
+
declare function toolSpanName(name: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* Spans from before `pre_tool` to settlement, so it covers the hooks, the gates and the body.
|
|
53
|
+
* Provider-run tools (search, maps, code execution) get no span: Theorem did not run them.
|
|
54
|
+
* `open` places it: a child of the turn's root, or the root of a host-invoked tool's own record.
|
|
55
|
+
*/
|
|
56
|
+
declare function startToolTrace(open: (name: string, attributes: TraceAttributes) => SpanHandle, start: ToolCallStart): ToolCallTrace;
|
|
57
|
+
export type { ToolCallEnd, ToolCallStart, ToolCallTrace, ToolCheck, ToolOutcome };
|
|
58
|
+
export { recordToolCheck, startToolTrace, toolSpanName };
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { errorKind } from '../../guardrails/error.js';
|
|
2
|
+
import { traceContent, traceJson, } from '../../observability/trace-span.js';
|
|
3
|
+
import { authScopeRefusedSchema } from '../auth/scope-refusal.js';
|
|
4
|
+
import { guardrailAttributes, guardrailCheckAttributes, optional, recordException, toolArgumentsText, tracePart, } from './turn-trace.js';
|
|
5
|
+
const TOOL_CHECK_STAGE = {
|
|
6
|
+
tool_arguments: 'tool_call',
|
|
7
|
+
taint: 'tool_call',
|
|
8
|
+
tool_result: 'tool_result',
|
|
9
|
+
tool_failure: 'tool_result',
|
|
10
|
+
network: 'network',
|
|
11
|
+
network_request: 'network',
|
|
12
|
+
};
|
|
13
|
+
/** Decisions already recorded with their check's time, so `observe` does not record them twice. */
|
|
14
|
+
const timedDecisions = new WeakSet();
|
|
15
|
+
/**
|
|
16
|
+
* Records one tool-boundary check on the tool's span with the time it took:
|
|
17
|
+
* its decision, or a pass (`allow`) when it let the call or result through.
|
|
18
|
+
* Trace only; the host still hears about decisions alone. An untraced call has no span.
|
|
19
|
+
*/
|
|
20
|
+
function recordToolCheck(span, check, durationMs, guardrail) {
|
|
21
|
+
if (!span)
|
|
22
|
+
return;
|
|
23
|
+
if (guardrail)
|
|
24
|
+
timedDecisions.add(guardrail);
|
|
25
|
+
span.event('theorem.guardrail', guardrailCheckAttributes(check, durationMs, guardrail, {
|
|
26
|
+
stage: TOOL_CHECK_STAGE[check],
|
|
27
|
+
trust: 'untrusted',
|
|
28
|
+
}));
|
|
29
|
+
}
|
|
30
|
+
function toolSpanName(name) {
|
|
31
|
+
return name ? `execute_tool ${name}` : 'execute_tool';
|
|
32
|
+
}
|
|
33
|
+
function toolSpanAttributes(start) {
|
|
34
|
+
return {
|
|
35
|
+
'gen_ai.operation.name': 'execute_tool',
|
|
36
|
+
'gen_ai.tool.name': start.name,
|
|
37
|
+
'gen_ai.tool.call.id': start.callId,
|
|
38
|
+
'gen_ai.tool.type': 'function',
|
|
39
|
+
'gen_ai.tool.call.arguments': traceContent(toolArgumentsText(start.call)),
|
|
40
|
+
...optional('theorem.tool.origin', start.origin),
|
|
41
|
+
...optional('theorem.tool.permission', start.permission),
|
|
42
|
+
...optional('theorem.tool.approved', start.approved),
|
|
43
|
+
...optional('theorem.step', start.step),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/** A gate as `theorem.gate` event attributes. The auth `state` is a secret and never recorded. */
|
|
47
|
+
function gateAttributes(gate) {
|
|
48
|
+
const auth = gate.kind === 'auth' ? gate.authChallenge : undefined;
|
|
49
|
+
return {
|
|
50
|
+
kind: gate.kind,
|
|
51
|
+
...optional('permission', gate.permission),
|
|
52
|
+
...(gate.summary ? { summary: traceContent(gate.summary) } : {}),
|
|
53
|
+
...(auth
|
|
54
|
+
? {
|
|
55
|
+
auth: {
|
|
56
|
+
slot: auth.slot,
|
|
57
|
+
type: auth.authType,
|
|
58
|
+
service: auth.service,
|
|
59
|
+
...optional('issuer', auth.issuer),
|
|
60
|
+
...optional('resource', auth.resource),
|
|
61
|
+
...optional('required_scopes', auth.requiredScopes),
|
|
62
|
+
},
|
|
63
|
+
}
|
|
64
|
+
: {}),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
const OUTCOME_STATUS = {
|
|
68
|
+
ok: 'OK',
|
|
69
|
+
error: 'ERROR',
|
|
70
|
+
denied: 'UNSET',
|
|
71
|
+
gated: 'UNSET',
|
|
72
|
+
paused: 'UNSET',
|
|
73
|
+
cancelled: 'UNSET',
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Spans from before `pre_tool` to settlement, so it covers the hooks, the gates and the body.
|
|
77
|
+
* Provider-run tools (search, maps, code execution) get no span: Theorem did not run them.
|
|
78
|
+
* `open` places it: a child of the turn's root, or the root of a host-invoked tool's own record.
|
|
79
|
+
*/
|
|
80
|
+
function startToolTrace(open, start) {
|
|
81
|
+
const span = open(toolSpanName(start.name), toolSpanAttributes(start));
|
|
82
|
+
return {
|
|
83
|
+
span,
|
|
84
|
+
observe: (event) => {
|
|
85
|
+
if (event.type === 'guardrail' && !timedDecisions.has(event.guardrail)) {
|
|
86
|
+
span.event('theorem.guardrail', guardrailAttributes(event.guardrail));
|
|
87
|
+
}
|
|
88
|
+
if (event.type === 'tool' && event.tool.phase === 'gate') {
|
|
89
|
+
span.event('theorem.gate', gateAttributes(event.tool.gate));
|
|
90
|
+
}
|
|
91
|
+
if (event.type === 'tool' && event.tool.phase === 'progress') {
|
|
92
|
+
const refused = authScopeRefusedSchema.safeParse(event.tool.data);
|
|
93
|
+
if (refused.success) {
|
|
94
|
+
const { slot, requested, declared } = refused.data;
|
|
95
|
+
span.event('theorem.auth.scope_refused', { slot, requested, declared });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (event.type === 'tool' && event.tool.phase === 'warning') {
|
|
99
|
+
const { code, message, severity } = event.tool.warning;
|
|
100
|
+
span.event('theorem.tool.warning', {
|
|
101
|
+
code,
|
|
102
|
+
message: traceContent(message),
|
|
103
|
+
...optional('severity', severity),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
if (event.type === 'citation') {
|
|
107
|
+
span.event('theorem.grounding', { sources: traceJson(event.sources) });
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
end: (end) => {
|
|
111
|
+
if (end.thrown !== undefined)
|
|
112
|
+
recordException(span, end.thrown);
|
|
113
|
+
const errorType = end.outcome === 'error'
|
|
114
|
+
? (end.failure?.kind ?? (end.thrown === undefined ? undefined : errorKind(end.thrown)))
|
|
115
|
+
: undefined;
|
|
116
|
+
span.set({
|
|
117
|
+
...(end.result ? { 'gen_ai.tool.call.result': traceContent(end.result.text) } : {}),
|
|
118
|
+
...(end.result?.parts?.length
|
|
119
|
+
? { 'theorem.tool.call.result.parts': end.result.parts.map(tracePart) }
|
|
120
|
+
: {}),
|
|
121
|
+
...(end.data ? { 'theorem.tool.data': traceJson(end.data.value) } : {}),
|
|
122
|
+
'theorem.tool.outcome': end.outcome,
|
|
123
|
+
...optional('theorem.tool.failure.code', end.failure?.code),
|
|
124
|
+
...optional('error.type', errorType),
|
|
125
|
+
});
|
|
126
|
+
const code = OUTCOME_STATUS[end.outcome];
|
|
127
|
+
span.end(code === 'ERROR' && errorType ? { code, message: errorType } : { code });
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
export { recordToolCheck, startToolTrace, toolSpanName };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
function mapStrings(value, map) {
|
|
2
|
+
if (typeof value === 'string') {
|
|
3
|
+
return map(value);
|
|
4
|
+
}
|
|
5
|
+
if (Array.isArray(value)) {
|
|
6
|
+
return value.map((item) => mapStrings(item, map));
|
|
7
|
+
}
|
|
8
|
+
if (value && typeof value === 'object') {
|
|
9
|
+
const out = {};
|
|
10
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
11
|
+
out[key] = mapStrings(nested, map);
|
|
12
|
+
}
|
|
13
|
+
return out;
|
|
14
|
+
}
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
export { mapStrings };
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import type { GuardrailEvent, GuardrailStage, TrustLevel } from '../../guardrails/types.js';
|
|
2
|
+
import { type SpanHandle, type SpanLinkInput, type SpanOptions, type TraceAttributes, type TraceAttributeValue } from '../../observability/trace-span.js';
|
|
3
|
+
import type { ToolFailure } from '../tools/types.js';
|
|
4
|
+
import type { InteractionPart, ModelBinding, ProviderCompleteRequest, ProviderEvent, ProviderTransport, TurnEvent, TurnRequest, TurnStop, TurnTokens, TurnTraceLink } from '../types.js';
|
|
5
|
+
import type { CallUsage } from './runner/usage.js';
|
|
6
|
+
type TracePart = {
|
|
7
|
+
[key: string]: TraceAttributeValue;
|
|
8
|
+
};
|
|
9
|
+
declare function tracePart(part: InteractionPart): TracePart;
|
|
10
|
+
/** Output parts folded from provider events: adjacent deltas of one kind merge. */
|
|
11
|
+
declare class OutputFold {
|
|
12
|
+
readonly parts: TracePart[];
|
|
13
|
+
private open?;
|
|
14
|
+
/** Each call's `tool_call` part by `callId`, for the failure that follows a malformed one. */
|
|
15
|
+
private readonly calls;
|
|
16
|
+
add(event: ProviderEvent, nowUnixNano: string): void;
|
|
17
|
+
private appendText;
|
|
18
|
+
/**
|
|
19
|
+
* The model's call as a `tool_call` part. A malformed call's failure follows
|
|
20
|
+
* it from the provider; the part then records the text the model sent.
|
|
21
|
+
*/
|
|
22
|
+
private addTool;
|
|
23
|
+
private push;
|
|
24
|
+
}
|
|
25
|
+
interface SentToolCall {
|
|
26
|
+
arguments: Record<string, unknown>;
|
|
27
|
+
failure?: ToolFailure;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* A tool call's arguments as the model sent them. Malformed arguments keep the
|
|
31
|
+
* provider's raw text; parsed ones are the same JSON the kernel sends back in
|
|
32
|
+
* history, so both hash alike.
|
|
33
|
+
*/
|
|
34
|
+
declare function toolArgumentsText(call: SentToolCall): string;
|
|
35
|
+
/**
|
|
36
|
+
* A guardrail decision as `theorem.guardrail` event attributes. The matched
|
|
37
|
+
* substring stays exact (not scrubbed: scrubbing it would erase what it shows);
|
|
38
|
+
* the record drops it unless the profile includes `guardrailMatchPreview`.
|
|
39
|
+
*/
|
|
40
|
+
declare function guardrailAttributes(guardrail: GuardrailEvent): TraceAttributes;
|
|
41
|
+
/**
|
|
42
|
+
* A guardrail check whose time the trace records: the turn's input and output,
|
|
43
|
+
* each tool boundary, the checks that run on streamed output, and a live
|
|
44
|
+
* session's own gates.
|
|
45
|
+
*/
|
|
46
|
+
type GuardrailCheck = 'input' | 'egress' | 'tool_arguments' | 'taint' | 'tool_result' | 'tool_failure' | 'network' | 'network_request' | StreamCheck | 'live_input';
|
|
47
|
+
/**
|
|
48
|
+
* A check that runs many times on one model call's streamed output: the
|
|
49
|
+
* progressive gate on each piece of reply text, the canary scan of every other
|
|
50
|
+
* streamed event, and a live session's gate on each batch it sends.
|
|
51
|
+
*/
|
|
52
|
+
type StreamCheck = 'output_stream' | 'stream_canary' | 'live_output';
|
|
53
|
+
/**
|
|
54
|
+
* One timed guardrail check as `theorem.guardrail` attributes: its decision,
|
|
55
|
+
* or a pass (`allow`, no hits) when it let the text through. Trace only; the
|
|
56
|
+
* host still hears about hits alone.
|
|
57
|
+
*/
|
|
58
|
+
declare function guardrailCheckAttributes(check: GuardrailCheck, durationMs: number, guardrail: GuardrailEvent | undefined, passed: {
|
|
59
|
+
stage: GuardrailStage;
|
|
60
|
+
trust: TrustLevel;
|
|
61
|
+
}): TraceAttributes;
|
|
62
|
+
declare function optional(key: string, value: TraceAttributeValue | undefined): TraceAttributes;
|
|
63
|
+
declare function usageAttributes(tokens: TurnTokens): TraceAttributes;
|
|
64
|
+
declare function urlAttributes(url: unknown): TraceAttributes;
|
|
65
|
+
declare function recordException(span: SpanHandle, err: unknown): void;
|
|
66
|
+
/** Close a root that a throw ended: the exception, its kind as `error.type`, and ERROR. */
|
|
67
|
+
declare function endThrownSpan(span: SpanHandle, err: unknown): void;
|
|
68
|
+
interface CallEnd {
|
|
69
|
+
/** The call's one usage report (estimated sides filled), when there is one. */
|
|
70
|
+
tokens?: TurnTokens;
|
|
71
|
+
stop?: TurnStop;
|
|
72
|
+
/** Thrown out of the stream (not an abort). */
|
|
73
|
+
thrown?: unknown;
|
|
74
|
+
}
|
|
75
|
+
interface CallTrace {
|
|
76
|
+
span: SpanHandle;
|
|
77
|
+
/** Pass as the adapter's `tapUpstream`. */
|
|
78
|
+
tap: (row: Record<string, unknown>) => void;
|
|
79
|
+
/** Every provider event, before guardrails. */
|
|
80
|
+
observe: (event: ProviderEvent) => void;
|
|
81
|
+
/** A stream check's decision; it carries the time that check has spent on the call so far. */
|
|
82
|
+
guardrail: (event: GuardrailEvent) => void;
|
|
83
|
+
/** Add one run of a stream check to this call. */
|
|
84
|
+
guardTime: (check: StreamCheck, ms: number) => void;
|
|
85
|
+
end: (end: CallEnd) => void;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* `open` places the span: a child of the turn's root, or the root of a Live response's own record.
|
|
89
|
+
* `usage.conversation` is read again at the end, since a Live response sets it only when it closes.
|
|
90
|
+
*/
|
|
91
|
+
declare function startCallTrace(open: (name: string, options: SpanOptions) => SpanHandle, args: {
|
|
92
|
+
req: ProviderCompleteRequest;
|
|
93
|
+
usage: CallUsage;
|
|
94
|
+
binding: ModelBinding | undefined;
|
|
95
|
+
transport: ProviderTransport;
|
|
96
|
+
step: number;
|
|
97
|
+
/** The runner's validation attempt; Live has none. */
|
|
98
|
+
attempt?: number;
|
|
99
|
+
}): CallTrace;
|
|
100
|
+
declare function traceLinks(links: readonly TurnTraceLink[] | undefined): SpanLinkInput[];
|
|
101
|
+
declare function turnSpanOptions(req: TurnRequest): SpanOptions;
|
|
102
|
+
interface TurnEnd {
|
|
103
|
+
seen: readonly TurnEvent[];
|
|
104
|
+
/** Those events as output parts, folded as they were delivered. */
|
|
105
|
+
delivered: OutputFold;
|
|
106
|
+
/** Validation / egress attempts that made a model call. */
|
|
107
|
+
attempts: number;
|
|
108
|
+
calls: number;
|
|
109
|
+
/** Thrown out of the turn (not an abort). */
|
|
110
|
+
thrown?: unknown;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Close a turn's `invoke_agent` span. Usage is the sum of this turn's own
|
|
114
|
+
* calls; a nested turn (compaction) carries its own.
|
|
115
|
+
*/
|
|
116
|
+
declare function endTurnSpan(root: SpanHandle, end: TurnEnd): void;
|
|
117
|
+
export type { CallEnd, CallTrace, GuardrailCheck, SentToolCall, StreamCheck, TracePart, TurnEnd };
|
|
118
|
+
export { endThrownSpan, endTurnSpan, guardrailAttributes, guardrailCheckAttributes, OutputFold, optional, recordException, startCallTrace, toolArgumentsText, traceLinks, tracePart, turnSpanOptions, urlAttributes, usageAttributes, };
|