autotel-genai 0.15.3 → 0.16.0

package/README.md

@@ -610,6 +610,81 @@ session-level latency are ordinary queries. This is the same hierarchy vendor
 LLM-analytics products sell as `$ai_session_id`; here it is spec, so it works
 wherever the spans land.
 
+### Signals: model judgments on requests (experimental)
+
+`autotel-genai/signals` asks an evaluation model questions about finished
+requests and records the answer as a typed attribute with a confidence:
+`signals.fault.value = 'upstream'`, `signals.fault.confidence = 0.93`. Group by
+it, alert on it, or use it to keep requests sampling would drop. It is built on
+the AI SDK's `experimental_evaluate` model contract, so it is experimental.
+
+You bring the model. There is no default model or provider and no key handling:
+pass any AI SDK evaluation model (`EvaluationModelV4`), configured the way that
+provider documents.
+
+```ts
+import { init } from 'autotel';
+import { createSignals, defineSignal } from 'autotel-genai/signals';
+
+const fault = defineSignal({
+  name: 'fault',
+  when: (e) => (e.status ?? 0) >= 500,
+  ask: 'Who is responsible for this failure?',
+  choice: {
+    client: 'Bad input, expired session, client mistake',
+    app: 'Our own code or configuration',
+    upstream: 'A third-party dependency failed',
+  },
+  cacheKey: (e) => String(e['error.type'] ?? e.name),
+});
+
+const silentFailure = defineSignal({
+  name: 'silent_failure',
+  when: (e) => e.status === 200 && e.name === 'POST /api/checkout',
+  ask: 'Returned 200, but the customer did not get what they came for',
+  keep: (v) => v.value && v.confidence > 0.8,
+});
+
+init({
+  service: 'shop',
+  spanEnrichers: [
+    createSignals({
+      model: evaluationModel, // any AI SDK evaluation model
+      signals: [fault, silentFailure],
+      budget: { perMinute: 600 },
+    }),
+  ],
+});
+```
+
+- **Register it in `spanEnrichers`,** not `spanProcessors`: autotel drains
+  enrichers before its exporters shut down, so answers still pending at exit
+  are exported. With your own `NodeSDK`, keep that order with
+  `new EnrichedSpanProcessor([signals], [exportProcessor])` from `autotel`.
+- **What is judged:** request spans, meaning spans with no parent or a remote
+  one. A signal's `when` sees the span's attributes plus `name`, `status`
+  (`http.response.status_code`), `error` and `durationMs`. All signals due for
+  one request go into one model call.
+- **Where answers land:** a span is immutable once it ends and a model call is
+  async, so fresh answers go on a `signals {request}` child span in the same
+  trace, which also parents the traced `evaluate` call. Answers served from
+  `cacheKey` are known at once and go on the request span itself.
+- **`keep`:** tail sampling decides per span when the span ends. A cached answer
+  that keeps marks the request span kept before tail sampling reads it. A fresh
+  answer arrives after that, so the request span and its children may already
+  be dropped; the signals span always exports, and when `keep` holds on a
+  dropped request it carries the request's attributes as `signals.event.*`.
+  Use `cacheKey` on keep signals that should promote the request span itself.
+- **What leaves the process:** the request's attributes, after
+  `redact` (default: the `default` redaction preset, because request spans reach
+  this processor before the export pipeline's redactor). Use `state` to send
+  only the fields a question needs, and `maxStateChars` to skip large requests.
+- **Failure:** over budget, during the cooldown after a failed call, or past
+  `timeoutMs` (default 2s), requests go unjudged and pass through unchanged.
+  `stats()` counts calls, skipped, errors, cached and kept.
+- **Tests:** `scriptedEvaluationModel((name, question, state) => answer)` stands
+  in for a provider and records every call.
+
 ### Sending the same spans to PostHog
 
 PostHog ingests plain OTLP: `PostHogSpanProcessor` from `@posthog/ai/otel`

package/dist/agent/index.d.cts

@@ -1,4 +1,4 @@
-import { $ as withScopedTool, A as TOOL_CALL_ID_LABEL, B as HoneyTokenToolOptions, C as SEQUENCE_RULES, Ct as hashPayload, D as SequenceSecurityEvent, E as SequenceRule, F as spansToSequenceEvents, G as CrossAgentMonitor, H as EVAL_IDENTITY_ATTR, I as EvalIncidentQueryResult, J as CrossAgentEvent, K as CrossAgentMonitorOptions, L as querySpansForEvalIncident, M as detectSequences, N as emitSequenceDetections, O as SequenceSeverity, P as sequenceDetectionsToSecurityEvents, Q as detectCrossAgentPattern, R as spansToCrossAgentEvents, S as POLICY_DECISION_LABEL, St as canonicalizeForHash, T as SequenceEvent, U as RecordEvalRunIdentityInput, V as createHoneyTokenTool, W as recordEvalRunIdentity, X as DetectCrossAgentPatternOptions, Y as CrossAgentSecurityEvent, Z as crossAgentDetectionsToSecurityEvents, _ as sequenceRulesToSigma, _t as recordAgentHandoff, a as AgentPlanRiskVerdict, at as RotateAgentIdentityInput, b as DETECTION_EVENT, bt as createAgentAuditMetadata, c as recordPlanRiskAssessment, ct as defineAgentAction, d as DETECTION_DISPOSITION_EVENT, dt as recordPolicyDecision, et as CreateSignedEventEnvelopeOptions, f as DetectionDispositionStatus, ft as withAgentAction, g as sequenceRuleToSigma, gt as delegateToAgent, h as SigmaDocument, ht as RecordAgentHandoffMetadata, i as AgentPlanClassifierResult, it as RevokeAgentIdentityInput, j as TOOL_NAME_LABEL, k as SequenceSpanLike, l as runAgentPlanClassifier, lt as defineAgentToolCall, m as recordDetectionDisposition, mt as DelegateToAgentInput, n as AgentPlanClassifier, nt as verifyEventEnvelopeHash, o as RecordPlanRiskAssessmentOptions, ot as createAgentIdentityRegistry, p as RecordDetectionDispositionInput, pt as withAgentToolCall, q as CrossAgentDetection, r as AgentPlanClassifierInput, rt as ProvisionAgentIdentityInput, s as heuristicPlanRiskClassifier, st as withAgentSession, t as AGENT_PLAN_RISK_ATTR, tt as createSignedEventEnvelope, u as DETECTION_DISPOSITION_ATTR, ut as recordDecisionBasis, v as AGENT_OUTCOME_LABEL, vt as flattenAgentAttributes, w as SequenceDetection, wt as AGENT_AUDIT_SCHEMA_VERSION, x as EmitSequenceDetectionsOptions, xt as HashPayloadOptions, y as DETECTION_ATTR, yt as setAgentAttributes, z as HoneyTokenToolDefinition } from "../index-Dg6xJssP.cjs";
+import { $ as withScopedTool, A as TOOL_CALL_ID_LABEL, B as HoneyTokenToolOptions, C as SEQUENCE_RULES, Ct as hashPayload, D as SequenceSecurityEvent, E as SequenceRule, F as spansToSequenceEvents, G as CrossAgentMonitor, H as EVAL_IDENTITY_ATTR, I as EvalIncidentQueryResult, J as CrossAgentEvent, K as CrossAgentMonitorOptions, L as querySpansForEvalIncident, M as detectSequences, N as emitSequenceDetections, O as SequenceSeverity, P as sequenceDetectionsToSecurityEvents, Q as detectCrossAgentPattern, R as spansToCrossAgentEvents, S as POLICY_DECISION_LABEL, St as canonicalizeForHash, T as SequenceEvent, U as RecordEvalRunIdentityInput, V as createHoneyTokenTool, W as recordEvalRunIdentity, X as DetectCrossAgentPatternOptions, Y as CrossAgentSecurityEvent, Z as crossAgentDetectionsToSecurityEvents, _ as sequenceRulesToSigma, _t as recordAgentHandoff, a as AgentPlanRiskVerdict, at as RotateAgentIdentityInput, b as DETECTION_EVENT, bt as createAgentAuditMetadata, c as recordPlanRiskAssessment, ct as defineAgentAction, d as DETECTION_DISPOSITION_EVENT, dt as recordPolicyDecision, et as CreateSignedEventEnvelopeOptions, f as DetectionDispositionStatus, ft as withAgentAction, g as sequenceRuleToSigma, gt as delegateToAgent, h as SigmaDocument, ht as RecordAgentHandoffMetadata, i as AgentPlanClassifierResult, it as RevokeAgentIdentityInput, j as TOOL_NAME_LABEL, k as SequenceSpanLike, l as runAgentPlanClassifier, lt as defineAgentToolCall, m as recordDetectionDisposition, mt as DelegateToAgentInput, n as AgentPlanClassifier, nt as verifyEventEnvelopeHash, o as RecordPlanRiskAssessmentOptions, ot as createAgentIdentityRegistry, p as RecordDetectionDispositionInput, pt as withAgentToolCall, q as CrossAgentDetection, r as AgentPlanClassifierInput, rt as ProvisionAgentIdentityInput, s as heuristicPlanRiskClassifier, st as withAgentSession, t as AGENT_PLAN_RISK_ATTR, tt as createSignedEventEnvelope, u as DETECTION_DISPOSITION_ATTR, ut as recordDecisionBasis, v as AGENT_OUTCOME_LABEL, vt as flattenAgentAttributes, w as SequenceDetection, wt as AGENT_AUDIT_SCHEMA_VERSION, x as EmitSequenceDetectionsOptions, xt as HashPayloadOptions, y as DETECTION_ATTR, yt as setAgentAttributes, z as HoneyTokenToolDefinition } from "../index-BjaNCZv3.cjs";
 import { i as ModelPricing, s as TokenUsage } from "../cost-B8rIvXTK.cjs";
 import { $ as PrivacyProfileName, A as AgentAiMetadata, B as AgentOutcome, C as recordMemoryAccess, D as AgentActionFactory, E as tryRecordHumanApproval, F as AgentIdentity, G as AgentToolCallOptions, H as AgentSessionMetadata, I as AgentIdentityRecord, K as AiLifecycleStage, L as AgentIdentityRegistry, M as AgentDecisionMetadata, N as AgentEventKind, O as AgentActionMetadata, P as AgentHandler, Q as PrivacyProfile, R as AgentIdentityStatus, S as recordInputProvenance, T as recordRenderOutput, U as AgentSessionStatus, V as AgentSessionActionMetadata, W as AgentToolCallActionMetadata, X as PolicyDecision, Y as GovernanceMetadata, Z as PolicyMetadata, _ as deriveActionRiskClass, a as AgentConsentOutcome, at as resolvePrivacyProfile, b as recordControllerId, c as AgentOutputFormat, ct as AgentContext, d as RecordControllerInput, et as ScopedToolDefinition, f as RecordHumanApprovalInput, g as RecordRenderOutputInput, h as RecordPlanStepInput, i as AgentConsentEvidence, it as SanitizationEvidence, j as AgentAuditEventEnvelope, k as AgentActionOptions, l as AgentSecurityRecordOptions, lt as agentContextFromSpan, m as RecordMemoryAccessInput, n as ActionRiskHints, nt as ToolStatus, o as AgentInputProvenance, ot as sanitizeAuditPayload, p as RecordInputProvenanceInput, q as DelegationContext, r as AgentActionRiskClass, rt as PrivacyProfileInput, s as AgentMemoryOperation, st as sanitizeAuditPayloadWithEvidence, t as AGENT_SECURITY_ATTR, tt as ToolCallMetadata, u as RecordActiveScopesInput, v as recordActionRiskClass, w as recordPlanStep, x as recordHumanApproval, y as recordActiveScopes, z as AgentMetadataInput } from "../agent-security-Bw7YX5sg.cjs";
 export { AGENT_AUDIT_SCHEMA_VERSION, AGENT_OUTCOME_LABEL, AGENT_PLAN_RISK_ATTR, AGENT_SECURITY_ATTR, type ActionRiskHints, type AgentActionFactory, type AgentActionMetadata, type AgentActionOptions, type AgentActionRiskClass, type AgentAiMetadata, type AgentAuditEventEnvelope, type AgentConsentEvidence, type AgentConsentOutcome, type AgentContext, type AgentDecisionMetadata, type AgentEventKind, type AgentHandler, type AgentIdentity, type AgentIdentityRecord, type AgentIdentityRegistry, type AgentIdentityStatus, type AgentInputProvenance, type AgentMemoryOperation, type AgentMetadataInput, type AgentOutcome, type AgentOutputFormat, type AgentPlanClassifier, type AgentPlanClassifierInput, type AgentPlanClassifierResult, type AgentPlanRiskVerdict, type AgentSecurityRecordOptions, type AgentSessionActionMetadata, type AgentSessionMetadata, type AgentSessionStatus, type AgentToolCallActionMetadata, type AgentToolCallOptions, type AiLifecycleStage, type CreateSignedEventEnvelopeOptions, type CrossAgentDetection, type CrossAgentEvent, CrossAgentMonitor, type CrossAgentMonitorOptions, type CrossAgentSecurityEvent, DETECTION_ATTR, DETECTION_DISPOSITION_ATTR, DETECTION_DISPOSITION_EVENT, DETECTION_EVENT, type DelegateToAgentInput, type DelegationContext, type DetectCrossAgentPatternOptions, type DetectionDispositionStatus, EVAL_IDENTITY_ATTR, type EmitSequenceDetectionsOptions, type EvalIncidentQueryResult, type GovernanceMetadata, type HashPayloadOptions, type HoneyTokenToolDefinition, type HoneyTokenToolOptions, type ModelPricing, POLICY_DECISION_LABEL, type PolicyDecision, type PolicyMetadata, type PrivacyProfile, type PrivacyProfileInput, type PrivacyProfileName, type ProvisionAgentIdentityInput, type RecordActiveScopesInput, type RecordAgentHandoffMetadata, type RecordControllerInput, type RecordDetectionDispositionInput, type RecordEvalRunIdentityInput, type RecordHumanApprovalInput, type RecordInputProvenanceInput, type RecordMemoryAccessInput, type RecordPlanRiskAssessmentOptions, type RecordPlanStepInput, type RecordRenderOutputInput, type RevokeAgentIdentityInput, type RotateAgentIdentityInput, SEQUENCE_RULES, type SanitizationEvidence, type ScopedToolDefinition, type SequenceDetection, type SequenceEvent, type SequenceRule, type SequenceSecurityEvent, type SequenceSeverity, type SequenceSpanLike, type SigmaDocument, TOOL_CALL_ID_LABEL, TOOL_NAME_LABEL, type TokenUsage, type ToolCallMetadata, type ToolStatus, agentContextFromSpan, canonicalizeForHash, createAgentAuditMetadata, createAgentIdentityRegistry, createHoneyTokenTool, createSignedEventEnvelope, crossAgentDetectionsToSecurityEvents, defineAgentAction, defineAgentToolCall, delegateToAgent, deriveActionRiskClass, detectCrossAgentPattern, detectSequences, emitSequenceDetections, flattenAgentAttributes, hashPayload, heuristicPlanRiskClassifier, querySpansForEvalIncident, recordActionRiskClass, recordActiveScopes, recordAgentHandoff, recordControllerId, recordDecisionBasis, recordDetectionDisposition, recordEvalRunIdentity, recordHumanApproval, recordInputProvenance, recordMemoryAccess, recordPlanRiskAssessment, recordPlanStep, recordPolicyDecision, recordRenderOutput, resolvePrivacyProfile, runAgentPlanClassifier, sanitizeAuditPayload, sanitizeAuditPayloadWithEvidence, sequenceDetectionsToSecurityEvents, sequenceRuleToSigma, sequenceRulesToSigma, setAgentAttributes, spansToCrossAgentEvents, spansToSequenceEvents, tryRecordHumanApproval, verifyEventEnvelopeHash, withAgentAction, withAgentSession, withAgentToolCall, withScopedTool };

package/dist/agent/index.d.ts

@@ -1,4 +1,4 @@
-import { $ as withScopedTool, A as TOOL_CALL_ID_LABEL, B as HoneyTokenToolOptions, C as SEQUENCE_RULES, Ct as hashPayload, D as SequenceSecurityEvent, E as SequenceRule, F as spansToSequenceEvents, G as CrossAgentMonitor, H as EVAL_IDENTITY_ATTR, I as EvalIncidentQueryResult, J as CrossAgentEvent, K as CrossAgentMonitorOptions, L as querySpansForEvalIncident, M as detectSequences, N as emitSequenceDetections, O as SequenceSeverity, P as sequenceDetectionsToSecurityEvents, Q as detectCrossAgentPattern, R as spansToCrossAgentEvents, S as POLICY_DECISION_LABEL, St as canonicalizeForHash, T as SequenceEvent, U as RecordEvalRunIdentityInput, V as createHoneyTokenTool, W as recordEvalRunIdentity, X as DetectCrossAgentPatternOptions, Y as CrossAgentSecurityEvent, Z as crossAgentDetectionsToSecurityEvents, _ as sequenceRulesToSigma, _t as recordAgentHandoff, a as AgentPlanRiskVerdict, at as RotateAgentIdentityInput, b as DETECTION_EVENT, bt as createAgentAuditMetadata, c as recordPlanRiskAssessment, ct as defineAgentAction, d as DETECTION_DISPOSITION_EVENT, dt as recordPolicyDecision, et as CreateSignedEventEnvelopeOptions, f as DetectionDispositionStatus, ft as withAgentAction, g as sequenceRuleToSigma, gt as delegateToAgent, h as SigmaDocument, ht as RecordAgentHandoffMetadata, i as AgentPlanClassifierResult, it as RevokeAgentIdentityInput, j as TOOL_NAME_LABEL, k as SequenceSpanLike, l as runAgentPlanClassifier, lt as defineAgentToolCall, m as recordDetectionDisposition, mt as DelegateToAgentInput, n as AgentPlanClassifier, nt as verifyEventEnvelopeHash, o as RecordPlanRiskAssessmentOptions, ot as createAgentIdentityRegistry, p as RecordDetectionDispositionInput, pt as withAgentToolCall, q as CrossAgentDetection, r as AgentPlanClassifierInput, rt as ProvisionAgentIdentityInput, s as heuristicPlanRiskClassifier, st as withAgentSession, t as AGENT_PLAN_RISK_ATTR, tt as createSignedEventEnvelope, u as DETECTION_DISPOSITION_ATTR, ut as recordDecisionBasis, v as AGENT_OUTCOME_LABEL, vt as flattenAgentAttributes, w as SequenceDetection, wt as AGENT_AUDIT_SCHEMA_VERSION, x as EmitSequenceDetectionsOptions, xt as HashPayloadOptions, y as DETECTION_ATTR, yt as setAgentAttributes, z as HoneyTokenToolDefinition } from "../index-C7NU4O3l.js";
+import { $ as withScopedTool, A as TOOL_CALL_ID_LABEL, B as HoneyTokenToolOptions, C as SEQUENCE_RULES, Ct as hashPayload, D as SequenceSecurityEvent, E as SequenceRule, F as spansToSequenceEvents, G as CrossAgentMonitor, H as EVAL_IDENTITY_ATTR, I as EvalIncidentQueryResult, J as CrossAgentEvent, K as CrossAgentMonitorOptions, L as querySpansForEvalIncident, M as detectSequences, N as emitSequenceDetections, O as SequenceSeverity, P as sequenceDetectionsToSecurityEvents, Q as detectCrossAgentPattern, R as spansToCrossAgentEvents, S as POLICY_DECISION_LABEL, St as canonicalizeForHash, T as SequenceEvent, U as RecordEvalRunIdentityInput, V as createHoneyTokenTool, W as recordEvalRunIdentity, X as DetectCrossAgentPatternOptions, Y as CrossAgentSecurityEvent, Z as crossAgentDetectionsToSecurityEvents, _ as sequenceRulesToSigma, _t as recordAgentHandoff, a as AgentPlanRiskVerdict, at as RotateAgentIdentityInput, b as DETECTION_EVENT, bt as createAgentAuditMetadata, c as recordPlanRiskAssessment, ct as defineAgentAction, d as DETECTION_DISPOSITION_EVENT, dt as recordPolicyDecision, et as CreateSignedEventEnvelopeOptions, f as DetectionDispositionStatus, ft as withAgentAction, g as sequenceRuleToSigma, gt as delegateToAgent, h as SigmaDocument, ht as RecordAgentHandoffMetadata, i as AgentPlanClassifierResult, it as RevokeAgentIdentityInput, j as TOOL_NAME_LABEL, k as SequenceSpanLike, l as runAgentPlanClassifier, lt as defineAgentToolCall, m as recordDetectionDisposition, mt as DelegateToAgentInput, n as AgentPlanClassifier, nt as verifyEventEnvelopeHash, o as RecordPlanRiskAssessmentOptions, ot as createAgentIdentityRegistry, p as RecordDetectionDispositionInput, pt as withAgentToolCall, q as CrossAgentDetection, r as AgentPlanClassifierInput, rt as ProvisionAgentIdentityInput, s as heuristicPlanRiskClassifier, st as withAgentSession, t as AGENT_PLAN_RISK_ATTR, tt as createSignedEventEnvelope, u as DETECTION_DISPOSITION_ATTR, ut as recordDecisionBasis, v as AGENT_OUTCOME_LABEL, vt as flattenAgentAttributes, w as SequenceDetection, wt as AGENT_AUDIT_SCHEMA_VERSION, x as EmitSequenceDetectionsOptions, xt as HashPayloadOptions, y as DETECTION_ATTR, yt as setAgentAttributes, z as HoneyTokenToolDefinition } from "../index-tyJsMeaN.js";
 import { i as ModelPricing, s as TokenUsage } from "../cost-B8rIvXTK.js";
 import { $ as PrivacyProfileName, A as AgentAiMetadata, B as AgentOutcome, C as recordMemoryAccess, D as AgentActionFactory, E as tryRecordHumanApproval, F as AgentIdentity, G as AgentToolCallOptions, H as AgentSessionMetadata, I as AgentIdentityRecord, K as AiLifecycleStage, L as AgentIdentityRegistry, M as AgentDecisionMetadata, N as AgentEventKind, O as AgentActionMetadata, P as AgentHandler, Q as PrivacyProfile, R as AgentIdentityStatus, S as recordInputProvenance, T as recordRenderOutput, U as AgentSessionStatus, V as AgentSessionActionMetadata, W as AgentToolCallActionMetadata, X as PolicyDecision, Y as GovernanceMetadata, Z as PolicyMetadata, _ as deriveActionRiskClass, a as AgentConsentOutcome, at as resolvePrivacyProfile, b as recordControllerId, c as AgentOutputFormat, ct as AgentContext, d as RecordControllerInput, et as ScopedToolDefinition, f as RecordHumanApprovalInput, g as RecordRenderOutputInput, h as RecordPlanStepInput, i as AgentConsentEvidence, it as SanitizationEvidence, j as AgentAuditEventEnvelope, k as AgentActionOptions, l as AgentSecurityRecordOptions, lt as agentContextFromSpan, m as RecordMemoryAccessInput, n as ActionRiskHints, nt as ToolStatus, o as AgentInputProvenance, ot as sanitizeAuditPayload, p as RecordInputProvenanceInput, q as DelegationContext, r as AgentActionRiskClass, rt as PrivacyProfileInput, s as AgentMemoryOperation, st as sanitizeAuditPayloadWithEvidence, t as AGENT_SECURITY_ATTR, tt as ToolCallMetadata, u as RecordActiveScopesInput, v as recordActionRiskClass, w as recordPlanStep, x as recordHumanApproval, y as recordActiveScopes, z as AgentMetadataInput } from "../agent-security-D3ZjviBy.js";
 export { AGENT_AUDIT_SCHEMA_VERSION, AGENT_OUTCOME_LABEL, AGENT_PLAN_RISK_ATTR, AGENT_SECURITY_ATTR, type ActionRiskHints, type AgentActionFactory, type AgentActionMetadata, type AgentActionOptions, type AgentActionRiskClass, type AgentAiMetadata, type AgentAuditEventEnvelope, type AgentConsentEvidence, type AgentConsentOutcome, type AgentContext, type AgentDecisionMetadata, type AgentEventKind, type AgentHandler, type AgentIdentity, type AgentIdentityRecord, type AgentIdentityRegistry, type AgentIdentityStatus, type AgentInputProvenance, type AgentMemoryOperation, type AgentMetadataInput, type AgentOutcome, type AgentOutputFormat, type AgentPlanClassifier, type AgentPlanClassifierInput, type AgentPlanClassifierResult, type AgentPlanRiskVerdict, type AgentSecurityRecordOptions, type AgentSessionActionMetadata, type AgentSessionMetadata, type AgentSessionStatus, type AgentToolCallActionMetadata, type AgentToolCallOptions, type AiLifecycleStage, type CreateSignedEventEnvelopeOptions, type CrossAgentDetection, type CrossAgentEvent, CrossAgentMonitor, type CrossAgentMonitorOptions, type CrossAgentSecurityEvent, DETECTION_ATTR, DETECTION_DISPOSITION_ATTR, DETECTION_DISPOSITION_EVENT, DETECTION_EVENT, type DelegateToAgentInput, type DelegationContext, type DetectCrossAgentPatternOptions, type DetectionDispositionStatus, EVAL_IDENTITY_ATTR, type EmitSequenceDetectionsOptions, type EvalIncidentQueryResult, type GovernanceMetadata, type HashPayloadOptions, type HoneyTokenToolDefinition, type HoneyTokenToolOptions, type ModelPricing, POLICY_DECISION_LABEL, type PolicyDecision, type PolicyMetadata, type PrivacyProfile, type PrivacyProfileInput, type PrivacyProfileName, type ProvisionAgentIdentityInput, type RecordActiveScopesInput, type RecordAgentHandoffMetadata, type RecordControllerInput, type RecordDetectionDispositionInput, type RecordEvalRunIdentityInput, type RecordHumanApprovalInput, type RecordInputProvenanceInput, type RecordMemoryAccessInput, type RecordPlanRiskAssessmentOptions, type RecordPlanStepInput, type RecordRenderOutputInput, type RevokeAgentIdentityInput, type RotateAgentIdentityInput, SEQUENCE_RULES, type SanitizationEvidence, type ScopedToolDefinition, type SequenceDetection, type SequenceEvent, type SequenceRule, type SequenceSecurityEvent, type SequenceSeverity, type SequenceSpanLike, type SigmaDocument, TOOL_CALL_ID_LABEL, TOOL_NAME_LABEL, type TokenUsage, type ToolCallMetadata, type ToolStatus, agentContextFromSpan, canonicalizeForHash, createAgentAuditMetadata, createAgentIdentityRegistry, createHoneyTokenTool, createSignedEventEnvelope, crossAgentDetectionsToSecurityEvents, defineAgentAction, defineAgentToolCall, delegateToAgent, deriveActionRiskClass, detectCrossAgentPattern, detectSequences, emitSequenceDetections, flattenAgentAttributes, hashPayload, heuristicPlanRiskClassifier, querySpansForEvalIncident, recordActionRiskClass, recordActiveScopes, recordAgentHandoff, recordControllerId, recordDecisionBasis, recordDetectionDisposition, recordEvalRunIdentity, recordHumanApproval, recordInputProvenance, recordMemoryAccess, recordPlanRiskAssessment, recordPlanStep, recordPolicyDecision, recordRenderOutput, resolvePrivacyProfile, runAgentPlanClassifier, sanitizeAuditPayload, sanitizeAuditPayloadWithEvidence, sequenceDetectionsToSecurityEvents, sequenceRuleToSigma, sequenceRulesToSigma, setAgentAttributes, spansToCrossAgentEvents, spansToSequenceEvents, tryRecordHumanApproval, verifyEventEnvelopeHash, withAgentAction, withAgentSession, withAgentToolCall, withScopedTool };

package/dist/ai-sdk-evaluate-6la6gWVI.js

@@ -0,0 +1,75 @@
+import { n as recordEvaluationResult } from "./events-KJTZlo22.js";
+import { recordGenAiResponse, recordGenAiUsage, traceGenAI } from "./trace.js";
+import { normalizeAiSdkProvider } from "./ai-sdk-bridge.js";
+
+//#region src/ai-sdk-evaluate.ts
+/** `gen_ai.operation.name` for an evaluation call; not yet in the semconv registry. */
+const GEN_AI_OPERATION_EVALUATE = "evaluate";
+/** Score value and label for the `gen_ai.evaluation.result` event of one answer. */
+function evaluationScore(answer, options = {}) {
+	switch (answer.type) {
+		case "choice": {
+			return {
+				scoreLabel: answer.choice,
+				scoreValue: answer.probabilities?.[answer.choice]
+			};
+		}
+		case "score": {
+			return { scoreValue: answer.score };
+		}
+		case "boolean": {
+			const threshold = options.name === undefined ? undefined : options.booleanThresholds?.[options.name];
+			return {
+				scoreValue: answer.probability,
+				...threshold === undefined ? {} : { scoreLabel: answer.probability >= threshold ? "yes" : "no" }
+			};
+		}
+	}
+}
+/**
+* Wrap an AI SDK evaluation model so each call is traced. Returns a model with
+* the same contract, so it drops into `experimental_evaluate({ model })`.
+*
+* @example
+* ```ts
+* const model = wrapEvaluationModel(typeSafeAi.evaluationModel('jev-latest'));
+* const { answers } = await evaluate({ model, state, questions });
+* ```
+*/
+function wrapEvaluationModel(model, options = {}) {
+	const traced = traceGenAI({
+		operation: GEN_AI_OPERATION_EVALUATE,
+		provider: normalizeAiSdkProvider(model.provider),
+		model: model.modelId,
+		attributes: options.attributes,
+		metrics: options.metrics
+	})((ctx) => async (call) => {
+		const result = await model.doEvaluate(call);
+		const seen = result;
+		recordGenAiResponse(ctx, {
+			model: seen.response?.modelId,
+			id: seen.response?.id
+		});
+		if (seen.usage) {
+			recordGenAiUsage(ctx, model.modelId, seen.usage, options.cost);
+		}
+		for (const [name, answer] of Object.entries(seen.answers)) {
+			recordEvaluationResult(ctx, {
+				name,
+				...evaluationScore(answer, {
+					name,
+					booleanThresholds: options.booleanThresholds
+				}),
+				responseId: seen.response?.id
+			});
+		}
+		return result;
+	});
+	return Object.create(model, { doEvaluate: {
+		value: traced,
+		enumerable: true
+	} });
+}
+
+//#endregion
+export { evaluationScore as n, wrapEvaluationModel as r, GEN_AI_OPERATION_EVALUATE as t };

package/dist/ai-sdk-evaluate-C0LpX5YV.d.ts

@@ -0,0 +1,74 @@
+import { t as EstimateCostOptions } from "./cost-B8rIvXTK.js";
+import { n as GenAiAttributeMap } from "./attributes-DAYkQi4W.js";
+//#region src/ai-sdk-evaluate.d.ts
+/** `gen_ai.operation.name` for an evaluation call; not yet in the semconv registry. */
+declare const GEN_AI_OPERATION_EVALUATE = "evaluate";
+/** One answer from `EvaluationModelV4Result.answers`, in the shape the AI SDK returns. */
+type AiSdkEvaluationAnswer = {
+  type: 'choice';
+  choice: string;
+  probabilities?: Record<string, number>;
+} | {
+  type: 'score';
+  score: number;
+  probabilities?: Record<string, number>;
+} | {
+  type: 'boolean';
+  probability: number;
+};
+/** The parts of `EvaluationModelV4Result` this module reads. */
+interface AiSdkEvaluationResult {
+  answers: Record<string, AiSdkEvaluationAnswer>;
+  usage?: {
+    inputTokens?: number;
+    outputTokens?: number;
+  };
+  response?: {
+    id?: string;
+    modelId?: string;
+  };
+}
+/** The parts of `EvaluationModelV4` this module needs; structural so `ai` is not a dependency. */
+interface AiSdkEvaluationModel {
+  readonly provider: string;
+  readonly modelId: string;
+  doEvaluate(options: never): PromiseLike<AiSdkEvaluationResult>;
+}
+interface WrapEvaluationModelOptions {
+  /** Pricing lookup for `gen_ai.usage.cost.usd`; `recordCost: false` skips it. */
+  cost?: EstimateCostOptions & {
+    recordCost?: boolean;
+  };
+  /** Extra attributes for every evaluation span. */
+  attributes?: GenAiAttributeMap;
+  /** Record `gen_ai.client.*` metrics alongside the span (default on). */
+  metrics?: boolean;
+  /**
+   * Per-question P(true) cutoffs for boolean answers. When a threshold is set
+   * for an answer's name, `gen_ai.evaluation.score.label` becomes `yes` or `no`.
+   */
+  booleanThresholds?: Record<string, number>;
+}
+interface EvaluationScoreOptions {
+  /** Question id; used with {@link WrapEvaluationModelOptions.booleanThresholds}. */
+  name?: string;
+  booleanThresholds?: Record<string, number>;
+}
+/** Score value and label for the `gen_ai.evaluation.result` event of one answer. */
+declare function evaluationScore(answer: AiSdkEvaluationAnswer, options?: EvaluationScoreOptions): {
+  scoreValue?: number;
+  scoreLabel?: string;
+};
+/**
+ * Wrap an AI SDK evaluation model so each call is traced. Returns a model with
+ * the same contract, so it drops into `experimental_evaluate({ model })`.
+ *
+ * @example
+ * ```ts
+ * const model = wrapEvaluationModel(typeSafeAi.evaluationModel('jev-latest'));
+ * const { answers } = await evaluate({ model, state, questions });
+ * ```
+ */
+declare function wrapEvaluationModel<M extends AiSdkEvaluationModel>(model: M, options?: WrapEvaluationModelOptions): M;
+//#endregion
+export { GEN_AI_OPERATION_EVALUATE as a, wrapEvaluationModel as c, EvaluationScoreOptions as i, AiSdkEvaluationModel as n, WrapEvaluationModelOptions as o, AiSdkEvaluationResult as r, evaluationScore as s, AiSdkEvaluationAnswer as t };

package/dist/ai-sdk-evaluate-CPyhbQFh.d.cts

@@ -0,0 +1,74 @@
+import { t as EstimateCostOptions } from "./cost-B8rIvXTK.cjs";
+import { n as GenAiAttributeMap } from "./attributes-DAOZehAi.cjs";
+//#region src/ai-sdk-evaluate.d.ts
+/** `gen_ai.operation.name` for an evaluation call; not yet in the semconv registry. */
+declare const GEN_AI_OPERATION_EVALUATE = "evaluate";
+/** One answer from `EvaluationModelV4Result.answers`, in the shape the AI SDK returns. */
+type AiSdkEvaluationAnswer = {
+  type: 'choice';
+  choice: string;
+  probabilities?: Record<string, number>;
+} | {
+  type: 'score';
+  score: number;
+  probabilities?: Record<string, number>;
+} | {
+  type: 'boolean';
+  probability: number;
+};
+/** The parts of `EvaluationModelV4Result` this module reads. */
+interface AiSdkEvaluationResult {
+  answers: Record<string, AiSdkEvaluationAnswer>;
+  usage?: {
+    inputTokens?: number;
+    outputTokens?: number;
+  };
+  response?: {
+    id?: string;
+    modelId?: string;
+  };
+}
+/** The parts of `EvaluationModelV4` this module needs; structural so `ai` is not a dependency. */
+interface AiSdkEvaluationModel {
+  readonly provider: string;
+  readonly modelId: string;
+  doEvaluate(options: never): PromiseLike<AiSdkEvaluationResult>;
+}
+interface WrapEvaluationModelOptions {
+  /** Pricing lookup for `gen_ai.usage.cost.usd`; `recordCost: false` skips it. */
+  cost?: EstimateCostOptions & {
+    recordCost?: boolean;
+  };
+  /** Extra attributes for every evaluation span. */
+  attributes?: GenAiAttributeMap;
+  /** Record `gen_ai.client.*` metrics alongside the span (default on). */
+  metrics?: boolean;
+  /**
+   * Per-question P(true) cutoffs for boolean answers. When a threshold is set
+   * for an answer's name, `gen_ai.evaluation.score.label` becomes `yes` or `no`.
+   */
+  booleanThresholds?: Record<string, number>;
+}
+interface EvaluationScoreOptions {
+  /** Question id; used with {@link WrapEvaluationModelOptions.booleanThresholds}. */
+  name?: string;
+  booleanThresholds?: Record<string, number>;
+}
+/** Score value and label for the `gen_ai.evaluation.result` event of one answer. */
+declare function evaluationScore(answer: AiSdkEvaluationAnswer, options?: EvaluationScoreOptions): {
+  scoreValue?: number;
+  scoreLabel?: string;
+};
+/**
+ * Wrap an AI SDK evaluation model so each call is traced. Returns a model with
+ * the same contract, so it drops into `experimental_evaluate({ model })`.
+ *
+ * @example
+ * ```ts
+ * const model = wrapEvaluationModel(typeSafeAi.evaluationModel('jev-latest'));
+ * const { answers } = await evaluate({ model, state, questions });
+ * ```
+ */
+declare function wrapEvaluationModel<M extends AiSdkEvaluationModel>(model: M, options?: WrapEvaluationModelOptions): M;
+//#endregion
+export { GEN_AI_OPERATION_EVALUATE as a, wrapEvaluationModel as c, EvaluationScoreOptions as i, AiSdkEvaluationModel as n, WrapEvaluationModelOptions as o, AiSdkEvaluationResult as r, evaluationScore as s, AiSdkEvaluationAnswer as t };

package/dist/ai-sdk-evaluate-DZgE7PIO.cjs

@@ -0,0 +1,92 @@
+const require_events = require('./events-CfLvw5tV.cjs');
+const require_trace = require('./trace.cjs');
+const require_ai_sdk_bridge = require('./ai-sdk-bridge.cjs');
+
+//#region src/ai-sdk-evaluate.ts
+/** `gen_ai.operation.name` for an evaluation call; not yet in the semconv registry. */
+const GEN_AI_OPERATION_EVALUATE = "evaluate";
+/** Score value and label for the `gen_ai.evaluation.result` event of one answer. */
+function evaluationScore(answer, options = {}) {
+	switch (answer.type) {
+		case "choice": {
+			return {
+				scoreLabel: answer.choice,
+				scoreValue: answer.probabilities?.[answer.choice]
+			};
+		}
+		case "score": {
+			return { scoreValue: answer.score };
+		}
+		case "boolean": {
+			const threshold = options.name === undefined ? undefined : options.booleanThresholds?.[options.name];
+			return {
+				scoreValue: answer.probability,
+				...threshold === undefined ? {} : { scoreLabel: answer.probability >= threshold ? "yes" : "no" }
+			};
+		}
+	}
+}
+/**
+* Wrap an AI SDK evaluation model so each call is traced. Returns a model with
+* the same contract, so it drops into `experimental_evaluate({ model })`.
+*
+* @example
+* ```ts
+* const model = wrapEvaluationModel(typeSafeAi.evaluationModel('jev-latest'));
+* const { answers } = await evaluate({ model, state, questions });
+* ```
+*/
+function wrapEvaluationModel(model, options = {}) {
+	const traced = require_trace.traceGenAI({
+		operation: GEN_AI_OPERATION_EVALUATE,
+		provider: require_ai_sdk_bridge.normalizeAiSdkProvider(model.provider),
+		model: model.modelId,
+		attributes: options.attributes,
+		metrics: options.metrics
+	})((ctx) => async (call) => {
+		const result = await model.doEvaluate(call);
+		const seen = result;
+		require_trace.recordGenAiResponse(ctx, {
+			model: seen.response?.modelId,
+			id: seen.response?.id
+		});
+		if (seen.usage) {
+			require_trace.recordGenAiUsage(ctx, model.modelId, seen.usage, options.cost);
+		}
+		for (const [name, answer] of Object.entries(seen.answers)) {
+			require_events.recordEvaluationResult(ctx, {
+				name,
+				...evaluationScore(answer, {
+					name,
+					booleanThresholds: options.booleanThresholds
+				}),
+				responseId: seen.response?.id
+			});
+		}
+		return result;
+	});
+	return Object.create(model, { doEvaluate: {
+		value: traced,
+		enumerable: true
+	} });
+}
+
+//#endregion
+Object.defineProperty(exports, 'GEN_AI_OPERATION_EVALUATE', {
+  enumerable: true,
+  get: function () {
+    return GEN_AI_OPERATION_EVALUATE;
+  }
+});
+Object.defineProperty(exports, 'evaluationScore', {
+  enumerable: true,
+  get: function () {
+    return evaluationScore;
+  }
+});
+Object.defineProperty(exports, 'wrapEvaluationModel', {
+  enumerable: true,
+  get: function () {
+    return wrapEvaluationModel;
+  }
+});