@zhivex-ai/core 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/README.md +36 -0
  2. package/dist/agent-handoff-contracts.d.ts +12 -0
  3. package/dist/agent-handoff-contracts.d.ts.map +1 -0
  4. package/dist/agent-handoff-contracts.js +18 -0
  5. package/dist/agent-handoff-contracts.js.map +1 -0
  6. package/dist/agent-handoff.d.ts +2 -11
  7. package/dist/agent-handoff.d.ts.map +1 -1
  8. package/dist/agent-handoff.js +1 -18
  9. package/dist/agent-handoff.js.map +1 -1
  10. package/dist/agent-trace.d.ts +1 -5
  11. package/dist/agent-trace.d.ts.map +1 -1
  12. package/dist/agent-trace.js.map +1 -1
  13. package/dist/agent.d.ts.map +1 -1
  14. package/dist/agent.js +39 -23
  15. package/dist/agent.js.map +1 -1
  16. package/dist/api-stability.d.ts.map +1 -1
  17. package/dist/api-stability.js +20 -3
  18. package/dist/api-stability.js.map +1 -1
  19. package/dist/artifact.js +1 -1
  20. package/dist/artifact.js.map +1 -1
  21. package/dist/catalog.d.ts +8 -1
  22. package/dist/catalog.d.ts.map +1 -1
  23. package/dist/catalog.js +7 -0
  24. package/dist/catalog.js.map +1 -1
  25. package/dist/contracts.d.ts +8 -0
  26. package/dist/contracts.d.ts.map +1 -0
  27. package/dist/contracts.js +2 -0
  28. package/dist/contracts.js.map +1 -0
  29. package/dist/generate-text.d.ts.map +1 -1
  30. package/dist/generate-text.js +6 -3
  31. package/dist/generate-text.js.map +1 -1
  32. package/dist/google.d.ts +1 -60
  33. package/dist/google.d.ts.map +1 -1
  34. package/dist/google.js +0 -198
  35. package/dist/google.js.map +1 -1
  36. package/dist/index.d.ts +10 -2
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +6 -2
  39. package/dist/index.js.map +1 -1
  40. package/dist/live-agent.js +1 -1
  41. package/dist/live-agent.js.map +1 -1
  42. package/dist/middleware.d.ts +16 -0
  43. package/dist/middleware.d.ts.map +1 -1
  44. package/dist/middleware.js +338 -58
  45. package/dist/middleware.js.map +1 -1
  46. package/dist/model-capabilities.d.ts +66 -0
  47. package/dist/model-capabilities.d.ts.map +1 -0
  48. package/dist/model-capabilities.js +136 -0
  49. package/dist/model-capabilities.js.map +1 -0
  50. package/dist/model-evaluation.d.ts +185 -0
  51. package/dist/model-evaluation.d.ts.map +1 -0
  52. package/dist/model-evaluation.js +545 -0
  53. package/dist/model-evaluation.js.map +1 -0
  54. package/dist/node.d.ts +8 -0
  55. package/dist/node.d.ts.map +1 -0
  56. package/dist/node.js +8 -0
  57. package/dist/node.js.map +1 -0
  58. package/dist/pricing.d.ts +7 -0
  59. package/dist/pricing.d.ts.map +1 -0
  60. package/dist/pricing.js +2 -0
  61. package/dist/pricing.js.map +1 -0
  62. package/dist/provider-resources.d.ts +61 -0
  63. package/dist/provider-resources.d.ts.map +1 -0
  64. package/dist/provider-resources.js +199 -0
  65. package/dist/provider-resources.js.map +1 -0
  66. package/dist/raw-provider-options.d.ts +20 -0
  67. package/dist/raw-provider-options.d.ts.map +1 -0
  68. package/dist/raw-provider-options.js +63 -0
  69. package/dist/raw-provider-options.js.map +1 -0
  70. package/dist/runner.js +1 -1
  71. package/dist/runner.js.map +1 -1
  72. package/dist/runtime-entry.d.ts +15 -0
  73. package/dist/runtime-entry.d.ts.map +1 -0
  74. package/dist/runtime-entry.js +14 -0
  75. package/dist/runtime-entry.js.map +1 -0
  76. package/dist/runtime.d.ts +9 -1
  77. package/dist/runtime.d.ts.map +1 -1
  78. package/dist/runtime.js +35 -8
  79. package/dist/runtime.js.map +1 -1
  80. package/dist/secure-id-node.d.ts +2 -0
  81. package/dist/secure-id-node.d.ts.map +1 -0
  82. package/dist/secure-id-node.js +3 -0
  83. package/dist/secure-id-node.js.map +1 -0
  84. package/dist/secure-id.d.ts.map +1 -1
  85. package/dist/secure-id.js +6 -2
  86. package/dist/secure-id.js.map +1 -1
  87. package/dist/testing.d.ts +4 -0
  88. package/dist/testing.d.ts.map +1 -0
  89. package/dist/testing.js +4 -0
  90. package/dist/testing.js.map +1 -0
  91. package/dist/types.d.ts +8 -3
  92. package/dist/types.d.ts.map +1 -1
  93. package/dist/ui-entry.d.ts +6 -0
  94. package/dist/ui-entry.d.ts.map +1 -0
  95. package/dist/ui-entry.js +5 -0
  96. package/dist/ui-entry.js.map +1 -0
  97. package/dist/ui.js +1 -1
  98. package/dist/ui.js.map +1 -1
  99. package/dist/workflow-state-contracts.d.ts +76 -0
  100. package/dist/workflow-state-contracts.d.ts.map +1 -0
  101. package/dist/workflow-state-contracts.js +49 -0
  102. package/dist/workflow-state-contracts.js.map +1 -0
  103. package/dist/workflow-state-service.d.ts +3 -34
  104. package/dist/workflow-state-service.d.ts.map +1 -1
  105. package/dist/workflow-state-service.js +2 -2
  106. package/dist/workflow-state-service.js.map +1 -1
  107. package/dist/workflow.d.ts +4 -42
  108. package/dist/workflow.d.ts.map +1 -1
  109. package/dist/workflow.js +3 -47
  110. package/dist/workflow.js.map +1 -1
  111. package/dist/workflows-entry.d.ts +9 -0
  112. package/dist/workflows-entry.d.ts.map +1 -0
  113. package/dist/workflows-entry.js +5 -0
  114. package/dist/workflows-entry.js.map +1 -0
  115. package/package.json +34 -1
  116. package/secure-id-internal.d.ts +1 -0
@@ -0,0 +1,136 @@
1
+ /**
2
+ * Describes how a model exposes a capability.
3
+ *
4
+ * `prompted` means that the SDK can provide the behavior through prompting
5
+ * rather than a provider-native API. `model-dependent` preserves conditional
6
+ * support that the legacy boolean contract cannot represent.
7
+ */
8
+ export const MODEL_CAPABILITY_SUPPORT_LEVELS = [
9
+ "unsupported",
10
+ "native",
11
+ "prompted",
12
+ "model-dependent"
13
+ ];
14
+ /**
15
+ * Returns whether a support level is representable as `true` in the legacy
16
+ * boolean contract. Consumers that need to distinguish native, prompted, and
17
+ * conditional support should use the v2 profile directly.
18
+ */
19
+ export const isModelCapabilitySupported = (support) => support !== undefined && support !== "unsupported";
20
+ /**
21
+ * Converts a kind-specific v2 capability profile to the existing flat
22
+ * `ModelCapabilities` contract.
23
+ */
24
+ export const deriveLegacyModelCapabilities = (profile) => {
25
+ const features = profile.capabilities;
26
+ const supported = (feature) => isModelCapabilitySupported(features[feature]);
27
+ const legacy = {
28
+ streaming: supported("streaming"),
29
+ tools: false,
30
+ structuredOutput: false,
31
+ jsonMode: false,
32
+ toolChoice: false,
33
+ parallelToolCalls: false,
34
+ vision: false,
35
+ files: false,
36
+ audioInput: false,
37
+ audioOutput: false,
38
+ embeddings: false,
39
+ reasoning: false,
40
+ webSearch: false
41
+ };
42
+ if (supported("batch")) {
43
+ legacy.batch = true;
44
+ }
45
+ switch (profile.kind) {
46
+ case "language": {
47
+ legacy.tools = supported("tools");
48
+ legacy.structuredOutput = supported("structuredOutput");
49
+ legacy.jsonMode = supported("jsonMode");
50
+ legacy.toolChoice = supported("toolChoice");
51
+ legacy.parallelToolCalls = supported("parallelToolCalls");
52
+ legacy.vision = supported("vision");
53
+ legacy.files = supported("files");
54
+ legacy.audioInput = supported("audioInput");
55
+ legacy.audioOutput = supported("audioOutput");
56
+ legacy.reasoning = supported("reasoning");
57
+ legacy.webSearch = supported("webSearch");
58
+ if (supported("fileSearch"))
59
+ legacy.fileSearch = true;
60
+ if (supported("urlContext"))
61
+ legacy.urlContext = true;
62
+ if (supported("contextCaching"))
63
+ legacy.contextCaching = true;
64
+ if (supported("explicitPromptCaching")) {
65
+ legacy.explicitPromptCaching = true;
66
+ }
67
+ if (supported("interactions"))
68
+ legacy.interactions = true;
69
+ if (supported("rawPrediction"))
70
+ legacy.rawPrediction = true;
71
+ if (supported("computerUse"))
72
+ legacy.computerUse = true;
73
+ if (legacy.reasoning && profile.reasoningDetails) {
74
+ legacy.reasoningEfforts = profile.reasoningDetails.efforts
75
+ ? [...profile.reasoningDetails.efforts]
76
+ : undefined;
77
+ legacy.reasoningModes = profile.reasoningDetails.modes
78
+ ? [...profile.reasoningDetails.modes]
79
+ : undefined;
80
+ legacy.reasoningContexts = profile.reasoningDetails.contexts
81
+ ? [...profile.reasoningDetails.contexts]
82
+ : undefined;
83
+ }
84
+ if (profile.agent && isModelCapabilitySupported(profile.agent.support)) {
85
+ legacy.agentCapabilities = { ...profile.agent.capabilities };
86
+ }
87
+ break;
88
+ }
89
+ case "embedding":
90
+ legacy.embeddings = supported("embeddings");
91
+ break;
92
+ case "transcription":
93
+ legacy.audioInput = supported("audioInput");
94
+ break;
95
+ case "speech":
96
+ legacy.audioOutput = supported("audioOutput");
97
+ break;
98
+ case "image-generation":
99
+ legacy.imageGeneration = supported("imageGeneration");
100
+ legacy.vision = supported("imageInput");
101
+ break;
102
+ case "video-generation":
103
+ legacy.videoGeneration = supported("videoGeneration");
104
+ legacy.vision = supported("imageInput");
105
+ legacy.audioOutput = supported("audioOutput");
106
+ break;
107
+ case "music-generation":
108
+ legacy.musicGeneration = supported("musicGeneration");
109
+ legacy.vision = supported("imageInput");
110
+ legacy.audioInput = supported("audioInput");
111
+ legacy.audioOutput = supported("audioOutput");
112
+ break;
113
+ case "realtime": {
114
+ const tools = supported("tools");
115
+ const imageInput = supported("imageInput");
116
+ legacy.tools = tools;
117
+ legacy.vision = imageInput;
118
+ legacy.audioInput = supported("audioInput");
119
+ legacy.audioOutput = supported("audioOutput");
120
+ legacy.realtime = {
121
+ sessions: supported("sessions"),
122
+ audioInput: legacy.audioInput,
123
+ audioOutput: legacy.audioOutput,
124
+ imageInput,
125
+ tools,
126
+ browserTokens: supported("browserTokens")
127
+ };
128
+ if (profile.agent && isModelCapabilitySupported(profile.agent.support)) {
129
+ legacy.agentCapabilities = { ...profile.agent.capabilities };
130
+ }
131
+ break;
132
+ }
133
+ }
134
+ return legacy;
135
+ };
136
+ //# sourceMappingURL=model-capabilities.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-capabilities.js","sourceRoot":"","sources":["../src/model-capabilities.ts"],"names":[],"mappings":"AAMA;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG;IAC7C,aAAa;IACb,QAAQ;IACR,UAAU;IACV,iBAAiB;CACT,CAAC;AA2KX;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACxC,OAAgD,EACvC,EAAE,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,aAAa,CAAC;AAEjE;;;GAGG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAC3C,OAA+B,EACZ,EAAE;IACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,YAExB,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAW,EAAE,CAC7D,0BAA0B,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAEhD,MAAM,MAAM,GAAsB;QAChC,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC;QACjC,KAAK,EAAE,KAAK;QACZ,gBAAgB,EAAE,KAAK;QACvB,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,iBAAiB,EAAE,KAAK;QACxB,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,KAAK;QACZ,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,KAAK;QAClB,UAAU,EAAE,KAAK;QACjB,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,CAAC,gBAAgB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACxD,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,iBAAiB,GAAG,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YAC1C,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YAE1C,IAAI,SAAS,CAAC,YAAY,CAAC;gBAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;YACtD,IAAI,SAAS,CAAC,YAAY,CAAC;gBAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;YACtD,IAAI,SAAS,CAAC,gBAAgB,CAAC;gBAAE,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;YAC9D,IAAI,SAAS,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACtC,CAAC;YACD,IAAI,SAAS,CAAC,cAAc,CAAC;gBAAE,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;YAC1D,IAAI,SAAS,CAAC,eAAe,CAAC;gBAAE,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;YAC5D,IAAI,SAAS,CAAC,aAAa,CAAC;gBAAE,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC;YAExD,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBACjD,MAAM,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO;oBACxD,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC;oBACvC,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK;oBACpD,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,CAAC,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ;oBAC1D,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC;oBACxC,CAAC,CAAC,SAAS,CAAC;YAChB,CAAC;YAED,IAAI,OAAO,CAAC,KAAK,IAAI,0BAA0B,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,iBAAiB,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC/D,CAAC;YACD,MAAM;QACR,CAAC;QACD,KAAK,WAAW;YACd,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM;QACR,KAAK,eAAe;YAClB,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAE3C,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAC3B,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,CAAC,QAAQ,GAAG;gBAChB,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC;gBAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,UAAU;gBACV,KAAK;gBACL,aAAa,EAAE,SAAS,CAAC,eAAe,CAAC;aAC1C,CAAC;YAEF,IAAI,OAAO,CAAC,KAAK,IAAI,0BAA0B,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,iBAAiB,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC/D,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
@@ -0,0 +1,185 @@
1
+ import type { ZodTypeAny } from "zod";
2
+ import { type CostEstimate, type TokenPricing } from "./agent-trace.js";
3
+ import type { ModelCatalog } from "./catalog.js";
4
+ import type { EmbeddingModel, FinishReason, GenerateTextOptions, JsonValue, LanguageModel, ModelMessage, TokenUsage, ToolExecutionResult } from "./types.js";
5
+ export declare const MODEL_EVALUATION_REPORT_SCHEMA_VERSION: 1;
6
+ export interface ModelEvaluationCase {
7
+ name: string;
8
+ prompt?: string;
9
+ messages?: ModelMessage[];
10
+ reference?: JsonValue;
11
+ expectedToolCalls?: string[];
12
+ settings?: ModelEvaluationGenerationSettings;
13
+ metadata?: Record<string, JsonValue>;
14
+ }
15
+ export type ModelEvaluationGenerationSettings = Omit<GenerateTextOptions<LanguageModel>, "model" | "prompt" | "messages">;
16
+ export interface ModelEvaluationCandidate {
17
+ id: string;
18
+ model: LanguageModel;
19
+ settings?: ModelEvaluationGenerationSettings;
20
+ pricing?: TokenPricing;
21
+ metadata?: Record<string, JsonValue>;
22
+ }
23
+ export interface ModelEvaluationScore {
24
+ scorerId: string;
25
+ score?: number;
26
+ feedback?: string;
27
+ metadata?: Record<string, JsonValue>;
28
+ error?: string;
29
+ }
30
+ export interface ModelEvaluationScorerResult {
31
+ score: number;
32
+ feedback?: string;
33
+ metadata?: Record<string, JsonValue>;
34
+ }
35
+ export interface ModelEvaluationScorerContext {
36
+ testCase: ModelEvaluationCase;
37
+ candidate: Pick<ModelEvaluationCandidate, "id" | "metadata"> & {
38
+ provider: string;
39
+ modelId: string;
40
+ };
41
+ repetition: number;
42
+ outputText?: string;
43
+ finishReason?: FinishReason;
44
+ usage?: TokenUsage;
45
+ toolResults: ToolExecutionResult[];
46
+ latencyMs: number;
47
+ cost?: CostEstimate;
48
+ error?: string;
49
+ }
50
+ export interface ModelEvaluationScorer {
51
+ id: string;
52
+ score(context: ModelEvaluationScorerContext): number | ModelEvaluationScorerResult | Promise<number | ModelEvaluationScorerResult>;
53
+ }
54
+ export interface ModelEvaluationRunResult {
55
+ caseName: string;
56
+ candidateId: string;
57
+ provider: string;
58
+ modelId: string;
59
+ repetition: number;
60
+ ok: boolean;
61
+ outputText?: string;
62
+ finishReason?: FinishReason;
63
+ usage?: TokenUsage;
64
+ toolCalls: string[];
65
+ latencyMs: number;
66
+ cost?: CostEstimate;
67
+ scores: ModelEvaluationScore[];
68
+ meanScore?: number;
69
+ error?: string;
70
+ caseMetadata?: Record<string, JsonValue>;
71
+ candidateMetadata?: Record<string, JsonValue>;
72
+ }
73
+ export interface ModelEvaluationCandidateSummary {
74
+ candidateId: string;
75
+ provider: string;
76
+ modelId: string;
77
+ runs: number;
78
+ successfulRuns: number;
79
+ failedRuns: number;
80
+ errorRate: number;
81
+ meanScore?: number;
82
+ minScore?: number;
83
+ maxScore?: number;
84
+ p50LatencyMs: number;
85
+ p95LatencyMs: number;
86
+ totalCost?: number;
87
+ meanCost?: number;
88
+ currency?: string;
89
+ usage?: TokenUsage;
90
+ metadata?: Record<string, JsonValue>;
91
+ }
92
+ export interface ModelEvaluationThresholds {
93
+ minMeanScore?: number;
94
+ maxTotalCost?: number;
95
+ maxP95LatencyMs?: number;
96
+ maxErrorRate?: number;
97
+ }
98
+ export type ModelEvaluationGateCheckCode = "min_mean_score" | "max_total_cost" | "max_p95_latency_ms" | "max_error_rate";
99
+ export interface ModelEvaluationGateCheck {
100
+ candidateId: string;
101
+ code: ModelEvaluationGateCheckCode;
102
+ ok: boolean;
103
+ actual?: number;
104
+ threshold: number;
105
+ message: string;
106
+ }
107
+ export interface ModelEvaluationGateResult {
108
+ ok: boolean;
109
+ thresholds: ModelEvaluationThresholds;
110
+ checks: ModelEvaluationGateCheck[];
111
+ }
112
+ export interface ModelEvaluationReport {
113
+ schemaVersion: typeof MODEL_EVALUATION_REPORT_SCHEMA_VERSION;
114
+ createdAt: string;
115
+ ok: boolean;
116
+ repetitions: number;
117
+ totalRuns: number;
118
+ cases: string[];
119
+ candidates: ModelEvaluationCandidateSummary[];
120
+ runs: ModelEvaluationRunResult[];
121
+ gate?: ModelEvaluationGateResult;
122
+ metadata?: Record<string, JsonValue>;
123
+ }
124
+ export interface ModelEvaluationSuite {
125
+ cases: ModelEvaluationCase[];
126
+ candidates: ModelEvaluationCandidate[];
127
+ scorers: ModelEvaluationScorer[];
128
+ repetitions?: number;
129
+ maxConcurrency?: number;
130
+ catalog?: ModelCatalog;
131
+ thresholds?: ModelEvaluationThresholds;
132
+ metadata?: Record<string, JsonValue>;
133
+ }
134
+ export interface ModelEvaluationCandidateComparison {
135
+ candidateId: string;
136
+ baseline?: ModelEvaluationCandidateSummary;
137
+ target?: ModelEvaluationCandidateSummary;
138
+ scoreDelta?: number;
139
+ totalCostDelta?: number;
140
+ p95LatencyDeltaMs?: number;
141
+ errorRateDelta?: number;
142
+ }
143
+ export interface ModelEvaluationComparison {
144
+ baselineCreatedAt: string;
145
+ targetCreatedAt: string;
146
+ candidates: ModelEvaluationCandidateComparison[];
147
+ }
148
+ export declare const evaluateModelEvaluationGate: (candidates: ModelEvaluationCandidateSummary[], thresholds: ModelEvaluationThresholds) => ModelEvaluationGateResult;
149
+ export declare const runModelEvaluation: (suite: ModelEvaluationSuite) => Promise<ModelEvaluationReport>;
150
+ export declare const compareModelEvaluationReports: (baseline: ModelEvaluationReport, target: ModelEvaluationReport) => ModelEvaluationComparison;
151
+ export declare const createExactMatchScorer: (options?: {
152
+ id?: string;
153
+ caseSensitive?: boolean;
154
+ trim?: boolean;
155
+ }) => ModelEvaluationScorer;
156
+ export declare const createContainsScorer: (options?: {
157
+ id?: string;
158
+ caseSensitive?: boolean;
159
+ }) => ModelEvaluationScorer;
160
+ export declare const createRegexScorer: (options: {
161
+ pattern: RegExp | ((testCase: ModelEvaluationCase) => RegExp);
162
+ id?: string;
163
+ }) => ModelEvaluationScorer;
164
+ export declare const createJsonValueScorer: (options?: {
165
+ id?: string;
166
+ }) => ModelEvaluationScorer;
167
+ export declare const createJsonSchemaScorer: (schema: ZodTypeAny, options?: {
168
+ id?: string;
169
+ }) => ModelEvaluationScorer;
170
+ export declare const createToolCallScorer: (options?: {
171
+ id?: string;
172
+ requireExactOrder?: boolean;
173
+ }) => ModelEvaluationScorer;
174
+ export declare const createEmbeddingSimilarityScorer: (options: {
175
+ model: EmbeddingModel;
176
+ id?: string;
177
+ minimumSimilarity?: number;
178
+ }) => ModelEvaluationScorer;
179
+ export declare const createModelJudgeScorer: (options: {
180
+ model: LanguageModel;
181
+ rubric: string;
182
+ id?: string;
183
+ maxTokens?: number;
184
+ }) => ModelEvaluationScorer;
185
+ //# sourceMappingURL=model-evaluation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model-evaluation.d.ts","sourceRoot":"","sources":["../src/model-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,KAAK,CAAC;AAEtC,OAAO,EAAqB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAC3F,OAAO,KAAK,EAAE,YAAY,EAAqB,MAAM,cAAc,CAAC;AAKpE,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,YAAY,EACZ,UAAU,EACV,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,sCAAsC,EAAG,CAAU,CAAC;AAEjE,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,EAAE,iCAAiC,CAAC;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,MAAM,iCAAiC,GAAG,IAAI,CAClD,mBAAmB,CAAC,aAAa,CAAC,EAClC,OAAO,GAAG,QAAQ,GAAG,UAAU,CAChC,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,EAAE,iCAAiC,CAAC;IAC7C,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,GAAG,UAAU,CAAC,GAAG;QAC7D,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CACH,OAAO,EAAE,4BAA4B,GACpC,MAAM,GAAG,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,2BAA2B,CAAC,CAAC;CACzF;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACzC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,+BAA+B;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,yBAAyB;IACxC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,4BAA4B,GACpC,gBAAgB,GAChB,gBAAgB,GAChB,oBAAoB,GACpB,gBAAgB,CAAC;AAErB,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,4BAA4B,CAAC;IACnC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,EAAE,yBAAyB,CAAC;IACtC,MAAM,EAAE,wBAAwB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,OAAO,sCAAsC,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,+BAA+B,EAAE,CAAC;IAC9C,IAAI,EAAE,wBAAwB,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,UAAU,EAAE,wBAAwB,EAAE,CAAC;IACvC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,kCAAkC;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,+BAA+B,CAAC;IAC3C,MAAM,CAAC,EAAE,+BAA+B,CAAC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,kCAAkC,EAAE,CAAC;CAClD;AAkSD,eAAO,MAAM,2BAA2B,GACtC,YAAY,+BAA+B,EAAE,EAC7C,YAAY,yBAAyB,KACpC,yBA6DF,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAU,OAAO,oBAAoB,KAAG,OAAO,CAAC,qBAAqB,CAuCnG,CAAC;AAEF,eAAO,MAAM,6BAA6B,GACxC,UAAU,qBAAqB,EAC/B,QAAQ,qBAAqB,KAC5B,yBA8BF,CAAC;AAoBF,eAAO,MAAM,sBAAsB,GAAI,UAAS;IAC9C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;CACX,KAAG,qBAaP,CAAC;AAEH,eAAO,MAAM,oBAAoB,GAAI,UAAS;IAC5C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,qBAUP,CAAC;AAEH,eAAO,MAAM,iBAAiB,GAAI,SAAS;IACzC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,QAAQ,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC;IAC9D,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,KAAG,qBAOF,CAAC;AAEH,eAAO,MAAM,qBAAqB,GAAI,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAA;CAAO,KAAG,qBAgBpE,CAAC;AAEH,eAAO,MAAM,sBAAsB,GAAI,QAAQ,UAAU,EAAE,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAA;CAAO,KAAG,qBAczF,CAAC;AAEH,eAAO,MAAM,oBAAoB,GAAI,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAO,KAAG,qBA6BhG,CAAC;AAEH,eAAO,MAAM,+BAA+B,GAAI,SAAS;IACvD,KAAK,EAAE,cAAc,CAAC;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,KAAG,qBAyCH,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,SAAS;IAC9C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAgCF,CAAC"}