@zhivex-ai/core 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -0
- package/dist/agent-handoff-contracts.d.ts +12 -0
- package/dist/agent-handoff-contracts.d.ts.map +1 -0
- package/dist/agent-handoff-contracts.js +18 -0
- package/dist/agent-handoff-contracts.js.map +1 -0
- package/dist/agent-handoff.d.ts +2 -11
- package/dist/agent-handoff.d.ts.map +1 -1
- package/dist/agent-handoff.js +1 -18
- package/dist/agent-handoff.js.map +1 -1
- package/dist/agent-trace.d.ts +1 -5
- package/dist/agent-trace.d.ts.map +1 -1
- package/dist/agent-trace.js.map +1 -1
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +39 -23
- package/dist/agent.js.map +1 -1
- package/dist/api-stability.d.ts.map +1 -1
- package/dist/api-stability.js +20 -3
- package/dist/api-stability.js.map +1 -1
- package/dist/artifact.js +1 -1
- package/dist/artifact.js.map +1 -1
- package/dist/catalog.d.ts +8 -1
- package/dist/catalog.d.ts.map +1 -1
- package/dist/catalog.js +7 -0
- package/dist/catalog.js.map +1 -1
- package/dist/contracts.d.ts +8 -0
- package/dist/contracts.d.ts.map +1 -0
- package/dist/contracts.js +2 -0
- package/dist/contracts.js.map +1 -0
- package/dist/generate-text.d.ts.map +1 -1
- package/dist/generate-text.js +6 -3
- package/dist/generate-text.js.map +1 -1
- package/dist/google.d.ts +1 -60
- package/dist/google.d.ts.map +1 -1
- package/dist/google.js +0 -198
- package/dist/google.js.map +1 -1
- package/dist/index.d.ts +10 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -2
- package/dist/index.js.map +1 -1
- package/dist/live-agent.js +1 -1
- package/dist/live-agent.js.map +1 -1
- package/dist/middleware.d.ts +16 -0
- package/dist/middleware.d.ts.map +1 -1
- package/dist/middleware.js +338 -58
- package/dist/middleware.js.map +1 -1
- package/dist/model-capabilities.d.ts +66 -0
- package/dist/model-capabilities.d.ts.map +1 -0
- package/dist/model-capabilities.js +136 -0
- package/dist/model-capabilities.js.map +1 -0
- package/dist/model-evaluation.d.ts +185 -0
- package/dist/model-evaluation.d.ts.map +1 -0
- package/dist/model-evaluation.js +545 -0
- package/dist/model-evaluation.js.map +1 -0
- package/dist/node.d.ts +8 -0
- package/dist/node.d.ts.map +1 -0
- package/dist/node.js +8 -0
- package/dist/node.js.map +1 -0
- package/dist/pricing.d.ts +7 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +2 -0
- package/dist/pricing.js.map +1 -0
- package/dist/provider-resources.d.ts +61 -0
- package/dist/provider-resources.d.ts.map +1 -0
- package/dist/provider-resources.js +199 -0
- package/dist/provider-resources.js.map +1 -0
- package/dist/raw-provider-options.d.ts +20 -0
- package/dist/raw-provider-options.d.ts.map +1 -0
- package/dist/raw-provider-options.js +63 -0
- package/dist/raw-provider-options.js.map +1 -0
- package/dist/runner.js +1 -1
- package/dist/runner.js.map +1 -1
- package/dist/runtime-entry.d.ts +15 -0
- package/dist/runtime-entry.d.ts.map +1 -0
- package/dist/runtime-entry.js +14 -0
- package/dist/runtime-entry.js.map +1 -0
- package/dist/runtime.d.ts +9 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +35 -8
- package/dist/runtime.js.map +1 -1
- package/dist/secure-id-node.d.ts +2 -0
- package/dist/secure-id-node.d.ts.map +1 -0
- package/dist/secure-id-node.js +3 -0
- package/dist/secure-id-node.js.map +1 -0
- package/dist/secure-id.d.ts.map +1 -1
- package/dist/secure-id.js +6 -2
- package/dist/secure-id.js.map +1 -1
- package/dist/testing.d.ts +4 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +4 -0
- package/dist/testing.js.map +1 -0
- package/dist/types.d.ts +8 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/ui-entry.d.ts +6 -0
- package/dist/ui-entry.d.ts.map +1 -0
- package/dist/ui-entry.js +5 -0
- package/dist/ui-entry.js.map +1 -0
- package/dist/ui.js +1 -1
- package/dist/ui.js.map +1 -1
- package/dist/workflow-state-contracts.d.ts +76 -0
- package/dist/workflow-state-contracts.d.ts.map +1 -0
- package/dist/workflow-state-contracts.js +49 -0
- package/dist/workflow-state-contracts.js.map +1 -0
- package/dist/workflow-state-service.d.ts +3 -34
- package/dist/workflow-state-service.d.ts.map +1 -1
- package/dist/workflow-state-service.js +2 -2
- package/dist/workflow-state-service.js.map +1 -1
- package/dist/workflow.d.ts +4 -42
- package/dist/workflow.d.ts.map +1 -1
- package/dist/workflow.js +3 -47
- package/dist/workflow.js.map +1 -1
- package/dist/workflows-entry.d.ts +9 -0
- package/dist/workflows-entry.d.ts.map +1 -0
- package/dist/workflows-entry.js +5 -0
- package/dist/workflows-entry.js.map +1 -0
- package/package.json +34 -1
- package/secure-id-internal.d.ts +1 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describes how a model exposes a capability.
|
|
3
|
+
*
|
|
4
|
+
* `prompted` means that the SDK can provide the behavior through prompting
|
|
5
|
+
* rather than a provider-native API. `model-dependent` preserves conditional
|
|
6
|
+
* support that the legacy boolean contract cannot represent.
|
|
7
|
+
*/
|
|
8
|
+
export const MODEL_CAPABILITY_SUPPORT_LEVELS = [
|
|
9
|
+
"unsupported",
|
|
10
|
+
"native",
|
|
11
|
+
"prompted",
|
|
12
|
+
"model-dependent"
|
|
13
|
+
];
|
|
14
|
+
/**
|
|
15
|
+
* Returns whether a support level is representable as `true` in the legacy
|
|
16
|
+
* boolean contract. Consumers that need to distinguish native, prompted, and
|
|
17
|
+
* conditional support should use the v2 profile directly.
|
|
18
|
+
*/
|
|
19
|
+
export const isModelCapabilitySupported = (support) => support !== undefined && support !== "unsupported";
|
|
20
|
+
/**
|
|
21
|
+
* Converts a kind-specific v2 capability profile to the existing flat
|
|
22
|
+
* `ModelCapabilities` contract.
|
|
23
|
+
*/
|
|
24
|
+
export const deriveLegacyModelCapabilities = (profile) => {
|
|
25
|
+
const features = profile.capabilities;
|
|
26
|
+
const supported = (feature) => isModelCapabilitySupported(features[feature]);
|
|
27
|
+
const legacy = {
|
|
28
|
+
streaming: supported("streaming"),
|
|
29
|
+
tools: false,
|
|
30
|
+
structuredOutput: false,
|
|
31
|
+
jsonMode: false,
|
|
32
|
+
toolChoice: false,
|
|
33
|
+
parallelToolCalls: false,
|
|
34
|
+
vision: false,
|
|
35
|
+
files: false,
|
|
36
|
+
audioInput: false,
|
|
37
|
+
audioOutput: false,
|
|
38
|
+
embeddings: false,
|
|
39
|
+
reasoning: false,
|
|
40
|
+
webSearch: false
|
|
41
|
+
};
|
|
42
|
+
if (supported("batch")) {
|
|
43
|
+
legacy.batch = true;
|
|
44
|
+
}
|
|
45
|
+
switch (profile.kind) {
|
|
46
|
+
case "language": {
|
|
47
|
+
legacy.tools = supported("tools");
|
|
48
|
+
legacy.structuredOutput = supported("structuredOutput");
|
|
49
|
+
legacy.jsonMode = supported("jsonMode");
|
|
50
|
+
legacy.toolChoice = supported("toolChoice");
|
|
51
|
+
legacy.parallelToolCalls = supported("parallelToolCalls");
|
|
52
|
+
legacy.vision = supported("vision");
|
|
53
|
+
legacy.files = supported("files");
|
|
54
|
+
legacy.audioInput = supported("audioInput");
|
|
55
|
+
legacy.audioOutput = supported("audioOutput");
|
|
56
|
+
legacy.reasoning = supported("reasoning");
|
|
57
|
+
legacy.webSearch = supported("webSearch");
|
|
58
|
+
if (supported("fileSearch"))
|
|
59
|
+
legacy.fileSearch = true;
|
|
60
|
+
if (supported("urlContext"))
|
|
61
|
+
legacy.urlContext = true;
|
|
62
|
+
if (supported("contextCaching"))
|
|
63
|
+
legacy.contextCaching = true;
|
|
64
|
+
if (supported("explicitPromptCaching")) {
|
|
65
|
+
legacy.explicitPromptCaching = true;
|
|
66
|
+
}
|
|
67
|
+
if (supported("interactions"))
|
|
68
|
+
legacy.interactions = true;
|
|
69
|
+
if (supported("rawPrediction"))
|
|
70
|
+
legacy.rawPrediction = true;
|
|
71
|
+
if (supported("computerUse"))
|
|
72
|
+
legacy.computerUse = true;
|
|
73
|
+
if (legacy.reasoning && profile.reasoningDetails) {
|
|
74
|
+
legacy.reasoningEfforts = profile.reasoningDetails.efforts
|
|
75
|
+
? [...profile.reasoningDetails.efforts]
|
|
76
|
+
: undefined;
|
|
77
|
+
legacy.reasoningModes = profile.reasoningDetails.modes
|
|
78
|
+
? [...profile.reasoningDetails.modes]
|
|
79
|
+
: undefined;
|
|
80
|
+
legacy.reasoningContexts = profile.reasoningDetails.contexts
|
|
81
|
+
? [...profile.reasoningDetails.contexts]
|
|
82
|
+
: undefined;
|
|
83
|
+
}
|
|
84
|
+
if (profile.agent && isModelCapabilitySupported(profile.agent.support)) {
|
|
85
|
+
legacy.agentCapabilities = { ...profile.agent.capabilities };
|
|
86
|
+
}
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
case "embedding":
|
|
90
|
+
legacy.embeddings = supported("embeddings");
|
|
91
|
+
break;
|
|
92
|
+
case "transcription":
|
|
93
|
+
legacy.audioInput = supported("audioInput");
|
|
94
|
+
break;
|
|
95
|
+
case "speech":
|
|
96
|
+
legacy.audioOutput = supported("audioOutput");
|
|
97
|
+
break;
|
|
98
|
+
case "image-generation":
|
|
99
|
+
legacy.imageGeneration = supported("imageGeneration");
|
|
100
|
+
legacy.vision = supported("imageInput");
|
|
101
|
+
break;
|
|
102
|
+
case "video-generation":
|
|
103
|
+
legacy.videoGeneration = supported("videoGeneration");
|
|
104
|
+
legacy.vision = supported("imageInput");
|
|
105
|
+
legacy.audioOutput = supported("audioOutput");
|
|
106
|
+
break;
|
|
107
|
+
case "music-generation":
|
|
108
|
+
legacy.musicGeneration = supported("musicGeneration");
|
|
109
|
+
legacy.vision = supported("imageInput");
|
|
110
|
+
legacy.audioInput = supported("audioInput");
|
|
111
|
+
legacy.audioOutput = supported("audioOutput");
|
|
112
|
+
break;
|
|
113
|
+
case "realtime": {
|
|
114
|
+
const tools = supported("tools");
|
|
115
|
+
const imageInput = supported("imageInput");
|
|
116
|
+
legacy.tools = tools;
|
|
117
|
+
legacy.vision = imageInput;
|
|
118
|
+
legacy.audioInput = supported("audioInput");
|
|
119
|
+
legacy.audioOutput = supported("audioOutput");
|
|
120
|
+
legacy.realtime = {
|
|
121
|
+
sessions: supported("sessions"),
|
|
122
|
+
audioInput: legacy.audioInput,
|
|
123
|
+
audioOutput: legacy.audioOutput,
|
|
124
|
+
imageInput,
|
|
125
|
+
tools,
|
|
126
|
+
browserTokens: supported("browserTokens")
|
|
127
|
+
};
|
|
128
|
+
if (profile.agent && isModelCapabilitySupported(profile.agent.support)) {
|
|
129
|
+
legacy.agentCapabilities = { ...profile.agent.capabilities };
|
|
130
|
+
}
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return legacy;
|
|
135
|
+
};
|
|
136
|
+
//# sourceMappingURL=model-capabilities.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-capabilities.js","sourceRoot":"","sources":["../src/model-capabilities.ts"],"names":[],"mappings":"AAMA;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG;IAC7C,aAAa;IACb,QAAQ;IACR,UAAU;IACV,iBAAiB;CACT,CAAC;AA2KX;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACxC,OAAgD,EACvC,EAAE,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,aAAa,CAAC;AAEjE;;;GAGG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,CAC3C,OAA+B,EACZ,EAAE;IACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,YAExB,CAAC;IACF,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAW,EAAE,CAC7D,0BAA0B,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAEhD,MAAM,MAAM,GAAsB;QAChC,SAAS,EAAE,SAAS,CAAC,WAAW,CAAC;QACjC,KAAK,EAAE,KAAK;QACZ,gBAAgB,EAAE,KAAK;QACvB,QAAQ,EAAE,KAAK;QACf,UAAU,EAAE,KAAK;QACjB,iBAAiB,EAAE,KAAK;QACxB,MAAM,EAAE,KAAK;QACb,KAAK,EAAE,KAAK;QACZ,UAAU,EAAE,KAAK;QACjB,WAAW,EAAE,KAAK;QAClB,UAAU,EAAE,KAAK;QACjB,SAAS,EAAE,KAAK;QAChB,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,IAAI,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,CAAC,gBAAgB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACxD,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,iBAAiB,GAAG,SAAS,CAAC,mBAAmB,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YAC1C,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;YAE1C,IAAI,SAAS,CAAC,YAAY,CAAC;gBAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;YACtD,IAAI,SAAS,CAAC,YAAY,CAAC;gBAAE,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;YACtD,IAAI,SAAS,CAAC,gBAAgB,CAAC;gBAAE,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;YAC9D,IAAI,SAAS,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACvC,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;YACtC,CAAC;YACD,IAAI,SAAS,CAAC,cAAc,CAAC;gBAAE,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;YAC1D,IAAI,SAAS,CAAC,eAAe,CAAC;gBAAE,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;YAC5D,IAAI,SAAS,CAAC,aAAa,CAAC;gBAAE,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC;YAExD,IAAI,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBACjD,MAAM,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO;oBACxD,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC;oBACvC,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,CAAC,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK;oBACpD,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC;oBACrC,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,CAAC,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ;oBAC1D,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC;oBACxC,CAAC,CAAC,SAAS,CAAC;YAChB,CAAC;YAED,IAAI,OAAO,CAAC,KAAK,IAAI,0BAA0B,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,iBAAiB,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC/D,CAAC;YACD,MAAM;QACR,CAAC;QACD,KAAK,WAAW;YACd,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM;QACR,KAAK,eAAe;YAClB,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,kBAAkB;YACrB,MAAM,CAAC,eAAe,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM;QACR,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAE3C,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACrB,MAAM,CAAC,MAAM,GAAG,UAAU,CAAC;YAC3B,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;YAC5C,MAAM,CAAC,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC;YAC9C,MAAM,CAAC,QAAQ,GAAG;gBAChB,QAAQ,EAAE,SAAS,CAAC,UAAU,CAAC;gBAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,UAAU;gBACV,KAAK;gBACL,aAAa,EAAE,SAAS,CAAC,eAAe,CAAC;aAC1C,CAAC;YAEF,IAAI,OAAO,CAAC,KAAK,IAAI,0BAA0B,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,iBAAiB,GAAG,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC/D,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import type { ZodTypeAny } from "zod";
|
|
2
|
+
import { type CostEstimate, type TokenPricing } from "./agent-trace.js";
|
|
3
|
+
import type { ModelCatalog } from "./catalog.js";
|
|
4
|
+
import type { EmbeddingModel, FinishReason, GenerateTextOptions, JsonValue, LanguageModel, ModelMessage, TokenUsage, ToolExecutionResult } from "./types.js";
|
|
5
|
+
export declare const MODEL_EVALUATION_REPORT_SCHEMA_VERSION: 1;
|
|
6
|
+
export interface ModelEvaluationCase {
|
|
7
|
+
name: string;
|
|
8
|
+
prompt?: string;
|
|
9
|
+
messages?: ModelMessage[];
|
|
10
|
+
reference?: JsonValue;
|
|
11
|
+
expectedToolCalls?: string[];
|
|
12
|
+
settings?: ModelEvaluationGenerationSettings;
|
|
13
|
+
metadata?: Record<string, JsonValue>;
|
|
14
|
+
}
|
|
15
|
+
export type ModelEvaluationGenerationSettings = Omit<GenerateTextOptions<LanguageModel>, "model" | "prompt" | "messages">;
|
|
16
|
+
export interface ModelEvaluationCandidate {
|
|
17
|
+
id: string;
|
|
18
|
+
model: LanguageModel;
|
|
19
|
+
settings?: ModelEvaluationGenerationSettings;
|
|
20
|
+
pricing?: TokenPricing;
|
|
21
|
+
metadata?: Record<string, JsonValue>;
|
|
22
|
+
}
|
|
23
|
+
export interface ModelEvaluationScore {
|
|
24
|
+
scorerId: string;
|
|
25
|
+
score?: number;
|
|
26
|
+
feedback?: string;
|
|
27
|
+
metadata?: Record<string, JsonValue>;
|
|
28
|
+
error?: string;
|
|
29
|
+
}
|
|
30
|
+
export interface ModelEvaluationScorerResult {
|
|
31
|
+
score: number;
|
|
32
|
+
feedback?: string;
|
|
33
|
+
metadata?: Record<string, JsonValue>;
|
|
34
|
+
}
|
|
35
|
+
export interface ModelEvaluationScorerContext {
|
|
36
|
+
testCase: ModelEvaluationCase;
|
|
37
|
+
candidate: Pick<ModelEvaluationCandidate, "id" | "metadata"> & {
|
|
38
|
+
provider: string;
|
|
39
|
+
modelId: string;
|
|
40
|
+
};
|
|
41
|
+
repetition: number;
|
|
42
|
+
outputText?: string;
|
|
43
|
+
finishReason?: FinishReason;
|
|
44
|
+
usage?: TokenUsage;
|
|
45
|
+
toolResults: ToolExecutionResult[];
|
|
46
|
+
latencyMs: number;
|
|
47
|
+
cost?: CostEstimate;
|
|
48
|
+
error?: string;
|
|
49
|
+
}
|
|
50
|
+
export interface ModelEvaluationScorer {
|
|
51
|
+
id: string;
|
|
52
|
+
score(context: ModelEvaluationScorerContext): number | ModelEvaluationScorerResult | Promise<number | ModelEvaluationScorerResult>;
|
|
53
|
+
}
|
|
54
|
+
export interface ModelEvaluationRunResult {
|
|
55
|
+
caseName: string;
|
|
56
|
+
candidateId: string;
|
|
57
|
+
provider: string;
|
|
58
|
+
modelId: string;
|
|
59
|
+
repetition: number;
|
|
60
|
+
ok: boolean;
|
|
61
|
+
outputText?: string;
|
|
62
|
+
finishReason?: FinishReason;
|
|
63
|
+
usage?: TokenUsage;
|
|
64
|
+
toolCalls: string[];
|
|
65
|
+
latencyMs: number;
|
|
66
|
+
cost?: CostEstimate;
|
|
67
|
+
scores: ModelEvaluationScore[];
|
|
68
|
+
meanScore?: number;
|
|
69
|
+
error?: string;
|
|
70
|
+
caseMetadata?: Record<string, JsonValue>;
|
|
71
|
+
candidateMetadata?: Record<string, JsonValue>;
|
|
72
|
+
}
|
|
73
|
+
export interface ModelEvaluationCandidateSummary {
|
|
74
|
+
candidateId: string;
|
|
75
|
+
provider: string;
|
|
76
|
+
modelId: string;
|
|
77
|
+
runs: number;
|
|
78
|
+
successfulRuns: number;
|
|
79
|
+
failedRuns: number;
|
|
80
|
+
errorRate: number;
|
|
81
|
+
meanScore?: number;
|
|
82
|
+
minScore?: number;
|
|
83
|
+
maxScore?: number;
|
|
84
|
+
p50LatencyMs: number;
|
|
85
|
+
p95LatencyMs: number;
|
|
86
|
+
totalCost?: number;
|
|
87
|
+
meanCost?: number;
|
|
88
|
+
currency?: string;
|
|
89
|
+
usage?: TokenUsage;
|
|
90
|
+
metadata?: Record<string, JsonValue>;
|
|
91
|
+
}
|
|
92
|
+
export interface ModelEvaluationThresholds {
|
|
93
|
+
minMeanScore?: number;
|
|
94
|
+
maxTotalCost?: number;
|
|
95
|
+
maxP95LatencyMs?: number;
|
|
96
|
+
maxErrorRate?: number;
|
|
97
|
+
}
|
|
98
|
+
export type ModelEvaluationGateCheckCode = "min_mean_score" | "max_total_cost" | "max_p95_latency_ms" | "max_error_rate";
|
|
99
|
+
export interface ModelEvaluationGateCheck {
|
|
100
|
+
candidateId: string;
|
|
101
|
+
code: ModelEvaluationGateCheckCode;
|
|
102
|
+
ok: boolean;
|
|
103
|
+
actual?: number;
|
|
104
|
+
threshold: number;
|
|
105
|
+
message: string;
|
|
106
|
+
}
|
|
107
|
+
export interface ModelEvaluationGateResult {
|
|
108
|
+
ok: boolean;
|
|
109
|
+
thresholds: ModelEvaluationThresholds;
|
|
110
|
+
checks: ModelEvaluationGateCheck[];
|
|
111
|
+
}
|
|
112
|
+
export interface ModelEvaluationReport {
|
|
113
|
+
schemaVersion: typeof MODEL_EVALUATION_REPORT_SCHEMA_VERSION;
|
|
114
|
+
createdAt: string;
|
|
115
|
+
ok: boolean;
|
|
116
|
+
repetitions: number;
|
|
117
|
+
totalRuns: number;
|
|
118
|
+
cases: string[];
|
|
119
|
+
candidates: ModelEvaluationCandidateSummary[];
|
|
120
|
+
runs: ModelEvaluationRunResult[];
|
|
121
|
+
gate?: ModelEvaluationGateResult;
|
|
122
|
+
metadata?: Record<string, JsonValue>;
|
|
123
|
+
}
|
|
124
|
+
export interface ModelEvaluationSuite {
|
|
125
|
+
cases: ModelEvaluationCase[];
|
|
126
|
+
candidates: ModelEvaluationCandidate[];
|
|
127
|
+
scorers: ModelEvaluationScorer[];
|
|
128
|
+
repetitions?: number;
|
|
129
|
+
maxConcurrency?: number;
|
|
130
|
+
catalog?: ModelCatalog;
|
|
131
|
+
thresholds?: ModelEvaluationThresholds;
|
|
132
|
+
metadata?: Record<string, JsonValue>;
|
|
133
|
+
}
|
|
134
|
+
export interface ModelEvaluationCandidateComparison {
|
|
135
|
+
candidateId: string;
|
|
136
|
+
baseline?: ModelEvaluationCandidateSummary;
|
|
137
|
+
target?: ModelEvaluationCandidateSummary;
|
|
138
|
+
scoreDelta?: number;
|
|
139
|
+
totalCostDelta?: number;
|
|
140
|
+
p95LatencyDeltaMs?: number;
|
|
141
|
+
errorRateDelta?: number;
|
|
142
|
+
}
|
|
143
|
+
export interface ModelEvaluationComparison {
|
|
144
|
+
baselineCreatedAt: string;
|
|
145
|
+
targetCreatedAt: string;
|
|
146
|
+
candidates: ModelEvaluationCandidateComparison[];
|
|
147
|
+
}
|
|
148
|
+
export declare const evaluateModelEvaluationGate: (candidates: ModelEvaluationCandidateSummary[], thresholds: ModelEvaluationThresholds) => ModelEvaluationGateResult;
|
|
149
|
+
export declare const runModelEvaluation: (suite: ModelEvaluationSuite) => Promise<ModelEvaluationReport>;
|
|
150
|
+
export declare const compareModelEvaluationReports: (baseline: ModelEvaluationReport, target: ModelEvaluationReport) => ModelEvaluationComparison;
|
|
151
|
+
export declare const createExactMatchScorer: (options?: {
|
|
152
|
+
id?: string;
|
|
153
|
+
caseSensitive?: boolean;
|
|
154
|
+
trim?: boolean;
|
|
155
|
+
}) => ModelEvaluationScorer;
|
|
156
|
+
export declare const createContainsScorer: (options?: {
|
|
157
|
+
id?: string;
|
|
158
|
+
caseSensitive?: boolean;
|
|
159
|
+
}) => ModelEvaluationScorer;
|
|
160
|
+
export declare const createRegexScorer: (options: {
|
|
161
|
+
pattern: RegExp | ((testCase: ModelEvaluationCase) => RegExp);
|
|
162
|
+
id?: string;
|
|
163
|
+
}) => ModelEvaluationScorer;
|
|
164
|
+
export declare const createJsonValueScorer: (options?: {
|
|
165
|
+
id?: string;
|
|
166
|
+
}) => ModelEvaluationScorer;
|
|
167
|
+
export declare const createJsonSchemaScorer: (schema: ZodTypeAny, options?: {
|
|
168
|
+
id?: string;
|
|
169
|
+
}) => ModelEvaluationScorer;
|
|
170
|
+
export declare const createToolCallScorer: (options?: {
|
|
171
|
+
id?: string;
|
|
172
|
+
requireExactOrder?: boolean;
|
|
173
|
+
}) => ModelEvaluationScorer;
|
|
174
|
+
export declare const createEmbeddingSimilarityScorer: (options: {
|
|
175
|
+
model: EmbeddingModel;
|
|
176
|
+
id?: string;
|
|
177
|
+
minimumSimilarity?: number;
|
|
178
|
+
}) => ModelEvaluationScorer;
|
|
179
|
+
export declare const createModelJudgeScorer: (options: {
|
|
180
|
+
model: LanguageModel;
|
|
181
|
+
rubric: string;
|
|
182
|
+
id?: string;
|
|
183
|
+
maxTokens?: number;
|
|
184
|
+
}) => ModelEvaluationScorer;
|
|
185
|
+
//# sourceMappingURL=model-evaluation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-evaluation.d.ts","sourceRoot":"","sources":["../src/model-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,KAAK,CAAC;AAEtC,OAAO,EAAqB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAC3F,OAAO,KAAK,EAAE,YAAY,EAAqB,MAAM,cAAc,CAAC;AAKpE,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACT,aAAa,EACb,YAAY,EACZ,UAAU,EACV,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,sCAAsC,EAAG,CAAU,CAAC;AAEjE,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,EAAE,iCAAiC,CAAC;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,MAAM,iCAAiC,GAAG,IAAI,CAClD,mBAAmB,CAAC,aAAa,CAAC,EAClC,OAAO,GAAG,QAAQ,GAAG,UAAU,CAChC,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,aAAa,CAAC;IACrB,QAAQ,CAAC,EAAE,iCAAiC,CAAC;IAC7C,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,GAAG,UAAU,CAAC,GAAG;QAC7D,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CACH,OAAO,EAAE,4BAA4B,GACpC,MAAM,GAAG,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,2BAA2B,CAAC,CAAC;CACzF;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,oBAAoB,EAAE,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACzC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,+BAA+B;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,yBAAyB;IACxC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,4BAA4B,GACpC,gBAAgB,GAChB,gBAAgB,GAChB,oBAAoB,GACpB,gBAAgB,CAAC;AAErB,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,4BAA4B,CAAC;IACnC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,yBAAyB;IACxC,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,EAAE,yBAAyB,CAAC;IACtC,MAAM,EAAE,wBAAwB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,OAAO,sCAAsC,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,UAAU,EAAE,+BAA+B,EAAE,CAAC;IAC9C,IAAI,EAAE,wBAAwB,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,yBAAyB,CAAC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,UAAU,EAAE,wBAAwB,EAAE,CAAC;IACvC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,UAAU,CAAC,EAAE,yBAAyB,CAAC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,kCAAkC;IACjD,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,+BAA+B,CAAC;IAC3C,MAAM,CAAC,EAAE,+BAA+B,CAAC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACxC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,kCAAkC,EAAE,CAAC;CAClD;AAkSD,eAAO,MAAM,2BAA2B,GACtC,YAAY,+BAA+B,EAAE,EAC7C,YAAY,yBAAyB,KACpC,yBA6DF,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAU,OAAO,oBAAoB,KAAG,OAAO,CAAC,qBAAqB,CAuCnG,CAAC;AAEF,eAAO,MAAM,6BAA6B,GACxC,UAAU,qBAAqB,EAC/B,QAAQ,qBAAqB,KAC5B,yBA8BF,CAAC;AAoBF,eAAO,MAAM,sBAAsB,GAAI,UAAS;IAC9C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,IAAI,CAAC,EAAE,OAAO,CAAC;CACX,KAAG,qBAaP,CAAC;AAEH,eAAO,MAAM,oBAAoB,GAAI,UAAS;IAC5C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,qBAUP,CAAC;AAEH,eAAO,MAAM,iBAAiB,GAAI,SAAS;IACzC,OAAO,EAAE,MAAM,GAAG,CAAC,CAAC,QAAQ,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC;IAC9D,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,KAAG,qBAOF,CAAC;AAEH,eAAO,MAAM,qBAAqB,GAAI,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAA;CAAO,KAAG,qBAgBpE,CAAC;AAEH,eAAO,MAAM,sBAAsB,GAAI,QAAQ,UAAU,EAAE,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAA;CAAO,KAAG,qBAczF,CAAC;AAEH,eAAO,MAAM,oBAAoB,GAAI,UAAS;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAAO,KAAG,qBA6BhG,CAAC;AAEH,eAAO,MAAM,+BAA+B,GAAI,SAAS;IACvD,KAAK,EAAE,cAAc,CAAC;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,KAAG,qBAyCH,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,SAAS;IAC9C,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,KAAG,qBAgCF,CAAC"}
|