@anvia/core 1.0.0-rc.6 → 1.0.0-rc.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -13
- package/dist/agent/index.d.ts +13 -13
- package/dist/agent/index.js +15 -15
- package/dist/agent/interactions/index.d.ts +1 -1
- package/dist/agent/interactions/index.js +2 -2
- package/dist/{agent-BK3jud9q.d.ts → agent-XPQBM4rX.d.ts} +10 -10
- package/dist/{chunk-UBE4WH6G.js → chunk-3RWESPUG.js} +122 -3
- package/dist/chunk-3RWESPUG.js.map +1 -0
- package/dist/{chunk-BAEFBTOW.js → chunk-3SKZ2BKP.js} +7 -5
- package/dist/chunk-3SKZ2BKP.js.map +1 -0
- package/dist/{chunk-SROQA7I7.js → chunk-3XQGVDU5.js} +2 -2
- package/dist/chunk-3XQGVDU5.js.map +1 -0
- package/dist/{chunk-QXQQ2DBM.js → chunk-5C6B6ZRH.js} +2 -2
- package/dist/{chunk-7O45T5F2.js → chunk-6BBOCFHV.js} +2 -2
- package/dist/{chunk-7YAH2F3J.js → chunk-76JLMBVT.js} +24 -12
- package/dist/chunk-76JLMBVT.js.map +1 -0
- package/dist/chunk-7JLAIN6E.js +15 -0
- package/dist/chunk-7JLAIN6E.js.map +1 -0
- package/dist/chunk-7PHMK7W4.js +1516 -0
- package/dist/chunk-7PHMK7W4.js.map +1 -0
- package/dist/{chunk-MGNTXGTX.js → chunk-D4PACGPA.js} +1 -1
- package/dist/{chunk-7OCVDT2X.js → chunk-EFLT7XZD.js} +9 -3
- package/dist/chunk-EFLT7XZD.js.map +1 -0
- package/dist/{chunk-RZDI7B6Z.js → chunk-EMZKPRE6.js} +9 -8
- package/dist/chunk-EMZKPRE6.js.map +1 -0
- package/dist/{chunk-HVHTVGPP.js → chunk-I3Q7XSBB.js} +9 -3
- package/dist/chunk-I3Q7XSBB.js.map +1 -0
- package/dist/{chunk-BOCJCECN.js → chunk-LTXABCOO.js} +4 -4
- package/dist/{chunk-BEU6FCTN.js → chunk-OJBFDBLG.js} +2 -2
- package/dist/{chunk-G5OACQEI.js → chunk-QGX73TSQ.js} +2 -2
- package/dist/{chunk-JXZTTNCR.js → chunk-SJMSS5SI.js} +3 -3
- package/dist/{chunk-JXZTTNCR.js.map → chunk-SJMSS5SI.js.map} +1 -1
- package/dist/{chunk-YXLDR2O4.js → chunk-T2C3CGDQ.js} +9 -3
- package/dist/chunk-T2C3CGDQ.js.map +1 -0
- package/dist/{chunk-VGJRXUPT.js → chunk-X4LK4ODN.js} +2 -2
- package/dist/chunk-X4LK4ODN.js.map +1 -0
- package/dist/{chunk-QXSVVYQK.js → chunk-YA2655FX.js} +169 -432
- package/dist/chunk-YA2655FX.js.map +1 -0
- package/dist/{chunk-H7BI3BSP.js → chunk-ZA564323.js} +2 -2
- package/dist/{client-BdQ_QZsY.d.ts → client-DTPl0hZv.d.ts} +2 -2
- package/dist/completion/index.d.ts +3 -3
- package/dist/completion/index.js +10 -5
- package/dist/{dynamic-tools--Aw37H9d.d.ts → dynamic-tools-aIZlg5x1.d.ts} +3 -3
- package/dist/embeddings/index.js +2 -2
- package/dist/evals/index.d.ts +6 -6
- package/dist/evals/index.js +51 -19
- package/dist/evals/index.js.map +1 -1
- package/dist/extractor/index.d.ts +1 -1
- package/dist/extractor/index.js +7 -6
- package/dist/guardrails/index.d.ts +4 -4
- package/dist/image-generation/index.d.ts +1 -1
- package/dist/image-generation/index.js +4 -2
- package/dist/index.d.ts +14 -14
- package/dist/index.js +23 -19
- package/dist/internal/agent.d.ts +11 -11
- package/dist/internal/agent.js +15 -15
- package/dist/mcp/index.d.ts +4 -4
- package/dist/mcp/index.js +10 -8
- package/dist/memory/index.d.ts +3 -3
- package/dist/memory/index.js +6 -5
- package/dist/{middleware-QZsbz7Hf.d.ts → middleware-kcF8AusP.d.ts} +8 -5
- package/dist/observability/index.d.ts +6 -6
- package/dist/observability/index.js +7 -6
- package/dist/pipeline/index.d.ts +11 -11
- package/dist/pipeline/index.js +8 -7
- package/dist/pipeline/index.js.map +1 -1
- package/dist/{message-schema-_sP_RKhh.d.ts → provider-output-error-BEwDgVv3.d.ts} +34 -3
- package/dist/skills/index.d.ts +4 -4
- package/dist/skills/index.js +7 -8
- package/dist/speech-generation/index.d.ts +1 -1
- package/dist/speech-generation/index.js +4 -2
- package/dist/{text-DTg1a8Mg.d.ts → text-ZLECpQE7.d.ts} +1 -1
- package/dist/{think-tool-C0n_dF2K.d.ts → think-tool-BwCsJ365.d.ts} +1 -1
- package/dist/tool/index.d.ts +6 -6
- package/dist/tool/index.js +6 -6
- package/dist/{tool-xHM_GmzT.d.ts → tool-BpqpoRSE.d.ts} +1 -1
- package/dist/transcription/index.d.ts +1 -1
- package/dist/transcription/index.js +4 -2
- package/dist/{types-D0Y2ZWS3.d.ts → types-CXNE592e.d.ts} +1 -1
- package/dist/{types-Dyp7arZQ.d.ts → types-DC1U1XwW.d.ts} +1 -1
- package/dist/{types-B1KL4w14.d.ts → types-DKOXVTcq.d.ts} +1 -1
- package/dist/{types-DboZvHR_.d.ts → types-DgvozfPc.d.ts} +3 -2
- package/dist/{types-D7A6CIGt.d.ts → types-DhkodEft.d.ts} +1 -1
- package/dist/{types-2DBdXArn.d.ts → types-Sue2hb_L.d.ts} +5 -5
- package/dist/vector-store/index.d.ts +4 -4
- package/dist/vector-store/index.js +5 -5
- package/package.json +1 -1
- package/dist/chunk-7OCVDT2X.js.map +0 -1
- package/dist/chunk-7YAH2F3J.js.map +0 -1
- package/dist/chunk-BAEFBTOW.js.map +0 -1
- package/dist/chunk-CQNNSZPG.js +0 -9
- package/dist/chunk-CQNNSZPG.js.map +0 -1
- package/dist/chunk-HT4P3SSG.js +0 -573
- package/dist/chunk-HT4P3SSG.js.map +0 -1
- package/dist/chunk-HVHTVGPP.js.map +0 -1
- package/dist/chunk-QXSVVYQK.js.map +0 -1
- package/dist/chunk-RZDI7B6Z.js.map +0 -1
- package/dist/chunk-SROQA7I7.js.map +0 -1
- package/dist/chunk-UBE4WH6G.js.map +0 -1
- package/dist/chunk-VGJRXUPT.js.map +0 -1
- package/dist/chunk-YXLDR2O4.js.map +0 -1
- /package/dist/{chunk-QXQQ2DBM.js.map → chunk-5C6B6ZRH.js.map} +0 -0
- /package/dist/{chunk-7O45T5F2.js.map → chunk-6BBOCFHV.js.map} +0 -0
- /package/dist/{chunk-MGNTXGTX.js.map → chunk-D4PACGPA.js.map} +0 -0
- /package/dist/{chunk-BOCJCECN.js.map → chunk-LTXABCOO.js.map} +0 -0
- /package/dist/{chunk-BEU6FCTN.js.map → chunk-OJBFDBLG.js.map} +0 -0
- /package/dist/{chunk-G5OACQEI.js.map → chunk-QGX73TSQ.js.map} +0 -0
- /package/dist/{chunk-H7BI3BSP.js.map → chunk-ZA564323.js.map} +0 -0
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
resolveRetryOptions,
|
|
6
6
|
runWithRetries,
|
|
7
7
|
throwIfAborted
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-3RWESPUG.js";
|
|
9
9
|
|
|
10
10
|
// src/embeddings/distance.ts
|
|
11
11
|
function dotProduct(left, right) {
|
|
@@ -328,4 +328,4 @@ export {
|
|
|
328
328
|
embedSparseQuery,
|
|
329
329
|
embedDocuments
|
|
330
330
|
};
|
|
331
|
-
//# sourceMappingURL=chunk-
|
|
331
|
+
//# sourceMappingURL=chunk-ZA564323.js.map
|
|
@@ -3,8 +3,8 @@ import { Stream } from 'node:stream';
|
|
|
3
3
|
import { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js';
|
|
4
4
|
import { StreamableHTTPReconnectionOptions } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
|
|
5
5
|
import { Transport } from '@modelcontextprotocol/sdk/shared/transport.js';
|
|
6
|
-
import { J as JsonObject } from './types-
|
|
7
|
-
import { A as AnyTool } from './tool-
|
|
6
|
+
import { J as JsonObject } from './types-DgvozfPc.js';
|
|
7
|
+
import { A as AnyTool } from './tool-BpqpoRSE.js';
|
|
8
8
|
|
|
9
9
|
type McpStdioTransport = {
|
|
10
10
|
readonly type: "stdio";
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export { M as ModelCallOptions } from '../model-call-options-CZkSw_xN.js';
|
|
2
2
|
export { R as RetryContext, a as RetryOptions, b as RetrySetting } from '../retry-CjvSlKGW.js';
|
|
3
|
-
import { D as Document, M as Message } from '../types-
|
|
4
|
-
export { A as AssistantContentPart, a as AssistantGenerationMetadata, b as AssistantMessage, Q as CompletionCapabilityError, C as CompletionFinishReason, c as CompletionModel, V as CompletionModelCapabilities, d as CompletionModelInfo, e as CompletionModelStreamEvent, f as CompletionRequest, g as CompletionResponse, h as CompletionResult, i as CompletionSource, j as CompletionStreamEvent, W as CompletionStreamPart, k as CompletionTool, l as ContextUsage, F as FileData, m as FilePart, I as ImageDetail, n as ImagePart, J as JsonObject, o as JsonPrimitive, p as JsonValue, q as ModelContextLimits, P as ProviderTool, r as ProviderToolCall, R as ReasoningContentType, s as ReasoningDetail, t as ReasoningPart, X as StreamingCompletionModel, S as SystemMessage, T as TextPart, Y as ToolApprovalResponsePart, Z as ToolCallArgumentsMode, u as ToolCallPart, v as ToolChoice, w as ToolDefinition, _ as ToolInteractionResponsePart, x as ToolMessage, $ as ToolQuestionAnswer, a0 as ToolQuestionResponsePart, y as ToolResultContentPart, z as ToolResultOutput, B as ToolResultPart, U as Usage, a1 as UsageDetails, E as UserContentPart, G as UserMessage, a2 as assertCompletionRequestSupported, H as calculateContextUsage, K as getAssistantGenerationMetadata, L as isProviderTool, a3 as reasoningDisplayText, N as resolveModelContextLimits, a4 as textFromAssistantContent, O as withContextUsage } from '../types-
|
|
5
|
-
export { C as
|
|
3
|
+
import { D as Document, M as Message } from '../types-DgvozfPc.js';
|
|
4
|
+
export { A as AssistantContentPart, a as AssistantGenerationMetadata, b as AssistantMessage, Q as CompletionCapabilityError, C as CompletionFinishReason, c as CompletionModel, V as CompletionModelCapabilities, d as CompletionModelInfo, e as CompletionModelStreamEvent, f as CompletionRequest, g as CompletionResponse, h as CompletionResult, i as CompletionSource, j as CompletionStreamEvent, W as CompletionStreamPart, k as CompletionTool, l as ContextUsage, F as FileData, m as FilePart, I as ImageDetail, n as ImagePart, J as JsonObject, o as JsonPrimitive, p as JsonValue, q as ModelContextLimits, P as ProviderTool, r as ProviderToolCall, R as ReasoningContentType, s as ReasoningDetail, t as ReasoningPart, X as StreamingCompletionModel, S as SystemMessage, T as TextPart, Y as ToolApprovalResponsePart, Z as ToolCallArgumentsMode, u as ToolCallPart, v as ToolChoice, w as ToolDefinition, _ as ToolInteractionResponsePart, x as ToolMessage, $ as ToolQuestionAnswer, a0 as ToolQuestionResponsePart, y as ToolResultContentPart, z as ToolResultOutput, B as ToolResultPart, U as Usage, a1 as UsageDetails, E as UserContentPart, G as UserMessage, a2 as assertCompletionRequestSupported, H as calculateContextUsage, K as getAssistantGenerationMetadata, L as isProviderTool, a3 as reasoningDisplayText, N as resolveModelContextLimits, a4 as textFromAssistantContent, O as withContextUsage } from '../types-DgvozfPc.js';
|
|
5
|
+
export { C as COMPLETION_PROVIDER_OUTPUT_ERROR_CODE, a as CompletionBaseOptions, b as CompletionInput, c as CompletionProviderOutputError, d as CompletionProviderOutputErrorKind, e as CompletionProviderOutputErrorOptions, f as CompletionStructuredOutputError, g as CompletionStructuredOutputPhase, G as GenerateCompletionOptions, h as GenerateStructuredCompletionOptions, M as MessageMetadataSchema, S as StreamCompletionOptions, i as StreamStructuredCompletionOptions, j as createMessageSchema, k as generateCompletion, l as isJsonValue, m as isMessage, r as isStreamingCompletionModel, n as messageSchema, o as messagesSchema, p as parseMessage, q as parseMessages, s as streamCompletion } from '../provider-output-error-BEwDgVv3.js';
|
|
6
6
|
import '../zod-schema-C7F4clpm.js';
|
|
7
7
|
import 'zod';
|
|
8
8
|
|
package/dist/completion/index.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
formatDocument,
|
|
3
3
|
normalizeDocuments
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-D4PACGPA.js";
|
|
5
5
|
import {
|
|
6
|
+
COMPLETION_PROVIDER_OUTPUT_ERROR_CODE,
|
|
6
7
|
CompletionCapabilityError,
|
|
8
|
+
CompletionProviderOutputError,
|
|
7
9
|
CompletionStructuredOutputError,
|
|
8
10
|
Usage,
|
|
9
11
|
assertCompletionRequestSupported,
|
|
@@ -17,7 +19,8 @@ import {
|
|
|
17
19
|
streamCompletion,
|
|
18
20
|
textFromAssistantContent,
|
|
19
21
|
withContextUsage
|
|
20
|
-
} from "../chunk-
|
|
22
|
+
} from "../chunk-7PHMK7W4.js";
|
|
23
|
+
import "../chunk-7JLAIN6E.js";
|
|
21
24
|
import {
|
|
22
25
|
createMessageSchema,
|
|
23
26
|
isMessage,
|
|
@@ -25,13 +28,15 @@ import {
|
|
|
25
28
|
messagesSchema,
|
|
26
29
|
parseMessage,
|
|
27
30
|
parseMessages
|
|
28
|
-
} from "../chunk-
|
|
31
|
+
} from "../chunk-QGX73TSQ.js";
|
|
29
32
|
import {
|
|
30
33
|
isJsonValue
|
|
31
|
-
} from "../chunk-
|
|
32
|
-
import "../chunk-
|
|
34
|
+
} from "../chunk-3XQGVDU5.js";
|
|
35
|
+
import "../chunk-3RWESPUG.js";
|
|
33
36
|
export {
|
|
37
|
+
COMPLETION_PROVIDER_OUTPUT_ERROR_CODE,
|
|
34
38
|
CompletionCapabilityError,
|
|
39
|
+
CompletionProviderOutputError,
|
|
35
40
|
CompletionStructuredOutputError,
|
|
36
41
|
Usage,
|
|
37
42
|
assertCompletionRequestSupported,
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { b as RetrySetting } from './retry-CjvSlKGW.js';
|
|
2
|
-
import { w as ToolDefinition } from './types-
|
|
2
|
+
import { w as ToolDefinition } from './types-DgvozfPc.js';
|
|
3
3
|
import { V as VectorMetadata, E as EmbeddingModel, a as EmbeddedDocument } from './types-Cr4uiYo5.js';
|
|
4
|
-
import { A as AnyTool } from './tool-
|
|
5
|
-
import { V as VectorFilter, a as VectorSearchResult, b as VectorInspectRequest, c as VectorInspectPage } from './types-
|
|
4
|
+
import { A as AnyTool } from './tool-BpqpoRSE.js';
|
|
5
|
+
import { V as VectorFilter, a as VectorSearchResult, b as VectorInspectRequest, c as VectorInspectPage } from './types-CXNE592e.js';
|
|
6
6
|
|
|
7
7
|
type ToolSearchDocument<Metadata extends VectorMetadata = VectorMetadata> = {
|
|
8
8
|
toolName: string;
|
package/dist/embeddings/index.js
CHANGED
|
@@ -10,9 +10,9 @@ import {
|
|
|
10
10
|
embedTexts,
|
|
11
11
|
euclideanDistance,
|
|
12
12
|
manhattanDistance
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-ZA564323.js";
|
|
14
14
|
import "../chunk-3RM57ZT2.js";
|
|
15
|
-
import "../chunk-
|
|
15
|
+
import "../chunk-3RWESPUG.js";
|
|
16
16
|
export {
|
|
17
17
|
angularDistance,
|
|
18
18
|
chebyshevDistance,
|
package/dist/evals/index.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { U as Usage, J as JsonObject, c as CompletionModel, M as Message } from '../types-
|
|
1
|
+
import { U as Usage, J as JsonObject, c as CompletionModel, M as Message } from '../types-DgvozfPc.js';
|
|
2
2
|
import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
|
|
3
3
|
import { AgentInteractionRequest, AgentInteractionResponse } from '../agent/interactions/index.js';
|
|
4
|
-
import { t as AgentSuspendedResult, j as AgentResponse, l as AgentRunOptions, k as AgentResult } from '../types-
|
|
4
|
+
import { t as AgentSuspendedResult, j as AgentResponse, l as AgentRunOptions, k as AgentResult } from '../types-Sue2hb_L.js';
|
|
5
5
|
import { E as EmbeddingModel } from '../types-Cr4uiYo5.js';
|
|
6
6
|
import '../model-call-options-CZkSw_xN.js';
|
|
7
7
|
import 'zod';
|
|
8
8
|
import '../retry-CjvSlKGW.js';
|
|
9
|
-
import '../types-
|
|
9
|
+
import '../types-DhkodEft.js';
|
|
10
10
|
import '../type-utils-CtHVDRn_.js';
|
|
11
|
-
import '../types-
|
|
12
|
-
import '../tool-
|
|
13
|
-
import '../middleware-
|
|
11
|
+
import '../types-DKOXVTcq.js';
|
|
12
|
+
import '../tool-BpqpoRSE.js';
|
|
13
|
+
import '../middleware-kcF8AusP.js';
|
|
14
14
|
|
|
15
15
|
type EvalOutcome<Score = unknown> = {
|
|
16
16
|
outcome: "pass";
|
package/dist/evals/index.js
CHANGED
|
@@ -1,23 +1,26 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AgentRunBlockedError
|
|
3
|
-
} from "../chunk-
|
|
3
|
+
} from "../chunk-X4LK4ODN.js";
|
|
4
4
|
import {
|
|
5
5
|
cosineSimilarity,
|
|
6
6
|
embedText
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-ZA564323.js";
|
|
8
8
|
import {
|
|
9
9
|
mapWithConcurrency
|
|
10
10
|
} from "../chunk-3RM57ZT2.js";
|
|
11
11
|
import {
|
|
12
12
|
extract
|
|
13
|
-
} from "../chunk-
|
|
14
|
-
import "../chunk-
|
|
13
|
+
} from "../chunk-LTXABCOO.js";
|
|
14
|
+
import "../chunk-D4PACGPA.js";
|
|
15
15
|
import {
|
|
16
16
|
Usage
|
|
17
|
-
} from "../chunk-
|
|
18
|
-
import "../chunk-
|
|
19
|
-
import "../chunk-
|
|
20
|
-
import
|
|
17
|
+
} from "../chunk-7PHMK7W4.js";
|
|
18
|
+
import "../chunk-7JLAIN6E.js";
|
|
19
|
+
import "../chunk-QGX73TSQ.js";
|
|
20
|
+
import {
|
|
21
|
+
isJsonValue
|
|
22
|
+
} from "../chunk-3XQGVDU5.js";
|
|
23
|
+
import "../chunk-3RWESPUG.js";
|
|
21
24
|
|
|
22
25
|
// src/evals/advanced-metrics.ts
|
|
23
26
|
import { z } from "zod";
|
|
@@ -238,13 +241,14 @@ ${actual}`,
|
|
|
238
241
|
usage = addUsage(usage, verdictResult.usage);
|
|
239
242
|
}
|
|
240
243
|
const score = verdicts.length === 0 ? 1 : verdicts.filter((verdict) => verdict.verdict !== "no").length / verdicts.length;
|
|
244
|
+
const serializedVerdicts = serializeVerdicts(verdicts);
|
|
241
245
|
const reasonResult = await maybeReason({
|
|
242
246
|
model: options.model,
|
|
243
247
|
includeReason: config.includeReason,
|
|
244
248
|
retries: config.retries,
|
|
245
249
|
metric: "answer relevancy",
|
|
246
250
|
score,
|
|
247
|
-
evidence: { input, verdicts }
|
|
251
|
+
evidence: { input, verdicts: serializedVerdicts }
|
|
248
252
|
});
|
|
249
253
|
usage = addUsage(usage, reasonResult.usage);
|
|
250
254
|
return higherOutcome({
|
|
@@ -252,7 +256,7 @@ ${actual}`,
|
|
|
252
256
|
threshold: config.threshold,
|
|
253
257
|
strictMode: config.strictMode,
|
|
254
258
|
comment: reasonResult.reason,
|
|
255
|
-
details: { statements, verdicts },
|
|
259
|
+
details: { statements, verdicts: serializedVerdicts },
|
|
256
260
|
usage
|
|
257
261
|
});
|
|
258
262
|
} catch (error) {
|
|
@@ -320,6 +324,9 @@ function jsonCorrectness(options) {
|
|
|
320
324
|
let validationError;
|
|
321
325
|
try {
|
|
322
326
|
parsed = JSON.parse(actual);
|
|
327
|
+
if (!isJsonValue(parsed)) {
|
|
328
|
+
throw new TypeError("Generated output is not a JSON value.");
|
|
329
|
+
}
|
|
323
330
|
const result = options.schema.safeParse(parsed);
|
|
324
331
|
if (!result.success) {
|
|
325
332
|
validationError = z.prettifyError(result.error);
|
|
@@ -452,13 +459,14 @@ ${actual}`,
|
|
|
452
459
|
(verdict) => verdict.verdict === "yes" || verdict.verdict === "idk" && !penalizeAmbiguousClaims
|
|
453
460
|
).length;
|
|
454
461
|
const score = verdicts.length === 0 ? 1 : supported / verdicts.length;
|
|
462
|
+
const serializedVerdicts = serializeVerdicts(verdicts);
|
|
455
463
|
const reasonResult = await maybeReason({
|
|
456
464
|
model: options.model,
|
|
457
465
|
includeReason: config.includeReason,
|
|
458
466
|
retries: config.retries,
|
|
459
467
|
metric: "faithfulness",
|
|
460
468
|
score,
|
|
461
|
-
evidence: { verdicts, penalizeAmbiguousClaims }
|
|
469
|
+
evidence: { verdicts: serializedVerdicts, penalizeAmbiguousClaims }
|
|
462
470
|
});
|
|
463
471
|
usage = addUsage(usage, reasonResult.usage);
|
|
464
472
|
return higherOutcome({
|
|
@@ -466,7 +474,7 @@ ${actual}`,
|
|
|
466
474
|
threshold: config.threshold,
|
|
467
475
|
strictMode: config.strictMode,
|
|
468
476
|
comment: reasonResult.reason,
|
|
469
|
-
details: { truths, claims, verdicts, penalizeAmbiguousClaims },
|
|
477
|
+
details: { truths, claims, verdicts: serializedVerdicts, penalizeAmbiguousClaims },
|
|
470
478
|
usage
|
|
471
479
|
});
|
|
472
480
|
} catch (error) {
|
|
@@ -630,13 +638,19 @@ ${actual}`,
|
|
|
630
638
|
});
|
|
631
639
|
const coverageScore = coverageTotal === 0 ? 0 : coverageMatched / coverageTotal;
|
|
632
640
|
const score = Math.min(alignmentScore, coverageScore);
|
|
641
|
+
const serializedAlignmentVerdicts = serializeVerdicts(alignmentVerdicts);
|
|
633
642
|
const reasonResult = await maybeReason({
|
|
634
643
|
model: options.model,
|
|
635
644
|
includeReason: config.includeReason,
|
|
636
645
|
retries: config.retries,
|
|
637
646
|
metric: "summarization",
|
|
638
647
|
score,
|
|
639
|
-
evidence: {
|
|
648
|
+
evidence: {
|
|
649
|
+
alignmentVerdicts: serializedAlignmentVerdicts,
|
|
650
|
+
coverageVerdicts,
|
|
651
|
+
alignmentScore,
|
|
652
|
+
coverageScore
|
|
653
|
+
}
|
|
640
654
|
});
|
|
641
655
|
const usage = addUsage(
|
|
642
656
|
truthResult.usage,
|
|
@@ -656,7 +670,7 @@ ${actual}`,
|
|
|
656
670
|
truths,
|
|
657
671
|
claims,
|
|
658
672
|
assessmentQuestions: questions,
|
|
659
|
-
alignmentVerdicts,
|
|
673
|
+
alignmentVerdicts: serializedAlignmentVerdicts,
|
|
660
674
|
coverageVerdicts,
|
|
661
675
|
scoreBreakdown: { alignment: alignmentScore, coverage: coverageScore }
|
|
662
676
|
},
|
|
@@ -1042,15 +1056,33 @@ function assertSameLength(label, inputs, outputs) {
|
|
|
1042
1056
|
throw new Error(`${label} count ${outputs.length} did not match input count ${inputs.length}.`);
|
|
1043
1057
|
}
|
|
1044
1058
|
}
|
|
1059
|
+
function serializeVerdicts(verdicts) {
|
|
1060
|
+
return verdicts.map((verdict) => {
|
|
1061
|
+
const serialized = { verdict: verdict.verdict };
|
|
1062
|
+
if (verdict.reason !== void 0) serialized.reason = verdict.reason;
|
|
1063
|
+
return serialized;
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1045
1066
|
function jsonPrompt(value) {
|
|
1046
1067
|
return JSON.stringify(value, null, 2);
|
|
1047
1068
|
}
|
|
1048
1069
|
function toJsonValue(value) {
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
|
|
1070
|
+
if (!isJsonValue(value)) {
|
|
1071
|
+
throw new TypeError("G-Eval expectedOutput must be a JSON value.");
|
|
1072
|
+
}
|
|
1073
|
+
return cloneJsonValue(value);
|
|
1074
|
+
}
|
|
1075
|
+
function cloneJsonValue(value) {
|
|
1076
|
+
if (value === null || typeof value !== "object") return value;
|
|
1077
|
+
if (Array.isArray(value)) return value.map(cloneJsonValue);
|
|
1078
|
+
return Object.fromEntries(
|
|
1079
|
+
Object.entries(value).map(([key, item]) => {
|
|
1080
|
+
if (item === void 0) {
|
|
1081
|
+
throw new TypeError("G-Eval expectedOutput must be a JSON value.");
|
|
1082
|
+
}
|
|
1083
|
+
return [key, cloneJsonValue(item)];
|
|
1084
|
+
})
|
|
1085
|
+
);
|
|
1054
1086
|
}
|
|
1055
1087
|
function normalizeEvalTurns(value) {
|
|
1056
1088
|
const source = conversationArray(value);
|