mesedi 0.1.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -31
- package/dist/_errorClassesGenerated.d.ts +18 -0
- package/dist/_errorClassesGenerated.d.ts.map +1 -0
- package/dist/_errorClassesGenerated.js +24 -0
- package/dist/_errorClassesGenerated.js.map +1 -0
- package/dist/anthropic_integration.d.ts +9 -9
- package/dist/anthropic_integration.d.ts.map +1 -1
- package/dist/anthropic_integration.js +67 -26
- package/dist/anthropic_integration.js.map +1 -1
- package/dist/client.d.ts +32 -12
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +46 -18
- package/dist/client.js.map +1 -1
- package/dist/cohere_integration.d.ts +42 -0
- package/dist/cohere_integration.d.ts.map +1 -0
- package/dist/cohere_integration.js +456 -0
- package/dist/cohere_integration.js.map +1 -0
- package/dist/compress.d.ts +69 -0
- package/dist/compress.d.ts.map +1 -0
- package/dist/compress.js +75 -0
- package/dist/compress.js.map +1 -0
- package/dist/context.d.ts +13 -4
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +15 -5
- package/dist/context.js.map +1 -1
- package/dist/errors.d.ts +137 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +462 -0
- package/dist/errors.js.map +1 -0
- package/dist/events.d.ts +19 -6
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js +31 -6
- package/dist/events.js.map +1 -1
- package/dist/gemini_integration.d.ts +33 -0
- package/dist/gemini_integration.d.ts.map +1 -0
- package/dist/gemini_integration.js +334 -0
- package/dist/gemini_integration.js.map +1 -0
- package/dist/halt.d.ts +12 -12
- package/dist/halt.js +16 -16
- package/dist/halt.js.map +1 -1
- package/dist/halt_stream.d.ts +6 -6
- package/dist/halt_stream.d.ts.map +1 -1
- package/dist/halt_stream.js +22 -12
- package/dist/halt_stream.js.map +1 -1
- package/dist/index.d.ts +26 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20 -12
- package/dist/index.js.map +1 -1
- package/dist/integrations/langgraph.d.ts +99 -0
- package/dist/integrations/langgraph.d.ts.map +1 -0
- package/dist/integrations/langgraph.js +572 -0
- package/dist/integrations/langgraph.js.map +1 -0
- package/dist/integrations/mastra.d.ts +114 -0
- package/dist/integrations/mastra.d.ts.map +1 -0
- package/dist/integrations/mastra.js +354 -0
- package/dist/integrations/mastra.js.map +1 -0
- package/dist/integrations/openai_agents.d.ts +66 -0
- package/dist/integrations/openai_agents.d.ts.map +1 -0
- package/dist/integrations/openai_agents.js +243 -0
- package/dist/integrations/openai_agents.js.map +1 -0
- package/dist/integrations/promptfoo.d.ts +62 -0
- package/dist/integrations/promptfoo.d.ts.map +1 -0
- package/dist/integrations/promptfoo.js +76 -0
- package/dist/integrations/promptfoo.js.map +1 -0
- package/dist/integrations/ragas.d.ts +65 -0
- package/dist/integrations/ragas.d.ts.map +1 -0
- package/dist/integrations/ragas.js +78 -0
- package/dist/integrations/ragas.js.map +1 -0
- package/dist/integrations/vectara.d.ts +60 -0
- package/dist/integrations/vectara.d.ts.map +1 -0
- package/dist/integrations/vectara.js +59 -0
- package/dist/integrations/vectara.js.map +1 -0
- package/dist/integrations/vercel_ai.d.ts +6 -6
- package/dist/integrations/vercel_ai.js +10 -10
- package/dist/integrations/vercel_ai.js.map +1 -1
- package/dist/observe.d.ts +358 -3
- package/dist/observe.d.ts.map +1 -1
- package/dist/observe.js +536 -4
- package/dist/observe.js.map +1 -1
- package/dist/ollama_integration.d.ts +126 -0
- package/dist/ollama_integration.d.ts.map +1 -0
- package/dist/ollama_integration.js +404 -0
- package/dist/ollama_integration.js.map +1 -0
- package/dist/openai_integration.d.ts +67 -0
- package/dist/openai_integration.d.ts.map +1 -0
- package/dist/openai_integration.js +574 -0
- package/dist/openai_integration.js.map +1 -0
- package/dist/tool.d.ts +1 -1
- package/dist/tool.d.ts.map +1 -1
- package/dist/tool.js +193 -10
- package/dist/tool.js.map +1 -1
- package/dist/truncate.d.ts +84 -0
- package/dist/truncate.d.ts.map +1 -0
- package/dist/truncate.js +134 -0
- package/dist/truncate.js.map +1 -0
- package/dist/vertex_gemini_integration.d.ts +27 -0
- package/dist/vertex_gemini_integration.d.ts.map +1 -0
- package/dist/vertex_gemini_integration.js +229 -0
- package/dist/vertex_gemini_integration.js.map +1 -0
- package/dist/wrap.d.ts +39 -3
- package/dist/wrap.d.ts.map +1 -1
- package/dist/wrap.js +15 -12
- package/dist/wrap.js.map +1 -1
- package/package.json +28 -4
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Agents SDK integration for Mesedi (Mesedi #28).
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
*
|
|
6
|
+
* import { wrap } from "mesedi";
|
|
7
|
+
* import { MesediRunHooks } from "mesedi/integrations/openai_agents";
|
|
8
|
+
* import { Agent, Runner } from "@openai/agents";
|
|
9
|
+
*
|
|
10
|
+
* const triage = new Agent({ name: "triage", ...});
|
|
11
|
+
* const planner = new Agent({ name: "planner", ...});
|
|
12
|
+
*
|
|
13
|
+
* export const runUserRequest = wrap(async (question: string) => {
|
|
14
|
+
* const result = await Runner.run(
|
|
15
|
+
* triage,
|
|
16
|
+
* question,
|
|
17
|
+
* { hooks: new MesediRunHooks() },
|
|
18
|
+
* );
|
|
19
|
+
* return result.finalOutput;
|
|
20
|
+
* });
|
|
21
|
+
*
|
|
22
|
+
* What the hooks emit:
|
|
23
|
+
*
|
|
24
|
+
* - `checkpoint` events at every agent start AND end with the
|
|
25
|
+
* agent name and a truncated input/output repr. Feeds the
|
|
26
|
+
* semantic_loop detector (#6).
|
|
27
|
+
* - `agent_handoff` events on every handoff between agents.
|
|
28
|
+
* from_agent / to_agent come straight from the hook arguments,
|
|
29
|
+
* handoff_kind defaults to `"transfer"`. Feeds cascading_failure
|
|
30
|
+
* (#12) and coordination_deadlock (#13).
|
|
31
|
+
* - `tool_call` events for every tool invocation, mirroring
|
|
32
|
+
* `mesedi.tool()` wire format.
|
|
33
|
+
*
|
|
34
|
+
* Out of scope for v1:
|
|
35
|
+
*
|
|
36
|
+
* - `llm_call` events. The OAI Agents SDK does not expose
|
|
37
|
+
* per-LLM-call hooks in its public RunHooks surface as of the
|
|
38
|
+
* version this integration was written against. For
|
|
39
|
+
* Anthropic-backed runs use the existing
|
|
40
|
+
* `instrumentAnthropic()` patch; an OpenAI-backed equivalent
|
|
41
|
+
* is not currently provided.
|
|
42
|
+
* - Streaming hooks (`Runner.runStreamed`). Not currently
|
|
43
|
+
* instrumented.
|
|
44
|
+
*/
|
|
45
|
+
import { getClient } from "../client.js";
|
|
46
|
+
import { currentExecutionContext, newEventId } from "../context.js";
|
|
47
|
+
import { EventType, utcNowRfc3339 } from "../events.js";
|
|
48
|
+
// Truncation budget for checkpoint state repr and tool args /
|
|
49
|
+
// return values. Wide enough to be useful for semantic_loop
|
|
50
|
+
// hashing; narrow enough to keep individual event payloads
|
|
51
|
+
// bounded.
|
|
52
|
+
const MAX_STATE_REPR = 1000;
|
|
53
|
+
const MAX_TOOL_INPUT_REPR = 200;
|
|
54
|
+
const MAX_TOOL_OUTPUT_REPR = 500;
|
|
55
|
+
/**
|
|
56
|
+
* Mesedi-emitting implementation of the `@openai/agents` RunHooks
|
|
57
|
+
* interface. Pass an instance to `Runner.run(..., { hooks: ... })`.
|
|
58
|
+
*
|
|
59
|
+
* All emissions go through Mesedi's standard event submitter, which
|
|
60
|
+
* is non-blocking (events are batched on a background shipper). The
|
|
61
|
+
* OAI Agents SDK's async runner is therefore never blocked on
|
|
62
|
+
* Mesedi I/O.
|
|
63
|
+
*
|
|
64
|
+
* Outside a `wrap()` execution context, all emissions silently
|
|
65
|
+
* no-op (matching the rest of Mesedi's observe layer).
|
|
66
|
+
*/
|
|
67
|
+
export class MesediRunHooks {
|
|
68
|
+
// The OAI Agents SDK reads this to identify the hooks instance
|
|
69
|
+
// in trace output. Required field on some versions of the
|
|
70
|
+
// dispatcher; ignored by older ones.
|
|
71
|
+
name = "mesedi-openai-agents";
|
|
72
|
+
// tool key → start time (ms epoch). Used to pair onToolStart
|
|
73
|
+
// with onToolEnd so we can compute duration. We key on the
|
|
74
|
+
// tool name because the SDK does not guarantee stable
|
|
75
|
+
// object identity across the start/end pair in all versions.
|
|
76
|
+
toolStarts = new Map();
|
|
77
|
+
// ── Agent lifecycle ─────────────────────────────────────────────
|
|
78
|
+
async onAgentStart(_context, agent) {
|
|
79
|
+
const agentName = extractAgentName(agent);
|
|
80
|
+
emitCheckpoint(agentName, extractContextState(_context), "enter");
|
|
81
|
+
}
|
|
82
|
+
async onAgentEnd(_context, agent, output) {
|
|
83
|
+
const agentName = extractAgentName(agent);
|
|
84
|
+
emitCheckpoint(agentName, output, "exit");
|
|
85
|
+
}
|
|
86
|
+
// ── Handoffs ────────────────────────────────────────────────────
|
|
87
|
+
async onHandoff(context, fromAgent, toAgent) {
|
|
88
|
+
emitHandoff(extractAgentName(fromAgent), extractAgentName(toAgent), "transfer", summarize(extractContextState(context)));
|
|
89
|
+
}
|
|
90
|
+
// ── Tools ───────────────────────────────────────────────────────
|
|
91
|
+
async onToolStart(_context, _agent, tool) {
|
|
92
|
+
const key = extractToolName(tool);
|
|
93
|
+
this.toolStarts.set(key, Date.now());
|
|
94
|
+
}
|
|
95
|
+
async onToolEnd(_context, _agent, tool, result) {
|
|
96
|
+
const key = extractToolName(tool);
|
|
97
|
+
const started = this.toolStarts.get(key);
|
|
98
|
+
this.toolStarts.delete(key);
|
|
99
|
+
const durationMs = started != null ? Math.max(0, Date.now() - started) : 0;
|
|
100
|
+
emitToolCall(key, summarizeToolInput(tool), summarizeToolOutput(result), durationMs, "ok");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// ── Helpers ─────────────────────────────────────────────────────────
|
|
104
|
+
function extractAgentName(agent) {
|
|
105
|
+
if (agent == null)
|
|
106
|
+
return "unknown";
|
|
107
|
+
if (typeof agent === "object") {
|
|
108
|
+
const a = agent;
|
|
109
|
+
if (typeof a.name === "string" && a.name)
|
|
110
|
+
return a.name;
|
|
111
|
+
const ctor = a.constructor?.name;
|
|
112
|
+
if (typeof ctor === "string" && ctor)
|
|
113
|
+
return ctor;
|
|
114
|
+
}
|
|
115
|
+
return String(agent);
|
|
116
|
+
}
|
|
117
|
+
function extractToolName(tool) {
|
|
118
|
+
if (tool == null)
|
|
119
|
+
return "unknown";
|
|
120
|
+
if (typeof tool === "object") {
|
|
121
|
+
const t = tool;
|
|
122
|
+
if (typeof t.name === "string" && t.name)
|
|
123
|
+
return t.name;
|
|
124
|
+
if (typeof t.toolName === "string" && t.toolName)
|
|
125
|
+
return t.toolName;
|
|
126
|
+
const ctor = t.constructor?.name;
|
|
127
|
+
if (typeof ctor === "string" && ctor)
|
|
128
|
+
return ctor;
|
|
129
|
+
}
|
|
130
|
+
return String(tool);
|
|
131
|
+
}
|
|
132
|
+
function extractContextState(context) {
|
|
133
|
+
// OAI Agents wraps the user-supplied context in a wrapper; the
|
|
134
|
+
// real context is typically at `.context`. Fall back to the
|
|
135
|
+
// wrapper itself when the attribute is missing.
|
|
136
|
+
if (context && typeof context === "object") {
|
|
137
|
+
const c = context;
|
|
138
|
+
if ("context" in c && c.context != null)
|
|
139
|
+
return c.context;
|
|
140
|
+
}
|
|
141
|
+
return context;
|
|
142
|
+
}
|
|
143
|
+
function summarize(obj) {
|
|
144
|
+
if (obj == null)
|
|
145
|
+
return "";
|
|
146
|
+
let r;
|
|
147
|
+
try {
|
|
148
|
+
r = typeof obj === "string" ? obj : JSON.stringify(obj);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
r = String(obj);
|
|
152
|
+
}
|
|
153
|
+
if (r.length > MAX_STATE_REPR) {
|
|
154
|
+
r = r.slice(0, MAX_STATE_REPR - 3) + "...";
|
|
155
|
+
}
|
|
156
|
+
return r;
|
|
157
|
+
}
|
|
158
|
+
function summarizeToolInput(tool) {
|
|
159
|
+
if (tool && typeof tool === "object") {
|
|
160
|
+
const t = tool;
|
|
161
|
+
for (const attr of ["arguments", "input", "params"]) {
|
|
162
|
+
if (t[attr] != null)
|
|
163
|
+
return summarize(t[attr]);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return summarize(extractToolName(tool));
|
|
167
|
+
}
|
|
168
|
+
function summarizeToolOutput(result) {
|
|
169
|
+
if (result == null)
|
|
170
|
+
return "";
|
|
171
|
+
if (typeof result === "object") {
|
|
172
|
+
const r = result;
|
|
173
|
+
if ("output" in r && r.output != null)
|
|
174
|
+
return summarize(r.output);
|
|
175
|
+
}
|
|
176
|
+
return summarize(result);
|
|
177
|
+
}
|
|
178
|
+
function emitCheckpoint(agentName, state, agentEvent) {
|
|
179
|
+
const ctx = currentExecutionContext();
|
|
180
|
+
if (!ctx)
|
|
181
|
+
return;
|
|
182
|
+
const payload = {
|
|
183
|
+
name: `openai_agents.${agentEvent}.${agentName}`,
|
|
184
|
+
agent: agentName,
|
|
185
|
+
agent_event: agentEvent,
|
|
186
|
+
state_repr: summarize(state),
|
|
187
|
+
};
|
|
188
|
+
const client = getClient();
|
|
189
|
+
const event = {
|
|
190
|
+
event_id: newEventId(),
|
|
191
|
+
execution_id: ctx.executionId,
|
|
192
|
+
event_type: EventType.CHECKPOINT,
|
|
193
|
+
sequence: ctx.nextSequence(),
|
|
194
|
+
timestamp: utcNowRfc3339(),
|
|
195
|
+
payload,
|
|
196
|
+
};
|
|
197
|
+
client.submitEvent(event);
|
|
198
|
+
}
|
|
199
|
+
function emitHandoff(fromAgent, toAgent, handoffKind, taskSummary) {
|
|
200
|
+
const ctx = currentExecutionContext();
|
|
201
|
+
if (!ctx)
|
|
202
|
+
return;
|
|
203
|
+
const payload = {
|
|
204
|
+
from_agent: fromAgent,
|
|
205
|
+
to_agent: toAgent,
|
|
206
|
+
handoff_kind: handoffKind,
|
|
207
|
+
task_summary: taskSummary,
|
|
208
|
+
};
|
|
209
|
+
const client = getClient();
|
|
210
|
+
const event = {
|
|
211
|
+
event_id: newEventId(),
|
|
212
|
+
execution_id: ctx.executionId,
|
|
213
|
+
event_type: EventType.AGENT_HANDOFF,
|
|
214
|
+
sequence: ctx.nextSequence(),
|
|
215
|
+
timestamp: utcNowRfc3339(),
|
|
216
|
+
payload,
|
|
217
|
+
};
|
|
218
|
+
client.submitEvent(event);
|
|
219
|
+
}
|
|
220
|
+
function emitToolCall(toolName, argumentsRepr, returnValueRepr, durationMs, status) {
|
|
221
|
+
const ctx = currentExecutionContext();
|
|
222
|
+
if (!ctx)
|
|
223
|
+
return;
|
|
224
|
+
const payload = {
|
|
225
|
+
tool_name: toolName,
|
|
226
|
+
arguments: argumentsRepr.slice(0, MAX_TOOL_INPUT_REPR),
|
|
227
|
+
return_value: returnValueRepr.slice(0, MAX_TOOL_OUTPUT_REPR),
|
|
228
|
+
latency_ms: Math.trunc(durationMs),
|
|
229
|
+
status,
|
|
230
|
+
};
|
|
231
|
+
const client = getClient();
|
|
232
|
+
const event = {
|
|
233
|
+
event_id: newEventId(),
|
|
234
|
+
execution_id: ctx.executionId,
|
|
235
|
+
event_type: EventType.TOOL_CALL,
|
|
236
|
+
sequence: ctx.nextSequence(),
|
|
237
|
+
timestamp: utcNowRfc3339(),
|
|
238
|
+
duration_ms: Math.trunc(durationMs),
|
|
239
|
+
payload,
|
|
240
|
+
};
|
|
241
|
+
client.submitEvent(event);
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=openai_agents.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai_agents.js","sourceRoot":"","sources":["../../src/integrations/openai_agents.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAS,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE/D,8DAA8D;AAC9D,4DAA4D;AAC5D,2DAA2D;AAC3D,WAAW;AACX,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAc;IACzB,+DAA+D;IAC/D,0DAA0D;IAC1D,qCAAqC;IAC5B,IAAI,GAAG,sBAAsB,CAAC;IAEvC,6DAA6D;IAC7D,2DAA2D;IAC3D,sDAAsD;IACtD,6DAA6D;IACrD,UAAU,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEpD,mEAAmE;IAEnE,KAAK,CAAC,YAAY,CAAC,QAAiB,EAAE,KAAc;QAClD,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC1C,cAAc,CAAC,SAAS,EAAE,mBAAmB,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,UAAU,CACd,QAAiB,EACjB,KAAc,EACd,MAAe;QAEf,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC1C,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,mEAAmE;IAEnE,KAAK,CAAC,SAAS,CACb,OAAgB,EAChB,SAAkB,EAClB,OAAgB;QAEhB,WAAW,CACT,gBAAgB,CAAC,SAAS,CAAC,EAC3B,gBAAgB,CAAC,OAAO,CAAC,EACzB,UAAU,EACV,SAAS,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CACxC,CAAC;IACJ,CAAC;IAED,mEAAmE;IAEnE,KAAK,CAAC,WAAW,CACf,QAAiB,EACjB,MAAe,EACf,IAAa;QAEb,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,SAAS,CACb,QAAiB,EACjB,MAAe,EACf,IAAa,EACb,MAAe;QAEf,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,UAAU,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,YAAY,CACV,GAAG,EACH,kBAAkB,CAAC,IAAI,CAAC,EACxB,mBAAmB,CAAC,MAAM,CAAC,EAC3B,UAAU,EACV,IAAI,CACL,CAAC;IACJ,CAAC;CACF;AAED,uEAAuE;AAEvE,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,KAAgC,CAAC;QAC3C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,IAAI,GAAI,CAAC,CAAC,WAA8C,EAAE,IAAI,CAAC;QACrE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,IAAa;IACpC,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACnC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACxD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QACpE,MAAM,IAAI,GAAI,CAAC,CAAC,WAA8C,EAAE,IAAI,CAAC;QACrE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAgB;IAC3C,+DAA+D;IAC/D,4DAA4D;IAC5D,gDAAgD;IAChD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,OAAkC,CAAC;QAC7C,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI;YAAE,OAAO,CAAC,CAAC,OAAO,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,GAAY;IAC7B,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC3B,IAAI,CAAS,CAAC;IACd,IAAI,CAAC;QACH,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAU,EAAE,CAAC;YAC7D,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI;gBAAE,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAe;IAC1C,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,cAAc,CACrB,SAAiB,EACjB,KAAc,EACd,UAA4B;IAE5B,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IAEjB,MAAM,OAAO,GAA4B;QACvC,IAAI,EAAE,iBAAiB,UAAU,IAAI,SAAS,EAAE;QAChD,KAAK,EAAE,SAAS;QAChB,WAAW,EAAE,UAAU;QACvB,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC;KAC7B,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAClB,SAAiB,EACjB,OAAe,EACf,WAAmB,EACnB,WAAmB;IAEnB,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,OAAO,GAA4B;QACvC,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,OAAO;QACjB,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,WAAW;KAC1B,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,aAAa;QACnC,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,YAAY,CACnB,QAAgB,EAChB,aAAqB,EACrB,eAAuB,EACvB,UAAkB,EAClB,MAAuB;IAEvB,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,OAAO,GAA4B;QACvC,SAAS,EAAE,QAAQ;QACnB,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC;QACtD,YAAY,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QAC5D,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QAClC,MAAM;KACP,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,SAAS;QAC/B,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QACnC,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promptfoo evaluator one-liner helpers (Mesedi #14, Wave 1.3).
|
|
3
|
+
*
|
|
4
|
+
* Promptfoo (https://www.promptfoo.dev) is a CLI + library for
|
|
5
|
+
* prompt evaluation. Customers using its assertion types
|
|
6
|
+
* (model-graded-factuality, llm-rubric, ...) can use these helpers
|
|
7
|
+
* to report verdicts into Mesedi without hand-writing the
|
|
8
|
+
* boilerplate around {@link emitEvalScore}.
|
|
9
|
+
*
|
|
10
|
+
* Report-only design: Mesedi does NOT import the `promptfoo` package
|
|
11
|
+
* as a dependency. The customer runs Promptfoo on their side; these
|
|
12
|
+
* helpers accept the resulting score/verdict and emit a
|
|
13
|
+
* correctly-shaped `eval_score` event.
|
|
14
|
+
*
|
|
15
|
+
* Typical usage:
|
|
16
|
+
*
|
|
17
|
+
* import { wrap } from "mesedi";
|
|
18
|
+
* import { reportFactuality } from "mesedi/integrations/promptfoo";
|
|
19
|
+
*
|
|
20
|
+
* export const myAgent = wrap(async (question: string) => {
|
|
21
|
+
* const answer = await runAgent(question);
|
|
22
|
+
* const verdict = await promptfooAssertFactuality(answer, expectedFacts);
|
|
23
|
+
* reportFactuality(verdict.score, verdict.pass);
|
|
24
|
+
* return answer;
|
|
25
|
+
* });
|
|
26
|
+
*
|
|
27
|
+
* Helpers cover the two most common Promptfoo assertion shapes. For
|
|
28
|
+
* other assertion types (similar, contains, equals, ...) call
|
|
29
|
+
* {@link emitEvalScore} directly with
|
|
30
|
+
* `evaluatorId: "promptfoo/<assertion>"`.
|
|
31
|
+
*/
|
|
32
|
+
export interface PromptfooReportOptions {
|
|
33
|
+
/** Threshold the Promptfoo assertion used. Default 0.5. */
|
|
34
|
+
threshold?: number;
|
|
35
|
+
/** Optional explanation the judge returned. */
|
|
36
|
+
reason?: string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Report a Promptfoo `model-graded-factuality` assertion.
|
|
40
|
+
*
|
|
41
|
+
* Promptfoo's model-graded factuality assertion uses an LLM judge
|
|
42
|
+
* to score how factually accurate the answer is. Higher is better.
|
|
43
|
+
*/
|
|
44
|
+
export declare function reportFactuality(score: number, passed: boolean, opts?: PromptfooReportOptions): void;
|
|
45
|
+
/**
|
|
46
|
+
* Report a Promptfoo `llm-rubric` assertion.
|
|
47
|
+
*
|
|
48
|
+
* The llm-rubric assertion runs a customer-defined rubric through
|
|
49
|
+
* an LLM judge. Because a single execution can run multiple rubrics
|
|
50
|
+
* against the same output, `metricName` distinguishes them so
|
|
51
|
+
* grounding_failure clusters per-rubric instead of collapsing every
|
|
52
|
+
* rubric into one bucket.
|
|
53
|
+
*
|
|
54
|
+
* @param metricName Short identifier for the specific rubric
|
|
55
|
+
* ("clarity", "helpfulness", "tone_appropriate", etc.). Becomes
|
|
56
|
+
* part of the eval_score's metric_type field.
|
|
57
|
+
* @param score The judge's numeric score for this rubric.
|
|
58
|
+
* @param passed Promptfoo's own pass/fail verdict for this rubric.
|
|
59
|
+
* @param opts Threshold + reason.
|
|
60
|
+
*/
|
|
61
|
+
export declare function reportLlmRubric(metricName: string, score: number, passed: boolean, opts?: PromptfooReportOptions): void;
|
|
62
|
+
//# sourceMappingURL=promptfoo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo.d.ts","sourceRoot":"","sources":["../../src/integrations/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,sBAAsB;IACrC,2DAA2D;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,EACf,IAAI,GAAE,sBAA2B,GAChC,IAAI,CAaN;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,EACf,IAAI,GAAE,sBAA2B,GAChC,IAAI,CAoBN"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promptfoo evaluator one-liner helpers (Mesedi #14, Wave 1.3).
|
|
3
|
+
*
|
|
4
|
+
* Promptfoo (https://www.promptfoo.dev) is a CLI + library for
|
|
5
|
+
* prompt evaluation. Customers using its assertion types
|
|
6
|
+
* (model-graded-factuality, llm-rubric, ...) can use these helpers
|
|
7
|
+
* to report verdicts into Mesedi without hand-writing the
|
|
8
|
+
* boilerplate around {@link emitEvalScore}.
|
|
9
|
+
*
|
|
10
|
+
* Report-only design: Mesedi does NOT import the `promptfoo` package
|
|
11
|
+
* as a dependency. The customer runs Promptfoo on their side; these
|
|
12
|
+
* helpers accept the resulting score/verdict and emit a
|
|
13
|
+
* correctly-shaped `eval_score` event.
|
|
14
|
+
*
|
|
15
|
+
* Typical usage:
|
|
16
|
+
*
|
|
17
|
+
* import { wrap } from "mesedi";
|
|
18
|
+
* import { reportFactuality } from "mesedi/integrations/promptfoo";
|
|
19
|
+
*
|
|
20
|
+
* export const myAgent = wrap(async (question: string) => {
|
|
21
|
+
* const answer = await runAgent(question);
|
|
22
|
+
* const verdict = await promptfooAssertFactuality(answer, expectedFacts);
|
|
23
|
+
* reportFactuality(verdict.score, verdict.pass);
|
|
24
|
+
* return answer;
|
|
25
|
+
* });
|
|
26
|
+
*
|
|
27
|
+
* Helpers cover the two most common Promptfoo assertion shapes. For
|
|
28
|
+
* other assertion types (similar, contains, equals, ...) call
|
|
29
|
+
* {@link emitEvalScore} directly with
|
|
30
|
+
* `evaluatorId: "promptfoo/<assertion>"`.
|
|
31
|
+
*/
|
|
32
|
+
import { emitEvalScore } from "../observe.js";
|
|
33
|
+
/**
|
|
34
|
+
* Report a Promptfoo `model-graded-factuality` assertion.
|
|
35
|
+
*
|
|
36
|
+
* Promptfoo's model-graded factuality assertion uses an LLM judge
|
|
37
|
+
* to score how factually accurate the answer is. Higher is better.
|
|
38
|
+
*/
|
|
39
|
+
export function reportFactuality(score, passed, opts = {}) {
|
|
40
|
+
const threshold = opts.threshold ?? 0.5;
|
|
41
|
+
emitEvalScore("promptfoo/factuality", "factuality", score, passed, {
|
|
42
|
+
threshold,
|
|
43
|
+
higherIsBetter: true,
|
|
44
|
+
reason: opts.reason,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Report a Promptfoo `llm-rubric` assertion.
|
|
49
|
+
*
|
|
50
|
+
* The llm-rubric assertion runs a customer-defined rubric through
|
|
51
|
+
* an LLM judge. Because a single execution can run multiple rubrics
|
|
52
|
+
* against the same output, `metricName` distinguishes them so
|
|
53
|
+
* grounding_failure clusters per-rubric instead of collapsing every
|
|
54
|
+
* rubric into one bucket.
|
|
55
|
+
*
|
|
56
|
+
* @param metricName Short identifier for the specific rubric
|
|
57
|
+
* ("clarity", "helpfulness", "tone_appropriate", etc.). Becomes
|
|
58
|
+
* part of the eval_score's metric_type field.
|
|
59
|
+
* @param score The judge's numeric score for this rubric.
|
|
60
|
+
* @param passed Promptfoo's own pass/fail verdict for this rubric.
|
|
61
|
+
* @param opts Threshold + reason.
|
|
62
|
+
*/
|
|
63
|
+
export function reportLlmRubric(metricName, score, passed, opts = {}) {
|
|
64
|
+
if (!metricName) {
|
|
65
|
+
throw new Error("reportLlmRubric: metricName is required so the " +
|
|
66
|
+
"grounding_failure detector can cluster per-rubric " +
|
|
67
|
+
"instead of collapsing every rubric into one bucket.");
|
|
68
|
+
}
|
|
69
|
+
const threshold = opts.threshold ?? 0.5;
|
|
70
|
+
emitEvalScore(`promptfoo/llm-rubric/${metricName}`, metricName, score, passed, {
|
|
71
|
+
threshold,
|
|
72
|
+
higherIsBetter: true,
|
|
73
|
+
reason: opts.reason,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=promptfoo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../../src/integrations/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAS9C;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAa,EACb,MAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,sBAAsB,EACtB,YAAY,EACZ,KAAK,EACL,MAAM,EACN;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,eAAe,CAC7B,UAAkB,EAClB,KAAa,EACb,MAAe,EACf,OAA+B,EAAE;IAEjC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,iDAAiD;YAC/C,oDAAoD;YACpD,qDAAqD,CACxD,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,wBAAwB,UAAU,EAAE,EACpC,UAAU,EACV,KAAK,EACL,MAAM,EACN;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ragas grounding-evaluator one-liner helpers (Mesedi #14, Wave 1.3).
|
|
3
|
+
*
|
|
4
|
+
* Ragas (https://docs.ragas.io) is a popular RAG evaluation library.
|
|
5
|
+
* Customers running Ragas against their agent outputs can use these
|
|
6
|
+
* helpers to report scores into Mesedi without hand-writing the
|
|
7
|
+
* boilerplate around {@link emitEvalScore}.
|
|
8
|
+
*
|
|
9
|
+
* Report-only design: Mesedi does NOT import the `ragas` package as
|
|
10
|
+
* a dependency. The customer runs the evaluator on their side with
|
|
11
|
+
* their own version pinning and configuration; these helpers accept
|
|
12
|
+
* the resulting score and emit a correctly-shaped `eval_score` event
|
|
13
|
+
* so the backend's grounding_failure detector aggregates it under
|
|
14
|
+
* the right evaluator_id + metric_type cluster.
|
|
15
|
+
*
|
|
16
|
+
* Typical usage (Ragas in TypeScript is typically called via a
|
|
17
|
+
* Python sidecar; the helper signature is identical regardless of
|
|
18
|
+
* how the customer obtained the score):
|
|
19
|
+
*
|
|
20
|
+
* import { wrap } from "mesedi";
|
|
21
|
+
* import { reportFaithfulness } from "mesedi/integrations/ragas";
|
|
22
|
+
*
|
|
23
|
+
* export const myAgent = wrap(async (question: string) => {
|
|
24
|
+
* const { answer, contexts } = await runRag(question);
|
|
25
|
+
* const score = await scoreWithRagas(question, answer, contexts);
|
|
26
|
+
* reportFaithfulness(score, { threshold: 0.7 });
|
|
27
|
+
* return answer;
|
|
28
|
+
* });
|
|
29
|
+
*
|
|
30
|
+
* Helpers cover the three most common Ragas metrics. For niche
|
|
31
|
+
* metrics (answer_correctness, answer_similarity, context_recall, ...)
|
|
32
|
+
* call {@link emitEvalScore} directly with
|
|
33
|
+
* `evaluatorId: "ragas/<metric>"`.
|
|
34
|
+
*/
|
|
35
|
+
export interface RagasReportOptions {
|
|
36
|
+
/**
|
|
37
|
+
* Cutoff below which the call counts as a failure. Defaults to
|
|
38
|
+
* 0.7 (Ragas's documented working threshold).
|
|
39
|
+
*/
|
|
40
|
+
threshold?: number;
|
|
41
|
+
/** Optional explanation Ragas returned. */
|
|
42
|
+
reason?: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Report a Ragas `faithfulness` score.
|
|
46
|
+
*
|
|
47
|
+
* Faithfulness measures how well the agent's answer can be inferred
|
|
48
|
+
* from the retrieved context. Higher is better; range [0, 1].
|
|
49
|
+
*/
|
|
50
|
+
export declare function reportFaithfulness(score: number, opts?: RagasReportOptions): void;
|
|
51
|
+
/**
|
|
52
|
+
* Report a Ragas `answer_relevancy` score.
|
|
53
|
+
*
|
|
54
|
+
* Answer relevance measures whether the answer addresses the
|
|
55
|
+
* question that was asked. Higher is better; range [0, 1].
|
|
56
|
+
*/
|
|
57
|
+
export declare function reportAnswerRelevance(score: number, opts?: RagasReportOptions): void;
|
|
58
|
+
/**
|
|
59
|
+
* Report a Ragas `context_precision` score.
|
|
60
|
+
*
|
|
61
|
+
* Context precision measures whether the retrieved context contained
|
|
62
|
+
* only relevant chunks. Higher is better; range [0, 1].
|
|
63
|
+
*/
|
|
64
|
+
export declare function reportContextPrecision(score: number, opts?: RagasReportOptions): void;
|
|
65
|
+
//# sourceMappingURL=ragas.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ragas.d.ts","sourceRoot":"","sources":["../../src/integrations/ragas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAIH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ragas grounding-evaluator one-liner helpers (Mesedi #14, Wave 1.3).
|
|
3
|
+
*
|
|
4
|
+
* Ragas (https://docs.ragas.io) is a popular RAG evaluation library.
|
|
5
|
+
* Customers running Ragas against their agent outputs can use these
|
|
6
|
+
* helpers to report scores into Mesedi without hand-writing the
|
|
7
|
+
* boilerplate around {@link emitEvalScore}.
|
|
8
|
+
*
|
|
9
|
+
* Report-only design: Mesedi does NOT import the `ragas` package as
|
|
10
|
+
* a dependency. The customer runs the evaluator on their side with
|
|
11
|
+
* their own version pinning and configuration; these helpers accept
|
|
12
|
+
* the resulting score and emit a correctly-shaped `eval_score` event
|
|
13
|
+
* so the backend's grounding_failure detector aggregates it under
|
|
14
|
+
* the right evaluator_id + metric_type cluster.
|
|
15
|
+
*
|
|
16
|
+
* Typical usage (Ragas in TypeScript is typically called via a
|
|
17
|
+
* Python sidecar; the helper signature is identical regardless of
|
|
18
|
+
* how the customer obtained the score):
|
|
19
|
+
*
|
|
20
|
+
* import { wrap } from "mesedi";
|
|
21
|
+
* import { reportFaithfulness } from "mesedi/integrations/ragas";
|
|
22
|
+
*
|
|
23
|
+
* export const myAgent = wrap(async (question: string) => {
|
|
24
|
+
* const { answer, contexts } = await runRag(question);
|
|
25
|
+
* const score = await scoreWithRagas(question, answer, contexts);
|
|
26
|
+
* reportFaithfulness(score, { threshold: 0.7 });
|
|
27
|
+
* return answer;
|
|
28
|
+
* });
|
|
29
|
+
*
|
|
30
|
+
* Helpers cover the three most common Ragas metrics. For niche
|
|
31
|
+
* metrics (answer_correctness, answer_similarity, context_recall, ...)
|
|
32
|
+
* call {@link emitEvalScore} directly with
|
|
33
|
+
* `evaluatorId: "ragas/<metric>"`.
|
|
34
|
+
*/
|
|
35
|
+
import { emitEvalScore } from "../observe.js";
|
|
36
|
+
/**
|
|
37
|
+
* Report a Ragas `faithfulness` score.
|
|
38
|
+
*
|
|
39
|
+
* Faithfulness measures how well the agent's answer can be inferred
|
|
40
|
+
* from the retrieved context. Higher is better; range [0, 1].
|
|
41
|
+
*/
|
|
42
|
+
export function reportFaithfulness(score, opts = {}) {
|
|
43
|
+
const threshold = opts.threshold ?? 0.7;
|
|
44
|
+
emitEvalScore("ragas/faithfulness", "faithfulness", score, score >= threshold, {
|
|
45
|
+
threshold,
|
|
46
|
+
higherIsBetter: true,
|
|
47
|
+
reason: opts.reason,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Report a Ragas `answer_relevancy` score.
|
|
52
|
+
*
|
|
53
|
+
* Answer relevance measures whether the answer addresses the
|
|
54
|
+
* question that was asked. Higher is better; range [0, 1].
|
|
55
|
+
*/
|
|
56
|
+
export function reportAnswerRelevance(score, opts = {}) {
|
|
57
|
+
const threshold = opts.threshold ?? 0.7;
|
|
58
|
+
emitEvalScore("ragas/answer_relevance", "answer_relevance", score, score >= threshold, {
|
|
59
|
+
threshold,
|
|
60
|
+
higherIsBetter: true,
|
|
61
|
+
reason: opts.reason,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Report a Ragas `context_precision` score.
|
|
66
|
+
*
|
|
67
|
+
* Context precision measures whether the retrieved context contained
|
|
68
|
+
* only relevant chunks. Higher is better; range [0, 1].
|
|
69
|
+
*/
|
|
70
|
+
export function reportContextPrecision(score, opts = {}) {
|
|
71
|
+
const threshold = opts.threshold ?? 0.7;
|
|
72
|
+
emitEvalScore("ragas/context_precision", "context_precision", score, score >= threshold, {
|
|
73
|
+
threshold,
|
|
74
|
+
higherIsBetter: true,
|
|
75
|
+
reason: opts.reason,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=ragas.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ragas.js","sourceRoot":"","sources":["../../src/integrations/ragas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAY9C;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,oBAAoB,EACpB,cAAc,EACd,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,wBAAwB,EACxB,kBAAkB,EAClB,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,yBAAyB,EACzB,mBAAmB,EACnB,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vectara HHEM (Hughes Hallucination Evaluation Model) helper
|
|
3
|
+
* (Mesedi #14, Wave 1.3).
|
|
4
|
+
*
|
|
5
|
+
* Vectara HHEM
|
|
6
|
+
* (https://huggingface.co/vectara/hallucination_evaluation_model)
|
|
7
|
+
* is a small open-source model that detects hallucinations in RAG
|
|
8
|
+
* agent outputs. Customers running HHEM against their answers can
|
|
9
|
+
* use this helper to report scores into Mesedi without hand-writing
|
|
10
|
+
* the boilerplate around {@link emitEvalScore}.
|
|
11
|
+
*
|
|
12
|
+
* IMPORTANT — direction note. HHEM scores have INVERTED semantics
|
|
13
|
+
* compared to Ragas / Promptfoo metrics: HIGHER means MORE faithful
|
|
14
|
+
* to context (less hallucination). The helper passes
|
|
15
|
+
* `higherIsBetter: true` to the backend and the parameter is named
|
|
16
|
+
* `faithfulnessScore` to mirror HHEM's documented output. Some HHEM
|
|
17
|
+
* consumers transform the output into a "hallucination_score" where
|
|
18
|
+
* LOWER is better — if your pipeline does that transformation,
|
|
19
|
+
* invert your value before calling this helper (i.e. pass
|
|
20
|
+
* `1 - hallucinationScore`).
|
|
21
|
+
*
|
|
22
|
+
* Report-only design: Mesedi does NOT import the vectara /
|
|
23
|
+
* transformers packages as dependencies. The customer runs HHEM on
|
|
24
|
+
* their side; this helper accepts the resulting score and emits a
|
|
25
|
+
* correctly-shaped `eval_score` event.
|
|
26
|
+
*
|
|
27
|
+
* Typical usage:
|
|
28
|
+
*
|
|
29
|
+
* import { wrap } from "mesedi";
|
|
30
|
+
* import { reportHhem } from "mesedi/integrations/vectara";
|
|
31
|
+
*
|
|
32
|
+
* export const myAgent = wrap(async (question: string) => {
|
|
33
|
+
* const { answer, contexts } = await runRag(question);
|
|
34
|
+
* const hhemScore = await runHhem(answer, contexts); // 0-1, higher = better
|
|
35
|
+
* reportHhem(hhemScore, { threshold: 0.5 });
|
|
36
|
+
* return answer;
|
|
37
|
+
* });
|
|
38
|
+
*/
|
|
39
|
+
export interface VectaraReportOptions {
|
|
40
|
+
/**
|
|
41
|
+
* Cutoff below which the call counts as a failure. Default 0.5
|
|
42
|
+
* (HHEM's documented "neutral" point).
|
|
43
|
+
*/
|
|
44
|
+
threshold?: number;
|
|
45
|
+
/** Optional explanation text. */
|
|
46
|
+
reason?: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Report a Vectara HHEM faithfulness score.
|
|
50
|
+
*
|
|
51
|
+
* HHEM produces a faithfulness probability in [0, 1]; HIGHER means
|
|
52
|
+
* more faithful to the retrieved context (i.e. LESS hallucinated).
|
|
53
|
+
* Read the module docstring for direction-sensitivity notes.
|
|
54
|
+
*
|
|
55
|
+
* @param faithfulnessScore HHEM's faithfulness probability in [0, 1].
|
|
56
|
+
* HIGHER is better.
|
|
57
|
+
* @param opts Threshold + reason.
|
|
58
|
+
*/
|
|
59
|
+
export declare function reportHhem(faithfulnessScore: number, opts?: VectaraReportOptions): void;
|
|
60
|
+
//# sourceMappingURL=vectara.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vectara.d.ts","sourceRoot":"","sources":["../../src/integrations/vectara.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAIH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CACxB,iBAAiB,EAAE,MAAM,EACzB,IAAI,GAAE,oBAAyB,GAC9B,IAAI,CAaN"}
|