mesedi 0.1.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/README.md +81 -31
  2. package/dist/_errorClassesGenerated.d.ts +18 -0
  3. package/dist/_errorClassesGenerated.d.ts.map +1 -0
  4. package/dist/_errorClassesGenerated.js +24 -0
  5. package/dist/_errorClassesGenerated.js.map +1 -0
  6. package/dist/anthropic_integration.d.ts +9 -9
  7. package/dist/anthropic_integration.d.ts.map +1 -1
  8. package/dist/anthropic_integration.js +67 -26
  9. package/dist/anthropic_integration.js.map +1 -1
  10. package/dist/client.d.ts +32 -12
  11. package/dist/client.d.ts.map +1 -1
  12. package/dist/client.js +46 -18
  13. package/dist/client.js.map +1 -1
  14. package/dist/cohere_integration.d.ts +42 -0
  15. package/dist/cohere_integration.d.ts.map +1 -0
  16. package/dist/cohere_integration.js +456 -0
  17. package/dist/cohere_integration.js.map +1 -0
  18. package/dist/compress.d.ts +69 -0
  19. package/dist/compress.d.ts.map +1 -0
  20. package/dist/compress.js +75 -0
  21. package/dist/compress.js.map +1 -0
  22. package/dist/context.d.ts +13 -4
  23. package/dist/context.d.ts.map +1 -1
  24. package/dist/context.js +15 -5
  25. package/dist/context.js.map +1 -1
  26. package/dist/errors.d.ts +137 -0
  27. package/dist/errors.d.ts.map +1 -0
  28. package/dist/errors.js +462 -0
  29. package/dist/errors.js.map +1 -0
  30. package/dist/events.d.ts +19 -6
  31. package/dist/events.d.ts.map +1 -1
  32. package/dist/events.js +31 -6
  33. package/dist/events.js.map +1 -1
  34. package/dist/gemini_integration.d.ts +33 -0
  35. package/dist/gemini_integration.d.ts.map +1 -0
  36. package/dist/gemini_integration.js +334 -0
  37. package/dist/gemini_integration.js.map +1 -0
  38. package/dist/halt.d.ts +12 -12
  39. package/dist/halt.js +16 -16
  40. package/dist/halt.js.map +1 -1
  41. package/dist/halt_stream.d.ts +6 -6
  42. package/dist/halt_stream.d.ts.map +1 -1
  43. package/dist/halt_stream.js +22 -12
  44. package/dist/halt_stream.js.map +1 -1
  45. package/dist/index.d.ts +26 -13
  46. package/dist/index.d.ts.map +1 -1
  47. package/dist/index.js +20 -12
  48. package/dist/index.js.map +1 -1
  49. package/dist/integrations/langgraph.d.ts +99 -0
  50. package/dist/integrations/langgraph.d.ts.map +1 -0
  51. package/dist/integrations/langgraph.js +572 -0
  52. package/dist/integrations/langgraph.js.map +1 -0
  53. package/dist/integrations/mastra.d.ts +114 -0
  54. package/dist/integrations/mastra.d.ts.map +1 -0
  55. package/dist/integrations/mastra.js +354 -0
  56. package/dist/integrations/mastra.js.map +1 -0
  57. package/dist/integrations/openai_agents.d.ts +66 -0
  58. package/dist/integrations/openai_agents.d.ts.map +1 -0
  59. package/dist/integrations/openai_agents.js +243 -0
  60. package/dist/integrations/openai_agents.js.map +1 -0
  61. package/dist/integrations/promptfoo.d.ts +62 -0
  62. package/dist/integrations/promptfoo.d.ts.map +1 -0
  63. package/dist/integrations/promptfoo.js +76 -0
  64. package/dist/integrations/promptfoo.js.map +1 -0
  65. package/dist/integrations/ragas.d.ts +65 -0
  66. package/dist/integrations/ragas.d.ts.map +1 -0
  67. package/dist/integrations/ragas.js +78 -0
  68. package/dist/integrations/ragas.js.map +1 -0
  69. package/dist/integrations/vectara.d.ts +60 -0
  70. package/dist/integrations/vectara.d.ts.map +1 -0
  71. package/dist/integrations/vectara.js +59 -0
  72. package/dist/integrations/vectara.js.map +1 -0
  73. package/dist/integrations/vercel_ai.d.ts +6 -6
  74. package/dist/integrations/vercel_ai.js +10 -10
  75. package/dist/integrations/vercel_ai.js.map +1 -1
  76. package/dist/observe.d.ts +358 -3
  77. package/dist/observe.d.ts.map +1 -1
  78. package/dist/observe.js +536 -4
  79. package/dist/observe.js.map +1 -1
  80. package/dist/ollama_integration.d.ts +126 -0
  81. package/dist/ollama_integration.d.ts.map +1 -0
  82. package/dist/ollama_integration.js +404 -0
  83. package/dist/ollama_integration.js.map +1 -0
  84. package/dist/openai_integration.d.ts +67 -0
  85. package/dist/openai_integration.d.ts.map +1 -0
  86. package/dist/openai_integration.js +574 -0
  87. package/dist/openai_integration.js.map +1 -0
  88. package/dist/tool.d.ts +1 -1
  89. package/dist/tool.d.ts.map +1 -1
  90. package/dist/tool.js +193 -10
  91. package/dist/tool.js.map +1 -1
  92. package/dist/truncate.d.ts +84 -0
  93. package/dist/truncate.d.ts.map +1 -0
  94. package/dist/truncate.js +134 -0
  95. package/dist/truncate.js.map +1 -0
  96. package/dist/vertex_gemini_integration.d.ts +27 -0
  97. package/dist/vertex_gemini_integration.d.ts.map +1 -0
  98. package/dist/vertex_gemini_integration.js +229 -0
  99. package/dist/vertex_gemini_integration.js.map +1 -0
  100. package/dist/wrap.d.ts +39 -3
  101. package/dist/wrap.d.ts.map +1 -1
  102. package/dist/wrap.js +15 -12
  103. package/dist/wrap.js.map +1 -1
  104. package/package.json +28 -4
@@ -0,0 +1,243 @@
1
+ /**
2
+ * OpenAI Agents SDK integration for Mesedi (Mesedi #28).
3
+ *
4
+ * Usage:
5
+ *
6
+ * import { wrap } from "mesedi";
7
+ * import { MesediRunHooks } from "mesedi/integrations/openai_agents";
8
+ * import { Agent, Runner } from "@openai/agents";
9
+ *
10
+ * const triage = new Agent({ name: "triage", ...});
11
+ * const planner = new Agent({ name: "planner", ...});
12
+ *
13
+ * export const runUserRequest = wrap(async (question: string) => {
14
+ * const result = await Runner.run(
15
+ * triage,
16
+ * question,
17
+ * { hooks: new MesediRunHooks() },
18
+ * );
19
+ * return result.finalOutput;
20
+ * });
21
+ *
22
+ * What the hooks emit:
23
+ *
24
+ * - `checkpoint` events at every agent start AND end with the
25
+ * agent name and a truncated input/output repr. Feeds the
26
+ * semantic_loop detector (#6).
27
+ * - `agent_handoff` events on every handoff between agents.
28
+ * from_agent / to_agent come straight from the hook arguments,
29
+ * handoff_kind defaults to `"transfer"`. Feeds cascading_failure
30
+ * (#12) and coordination_deadlock (#13).
31
+ * - `tool_call` events for every tool invocation, mirroring
32
+ * `mesedi.tool()` wire format.
33
+ *
34
+ * Out of scope for v1:
35
+ *
36
+ * - `llm_call` events. The OAI Agents SDK does not expose
37
+ * per-LLM-call hooks in its public RunHooks surface as of the
38
+ * version this integration was written against. For
39
+ * Anthropic-backed runs use the existing
40
+ * `instrumentAnthropic()` patch; an OpenAI-backed equivalent
41
+ * is not currently provided.
42
+ * - Streaming hooks (`Runner.runStreamed`). Not currently
43
+ * instrumented.
44
+ */
45
+ import { getClient } from "../client.js";
46
+ import { currentExecutionContext, newEventId } from "../context.js";
47
+ import { EventType, utcNowRfc3339 } from "../events.js";
48
+ // Truncation budget for checkpoint state repr and tool args /
49
+ // return values. Wide enough to be useful for semantic_loop
50
+ // hashing; narrow enough to keep individual event payloads
51
+ // bounded.
52
+ const MAX_STATE_REPR = 1000;
53
+ const MAX_TOOL_INPUT_REPR = 200;
54
+ const MAX_TOOL_OUTPUT_REPR = 500;
55
+ /**
56
+ * Mesedi-emitting implementation of the `@openai/agents` RunHooks
57
+ * interface. Pass an instance to `Runner.run(..., { hooks: ... })`.
58
+ *
59
+ * All emissions go through Mesedi's standard event submitter, which
60
+ * is non-blocking (events are batched on a background shipper). The
61
+ * OAI Agents SDK's async runner is therefore never blocked on
62
+ * Mesedi I/O.
63
+ *
64
+ * Outside a `wrap()` execution context, all emissions silently
65
+ * no-op (matching the rest of Mesedi's observe layer).
66
+ */
67
+ export class MesediRunHooks {
68
+ // The OAI Agents SDK reads this to identify the hooks instance
69
+ // in trace output. Required field on some versions of the
70
+ // dispatcher; ignored by older ones.
71
+ name = "mesedi-openai-agents";
72
+ // tool key → start time (ms epoch). Used to pair onToolStart
73
+ // with onToolEnd so we can compute duration. We key on the
74
+ // tool name because the SDK does not guarantee stable
75
+ // object identity across the start/end pair in all versions.
76
+ toolStarts = new Map();
77
+ // ── Agent lifecycle ─────────────────────────────────────────────
78
+ async onAgentStart(_context, agent) {
79
+ const agentName = extractAgentName(agent);
80
+ emitCheckpoint(agentName, extractContextState(_context), "enter");
81
+ }
82
+ async onAgentEnd(_context, agent, output) {
83
+ const agentName = extractAgentName(agent);
84
+ emitCheckpoint(agentName, output, "exit");
85
+ }
86
+ // ── Handoffs ────────────────────────────────────────────────────
87
+ async onHandoff(context, fromAgent, toAgent) {
88
+ emitHandoff(extractAgentName(fromAgent), extractAgentName(toAgent), "transfer", summarize(extractContextState(context)));
89
+ }
90
+ // ── Tools ───────────────────────────────────────────────────────
91
+ async onToolStart(_context, _agent, tool) {
92
+ const key = extractToolName(tool);
93
+ this.toolStarts.set(key, Date.now());
94
+ }
95
+ async onToolEnd(_context, _agent, tool, result) {
96
+ const key = extractToolName(tool);
97
+ const started = this.toolStarts.get(key);
98
+ this.toolStarts.delete(key);
99
+ const durationMs = started != null ? Math.max(0, Date.now() - started) : 0;
100
+ emitToolCall(key, summarizeToolInput(tool), summarizeToolOutput(result), durationMs, "ok");
101
+ }
102
+ }
103
+ // ── Helpers ─────────────────────────────────────────────────────────
104
+ function extractAgentName(agent) {
105
+ if (agent == null)
106
+ return "unknown";
107
+ if (typeof agent === "object") {
108
+ const a = agent;
109
+ if (typeof a.name === "string" && a.name)
110
+ return a.name;
111
+ const ctor = a.constructor?.name;
112
+ if (typeof ctor === "string" && ctor)
113
+ return ctor;
114
+ }
115
+ return String(agent);
116
+ }
117
+ function extractToolName(tool) {
118
+ if (tool == null)
119
+ return "unknown";
120
+ if (typeof tool === "object") {
121
+ const t = tool;
122
+ if (typeof t.name === "string" && t.name)
123
+ return t.name;
124
+ if (typeof t.toolName === "string" && t.toolName)
125
+ return t.toolName;
126
+ const ctor = t.constructor?.name;
127
+ if (typeof ctor === "string" && ctor)
128
+ return ctor;
129
+ }
130
+ return String(tool);
131
+ }
132
+ function extractContextState(context) {
133
+ // OAI Agents wraps the user-supplied context in a wrapper; the
134
+ // real context is typically at `.context`. Fall back to the
135
+ // wrapper itself when the attribute is missing.
136
+ if (context && typeof context === "object") {
137
+ const c = context;
138
+ if ("context" in c && c.context != null)
139
+ return c.context;
140
+ }
141
+ return context;
142
+ }
143
+ function summarize(obj) {
144
+ if (obj == null)
145
+ return "";
146
+ let r;
147
+ try {
148
+ r = typeof obj === "string" ? obj : JSON.stringify(obj);
149
+ }
150
+ catch {
151
+ r = String(obj);
152
+ }
153
+ if (r.length > MAX_STATE_REPR) {
154
+ r = r.slice(0, MAX_STATE_REPR - 3) + "...";
155
+ }
156
+ return r;
157
+ }
158
+ function summarizeToolInput(tool) {
159
+ if (tool && typeof tool === "object") {
160
+ const t = tool;
161
+ for (const attr of ["arguments", "input", "params"]) {
162
+ if (t[attr] != null)
163
+ return summarize(t[attr]);
164
+ }
165
+ }
166
+ return summarize(extractToolName(tool));
167
+ }
168
+ function summarizeToolOutput(result) {
169
+ if (result == null)
170
+ return "";
171
+ if (typeof result === "object") {
172
+ const r = result;
173
+ if ("output" in r && r.output != null)
174
+ return summarize(r.output);
175
+ }
176
+ return summarize(result);
177
+ }
178
+ function emitCheckpoint(agentName, state, agentEvent) {
179
+ const ctx = currentExecutionContext();
180
+ if (!ctx)
181
+ return;
182
+ const payload = {
183
+ name: `openai_agents.${agentEvent}.${agentName}`,
184
+ agent: agentName,
185
+ agent_event: agentEvent,
186
+ state_repr: summarize(state),
187
+ };
188
+ const client = getClient();
189
+ const event = {
190
+ event_id: newEventId(),
191
+ execution_id: ctx.executionId,
192
+ event_type: EventType.CHECKPOINT,
193
+ sequence: ctx.nextSequence(),
194
+ timestamp: utcNowRfc3339(),
195
+ payload,
196
+ };
197
+ client.submitEvent(event);
198
+ }
199
+ function emitHandoff(fromAgent, toAgent, handoffKind, taskSummary) {
200
+ const ctx = currentExecutionContext();
201
+ if (!ctx)
202
+ return;
203
+ const payload = {
204
+ from_agent: fromAgent,
205
+ to_agent: toAgent,
206
+ handoff_kind: handoffKind,
207
+ task_summary: taskSummary,
208
+ };
209
+ const client = getClient();
210
+ const event = {
211
+ event_id: newEventId(),
212
+ execution_id: ctx.executionId,
213
+ event_type: EventType.AGENT_HANDOFF,
214
+ sequence: ctx.nextSequence(),
215
+ timestamp: utcNowRfc3339(),
216
+ payload,
217
+ };
218
+ client.submitEvent(event);
219
+ }
220
+ function emitToolCall(toolName, argumentsRepr, returnValueRepr, durationMs, status) {
221
+ const ctx = currentExecutionContext();
222
+ if (!ctx)
223
+ return;
224
+ const payload = {
225
+ tool_name: toolName,
226
+ arguments: argumentsRepr.slice(0, MAX_TOOL_INPUT_REPR),
227
+ return_value: returnValueRepr.slice(0, MAX_TOOL_OUTPUT_REPR),
228
+ latency_ms: Math.trunc(durationMs),
229
+ status,
230
+ };
231
+ const client = getClient();
232
+ const event = {
233
+ event_id: newEventId(),
234
+ execution_id: ctx.executionId,
235
+ event_type: EventType.TOOL_CALL,
236
+ sequence: ctx.nextSequence(),
237
+ timestamp: utcNowRfc3339(),
238
+ duration_ms: Math.trunc(durationMs),
239
+ payload,
240
+ };
241
+ client.submitEvent(event);
242
+ }
243
+ //# sourceMappingURL=openai_agents.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai_agents.js","sourceRoot":"","sources":["../../src/integrations/openai_agents.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,EAAS,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE/D,8DAA8D;AAC9D,4DAA4D;AAC5D,2DAA2D;AAC3D,WAAW;AACX,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAc;IACzB,+DAA+D;IAC/D,0DAA0D;IAC1D,qCAAqC;IAC5B,IAAI,GAAG,sBAAsB,CAAC;IAEvC,6DAA6D;IAC7D,2DAA2D;IAC3D,sDAAsD;IACtD,6DAA6D;IACrD,UAAU,GAAwB,IAAI,GAAG,EAAE,CAAC;IAEpD,mEAAmE;IAEnE,KAAK,CAAC,YAAY,CAAC,QAAiB,EAAE,KAAc;QAClD,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC1C,cAAc,CAAC,SAAS,EAAE,mBAAmB,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,UAAU,CACd,QAAiB,EACjB,KAAc,EACd,MAAe;QAEf,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAC1C,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IAED,mEAAmE;IAEnE,KAAK,CAAC,SAAS,CACb,OAAgB,EAChB,SAAkB,EAClB,OAAgB;QAEhB,WAAW,CACT,gBAAgB,CAAC,SAAS,CAAC,EAC3B,gBAAgB,CAAC,OAAO,CAAC,EACzB,UAAU,EACV,SAAS,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CACxC,CAAC;IACJ,CAAC;IAED,mEAAmE;IAEnE,KAAK,CAAC,WAAW,CACf,QAAiB,EACjB,MAAe,EACf,IAAa;QAEb,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,SAAS,CACb,QAAiB,EACjB,MAAe,EACf,IAAa,EACb,MAAe;QAEf,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,UAAU,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,YAAY,CACV,GAAG,EACH,kBAAkB,CAAC,IAAI,CAAC,EACxB,mBAAmB,CAAC,MAAM,CAAC,EAC3B,UAAU,EACV,IAAI,CACL,CAAC;IACJ,CAAC;CACF;AAED,uEAAuE;AAEvE,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,KAAgC,CAAC;QAC3C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,IAAI,GAAI,CAAC,CAAC,WAA8C,EAAE,IAAI,CAAC;QACrE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,IAAa;IACpC,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACnC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACxD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QACpE,MAAM,IAAI,GAAI,CAAC,CAAC,WAA8C,EAAE,IAAI,CAAC;QACrE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACpD,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAgB;IAC3C,+DAA+D;IAC/D,4DAA4D;IAC5D,gDAAgD;IAChD,IAAI,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,OAAkC,CAAC;QAC7C,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI;YAAE,OAAO,CAAC,CAAC,OAAO,CAAC;IAC5D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS,CAAC,GAAY;IAC7B,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC3B,IAAI,CAAS,CAAC;IACd,IAAI,CAAC;QACH,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QAC9B,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAA+B,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAU,EAAE,CAAC;YAC7D,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI;gBAAE,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAe;IAC1C,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI;YAAE,OAAO,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,cAAc,CACrB,SAAiB,EACjB,KAAc,EACd,UAA4B;IAE5B,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IAEjB,MAAM,OAAO,GAA4B;QACvC,IAAI,EAAE,iBAAiB,UAAU,IAAI,SAAS,EAAE;QAChD,KAAK,EAAE,SAAS;QAChB,WAAW,EAAE,UAAU;QACvB,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC;KAC7B,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,WAAW,CAClB,SAAiB,EACjB,OAAe,EACf,WAAmB,EACnB,WAAmB;IAEnB,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,OAAO,GAA4B;QACvC,UAAU,EAAE,SAAS;QACrB,QAAQ,EAAE,OAAO;QACjB,YAAY,EAAE,WAAW;QACzB,YAAY,EAAE,WAAW;KAC1B,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,aAAa;QACnC,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,YAAY,CACnB,QAAgB,EAChB,aAAqB,EACrB,eAAuB,EACvB,UAAkB,EAClB,MAAuB;IAEvB,MAAM,GAAG,GAAG,uBAAuB,EAAE,CAAC;IACtC,IAAI,CAAC,GAAG;QAAE,OAAO;IACjB,MAAM,OAAO,GAA4B;QACvC,SAAS,EAAE,QAAQ;QACnB,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC;QACtD,YAAY,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC;QAC5D,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QAClC,MAAM;KACP,CAAC;IACF,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAU;QACnB,QAAQ,EAAE,UAAU,EAAE;QACtB,YAAY,EAAE,GAAG,CAAC,WAAW;QAC7B,UAAU,EAAE,SAAS,CAAC,SAAS;QAC/B,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE;QAC5B,SAAS,EAAE,aAAa,EAAE;QAC1B,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;QACnC,OAAO;KACR,CAAC;IACF,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC"}
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Promptfoo evaluator one-liner helpers (Mesedi #14, Wave 1.3).
3
+ *
4
+ * Promptfoo (https://www.promptfoo.dev) is a CLI + library for
5
+ * prompt evaluation. Customers using its assertion types
6
+ * (model-graded-factuality, llm-rubric, ...) can use these helpers
7
+ * to report verdicts into Mesedi without hand-writing the
8
+ * boilerplate around {@link emitEvalScore}.
9
+ *
10
+ * Report-only design: Mesedi does NOT import the `promptfoo` package
11
+ * as a dependency. The customer runs Promptfoo on their side; these
12
+ * helpers accept the resulting score/verdict and emit a
13
+ * correctly-shaped `eval_score` event.
14
+ *
15
+ * Typical usage:
16
+ *
17
+ * import { wrap } from "mesedi";
18
+ * import { reportFactuality } from "mesedi/integrations/promptfoo";
19
+ *
20
+ * export const myAgent = wrap(async (question: string) => {
21
+ * const answer = await runAgent(question);
22
+ * const verdict = await promptfooAssertFactuality(answer, expectedFacts);
23
+ * reportFactuality(verdict.score, verdict.pass);
24
+ * return answer;
25
+ * });
26
+ *
27
+ * Helpers cover the two most common Promptfoo assertion shapes. For
28
+ * other assertion types (similar, contains, equals, ...) call
29
+ * {@link emitEvalScore} directly with
30
+ * `evaluatorId: "promptfoo/<assertion>"`.
31
+ */
32
+ export interface PromptfooReportOptions {
33
+ /** Threshold the Promptfoo assertion used. Default 0.5. */
34
+ threshold?: number;
35
+ /** Optional explanation the judge returned. */
36
+ reason?: string;
37
+ }
38
+ /**
39
+ * Report a Promptfoo `model-graded-factuality` assertion.
40
+ *
41
+ * Promptfoo's model-graded factuality assertion uses an LLM judge
42
+ * to score how factually accurate the answer is. Higher is better.
43
+ */
44
+ export declare function reportFactuality(score: number, passed: boolean, opts?: PromptfooReportOptions): void;
45
+ /**
46
+ * Report a Promptfoo `llm-rubric` assertion.
47
+ *
48
+ * The llm-rubric assertion runs a customer-defined rubric through
49
+ * an LLM judge. Because a single execution can run multiple rubrics
50
+ * against the same output, `metricName` distinguishes them so
51
+ * grounding_failure clusters per-rubric instead of collapsing every
52
+ * rubric into one bucket.
53
+ *
54
+ * @param metricName Short identifier for the specific rubric
55
+ * ("clarity", "helpfulness", "tone_appropriate", etc.). Becomes
56
+ * part of the eval_score's metric_type field.
57
+ * @param score The judge's numeric score for this rubric.
58
+ * @param passed Promptfoo's own pass/fail verdict for this rubric.
59
+ * @param opts Threshold + reason.
60
+ */
61
+ export declare function reportLlmRubric(metricName: string, score: number, passed: boolean, opts?: PromptfooReportOptions): void;
62
+ //# sourceMappingURL=promptfoo.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo.d.ts","sourceRoot":"","sources":["../../src/integrations/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,sBAAsB;IACrC,2DAA2D;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,EACf,IAAI,GAAE,sBAA2B,GAChC,IAAI,CAaN;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,OAAO,EACf,IAAI,GAAE,sBAA2B,GAChC,IAAI,CAoBN"}
@@ -0,0 +1,76 @@
1
+ /**
2
+ * Promptfoo evaluator one-liner helpers (Mesedi #14, Wave 1.3).
3
+ *
4
+ * Promptfoo (https://www.promptfoo.dev) is a CLI + library for
5
+ * prompt evaluation. Customers using its assertion types
6
+ * (model-graded-factuality, llm-rubric, ...) can use these helpers
7
+ * to report verdicts into Mesedi without hand-writing the
8
+ * boilerplate around {@link emitEvalScore}.
9
+ *
10
+ * Report-only design: Mesedi does NOT import the `promptfoo` package
11
+ * as a dependency. The customer runs Promptfoo on their side; these
12
+ * helpers accept the resulting score/verdict and emit a
13
+ * correctly-shaped `eval_score` event.
14
+ *
15
+ * Typical usage:
16
+ *
17
+ * import { wrap } from "mesedi";
18
+ * import { reportFactuality } from "mesedi/integrations/promptfoo";
19
+ *
20
+ * export const myAgent = wrap(async (question: string) => {
21
+ * const answer = await runAgent(question);
22
+ * const verdict = await promptfooAssertFactuality(answer, expectedFacts);
23
+ * reportFactuality(verdict.score, verdict.pass);
24
+ * return answer;
25
+ * });
26
+ *
27
+ * Helpers cover the two most common Promptfoo assertion shapes. For
28
+ * other assertion types (similar, contains, equals, ...) call
29
+ * {@link emitEvalScore} directly with
30
+ * `evaluatorId: "promptfoo/<assertion>"`.
31
+ */
32
+ import { emitEvalScore } from "../observe.js";
33
+ /**
34
+ * Report a Promptfoo `model-graded-factuality` assertion.
35
+ *
36
+ * Promptfoo's model-graded factuality assertion uses an LLM judge
37
+ * to score how factually accurate the answer is. Higher is better.
38
+ */
39
+ export function reportFactuality(score, passed, opts = {}) {
40
+ const threshold = opts.threshold ?? 0.5;
41
+ emitEvalScore("promptfoo/factuality", "factuality", score, passed, {
42
+ threshold,
43
+ higherIsBetter: true,
44
+ reason: opts.reason,
45
+ });
46
+ }
47
+ /**
48
+ * Report a Promptfoo `llm-rubric` assertion.
49
+ *
50
+ * The llm-rubric assertion runs a customer-defined rubric through
51
+ * an LLM judge. Because a single execution can run multiple rubrics
52
+ * against the same output, `metricName` distinguishes them so
53
+ * grounding_failure clusters per-rubric instead of collapsing every
54
+ * rubric into one bucket.
55
+ *
56
+ * @param metricName Short identifier for the specific rubric
57
+ * ("clarity", "helpfulness", "tone_appropriate", etc.). Becomes
58
+ * part of the eval_score's metric_type field.
59
+ * @param score The judge's numeric score for this rubric.
60
+ * @param passed Promptfoo's own pass/fail verdict for this rubric.
61
+ * @param opts Threshold + reason.
62
+ */
63
+ export function reportLlmRubric(metricName, score, passed, opts = {}) {
64
+ if (!metricName) {
65
+ throw new Error("reportLlmRubric: metricName is required so the " +
66
+ "grounding_failure detector can cluster per-rubric " +
67
+ "instead of collapsing every rubric into one bucket.");
68
+ }
69
+ const threshold = opts.threshold ?? 0.5;
70
+ emitEvalScore(`promptfoo/llm-rubric/${metricName}`, metricName, score, passed, {
71
+ threshold,
72
+ higherIsBetter: true,
73
+ reason: opts.reason,
74
+ });
75
+ }
76
+ //# sourceMappingURL=promptfoo.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../../src/integrations/promptfoo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAS9C;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAa,EACb,MAAe,EACf,OAA+B,EAAE;IAEjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,sBAAsB,EACtB,YAAY,EACZ,KAAK,EACL,MAAM,EACN;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,eAAe,CAC7B,UAAkB,EAClB,KAAa,EACb,MAAe,EACf,OAA+B,EAAE;IAEjC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,iDAAiD;YAC/C,oDAAoD;YACpD,qDAAqD,CACxD,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,wBAAwB,UAAU,EAAE,EACpC,UAAU,EACV,KAAK,EACL,MAAM,EACN;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Ragas grounding-evaluator one-liner helpers (Mesedi #14, Wave 1.3).
3
+ *
4
+ * Ragas (https://docs.ragas.io) is a popular RAG evaluation library.
5
+ * Customers running Ragas against their agent outputs can use these
6
+ * helpers to report scores into Mesedi without hand-writing the
7
+ * boilerplate around {@link emitEvalScore}.
8
+ *
9
+ * Report-only design: Mesedi does NOT import the `ragas` package as
10
+ * a dependency. The customer runs the evaluator on their side with
11
+ * their own version pinning and configuration; these helpers accept
12
+ * the resulting score and emit a correctly-shaped `eval_score` event
13
+ * so the backend's grounding_failure detector aggregates it under
14
+ * the right evaluator_id + metric_type cluster.
15
+ *
16
+ * Typical usage (Ragas in TypeScript is typically called via a
17
+ * Python sidecar; the helper signature is identical regardless of
18
+ * how the customer obtained the score):
19
+ *
20
+ * import { wrap } from "mesedi";
21
+ * import { reportFaithfulness } from "mesedi/integrations/ragas";
22
+ *
23
+ * export const myAgent = wrap(async (question: string) => {
24
+ * const { answer, contexts } = await runRag(question);
25
+ * const score = await scoreWithRagas(question, answer, contexts);
26
+ * reportFaithfulness(score, { threshold: 0.7 });
27
+ * return answer;
28
+ * });
29
+ *
30
+ * Helpers cover the three most common Ragas metrics. For niche
31
+ * metrics (answer_correctness, answer_similarity, context_recall, ...)
32
+ * call {@link emitEvalScore} directly with
33
+ * `evaluatorId: "ragas/<metric>"`.
34
+ */
35
+ export interface RagasReportOptions {
36
+ /**
37
+ * Cutoff below which the call counts as a failure. Defaults to
38
+ * 0.7 (Ragas's documented working threshold).
39
+ */
40
+ threshold?: number;
41
+ /** Optional explanation Ragas returned. */
42
+ reason?: string;
43
+ }
44
+ /**
45
+ * Report a Ragas `faithfulness` score.
46
+ *
47
+ * Faithfulness measures how well the agent's answer can be inferred
48
+ * from the retrieved context. Higher is better; range [0, 1].
49
+ */
50
+ export declare function reportFaithfulness(score: number, opts?: RagasReportOptions): void;
51
+ /**
52
+ * Report a Ragas `answer_relevancy` score.
53
+ *
54
+ * Answer relevance measures whether the answer addresses the
55
+ * question that was asked. Higher is better; range [0, 1].
56
+ */
57
+ export declare function reportAnswerRelevance(score: number, opts?: RagasReportOptions): void;
58
+ /**
59
+ * Report a Ragas `context_precision` score.
60
+ *
61
+ * Context precision measures whether the retrieved context contained
62
+ * only relevant chunks. Higher is better; range [0, 1].
63
+ */
64
+ export declare function reportContextPrecision(score: number, opts?: RagasReportOptions): void;
65
+ //# sourceMappingURL=ragas.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ragas.d.ts","sourceRoot":"","sources":["../../src/integrations/ragas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAIH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,kBAAuB,GAC5B,IAAI,CAaN"}
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Ragas grounding-evaluator one-liner helpers (Mesedi #14, Wave 1.3).
3
+ *
4
+ * Ragas (https://docs.ragas.io) is a popular RAG evaluation library.
5
+ * Customers running Ragas against their agent outputs can use these
6
+ * helpers to report scores into Mesedi without hand-writing the
7
+ * boilerplate around {@link emitEvalScore}.
8
+ *
9
+ * Report-only design: Mesedi does NOT import the `ragas` package as
10
+ * a dependency. The customer runs the evaluator on their side with
11
+ * their own version pinning and configuration; these helpers accept
12
+ * the resulting score and emit a correctly-shaped `eval_score` event
13
+ * so the backend's grounding_failure detector aggregates it under
14
+ * the right evaluator_id + metric_type cluster.
15
+ *
16
+ * Typical usage (Ragas in TypeScript is typically called via a
17
+ * Python sidecar; the helper signature is identical regardless of
18
+ * how the customer obtained the score):
19
+ *
20
+ * import { wrap } from "mesedi";
21
+ * import { reportFaithfulness } from "mesedi/integrations/ragas";
22
+ *
23
+ * export const myAgent = wrap(async (question: string) => {
24
+ * const { answer, contexts } = await runRag(question);
25
+ * const score = await scoreWithRagas(question, answer, contexts);
26
+ * reportFaithfulness(score, { threshold: 0.7 });
27
+ * return answer;
28
+ * });
29
+ *
30
+ * Helpers cover the three most common Ragas metrics. For niche
31
+ * metrics (answer_correctness, answer_similarity, context_recall, ...)
32
+ * call {@link emitEvalScore} directly with
33
+ * `evaluatorId: "ragas/<metric>"`.
34
+ */
35
+ import { emitEvalScore } from "../observe.js";
36
+ /**
37
+ * Report a Ragas `faithfulness` score.
38
+ *
39
+ * Faithfulness measures how well the agent's answer can be inferred
40
+ * from the retrieved context. Higher is better; range [0, 1].
41
+ */
42
+ export function reportFaithfulness(score, opts = {}) {
43
+ const threshold = opts.threshold ?? 0.7;
44
+ emitEvalScore("ragas/faithfulness", "faithfulness", score, score >= threshold, {
45
+ threshold,
46
+ higherIsBetter: true,
47
+ reason: opts.reason,
48
+ });
49
+ }
50
+ /**
51
+ * Report a Ragas `answer_relevancy` score.
52
+ *
53
+ * Answer relevance measures whether the answer addresses the
54
+ * question that was asked. Higher is better; range [0, 1].
55
+ */
56
+ export function reportAnswerRelevance(score, opts = {}) {
57
+ const threshold = opts.threshold ?? 0.7;
58
+ emitEvalScore("ragas/answer_relevance", "answer_relevance", score, score >= threshold, {
59
+ threshold,
60
+ higherIsBetter: true,
61
+ reason: opts.reason,
62
+ });
63
+ }
64
+ /**
65
+ * Report a Ragas `context_precision` score.
66
+ *
67
+ * Context precision measures whether the retrieved context contained
68
+ * only relevant chunks. Higher is better; range [0, 1].
69
+ */
70
+ export function reportContextPrecision(score, opts = {}) {
71
+ const threshold = opts.threshold ?? 0.7;
72
+ emitEvalScore("ragas/context_precision", "context_precision", score, score >= threshold, {
73
+ threshold,
74
+ higherIsBetter: true,
75
+ reason: opts.reason,
76
+ });
77
+ }
78
+ //# sourceMappingURL=ragas.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ragas.js","sourceRoot":"","sources":["../../src/integrations/ragas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAY9C;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,oBAAoB,EACpB,cAAc,EACd,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,wBAAwB,EACxB,kBAAkB,EAClB,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAa,EACb,OAA2B,EAAE;IAE7B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,aAAa,CACX,yBAAyB,EACzB,mBAAmB,EACnB,KAAK,EACL,KAAK,IAAI,SAAS,EAClB;QACE,SAAS;QACT,cAAc,EAAE,IAAI;QACpB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Vectara HHEM (Hughes Hallucination Evaluation Model) helper
3
+ * (Mesedi #14, Wave 1.3).
4
+ *
5
+ * Vectara HHEM
6
+ * (https://huggingface.co/vectara/hallucination_evaluation_model)
7
+ * is a small open-source model that detects hallucinations in RAG
8
+ * agent outputs. Customers running HHEM against their answers can
9
+ * use this helper to report scores into Mesedi without hand-writing
10
+ * the boilerplate around {@link emitEvalScore}.
11
+ *
12
+ * IMPORTANT — direction note. HHEM scores have INVERTED semantics
13
+ * compared to Ragas / Promptfoo metrics: HIGHER means MORE faithful
14
+ * to context (less hallucination). The helper passes
15
+ * `higherIsBetter: true` to the backend and the parameter is named
16
+ * `faithfulnessScore` to mirror HHEM's documented output. Some HHEM
17
+ * consumers transform the output into a "hallucination_score" where
18
+ * LOWER is better — if your pipeline does that transformation,
19
+ * invert your value before calling this helper (i.e. pass
20
+ * `1 - hallucinationScore`).
21
+ *
22
+ * Report-only design: Mesedi does NOT import the vectara /
23
+ * transformers packages as dependencies. The customer runs HHEM on
24
+ * their side; this helper accepts the resulting score and emits a
25
+ * correctly-shaped `eval_score` event.
26
+ *
27
+ * Typical usage:
28
+ *
29
+ * import { wrap } from "mesedi";
30
+ * import { reportHhem } from "mesedi/integrations/vectara";
31
+ *
32
+ * export const myAgent = wrap(async (question: string) => {
33
+ * const { answer, contexts } = await runRag(question);
34
+ * const hhemScore = await runHhem(answer, contexts); // 0-1, higher = better
35
+ * reportHhem(hhemScore, { threshold: 0.5 });
36
+ * return answer;
37
+ * });
38
+ */
39
+ export interface VectaraReportOptions {
40
+ /**
41
+ * Cutoff below which the call counts as a failure. Default 0.5
42
+ * (HHEM's documented "neutral" point).
43
+ */
44
+ threshold?: number;
45
+ /** Optional explanation text. */
46
+ reason?: string;
47
+ }
48
+ /**
49
+ * Report a Vectara HHEM faithfulness score.
50
+ *
51
+ * HHEM produces a faithfulness probability in [0, 1]; HIGHER means
52
+ * more faithful to the retrieved context (i.e. LESS hallucinated).
53
+ * Read the module docstring for direction-sensitivity notes.
54
+ *
55
+ * @param faithfulnessScore HHEM's faithfulness probability in [0, 1].
56
+ * HIGHER is better.
57
+ * @param opts Threshold + reason.
58
+ */
59
+ export declare function reportHhem(faithfulnessScore: number, opts?: VectaraReportOptions): void;
60
+ //# sourceMappingURL=vectara.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vectara.d.ts","sourceRoot":"","sources":["../../src/integrations/vectara.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAIH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CACxB,iBAAiB,EAAE,MAAM,EACzB,IAAI,GAAE,oBAAyB,GAC9B,IAAI,CAaN"}