incremnt 0.7.1 → 0.8.0

package/src/promptfoo-langfuse-scores.js

@@ -100,6 +100,23 @@ function failedChecks(result) {
   return (result.checks ?? []).filter((check) => !check.passed);
 }
 
+function nonEmptyStrings(values, { max = 12 } = {}) {
+  const unique = [];
+  const seen = new Set();
+  for (const value of Array.isArray(values) ? values : []) {
+    const text = typeof value === 'string' ? value.trim() : '';
+    if (!text || seen.has(text)) continue;
+    seen.add(text);
+    unique.push(text);
+    if (unique.length >= max) break;
+  }
+  return unique.length > 0 ? unique : undefined;
+}
+
+function countArray(values) {
+  return Array.isArray(values) ? values.length : undefined;
+}
+
 function langfuseScoreTarget({ testCase = {}, context = {}, sessionId }) {
   const providerMetadata = context.providerResponse?.metadata ?? {};
   const caseMetadata = testCase.metadata ?? {};
@@ -138,6 +155,21 @@ export function buildPromptfooLangfuseScorePayload({
   const runId = scoreRunId(context, env, now);
   const mode = promptfooEvalMode(env);
   const provider = providerLabel(context);
+  const providerMetadata = context.providerResponse?.metadata ?? {};
+  const routingMetadata = providerMetadata.routingMetadata && typeof providerMetadata.routingMetadata === 'object'
+    ? providerMetadata.routingMetadata
+    : {};
+  const evidencePlan = routingMetadata.evidencePlan && typeof routingMetadata.evidencePlan === 'object'
+    ? routingMetadata.evidencePlan
+    : providerMetadata.evidencePlan && typeof providerMetadata.evidencePlan === 'object'
+      ? providerMetadata.evidencePlan
+      : {};
+  const contextBundle = routingMetadata.contextBundle && typeof routingMetadata.contextBundle === 'object'
+    ? routingMetadata.contextBundle
+    : {};
+  const structured = providerMetadata.structured && typeof providerMetadata.structured === 'object'
+    ? providerMetadata.structured
+    : {};
   const failed = failedChecks(result);
   const promptVersion = vars.promptVersion
     ?? testCase.metadata?.promptVersion
@@ -179,6 +211,29 @@ export function buildPromptfooLangfuseScorePayload({
       assertionCount: result.checks?.length ?? 0,
       failedAssertionKeys: failed.map((check) => check.key),
       failedAssertionReasons: failed.map((check) => check.reason).slice(0, 10),
+      route: firstString(routingMetadata.route, evidencePlan.route),
+      effectiveRoute: firstString(routingMetadata.effectiveRoute, evidencePlan.effectiveRoute),
+      requestedAction: firstString(routingMetadata.intent?.requestedAction),
+      intentConfidence: typeof routingMetadata.intent?.confidence === 'number' ? routingMetadata.intent.confidence : undefined,
+      contextCharCount: typeof routingMetadata.contextCharCount === 'number' ? routingMetadata.contextCharCount : undefined,
+      historyTurnCount: typeof routingMetadata.historyTurnCount === 'number' ? routingMetadata.historyTurnCount : undefined,
+      requiredTools: nonEmptyStrings(evidencePlan.requiredTools),
+      executedTools: nonEmptyStrings(evidencePlan.executedTools ?? routingMetadata.toolsUsed ?? contextBundle.executedTools),
+      evidenceGaps: nonEmptyStrings(evidencePlan.evidenceGaps),
+      missingDataFlags: nonEmptyStrings([
+        ...(routingMetadata.missingDataFlags ?? []),
+        ...(contextBundle.missingDataFlags ?? []),
+        ...(evidencePlan.evidenceGaps ?? [])
+      ]),
+      structuredResponsePresent: Object.keys(structured).length > 0 ? true : undefined,
+      structuredConfidence: firstString(structured.confidence),
+      followUpSuggestionCount: countArray(structured.followUpSuggestions),
+      limitationCount: countArray(structured.limitations),
+      evidenceUsedLabels: nonEmptyStrings((structured.evidenceUsed ?? []).map((item) => item?.label)),
+      evidenceUsedTools: nonEmptyStrings((structured.evidenceUsed ?? []).map((item) => item?.toolName)),
+      recommendedActionIds: nonEmptyStrings((structured.recommendedActions ?? []).map((item) => item?.id)),
+      recommendedActionLabels: nonEmptyStrings((structured.recommendedActions ?? []).map((item) => item?.label)),
+      hasProgramDraft: structured.programDraft != null ? true : undefined,
       generatedAt: now.toISOString()
     }),
     environment: env.LANGFUSE_ENVIRONMENT ?? env.NODE_ENV ?? 'development'