incremnt 0.3.0 → 0.5.0

package/src/promptfoo-langfuse-scores.js

@@ -0,0 +1,354 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { AI_PROMPT_VERSIONS } from './openrouter.js';
+
+const DEFAULT_LANGFUSE_HOST = 'https://cloud.langfuse.com';
+const SCORE_NAME = 'promptfoo_domain_pass';
+const promptfooRunTracesCreated = new Set();
+
+function compactObject(obj) {
+  return Object.fromEntries(
+    Object.entries(obj).filter(([, value]) => value !== undefined && value !== null)
+  );
+}
+
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value === 'string' && value.trim()) return value.trim();
+  }
+  return null;
+}
+
+function envFlag(name, env = process.env) {
+  return ['1', 'true', 'yes'].includes(String(env[name] ?? '').toLowerCase());
+}
+
+function currentGitSha(env = process.env) {
+  return env.RENDER_GIT_COMMIT
+    ?? env.GIT_SHA
+    ?? env.COMMIT_SHA
+    ?? env.VERCEL_GIT_COMMIT_SHA
+    ?? null;
+}
+
+export function shouldPublishPromptfooLangfuseScores(env = process.env) {
+  if (String(env.PROMPTFOO_LANGFUSE_SCORES ?? '').toLowerCase() === '0') return false;
+  return Boolean(env.LANGFUSE_PUBLIC_KEY && env.LANGFUSE_SECRET_KEY);
+}
+
+export function langfuseHost(env = process.env) {
+  return (firstString(env.LANGFUSE_BASE_URL, env.LANGFUSE_HOST) ?? DEFAULT_LANGFUSE_HOST).replace(/\/+$/, '');
+}
+
+export function promptVersionForSurface(surface) {
+  switch (surface) {
+  case 'ask':
+    return AI_PROMPT_VERSIONS.ask;
+  case 'workout':
+    return AI_PROMPT_VERSIONS.workout;
+  case 'cycle':
+    return AI_PROMPT_VERSIONS.cycle;
+  case 'vitals':
+    return AI_PROMPT_VERSIONS.vitals;
+  case 'checkpoint':
+    return AI_PROMPT_VERSIONS.checkpoint;
+  case 'weekly-checkin':
+    return AI_PROMPT_VERSIONS.weeklyCheckin;
+  default:
+    return 'unknown';
+  }
+}
+
+export function promptfooEvalMode(env = process.env) {
+  return envFlag('PROMPTFOO_LIVE', env) || envFlag('SUMMARY_EVALS_LIVE', env) ? 'live' : 'stored';
+}
+
+function stableScoreId({ runId, caseSet, caseId, provider, mode }) {
+  const key = [SCORE_NAME, runId, caseSet, caseId, provider, mode].join(':');
+  return `pf-${createHash('sha256').update(key).digest('hex').slice(0, 24)}`;
+}
+
+function stablePromptfooRunTraceId({ sessionId, runId, mode }) {
+  const key = ['promptfoo_run_trace', sessionId, runId, mode].join(':');
+  return createHash('sha256').update(key).digest('hex').slice(0, 32);
+}
+
+function scoreRunId(context = {}, env = process.env, now = new Date()) {
+  return context.evaluationId
+    ?? env.PROMPTFOO_RUN_ID
+    ?? env.GITHUB_RUN_ID
+    ?? env.CI_PIPELINE_ID
+    ?? `local-${now.toISOString().slice(0, 19)}`;
+}
+
+function providerLabel(context = {}) {
+  return context.provider?.label
+    ?? context.provider?.id
+    ?? context.vars?.provider
+    ?? 'incremnt-coach-current';
+}
+
+function modelLabel(testCase, context = {}, env = process.env) {
+  return context.providerResponse?.metadata?.model
+    ?? context.vars?.model
+    ?? testCase.metadata?.model
+    ?? env.SUMMARY_EVAL_MODEL
+    ?? env.OPENROUTER_MODEL
+    ?? 'current-chain';
+}
+
+function failedChecks(result) {
+  return (result.checks ?? []).filter((check) => !check.passed);
+}
+
+function langfuseScoreTarget({ testCase = {}, context = {}, sessionId }) {
+  const providerMetadata = context.providerResponse?.metadata ?? {};
+  const caseMetadata = testCase.metadata ?? {};
+  const traceId = firstString(
+    providerMetadata.langfuseTraceId,
+    providerMetadata.traceId,
+    caseMetadata.langfuseTraceId,
+    caseMetadata.traceId
+  );
+  const observationId = firstString(
+    providerMetadata.langfuseObservationId,
+    providerMetadata.observationId,
+    caseMetadata.langfuseObservationId,
+    caseMetadata.observationId
+  );
+
+  if (traceId) {
+    return compactObject({
+      traceId,
+      observationId
+    });
+  }
+
+  return { sessionId };
+}
+
+export function buildPromptfooLangfuseScorePayload({
+  result,
+  testCase,
+  context = {},
+  now = new Date(),
+  env = process.env
+}) {
+  const vars = context.vars ?? {};
+  const caseSet = vars.caseSet ?? testCase.caseSet ?? env.SUMMARY_EVAL_CASE_SET ?? 'synthetic';
+  const runId = scoreRunId(context, env, now);
+  const mode = promptfooEvalMode(env);
+  const provider = providerLabel(context);
+  const failed = failedChecks(result);
+  const promptVersion = vars.promptVersion
+    ?? testCase.metadata?.promptVersion
+    ?? promptVersionForSurface(result.surface);
+  const sessionId = `promptfoo:${runId}`;
+  const target = langfuseScoreTarget({ testCase, context, sessionId });
+  const targetKind = target.observationId ? 'observation' : target.traceId ? 'trace' : 'session';
+
+  return {
+    id: stableScoreId({ runId, caseSet, caseId: result.id, provider, mode }),
+    name: SCORE_NAME,
+    value: result.passed ? 1 : 0,
+    dataType: 'BOOLEAN',
+    ...target,
+    comment: result.passed
+      ? 'Promptfoo domain checks passed.'
+      : failed.map((check) => `${check.key}: ${check.reason}`).join(' | ').slice(0, 1000),
+    metadata: compactObject({
+      kind: 'promptfoo_eval',
+      caseId: result.id,
+      caseName: result.name,
+      caseSet,
+      fixtureFile: vars.fixtureFile ?? testCase.fixtureFile,
+      snapshotFile: vars.snapshotFile ?? testCase.snapshotFile,
+      surface: result.surface,
+      promptVersion,
+      model: modelLabel(testCase, context, env),
+      provider,
+      mode,
+      runId,
+      shouldPass: vars.shouldPass ?? testCase.shouldPass !== false,
+      source: testCase.source ?? 'fixture',
+      gitSha: currentGitSha(env),
+      passed: result.passed,
+      scoreTarget: targetKind,
+      promptfooSessionId: sessionId,
+      langfuseTraceId: target.traceId,
+      langfuseObservationId: target.observationId,
+      assertionCount: result.checks?.length ?? 0,
+      failedAssertionKeys: failed.map((check) => check.key),
+      failedAssertionReasons: failed.map((check) => check.reason).slice(0, 10),
+      generatedAt: now.toISOString()
+    }),
+    environment: env.LANGFUSE_ENVIRONMENT ?? env.NODE_ENV ?? 'development'
+  };
+}
+
+export function buildPromptfooLangfuseRunTracePayload(scorePayload, {
+  now = new Date(),
+  eventId = randomUUID()
+} = {}) {
+  if (!scorePayload?.sessionId) return null;
+
+  const metadata = scorePayload.metadata ?? {};
+  const runId = metadata.runId ?? String(scorePayload.sessionId).replace(/^promptfoo:/, '');
+  const traceId = stablePromptfooRunTraceId({
+    sessionId: scorePayload.sessionId,
+    runId,
+    mode: metadata.mode
+  });
+
+  return {
+    batch: [
+      {
+        type: 'trace-create',
+        id: eventId,
+        timestamp: now.toISOString(),
+        body: {
+          id: traceId,
+          timestamp: now.toISOString(),
+          name: 'promptfoo eval run',
+          sessionId: scorePayload.sessionId,
+          userId: 'promptfoo',
+          input: {
+            redacted: true,
+            source: 'promptfoo',
+            runId,
+            caseSet: metadata.caseSet,
+            mode: metadata.mode
+          },
+          output: {
+            redacted: true,
+            scoreName: scorePayload.name,
+            scoreTarget: metadata.scoreTarget
+          },
+          metadata: compactObject({
+            kind: 'promptfoo_eval_run',
+            runId,
+            caseSet: metadata.caseSet,
+            mode: metadata.mode,
+            provider: metadata.provider,
+            model: metadata.model,
+            promptfooSessionId: scorePayload.sessionId,
+            scoreName: scorePayload.name,
+            scoreTarget: metadata.scoreTarget,
+            gitSha: metadata.gitSha,
+            generatedAt: now.toISOString()
+          }),
+          tags: [
+            'source:promptfoo',
+            metadata.mode ? `mode:${metadata.mode}` : null,
+            metadata.caseSet ? `case-set:${metadata.caseSet}` : null
+          ].filter(Boolean),
+          environment: scorePayload.environment
+        }
+      }
+    ],
+    metadata: {
+      source: 'incremnt-promptfoo-langfuse-scores',
+      runId
+    }
+  };
+}
+
+function authHeader(env) {
+  return `Basic ${Buffer.from(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`).toString('base64')}`;
+}
+
+async function ensurePromptfooRunTrace(payload, { env, fetchImpl, now }) {
+  if (!payload.sessionId) return null;
+  const runTracePayload = buildPromptfooLangfuseRunTracePayload(payload, { now });
+  const runTraceId = runTracePayload.batch[0].body.id;
+
+  if (promptfooRunTracesCreated.has(runTraceId)) {
+    return { status: 'already-created', traceId: runTraceId };
+  }
+
+  const response = await fetchImpl(`${langfuseHost(env)}/api/public/ingestion`, {
+    method: 'POST',
+    headers: {
+      Authorization: authHeader(env),
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(runTracePayload)
+  });
+
+  let body = null;
+  let parseError = null;
+  let rawText = '';
+
+  if (typeof response?.json === 'function') {
+    try {
+      body = await response.json();
+    } catch (error) {
+      parseError = error;
+      rawText = typeof response?.text === 'function' ? await response.text() : '';
+    }
+  }
+
+  if (!response?.ok || parseError || (Array.isArray(body?.errors) && body.errors.length > 0)) {
+    const detail = body ? JSON.stringify(body) : rawText;
+    throw new Error(`Langfuse promptfoo run trace publish failed: ${response?.status ?? 'unknown'} ${detail}`.trim());
+  }
+
+  promptfooRunTracesCreated.add(runTraceId);
+  return { status: 'published', traceId: runTraceId };
+}
+
+export async function publishPromptfooLangfuseScore(input, {
+  env = process.env,
+  fetchImpl = globalThis.fetch,
+  logger = console,
+  now = new Date()
+} = {}) {
+  if (!shouldPublishPromptfooLangfuseScores(env)) {
+    return {
+      status: 'disabled',
+      reason: env.LANGFUSE_PUBLIC_KEY && env.LANGFUSE_SECRET_KEY ? 'disabled-by-env' : 'missing-credentials'
+    };
+  }
+
+  if (typeof fetchImpl !== 'function') {
+    return { status: 'failed', error: 'fetch is unavailable' };
+  }
+
+  const payload = buildPromptfooLangfuseScorePayload({ ...input, env, now });
+  try {
+    let runTrace = null;
+    try {
+      runTrace = await ensurePromptfooRunTrace(payload, { env, fetchImpl, now });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      logger?.warn?.('Promptfoo Langfuse run trace publish failed:', message);
+      runTrace = { status: 'failed', error: message };
+    }
+
+    const response = await fetchImpl(`${langfuseHost(env)}/api/public/scores`, {
+      method: 'POST',
+      headers: {
+        Authorization: authHeader(env),
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(payload)
+    });
+
+    if (!response?.ok) {
+      const body = typeof response?.text === 'function' ? await response.text() : '';
+      throw new Error(`Langfuse score publish failed: ${response?.status ?? 'unknown'} ${body}`.trim());
+    }
+
+    const data = typeof response.json === 'function' ? await response.json() : {};
+    return compactObject({
+      status: 'published',
+      scoreId: data.id,
+      sessionId: payload.sessionId,
+      runTraceId: runTrace?.traceId,
+      runTraceStatus: runTrace?.status
+    });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger?.warn?.('Promptfoo Langfuse score publish skipped:', message);
+    return { status: 'failed', error: message };
+  }
+}

package/src/promptfoo-provider.cjs

@@ -0,0 +1,14 @@
+module.exports = class IncremntCoachProvider {
+  constructor(options = {}) {
+    this.providerId = options.id || 'incremnt-coach';
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt, context) {
+    const { callPromptfooProvider } = await import('./promptfoo-evals.js');
+    return callPromptfooProvider(prompt, context);
+  }
+};

package/src/promptfoo-tests.cjs

@@ -0,0 +1,4 @@
+module.exports = async function promptfooTests() {
+  const { buildPromptfooTests } = await import('./promptfoo-evals.js');
+  return buildPromptfooTests();
+};