@agent-native/core 0.113.0 → 0.114.0

package/src/observability/experiments.ts

@@ -0,0 +1,359 @@
+import { getDbExec } from "../db/client.js";
+import {
+  insertExperiment,
+  updateExperiment,
+  listExperiments,
+  getExperiment,
+  upsertAssignment,
+  getAssignment,
+  insertExperimentResult,
+  ensureObservabilityTables,
+} from "./store.js";
+import type {
+  Experiment,
+  ExperimentVariant,
+  ExperimentMetricResult,
+} from "./types.js";
+
+// ─── Hashing ────────────────────────────────────────────────────────
+
+function simpleHash(str: string): number {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
+  }
+  return Math.abs(hash);
+}
+
+function generateId(prefix: string): string {
+  return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+// ─── Active experiments cache (short TTL for hot path) ──────────────
+
+let _cachedActive: Experiment[] | null = null;
+let _cachedActiveAt = 0;
+const CACHE_TTL_MS = 5_000;
+
+async function getActiveExperiments(): Promise<Experiment[]> {
+  const now = Date.now();
+  if (_cachedActive && now - _cachedActiveAt < CACHE_TTL_MS) {
+    return _cachedActive;
+  }
+  const all = await listExperiments();
+  _cachedActive = all.filter((e) => e.status === "running");
+  _cachedActiveAt = now;
+  return _cachedActive;
+}
+
+function invalidateCache(): void {
+  _cachedActive = null;
+  _cachedActiveAt = 0;
+}
+
+// ─── Experiment lifecycle ───────────────────────────────────────────
+
+export async function createExperiment(opts: {
+  name: string;
+  variants: Array<{
+    id: string;
+    weight: number;
+    config: Record<string, unknown>;
+  }>;
+  metrics: string[];
+  assignmentLevel?: "user" | "session";
+}): Promise<Experiment> {
+  const experiment: Experiment = {
+    id: generateId("exp"),
+    name: opts.name,
+    status: "draft",
+    variants: opts.variants,
+    metrics: opts.metrics,
+    assignmentLevel: opts.assignmentLevel ?? "user",
+    startedAt: null,
+    endedAt: null,
+    createdAt: Date.now(),
+  };
+  await insertExperiment(experiment);
+  return experiment;
+}
+
+export async function startExperiment(id: string): Promise<void> {
+  await updateExperiment(id, { status: "running" });
+  invalidateCache();
+}
+
+export async function pauseExperiment(id: string): Promise<void> {
+  await updateExperiment(id, { status: "paused" });
+  invalidateCache();
+}
+
+export async function completeExperiment(id: string): Promise<void> {
+  await updateExperiment(id, { status: "completed", endedAt: Date.now() });
+  invalidateCache();
+}
+
+// ─── Variant assignment ─────────────────────────────────────────────
+
+export async function resolveVariant(
+  experimentId: string,
+  userId: string,
+): Promise<ExperimentVariant> {
+  const existing = await getAssignment(experimentId, userId);
+  if (existing) {
+    const experiment = await getExperiment(experimentId);
+    if (!experiment) throw new Error(`Experiment ${experimentId} not found`);
+    const variant = experiment.variants.find(
+      (v) => v.id === existing.variantId,
+    );
+    if (!variant)
+      throw new Error(
+        `Variant ${existing.variantId} not found in experiment ${experimentId}`,
+      );
+    return variant;
+  }
+
+  const experiment = await getExperiment(experimentId);
+  if (!experiment) throw new Error(`Experiment ${experimentId} not found`);
+
+  if (experiment.variants.length === 0)
+    throw new Error("Experiment has no variants");
+  const totalWeight = experiment.variants.reduce((sum, v) => sum + v.weight, 0);
+  if (totalWeight <= 0)
+    throw new Error("Experiment has no valid variant weights");
+  const hashValue = simpleHash(experimentId + userId) % totalWeight;
+
+  let cumulative = 0;
+  let chosen: ExperimentVariant | undefined;
+  for (const variant of experiment.variants) {
+    cumulative += variant.weight;
+    if (hashValue < cumulative) {
+      chosen = variant;
+      break;
+    }
+  }
+  // Fallback to last variant if rounding causes no match
+  if (!chosen) chosen = experiment.variants[experiment.variants.length - 1];
+
+  // Fire-and-forget persistence
+  upsertAssignment({
+    experimentId,
+    userId,
+    variantId: chosen.id,
+    assignedAt: Date.now(),
+  }).catch(() => {});
+
+  return chosen;
+}
+
+export async function resolveActiveExperimentConfig(userId: string): Promise<{
+  configs: Record<string, unknown>;
+  assignments: Array<{ experimentId: string; variantId: string }>;
+} | null> {
+  const active = await getActiveExperiments();
+  if (active.length === 0) return null;
+
+  const assignments: Array<{ experimentId: string; variantId: string }> = [];
+  const merged: Record<string, unknown> = {};
+
+  const variants = await Promise.all(
+    active.map((exp) => resolveVariant(exp.id, userId)),
+  );
+
+  for (let i = 0; i < active.length; i++) {
+    const exp = active[i];
+    const variant = variants[i];
+    assignments.push({ experimentId: exp.id, variantId: variant.id });
+    Object.assign(merged, variant.config);
+  }
+
+  return { configs: merged, assignments };
+}
+
+// ─── Results computation ────────────────────────────────────────────
+
+export async function computeExperimentResults(
+  experimentId: string,
+): Promise<ExperimentMetricResult[]> {
+  const experiment = await getExperiment(experimentId);
+  if (!experiment) throw new Error(`Experiment ${experimentId} not found`);
+
+  await ensureObservabilityTables();
+  const client = getDbExec();
+
+  const results: ExperimentMetricResult[] = [];
+  const now = Date.now();
+
+  for (const variant of experiment.variants) {
+    const { rows: assignmentRows } = await client.execute({
+      sql: `SELECT user_id FROM agent_experiment_assignments WHERE experiment_id = ? AND variant_id = ?`,
+      args: [experimentId, variant.id],
+    });
+
+    if (assignmentRows.length === 0) {
+      const emptyMetrics = [
+        "avg_cost",
+        "avg_latency",
+        "avg_eval_score",
+        "tool_success_rate",
+        "satisfaction",
+        "sample_size",
+      ];
+      for (const metric of emptyMetrics) {
+        const result: ExperimentMetricResult = {
+          id: generateId("expres"),
+          experimentId,
+          variantId: variant.id,
+          metric,
+          value: 0,
+          sampleSize: 0,
+          confidenceLow: 0,
+          confidenceHigh: 0,
+          computedAt: now,
+        };
+        results.push(result);
+        insertExperimentResult(result).catch(() => {});
+      }
+      continue;
+    }
+
+    const userIds = assignmentRows.map((r: any) => String(r.user_id));
+    const placeholders = userIds.map(() => "?").join(", ");
+
+    // Scope runs to this variant's assigned users via user_id on the summary.
+    // Previously used INNER JOIN agent_feedback which excluded runs without
+    // any feedback — silently underreporting cost/latency/tool metrics.
+    const { rows: userTraceRows } = await client.execute({
+      sql: `SELECT s.total_cost_cents_x100, s.total_duration_ms, s.successful_tools, s.tool_calls, s.run_id
+            FROM agent_trace_summaries s
+            WHERE s.user_id IN (${placeholders})
+            ${experiment.startedAt ? "AND s.created_at >= ?" : ""}`,
+      args: experiment.startedAt ? [...userIds, experiment.startedAt] : userIds,
+    });
+
+    const costs: number[] = [];
+    const latencies: number[] = [];
+    const toolRates: number[] = [];
+
+    for (const row of userTraceRows as any[]) {
+      costs.push(Number(row.total_cost_cents_x100) / 100);
+      latencies.push(Number(row.total_duration_ms));
+      const totalTools = Number(row.tool_calls);
+      const successTools = Number(row.successful_tools);
+      toolRates.push(totalTools > 0 ? successTools / totalTools : 1);
+    }
+
+    // Eval scores for these runs
+    const runIds = (userTraceRows as any[]).map((r) => String(r.run_id));
+    let evalScores: number[] = [];
+    if (runIds.length > 0) {
+      const runPlaceholders = runIds.map(() => "?").join(", ");
+      const { rows: evalRows } = await client.execute({
+        sql: `SELECT score FROM agent_evals WHERE run_id IN (${runPlaceholders})`,
+        args: runIds,
+      });
+      evalScores = (evalRows as any[]).map((r) => Number(r.score));
+    }
+
+    // Satisfaction scores (inverse of frustration) for these users' threads
+    const { rows: satRows } = await client.execute({
+      sql: `SELECT frustration_score FROM agent_satisfaction_scores
+            WHERE thread_id IN (
+              SELECT DISTINCT f.thread_id FROM agent_feedback f
+              WHERE f.user_id IN (${placeholders}) AND f.thread_id IS NOT NULL
+            )
+            ${experiment.startedAt ? "AND computed_at >= ?" : ""}`,
+      args: experiment.startedAt ? [...userIds, experiment.startedAt] : userIds,
+    });
+    const satisfactionScores = (satRows as any[]).map(
+      (r) => 1 - Number(r.frustration_score) / 100,
+    );
+
+    const sampleSize = userTraceRows.length;
+
+    const metricEntries: Array<{
+      metric: string;
+      value: number;
+      std: number;
+      n: number;
+    }> = [
+      {
+        metric: "avg_cost",
+        value: mean(costs),
+        std: stddev(costs),
+        n: costs.length,
+      },
+      {
+        metric: "avg_latency",
+        value: mean(latencies),
+        std: stddev(latencies),
+        n: latencies.length,
+      },
+      {
+        metric: "avg_eval_score",
+        value: mean(evalScores),
+        std: stddev(evalScores),
+        n: evalScores.length,
+      },
+      {
+        metric: "tool_success_rate",
+        value: mean(toolRates),
+        std: stddev(toolRates),
+        n: toolRates.length,
+      },
+      {
+        metric: "satisfaction",
+        value: mean(satisfactionScores),
+        std: stddev(satisfactionScores),
+        n: satisfactionScores.length,
+      },
+      { metric: "sample_size", value: sampleSize, std: 0, n: sampleSize },
+    ];
+
+    for (const entry of metricEntries) {
+      const [low, high] = confidenceInterval(entry.value, entry.std, entry.n);
+      const result: ExperimentMetricResult = {
+        id: generateId("expres"),
+        experimentId,
+        variantId: variant.id,
+        metric: entry.metric,
+        value: entry.value,
+        sampleSize,
+        confidenceLow: low,
+        confidenceHigh: high,
+        computedAt: now,
+      };
+      results.push(result);
+      insertExperimentResult(result).catch(() => {});
+    }
+  }
+
+  return results;
+}
+
+// ─── Stats helpers ──────────────────────────────────────────────────
+
+function mean(values: number[]): number {
+  if (values.length === 0) return 0;
+  return values.reduce((a, b) => a + b, 0) / values.length;
+}
+
+function stddev(values: number[]): number {
+  if (values.length < 2) return 0;
+  const avg = mean(values);
+  const squaredDiffs = values.map((v) => (v - avg) ** 2);
+  return Math.sqrt(
+    squaredDiffs.reduce((a, b) => a + b, 0) / (values.length - 1),
+  );
+}
+
+function confidenceInterval(
+  avg: number,
+  std: number,
+  n: number,
+): [low: number, high: number] {
+  if (n < 2) return [avg, avg];
+  const se = std / Math.sqrt(n);
+  const margin = 1.96 * se;
+  return [avg - margin, avg + margin];
+}

package/src/observability/feedback.ts

@@ -0,0 +1,332 @@
+import { getDbExec } from "../db/client.js";
+import {
+  insertFeedback,
+  upsertSatisfactionScore,
+  ensureObservabilityTables,
+} from "./store.js";
+import type {
+  FeedbackEntry,
+  FeedbackType,
+  SatisfactionScore,
+} from "./types.js";
+
+function generateId(): string {
+  return `fb-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+// ─── Feedback submission ────────────────────────────────────────────
+
+export interface SubmitFeedbackOpts {
+  threadId: string;
+  runId?: string;
+  messageSeq?: number;
+  feedbackType: FeedbackType;
+  value?: string;
+  userId?: string;
+}
+
+export async function submitFeedback(
+  opts: SubmitFeedbackOpts,
+): Promise<FeedbackEntry> {
+  if (!opts.threadId) throw new Error("threadId is required");
+  if (!opts.feedbackType) throw new Error("feedbackType is required");
+
+  const validTypes: FeedbackType[] = [
+    "thumbs_up",
+    "thumbs_down",
+    "category",
+    "text",
+  ];
+  if (!validTypes.includes(opts.feedbackType)) {
+    throw new Error(`Invalid feedbackType: ${opts.feedbackType}`);
+  }
+
+  const entry: FeedbackEntry = {
+    id: generateId(),
+    runId: opts.runId ?? null,
+    threadId: opts.threadId,
+    messageSeq: opts.messageSeq ?? null,
+    feedbackType: opts.feedbackType,
+    value: opts.value ?? "",
+    userId: opts.userId ?? null,
+    createdAt: Date.now(),
+  };
+
+  await insertFeedback(entry);
+  return entry;
+}
+
+// ─── Satisfaction scoring ───────────────────────────────────────────
+
+interface ThreadMessage {
+  role: "user" | "assistant";
+  content: string;
+  createdAt?: number;
+}
+
+async function getThreadMessages(threadId: string): Promise<ThreadMessage[]> {
+  await ensureObservabilityTables();
+  const client = getDbExec();
+
+  const { rows } = await client.execute({
+    sql: `SELECT thread_data FROM chat_threads WHERE id = ?`,
+    args: [threadId],
+  });
+
+  if (rows.length === 0) return [];
+
+  const raw = (rows[0] as Record<string, unknown>).thread_data;
+  if (!raw) return [];
+
+  try {
+    const data = JSON.parse(String(raw));
+    const messages: unknown[] = data.messages ?? data;
+    if (!Array.isArray(messages)) return [];
+
+    return messages
+      .filter(
+        (m: any) =>
+          m &&
+          typeof m.role === "string" &&
+          (typeof m.content === "string" ||
+            (Array.isArray(m.content) &&
+              m.content.some((p: any) => p.type === "text"))),
+      )
+      .map((m: any) => ({
+        role: m.role as "user" | "assistant",
+        content:
+          typeof m.content === "string"
+            ? m.content
+            : (m.content as any[])
+                .filter((p: any) => p.type === "text")
+                .map((p: any) => p.text ?? "")
+                .join(""),
+        createdAt: m.createdAt ? Number(m.createdAt) : undefined,
+      }));
+  } catch {
+    return [];
+  }
+}
+
+function tokenize(text: string): Set<string> {
+  return new Set(
+    text
+      .toLowerCase()
+      .replace(/[^a-z0-9\s]/g, "")
+      .split(/\s+/)
+      .filter((w) => w.length > 1),
+  );
+}
+
+function jaccardSimilarity(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 && b.size === 0) return 0;
+  let intersection = 0;
+  for (const word of a) {
+    if (b.has(word)) intersection++;
+  }
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+function computeRephrasingScore(userMessages: string[]): number {
+  if (userMessages.length < 2) return 0;
+
+  const tokenSets = userMessages.map(tokenize);
+  let maxConsecutiveSimilarity = 0;
+  let highSimilarityCount = 0;
+
+  for (let i = 1; i < tokenSets.length; i++) {
+    const sim = jaccardSimilarity(tokenSets[i - 1], tokenSets[i]);
+    if (sim > maxConsecutiveSimilarity) maxConsecutiveSimilarity = sim;
+    if (sim >= 0.4) highSimilarityCount++;
+  }
+
+  const pairCount = tokenSets.length - 1;
+  const rephrasingRatio = pairCount > 0 ? highSimilarityCount / pairCount : 0;
+
+  // Blend peak similarity with overall rephrasing frequency
+  return Math.min(
+    100,
+    ((maxConsecutiveSimilarity * 60 + rephrasingRatio * 40) * 100) / 100,
+  );
+}
+
+function computeAbandonmentScore(messages: ThreadMessage[]): number {
+  if (messages.length === 0) return 0;
+
+  const last = messages[messages.length - 1];
+
+  // Thread ends with a user message and no agent response
+  if (last.role === "user") return 80;
+
+  // Thread ends with agent response, but check if last user message
+  // was very close to it (agent responded but user never replied back)
+  if (messages.length >= 3) {
+    const secondToLast = messages[messages.length - 2];
+    if (secondToLast.role === "user") {
+      const userMsg = secondToLast.content.trim();
+      // Short user messages right before end suggest giving up
+      if (userMsg.length < 15) return 40;
+    }
+  }
+
+  return 0;
+}
+
+const NEGATIVE_PATTERNS = [
+  /\bno\b/i,
+  /\bwrong\b/i,
+  /\bnot what i/i,
+  /\btry again\b/i,
+  /\bnever mind\b/i,
+  /\bnevermind\b/i,
+  /\bthat's not\b/i,
+  /\bthats not\b/i,
+  /\bincorrect\b/i,
+  /\bdoesn't work\b/i,
+  /\bdoesnt work\b/i,
+  /\bstill wrong\b/i,
+  /\bnope\b/i,
+  /\bstop\b/i,
+  /\bforget it\b/i,
+  /\buseless\b/i,
+  /\bbroken\b/i,
+];
+
+function computeSentimentScore(userMessages: string[]): number {
+  if (userMessages.length === 0) return 0;
+
+  let negativeCount = 0;
+  let terseCount = 0;
+
+  for (const msg of userMessages) {
+    const trimmed = msg.trim();
+
+    // Terse single-word or very short responses
+    if (trimmed.split(/\s+/).length <= 2 && trimmed.length < 20) {
+      terseCount++;
+    }
+
+    for (const pattern of NEGATIVE_PATTERNS) {
+      if (pattern.test(trimmed)) {
+        negativeCount++;
+        break;
+      }
+    }
+  }
+
+  const negativeRatio = negativeCount / userMessages.length;
+  const terseRatio = terseCount / userMessages.length;
+
+  // negativeRatio/terseRatio are already in [0,1], so the weighted sum is in
+  // [0,100] — it must NOT be multiplied by another 100 (that would saturate
+  // sentiment to 100 the moment a single message matched any negative pattern).
+  return Math.min(100, negativeRatio * 70 + terseRatio * 30);
+}
+
+function computeLengthTrendScore(userMessages: string[]): number {
+  if (userMessages.length < 3) return 0;
+
+  const lengths = userMessages.map((m) => m.trim().length);
+  const n = lengths.length;
+
+  // Simple linear regression: y = mx + b, we care about slope m
+  const xMean = (n - 1) / 2;
+  const yMean = lengths.reduce((a, b) => a + b, 0) / n;
+
+  let numerator = 0;
+  let denominator = 0;
+  for (let i = 0; i < n; i++) {
+    const xDiff = i - xMean;
+    numerator += xDiff * (lengths[i] - yMean);
+    denominator += xDiff * xDiff;
+  }
+
+  if (denominator === 0) return 0;
+
+  const slope = numerator / denominator;
+
+  // Normalize: negative slope = messages getting shorter = frustration
+  // Scale by average length to get a relative measure
+  if (yMean === 0) return 0;
+  const normalizedSlope = slope / yMean;
+
+  // Only negative slopes (shrinking messages) contribute to frustration
+  if (normalizedSlope >= 0) return 0;
+
+  // Map normalized slope to 0-100; -1 (halving each message) = 100
+  return Math.min(100, Math.abs(normalizedSlope) * 100);
+}
+
+const RETRY_PATTERNS = [
+  /\btry again\b/i,
+  /\bthat's wrong\b/i,
+  /\bthats wrong\b/i,
+  /\bno,?\s*(that's|thats|it's|its)\b/i,
+  /\bredo\b/i,
+  /\bdo it again\b/i,
+  /\bone more time\b/i,
+  /\bregenerate\b/i,
+  /\bfix (it|this|that)\b/i,
+  /\btry (this|that) instead\b/i,
+  /\bi (said|meant|asked)\b/i,
+  /\bstill (not|wrong|broken|doesn't|doesnt)\b/i,
+];
+
+function computeRetryScore(userMessages: string[]): number {
+  if (userMessages.length === 0) return 0;
+
+  let retryCount = 0;
+  for (const msg of userMessages) {
+    for (const pattern of RETRY_PATTERNS) {
+      if (pattern.test(msg)) {
+        retryCount++;
+        break;
+      }
+    }
+  }
+
+  const retryRatio = retryCount / userMessages.length;
+  return Math.min(100, retryRatio * 150);
+}
+
+export async function computeSatisfactionScore(
+  threadId: string,
+  opts: { userId?: string | null } = {},
+): Promise<SatisfactionScore> {
+  const messages = await getThreadMessages(threadId);
+  const userMessages = messages
+    .filter((m) => m.role === "user")
+    .map((m) => m.content);
+
+  const rephrasingScore = computeRephrasingScore(userMessages);
+  const abandonmentScore = computeAbandonmentScore(messages);
+  const sentimentScore = computeSentimentScore(userMessages);
+  const lengthTrendScore = computeLengthTrendScore(userMessages);
+  const retryScore = computeRetryScore(userMessages);
+
+  // Weighted composite: rephrasing 30, abandonment 20, sentiment 15, length trend 15, retry 20
+  const frustrationScore = Math.min(
+    100,
+    rephrasingScore * 0.3 +
+      abandonmentScore * 0.2 +
+      sentimentScore * 0.15 +
+      lengthTrendScore * 0.15 +
+      retryScore * 0.2,
+  );
+
+  const score: SatisfactionScore = {
+    id: `sat-${threadId}`,
+    threadId,
+    userId: opts.userId ?? null,
+    frustrationScore: Math.round(frustrationScore * 100) / 100,
+    rephrasingScore: Math.round(rephrasingScore * 100) / 100,
+    abandonmentScore: Math.round(abandonmentScore * 100) / 100,
+    sentimentScore: Math.round(sentimentScore * 100) / 100,
+    lengthTrendScore: Math.round(lengthTrendScore * 100) / 100,
+    computedAt: Date.now(),
+  };
+
+  await upsertSatisfactionScore(score);
+  return score;
+}