behavior-wrapped 0.2.12 → 0.2.14

package/scripts/compare-free-judges.mjs

@@ -1,162 +0,0 @@
-#!/usr/bin/env node
-import { discoverAllSessionsAsync, readRecordsAsync, sessionsInDefaultWindow } from "../server/discovery.mjs";
-import { buildPhraseCandidates, judgePhraseCard } from "../server/phrase-card.mjs";
-import { buildInteractionToneCandidates, judgeInteractionTone } from "../server/interaction-tone.mjs";
-import { buildSessionTopicCandidates, judgeSessionTopics } from "../server/session-topics.mjs";
-import { buildWorkaroundTrajectories, judgeWorkarounds } from "../server/instrumental-workarounds.mjs";
-
-const apiKey = process.env.OPENROUTER_API_KEY;
-if (!apiKey) throw new Error("OPENROUTER_API_KEY is required.");
-
-const configuredModels = [
-  { label: "Gemma 4 31B", id: "google/gemma-4-31b-it:free", reasoningEffort: "none" },
-  { label: "GPT-OSS 20B", id: "openai/gpt-oss-20b:free", reasoningEffort: "low", minimumMaxTokens: 16_384 },
-  { label: "GPT-5.6 Luna (ZDR)", id: "openai/gpt-5.6-luna", reasoningEffort: "none", zdr: true },
-];
-const models = process.env.COMPARE_MODEL
-  ? configuredModels.filter((model) => model.id.includes(process.env.COMPARE_MODEL))
-  : configuredModels;
-if (!models.length) throw new Error(`No configured model matched COMPARE_MODEL=${process.env.COMPARE_MODEL}.`);
-
-const providersSeen = new Set();
-const usageSeen = [];
-function privacyFilteredFetch(model) {
-  return async (url, options = {}) => {
-    const requestBody = JSON.parse(options.body);
-    requestBody.provider = { ...(requestBody.provider || {}), data_collection: "deny", ...(model.zdr ? { zdr: true } : {}) };
-    requestBody.reasoning = { ...(requestBody.reasoning || {}), effort: model.reasoningEffort, exclude: true };
-    requestBody.max_tokens = Math.max(Number(requestBody.max_tokens) || 0, model.minimumMaxTokens || 0);
-    let response;
-    for (let attempt = 0; attempt < 4; attempt++) {
-      response = await fetch(url, { ...options, body: JSON.stringify(requestBody) });
-      if (response.status !== 429 || attempt === 3) break;
-      const waitMs = 5_000 * (2 ** attempt);
-      process.stdout.write(`capacity retry in ${waitMs / 1000}s... `);
-      await new Promise((resolve) => setTimeout(resolve, waitMs));
-    }
-    try {
-      const responseBody = await response.clone().json();
-      if (responseBody?.provider) providersSeen.add(responseBody.provider);
-      if (responseBody?.usage) usageSeen.push({
-        provider: responseBody.provider || null,
-        promptTokens: Number(responseBody.usage.prompt_tokens) || 0,
-        completionTokens: Number(responseBody.usage.completion_tokens) || 0,
-        totalTokens: Number(responseBody.usage.total_tokens) || 0,
-        costUsd: Number(responseBody.usage.cost) || 0,
-      });
-    } catch { /* The judge functions report malformed responses. */ }
-    return response;
-  };
-}
-
-function errorSummary(error) {
-  return {
-    error: error?.message || String(error),
-    ...(error?.judgeDetails?.http_status ? { status: error.judgeDetails.http_status } : {}),
-    ...(error?.judgeDetails?.upstream_code ? { code: error.judgeDetails.upstream_code } : {}),
-  };
-}
-
-async function attempt(label, operation) {
-  process.stdout.write(`  ${label}... `);
-  const startedAt = Date.now();
-  try {
-    const value = await operation();
-    console.log(`ok (${((Date.now() - startedAt) / 1000).toFixed(1)}s)`);
-    return { ok: true, value };
-  } catch (error) {
-    console.log(`failed (${((Date.now() - startedAt) / 1000).toFixed(1)}s)`);
-    return { ok: false, ...errorSummary(error) };
-  }
-}
-
-function summarize(result) {
-  const phrase = result.phrase?.ok ? {
-    phrase: result.phrase.value.phrase,
-    occurrences: result.phrase.value.occurrences,
-    latencyMs: result.phrase.value.latencyMs,
-  } : result.phrase;
-  const tone = result.tone?.ok ? {
-    frustratedMessages: result.tone.value.frustratedMessages,
-    gratefulMessages: result.tone.value.gratefulMessages,
-    frustrationQuote: result.tone.value.frustrationQuote,
-    latencyMs: result.tone.value.latencyMs,
-  } : result.tone;
-  const topics = result.topics?.ok ? {
-    topics: result.topics.value.topics.map(({ topic, percentage }) => ({ topic, percentage })),
-    latencyMs: result.topics.value.latencyMs,
-  } : result.topics;
-  const workarounds = result.workarounds?.ok ? {
-    confirmed: result.workarounds.value.card.count,
-    borderline: result.workarounds.value.review.borderline.length,
-    examples: result.workarounds.value.review.occurrences.map(({ summary, confidence, disclosure }) => ({ summary, confidence, disclosure })),
-    latencyMs: result.workarounds.value.review.latencyMs,
-  } : result.workarounds;
-  return { phrase, tone, topics, workarounds };
-}
-
-function summarizeUsage(records) {
-  return records.reduce((total, record) => ({
-    requests: total.requests + 1,
-    promptTokens: total.promptTokens + record.promptTokens,
-    completionTokens: total.completionTokens + record.completionTokens,
-    totalTokens: total.totalTokens + record.totalTokens,
-    costUsd: Number((total.costUsd + record.costUsd).toFixed(12)),
-  }), { requests: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0 });
-}
-
-console.log("Preparing the same 30-day, locally redacted Behavior Wrapped input for both models...");
-const catalog = await discoverAllSessionsAsync();
-const sessions = sessionsInDefaultWindow(catalog.sessions, { days: 30 });
-const records = [];
-for (const publicSession of sessions) {
-  const session = catalog.index.get(publicSession.id);
-  records.push({ sessionId: session.id, agent: session.agent, records: await readRecordsAsync(session.file, session.agent) });
-}
-
-const phraseCandidates = buildPhraseCandidates(records, { maximumCandidates: 100 });
-const toneCandidates = buildInteractionToneCandidates(records);
-const topicBundle = buildSessionTopicCandidates(records);
-const workaroundBundle = buildWorkaroundTrajectories(records);
-console.log(`Snapshot: ${sessions.length} sessions; ${phraseCandidates.length} phrase candidates; ${toneCandidates.length} tone candidates; ${topicBundle.candidates.length} topic candidates; ${workaroundBundle.chunks.length} workaround chunks.`);
-
-const comparison = {};
-for (const model of models) {
-  providersSeen.clear();
-  usageSeen.length = 0;
-  console.log(`\n${model.label} (${model.id})`);
-  const options = { model: model.id, fetchImpl: privacyFilteredFetch(model) };
-  const phrase = phraseCandidates.length
-    ? await attempt("favorite phrase", () => judgePhraseCard(phraseCandidates, apiKey, options))
-    : { ok: false, error: "No candidates" };
-  const tone = toneCandidates.length
-    ? await attempt("interaction tone", () => judgeInteractionTone(toneCandidates, apiKey, options))
-    : { ok: false, error: "No candidates" };
-  const topics = topicBundle.candidates.length
-    ? await attempt("session topics", () => judgeSessionTopics(topicBundle, apiKey, options))
-    : { ok: false, error: "No candidates" };
-  const workarounds = workaroundBundle.chunks.length
-    ? await attempt("workarounds", () => judgeWorkarounds(workaroundBundle, apiKey, {
-      ...options,
-      onProgress: ({ index, total }) => process.stdout.write(index === 1 ? `[${index}/${total}] ` : `${index}/${total} `),
-    }))
-    : { ok: false, error: "No candidates" };
-  comparison[model.label] = {
-    providers: [...providersSeen],
-    privacy: { dataCollectionDenied: true, zeroDataRetentionRequired: Boolean(model.zdr) },
-    usage: summarizeUsage(usageSeen),
-    ...summarize({ phrase, tone, topics, workarounds }),
-  };
-}
-
-console.log("\nCOMPARISON_JSON");
-console.log(JSON.stringify({
-  input: {
-    sessions: sessions.length,
-    phraseCandidates: phraseCandidates.length,
-    toneCandidates: toneCandidates.length,
-    topicCandidates: topicBundle.candidates.length,
-    workaroundChunks: workaroundBundle.chunks.length,
-  },
-  models: comparison,
-}, null, 2));