@mrclrchtr/supi-insights 0.1.0

package/src/aggregator.ts

@@ -0,0 +1,245 @@
+// Data aggregator — combine session metadata and facets into aggregated statistics.
+
+import type { AggregatedData, SessionFacets, SessionMeta } from "./types.ts";
+
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: aggregation intentionally combines many independent report counters.
+// biome-ignore lint/complexity/noExcessiveLinesPerFunction: keeping the aggregation flow together makes metric interactions easier to audit.
+export function aggregateData(
+  sessions: SessionMeta[],
+  facets: Map<string, SessionFacets>,
+): AggregatedData {
+  const result: AggregatedData = {
+    totalSessions: sessions.length,
+    sessionsWithFacets: facets.size,
+    dateRange: { start: "", end: "" },
+    totalMessages: 0,
+    totalDurationHours: 0,
+    totalInputTokens: 0,
+    totalOutputTokens: 0,
+    toolCounts: {},
+    languages: {},
+    gitCommits: 0,
+    gitPushes: 0,
+    projects: {},
+    goalCategories: {},
+    outcomes: {},
+    satisfaction: {},
+    helpfulness: {},
+    sessionTypes: {},
+    friction: {},
+    success: {},
+    sessionSummaries: [],
+    totalInterruptions: 0,
+    totalToolErrors: 0,
+    toolErrorCategories: {},
+    userResponseTimes: [],
+    medianResponseTime: 0,
+    avgResponseTime: 0,
+    sessionsUsingTaskAgent: 0,
+    sessionsUsingMcp: 0,
+    sessionsUsingWebSearch: 0,
+    sessionsUsingWebFetch: 0,
+    totalLinesAdded: 0,
+    totalLinesRemoved: 0,
+    totalFilesModified: 0,
+    daysActive: 0,
+    messagesPerDay: 0,
+    messageHours: [],
+    multiClauding: {
+      overlapEvents: 0,
+      sessionsInvolved: 0,
+      userMessagesDuring: 0,
+    },
+    facetExtractionAttempted: 0,
+    facetExtractionFailed: 0,
+    insightSectionsFailed: [],
+  };
+
+  const dates: string[] = [];
+  const allResponseTimes: number[] = [];
+  const allMessageHours: number[] = [];
+
+  for (const session of sessions) {
+    dates.push(session.startTime);
+    result.totalMessages += session.userMessageCount;
+    result.totalDurationHours += session.durationMinutes / 60;
+    result.totalInputTokens += session.inputTokens;
+    result.totalOutputTokens += session.outputTokens;
+    result.gitCommits += session.gitCommits;
+    result.gitPushes += session.gitPushes;
+
+    result.totalInterruptions += session.userInterruptions;
+    result.totalToolErrors += session.toolErrors;
+    for (const [cat, count] of Object.entries(session.toolErrorCategories)) {
+      result.toolErrorCategories[cat] = (result.toolErrorCategories[cat] ?? 0) + count;
+    }
+    allResponseTimes.push(...session.userResponseTimes);
+    if (session.usesTaskAgent) result.sessionsUsingTaskAgent++;
+    if (session.usesMcp) result.sessionsUsingMcp++;
+    if (session.usesWebSearch) result.sessionsUsingWebSearch++;
+    if (session.usesWebFetch) result.sessionsUsingWebFetch++;
+
+    result.totalLinesAdded += session.linesAdded;
+    result.totalLinesRemoved += session.linesRemoved;
+    result.totalFilesModified += session.filesModified;
+    allMessageHours.push(...session.messageHours);
+
+    for (const [tool, count] of Object.entries(session.toolCounts)) {
+      result.toolCounts[tool] = (result.toolCounts[tool] ?? 0) + count;
+    }
+
+    for (const [lang, count] of Object.entries(session.languages)) {
+      result.languages[lang] = (result.languages[lang] ?? 0) + count;
+    }
+
+    if (session.projectPath) {
+      result.projects[session.projectPath] = (result.projects[session.projectPath] ?? 0) + 1;
+    }
+
+    const sessionFacets = facets.get(session.sessionId);
+    if (sessionFacets) {
+      for (const [cat, count] of safeEntries(sessionFacets.goalCategories)) {
+        if (count > 0) {
+          result.goalCategories[cat] = (result.goalCategories[cat] ?? 0) + count;
+        }
+      }
+
+      result.outcomes[sessionFacets.outcome] = (result.outcomes[sessionFacets.outcome] ?? 0) + 1;
+
+      for (const [level, count] of safeEntries(sessionFacets.userSatisfactionCounts)) {
+        if (count > 0) {
+          result.satisfaction[level] = (result.satisfaction[level] ?? 0) + count;
+        }
+      }
+
+      result.helpfulness[sessionFacets.claudeHelpfulness] =
+        (result.helpfulness[sessionFacets.claudeHelpfulness] ?? 0) + 1;
+
+      result.sessionTypes[sessionFacets.sessionType] =
+        (result.sessionTypes[sessionFacets.sessionType] ?? 0) + 1;
+
+      for (const [type, count] of safeEntries(sessionFacets.frictionCounts)) {
+        if (count > 0) {
+          result.friction[type] = (result.friction[type] ?? 0) + count;
+        }
+      }
+
+      if (sessionFacets.primarySuccess !== "none") {
+        result.success[sessionFacets.primarySuccess] =
+          (result.success[sessionFacets.primarySuccess] ?? 0) + 1;
+      }
+    }
+
+    if (result.sessionSummaries.length < 50) {
+      result.sessionSummaries.push({
+        id: session.sessionId.slice(0, 8),
+        date: session.startTime.split("T")[0] ?? "",
+        summary: session.summary ?? session.firstPrompt.slice(0, 100),
+        goal: sessionFacets?.underlyingGoal,
+      });
+    }
+  }
+
+  dates.sort((a, b) => a.localeCompare(b));
+  result.dateRange.start = dates[0]?.split("T")[0] ?? "";
+  result.dateRange.end = dates[dates.length - 1]?.split("T")[0] ?? "";
+
+  result.userResponseTimes = allResponseTimes;
+  if (allResponseTimes.length > 0) {
+    const sorted = [...allResponseTimes].sort((a, b) => a - b);
+    result.medianResponseTime = sorted[Math.floor(sorted.length / 2)] ?? 0;
+    result.avgResponseTime = allResponseTimes.reduce((a, b) => a + b, 0) / allResponseTimes.length;
+  }
+
+  const uniqueDays = new Set(dates.map((d) => d.split("T")[0]));
+  result.daysActive = uniqueDays.size;
+  result.messagesPerDay =
+    result.daysActive > 0 ? Math.round((result.totalMessages / result.daysActive) * 10) / 10 : 0;
+
+  result.messageHours = allMessageHours;
+  result.multiClauding = detectMultiClauding(sessions);
+
+  return result;
+}
+
+// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: sliding-window overlap detection is clearer in one pass.
+function detectMultiClauding(
+  sessions: Array<{ sessionId: string; userMessageTimestamps: string[] }>,
+): { overlapEvents: number; sessionsInvolved: number; userMessagesDuring: number } {
+  const OVERLAP_WINDOW_MS = 30 * 60000;
+  const allMessages: Array<{ ts: number; sessionId: string }> = [];
+
+  for (const session of sessions) {
+    for (const timestamp of session.userMessageTimestamps) {
+      try {
+        const ts = new Date(timestamp).getTime();
+        allMessages.push({ ts, sessionId: session.sessionId });
+      } catch {
+        // skip
+      }
+    }
+  }
+
+  allMessages.sort((a, b) => a.ts - b.ts);
+
+  const multiClaudeSessionPairs = new Set<string>();
+  const messagesDuringMulticlaude = new Set<string>();
+
+  let windowStart = 0;
+  const sessionLastIndex = new Map<string, number>();
+
+  for (let i = 0; i < allMessages.length; i++) {
+    const msg = allMessages[i];
+    if (!msg) continue;
+
+    let windowStartMessage = allMessages[windowStart];
+    while (
+      windowStart < i &&
+      windowStartMessage &&
+      msg.ts - windowStartMessage.ts > OVERLAP_WINDOW_MS
+    ) {
+      if (sessionLastIndex.get(windowStartMessage.sessionId) === windowStart) {
+        sessionLastIndex.delete(windowStartMessage.sessionId);
+      }
+      windowStart++;
+      windowStartMessage = allMessages[windowStart];
+    }
+
+    const prevIndex = sessionLastIndex.get(msg.sessionId);
+    if (prevIndex !== undefined) {
+      const previous = allMessages[prevIndex];
+      for (let j = prevIndex + 1; j < i; j++) {
+        const between = allMessages[j];
+        if (between && previous && between.sessionId !== msg.sessionId) {
+          const pair = [msg.sessionId, between.sessionId]
+            .sort((a, b) => a.localeCompare(b))
+            .join(":");
+          multiClaudeSessionPairs.add(pair);
+          messagesDuringMulticlaude.add(`${previous.ts}:${msg.sessionId}`);
+          messagesDuringMulticlaude.add(`${between.ts}:${between.sessionId}`);
+          messagesDuringMulticlaude.add(`${msg.ts}:${msg.sessionId}`);
+          break;
+        }
+      }
+    }
+
+    sessionLastIndex.set(msg.sessionId, i);
+  }
+
+  const sessionsWithOverlaps = new Set<string>();
+  for (const pair of multiClaudeSessionPairs) {
+    const [s1, s2] = pair.split(":");
+    if (s1) sessionsWithOverlaps.add(s1);
+    if (s2) sessionsWithOverlaps.add(s2);
+  }
+
+  return {
+    overlapEvents: multiClaudeSessionPairs.size,
+    sessionsInvolved: sessionsWithOverlaps.size,
+    userMessagesDuring: messagesDuringMulticlaude.size,
+  };
+}
+
+function safeEntries<V>(obj: Record<string, V> | undefined | null): [string, V][] {
+  return obj ? Object.entries(obj) : [];
+}

package/src/cache.ts

@@ -0,0 +1,94 @@
+// Cache management for supi-insights
+// Stores extracted facets and session metadata to avoid re-processing.
+
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import type { SessionFacets, SessionMeta } from "./types.ts";
+
+function getInsightsDir(): string {
+  return join(getAgentDir(), "supi", "insights");
+}
+
+function getFacetsDir(): string {
+  return join(getInsightsDir(), "facets");
+}
+
+function getMetaDir(): string {
+  return join(getInsightsDir(), "meta");
+}
+
+/** Build a stable cache key from session id, file path, and version so branched/stale sessions do not collide. */
+export function makeCacheKey(
+  sessionId: string,
+  filePath: string,
+  version: number | string,
+): string {
+  return `${sessionId}_${djb2(filePath)}_${djb2(String(version))}`;
+}
+
+function djb2(str: string): string {
+  let hash = 5381;
+  for (let i = 0; i < str.length; i++) {
+    hash = ((hash << 5) + hash + str.charCodeAt(i)) >>> 0;
+  }
+  return hash.toString(16).slice(0, 8);
+}
+
+export async function loadCachedFacets(cacheKey: string): Promise<SessionFacets | null> {
+  try {
+    const content = await readFile(join(getFacetsDir(), `${cacheKey}.json`), { encoding: "utf-8" });
+    const parsed = JSON.parse(content) as unknown;
+    if (!isValidSessionFacets(parsed)) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+export async function saveCachedFacets(cacheKey: string, facets: SessionFacets): Promise<void> {
+  const dir = getFacetsDir();
+  await mkdir(dir, { recursive: true });
+  await writeFile(join(dir, `${cacheKey}.json`), JSON.stringify(facets, null, 2), {
+    encoding: "utf-8",
+    mode: 0o600,
+  });
+}
+
+export async function loadCachedMeta(cacheKey: string): Promise<SessionMeta | null> {
+  try {
+    const content = await readFile(join(getMetaDir(), `${cacheKey}.json`), { encoding: "utf-8" });
+    return JSON.parse(content) as SessionMeta;
+  } catch {
+    return null;
+  }
+}
+
+export async function saveCachedMeta(cacheKey: string, meta: SessionMeta): Promise<void> {
+  const dir = getMetaDir();
+  await mkdir(dir, { recursive: true });
+  await writeFile(join(dir, `${cacheKey}.json`), JSON.stringify(meta, null, 2), {
+    encoding: "utf-8",
+    mode: 0o600,
+  });
+}
+
+export async function ensureInsightsDir(): Promise<void> {
+  await mkdir(getInsightsDir(), { recursive: true });
+}
+
+function isValidSessionFacets(obj: unknown): obj is SessionFacets {
+  if (!obj || typeof obj !== "object") return false;
+  const o = obj as Record<string, unknown>;
+  return (
+    typeof o.underlyingGoal === "string" &&
+    typeof o.outcome === "string" &&
+    typeof o.briefSummary === "string" &&
+    o.goalCategories !== null &&
+    typeof o.goalCategories === "object" &&
+    o.userSatisfactionCounts !== null &&
+    typeof o.userSatisfactionCounts === "object" &&
+    o.frictionCounts !== null &&
+    typeof o.frictionCounts === "object"
+  );
+}

package/src/extractor.ts

@@ -0,0 +1,189 @@
+// LLM facet extraction — analyze session transcripts and extract structured facets.
+
+import { complete } from "@earendil-works/pi-ai";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { SessionFacets } from "./types.ts";
+import { withRetry } from "./utils.ts";
+
+const FACET_EXTRACTION_PROMPT = `Analyze this PI coding agent session and extract structured facets.
+
+CRITICAL GUIDELINES:
+
+1. **goalCategories**: Count ONLY what the USER explicitly asked for.
+   - DO NOT count the agent's autonomous codebase exploration
+   - DO NOT count work the agent decided to do on its own
+   - ONLY count when user says "can you...", "please...", "I need...", "let's..."
+
+2. **userSatisfactionCounts**: Base ONLY on explicit user signals.
+   - "Yay!", "great!", "perfect!" → happy
+   - "thanks", "looks good", "that works" → satisfied
+   - "ok, now let's..." (continuing without complaint) → likely_satisfied
+   - "that's not right", "try again" → dissatisfied
+   - "this is broken", "I give up" → frustrated
+
+3. **frictionCounts**: Be specific about what went wrong.
+   - misunderstood_request: Agent interpreted incorrectly
+   - wrong_approach: Right goal, wrong solution method
+   - buggy_code: Code didn't work correctly
+   - user_rejected_action: User said no/stop to a tool call
+   - excessive_changes: Over-engineered or changed too much
+
+4. If very short or just warmup, use warmup_minimal for goal_category
+
+SESSION:
+`;
+
+const SUMMARIZE_CHUNK_PROMPT = `Summarize this portion of a PI session transcript. Focus on:
+1. What the user asked for
+2. What the agent did (tools used, files modified)
+3. Any friction or issues
+4. The outcome
+
+Keep it concise - 3-5 sentences. Preserve specific details like file names, error messages, and user feedback.
+
+TRANSCRIPT CHUNK:
+`;
+
+export async function extractFacets(
+  transcript: string,
+  sessionId: string,
+  ctx: ExtensionContext,
+): Promise<SessionFacets | null> {
+  // Resolve model: prefer active model, fall back to any available configured model
+  const model = ctx.model ?? ctx.modelRegistry.getAvailable()[0];
+  if (!model) return null;
+
+  const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+  if (!auth.ok || !auth.apiKey) return null;
+
+  // For long transcripts, summarize in chunks first
+  const processedTranscript =
+    transcript.length > 30000 ? await summarizeTranscript(transcript, ctx) : transcript;
+
+  const jsonPrompt = `${FACET_EXTRACTION_PROMPT}${processedTranscript}
+
+RESPOND WITH ONLY A VALID JSON OBJECT matching this schema:
+{
+  "underlyingGoal": "What the user fundamentally wanted to achieve",
+  "goalCategories": {"category_name": count, ...},
+  "outcome": "fully_achieved|mostly_achieved|partially_achieved|not_achieved|unclear_from_transcript",
+  "userSatisfactionCounts": {"level": count, ...},
+  "claudeHelpfulness": "unhelpful|slightly_helpful|moderately_helpful|very_helpful|essential",
+  "sessionType": "single_task|multi_task|iterative_refinement|exploration|quick_question",
+  "frictionCounts": {"friction_type": count, ...},
+  "frictionDetail": "One sentence describing friction or empty",
+  "primarySuccess": "none|fast_accurate_search|correct_code_edits|good_explanations|proactive_help|multi_file_changes|good_debugging",
+  "briefSummary": "One sentence: what user wanted and whether they got it"
+}`;
+
+  // Attempt the LLM call with up to 2 retries
+  const response = await withRetry(
+    async () => {
+      const res = await complete(
+        model,
+        {
+          systemPrompt: "",
+          messages: [
+            {
+              role: "user",
+              content: [{ type: "text", text: jsonPrompt }],
+              timestamp: Date.now(),
+            },
+          ],
+        },
+        {
+          apiKey: auth.apiKey,
+          headers: auth.headers,
+          signal: ctx.signal,
+          maxTokens: 4096,
+        },
+      );
+      return res;
+    },
+    2,
+    1000,
+  );
+
+  if (!response) return null;
+
+  try {
+    const text = response.content
+      .filter((c): c is { type: "text"; text: string } => c.type === "text")
+      .map((c) => c.text)
+      .join("");
+
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return null;
+
+    const parsed = JSON.parse(jsonMatch[0]) as unknown;
+    if (!isValidSessionFacets(parsed)) return null;
+
+    return { ...parsed, sessionId };
+  } catch {
+    return null;
+  }
+}
+
+async function summarizeTranscript(transcript: string, ctx: ExtensionContext): Promise<string> {
+  const model = ctx.model ?? ctx.modelRegistry.getAvailable()[0];
+  if (!model) return transcript.slice(0, 30000);
+
+  const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+  if (!auth.ok || !auth.apiKey) return transcript.slice(0, 30000);
+
+  const CHUNK_SIZE = 25000;
+  const chunks: string[] = [];
+  for (let i = 0; i < transcript.length; i += CHUNK_SIZE) {
+    chunks.push(transcript.slice(i, i + CHUNK_SIZE));
+  }
+
+  const summaries = await Promise.all(
+    chunks.map(async (chunk) => {
+      try {
+        const response = await complete(
+          model,
+          {
+            systemPrompt: "",
+            messages: [
+              {
+                role: "user",
+                content: [{ type: "text", text: SUMMARIZE_CHUNK_PROMPT + chunk }],
+                timestamp: Date.now(),
+              },
+            ],
+          },
+          {
+            apiKey: auth.apiKey,
+            headers: auth.headers,
+            signal: ctx.signal,
+            maxTokens: 500,
+          },
+        );
+        return response.content
+          .filter((c): c is { type: "text"; text: string } => c.type === "text")
+          .map((c) => c.text)
+          .join("");
+      } catch {
+        return chunk.slice(0, 2000);
+      }
+    }),
+  );
+
+  return summaries.join("\n\n---\n\n");
+}
+
+function isValidSessionFacets(obj: unknown): obj is Omit<SessionFacets, "sessionId"> {
+  if (!obj || typeof obj !== "object") return false;
+  const o = obj as Record<string, unknown>;
+  return (
+    typeof o.underlyingGoal === "string" &&
+    typeof o.outcome === "string" &&
+    typeof o.briefSummary === "string" &&
+    o.goalCategories !== null &&
+    typeof o.goalCategories === "object" &&
+    o.userSatisfactionCounts !== null &&
+    typeof o.userSatisfactionCounts === "object" &&
+    o.frictionCounts !== null &&
+    typeof o.frictionCounts === "object"
+  );
+}