assistme 0.4.0 → 0.6.0

package/src/agent/processor.ts

@@ -22,6 +22,7 @@ import { MemoryManager } from "./memory.js";
 import { SkillManager } from "./skills.js";
 import { type ToolCallRecord } from "./skill-extractor.js";
 import { evaluateAndMaybeCreateSkill } from "./skill-evaluator.js";
+import { analyzeSelfPostTask } from "./self-analyzer.js";
 import { withRetry } from "../utils/retry.js";
 import {
   createBrowserMcpServer,
@@ -355,11 +356,30 @@ export class TaskProcessor {
         );
       }
 
-      // Post-task: resume the same session to evaluate skill creation
+      // Post-task: skill evaluation → self-analysis (serial to avoid session resume conflicts)
       if (agentSessionId) {
-        this.evaluateSkillPostTask(agentSessionId, config.model).catch((err) =>
-          log.debug(`Post-task skill evaluation skipped: ${err}`)
-        );
+        (async () => {
+          try {
+            await this.evaluateSkillPostTask(agentSessionId, config.model);
+          } catch (err) {
+            log.debug(`Post-task skill evaluation skipped: ${err}`);
+          }
+          try {
+            await analyzeSelfPostTask({
+              model: config.model,
+              taskId: task.id,
+              conversationId: task.conversation_id,
+              taskPrompt: task.prompt,
+              taskResponse: finalResponse,
+              toolCallRecords,
+              toolFailures,
+              tokenUsage,
+              sessionId: this.sessionId || "",
+            });
+          } catch (err) {
+            log.debug(`Post-task self-analysis skipped: ${err}`);
+          }
+        })().catch(() => {});
       }
     } catch (err) {
       const errMsg = errorMessage(err);

package/src/agent/self-analyzer.ts

@@ -0,0 +1,444 @@
+import {
+  query,
+  type SDKResultMessage,
+  type SDKResultSuccess,
+  type OutputFormat,
+} from "@anthropic-ai/claude-agent-sdk";
+import { submitFeedback, FeedbackError } from "edsger-feedback";
+import { log } from "../utils/logger.js";
+import {
+  SelfAnalysisResultSchema,
+  type SelfAnalysisResult,
+  safeParse,
+} from "../utils/schemas.js";
+import { errorMessage } from "../utils/errors.js";
+import {
+  getSessionLogs,
+  getMessageEvents,
+  getConversationMessages,
+} from "../db/analysis-data.js";
+import {
+  SELF_ANALYSIS_MAX_SESSION_LOGS,
+  SELF_ANALYSIS_MAX_MESSAGE_EVENTS,
+  SELF_ANALYSIS_MAX_CONVERSATION_MESSAGES,
+  SELF_ANALYSIS_LOG_CONTEXT_CHARS,
+  SELF_ANALYSIS_EVENT_CONTEXT_CHARS,
+  EDSGER_PRODUCT_SLUG,
+} from "../utils/constants.js";
+import type { ToolCallRecord } from "./skill-extractor.js";
+import type { ToolFailureRecord } from "./event-hooks.js";
+
+// ── Structured Output Schema ────────────────────────────────────
+
+const SELF_ANALYSIS_OUTPUT_FORMAT: OutputFormat = {
+  type: "json_schema",
+  schema: {
+    type: "object",
+    properties: {
+      is_perfect: { type: "boolean" },
+      overall_score: { type: "number" },
+      task_completion_quality: {
+        type: "object",
+        properties: {
+          score: { type: "number" },
+          assessment: { type: "string" },
+        },
+        required: ["score", "assessment"],
+      },
+      improvements: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            area: { type: "string" },
+            severity: {
+              type: "string",
+              enum: ["critical", "major", "minor", "suggestion"],
+            },
+            description: { type: "string" },
+            suggestion: { type: "string" },
+          },
+          required: ["area", "severity", "description", "suggestion"],
+        },
+      },
+      data_quality: {
+        type: "object",
+        properties: {
+          session_logs_useful: { type: "boolean" },
+          session_logs_gaps: { type: "string" },
+          message_events_useful: { type: "boolean" },
+          message_events_gaps: { type: "string" },
+          conversation_context_useful: { type: "boolean" },
+          conversation_context_gaps: { type: "string" },
+        },
+        required: [
+          "session_logs_useful",
+          "message_events_useful",
+          "conversation_context_useful",
+        ],
+      },
+      summary: { type: "string" },
+    },
+    required: [
+      "is_perfect",
+      "overall_score",
+      "task_completion_quality",
+      "improvements",
+      "data_quality",
+      "summary",
+    ],
+  },
+};
+
+// ── Analysis Prompt ─────────────────────────────────────────────
+
+const SELF_ANALYSIS_PROMPT = `You just completed a task as the AssistMe agent. Now critically analyze AssistMe's own implementation — NOT the user's task itself, but how well AssistMe (the agent system) performed and whether AssistMe's codebase can be improved.
+
+## Your Role
+You are a perfectionist code reviewer analyzing the AssistMe agent system itself. Be critical, thorough, and constructive. Focus on:
+
+1. **Task Completion Quality**: Did AssistMe handle the task optimally? Were there unnecessary steps, missed edge cases, or suboptimal tool usage?
+2. **Agent Architecture**: Based on what you observed during execution, are there architectural improvements to AssistMe's code (processor, event hooks, MCP servers, skill system, etc.)?
+3. **Data & Observability**: Evaluate the quality of the context data provided (session logs, message events, conversation messages). What information is missing that would help diagnose issues or improve the system?
+4. **Error Handling & Resilience**: Were there any failures or retries? Could the error handling be improved?
+5. **Performance & Efficiency**: Were there unnecessary API calls, redundant operations, or opportunities to optimize?
+6. **User Experience**: Could the interaction flow be smoother? Is the feedback to the user adequate?
+
+## Context Data Provided
+Below you will find:
+- **Session Logs**: stdout/stderr/status logs from the agent session
+- **Message Events**: Real-time events emitted during task execution (tool calls, results, status changes, thinking blocks)
+- **Conversation Messages**: User interaction history for this conversation
+- **Tool Call Records**: Summary of all tool calls made during the task
+- **Tool Failures**: Any tool calls that failed during execution
+
+## Instructions
+Analyze all provided data critically. Consider:
+- Are the session logs capturing enough detail for debugging?
+- Do the message events provide sufficient visibility into the agent's decision-making?
+- Is the conversation context giving enough user intent signal?
+- Were tools used efficiently?
+- Could the overall execution flow be improved?
+
+Set is_perfect to true ONLY if there are genuinely zero improvements to suggest (this should be rare).
+The overall_score should be 1-10 where 10 means absolutely perfect.
+
+Respond with a JSON object now.`;
+
+// ── Context Building ────────────────────────────────────────────
+
+interface SelfAnalysisContext {
+  taskId: string;
+  conversationId: string;
+  sessionId: string;
+  taskPrompt: string;
+  taskResponse: string;
+  toolCallRecords: ToolCallRecord[];
+  toolFailures: ToolFailureRecord[];
+  tokenUsage?: Record<string, number>;
+}
+
+function truncateToChars(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  return text.slice(0, maxChars) + "\n... [truncated]";
+}
+
+/** Max characters for tool result in event summary */
+const EVENT_RESULT_SUMMARY_CHARS = 150;
+
+/**
+ * Aggregate and filter events intelligently:
+ * - Merge consecutive text_delta events into a single entry
+ * - Summarize tool_result data (keep name + truncated result)
+ * - Keep tool_failure, status_change, error events in full
+ * - Drop thinking events (verbose, low signal for analysis)
+ */
+function filterAndAggregateEvents(
+  events: Array<{ event_type: string; event_data: Record<string, unknown>; sequence_number: number; created_at: string }>
+): string[] {
+  const lines: string[] = [];
+  let pendingTextChunks: string[] = [];
+
+  const flushTextDelta = () => {
+    if (pendingTextChunks.length > 0) {
+      const merged = pendingTextChunks.join("");
+      const summary = merged.length > 200
+        ? merged.slice(0, 200) + "..."
+        : merged;
+      lines.push(`[text_delta x${pendingTextChunks.length}] ${summary}`);
+      pendingTextChunks = [];
+    }
+  };
+
+  for (const e of events) {
+    switch (e.event_type) {
+      case "text_delta": {
+        const text = (e.event_data.text as string) || "";
+        pendingTextChunks.push(text);
+        break;
+      }
+
+      case "thinking":
+        // Skip thinking events — verbose and low signal for self-analysis
+        flushTextDelta();
+        break;
+
+      case "tool_use_input":
+        // Skip raw input — tool_use_start + tool_result cover the essentials
+        flushTextDelta();
+        break;
+
+      case "tool_result": {
+        flushTextDelta();
+        const name = (e.event_data.name as string) || "unknown";
+        const result = ((e.event_data.result as string) || "").slice(0, EVENT_RESULT_SUMMARY_CHARS);
+        lines.push(`[tool_result] ${name}: ${result}`);
+        break;
+      }
+
+      case "tool_failure":
+      case "status_change":
+      case "error": {
+        flushTextDelta();
+        lines.push(`[${e.event_type}] ${JSON.stringify(e.event_data)}`);
+        break;
+      }
+
+      default: {
+        flushTextDelta();
+        const dataStr = JSON.stringify(e.event_data);
+        lines.push(`[${e.event_type}] ${dataStr.slice(0, 200)}`);
+        break;
+      }
+    }
+  }
+
+  flushTextDelta();
+  return lines;
+}
+
+async function buildAnalysisContext(ctx: SelfAnalysisContext): Promise<string> {
+  // Fetch all data sources in parallel
+  const [sessionLogs, messageEvents, conversationMessages] = await Promise.all([
+    getSessionLogs(ctx.sessionId, SELF_ANALYSIS_MAX_SESSION_LOGS),
+    getMessageEvents(ctx.taskId, SELF_ANALYSIS_MAX_MESSAGE_EVENTS),
+    getConversationMessages(ctx.conversationId, SELF_ANALYSIS_MAX_CONVERSATION_MESSAGES),
+  ]);
+
+  let context = "";
+
+  // Session Logs
+  if (sessionLogs.length > 0) {
+    const logText = sessionLogs
+      .map((l) => `[${l.log_type}] ${l.message}`)
+      .join("\n");
+    context += `\n## Session Logs (${sessionLogs.length} entries)\n`;
+    context += truncateToChars(logText, SELF_ANALYSIS_LOG_CONTEXT_CHARS);
+  } else {
+    context += "\n## Session Logs\n(No session logs available — this is itself a data gap to note)\n";
+  }
+
+  // Message Events (filtered & aggregated)
+  if (messageEvents.length > 0) {
+    const filteredLines = filterAndAggregateEvents(messageEvents);
+    const eventText = filteredLines.join("\n");
+    context += `\n## Message Events (${messageEvents.length} raw → ${filteredLines.length} aggregated)\n`;
+    context += truncateToChars(eventText, SELF_ANALYSIS_EVENT_CONTEXT_CHARS);
+  } else {
+    context += "\n## Message Events\n(No message events available — this is itself a data gap to note)\n";
+  }
+
+  // Conversation Messages
+  if (conversationMessages.length > 0) {
+    context += `\n## Conversation Messages (${conversationMessages.length} entries)\n`;
+    for (const msg of conversationMessages) {
+      const role = (msg.role as string) || "unknown";
+      const content = ((msg.content as string) || "").slice(0, 500);
+      const status = (msg.status as string) || "";
+      context += `[${role}${status ? ` (${status})` : ""}] ${content}\n`;
+    }
+  } else {
+    context += "\n## Conversation Messages\n(No conversation messages available)\n";
+  }
+
+  // Tool Call Records
+  if (ctx.toolCallRecords.length > 0) {
+    context += `\n## Tool Call Records (${ctx.toolCallRecords.length} calls)\n`;
+    for (const tc of ctx.toolCallRecords) {
+      const inputStr = JSON.stringify(tc.input).slice(0, 200);
+      context += `- ${tc.name}: ${inputStr} → ${tc.result.slice(0, 100)}\n`;
+    }
+  }
+
+  // Tool Failures
+  if (ctx.toolFailures.length > 0) {
+    context += `\n## Tool Failures (${ctx.toolFailures.length} failures)\n`;
+    for (const tf of ctx.toolFailures) {
+      context += `- ${tf.toolName}: ${tf.error}\n`;
+    }
+  }
+
+  // Token Usage
+  if (ctx.tokenUsage) {
+    context += `\n## Token Usage\n`;
+    context += `Input: ${ctx.tokenUsage.input_tokens}, Output: ${ctx.tokenUsage.output_tokens}\n`;
+  }
+
+  // Task Summary
+  context += `\n## Task\n`;
+  context += `Prompt: ${ctx.taskPrompt.slice(0, 500)}\n`;
+  context += `Response length: ${ctx.taskResponse.length} chars\n`;
+
+  return context;
+}
+
+// ── Feedback Submission ─────────────────────────────────────────
+
+async function submitSelfAnalysisFeedback(analysis: SelfAnalysisResult): Promise<void> {
+  const title = `Self-Analysis: Score ${analysis.overall_score}/10 — ${analysis.improvements.length} improvement(s)`;
+
+  const improvementDetails = analysis.improvements
+    .map((imp, i) => `${i + 1}. [${imp.severity}] **${imp.area}**: ${imp.description}\n   → ${imp.suggestion}`)
+    .join("\n");
+
+  const dataQualityNotes = [
+    analysis.data_quality.session_logs_gaps
+      ? `Session logs: ${analysis.data_quality.session_logs_gaps}`
+      : null,
+    analysis.data_quality.message_events_gaps
+      ? `Message events: ${analysis.data_quality.message_events_gaps}`
+      : null,
+    analysis.data_quality.conversation_context_gaps
+      ? `Conversation context: ${analysis.data_quality.conversation_context_gaps}`
+      : null,
+  ]
+    .filter(Boolean)
+    .join("\n");
+
+  let description = `## Summary\n${analysis.summary}\n\n`;
+  description += `## Task Completion Quality (${analysis.task_completion_quality.score}/10)\n${analysis.task_completion_quality.assessment}\n\n`;
+  description += `## Improvements\n${improvementDetails}\n`;
+
+  if (dataQualityNotes) {
+    description += `\n## Data Quality Gaps\n${dataQualityNotes}\n`;
+  }
+
+  // Truncate to fit edsger-feedback's 5000 char limit
+  if (description.length > 4900) {
+    description = description.slice(0, 4900) + "\n...[truncated]";
+  }
+
+  try {
+    const result = await submitFeedback({
+      slug: EDSGER_PRODUCT_SLUG,
+      title: title.slice(0, 200),
+      description,
+      category: "improvement",
+    });
+    log.info(`Self-analysis feedback submitted: ${result.id}`);
+  } catch (err) {
+    if (err instanceof FeedbackError) {
+      log.debug(`Feedback submission failed (${err.statusCode}): ${err.message}`);
+    } else {
+      log.debug(`Feedback submission failed: ${errorMessage(err)}`);
+    }
+  }
+}
+
+// ── Main Entry Point ────────────────────────────────────────────
+
+/**
+ * Post-task self-analysis: resume the agent SDK session to critically
+ * analyze AssistMe's own performance and code quality.
+ *
+ * If improvements are found, submits feedback via edsger-feedback.
+ * If the analysis determines everything is perfect, no feedback is sent.
+ */
+export async function analyzeSelfPostTask(opts: {
+  model: string;
+  taskId: string;
+  conversationId: string;
+  taskPrompt: string;
+  taskResponse: string;
+  toolCallRecords: ToolCallRecord[];
+  toolFailures: ToolFailureRecord[];
+  tokenUsage?: Record<string, number>;
+  sessionId: string;
+}): Promise<void> {
+  const {
+    model,
+    taskId,
+    conversationId,
+    taskPrompt,
+    taskResponse,
+    toolCallRecords,
+    toolFailures,
+    tokenUsage,
+    sessionId,
+  } = opts;
+
+  try {
+    // Build context from all available data sources
+    const analysisContext = await buildAnalysisContext({
+      taskId,
+      conversationId,
+      sessionId,
+      taskPrompt,
+      taskResponse,
+      toolCallRecords,
+      toolFailures,
+      tokenUsage,
+    });
+
+    const prompt = `${SELF_ANALYSIS_PROMPT}\n${analysisContext}\n\nRespond with a JSON object now.`;
+
+    let structuredOutput: unknown;
+
+    // Use independent query() instead of session resume to avoid
+    // conflicts with skill evaluation which also resumes the session
+    for await (const message of query({
+      prompt,
+      options: {
+        model,
+        maxTurns: 1,
+        allowedTools: [],
+        effort: "low",
+        outputFormat: SELF_ANALYSIS_OUTPUT_FORMAT,
+      },
+    })) {
+      if (message.type === "result") {
+        const resultMsg = message as SDKResultMessage;
+        if (resultMsg.subtype === "success") {
+          const successMsg = resultMsg as SDKResultSuccess;
+          structuredOutput = successMsg.structured_output;
+          log.debug(
+            `Self-analysis cost: $${successMsg.total_cost_usd.toFixed(4)}`
+          );
+        }
+      }
+    }
+
+    // Validate against Zod schema
+    const analysis = structuredOutput
+      ? safeParse(SelfAnalysisResultSchema, structuredOutput)
+      : null;
+
+    if (!analysis) {
+      log.debug("Self-analysis: no valid structured output");
+      return;
+    }
+
+    log.info(
+      `Self-analysis complete: score=${analysis.overall_score}/10, perfect=${analysis.is_perfect}, improvements=${analysis.improvements.length}`
+    );
+
+    // Only submit feedback if not perfect
+    if (!analysis.is_perfect && analysis.improvements.length > 0) {
+      await submitSelfAnalysisFeedback(analysis);
+    } else {
+      log.debug("Self-analysis: no improvements to report — skipping feedback");
+    }
+  } catch (err) {
+    log.debug(`Self-analysis error: ${errorMessage(err)}`);
+  }
+}

package/src/commands/start.ts

@@ -4,10 +4,11 @@ import ora from "ora";
 import { createInterface } from "readline";
 import { getCurrentUserId } from "../db/supabase.js";
 import { setConfig } from "../utils/config.js";
-import { log, setLogLevel } from "../utils/logger.js";
+import { log, setLogLevel, setLogHook } from "../utils/logger.js";
 import { SessionManager } from "../agent/session.js";
 import { TaskProcessor } from "../agent/processor.js";
 import { getBrowser, ensureBrowserAvailable } from "../tools/browser.js";
+import { SessionLogEmitter } from "../db/session-log.js";
 
 export function registerStartCommand(program: Command): void {
   program
@@ -85,12 +86,19 @@ async function runAgent(opts: { workspace?: string; name?: string; verbose?: boo
   const processor = new TaskProcessor();
   processor.init(userId);
   const sessionManager = new SessionManager();
+  let logEmitter: SessionLogEmitter | null = null;
 
   // Graceful shutdown
   const browserRef = getBrowser();
   const shutdown = async () => {
     console.log();
     log.info("Shutting down...");
+    setLogHook(null);
+    try {
+      if (logEmitter) await logEmitter.stop();
+    } catch {
+      /* ignore */
+    }
     try {
       if (browserRef.isConnected()) await browserRef.disconnect();
     } catch {
@@ -109,6 +117,12 @@ async function runAgent(opts: { workspace?: string; name?: string; verbose?: boo
     });
     processor.setSessionId(session.id);
 
+    // Start persisting logs to Supabase
+    logEmitter = new SessionLogEmitter(session.id);
+    setLogHook((logType, message) => {
+      logEmitter?.push(logType, message);
+    });
+
     log.info("Listening for tasks (chat + jobs) from web UI...");
     log.info("Press Ctrl+C to stop.\n");
 

package/src/db/analysis-data.ts

@@ -0,0 +1,79 @@
+import { callMcpHandler } from "./api-client.js";
+import { log } from "../utils/logger.js";
+
+export interface SessionLogEntry {
+  log_type: "stdout" | "stderr" | "status";
+  message: string;
+  sequence_number: number;
+  created_at: string;
+}
+
+export interface MessageEventEntry {
+  event_type: string;
+  event_data: Record<string, unknown>;
+  sequence_number: number;
+  created_at: string;
+}
+
+/**
+ * Fetch agent session logs for a given session.
+ * Uses the log.list MCP handler endpoint.
+ */
+export async function getSessionLogs(
+  sessionId: string,
+  limit = 500
+): Promise<SessionLogEntry[]> {
+  try {
+    const data = await callMcpHandler<SessionLogEntry[]>("log.list", {
+      session_id: sessionId,
+      limit,
+    });
+    return data || [];
+  } catch (err) {
+    log.debug(`Failed to fetch session logs: ${err instanceof Error ? err.message : err}`);
+    return [];
+  }
+}
+
+/**
+ * Fetch message events for a given task (message_id).
+ * Uses the event.list MCP handler endpoint.
+ */
+export async function getMessageEvents(
+  messageId: string,
+  limit = 500
+): Promise<MessageEventEntry[]> {
+  try {
+    const data = await callMcpHandler<MessageEventEntry[]>("event.list", {
+      message_id: messageId,
+      limit,
+    });
+    return data || [];
+  } catch (err) {
+    log.debug(`Failed to fetch message events: ${err instanceof Error ? err.message : err}`);
+    return [];
+  }
+}
+
+/**
+ * Fetch conversation messages for context.
+ * Returns the most recent messages including the current task.
+ */
+export async function getConversationMessages(
+  conversationId: string,
+  limit = 10
+): Promise<Array<Record<string, unknown>>> {
+  try {
+    const data = await callMcpHandler<Array<Record<string, unknown>>>(
+      "conversation.list_messages",
+      {
+        conversation_id: conversationId,
+        limit,
+      }
+    );
+    return data || [];
+  } catch (err) {
+    log.debug(`Failed to fetch conversation messages: ${err instanceof Error ? err.message : err}`);
+    return [];
+  }
+}

package/src/db/session-log.ts

@@ -0,0 +1,71 @@
+import { callMcpHandler } from "./api-client.js";
+import { log } from "../utils/logger.js";
+
+const FLUSH_INTERVAL_MS = 3_000;
+const MAX_BATCH_SIZE = 100;
+
+interface PendingLog {
+  log_type: "stdout" | "stderr" | "status";
+  message: string;
+  seq: number;
+}
+
+/**
+ * Batches agent process logs and flushes them to Supabase periodically.
+ * Each session gets its own emitter with an auto-incrementing sequence.
+ */
+export class SessionLogEmitter {
+  private sequence = 0;
+  private buffer: PendingLog[] = [];
+  private flushTimer: ReturnType<typeof setInterval> | null = null;
+  private flushing = false;
+
+  constructor(private sessionId: string) {
+    this.flushTimer = setInterval(() => this.flush(), FLUSH_INTERVAL_MS);
+  }
+
+  /** Queue a log entry for batch insertion */
+  push(logType: "stdout" | "stderr" | "status", message: string): void {
+    this.sequence++;
+    this.buffer.push({ log_type: logType, message, seq: this.sequence });
+
+    // Flush immediately if buffer is large
+    if (this.buffer.length >= MAX_BATCH_SIZE) {
+      this.flush();
+    }
+  }
+
+  /** Flush buffered logs to Supabase */
+  async flush(): Promise<void> {
+    if (this.flushing || this.buffer.length === 0) return;
+
+    const batch = this.buffer.splice(0);
+    this.flushing = true;
+
+    try {
+      await callMcpHandler("log.emit_batch", {
+        session_id: this.sessionId,
+        logs: batch,
+      });
+    } catch (err) {
+      log.debug(
+        `Failed to flush session logs: ${err instanceof Error ? err.message : err}`
+      );
+      // Re-queue failed batch at the front (best-effort, drop if too old)
+      if (this.buffer.length < MAX_BATCH_SIZE * 5) {
+        this.buffer.unshift(...batch);
+      }
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  /** Stop the emitter and flush remaining logs */
+  async stop(): Promise<void> {
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+    await this.flush();
+  }
+}