assistme 0.4.0 → 0.6.0

package/src/utils/constants.ts

@@ -89,6 +89,26 @@ export const MEMORY_COMPRESSION_TARGET = 30;
 /** Max USD spend per task (safety net) */
 export const MAX_BUDGET_USD = 2.0;
 
+// ── Self-Analysis ─────────────────────────────────────────────────
+
+/** Max session log entries to include in self-analysis context */
+export const SELF_ANALYSIS_MAX_SESSION_LOGS = 200;
+
+/** Max message events to include in self-analysis context */
+export const SELF_ANALYSIS_MAX_MESSAGE_EVENTS = 300;
+
+/** Max conversation messages to include in self-analysis context */
+export const SELF_ANALYSIS_MAX_CONVERSATION_MESSAGES = 10;
+
+/** Max characters for session log context in self-analysis */
+export const SELF_ANALYSIS_LOG_CONTEXT_CHARS = 20_000;
+
+/** Max characters for event context in self-analysis */
+export const SELF_ANALYSIS_EVENT_CONTEXT_CHARS = 20_000;
+
+/** Edsger feedback product slug for assistme */
+export const EDSGER_PRODUCT_SLUG = "assistme";
+
 // ── Retry & Polling ────────────────────────────────────────────────
 
 /** Max retries for event emission */

package/src/utils/logger.ts

@@ -2,9 +2,11 @@ import chalk from "chalk";
 import { randomUUID } from "crypto";
 
 export type LogLevel = "debug" | "info" | "warn" | "error";
+export type LogHook = (logType: "stdout" | "stderr", message: string) => void;
 
 let currentLevel: LogLevel = "info";
 let currentCorrelationId: string | null = null;
+let logHook: LogHook | null = null;
 
 const LEVEL_ORDER: Record<LogLevel, number> = {
   debug: 0,
@@ -17,6 +19,14 @@ export function setLogLevel(level: LogLevel) {
   currentLevel = level;
 }
 
+/**
+ * Set a hook to capture all log output for persistence.
+ * Call with null to remove.
+ */
+export function setLogHook(hook: LogHook | null) {
+  logHook = hook;
+}
+
 /**
  * Set a correlation ID that will be included in all subsequent log messages.
  * Call with null to clear.
@@ -54,36 +64,54 @@ function prefix(): string {
 export const log = {
   debug(msg: string, ...args: unknown[]) {
     if (shouldLog("debug")) {
+      const text = formatArgs(msg, args);
+      logHook?.("stdout", `[DEBUG] ${text}`);
       console.log(chalk.gray(`[${prefix()}] DEBUG`), msg, ...args);
     }
   },
   info(msg: string, ...args: unknown[]) {
     if (shouldLog("info")) {
+      const text = formatArgs(msg, args);
+      logHook?.("stdout", text);
       console.log(chalk.blue(`[${prefix()}]`), msg, ...args);
     }
   },
   success(msg: string, ...args: unknown[]) {
     if (shouldLog("info")) {
+      const text = formatArgs(msg, args);
+      logHook?.("stdout", `✓ ${text}`);
       console.log(chalk.green(`[${prefix()}] ✓`), msg, ...args);
     }
   },
   warn(msg: string, ...args: unknown[]) {
     if (shouldLog("warn")) {
+      const text = formatArgs(msg, args);
+      logHook?.("stderr", `[WARN] ${text}`);
       console.log(chalk.yellow(`[${prefix()}] WARN`), msg, ...args);
     }
   },
   error(msg: string, ...args: unknown[]) {
     if (shouldLog("error")) {
+      const text = formatArgs(msg, args);
+      logHook?.("stderr", `[ERROR] ${text}`);
       console.error(chalk.red(`[${prefix()}] ERROR`), msg, ...args);
     }
   },
   agent(msg: string) {
+    logHook?.("stdout", `▸ ${msg}`);
     console.log(chalk.cyan("  ▸"), msg);
   },
   tool(name: string, msg: string) {
+    logHook?.("stdout", `⚡ ${name}: ${msg}`);
     console.log(chalk.magenta(`  ⚡ ${name}:`), msg);
   },
   result(msg: string) {
+    logHook?.("stdout", `← ${msg}`);
     console.log(chalk.green("  ←"), msg);
   },
 };
+
+function formatArgs(msg: string, args: unknown[]): string {
+  if (args.length === 0) return msg;
+  return `${msg} ${args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ")}`;
+}

package/src/utils/schemas.ts

@@ -114,6 +114,36 @@ export const BrowseSkillRowSchema = z.object({
   rating_count: z.number().optional().default(0),
 });
 
+// ── Self-Analysis Schemas ────────────────────────────────────────
+
+export const SelfAnalysisResultSchema = z.object({
+  is_perfect: z.boolean(),
+  overall_score: z.number().min(1).max(10),
+  task_completion_quality: z.object({
+    score: z.number().min(1).max(10),
+    assessment: z.string(),
+  }),
+  improvements: z.array(
+    z.object({
+      area: z.string(),
+      severity: z.enum(["critical", "major", "minor", "suggestion"]),
+      description: z.string(),
+      suggestion: z.string(),
+    })
+  ),
+  data_quality: z.object({
+    session_logs_useful: z.boolean(),
+    session_logs_gaps: z.string().nullable().optional(),
+    message_events_useful: z.boolean(),
+    message_events_gaps: z.string().nullable().optional(),
+    conversation_context_useful: z.boolean(),
+    conversation_context_gaps: z.string().nullable().optional(),
+  }),
+  summary: z.string(),
+});
+
+export type SelfAnalysisResult = z.infer<typeof SelfAnalysisResultSchema>;
+
 // ── JSON Extraction ──────────────────────────────────────────────
 
 /**

package/tests/agent/processor.test.ts

@@ -131,6 +131,10 @@ vi.mock("../../src/agent/event-hooks.js", () => ({
   })),
 }));
 
+vi.mock("../../src/agent/self-analyzer.js", () => ({
+  analyzeSelfPostTask: vi.fn().mockResolvedValue(undefined),
+}));
+
 const { TaskProcessor } = await import("../../src/agent/processor.js");
 
 describe("TaskProcessor", () => {

package/tests/agent/self-analyzer.test.ts

@@ -0,0 +1,349 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+// ── Mocks ────────────────────────────────────────────────────────
+
+// Track what was passed to query()
+let mockQueryCalls: unknown[] = [];
+const mockQueryMessages: unknown[] = [];
+vi.mock("@anthropic-ai/claude-agent-sdk", () => ({
+  query: vi.fn((params: unknown) => {
+    mockQueryCalls.push(params);
+    return {
+      [Symbol.asyncIterator]: async function* () {
+        for (const msg of mockQueryMessages) {
+          yield msg;
+        }
+      },
+    };
+  }),
+}));
+
+// Mock edsger-feedback
+const mockSubmitFeedback = vi.fn().mockResolvedValue({ id: "fb-001", createdAt: "2026-01-01" });
+vi.mock("edsger-feedback", () => ({
+  submitFeedback: (...args: unknown[]) => mockSubmitFeedback(...args),
+  FeedbackError: class FeedbackError extends Error {
+    statusCode: number;
+    constructor(message: string, statusCode: number) {
+      super(message);
+      this.statusCode = statusCode;
+    }
+  },
+}));
+
+// Mock analysis data fetchers
+const mockGetSessionLogs = vi.fn().mockResolvedValue([]);
+const mockGetMessageEvents = vi.fn().mockResolvedValue([]);
+const mockGetConversationMessages = vi.fn().mockResolvedValue([]);
+vi.mock("../../src/db/analysis-data.js", () => ({
+  getSessionLogs: (...args: unknown[]) => mockGetSessionLogs(...args),
+  getMessageEvents: (...args: unknown[]) => mockGetMessageEvents(...args),
+  getConversationMessages: (...args: unknown[]) => mockGetConversationMessages(...args),
+}));
+
+vi.mock("../../src/utils/logger.js", () => ({
+  log: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    success: vi.fn(),
+  },
+}));
+
+vi.mock("../../src/utils/errors.js", () => ({
+  errorMessage: (err: unknown) => (err instanceof Error ? err.message : String(err)),
+}));
+
+const { analyzeSelfPostTask } = await import("../../src/agent/self-analyzer.js");
+
+// ── Helper ──────────────────────────────────────────────────────
+
+function makeAnalysisResult(overrides: Record<string, unknown> = {}) {
+  return {
+    is_perfect: false,
+    overall_score: 7,
+    task_completion_quality: {
+      score: 8,
+      assessment: "Task completed successfully with minor inefficiencies.",
+    },
+    improvements: [
+      {
+        area: "Error Handling",
+        severity: "minor",
+        description: "Browser tool errors could be more descriptive.",
+        suggestion: "Include the CSS selector in error messages for easier debugging.",
+      },
+    ],
+    data_quality: {
+      session_logs_useful: true,
+      session_logs_gaps: "Missing timing information for each tool call.",
+      message_events_useful: true,
+      message_events_gaps: null,
+      conversation_context_useful: true,
+      conversation_context_gaps: null,
+    },
+    summary: "AssistMe handled the task well but has room for improvement in error reporting.",
+    ...overrides,
+  };
+}
+
+const baseOpts = {
+  model: "claude-sonnet-4-20250514",
+  taskId: "task-001",
+  conversationId: "conv-001",
+  taskPrompt: "Navigate to example.com and extract the title",
+  taskResponse: "The title of example.com is 'Example Domain'.",
+  toolCallRecords: [
+    { name: "browser_navigate", input: { url: "https://example.com" }, result: "Navigated" },
+    { name: "browser_read_page", input: {}, result: "Example Domain" },
+  ],
+  toolFailures: [],
+  tokenUsage: { input_tokens: 1000, output_tokens: 500 },
+  sessionId: "session-001",
+};
+
+// ── Tests ────────────────────────────────────────────────────────
+
+describe("analyzeSelfPostTask", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockQueryMessages.length = 0;
+    mockQueryCalls = [];
+  });
+
+  it("submits feedback when improvements are found", async () => {
+    const analysis = makeAnalysisResult();
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: analysis,
+      total_cost_usd: 0.005,
+      num_turns: 1,
+      usage: { input_tokens: 2000, output_tokens: 500 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    expect(mockSubmitFeedback).toHaveBeenCalledTimes(1);
+    const feedbackCall = mockSubmitFeedback.mock.calls[0][0];
+    expect(feedbackCall.slug).toBe("assistme");
+    expect(feedbackCall.title).toContain("Score 7/10");
+    expect(feedbackCall.title).toContain("1 improvement");
+    expect(feedbackCall.description).toContain("Error Handling");
+    expect(feedbackCall.category).toBe("improvement");
+  });
+
+  it("does NOT submit feedback when analysis is perfect", async () => {
+    const analysis = makeAnalysisResult({
+      is_perfect: true,
+      overall_score: 10,
+      improvements: [],
+    });
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: analysis,
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1500, output_tokens: 300 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    expect(mockSubmitFeedback).not.toHaveBeenCalled();
+  });
+
+  it("uses independent query (no session resume)", async () => {
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult({ is_perfect: true, improvements: [] }),
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1000, output_tokens: 200 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    expect(mockQueryCalls).toHaveLength(1);
+    const queryParams = mockQueryCalls[0] as { options: Record<string, unknown> };
+    expect(queryParams.options.resume).toBeUndefined();
+    expect(queryParams.options.maxTurns).toBe(1);
+    expect(queryParams.options.allowedTools).toEqual([]);
+  });
+
+  it("includes session logs and filtered events in context prompt", async () => {
+    mockGetSessionLogs.mockResolvedValueOnce([
+      { log_type: "stdout", message: "Processing task...", sequence_number: 1, created_at: "2026-01-01" },
+      { log_type: "stderr", message: "Warning: slow response", sequence_number: 2, created_at: "2026-01-01" },
+    ]);
+    mockGetMessageEvents.mockResolvedValueOnce([
+      { event_type: "tool_use_start", event_data: { name: "browser_navigate" }, sequence_number: 1, created_at: "2026-01-01" },
+      { event_type: "tool_result", event_data: { name: "browser_navigate", result: "ok" }, sequence_number: 2, created_at: "2026-01-01" },
+    ]);
+    mockGetConversationMessages.mockResolvedValueOnce([
+      { role: "user", content: "Go to example.com", status: null },
+    ]);
+
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult({ is_perfect: true, improvements: [] }),
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1000, output_tokens: 200 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    const queryParams = mockQueryCalls[0] as { prompt: string };
+    expect(queryParams.prompt).toContain("Session Logs (2 entries)");
+    expect(queryParams.prompt).toContain("Processing task...");
+    // Events are now aggregated — shows raw → aggregated count
+    expect(queryParams.prompt).toContain("Message Events (2 raw");
+    expect(queryParams.prompt).toContain("aggregated)");
+    expect(queryParams.prompt).toContain("[tool_result] browser_navigate: ok");
+    expect(queryParams.prompt).toContain("Conversation Messages (1 entries)");
+    expect(queryParams.prompt).toContain("Go to example.com");
+  });
+
+  it("aggregates consecutive text_delta events and drops thinking", async () => {
+    mockGetMessageEvents.mockResolvedValueOnce([
+      { event_type: "text_delta", event_data: { text: "Hello " }, sequence_number: 1, created_at: "2026-01-01" },
+      { event_type: "text_delta", event_data: { text: "world" }, sequence_number: 2, created_at: "2026-01-01" },
+      { event_type: "thinking", event_data: { text: "I should analyze this carefully..." }, sequence_number: 3, created_at: "2026-01-01" },
+      { event_type: "tool_use_start", event_data: { name: "Bash" }, sequence_number: 4, created_at: "2026-01-01" },
+      { event_type: "tool_use_input", event_data: { input: { command: "ls" } }, sequence_number: 5, created_at: "2026-01-01" },
+      { event_type: "tool_result", event_data: { name: "Bash", result: "file1.ts file2.ts" }, sequence_number: 6, created_at: "2026-01-01" },
+      { event_type: "text_delta", event_data: { text: "Done!" }, sequence_number: 7, created_at: "2026-01-01" },
+    ]);
+
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult({ is_perfect: true, improvements: [] }),
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1000, output_tokens: 200 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    const queryParams = mockQueryCalls[0] as { prompt: string };
+    // 7 raw events → 4 aggregated (2 text_delta merged, thinking dropped, tool_use_input dropped)
+    expect(queryParams.prompt).toContain("7 raw");
+    // Merged text_delta
+    expect(queryParams.prompt).toContain("[text_delta x2] Hello world");
+    // Thinking should be dropped
+    expect(queryParams.prompt).not.toContain("I should analyze this carefully");
+    // tool_use_input should be dropped
+    expect(queryParams.prompt).not.toContain("[tool_use_input]");
+    // tool_result kept with summary
+    expect(queryParams.prompt).toContain("[tool_result] Bash: file1.ts file2.ts");
+    // Second text_delta batch
+    expect(queryParams.prompt).toContain("[text_delta x1] Done!");
+  });
+
+  it("includes tool failures in context when present", async () => {
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult({ is_perfect: true, improvements: [] }),
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1000, output_tokens: 200 },
+    });
+
+    await analyzeSelfPostTask({
+      ...baseOpts,
+      toolFailures: [
+        { toolName: "browser_click", input: { selector: "#btn" }, error: "Element not found", timestamp: Date.now() },
+      ],
+    });
+
+    const queryParams = mockQueryCalls[0] as { prompt: string };
+    expect(queryParams.prompt).toContain("Tool Failures (1 failures)");
+    expect(queryParams.prompt).toContain("browser_click");
+    expect(queryParams.prompt).toContain("Element not found");
+  });
+
+  it("handles SDK errors gracefully without throwing", async () => {
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "error_during_execution",
+      errors: ["Analysis failed"],
+      total_cost_usd: 0,
+      num_turns: 0,
+      usage: { input_tokens: 10, output_tokens: 0 },
+    });
+
+    // Should not throw
+    await expect(analyzeSelfPostTask(baseOpts)).resolves.toBeUndefined();
+    expect(mockSubmitFeedback).not.toHaveBeenCalled();
+  });
+
+  it("handles feedback submission failure gracefully", async () => {
+    const { FeedbackError } = await import("edsger-feedback");
+    mockSubmitFeedback.mockRejectedValueOnce(new FeedbackError("Rate limited", 429));
+
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult(),
+      total_cost_usd: 0.005,
+      num_turns: 1,
+      usage: { input_tokens: 2000, output_tokens: 500 },
+    });
+
+    // Should not throw
+    await expect(analyzeSelfPostTask(baseOpts)).resolves.toBeUndefined();
+  });
+
+  it("handles multiple improvements with correct count in title", async () => {
+    const analysis = makeAnalysisResult({
+      improvements: [
+        { area: "Error Handling", severity: "minor", description: "desc1", suggestion: "sug1" },
+        { area: "Performance", severity: "major", description: "desc2", suggestion: "sug2" },
+        { area: "Observability", severity: "suggestion", description: "desc3", suggestion: "sug3" },
+      ],
+    });
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: analysis,
+      total_cost_usd: 0.005,
+      num_turns: 1,
+      usage: { input_tokens: 2000, output_tokens: 500 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    const feedbackCall = mockSubmitFeedback.mock.calls[0][0];
+    expect(feedbackCall.title).toContain("3 improvement(s)");
+    expect(feedbackCall.description).toContain("Performance");
+    expect(feedbackCall.description).toContain("Observability");
+  });
+
+  it("notes missing data when no logs/events available", async () => {
+    mockGetSessionLogs.mockResolvedValueOnce([]);
+    mockGetMessageEvents.mockResolvedValueOnce([]);
+    mockGetConversationMessages.mockResolvedValueOnce([]);
+
+    mockQueryMessages.push({
+      type: "result",
+      subtype: "success",
+      structured_output: makeAnalysisResult({ is_perfect: true, improvements: [] }),
+      total_cost_usd: 0.003,
+      num_turns: 1,
+      usage: { input_tokens: 1000, output_tokens: 200 },
+    });
+
+    await analyzeSelfPostTask(baseOpts);
+
+    const queryParams = mockQueryCalls[0] as { prompt: string };
+    expect(queryParams.prompt).toContain("No session logs available");
+    expect(queryParams.prompt).toContain("No message events available");
+    expect(queryParams.prompt).toContain("data gap");
+  });
+});