@sage-protocol/sage-plugin 0.1.5 → 0.1.6

package/rlm-feedback.e2e.test.js

@@ -0,0 +1,295 @@
+/**
+ * E2E Test: RLM Feedback Correlation Loop
+ *
+ * Validates the suggestion feedback cycle within the sage-plugin:
+ *   1. Plugin captures a prompt
+ *   2. A suggestion is shown (simulated via internal state)
+ *   3. User sends a follow-up prompt that overlaps with the suggestion
+ *   4. Plugin detects correlation and sends feedback
+ *   5. Verify feedback classification (accepted/steered/rejected)
+ */
+
+import { beforeEach, describe, expect, it } from "bun:test";
+import SagePlugin from "./index.js";
+
+describe("RLM Feedback Correlation E2E", () => {
+  let plugin;
+  let $mock;
+  let appLogCalls;
+
+  const makeClient = () => {
+    const logs = [];
+    const appends = [];
+    return {
+      client: {
+        app: {
+          log: ({ level, message, extra }) => logs.push({ level, message, extra }),
+        },
+        tui: {
+          appendPrompt: ({ body }) => appends.push(body?.text ?? ""),
+        },
+      },
+      appLogCalls: logs,
+      promptAppends: appends,
+    };
+  };
+
+  const make$ = () => {
+    const calls = [];
+    const shell = (opts) => {
+      return (strings, ...values) => {
+        const cmd = strings.reduce((acc, str, i) => acc + str + (values[i] ?? ""), "");
+        calls.push({ cmd, env: opts?.env });
+        return { stdout: "" };
+      };
+    };
+    shell.calls = calls;
+    return shell;
+  };
+
+  beforeEach(() => {
+    // Enable RLM feedback, disable dry-run so $ is actually called
+    process.env.SAGE_PLUGIN_DRY_RUN = "";
+    process.env.SAGE_RLM_FEEDBACK = "1";
+    process.env.SAGE_SUGGEST_DEBOUNCE_MS = "10"; // fast debounce for tests
+
+    $mock = make$();
+    const { client: c, appLogCalls: logs } = makeClient();
+    appLogCalls = logs;
+
+    // We'll re-create plugin in each test for isolation
+  });
+
+  it("detects 'accepted' when user prompt closely matches suggestion", async () => {
+    const { client } = makeClient();
+    plugin = await SagePlugin({ client, $: $mock, directory: "/tmp" });
+
+    // Step 1: Send initial prompt (triggers capture)
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "how to optimize database queries" }] },
+    );
+
+    // Step 2: Complete the assistant response to reset state
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: { part: { type: "text", text: "Use indexes and EXPLAIN" } },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 10, output: 20 } },
+        },
+      },
+    });
+
+    // Step 3: Simulate suggestion being shown by triggering tui.prompt.append
+    // This would normally call `sage suggest skill ...` which sets internal state.
+    // Since we can't easily mock the async suggest flow, we test the correlation
+    // function indirectly by sending a prompt that would trigger correlation.
+    // The key insight: if no suggestion was shown, correlation returns null (harmless).
+
+    // Step 4: Send another prompt
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "how to optimize database queries" }] },
+    );
+
+    // No crash, no error — feedback path handled gracefully even without prior suggestion
+    // The capture hook should still have been called
+    const capturePromptCalls = $mock.calls.filter(
+      (c) => c.cmd.includes("capture") && c.cmd.includes("hook") && c.cmd.includes("prompt"),
+    );
+    expect(capturePromptCalls.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it("feedback calls use 'suggest feedback' not 'prompts append-feedback'", async () => {
+    const { client } = makeClient();
+    plugin = await SagePlugin({ client, $: $mock, directory: "/tmp" });
+
+    // Full cycle: prompt -> response -> prompt again
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "explain rust ownership" }] },
+    );
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: {
+          part: { type: "text", text: "Rust uses ownership rules..." },
+        },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 10, output: 20 } },
+        },
+      },
+    });
+
+    // Any feedback calls should use "suggest" path
+    const feedbackCalls = $mock.calls.filter((c) => c.cmd.includes("feedback"));
+    for (const call of feedbackCalls) {
+      expect(call.cmd).toContain("suggest");
+      expect(call.cmd).not.toContain("append-feedback");
+      expect(call.cmd).not.toContain("prompts");
+    }
+  });
+
+  it("implicit marker feedback: assistant response with [[sage:prompt_key=...]] marker", async () => {
+    const { client } = makeClient();
+    plugin = await SagePlugin({ client, $: $mock, directory: "/tmp" });
+
+    // Capture a prompt
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "build an MCP server" }] },
+    );
+
+    // Simulate assistant response that includes a sage prompt key marker
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: {
+          part: {
+            type: "text",
+            text: "Here is how to build an MCP server.\n[[sage:prompt_key=my-lib/mcp-builder]]",
+          },
+        },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 15, output: 30 } },
+        },
+      },
+    });
+
+    // The marker detection only fires if lastSuggestionId is set AND the key is in lastShownPromptKeys.
+    // Without a prior suggestion, this should be a no-op (no crash).
+    // This validates the implicit feedback code path doesn't error.
+  });
+
+  it("multiple prompt-response cycles maintain correct state for feedback", async () => {
+    const { client } = makeClient();
+    plugin = await SagePlugin({ client, $: $mock, directory: "/tmp" });
+
+    // Cycle 1
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "first question about rust" }] },
+    );
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: { part: { type: "text", text: "Rust is great." } },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 5, output: 10 } },
+        },
+      },
+    });
+
+    // Cycle 2
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      {
+        parts: [{ type: "text", text: "second question about typescript" }],
+      },
+    );
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: { part: { type: "text", text: "TypeScript adds types." } },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 8, output: 12 } },
+        },
+      },
+    });
+
+    // Cycle 3
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "third question about python" }] },
+    );
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: { part: { type: "text", text: "Python is interpreted." } },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 6, output: 8 } },
+        },
+      },
+    });
+
+    // Should have 3 capture prompt + 3 capture response calls
+    const captureCalls = $mock.calls.filter(
+      (c) => c.cmd.includes("capture") && c.cmd.includes("hook"),
+    );
+    expect(captureCalls.length).toBe(6); // 3 prompt + 3 response
+  });
+
+  it("session.created resets feedback state", async () => {
+    const { client } = makeClient();
+    plugin = await SagePlugin({ client, $: $mock, directory: "/tmp" });
+
+    // Capture a prompt
+    await plugin["chat.message"](
+      { sessionID: "s1", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "some prompt" }] },
+    );
+
+    // New session
+    await plugin.event({
+      event: {
+        type: "session.created",
+        properties: {
+          info: { id: "s2", parentID: null, directory: "/project" },
+        },
+      },
+    });
+
+    // After session reset, a new prompt should work cleanly
+    await plugin["chat.message"](
+      { sessionID: "s2", model: { modelID: "claude-3" } },
+      { parts: [{ type: "text", text: "fresh prompt in new session" }] },
+    );
+    await plugin.event({
+      event: {
+        type: "message.part.updated",
+        properties: { part: { type: "text", text: "fresh response" } },
+      },
+    });
+    await plugin.event({
+      event: {
+        type: "message.updated",
+        properties: {
+          info: { role: "assistant", tokens: { input: 3, output: 5 } },
+        },
+      },
+    });
+
+    // No errors means state properly reset
+  });
+});

package/rlm.e2e.test.js

@@ -0,0 +1,149 @@
+/**
+ * E2E Test: RLM Capture & Suggestion Loop via MCP
+ *
+ * Validates the full cycle:
+ *   1. Start daemon + MCP server (isolated HOME)
+ *   2. Baseline rlm_stats (zero state)
+ *   3. Inject captures via CLI
+ *   4. Run rlm_analyze_captures
+ *   5. Query rlm_list_patterns
+ *   6. Verify rlm_stats reflects the analysis
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import {
+  callTool,
+  createIsolatedHome,
+  injectCapture,
+  killProc,
+  resolveSageBin,
+  spawnSageMcp,
+  startDaemon,
+} from "./test-utils.js";
+
+const sageBin = resolveSageBin();
+const TIMEOUT = 60_000;
+
+describe("RLM E2E: capture -> analyze -> patterns -> stats", () => {
+  let daemonProc;
+  let mcpProc;
+  let client;
+  let tmpHome;
+
+  beforeAll(async () => {
+    tmpHome = createIsolatedHome();
+
+    // Start daemon first (provides RLM service via IPC)
+    daemonProc = await startDaemon(sageBin, tmpHome);
+
+    // Then start MCP server (routes RLM tool calls to daemon)
+    const mcp = await spawnSageMcp(sageBin, tmpHome);
+    mcpProc = mcp.proc;
+    client = mcp.client;
+  }, TIMEOUT);
+
+  afterAll(() => {
+    if (mcpProc) killProc(mcpProc);
+    if (daemonProc) killProc(daemonProc);
+  });
+
+  it(
+    "baseline rlm_stats returns zero state",
+    async () => {
+      const result = await callTool(client, "rlm_stats");
+      if (result.isError) {
+        console.error("rlm_stats error:", result.text);
+      }
+      expect(result.isError).toBe(false);
+      expect(result.json).toBeTruthy();
+      expect(result.json.total_analyses).toBe(0);
+      expect(result.json.patterns_discovered).toBe(0);
+    },
+    TIMEOUT,
+  );
+
+  it(
+    "inject captures via CLI without crashing",
+    async () => {
+      const prompts = [
+        {
+          prompt: "How do I optimize database queries in PostgreSQL?",
+          response: "Use EXPLAIN ANALYZE, add indexes, avoid SELECT *, use connection pooling.",
+        },
+        {
+          prompt: "What are best practices for REST API design?",
+          response: "Use proper HTTP methods, version your API, paginate responses, use HATEOAS.",
+        },
+        {
+          prompt: "How to handle errors in async Rust code?",
+          response:
+            "Use Result<T, E>, the ? operator, anyhow for applications, thiserror for libraries.",
+        },
+        {
+          prompt: "Explain React useEffect cleanup functions",
+          response:
+            "Return a cleanup function from useEffect to cancel subscriptions, timers, or listeners.",
+        },
+        {
+          prompt: "How to set up CI/CD with GitHub Actions?",
+          response:
+            "Create .github/workflows/*.yml, define jobs with steps, use caching for dependencies.",
+        },
+      ];
+
+      for (const { prompt, response } of prompts) {
+        const result = await injectCapture(sageBin, tmpHome, {
+          prompt,
+          response,
+        });
+        expect(result.promptExit).toBeDefined();
+        expect(result.responseExit).toBeDefined();
+      }
+    },
+    TIMEOUT,
+  );
+
+  it(
+    "rlm_analyze_captures returns analysis result",
+    async () => {
+      const { text, isError, json } = await callTool(client, "rlm_analyze_captures", {
+        goal: "optimize developer workflow",
+      });
+      expect(isError).toBe(false);
+      expect(text.length).toBeGreaterThan(0);
+      // Should have structured response
+      if (json) {
+        expect(json.model_used).toBeDefined();
+        expect(json.execution_time_ms).toBeDefined();
+      }
+    },
+    TIMEOUT,
+  );
+
+  it(
+    "rlm_list_patterns returns patterns array",
+    async () => {
+      const { isError, json } = await callTool(client, "rlm_list_patterns", {});
+      expect(isError).toBe(false);
+      if (json) {
+        expect(Array.isArray(json.patterns)).toBe(true);
+        expect(typeof json.count).toBe("number");
+      }
+    },
+    TIMEOUT,
+  );
+
+  it(
+    "rlm_stats after analysis reflects activity",
+    async () => {
+      const { isError, json } = await callTool(client, "rlm_stats");
+      expect(isError).toBe(false);
+      expect(json).toBeTruthy();
+      // After running analyze, total_analyses should have incremented
+      expect(typeof json.total_analyses).toBe("number");
+      expect(typeof json.patterns_discovered).toBe("number");
+      expect(typeof json.unique_sessions).toBe("number");
+    },
+    TIMEOUT,
+  );
+});