openclaw-triage-gate 1.0.1 → 1.1.0

package/ROADMAP.md

@@ -0,0 +1,33 @@
+# Roadmap
+
+Future enhancements planned for openclaw-triage-gate. None of these are committed to a timeline — they'll be built as needed.
+
+## Implemented
+
+### Keyword bypass list
+Always respond to messages containing certain keywords regardless of triage (e.g. "help", "urgent", the bot's name). Simple case-insensitive substring matching — no model call needed. Configure via `bypassKeywords` in the plugin config.
+
+### Confidence scores with configurable threshold
+Instead of binary RESPOND/SKIP, the triage model returns a confidence score (1-10). Users configure a threshold — messages scoring at or above it proceed to the main model. Enable with `useConfidenceScores: true` and tune via `confidenceThreshold` (default: 5).
+
+### Recent message history in triage context
+Include the last N messages from the group conversation in the triage prompt. This gives the triage model better context for deciding whether the bot should respond (e.g. understanding ongoing threads, follow-up questions). Configure via `historyCount` (default: 0, max: 20). Trade-off: increases triage token cost by ~500-1000 tokens per call.
+
+## Planned
+
+### Custom per-group triage prompts
+Different groups may need different triage criteria. A caregiving group might want the bot to respond more aggressively, while a social group should be more conservative. Allow `triagePrompt` to be overridden per group ID.
+
+### Analytics and metrics
+Track triage decisions over time: hit/miss ratio, tokens saved, false negatives, response times. Expose via a CLI command or dashboard. Helps users tune their triage prompt and verify cost savings.
+
+### Rate-based bypass
+In quiet groups (low message rate), skip triage and always respond. The cost savings from triage matter most in active groups. Configurable threshold (e.g. "if fewer than 5 messages in the last hour, skip triage").
+
+### Feedback loop
+Let users mark false negatives ("the bot should have responded to this") via a reaction or command. Store these examples and optionally include them in the triage prompt as few-shot examples.
+
+## Considered but not planned
+
+### Non-Anthropic/OpenAI provider formats
+Currently supports Anthropic Messages API and OpenAI-compatible Chat Completions API. Other provider formats (e.g. Google Gemini, Cohere) could be added to `providers.ts` if there's demand.

package/openclaw.plugin.json

@@ -31,6 +31,31 @@
       "logDecisions": {
         "type": "boolean",
         "description": "Whether to log each triage decision."
+      },
+      "bypassKeywords": {
+        "type": "array",
+        "items": { "type": "string" },
+        "description": "Keywords that bypass triage entirely. Messages containing any keyword (case-insensitive) always get a response."
+      },
+      "useConfidenceScores": {
+        "type": "boolean",
+        "description": "When true, the triage model returns a 1-10 confidence score instead of binary RESPOND/SKIP."
+      },
+      "confidenceThreshold": {
+        "type": "number",
+        "minimum": 1,
+        "maximum": 10,
+        "description": "Confidence threshold (1-10). Messages scoring at or above this value proceed to the main model. Only used when useConfidenceScores is true."
+      },
+      "historyCount": {
+        "type": "number",
+        "minimum": 0,
+        "maximum": 20,
+        "description": "Number of recent group messages to include in the triage prompt for context. Default: 0 (disabled), max: 20."
+      },
+      "botName": {
+        "type": "string",
+        "description": "The bot's name. Messages containing this name bypass triage and always get a response."
       }
     }
   },
@@ -58,6 +83,26 @@
     "logDecisions": {
       "label": "Log Decisions",
       "help": "Log each triage decision to the plugin logger (default: true)"
+    },
+    "bypassKeywords": {
+      "label": "Bypass Keywords",
+      "help": "Keywords that bypass triage entirely — messages containing any keyword always get a response (case-insensitive)"
+    },
+    "useConfidenceScores": {
+      "label": "Use Confidence Scores",
+      "help": "Use 1-10 confidence scoring instead of binary RESPOND/SKIP (default: false)"
+    },
+    "confidenceThreshold": {
+      "label": "Confidence Threshold",
+      "help": "Minimum confidence score (1-10) required for the bot to respond (default: 5)"
+    },
+    "historyCount": {
+      "label": "History Count",
+      "help": "Number of recent messages to include in the triage prompt for context (default: 0, max: 20)"
+    },
+    "botName": {
+      "label": "Bot Name",
+      "help": "The bot's name — messages containing this name always get a response (e.g. 'Nox')"
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openclaw-triage-gate",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "A lightweight triage gate for OpenClaw group chats. Uses a cheap model to decide if the bot should respond before the expensive main model runs.",
   "type": "module",
   "main": "src/index.ts",
@@ -31,5 +31,8 @@
     "install": {
       "npmSpec": "openclaw-triage-gate"
     }
+  },
+  "devDependencies": {
+    "vitest": "^4.1.2"
   }
 }

package/src/config.ts

@@ -51,6 +51,45 @@ export type TriageGateConfig = {
    * Default: true
    */
   logDecisions?: boolean;
+
+  /**
+   * Keywords that bypass triage entirely. When a message contains any of
+   * these keywords (case-insensitive substring match), the bot always
+   * responds without calling the triage model.
+   */
+  bypassKeywords?: string[];
+
+  /**
+   * When true, the triage model returns a 1-10 confidence score instead
+   * of binary RESPOND/SKIP. Messages scoring at or above the threshold
+   * proceed to the main model.
+   * Default: false
+   */
+  useConfidenceScores?: boolean;
+
+  /**
+   * Confidence threshold (1-10). Messages with a score at or above this
+   * value proceed to the main model. Only used when useConfidenceScores
+   * is true.
+   * Default: 5
+   */
+  confidenceThreshold?: number;
+
+  /**
+   * Number of recent group messages to include in the triage prompt for
+   * additional context. Helps the model make better decisions by seeing
+   * the conversation flow. The plugin maintains its own in-memory buffer
+   * per group (resets on plugin restart).
+   * Default: 0 (disabled), Max: 20
+   */
+  historyCount?: number;
+
+  /**
+   * The bot's name. When set, messages containing this name (case-insensitive)
+   * bypass triage and always get a response — similar to an @mention.
+   * Example: "Nox"
+   */
+  botName?: string;
 };
 
 /** The default triage model when none is configured. */
@@ -80,3 +119,15 @@ SKIP when:
 - A response would just be acknowledgment ("nice", "yeah", "lol")
 - The conversation is flowing fine without the bot
 - The message is a reaction, emoji, or sticker`;
+
+/** Default confidence threshold when useConfidenceScores is enabled. */
+export const DEFAULT_CONFIDENCE_THRESHOLD = 5;
+
+/** Default number of recent messages to include in triage context. */
+export const DEFAULT_HISTORY_COUNT = 0;
+
+/**
+ * The built-in confidence-scoring prompt. Instructs the model to reply
+ * with a single number 1-10 indicating response likelihood.
+ */
+export const DEFAULT_CONFIDENCE_PROMPT = `Reply with a single number 1-10 indicating how likely the bot should respond. 1 = definitely skip, 10 = definitely respond. Reply with ONLY the number.`;

package/src/index.ts

@@ -10,9 +10,62 @@
  */
 
 import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
-import { evaluateMessage } from "./triage.js";
+import { evaluateMessage, containsBypassKeyword } from "./triage.js";
 import { type TriageGateConfig } from "./config.js";
 
+// Guard against multiple registrations. OpenClaw calls register() for each
+// agent context, but we only need one before_dispatch hook globally.
+let registered = false;
+
+/**
+ * In-memory ring buffer that accumulates recent messages per group.
+ * Keyed by sessionKey (group ID). Resets on plugin restart — this is
+ * acceptable for triage context since it's best-effort.
+ */
+const groupHistoryBuffers = new Map<
+  string,
+  Array<{ role: string; content: string; ts: number }>
+>();
+
+/** Evict groups with no activity in the last hour to prevent unbounded growth. */
+const EVICT_AFTER_MS = 60 * 60 * 1000;
+
+function pushToBuffer(
+  groupId: string,
+  senderId: string,
+  content: string,
+  maxSize: number,
+): void {
+  let buffer = groupHistoryBuffers.get(groupId);
+  if (!buffer) {
+    buffer = [];
+    groupHistoryBuffers.set(groupId, buffer);
+  }
+  buffer.push({ role: senderId, content, ts: Date.now() });
+  // Keep only the last maxSize entries
+  if (buffer.length > maxSize) {
+    buffer.splice(0, buffer.length - maxSize);
+  }
+}
+
+function getBufferedHistory(
+  groupId: string,
+  count: number,
+): Array<{ role: string; content: string }> | undefined {
+  const buffer = groupHistoryBuffers.get(groupId);
+  if (!buffer?.length) return undefined;
+  return buffer.slice(-count).map(({ role, content }) => ({ role, content }));
+}
+
+function evictStaleBuffers(): void {
+  const cutoff = Date.now() - EVICT_AFTER_MS;
+  for (const [key, buffer] of groupHistoryBuffers) {
+    if (!buffer.length || buffer[buffer.length - 1].ts < cutoff) {
+      groupHistoryBuffers.delete(key);
+    }
+  }
+}
+
 export default definePluginEntry({
   id: "openclaw-triage-gate",
   name: "Triage Gate",
@@ -20,8 +73,17 @@ export default definePluginEntry({
     "Uses a cheap model to decide if the bot should respond in group chats, saving 75-90% of group chat token costs.",
 
   register(api: OpenClawPluginApi) {
+    if (registered) return;
+    registered = true;
+
     const config = (api.pluginConfig ?? {}) as TriageGateConfig;
     const logDecisions = config.logDecisions !== false; // default: true
+    const historyCount = Math.min(Math.max(config.historyCount ?? 0, 0), 20);
+
+    // Resolve bot name: explicit config > first agent's identity.name from OpenClaw config
+    const ocConfig = api.config as { agents?: { list?: Array<{ identity?: { name?: string } }> } };
+    const agentIdentityName = ocConfig.agents?.list?.[0]?.identity?.name;
+    const botNameLower = (config.botName ?? agentIdentityName ?? "").toLowerCase();
 
     // Pre-compute the set of groups to include/exclude for fast lookups
     const includeGroups = config.groups?.length
@@ -31,6 +93,11 @@ export default definePluginEntry({
       ? new Set(config.excludeGroups)
       : null;
 
+    // Periodically evict stale group buffers (every 10 minutes)
+    if (historyCount > 0) {
+      setInterval(evictStaleBuffers, 10 * 60 * 1000);
+    }
+
     /**
      * Resolve an API key for a provider/model using OpenClaw's auth system.
      * This keeps the plugin model-agnostic — it works with any provider
@@ -69,18 +136,60 @@ export default definePluginEntry({
         return { handled: true }; // Skip silently
       }
 
+      // Always respond when the bot's name is mentioned in the message
+      if (botNameLower && event.content.toLowerCase().includes(botNameLower)) {
+        if (logDecisions) {
+          api.logger.info?.(`triage-gate: RESPOND (bot name mentioned) — "${event.content.slice(0, 80)}"`);
+        }
+        // Still record in history buffer before passing through
+        if (historyCount > 0) {
+          pushToBuffer(groupId, event.senderId ?? "unknown", event.content, historyCount);
+        }
+        return; // let message through without triage
+      }
+
+      // Check for bypass keywords — if matched, skip triage entirely
+      if (config.bypassKeywords?.length) {
+        const matched = containsBypassKeyword(event.content, config.bypassKeywords);
+        if (matched) {
+          if (logDecisions) {
+            api.logger.info?.(`triage-gate: BYPASS (keyword: ${matched})`);
+          }
+          if (historyCount > 0) {
+            pushToBuffer(groupId, event.senderId ?? "unknown", event.content, historyCount);
+          }
+          return; // undefined = let message through without triage
+        }
+      }
+
+      // Get recent messages from the in-memory buffer
+      const recentMessages = historyCount > 0
+        ? getBufferedHistory(groupId, historyCount)
+        : undefined;
+
+      // Record this message in the buffer (after reading history so this
+      // message isn't included as "recent" context for itself)
+      if (historyCount > 0) {
+        pushToBuffer(groupId, event.senderId ?? "unknown", event.content, historyCount);
+      }
+
       // Run the triage model
       const result = await evaluateMessage({
         content: event.content,
+        senderName: event.senderId,
         config,
         resolveApiKey,
         logger: logDecisions ? api.logger : undefined,
+        recentMessages,
       });
 
       if (logDecisions) {
         const decision = result.shouldRespond ? "RESPOND" : "SKIP";
+        const scoreInfo = result.confidenceScore != null
+          ? `score: ${result.confidenceScore}/10, `
+          : "";
         api.logger.info?.(
-          `triage-gate: ${decision} (${result.durationMs}ms) — "${event.content.slice(0, 80)}"`,
+          `triage-gate: ${decision} (${scoreInfo}${result.durationMs}ms) — "${event.content.slice(0, 80)}"`,
         );
       }
 

package/src/triage.ts

@@ -12,6 +12,8 @@ import {
   DEFAULT_TRIAGE_MODEL,
   DEFAULT_TRIAGE_PROMPT,
   DEFAULT_MAX_TRIAGE_TOKENS,
+  DEFAULT_CONFIDENCE_THRESHOLD,
+  DEFAULT_CONFIDENCE_PROMPT,
   type TriageGateConfig,
 } from "./config.js";
 import { parseModelString, getProviderAdapter } from "./providers.js";
@@ -24,6 +26,9 @@ type TriageParams = {
   /** The message content to evaluate. */
   content: string;
 
+  /** Sender identifier (when available). */
+  senderName?: string;
+
   /** Plugin config. */
   config: TriageGateConfig;
 
@@ -38,6 +43,9 @@ type TriageParams = {
     info?: (msg: string) => void;
     warn?: (msg: string) => void;
   };
+
+  /** Recent messages from the group conversation for additional context. */
+  recentMessages?: Array<{ role: string; content: string }>;
 };
 
 export type TriageResult = {
@@ -49,6 +57,9 @@ export type TriageResult = {
 
   /** How long the triage call took in milliseconds. */
   durationMs: number;
+
+  /** Confidence score 1-10 when useConfidenceScores is enabled. */
+  confidenceScore?: number;
 };
 
 // ---------------------------------------------------------------------------
@@ -66,12 +77,14 @@ export type TriageResult = {
  * silently drop messages.
  */
 export async function evaluateMessage(params: TriageParams): Promise<TriageResult> {
-  const { content, config, resolveApiKey, logger } = params;
+  const { content, senderName, config, resolveApiKey, logger, recentMessages } = params;
   const startTime = Date.now();
 
   const modelString = config.triageModel ?? DEFAULT_TRIAGE_MODEL;
   const { provider, model } = parseModelString(modelString);
-  const prompt = config.triagePrompt ?? DEFAULT_TRIAGE_PROMPT;
+  const useConfidence = config.useConfidenceScores === true;
+  const prompt = config.triagePrompt
+    ?? (useConfidence ? DEFAULT_CONFIDENCE_PROMPT : DEFAULT_TRIAGE_PROMPT);
   const maxTokens = config.maxTriageTokens ?? DEFAULT_MAX_TRIAGE_TOKENS;
 
   try {
@@ -85,6 +98,9 @@ export async function evaluateMessage(params: TriageParams): Promise<TriageResul
     // Get the right adapter for this provider's API format
     const adapter = getProviderAdapter(provider);
 
+    // Build the user message with available context
+    const userMessage = buildTriageUserMessage({ content, senderName, recentMessages });
+
     // Make the API call
     const response = await fetch(adapter.endpoint, {
       method: "POST",
@@ -92,7 +108,7 @@ export async function evaluateMessage(params: TriageParams): Promise<TriageResul
       body: adapter.buildRequestBody({
         model,
         systemPrompt: prompt,
-        userMessage: `Message: ${content}`,
+        userMessage,
         maxTokens,
       }),
       signal: AbortSignal.timeout(5000), // 5s timeout — triage should be fast
@@ -108,8 +124,15 @@ export async function evaluateMessage(params: TriageParams): Promise<TriageResul
 
     const body = await response.json();
     const rawResponse = adapter.extractResponse(body);
-    const shouldRespond = parseTriageDecision(rawResponse);
 
+    if (useConfidence) {
+      const confidenceScore = parseConfidenceScore(rawResponse);
+      const threshold = config.confidenceThreshold ?? DEFAULT_CONFIDENCE_THRESHOLD;
+      const shouldRespond = confidenceScore >= threshold;
+      return { shouldRespond, rawResponse, durationMs: Date.now() - startTime, confidenceScore };
+    }
+
+    const shouldRespond = parseTriageDecision(rawResponse);
     return { shouldRespond, rawResponse, durationMs: Date.now() - startTime };
   } catch (error) {
     // On any error, default to letting the message through.
@@ -119,6 +142,92 @@ export async function evaluateMessage(params: TriageParams): Promise<TriageResul
   }
 }
 
+// ---------------------------------------------------------------------------
+// User message construction
+// ---------------------------------------------------------------------------
+
+/**
+ * Build the user message sent to the triage model, including available context.
+ * This gives the triage model enough information to make informed decisions
+ * about messages like "yes please" or "can you elaborate?" that only make
+ * sense in context.
+ */
+export function buildTriageUserMessage(params: {
+  content: string;
+  senderName?: string;
+  recentMessages?: Array<{ role: string; content: string }>;
+}): string {
+  const parts: string[] = [];
+
+  if (params.senderName) {
+    parts.push(`From: ${params.senderName}`);
+  }
+
+  parts.push(`Message: ${params.content}`);
+
+  if (params.recentMessages?.length) {
+    parts.push("");
+    parts.push(formatMessageHistory(params.recentMessages));
+  }
+
+  return parts.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Bypass keyword check
+// ---------------------------------------------------------------------------
+
+/**
+ * Check whether the message content contains any of the configured bypass
+ * keywords.  Matching is case-insensitive and uses substring containment.
+ *
+ * @returns The first matched keyword (lowercased), or `null` if none match.
+ */
+export function containsBypassKeyword(
+  content: string,
+  keywords: string[],
+): string | null {
+  if (!content || keywords.length === 0) return null;
+
+  const lowerContent = content.toLowerCase();
+  for (const keyword of keywords) {
+    const lowerKeyword = keyword.toLowerCase();
+    if (lowerContent.includes(lowerKeyword)) {
+      return lowerKeyword;
+    }
+  }
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Message history formatting
+// ---------------------------------------------------------------------------
+
+const MAX_MESSAGE_CONTENT_LENGTH = 200;
+
+/**
+ * Format an array of recent messages into a human-readable string for the
+ * triage prompt. Each message is rendered as "- [role]: [content]".
+ *
+ * Message content longer than 200 characters is truncated with "...".
+ * Returns an empty string for an empty array.
+ */
+export function formatMessageHistory(
+  messages: Array<{ role: string; content: string }>,
+): string {
+  if (messages.length === 0) return "";
+
+  const lines = messages.map(({ role, content }) => {
+    const truncated =
+      content.length > MAX_MESSAGE_CONTENT_LENGTH
+        ? content.slice(0, MAX_MESSAGE_CONTENT_LENGTH) + "..."
+        : content;
+    return `- ${role}: ${truncated}`;
+  });
+
+  return `Recent conversation:\n${lines.join("\n")}`;
+}
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
@@ -140,3 +249,30 @@ export function parseTriageDecision(response: string): boolean {
   // are worse than false positives (bot responds when it didn't need to).
   return true;
 }
+
+/**
+ * Parse a confidence score (1-10) from the triage model's response.
+ *
+ * Extraction strategy:
+ *   1. Look for the first integer 1-10 in the text
+ *   2. Fall back to keyword matching: "RESPOND" -> 10, "SKIP" -> 1
+ *   3. Default to 10 for ambiguous/empty responses (safe default — respond)
+ */
+export function parseConfidenceScore(response: string): number {
+  const trimmed = response.trim();
+
+  // Try to extract the first integer 1-10 from the response
+  const match = trimmed.match(/\b(10|[1-9])\b/);
+  if (match) {
+    return parseInt(match[1], 10);
+  }
+
+  // Backward compatibility: map RESPOND/SKIP keywords to scores
+  const upper = trimmed.toUpperCase();
+  if (upper.startsWith("RESPOND")) return 10;
+  if (upper.startsWith("SKIP")) return 1;
+
+  // Default to 10 (respond) for ambiguous or empty input.
+  // Same philosophy as parseTriageDecision: better to respond than drop.
+  return 10;
+}