@datafog/fogclaw 0.1.5 → 0.2.0

package/src/index.ts

@@ -1,7 +1,18 @@
 import { Scanner } from "./scanner.js";
 import { redact } from "./redactor.js";
 import { loadConfig } from "./config.js";
-import type { GuardrailAction } from "./types.js";
+import { RegexEngine } from "./engines/regex.js";
+import { createToolResultHandler } from "./tool-result-handler.js";
+import { createMessageSendingHandler } from "./message-sending-handler.js";
+import { resolveAction } from "./types.js";
+import type {
+  Entity,
+  FogClawConfig,
+  GuardrailAction,
+  RedactResult,
+  RedactStrategy,
+  ScanResult,
+} from "./types.js";
 
 export { Scanner } from "./scanner.js";
 export { redact } from "./redactor.js";
@@ -15,12 +26,80 @@ export type {
   GuardrailAction,
 } from "./types.js";
 
+function buildGuardrailPlan(entities: Entity[], config: FogClawConfig) {
+  const blocked: Entity[] = [];
+  const warned: Entity[] = [];
+  const redacted: Entity[] = [];
+
+  for (const entity of entities) {
+    const action = resolveAction(entity, config);
+    if (action === "block") blocked.push(entity);
+    else if (action === "warn") warned.push(entity);
+    else redacted.push(entity);
+  }
+
+  return { blocked, warned, redacted };
+}
+
+function planToSummary(plan: ReturnType<typeof buildGuardrailPlan>): {
+  total: number;
+  blocked: number;
+  warned: number;
+  redacted: number;
+  labels: {
+    blocked: string[];
+    warned: string[];
+    redacted: string[];
+  };
+} {
+  return {
+    total: plan.blocked.length + plan.warned.length + plan.redacted.length,
+    blocked: plan.blocked.length,
+    warned: plan.warned.length,
+    redacted: plan.redacted.length,
+    labels: {
+      blocked: [...new Set(plan.blocked.map((entity) => entity.label))],
+      warned: [...new Set(plan.warned.map((entity) => entity.label))],
+      redacted: [...new Set(plan.redacted.map((entity) => entity.label))],
+    },
+  };
+}
+
+function buildGuardrailContext(plan: ReturnType<typeof buildGuardrailPlan>, config: FogClawConfig): string[] {
+  const contextParts: string[] = [];
+
+  if (plan.blocked.length > 0) {
+    const types = [...new Set(plan.blocked.map((entity) => entity.label))].join(", ");
+    contextParts.push(
+      `[FOGCLAW GUARDRAIL — BLOCKED] The user's message contains sensitive information (${types}). ` +
+        `Do NOT process or repeat this information. Ask the user to rephrase without sensitive data.`,
+    );
+  }
+
+  if (plan.warned.length > 0) {
+    const types = [...new Set(plan.warned.map((entity) => entity.label))].join(", ");
+    contextParts.push(
+      `[FOGCLAW NOTICE] PII detected in user message: ${types}. Handle with care.`,
+    );
+  }
+
+  if (plan.redacted.length > 0) {
+    const labels = [...new Set(plan.redacted.map((entity) => entity.label))].join(", ");
+    contextParts.push(
+      `[FOGCLAW REDACTED] ${plan.redacted.length} entity(ies) prepared for ${config.redactStrategy} redaction (${labels}).`,
+    );
+  }
+
+  return contextParts;
+}
+
 /**
  * OpenClaw plugin definition.
  *
  * Registers:
  * - `before_agent_start` hook for automatic PII guardrail
  * - `fogclaw_scan` tool for on-demand entity detection
+ * - `fogclaw_preview` tool for dry-run policy simulation
  * - `fogclaw_redact` tool for on-demand redaction
  */
 const fogclaw = {
@@ -48,47 +127,35 @@ const fogclaw = {
       const message = event.prompt ?? "";
       if (!message) return;
 
-      const result = await scanner.scan(message);
-
+      const result: ScanResult = await scanner.scan(message);
       if (result.entities.length === 0) return;
 
-      // Classify entities by their configured action
-      const blocked: typeof result.entities = [];
-      const warned: typeof result.entities = [];
-      const toRedact: typeof result.entities = [];
-
-      for (const entity of result.entities) {
-        const action: GuardrailAction =
-          config.entityActions[entity.label] ?? config.guardrail_mode;
-        if (action === "block") blocked.push(entity);
-        else if (action === "warn") warned.push(entity);
-        else if (action === "redact") toRedact.push(entity);
-      }
-
-      const contextParts: string[] = [];
+      const plan = buildGuardrailPlan(result.entities, config);
+      const contextParts = buildGuardrailContext(plan, config);
 
-      // "block" — inject a strong instruction to refuse
-      if (blocked.length > 0) {
-        const types = [...new Set(blocked.map((e) => e.label))].join(", ");
-        contextParts.push(
-          `[FOGCLAW GUARDRAIL — BLOCKED] The user's message contains sensitive information (${types}). ` +
-          `Do NOT process or repeat this information. Ask the user to rephrase without sensitive data.`,
+      if (config.auditEnabled) {
+        const summary = planToSummary(plan);
+        api.logger?.info(
+          `[FOGCLAW AUDIT] guardrail_scan ${JSON.stringify({
+            totalEntities: summary.total,
+            blocked: summary.blocked,
+            warned: summary.warned,
+            redacted: summary.redacted,
+            blockedLabels: summary.labels.blocked,
+            warnedLabels: summary.labels.warned,
+            redactedLabels: summary.labels.redacted,
+          })}`,
         );
       }
 
-      // "warn" — inject a warning notice
-      if (warned.length > 0) {
-        const types = [...new Set(warned.map((e) => e.label))].join(", ");
-        contextParts.push(
-          `[FOGCLAW NOTICE] PII detected in user message: ${types}. Handle with care.`,
+      if (plan.redacted.length > 0) {
+        const redactedResult: RedactResult = redact(
+          message,
+          plan.redacted,
+          config.redactStrategy,
         );
-      }
-
-      // "redact" — replace PII with tokens
-      if (toRedact.length > 0) {
-        const redacted = redact(message, toRedact, config.redactStrategy);
         contextParts.push(
-          `[FOGCLAW REDACTED] The following is the user's message with PII redacted:\n${redacted.redacted_text}`,
+          `[FOGCLAW REDACTED] The following is the user's message with PII redacted:\n${redactedResult.redacted_text}`,
         );
       }
 
@@ -97,6 +164,15 @@ const fogclaw = {
       }
     });
 
+    // --- HOOK: Scan tool results for PII before persistence ---
+    const toolResultRegex = new RegexEngine();
+    const toolResultHandler = createToolResultHandler(config, toolResultRegex, api.logger);
+    api.on("tool_result_persist", toolResultHandler);
+
+    // --- HOOK: Scan outbound messages for PII before delivery ---
+    const messageSendingHandler = createMessageSendingHandler(config, scanner, api.logger);
+    api.on("message_sending", messageSendingHandler);
+
     // --- TOOL: On-demand scan ---
     api.registerTool(
       {
@@ -138,7 +214,7 @@ const fogclaw = {
                     count: result.entities.length,
                     summary:
                       result.entities.length > 0
-                        ? `Found ${result.entities.length} entities: ${[...new Set(result.entities.map((e) => e.label))].join(", ")}`
+                        ? `Found ${result.entities.length} entities: ${[...new Set(result.entities.map((entity) => entity.label))].join(", ")}`
                         : "No entities detected",
                   },
                   null,
@@ -151,6 +227,88 @@ const fogclaw = {
       }
     );
 
+    // --- TOOL: Policy preview ---
+    api.registerTool(
+      {
+        name: "fogclaw_preview",
+        id: "fogclaw_preview",
+        description:
+          "Preview which entities will be blocked, warned, or redacted and the redacted message, without changing runtime behavior.",
+        schema: {
+          type: "object",
+          properties: {
+            text: {
+              type: "string",
+              description: "Text to run through FogClaw policy preview",
+            },
+            strategy: {
+              type: "string",
+              description:
+                'Override redaction strategy for the preview: "token" ([EMAIL_1]), "mask" (****), or "hash" ([EMAIL_a1b2c3...]).',
+              enum: ["token", "mask", "hash"],
+            },
+            custom_labels: {
+              type: "array",
+              items: { type: "string" },
+              description: "Additional entity labels for zero-shot detection",
+            },
+          },
+          required: ["text"],
+        },
+        handler: async ({
+          text,
+          strategy,
+          custom_labels,
+        }: {
+          text: string;
+          strategy?: "token" | "mask" | "hash";
+          custom_labels?: string[];
+        }) => {
+          const result = await scanner.scan(text, custom_labels);
+          const plan = buildGuardrailPlan(result.entities, config);
+          const summary = planToSummary(plan);
+          const redacted = redact(
+            text,
+            plan.redacted,
+            strategy ?? config.redactStrategy,
+          );
+
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify(
+                  {
+                    entities: result.entities,
+                    totalEntities: summary.total,
+                    actionPlan: {
+                      blocked: {
+                        count: summary.blocked,
+                        labels: summary.labels.blocked,
+                      },
+                      warned: {
+                        count: summary.warned,
+                        labels: summary.labels.warned,
+                      },
+                      redacted: {
+                        count: summary.redacted,
+                        labels: summary.labels.redacted,
+                      },
+                    },
+                    redactedText: redacted.redacted_text,
+                    redactionStrategy: strategy ?? config.redactStrategy,
+                    mapping: redacted.mapping,
+                  },
+                  null,
+                  2,
+                ),
+              },
+            ],
+          };
+        },
+      }
+    );
+
     // --- TOOL: On-demand redact ---
     api.registerTool(
       {
@@ -215,7 +373,7 @@ const fogclaw = {
     );
 
     api.logger?.info(
-      `[fogclaw] Plugin registered — guardrail: ${config.guardrail_mode}, model: ${config.model}, custom entities: ${config.custom_entities.length}`,
+      `[fogclaw] Plugin registered — guardrail: ${config.guardrail_mode}, model: ${config.model}, custom entities: ${config.custom_entities.length}, audit: ${config.auditEnabled}`,
     );
   },
 };

package/src/message-sending-handler.ts

@@ -0,0 +1,87 @@
+/**
+ * Async message_sending hook handler for FogClaw.
+ *
+ * Scans outbound message text for PII using the full Scanner
+ * (regex + GLiNER), redacts detected entities, and returns
+ * modified content. Never cancels message delivery.
+ *
+ * Note: message_sending is defined in OpenClaw but not yet invoked
+ * upstream. This handler activates automatically when wired.
+ */
+
+import type { Scanner } from "./scanner.js";
+import { redact } from "./redactor.js";
+import { resolveAction } from "./types.js";
+import type { Entity, FogClawConfig } from "./types.js";
+
+interface Logger {
+  info(msg: string): void;
+  warn(msg: string): void;
+}
+
+export interface MessageSendingEvent {
+  to: string;
+  content: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface MessageSendingContext {
+  channelId: string;
+  accountId?: string;
+  conversationId?: string;
+}
+
+export interface MessageSendingResult {
+  content?: string;
+  cancel?: boolean;
+}
+
+/**
+ * Create an async message_sending hook handler.
+ *
+ * Uses the full Scanner (regex + GLiNER) since this hook supports
+ * async handlers. All guardrail modes produce span-level redaction;
+ * cancel is never returned.
+ */
+export function createMessageSendingHandler(
+  config: FogClawConfig,
+  scanner: Scanner,
+  logger?: Logger,
+): (event: MessageSendingEvent, ctx: MessageSendingContext) => Promise<MessageSendingResult | void> {
+  return async (
+    event: MessageSendingEvent,
+    _ctx: MessageSendingContext,
+  ): Promise<MessageSendingResult | void> => {
+    const text = event.content;
+    if (!text) return;
+
+    const result = await scanner.scan(text);
+    if (result.entities.length === 0) return;
+
+    // All modes produce span-level redaction for outbound messages.
+    const actionableEntities = result.entities.filter((entity) => {
+      const action = resolveAction(entity, config);
+      return action === "redact" || action === "block" || action === "warn";
+    });
+
+    if (actionableEntities.length === 0) return;
+
+    const redacted = redact(text, actionableEntities, config.redactStrategy);
+
+    // Audit logging
+    if (config.auditEnabled && logger) {
+      const labels = [...new Set(actionableEntities.map((e) => e.label))];
+      logger.info(
+        `[FOGCLAW AUDIT] outbound_scan ${JSON.stringify({
+          totalEntities: actionableEntities.length,
+          labels,
+          channelId: _ctx.channelId ?? null,
+          source: "outbound",
+        })}`,
+      );
+    }
+
+    // Never cancel — always deliver the redacted version.
+    return { content: redacted.redacted_text };
+  };
+}

package/src/scanner.ts

@@ -1,23 +1,44 @@
-import type { Entity, FogClawConfig, ScanResult } from "./types.js";
+import type { Entity, FogClawConfig } from "./types.js";
+import { canonicalType } from "./types.js";
 import { RegexEngine } from "./engines/regex.js";
 import { GlinerEngine } from "./engines/gliner.js";
 
+type AllowlistPatternCache = {
+  values: Set<string>;
+  patterns: RegExp[];
+  entityValues: Map<string, Set<string>>;
+};
+
+function normalizeAllowlistValue(value: string): string {
+  return value.trim().toLowerCase();
+}
+
+function buildPatternMaps(value: string[] | undefined): RegExp[] {
+  if (!value || value.length === 0) {
+    return [];
+  }
+
+  return value.map((pattern) => new RegExp(pattern, "i"));
+}
+
 export class Scanner {
   private regexEngine: RegexEngine;
   private glinerEngine: GlinerEngine;
   private glinerAvailable = false;
   private config: FogClawConfig;
+  private allowlist: AllowlistPatternCache;
 
   constructor(config: FogClawConfig) {
     this.config = config;
     this.regexEngine = new RegexEngine();
-    this.glinerEngine = new GlinerEngine(
-      config.model,
-      config.confidence_threshold,
-    );
+
+    const glinerThreshold = this.computeGlinerThreshold(config);
+    this.glinerEngine = new GlinerEngine(config.model, glinerThreshold);
     if (config.custom_entities.length > 0) {
       this.glinerEngine.setCustomLabels(config.custom_entities);
     }
+
+    this.allowlist = this.buildAllowlistCache(config.allowlist);
   }
 
   async initialize(): Promise<void> {
@@ -32,19 +53,25 @@ export class Scanner {
     }
   }
 
-  async scan(text: string, extraLabels?: string[]): Promise<ScanResult> {
+  async scan(text: string, extraLabels?: string[]): Promise<{ entities: Entity[]; text: string }> {
     if (!text) return { entities: [], text };
 
     // Step 1: Regex pass (always runs, synchronous)
-    const regexEntities = this.regexEngine.scan(text);
+    const regexEntities = this.filterByPolicy(this.regexEngine.scan(text));
 
     // Step 2: GLiNER pass (if available)
     let glinerEntities: Entity[] = [];
     if (this.glinerAvailable) {
       try {
         glinerEntities = await this.glinerEngine.scan(text, extraLabels);
+        glinerEntities = this.filterByConfidence(glinerEntities);
+        glinerEntities = this.filterByPolicy(glinerEntities);
       } catch (err) {
-        console.warn(`[fogclaw] GLiNER scan failed, using regex results only: ${err instanceof Error ? err.message : String(err)}`);
+        console.warn(
+          `[fogclaw] GLiNER scan failed, using regex results only: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
       }
     }
 
@@ -53,6 +80,85 @@ export class Scanner {
 
     return { entities: merged, text };
   }
+
+  private filterByConfidence(entities: Entity[]): Entity[] {
+    return entities.filter((entity) => {
+      const threshold = this.getThresholdForLabel(entity.label);
+      return entity.confidence >= threshold;
+    });
+  }
+
+  private filterByPolicy(entities: Entity[]): Entity[] {
+    if (
+      this.allowlist.values.size === 0 &&
+      this.allowlist.patterns.length === 0 &&
+      this.allowlist.entityValues.size === 0
+    ) {
+      return entities;
+    }
+
+    return entities.filter((entity) => !this.shouldAllowlistEntity(entity));
+  }
+
+  private shouldAllowlistEntity(entity: Entity): boolean {
+    const normalizedText = normalizeAllowlistValue(entity.text);
+
+    if (this.allowlist.values.has(normalizedText)) {
+      return true;
+    }
+
+    if (this.allowlist.patterns.some((pattern) => pattern.test(entity.text))) {
+      return true;
+    }
+
+    const entityValues = this.allowlist.entityValues.get(entity.label);
+    if (entityValues && entityValues.has(normalizedText)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private getThresholdForLabel(label: string): number {
+    const canonicalLabel = canonicalType(label);
+    return this.config.entityConfidenceThresholds[canonicalLabel] ?? this.config.confidence_threshold;
+  }
+
+  private computeGlinerThreshold(config: FogClawConfig): number {
+    const thresholds = Object.values(config.entityConfidenceThresholds);
+    if (thresholds.length === 0) {
+      return config.confidence_threshold;
+    }
+
+    return Math.min(config.confidence_threshold, ...thresholds);
+  }
+
+  private buildAllowlistCache(allowlist: FogClawConfig["allowlist"]): AllowlistPatternCache {
+    const globalValues = new Set(
+      allowlist.values.map((value) => normalizeAllowlistValue(value)),
+    );
+
+    const globalPatterns = buildPatternMaps(allowlist.patterns);
+
+    const entityValues = new Map<string, Set<string>>();
+    for (const [entityType, values] of Object.entries(allowlist.entities)) {
+      const canonical = canonicalType(entityType);
+      const uniqueValues = values
+        .map((value) => normalizeAllowlistValue(value))
+        .filter((value) => value.length > 0);
+      entityValues.set(canonical, new Set(uniqueValues));
+    }
+
+    return {
+      values: globalValues,
+      patterns: globalPatterns,
+      entityValues,
+    };
+  }
+
+  get isGlinerAvailable(): boolean {
+    return this.glinerAvailable;
+  }
 }
 
 /**

package/src/tool-result-handler.ts

@@ -0,0 +1,133 @@
+/**
+ * Synchronous tool_result_persist hook handler for FogClaw.
+ *
+ * Scans tool result text for PII using the regex engine (synchronous),
+ * redacts detected entities, and returns the transformed message.
+ * GLiNER is not used here because tool_result_persist is synchronous-only.
+ */
+
+import { RegexEngine } from "./engines/regex.js";
+import { redact } from "./redactor.js";
+import { extractText, replaceText } from "./extract.js";
+import { canonicalType, resolveAction } from "./types.js";
+import type { Entity, FogClawConfig } from "./types.js";
+
+interface Logger {
+  info(msg: string): void;
+  warn(msg: string): void;
+}
+
+export interface ToolResultPersistEvent {
+  toolName?: string;
+  toolCallId?: string;
+  message: unknown;
+  isSynthetic?: boolean;
+}
+
+export interface ToolResultPersistContext {
+  agentId?: string;
+  sessionKey?: string;
+  toolName?: string;
+  toolCallId?: string;
+}
+
+/**
+ * Build an allowlist filter from config. Replicates Scanner.filterByPolicy
+ * and Scanner.shouldAllowlistEntity logic synchronously.
+ */
+function buildAllowlistFilter(config: FogClawConfig): (entity: Entity) => boolean {
+  const globalValues = new Set(
+    config.allowlist.values.map((v) => v.trim().toLowerCase()),
+  );
+
+  const globalPatterns = config.allowlist.patterns
+    .filter((p) => p.length > 0)
+    .map((p) => new RegExp(p, "i"));
+
+  const entityValues = new Map<string, Set<string>>();
+  for (const [entityType, values] of Object.entries(config.allowlist.entities)) {
+    const canonical = canonicalType(entityType);
+    const set = new Set(
+      values
+        .map((v) => v.trim().toLowerCase())
+        .filter((v) => v.length > 0),
+    );
+    entityValues.set(canonical, set);
+  }
+
+  // Short-circuit: if no allowlist entries, keep everything
+  if (globalValues.size === 0 && globalPatterns.length === 0 && entityValues.size === 0) {
+    return () => true;
+  }
+
+  // Return true if entity should be KEPT (not allowlisted)
+  return (entity: Entity): boolean => {
+    const normalizedText = entity.text.trim().toLowerCase();
+
+    if (globalValues.has(normalizedText)) return false;
+    if (globalPatterns.some((pattern) => pattern.test(entity.text))) return false;
+
+    const perEntity = entityValues.get(entity.label);
+    if (perEntity && perEntity.has(normalizedText)) return false;
+
+    return true;
+  };
+}
+
+/**
+ * Create a synchronous tool_result_persist hook handler.
+ *
+ * The returned function must NOT return a Promise — OpenClaw rejects
+ * async tool_result_persist handlers.
+ */
+export function createToolResultHandler(
+  config: FogClawConfig,
+  regexEngine: RegexEngine,
+  logger?: Logger,
+): (event: ToolResultPersistEvent, ctx: ToolResultPersistContext) => { message: unknown } | void {
+  const shouldKeep = buildAllowlistFilter(config);
+
+  return (event: ToolResultPersistEvent, _ctx: ToolResultPersistContext): { message: unknown } | void => {
+    const text = extractText(event.message);
+    if (!text) return;
+
+    // Scan with regex engine (synchronous)
+    let entities = regexEngine.scan(text);
+    if (entities.length === 0) return;
+
+    // Apply allowlist filtering
+    entities = entities.filter(shouldKeep);
+    if (entities.length === 0) return;
+
+    // All guardrail modes produce span-level redaction in tool results.
+    // Determine which entities are actionable (all of them — block/warn/redact
+    // all produce redaction at the tool result level).
+    const actionableEntities = entities.filter((entity) => {
+      const action = resolveAction(entity, config);
+      return action === "redact" || action === "block" || action === "warn";
+    });
+
+    if (actionableEntities.length === 0) return;
+
+    // Redact
+    const result = redact(text, actionableEntities, config.redactStrategy);
+
+    // Replace text in the message
+    const modifiedMessage = replaceText(event.message, result.redacted_text);
+
+    // Audit logging
+    if (config.auditEnabled && logger) {
+      const labels = [...new Set(actionableEntities.map((e) => e.label))];
+      logger.info(
+        `[FOGCLAW AUDIT] tool_result_scan ${JSON.stringify({
+          totalEntities: actionableEntities.length,
+          labels,
+          toolName: event.toolName ?? null,
+          source: "tool_result",
+        })}`,
+      );
+    }
+
+    return { message: modifiedMessage };
+  };
+}