@datafog/fogclaw 0.2.0 → 0.3.0

package/src/scanner.ts

@@ -1,196 +0,0 @@
-import type { Entity, FogClawConfig } from "./types.js";
-import { canonicalType } from "./types.js";
-import { RegexEngine } from "./engines/regex.js";
-import { GlinerEngine } from "./engines/gliner.js";
-
-type AllowlistPatternCache = {
-  values: Set<string>;
-  patterns: RegExp[];
-  entityValues: Map<string, Set<string>>;
-};
-
-function normalizeAllowlistValue(value: string): string {
-  return value.trim().toLowerCase();
-}
-
-function buildPatternMaps(value: string[] | undefined): RegExp[] {
-  if (!value || value.length === 0) {
-    return [];
-  }
-
-  return value.map((pattern) => new RegExp(pattern, "i"));
-}
-
-export class Scanner {
-  private regexEngine: RegexEngine;
-  private glinerEngine: GlinerEngine;
-  private glinerAvailable = false;
-  private config: FogClawConfig;
-  private allowlist: AllowlistPatternCache;
-
-  constructor(config: FogClawConfig) {
-    this.config = config;
-    this.regexEngine = new RegexEngine();
-
-    const glinerThreshold = this.computeGlinerThreshold(config);
-    this.glinerEngine = new GlinerEngine(config.model, glinerThreshold);
-    if (config.custom_entities.length > 0) {
-      this.glinerEngine.setCustomLabels(config.custom_entities);
-    }
-
-    this.allowlist = this.buildAllowlistCache(config.allowlist);
-  }
-
-  async initialize(): Promise<void> {
-    try {
-      await this.glinerEngine.initialize();
-      this.glinerAvailable = true;
-    } catch (err) {
-      console.warn(
-        `[fogclaw] GLiNER failed to initialize, falling back to regex-only mode: ${err instanceof Error ? err.message : String(err)}`,
-      );
-      this.glinerAvailable = false;
-    }
-  }
-
-  async scan(text: string, extraLabels?: string[]): Promise<{ entities: Entity[]; text: string }> {
-    if (!text) return { entities: [], text };
-
-    // Step 1: Regex pass (always runs, synchronous)
-    const regexEntities = this.filterByPolicy(this.regexEngine.scan(text));
-
-    // Step 2: GLiNER pass (if available)
-    let glinerEntities: Entity[] = [];
-    if (this.glinerAvailable) {
-      try {
-        glinerEntities = await this.glinerEngine.scan(text, extraLabels);
-        glinerEntities = this.filterByConfidence(glinerEntities);
-        glinerEntities = this.filterByPolicy(glinerEntities);
-      } catch (err) {
-        console.warn(
-          `[fogclaw] GLiNER scan failed, using regex results only: ${
-            err instanceof Error ? err.message : String(err)
-          }`,
-        );
-      }
-    }
-
-    // Step 3: Merge and deduplicate
-    const merged = deduplicateEntities([...regexEntities, ...glinerEntities]);
-
-    return { entities: merged, text };
-  }
-
-  private filterByConfidence(entities: Entity[]): Entity[] {
-    return entities.filter((entity) => {
-      const threshold = this.getThresholdForLabel(entity.label);
-      return entity.confidence >= threshold;
-    });
-  }
-
-  private filterByPolicy(entities: Entity[]): Entity[] {
-    if (
-      this.allowlist.values.size === 0 &&
-      this.allowlist.patterns.length === 0 &&
-      this.allowlist.entityValues.size === 0
-    ) {
-      return entities;
-    }
-
-    return entities.filter((entity) => !this.shouldAllowlistEntity(entity));
-  }
-
-  private shouldAllowlistEntity(entity: Entity): boolean {
-    const normalizedText = normalizeAllowlistValue(entity.text);
-
-    if (this.allowlist.values.has(normalizedText)) {
-      return true;
-    }
-
-    if (this.allowlist.patterns.some((pattern) => pattern.test(entity.text))) {
-      return true;
-    }
-
-    const entityValues = this.allowlist.entityValues.get(entity.label);
-    if (entityValues && entityValues.has(normalizedText)) {
-      return true;
-    }
-
-    return false;
-  }
-
-  private getThresholdForLabel(label: string): number {
-    const canonicalLabel = canonicalType(label);
-    return this.config.entityConfidenceThresholds[canonicalLabel] ?? this.config.confidence_threshold;
-  }
-
-  private computeGlinerThreshold(config: FogClawConfig): number {
-    const thresholds = Object.values(config.entityConfidenceThresholds);
-    if (thresholds.length === 0) {
-      return config.confidence_threshold;
-    }
-
-    return Math.min(config.confidence_threshold, ...thresholds);
-  }
-
-  private buildAllowlistCache(allowlist: FogClawConfig["allowlist"]): AllowlistPatternCache {
-    const globalValues = new Set(
-      allowlist.values.map((value) => normalizeAllowlistValue(value)),
-    );
-
-    const globalPatterns = buildPatternMaps(allowlist.patterns);
-
-    const entityValues = new Map<string, Set<string>>();
-    for (const [entityType, values] of Object.entries(allowlist.entities)) {
-      const canonical = canonicalType(entityType);
-      const uniqueValues = values
-        .map((value) => normalizeAllowlistValue(value))
-        .filter((value) => value.length > 0);
-      entityValues.set(canonical, new Set(uniqueValues));
-    }
-
-    return {
-      values: globalValues,
-      patterns: globalPatterns,
-      entityValues,
-    };
-  }
-
-  get isGlinerAvailable(): boolean {
-    return this.glinerAvailable;
-  }
-}
-
-/**
- * Remove overlapping entity spans. When two entities overlap,
- * keep the one with higher confidence. If equal, prefer regex.
- */
-function deduplicateEntities(entities: Entity[]): Entity[] {
-  if (entities.length <= 1) return entities;
-
-  // Sort by start position, then by confidence descending
-  const sorted = [...entities].sort((a, b) => {
-    if (a.start !== b.start) return a.start - b.start;
-    return b.confidence - a.confidence;
-  });
-
-  const result: Entity[] = [sorted[0]];
-
-  for (let i = 1; i < sorted.length; i++) {
-    const current = sorted[i];
-    const last = result[result.length - 1];
-
-    // Check for overlap
-    if (current.start < last.end) {
-      // Overlapping: keep higher confidence (already in result if first)
-      if (current.confidence > last.confidence) {
-        result[result.length - 1] = current;
-      }
-      // Otherwise keep what's already in result
-    } else {
-      result.push(current);
-    }
-  }
-
-  return result;
-}

package/src/tool-result-handler.ts

@@ -1,133 +0,0 @@
-/**
- * Synchronous tool_result_persist hook handler for FogClaw.
- *
- * Scans tool result text for PII using the regex engine (synchronous),
- * redacts detected entities, and returns the transformed message.
- * GLiNER is not used here because tool_result_persist is synchronous-only.
- */
-
-import { RegexEngine } from "./engines/regex.js";
-import { redact } from "./redactor.js";
-import { extractText, replaceText } from "./extract.js";
-import { canonicalType, resolveAction } from "./types.js";
-import type { Entity, FogClawConfig } from "./types.js";
-
-interface Logger {
-  info(msg: string): void;
-  warn(msg: string): void;
-}
-
-export interface ToolResultPersistEvent {
-  toolName?: string;
-  toolCallId?: string;
-  message: unknown;
-  isSynthetic?: boolean;
-}
-
-export interface ToolResultPersistContext {
-  agentId?: string;
-  sessionKey?: string;
-  toolName?: string;
-  toolCallId?: string;
-}
-
-/**
- * Build an allowlist filter from config. Replicates Scanner.filterByPolicy
- * and Scanner.shouldAllowlistEntity logic synchronously.
- */
-function buildAllowlistFilter(config: FogClawConfig): (entity: Entity) => boolean {
-  const globalValues = new Set(
-    config.allowlist.values.map((v) => v.trim().toLowerCase()),
-  );
-
-  const globalPatterns = config.allowlist.patterns
-    .filter((p) => p.length > 0)
-    .map((p) => new RegExp(p, "i"));
-
-  const entityValues = new Map<string, Set<string>>();
-  for (const [entityType, values] of Object.entries(config.allowlist.entities)) {
-    const canonical = canonicalType(entityType);
-    const set = new Set(
-      values
-        .map((v) => v.trim().toLowerCase())
-        .filter((v) => v.length > 0),
-    );
-    entityValues.set(canonical, set);
-  }
-
-  // Short-circuit: if no allowlist entries, keep everything
-  if (globalValues.size === 0 && globalPatterns.length === 0 && entityValues.size === 0) {
-    return () => true;
-  }
-
-  // Return true if entity should be KEPT (not allowlisted)
-  return (entity: Entity): boolean => {
-    const normalizedText = entity.text.trim().toLowerCase();
-
-    if (globalValues.has(normalizedText)) return false;
-    if (globalPatterns.some((pattern) => pattern.test(entity.text))) return false;
-
-    const perEntity = entityValues.get(entity.label);
-    if (perEntity && perEntity.has(normalizedText)) return false;
-
-    return true;
-  };
-}
-
-/**
- * Create a synchronous tool_result_persist hook handler.
- *
- * The returned function must NOT return a Promise — OpenClaw rejects
- * async tool_result_persist handlers.
- */
-export function createToolResultHandler(
-  config: FogClawConfig,
-  regexEngine: RegexEngine,
-  logger?: Logger,
-): (event: ToolResultPersistEvent, ctx: ToolResultPersistContext) => { message: unknown } | void {
-  const shouldKeep = buildAllowlistFilter(config);
-
-  return (event: ToolResultPersistEvent, _ctx: ToolResultPersistContext): { message: unknown } | void => {
-    const text = extractText(event.message);
-    if (!text) return;
-
-    // Scan with regex engine (synchronous)
-    let entities = regexEngine.scan(text);
-    if (entities.length === 0) return;
-
-    // Apply allowlist filtering
-    entities = entities.filter(shouldKeep);
-    if (entities.length === 0) return;
-
-    // All guardrail modes produce span-level redaction in tool results.
-    // Determine which entities are actionable (all of them — block/warn/redact
-    // all produce redaction at the tool result level).
-    const actionableEntities = entities.filter((entity) => {
-      const action = resolveAction(entity, config);
-      return action === "redact" || action === "block" || action === "warn";
-    });
-
-    if (actionableEntities.length === 0) return;
-
-    // Redact
-    const result = redact(text, actionableEntities, config.redactStrategy);
-
-    // Replace text in the message
-    const modifiedMessage = replaceText(event.message, result.redacted_text);
-
-    // Audit logging
-    if (config.auditEnabled && logger) {
-      const labels = [...new Set(actionableEntities.map((e) => e.label))];
-      logger.info(
-        `[FOGCLAW AUDIT] tool_result_scan ${JSON.stringify({
-          totalEntities: actionableEntities.length,
-          labels,
-          toolName: event.toolName ?? null,
-          source: "tool_result",
-        })}`,
-      );
-    }
-
-    return { message: modifiedMessage };
-  };
-}

package/src/types.ts

@@ -1,75 +0,0 @@
-export interface Entity {
-  text: string;
-  label: string;
-  start: number;
-  end: number;
-  confidence: number;
-  source: "regex" | "gliner";
-}
-
-export type RedactStrategy = "token" | "mask" | "hash";
-
-export type GuardrailAction = "redact" | "block" | "warn";
-
-export interface EntityConfidenceThresholds {
-  [entityType: string]: number;
-}
-
-export interface EntityAllowlist {
-  values: string[];
-  patterns: string[];
-  entities: Record<string, string[]>;
-}
-
-export interface FogClawConfig {
-  enabled: boolean;
-  guardrail_mode: GuardrailAction;
-  redactStrategy: RedactStrategy;
-  model: string;
-  confidence_threshold: number;
-  custom_entities: string[];
-  entityActions: Record<string, GuardrailAction>;
-  entityConfidenceThresholds: EntityConfidenceThresholds;
-  allowlist: EntityAllowlist;
-  auditEnabled: boolean;
-}
-
-export interface ScanResult {
-  entities: Entity[];
-  text: string;
-}
-
-export interface RedactResult {
-  redacted_text: string;
-  mapping: Record<string, string>;
-  entities: Entity[];
-}
-
-export interface GuardrailPlan {
-  blocked: Entity[];
-  warned: Entity[];
-  redacted: Entity[];
-}
-
-export const CANONICAL_TYPE_MAP: Record<string, string> = {
-  DOB: "DATE",
-  ZIP: "ZIP_CODE",
-  PER: "PERSON",
-  ORG: "ORGANIZATION",
-  GPE: "LOCATION",
-  LOC: "LOCATION",
-  FAC: "ADDRESS",
-  PHONE_NUMBER: "PHONE",
-  SOCIAL_SECURITY_NUMBER: "SSN",
-  CREDIT_CARD_NUMBER: "CREDIT_CARD",
-  DATE_OF_BIRTH: "DATE",
-};
-
-export function canonicalType(entityType: string): string {
-  const normalized = entityType.toUpperCase().trim();
-  return CANONICAL_TYPE_MAP[normalized] ?? normalized;
-}
-
-export function resolveAction(entity: Entity, config: FogClawConfig): GuardrailAction {
-  return config.entityActions[entity.label] ?? config.guardrail_mode;
-}

package/tests/config.test.ts

@@ -1,78 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-import { loadConfig } from "../src/config.js";
-
-describe("FogClaw config", () => {
-  it("loads defaults for new policy fields", () => {
-    const config = loadConfig({});
-
-    expect(config.entityConfidenceThresholds).toEqual({});
-    expect(config.allowlist).toMatchObject({
-      values: [],
-      patterns: [],
-      entities: {},
-    });
-  });
-
-  it("canonicalizes per-entity confidence threshold keys", () => {
-    const config = loadConfig({
-      entityConfidenceThresholds: {
-        person: 0.7,
-      },
-    });
-
-    expect(config.entityConfidenceThresholds).toEqual({
-      PERSON: 0.7,
-    });
-  });
-
-  it("rejects invalid per-entity confidence thresholds", () => {
-    expect(() =>
-      loadConfig({
-        entityConfidenceThresholds: {
-          PERSON: 1.2,
-        },
-      }),
-    ).toThrow('entityConfidenceThresholds["PERSON"] must be between 0 and 1, got 1.2');
-  });
-
-  it("validates allowlist regex patterns", () => {
-    expect(() =>
-      loadConfig({
-        allowlist: {
-          values: ["ok@example.com"],
-          patterns: ["["],
-          entities: {
-            PERSON: ["John"],
-          },
-        },
-      }),
-    ).toThrow(/invalid regex pattern/);
-  });
-
-  it("canonicalizes allowlist entity keys", () => {
-    const config = loadConfig({
-      allowlist: {
-        entities: {
-          person: ["John"],
-        },
-      },
-    });
-
-    expect(config.allowlist.entities).toEqual({
-      PERSON: ["John"],
-    });
-  });
-
-  it("canonicalizes entity action labels", () => {
-    const config = loadConfig({
-      entityActions: {
-        person: "block",
-      },
-    });
-
-    expect(config.entityActions).toEqual({
-      PERSON: "block",
-    });
-  });
-});

package/tests/extract.test.ts

@@ -1,185 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { extractText, replaceText } from "../src/extract.js";
-
-describe("extractText", () => {
-  it("extracts from a plain string", () => {
-    expect(extractText("hello world")).toBe("hello world");
-  });
-
-  it("extracts from an object with content as string", () => {
-    expect(extractText({ role: "toolResult", content: "file contents here" })).toBe(
-      "file contents here",
-    );
-  });
-
-  it("extracts from content block array with single text block", () => {
-    const msg = {
-      role: "toolResult",
-      content: [{ type: "text", text: "block one" }],
-    };
-    expect(extractText(msg)).toBe("block one");
-  });
-
-  it("extracts from content block array with multiple text blocks", () => {
-    const msg = {
-      content: [
-        { type: "text", text: "first" },
-        { type: "text", text: "second" },
-      ],
-    };
-    expect(extractText(msg)).toBe("first\0second");
-  });
-
-  it("skips non-text blocks in content array", () => {
-    const msg = {
-      content: [
-        { type: "text", text: "visible" },
-        { type: "image", source: { data: "base64..." } },
-        { type: "text", text: "also visible" },
-      ],
-    };
-    expect(extractText(msg)).toBe("visible\0also visible");
-  });
-
-  it("returns empty string for null", () => {
-    expect(extractText(null)).toBe("");
-  });
-
-  it("returns empty string for undefined", () => {
-    expect(extractText(undefined)).toBe("");
-  });
-
-  it("returns empty string for a number", () => {
-    expect(extractText(42)).toBe("");
-  });
-
-  it("returns empty string for object with no content", () => {
-    expect(extractText({ role: "toolResult" })).toBe("");
-  });
-
-  it("returns empty string for object with null content", () => {
-    expect(extractText({ content: null })).toBe("");
-  });
-
-  it("returns empty string for empty content array", () => {
-    expect(extractText({ content: [] })).toBe("");
-  });
-
-  it("returns empty string for content array with only image blocks", () => {
-    const msg = {
-      content: [
-        { type: "image", source: { data: "..." } },
-        { type: "image", source: { data: "..." } },
-      ],
-    };
-    expect(extractText(msg)).toBe("");
-  });
-
-  it("handles empty string content", () => {
-    expect(extractText({ content: "" })).toBe("");
-  });
-
-  it("handles text block with empty text", () => {
-    const msg = { content: [{ type: "text", text: "" }] };
-    expect(extractText(msg)).toBe("");
-  });
-
-  it("handles content array with mixed valid and invalid blocks", () => {
-    const msg = {
-      content: [
-        { type: "text", text: "valid" },
-        { type: "text" }, // missing text property
-        null,
-        { type: "text", text: "also valid" },
-      ],
-    };
-    expect(extractText(msg)).toBe("valid\0also valid");
-  });
-});
-
-describe("replaceText", () => {
-  it("replaces plain string message", () => {
-    expect(replaceText("original", "redacted")).toBe("redacted");
-  });
-
-  it("replaces content string in object", () => {
-    const msg = { role: "toolResult", content: "original text" };
-    const result = replaceText(msg, "redacted text") as Record<string, unknown>;
-    expect(result.content).toBe("redacted text");
-    expect(result.role).toBe("toolResult");
-  });
-
-  it("does not mutate the original message object", () => {
-    const msg = { role: "toolResult", content: "original" };
-    replaceText(msg, "redacted");
-    expect(msg.content).toBe("original");
-  });
-
-  it("replaces single text block in content array", () => {
-    const msg = {
-      content: [{ type: "text", text: "original" }],
-    };
-    const result = replaceText(msg, "redacted") as Record<string, unknown>;
-    const content = result.content as Array<Record<string, unknown>>;
-    expect(content[0].text).toBe("redacted");
-    expect(content[0].type).toBe("text");
-  });
-
-  it("replaces multiple text blocks using segment separator", () => {
-    const msg = {
-      content: [
-        { type: "text", text: "first original" },
-        { type: "text", text: "second original" },
-      ],
-    };
-    const result = replaceText(msg, "first redacted\0second redacted") as Record<string, unknown>;
-    const content = result.content as Array<Record<string, unknown>>;
-    expect(content[0].text).toBe("first redacted");
-    expect(content[1].text).toBe("second redacted");
-  });
-
-  it("preserves non-text blocks in content array", () => {
-    const msg = {
-      content: [
-        { type: "text", text: "original" },
-        { type: "image", source: { data: "base64" } },
-        { type: "text", text: "also original" },
-      ],
-    };
-    const result = replaceText(msg, "redacted\0also redacted") as Record<string, unknown>;
-    const content = result.content as Array<Record<string, unknown>>;
-    expect(content[0].text).toBe("redacted");
-    expect((content[1] as any).type).toBe("image");
-    expect((content[1] as any).source.data).toBe("base64");
-    expect(content[2].text).toBe("also redacted");
-  });
-
-  it("returns null unchanged", () => {
-    expect(replaceText(null, "x")).toBe(null);
-  });
-
-  it("returns undefined unchanged", () => {
-    expect(replaceText(undefined, "x")).toBe(undefined);
-  });
-
-  it("returns number unchanged", () => {
-    expect(replaceText(42, "x")).toBe(42);
-  });
-
-  it("returns message unchanged if content is null", () => {
-    const msg = { content: null };
-    expect(replaceText(msg, "x")).toBe(msg);
-  });
-
-  it("returns message unchanged if content is not string or array", () => {
-    const msg = { content: 123 };
-    expect(replaceText(msg, "x")).toBe(msg);
-  });
-
-  it("preserves extra properties on the message", () => {
-    const msg = { role: "toolResult", content: "original", toolCallId: "abc123" };
-    const result = replaceText(msg, "redacted") as Record<string, unknown>;
-    expect(result.toolCallId).toBe("abc123");
-    expect(result.role).toBe("toolResult");
-  });
-});