@datafog/fogclaw 0.1.6 → 0.3.0

package/src/redactor.ts

@@ -1,51 +0,0 @@
-import { createHash } from "node:crypto";
-import type { Entity, RedactResult, RedactStrategy } from "./types.js";
-
-export function redact(
-  text: string,
-  entities: Entity[],
-  strategy: RedactStrategy = "token",
-): RedactResult {
-  if (entities.length === 0) {
-    return { redacted_text: text, mapping: {}, entities: [] };
-  }
-
-  // Sort by start position descending so we replace from end to start
-  // without corrupting earlier offsets
-  const sorted = [...entities].sort((a, b) => b.start - a.start);
-
-  const counters: Record<string, number> = {};
-  const mapping: Record<string, string> = {};
-  let result = text;
-
-  for (const entity of sorted) {
-    const replacement = makeReplacement(entity, strategy, counters);
-    mapping[replacement] = entity.text;
-    result = result.slice(0, entity.start) + replacement + result.slice(entity.end);
-  }
-
-  return { redacted_text: result, mapping, entities };
-}
-
-function makeReplacement(
-  entity: Entity,
-  strategy: RedactStrategy,
-  counters: Record<string, number>,
-): string {
-  switch (strategy) {
-    case "token": {
-      counters[entity.label] = (counters[entity.label] ?? 0) + 1;
-      return `[${entity.label}_${counters[entity.label]}]`;
-    }
-    case "mask": {
-      return "*".repeat(Math.max(entity.text.length, 1));
-    }
-    case "hash": {
-      const digest = createHash("sha256")
-        .update(entity.text)
-        .digest("hex")
-        .slice(0, 12);
-      return `[${entity.label}_${digest}]`;
-    }
-  }
-}

package/src/scanner.ts

@@ -1,196 +0,0 @@
-import type { Entity, FogClawConfig } from "./types.js";
-import { canonicalType } from "./types.js";
-import { RegexEngine } from "./engines/regex.js";
-import { GlinerEngine } from "./engines/gliner.js";
-
-type AllowlistPatternCache = {
-  values: Set<string>;
-  patterns: RegExp[];
-  entityValues: Map<string, Set<string>>;
-};
-
-function normalizeAllowlistValue(value: string): string {
-  return value.trim().toLowerCase();
-}
-
-function buildPatternMaps(value: string[] | undefined): RegExp[] {
-  if (!value || value.length === 0) {
-    return [];
-  }
-
-  return value.map((pattern) => new RegExp(pattern, "i"));
-}
-
-export class Scanner {
-  private regexEngine: RegexEngine;
-  private glinerEngine: GlinerEngine;
-  private glinerAvailable = false;
-  private config: FogClawConfig;
-  private allowlist: AllowlistPatternCache;
-
-  constructor(config: FogClawConfig) {
-    this.config = config;
-    this.regexEngine = new RegexEngine();
-
-    const glinerThreshold = this.computeGlinerThreshold(config);
-    this.glinerEngine = new GlinerEngine(config.model, glinerThreshold);
-    if (config.custom_entities.length > 0) {
-      this.glinerEngine.setCustomLabels(config.custom_entities);
-    }
-
-    this.allowlist = this.buildAllowlistCache(config.allowlist);
-  }
-
-  async initialize(): Promise<void> {
-    try {
-      await this.glinerEngine.initialize();
-      this.glinerAvailable = true;
-    } catch (err) {
-      console.warn(
-        `[fogclaw] GLiNER failed to initialize, falling back to regex-only mode: ${err instanceof Error ? err.message : String(err)}`,
-      );
-      this.glinerAvailable = false;
-    }
-  }
-
-  async scan(text: string, extraLabels?: string[]): Promise<{ entities: Entity[]; text: string }> {
-    if (!text) return { entities: [], text };
-
-    // Step 1: Regex pass (always runs, synchronous)
-    const regexEntities = this.filterByPolicy(this.regexEngine.scan(text));
-
-    // Step 2: GLiNER pass (if available)
-    let glinerEntities: Entity[] = [];
-    if (this.glinerAvailable) {
-      try {
-        glinerEntities = await this.glinerEngine.scan(text, extraLabels);
-        glinerEntities = this.filterByConfidence(glinerEntities);
-        glinerEntities = this.filterByPolicy(glinerEntities);
-      } catch (err) {
-        console.warn(
-          `[fogclaw] GLiNER scan failed, using regex results only: ${
-            err instanceof Error ? err.message : String(err)
-          }`,
-        );
-      }
-    }
-
-    // Step 3: Merge and deduplicate
-    const merged = deduplicateEntities([...regexEntities, ...glinerEntities]);
-
-    return { entities: merged, text };
-  }
-
-  private filterByConfidence(entities: Entity[]): Entity[] {
-    return entities.filter((entity) => {
-      const threshold = this.getThresholdForLabel(entity.label);
-      return entity.confidence >= threshold;
-    });
-  }
-
-  private filterByPolicy(entities: Entity[]): Entity[] {
-    if (
-      this.allowlist.values.size === 0 &&
-      this.allowlist.patterns.length === 0 &&
-      this.allowlist.entityValues.size === 0
-    ) {
-      return entities;
-    }
-
-    return entities.filter((entity) => !this.shouldAllowlistEntity(entity));
-  }
-
-  private shouldAllowlistEntity(entity: Entity): boolean {
-    const normalizedText = normalizeAllowlistValue(entity.text);
-
-    if (this.allowlist.values.has(normalizedText)) {
-      return true;
-    }
-
-    if (this.allowlist.patterns.some((pattern) => pattern.test(entity.text))) {
-      return true;
-    }
-
-    const entityValues = this.allowlist.entityValues.get(entity.label);
-    if (entityValues && entityValues.has(normalizedText)) {
-      return true;
-    }
-
-    return false;
-  }
-
-  private getThresholdForLabel(label: string): number {
-    const canonicalLabel = canonicalType(label);
-    return this.config.entityConfidenceThresholds[canonicalLabel] ?? this.config.confidence_threshold;
-  }
-
-  private computeGlinerThreshold(config: FogClawConfig): number {
-    const thresholds = Object.values(config.entityConfidenceThresholds);
-    if (thresholds.length === 0) {
-      return config.confidence_threshold;
-    }
-
-    return Math.min(config.confidence_threshold, ...thresholds);
-  }
-
-  private buildAllowlistCache(allowlist: FogClawConfig["allowlist"]): AllowlistPatternCache {
-    const globalValues = new Set(
-      allowlist.values.map((value) => normalizeAllowlistValue(value)),
-    );
-
-    const globalPatterns = buildPatternMaps(allowlist.patterns);
-
-    const entityValues = new Map<string, Set<string>>();
-    for (const [entityType, values] of Object.entries(allowlist.entities)) {
-      const canonical = canonicalType(entityType);
-      const uniqueValues = values
-        .map((value) => normalizeAllowlistValue(value))
-        .filter((value) => value.length > 0);
-      entityValues.set(canonical, new Set(uniqueValues));
-    }
-
-    return {
-      values: globalValues,
-      patterns: globalPatterns,
-      entityValues,
-    };
-  }
-
-  get isGlinerAvailable(): boolean {
-    return this.glinerAvailable;
-  }
-}
-
-/**
- * Remove overlapping entity spans. When two entities overlap,
- * keep the one with higher confidence. If equal, prefer regex.
- */
-function deduplicateEntities(entities: Entity[]): Entity[] {
-  if (entities.length <= 1) return entities;
-
-  // Sort by start position, then by confidence descending
-  const sorted = [...entities].sort((a, b) => {
-    if (a.start !== b.start) return a.start - b.start;
-    return b.confidence - a.confidence;
-  });
-
-  const result: Entity[] = [sorted[0]];
-
-  for (let i = 1; i < sorted.length; i++) {
-    const current = sorted[i];
-    const last = result[result.length - 1];
-
-    // Check for overlap
-    if (current.start < last.end) {
-      // Overlapping: keep higher confidence (already in result if first)
-      if (current.confidence > last.confidence) {
-        result[result.length - 1] = current;
-      }
-      // Otherwise keep what's already in result
-    } else {
-      result.push(current);
-    }
-  }
-
-  return result;
-}

package/src/types.ts

@@ -1,71 +0,0 @@
-export interface Entity {
-  text: string;
-  label: string;
-  start: number;
-  end: number;
-  confidence: number;
-  source: "regex" | "gliner";
-}
-
-export type RedactStrategy = "token" | "mask" | "hash";
-
-export type GuardrailAction = "redact" | "block" | "warn";
-
-export interface EntityConfidenceThresholds {
-  [entityType: string]: number;
-}
-
-export interface EntityAllowlist {
-  values: string[];
-  patterns: string[];
-  entities: Record<string, string[]>;
-}
-
-export interface FogClawConfig {
-  enabled: boolean;
-  guardrail_mode: GuardrailAction;
-  redactStrategy: RedactStrategy;
-  model: string;
-  confidence_threshold: number;
-  custom_entities: string[];
-  entityActions: Record<string, GuardrailAction>;
-  entityConfidenceThresholds: EntityConfidenceThresholds;
-  allowlist: EntityAllowlist;
-  auditEnabled: boolean;
-}
-
-export interface ScanResult {
-  entities: Entity[];
-  text: string;
-}
-
-export interface RedactResult {
-  redacted_text: string;
-  mapping: Record<string, string>;
-  entities: Entity[];
-}
-
-export interface GuardrailPlan {
-  blocked: Entity[];
-  warned: Entity[];
-  redacted: Entity[];
-}
-
-export const CANONICAL_TYPE_MAP: Record<string, string> = {
-  DOB: "DATE",
-  ZIP: "ZIP_CODE",
-  PER: "PERSON",
-  ORG: "ORGANIZATION",
-  GPE: "LOCATION",
-  LOC: "LOCATION",
-  FAC: "ADDRESS",
-  PHONE_NUMBER: "PHONE",
-  SOCIAL_SECURITY_NUMBER: "SSN",
-  CREDIT_CARD_NUMBER: "CREDIT_CARD",
-  DATE_OF_BIRTH: "DATE",
-};
-
-export function canonicalType(entityType: string): string {
-  const normalized = entityType.toUpperCase().trim();
-  return CANONICAL_TYPE_MAP[normalized] ?? normalized;
-}

package/tests/config.test.ts

@@ -1,78 +0,0 @@
-import { describe, it, expect } from "vitest";
-
-import { loadConfig } from "../src/config.js";
-
-describe("FogClaw config", () => {
-  it("loads defaults for new policy fields", () => {
-    const config = loadConfig({});
-
-    expect(config.entityConfidenceThresholds).toEqual({});
-    expect(config.allowlist).toMatchObject({
-      values: [],
-      patterns: [],
-      entities: {},
-    });
-  });
-
-  it("canonicalizes per-entity confidence threshold keys", () => {
-    const config = loadConfig({
-      entityConfidenceThresholds: {
-        person: 0.7,
-      },
-    });
-
-    expect(config.entityConfidenceThresholds).toEqual({
-      PERSON: 0.7,
-    });
-  });
-
-  it("rejects invalid per-entity confidence thresholds", () => {
-    expect(() =>
-      loadConfig({
-        entityConfidenceThresholds: {
-          PERSON: 1.2,
-        },
-      }),
-    ).toThrow('entityConfidenceThresholds["PERSON"] must be between 0 and 1, got 1.2');
-  });
-
-  it("validates allowlist regex patterns", () => {
-    expect(() =>
-      loadConfig({
-        allowlist: {
-          values: ["ok@example.com"],
-          patterns: ["["],
-          entities: {
-            PERSON: ["John"],
-          },
-        },
-      }),
-    ).toThrow(/invalid regex pattern/);
-  });
-
-  it("canonicalizes allowlist entity keys", () => {
-    const config = loadConfig({
-      allowlist: {
-        entities: {
-          person: ["John"],
-        },
-      },
-    });
-
-    expect(config.allowlist.entities).toEqual({
-      PERSON: ["John"],
-    });
-  });
-
-  it("canonicalizes entity action labels", () => {
-    const config = loadConfig({
-      entityActions: {
-        person: "block",
-      },
-    });
-
-    expect(config.entityActions).toEqual({
-      PERSON: "block",
-    });
-  });
-});

package/tests/gliner.test.ts

@@ -1,289 +0,0 @@
-import { beforeAll, beforeEach, afterAll, describe, it, expect, vi } from "vitest";
-import fs from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-
-// Mock the gliner npm package so we don't need the actual 1.4GB model
-vi.mock("gliner", () => {
-  class MockGliner {
-    private config: any;
-
-    constructor(config: any) {
-      this.config = config;
-    }
-
-    async initialize(): Promise<void> {
-      // No-op in mock
-    }
-
-    async inference(
-      request: { texts: string[]; entities: string[] } | string | string[],
-      maybeEntities?: string[],
-      _flatNer = false,
-      _threshold = 0.5,
-    ): Promise<Array<{ text: string; label: string; score: number; start: number; end: number }>> {
-      const text =
-        typeof request === "string"
-          ? request
-          : Array.isArray(request)
-            ? request[0] ?? ""
-            : request.texts[0] ?? "";
-      const requestEntities =
-        typeof request === "object" && request !== null && "entities" in request
-          ? request.entities
-          : undefined;
-      const labels =
-        Array.isArray(maybeEntities)
-          ? maybeEntities
-          : requestEntities ?? [];
-      const results: Array<{ text: string; label: string; score: number; start: number; end: number }> = [];
-
-      // Simulate entity detection for "John Smith"
-      const johnIndex = text.indexOf("John Smith");
-      if (johnIndex !== -1 && labels.includes("person")) {
-        results.push({
-          text: "John Smith",
-          label: "person",
-          score: 0.95,
-          start: johnIndex,
-          end: johnIndex + "John Smith".length,
-        });
-      }
-
-      // Simulate entity detection for "Acme Corp"
-      const acmeIndex = text.indexOf("Acme Corp");
-      if (acmeIndex !== -1 && labels.includes("organization")) {
-        results.push({
-          text: "Acme Corp",
-          label: "organization",
-          score: 0.88,
-          start: acmeIndex,
-          end: acmeIndex + "Acme Corp".length,
-        });
-      }
-
-      // Simulate entity detection for "New York"
-      const nyIndex = text.indexOf("New York");
-      if (nyIndex !== -1 && labels.includes("location")) {
-        results.push({
-          text: "New York",
-          label: "location",
-          score: 0.91,
-          start: nyIndex,
-          end: nyIndex + "New York".length,
-        });
-      }
-
-      return results;
-    }
-  }
-
-  return { Gliner: MockGliner };
-});
-
-vi.mock("gliner/node", () => {
-  class MockGliner {
-    private config: any;
-
-    constructor(config: any) {
-      this.config = config;
-    }
-
-    async initialize(): Promise<void> {
-      // No-op in mock
-    }
-
-    async inference(
-      request: { texts: string[]; entities: string[] } | string | string[],
-      maybeEntities?: string[],
-      _flatNer = false,
-      _threshold = 0.5,
-    ): Promise<Array<{ text: string; label: string; score: number; start: number; end: number }>> {
-      const text =
-        typeof request === "string"
-          ? request
-          : Array.isArray(request)
-            ? request[0] ?? ""
-            : request.texts[0] ?? "";
-      const requestEntities =
-        typeof request === "object" && request !== null && "entities" in request
-          ? request.entities
-          : undefined;
-      const labels =
-        Array.isArray(maybeEntities)
-          ? maybeEntities
-          : requestEntities ?? [];
-      const results: Array<{ text: string; label: string; score: number; start: number; end: number }> = [];
-
-      // Simulate entity detection for "John Smith"
-      const johnIndex = text.indexOf("John Smith");
-      if (johnIndex !== -1 && labels.includes("person")) {
-        results.push({
-          text: "John Smith",
-          label: "person",
-          score: 0.95,
-          start: johnIndex,
-          end: johnIndex + "John Smith".length,
-        });
-      }
-
-      // Simulate entity detection for "Acme Corp"
-      const acmeIndex = text.indexOf("Acme Corp");
-      if (acmeIndex !== -1 && labels.includes("organization")) {
-        results.push({
-          text: "Acme Corp",
-          label: "organization",
-          score: 0.88,
-          start: acmeIndex,
-          end: acmeIndex + "Acme Corp".length,
-        });
-      }
-
-      // Simulate entity detection for "New York"
-      const nyIndex = text.indexOf("New York");
-      if (nyIndex !== -1 && labels.includes("location")) {
-        results.push({
-          text: "New York",
-          label: "location",
-          score: 0.91,
-          start: nyIndex,
-          end: nyIndex + "New York".length,
-        });
-      }
-
-      return results;
-    }
-  }
-
-  return { Gliner: MockGliner };
-});
-
-import { GlinerEngine } from "../src/engines/gliner.js";
-
-const TEST_ONNX_MODEL_PATH = path.join(os.tmpdir(), "fogclaw-gliner-model-test.onnx");
-
-beforeAll(async () => {
-  await fs.writeFile(TEST_ONNX_MODEL_PATH, "mock-onnx-model", "utf8");
-});
-
-afterAll(async () => {
-  await fs.unlink(TEST_ONNX_MODEL_PATH).catch(() => undefined);
-});
-
-describe("GlinerEngine", () => {
-  let engine: GlinerEngine;
-
-  beforeEach(async () => {
-    engine = new GlinerEngine(TEST_ONNX_MODEL_PATH, 0.5);
-    await engine.initialize();
-  });
-
-  it("detects person entities with canonical PERSON label", async () => {
-    const entities = await engine.scan("My name is John Smith and I live here.");
-
-    expect(entities).toHaveLength(1);
-    expect(entities[0].text).toBe("John Smith");
-    expect(entities[0].label).toBe("PERSON");
-  });
-
-  it("detects organization entities with canonical ORGANIZATION label", async () => {
-    const entities = await engine.scan("I work at Acme Corp downtown.");
-
-    expect(entities).toHaveLength(1);
-    expect(entities[0].text).toBe("Acme Corp");
-    expect(entities[0].label).toBe("ORGANIZATION");
-  });
-
-  it("detects multiple entity types in the same text", async () => {
-    const entities = await engine.scan(
-      "John Smith works at Acme Corp in New York.",
-    );
-
-    expect(entities).toHaveLength(3);
-
-    const labels = entities.map((e) => e.label);
-    expect(labels).toContain("PERSON");
-    expect(labels).toContain("ORGANIZATION");
-    expect(labels).toContain("LOCATION");
-  });
-
-  it("returns empty array for text with no entities", async () => {
-    const entities = await engine.scan("Hello world, this is a test.");
-
-    expect(entities).toEqual([]);
-  });
-
-  it("returns empty array for empty string input", async () => {
-    const entities = await engine.scan("");
-
-    expect(entities).toEqual([]);
-  });
-
-  it("allows setting custom labels without crashing", async () => {
-    expect(() => engine.setCustomLabels(["product", "event"])).not.toThrow();
-
-    // Scan still works after setting custom labels
-    const entities = await engine.scan("John Smith attended the event.");
-    expect(entities).toHaveLength(1);
-    expect(entities[0].label).toBe("PERSON");
-  });
-
-  it("applies canonical type mapping (lowercase person -> PERSON)", async () => {
-    // The mock returns lowercase "person" as label; canonicalType should map it to "PERSON"
-    const entities = await engine.scan("John Smith is here.");
-
-    expect(entities[0].label).toBe("PERSON");
-    // Verify it's not lowercase
-    expect(entities[0].label).not.toBe("person");
-  });
-
-  it("sets source to gliner for all detected entities", async () => {
-    const entities = await engine.scan(
-      "John Smith works at Acme Corp in New York.",
-    );
-
-    for (const entity of entities) {
-      expect(entity.source).toBe("gliner");
-    }
-  });
-
-  it("confidence comes from model score", async () => {
-    const entities = await engine.scan(
-      "John Smith works at Acme Corp in New York.",
-    );
-
-    const person = entities.find((e) => e.label === "PERSON");
-    const org = entities.find((e) => e.label === "ORGANIZATION");
-    const loc = entities.find((e) => e.label === "LOCATION");
-
-    // These match the scores set in our mock
-    expect(person?.confidence).toBe(0.95);
-    expect(org?.confidence).toBe(0.88);
-    expect(loc?.confidence).toBe(0.91);
-  });
-
-  it("throws if scan is called before initialize", async () => {
-    const uninitializedEngine = new GlinerEngine("some-model", 0.5);
-
-    await expect(uninitializedEngine.scan("test")).rejects.toThrow(
-      "GLiNER engine not initialized. Call initialize() first.",
-    );
-  });
-
-  it("reports isInitialized correctly", async () => {
-    const freshEngine = new GlinerEngine(TEST_ONNX_MODEL_PATH, 0.5);
-    expect(freshEngine.isInitialized).toBe(false);
-
-    await freshEngine.initialize();
-    expect(freshEngine.isInitialized).toBe(true);
-  });
-
-  it("includes correct start and end offsets", async () => {
-    const text = "Contact John Smith for details.";
-    const entities = await engine.scan(text);
-
-    expect(entities).toHaveLength(1);
-    expect(entities[0].start).toBe(8); // "Contact " is 8 chars
-    expect(entities[0].end).toBe(18); // 8 + "John Smith".length = 18
-  });
-});