@datafog/fogclaw 0.1.6 → 0.3.0

package/tests/regex.test.ts

@@ -1,345 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { RegexEngine } from "../src/engines/regex.js";
-
-const engine = new RegexEngine();
-
-/** Helper: assert every returned entity has correct span offsets */
-function assertSpans(text: string) {
-  const entities = engine.scan(text);
-  for (const e of entities) {
-    expect(text.slice(e.start, e.end)).toBe(e.text);
-  }
-  return entities;
-}
-
-// ---------------------------------------------------------------------------
-// EMAIL
-// ---------------------------------------------------------------------------
-describe("EMAIL", () => {
-  it("detects a simple email", () => {
-    const entities = assertSpans("Contact alice@example.com for info.");
-    const emails = entities.filter((e) => e.label === "EMAIL");
-    expect(emails).toHaveLength(1);
-    expect(emails[0].text).toBe("alice@example.com");
-    expect(emails[0].confidence).toBe(1.0);
-    expect(emails[0].source).toBe("regex");
-  });
-
-  it("detects email with subdomains", () => {
-    const entities = assertSpans("Send to bob@mail.example.co.uk now");
-    const emails = entities.filter((e) => e.label === "EMAIL");
-    expect(emails).toHaveLength(1);
-    expect(emails[0].text).toBe("bob@mail.example.co.uk");
-  });
-
-  it("detects email with special chars in local part", () => {
-    const entities = assertSpans("user+tag@example.org");
-    const emails = entities.filter((e) => e.label === "EMAIL");
-    expect(emails).toHaveLength(1);
-    expect(emails[0].text).toBe("user+tag@example.org");
-  });
-
-  it("does not match bare @-signs or partial addresses", () => {
-    const entities = engine.scan("@ or foo@ or @bar");
-    const emails = entities.filter((e) => e.label === "EMAIL");
-    expect(emails).toHaveLength(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// PHONE
-// ---------------------------------------------------------------------------
-describe("PHONE", () => {
-  it("detects US phone with dashes", () => {
-    const entities = assertSpans("Call 555-123-4567 today.");
-    const phones = entities.filter((e) => e.label === "PHONE");
-    expect(phones).toHaveLength(1);
-    expect(phones[0].text).toBe("555-123-4567");
-  });
-
-  it("detects US phone with parentheses", () => {
-    const entities = assertSpans("Phone: (555) 123-4567");
-    const phones = entities.filter((e) => e.label === "PHONE");
-    expect(phones).toHaveLength(1);
-    expect(phones[0].text).toBe("(555) 123-4567");
-  });
-
-  it("detects +1 prefix", () => {
-    const entities = assertSpans("Reach me at +1-800-555-1234.");
-    const phones = entities.filter((e) => e.label === "PHONE");
-    expect(phones).toHaveLength(1);
-    expect(phones[0].text).toBe("+1-800-555-1234");
-  });
-
-  it("detects international format", () => {
-    const entities = assertSpans("Number: +44 20 7946 0958");
-    const phones = entities.filter((e) => e.label === "PHONE");
-    expect(phones).toHaveLength(1);
-    expect(phones[0].text).toBe("+44 20 7946 0958");
-  });
-
-  it("does not match short digit sequences", () => {
-    const entities = engine.scan("Code 12345 here");
-    const phones = entities.filter((e) => e.label === "PHONE");
-    expect(phones).toHaveLength(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// SSN
-// ---------------------------------------------------------------------------
-describe("SSN", () => {
-  it("detects a valid SSN with dashes", () => {
-    const entities = assertSpans("SSN: 123-45-6789");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(1);
-    expect(ssns[0].text).toBe("123-45-6789");
-  });
-
-  it("detects a valid SSN without dashes", () => {
-    const entities = assertSpans("SSN 123456789 filed.");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(1);
-    expect(ssns[0].text).toBe("123456789");
-  });
-
-  it("rejects SSN starting with 000", () => {
-    const entities = engine.scan("Invalid SSN 000-12-3456");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(0);
-  });
-
-  it("rejects SSN starting with 666", () => {
-    const entities = engine.scan("Invalid SSN 666-12-3456");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(0);
-  });
-
-  it("rejects SSN with 00 in middle group", () => {
-    const entities = engine.scan("Invalid SSN 123-00-6789");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(0);
-  });
-
-  it("rejects SSN with 0000 in last group", () => {
-    const entities = engine.scan("Invalid SSN 123-45-0000");
-    const ssns = entities.filter((e) => e.label === "SSN");
-    expect(ssns).toHaveLength(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// CREDIT_CARD
-// ---------------------------------------------------------------------------
-describe("CREDIT_CARD", () => {
-  it("detects a Visa card (16 digits)", () => {
-    const entities = assertSpans("Card: 4111111111111111");
-    const cards = entities.filter((e) => e.label === "CREDIT_CARD");
-    expect(cards).toHaveLength(1);
-    expect(cards[0].text).toBe("4111111111111111");
-  });
-
-  it("detects a Mastercard", () => {
-    const entities = assertSpans("MC 5500000000000004");
-    const cards = entities.filter((e) => e.label === "CREDIT_CARD");
-    expect(cards).toHaveLength(1);
-    expect(cards[0].text).toBe("5500000000000004");
-  });
-
-  it("detects an Amex card", () => {
-    const entities = assertSpans("Amex 378282246310005");
-    const cards = entities.filter((e) => e.label === "CREDIT_CARD");
-    expect(cards).toHaveLength(1);
-    expect(cards[0].text).toBe("378282246310005");
-  });
-
-  it("detects card number with dashes", () => {
-    const entities = assertSpans("Card 4111-1111-1111-1111 charged");
-    const cards = entities.filter((e) => e.label === "CREDIT_CARD");
-    expect(cards).toHaveLength(1);
-    expect(cards[0].text).toBe("4111-1111-1111-1111");
-  });
-
-  it("detects card number with spaces", () => {
-    const entities = assertSpans("Card 5500 0000 0000 0004 charged");
-    const cards = entities.filter((e) => e.label === "CREDIT_CARD");
-    expect(cards).toHaveLength(1);
-    expect(cards[0].text).toBe("5500 0000 0000 0004");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// IP_ADDRESS
-// ---------------------------------------------------------------------------
-describe("IP_ADDRESS", () => {
-  it("detects a valid IPv4 address", () => {
-    const entities = assertSpans("Server at 192.168.1.1 is up.");
-    const ips = entities.filter((e) => e.label === "IP_ADDRESS");
-    expect(ips).toHaveLength(1);
-    expect(ips[0].text).toBe("192.168.1.1");
-  });
-
-  it("detects 0.0.0.0", () => {
-    const entities = assertSpans("Bind to 0.0.0.0 for all interfaces.");
-    const ips = entities.filter((e) => e.label === "IP_ADDRESS");
-    expect(ips).toHaveLength(1);
-    expect(ips[0].text).toBe("0.0.0.0");
-  });
-
-  it("detects 255.255.255.255", () => {
-    const entities = assertSpans("Broadcast: 255.255.255.255");
-    const ips = entities.filter((e) => e.label === "IP_ADDRESS");
-    expect(ips).toHaveLength(1);
-    expect(ips[0].text).toBe("255.255.255.255");
-  });
-
-  it("rejects IP with octet > 255", () => {
-    const entities = engine.scan("Invalid 256.1.2.3 address");
-    const ips = entities.filter((e) => e.label === "IP_ADDRESS");
-    // Should not match 256.1.2.3 as a complete valid IP
-    for (const ip of ips) {
-      expect(ip.text).not.toBe("256.1.2.3");
-    }
-  });
-
-  it("rejects IP with octet 999", () => {
-    const entities = engine.scan("Bad IP 999.999.999.999");
-    const ips = entities.filter((e) => e.label === "IP_ADDRESS");
-    for (const ip of ips) {
-      expect(ip.text).not.toBe("999.999.999.999");
-    }
-  });
-});
-
-// ---------------------------------------------------------------------------
-// DATE
-// ---------------------------------------------------------------------------
-describe("DATE", () => {
-  it("detects MM/DD/YYYY format", () => {
-    const entities = assertSpans("Born on 01/15/1990 in NY.");
-    const dates = entities.filter((e) => e.label === "DATE");
-    expect(dates).toHaveLength(1);
-    expect(dates[0].text).toBe("01/15/1990");
-  });
-
-  it("detects YYYY-MM-DD format", () => {
-    const entities = assertSpans("Date: 2024-03-15 confirmed.");
-    const dates = entities.filter((e) => e.label === "DATE");
-    expect(dates).toHaveLength(1);
-    expect(dates[0].text).toBe("2024-03-15");
-  });
-
-  it("detects Month DD, YYYY format", () => {
-    const entities = assertSpans("On January 5, 2023 we met.");
-    const dates = entities.filter((e) => e.label === "DATE");
-    expect(dates).toHaveLength(1);
-    expect(dates[0].text).toBe("January 5, 2023");
-  });
-
-  it("detects abbreviated month", () => {
-    const entities = assertSpans("Meeting: Dec 25, 2022 at noon.");
-    const dates = entities.filter((e) => e.label === "DATE");
-    expect(dates).toHaveLength(1);
-    expect(dates[0].text).toBe("Dec 25, 2022");
-  });
-
-  it("detects MM-DD-YY format", () => {
-    const entities = assertSpans("Filed 03-15-90 in records.");
-    const dates = entities.filter((e) => e.label === "DATE");
-    expect(dates).toHaveLength(1);
-    expect(dates[0].text).toBe("03-15-90");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// ZIP_CODE
-// ---------------------------------------------------------------------------
-describe("ZIP_CODE", () => {
-  it("detects a 5-digit ZIP", () => {
-    const entities = assertSpans("ZIP 90210 area.");
-    const zips = entities.filter((e) => e.label === "ZIP_CODE");
-    expect(zips).toHaveLength(1);
-    expect(zips[0].text).toBe("90210");
-  });
-
-  it("detects a ZIP+4", () => {
-    const entities = assertSpans("Mailing: 90210-1234 confirmed.");
-    const zips = entities.filter((e) => e.label === "ZIP_CODE");
-    expect(zips).toHaveLength(1);
-    expect(zips[0].text).toBe("90210-1234");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// COMBINED / EDGE CASES
-// ---------------------------------------------------------------------------
-describe("Multiple entities in one text", () => {
-  it("finds email, phone, and SSN in same text", () => {
-    const text =
-      "Contact alice@example.com or 555-123-4567. SSN: 123-45-6789.";
-    const entities = assertSpans(text);
-
-    const labels = entities.map((e) => e.label);
-    expect(labels).toContain("EMAIL");
-    expect(labels).toContain("PHONE");
-    expect(labels).toContain("SSN");
-  });
-
-  it("finds multiple emails", () => {
-    const text = "Send to a@b.com and c@d.org please.";
-    const entities = assertSpans(text);
-    const emails = entities.filter((e) => e.label === "EMAIL");
-    expect(emails).toHaveLength(2);
-    expect(emails[0].text).toBe("a@b.com");
-    expect(emails[1].text).toBe("c@d.org");
-  });
-});
-
-describe("Empty and no-match inputs", () => {
-  it("returns empty array for empty string", () => {
-    const entities = engine.scan("");
-    expect(entities).toEqual([]);
-  });
-
-  it("returns empty array for text with no PII", () => {
-    const entities = engine.scan("The quick brown fox jumps over the lazy dog.");
-    // Filter out anything that might false-positive
-    const meaningful = entities.filter(
-      (e) => !["ZIP_CODE"].includes(e.label) || e.text.length >= 5
-    );
-    // This sentence has no PII
-    expect(entities).toEqual([]);
-  });
-});
-
-describe("Entity shape", () => {
-  it("every entity has correct confidence and source", () => {
-    const text = "Email: test@test.com Phone: 555-123-4567";
-    const entities = engine.scan(text);
-    for (const e of entities) {
-      expect(e.confidence).toBe(1.0);
-      expect(e.source).toBe("regex");
-      expect(typeof e.start).toBe("number");
-      expect(typeof e.end).toBe("number");
-      expect(e.end).toBeGreaterThan(e.start);
-      expect(typeof e.text).toBe("string");
-      expect(typeof e.label).toBe("string");
-    }
-  });
-
-  it("span offsets are correct for all entity types", () => {
-    const text =
-      "Email: user@site.com, Phone: (800) 555-0199, SSN: 321-54-9876, " +
-      "Card: 4111111111111111, IP: 10.0.0.1, Date: 2024-06-15, ZIP: 60601";
-    assertSpans(text);
-  });
-});
-
-describe("Repeated scan calls (lastIndex reset)", () => {
-  it("produces the same results on consecutive calls", () => {
-    const text = "Email alice@example.com and call 555-123-4567.";
-    const first = engine.scan(text);
-    const second = engine.scan(text);
-    expect(first).toEqual(second);
-  });
-});

package/tests/scanner.test.ts

@@ -1,348 +0,0 @@
-import {
-  beforeAll,
-  beforeEach,
-  afterAll,
-  describe,
-  it,
-  expect,
-  vi,
-} from "vitest";
-import fs from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-
-// Mock the gliner npm package so we don't need the actual model
-vi.mock("gliner", () => {
-  return {
-    Gliner: class MockGliner {
-      async initialize() {}
-      async inference(
-        request: { texts: string[]; entities: string[] } | string | string[],
-        maybeEntities?: string[],
-        _flatNer = false,
-        _threshold = 0.5,
-      ) {
-        const text =
-          typeof request === "string"
-            ? request
-            : Array.isArray(request)
-              ? request[0] ?? ""
-              : request.texts[0] ?? "";
-        const requestEntities =
-          typeof request === "object" && request !== null && "entities" in request
-            ? request.entities
-            : undefined;
-        const labels =
-          Array.isArray(maybeEntities)
-            ? maybeEntities
-            : requestEntities ?? [];
-        const results: any[] = [];
-
-        // Simulate person detection for "John Smith"
-        if (text.includes("John Smith")) {
-          const idx = text.indexOf("John Smith");
-          results.push({
-            text: "John Smith",
-            label: "person",
-            score: 0.95,
-            start: idx,
-            end: idx + 10,
-          });
-        }
-
-        // Simulate organization detection for "Acme Corp"
-        if (text.includes("Acme Corp")) {
-          const idx = text.indexOf("Acme Corp");
-          results.push({
-            text: "Acme Corp",
-            label: "organization",
-            score: 0.88,
-            start: idx,
-            end: idx + 9,
-          });
-        }
-
-        // Only return results whose labels are requested
-        return results.filter((r) => labels.includes(r.label));
-      }
-    }
-  };
-});
-
-vi.mock("gliner/node", () => {
-  return {
-    Gliner: class MockGliner {
-      async initialize() {}
-      async inference(
-        request: { texts: string[]; entities: string[] } | string | string[],
-        maybeEntities?: string[],
-        _flatNer = false,
-        _threshold = 0.5,
-      ) {
-        const text =
-          typeof request === "string"
-            ? request
-            : Array.isArray(request)
-              ? request[0] ?? ""
-              : request.texts[0] ?? "";
-        const requestEntities =
-          typeof request === "object" && request !== null && "entities" in request
-            ? request.entities
-            : undefined;
-        const labels =
-          Array.isArray(maybeEntities)
-            ? maybeEntities
-            : requestEntities ?? [];
-        const results: any[] = [];
-
-        // Simulate person detection for "John Smith"
-        if (text.includes("John Smith")) {
-          const idx = text.indexOf("John Smith");
-          results.push({
-            text: "John Smith",
-            label: "person",
-            score: 0.95,
-            start: idx,
-            end: idx + 10,
-          });
-        }
-
-        // Simulate organization detection for "Acme Corp"
-        if (text.includes("Acme Corp")) {
-          const idx = text.indexOf("Acme Corp");
-          results.push({
-            text: "Acme Corp",
-            label: "organization",
-            score: 0.88,
-            start: idx,
-            end: idx + 9,
-          });
-        }
-
-        // Only return results whose labels are requested
-        return results.filter((r) => labels.includes(r.label));
-      }
-    }
-  };
-});
-
-import { Scanner } from "../src/scanner.js";
-import { DEFAULT_CONFIG } from "../src/config.js";
-import type { FogClawConfig } from "../src/types.js";
-
-const TEST_ONNX_MODEL_PATH = path.join(os.tmpdir(), "fogclaw-scanner-gliner-model-test.onnx");
-
-beforeAll(async () => {
-  await fs.writeFile(TEST_ONNX_MODEL_PATH, "mock-onnx-model", "utf8");
-});
-
-afterAll(async () => {
-  await fs.unlink(TEST_ONNX_MODEL_PATH).catch(() => undefined);
-});
-
-function makeConfig(overrides: Partial<FogClawConfig> = {}): FogClawConfig {
-  return {
-    ...DEFAULT_CONFIG,
-    model: TEST_ONNX_MODEL_PATH,
-    ...overrides,
-  };
-}
-
-describe("Scanner", () => {
-  let scanner: Scanner;
-
-  beforeEach(async () => {
-    scanner = new Scanner(makeConfig());
-    await scanner.initialize();
-  });
-
-  it("detects regex entities (email) without needing GLiNER", async () => {
-    // Even without initialize, regex should work
-    const regexOnly = new Scanner(makeConfig());
-    // Deliberately NOT calling initialize — GLiNER unavailable
-
-    const result = await regexOnly.scan("Contact us at test@example.com please.");
-
-    expect(result.entities.length).toBeGreaterThanOrEqual(1);
-    const email = result.entities.find((e) => e.label === "EMAIL");
-    expect(email).toBeDefined();
-    expect(email!.text).toBe("test@example.com");
-    expect(email!.source).toBe("regex");
-  });
-
-  it("detects GLiNER entities (person names)", async () => {
-    const result = await scanner.scan("My name is John Smith.");
-
-    const person = result.entities.find((e) => e.label === "PERSON");
-    expect(person).toBeDefined();
-    expect(person!.text).toBe("John Smith");
-    expect(person!.source).toBe("gliner");
-    expect(person!.confidence).toBe(0.95);
-  });
-
-  it("merges results from both engines (email + person in same text)", async () => {
-    const result = await scanner.scan(
-      "John Smith can be reached at john@example.com for details.",
-    );
-
-    const person = result.entities.find((e) => e.label === "PERSON");
-    const email = result.entities.find((e) => e.label === "EMAIL");
-
-    expect(person).toBeDefined();
-    expect(email).toBeDefined();
-    expect(person!.source).toBe("gliner");
-    expect(email!.source).toBe("regex");
-  });
-
-  it("applies per-entity confidence threshold overrides", async () => {
-    const strictScanner = new Scanner(
-      makeConfig({
-        entityConfidenceThresholds: {
-          PERSON: 0.98,
-        },
-      }),
-    );
-    await strictScanner.initialize();
-
-    const result = await strictScanner.scan("My name is John Smith.");
-    expect(result.entities.find((e) => e.label === "PERSON")).toBeUndefined();
-  });
-
-  it("supports allowlist exact matches across global and per-entity rules", async () => {
-    const allowlistScanner = new Scanner(
-      makeConfig({
-        allowlist: {
-          values: ["john@example.com"],
-          patterns: ["^internal-"],
-          entities: {
-            PERSON: ["john smith"],
-          },
-        },
-      }),
-    );
-    await allowlistScanner.initialize();
-
-    const result = await allowlistScanner.scan(
-      "John Smith can be reached at john@example.com.",
-    );
-
-    expect(result.entities.find((e) => e.label === "EMAIL")).toBeUndefined();
-    expect(result.entities.find((e) => e.label === "PERSON")).toBeUndefined();
-  });
-
-  it("applies allowlist regex patterns", async () => {
-    const allowlistScanner = new Scanner(
-      makeConfig({
-        allowlist: {
-          values: [],
-          patterns: ["test@example\\.com"],
-          entities: {},
-        },
-      }),
-    );
-    await allowlistScanner.initialize();
-
-    const result = await allowlistScanner.scan("This is test@example.com for redaction.");
-    expect(result.entities.find((e) => e.label === "EMAIL")).toBeUndefined();
-  });
-
-  it("deduplicates overlapping spans keeping higher confidence", async () => {
-    // Scan text that might produce overlapping entities
-    // The dedup logic should keep higher confidence when spans overlap
-    const result = await scanner.scan("Contact John Smith today.");
-
-    // We shouldn't have duplicate entities for the same span
-    const starts = result.entities.map((e) => e.start);
-    const uniqueStarts = [...new Set(starts)];
-    // If there were overlapping entities, dedup should have resolved them
-    expect(starts.length).toBe(uniqueStarts.length);
-  });
-
-  it("returns original text in result", async () => {
-    const text = "Hello John Smith, your email is test@example.com.";
-    const result = await scanner.scan(text);
-
-    expect(result.text).toBe(text);
-  });
-
-  it("accepts extra labels at scan time", async () => {
-    // The mock only returns results for labels that are in the labels array
-    // Extra labels get passed through to GLiNER
-    const result = await scanner.scan(
-      "John Smith works at Acme Corp.",
-      ["organization"],
-    );
-
-    // Person is always in default labels, organization should be detected too
-    const person = result.entities.find((e) => e.label === "PERSON");
-    const org = result.entities.find((e) => e.label === "ORGANIZATION");
-
-    expect(person).toBeDefined();
-    expect(org).toBeDefined();
-  });
-
-  it("falls back to regex-only when GLiNER is not initialized", async () => {
-    const fallbackScanner = new Scanner(makeConfig());
-    // Do NOT call initialize — GLiNER stays unavailable
-
-    const result = await fallbackScanner.scan(
-      "John Smith at john@example.com",
-    );
-
-    // Should still find the email via regex
-    const email = result.entities.find((e) => e.label === "EMAIL");
-    expect(email).toBeDefined();
-    expect(email!.source).toBe("regex");
-
-    // Should NOT find person because GLiNER is not available
-    const person = result.entities.find((e) => e.label === "PERSON");
-    expect(person).toBeUndefined();
-  });
-
-  it("empty text returns empty entities", async () => {
-    const result = await scanner.scan("");
-
-    expect(result.entities).toEqual([]);
-    expect(result.text).toBe("");
-  });
-
-  it("entities are sorted by start position after merge", async () => {
-    const result = await scanner.scan(
-      "John Smith can be reached at john@example.com for details.",
-    );
-
-    for (let i = 1; i < result.entities.length; i++) {
-      expect(result.entities[i].start).toBeGreaterThanOrEqual(
-        result.entities[i - 1].start,
-      );
-    }
-  });
-
-  it("passes custom_entities from config to GLiNER engine", async () => {
-    const customScanner = new Scanner(
-      makeConfig({ custom_entities: ["product", "event"] }),
-    );
-    await customScanner.initialize();
-
-    // Should not throw, custom labels are set on the engine
-    const result = await customScanner.scan("John Smith attended the event.");
-    expect(result.entities.length).toBeGreaterThanOrEqual(1);
-  });
-
-  it("handles text with only regex-detectable entities", async () => {
-    const result = await scanner.scan(
-      "Send to test@example.com and call 555-123-4567.",
-    );
-
-    expect(result.entities.length).toBeGreaterThanOrEqual(1);
-    const email = result.entities.find((e) => e.label === "EMAIL");
-    expect(email).toBeDefined();
-  });
-
-  it("handles text with no detectable entities", async () => {
-    const result = await scanner.scan("Hello world, this is a simple test.");
-
-    expect(result.entities).toEqual([]);
-    expect(result.text).toBe("Hello world, this is a simple test.");
-  });
-});