@datafog/fogclaw 0.2.0 → 0.3.0

package/src/engines/gliner.ts

@@ -1,240 +0,0 @@
-import fs from "node:fs/promises";
-import path from "node:path";
-import { env } from "@xenova/transformers";
-
-import type { Entity } from "../types.js";
-import { canonicalType } from "../types.js";
-
-const DEFAULT_NER_LABELS = [
-  "person",
-  "organization",
-  "location",
-  "address",
-  "date of birth",
-  "medical record number",
-  "account number",
-  "passport number",
-];
-
-const GLINER_MODEL_FILES = [
-  "onnx/model_q4f16.onnx",
-  "onnx/model_q4.onnx",
-  "onnx/model_bnb4.onnx",
-  "onnx/model_int8.onnx",
-  "onnx/model_uint8.onnx",
-  "onnx/model_quantized.onnx",
-  "onnx/model_fp16.onnx",
-  "onnx/model.onnx",
-];
-
-const MODEL_DOWNLOAD_TIMEOUT_MS = 120_000;
-
-function isLikelyLocalPath(modelPath: string): boolean {
-  const trimmed = modelPath.trim();
-  if (!trimmed) {
-    return false;
-  }
-
-  const lower = trimmed.toLowerCase();
-  const hasExtension = [".onnx", ".ort", ".bin"].some((ext) => lower.endsWith(ext));
-  if (hasExtension) {
-    return true;
-  }
-
-  if (trimmed.startsWith(".") || path.isAbsolute(trimmed)) {
-    return true;
-  }
-
-  return false;
-}
-
-function toAbsolutePath(value: string): string {
-  return path.isAbsolute(value) ? value : path.resolve(process.cwd(), value);
-}
-
-function getModelCacheDir(): string {
-  return env.localModelPath ?? path.join(process.cwd(), ".cache");
-}
-
-function sanitizeModelReference(modelPath: string): string {
-  return modelPath.trim();
-}
-
-async function fileExists(filePath: string): Promise<boolean> {
-  try {
-    await fs.access(filePath);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-async function downloadModelIfNeeded(modelRepo: string, filename: string): Promise<string> {
-  const cacheDir = getModelCacheDir();
-  const localPath = path.join(cacheDir, modelRepo, filename);
-
-  if (await fileExists(localPath)) {
-    return localPath;
-  }
-
-  const url = `https://huggingface.co/${modelRepo}/resolve/main/${filename}`;
-  const headers = new Headers();
-  const token = process.env.HF_TOKEN ?? process.env.HF_ACCESS_TOKEN;
-  if (token) {
-    headers.set("Authorization", `Bearer ${token}`);
-  }
-
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), MODEL_DOWNLOAD_TIMEOUT_MS);
-
-  try {
-    const response = await fetch(url, { headers, signal: controller.signal });
-    if (!response.ok) {
-      throw new Error(`Unable to download model artifact: ${response.status}`);
-    }
-
-    const bytes = new Uint8Array(await response.arrayBuffer());
-    await fs.mkdir(path.dirname(localPath), { recursive: true });
-    await fs.writeFile(localPath, bytes);
-
-    return localPath;
-  } catch (err) {
-    if (err instanceof Error && err.name === "AbortError") {
-      throw new Error(`Model download timed out after ${MODEL_DOWNLOAD_TIMEOUT_MS}ms`);
-    }
-
-    throw err;
-  } finally {
-    clearTimeout(timeout);
-  }
-}
-
-async function resolveModelPath(modelPath: string): Promise<string> {
-  const sanitized = sanitizeModelReference(modelPath);
-  if (!sanitized) {
-    throw new Error("Model path is empty");
-  }
-
-  if (isLikelyLocalPath(sanitized)) {
-    const absolutePath = toAbsolutePath(sanitized);
-    if (!(await fileExists(absolutePath))) {
-      throw new Error(`Local GLiNER model file not found at: ${absolutePath}`);
-    }
-
-    return absolutePath;
-  }
-
-  const candidates = GLINER_MODEL_FILES;
-  let lastError: Error | undefined;
-
-  for (const filename of candidates) {
-    const localPath = path.join(getModelCacheDir(), sanitized, filename);
-    if (await fileExists(localPath)) {
-      return localPath;
-    }
-  }
-
-  for (const filename of candidates) {
-    try {
-      return await downloadModelIfNeeded(sanitized, filename);
-    } catch (err) {
-      lastError = err instanceof Error ? err : new Error(String(err));
-    }
-  }
-
-  throw new Error(
-    `Failed to resolve GLiNER model "${sanitized}". Tried ${candidates.join(", ")}: ${
-      lastError?.message ?? "unknown"
-    }`,
-  );
-}
-
-export class GlinerEngine {
-  private model: any = null;
-  private modelPath: string;
-  private threshold: number;
-  private customLabels: string[] = [];
-  private initialized = false;
-
-  constructor(modelPath: string, threshold: number = 0.5) {
-    this.modelPath = modelPath;
-    this.threshold = threshold;
-  }
-
-  async initialize(): Promise<void> {
-    if (this.initialized) return;
-
-    try {
-      const resolvedModelPath = await resolveModelPath(this.modelPath);
-      const glinerModule = await import("gliner/node").catch(async () => import("gliner"));
-      const { Gliner } = glinerModule;
-      this.model = new Gliner({
-        tokenizerPath: this.modelPath,
-        onnxSettings: {
-          modelPath: resolvedModelPath,
-          executionProvider: "cpu",
-        },
-        maxWidth: 12,
-        modelType: "span-level",
-      });
-      await this.model.initialize();
-      this.initialized = true;
-    } catch (err) {
-      throw new Error(
-        `Failed to initialize GLiNER model "${this.modelPath}": ${err instanceof Error ? err.message : String(err)}`,
-      );
-    }
-  }
-
-  setCustomLabels(labels: string[]): void {
-    this.customLabels = labels;
-  }
-
-  async scan(text: string, extraLabels?: string[]): Promise<Entity[]> {
-    if (!text) return [];
-    if (!this.model) {
-      throw new Error("GLiNER engine not initialized. Call initialize() first.");
-    }
-
-    const labels = [
-      ...DEFAULT_NER_LABELS,
-      ...this.customLabels,
-      ...(extraLabels ?? []),
-    ];
-
-    // Deduplicate labels
-    const uniqueLabels = [...new Set(labels)];
-
-    const rawResults = await this.model.inference({
-      texts: [text],
-      entities: uniqueLabels,
-      flatNer: false,
-      threshold: this.threshold,
-    });
-    const flatResults = Array.isArray(rawResults) ? rawResults.flat() : [];
-
-    return flatResults.map(
-      (
-        r: {
-          spanText?: string;
-          text: string;
-          label: string;
-          score: number;
-          start: number;
-          end: number;
-        },
-      ) => ({
-        text: r.spanText ?? r.text,
-        label: canonicalType(r.label),
-        start: r.start,
-        end: r.end,
-        confidence: r.score,
-        source: "gliner" as const,
-      }),
-    );
-  }
-
-  get isInitialized(): boolean {
-    return this.initialized;
-  }
-}

package/src/engines/regex.ts

@@ -1,71 +0,0 @@
-import type { Entity } from "../types.js";
-
-interface PatternDef {
-  label: string;
-  pattern: RegExp;
-}
-
-const PATTERNS: PatternDef[] = [
-  {
-    label: "EMAIL",
-    pattern:
-      /(?<![A-Za-z0-9._%+\-@])(?![A-Za-z_]{2,20}=)[A-Za-z0-9!#$%&*+\-/=^_`{|}~][A-Za-z0-9!#$%&'*+\-/=?^_`{|}~.]*@(?:\.?[A-Za-z0-9-]+\.)+[A-Za-z]{2,}(?=$|[^A-Za-z])/gi,
-  },
-  {
-    label: "PHONE",
-    pattern:
-      /(?<![A-Za-z0-9])(?:(?:(?:\+?1)[-.\s]?)?(?:\(\d{3}\)|\d{3})[-.\s]?\d{3}[-.\s]?\d{4}|\+\d{1,3}[\s\-.]?\d{1,4}(?:[\s\-.]?\d{2,4}){2,3})(?![-A-Za-z0-9])/gi,
-  },
-  {
-    label: "SSN",
-    pattern:
-      /(?<!\d)(?:(?!000|666)\d{3}-(?!00)\d{2}-(?!0000)\d{4}|(?!000|666)\d{3}(?!00)\d{2}(?!0000)\d{4})(?!\d)/g,
-  },
-  {
-    label: "CREDIT_CARD",
-    pattern:
-      /\b(?:4\d{12}(?:\d{3})?|5[1-5]\d{14}|3[47]\d{13}|(?:(?:4\d{3}|5[1-5]\d{2}|3[47]\d{2})[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4})|(?:3[47]\d{2}[-\s]?\d{6}[-\s]?\d{5}))\b/g,
-  },
-  {
-    label: "IP_ADDRESS",
-    pattern:
-      /\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.(?:25[0-5]|2[0-4]\d|1?\d?\d)\.(?:25[0-5]|2[0-4]\d|1?\d?\d)\.(?:25[0-5]|2[0-4]\d|1?\d?\d))\b/g,
-  },
-  {
-    label: "DATE",
-    pattern:
-      /\b(?:(?:0?[1-9]|1[0-2])[/-](?:0?[1-9]|[12]\d|3[01])[/-](?:\d{2}|\d{4})|(?:\d{4})-(?:0?[1-9]|1[0-2])-(?:0?[1-9]|[12]\d|3[01])|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:0?[1-9]|[12]\d|3[01]),\s+(?:19|20)\d{2})\b/gi,
-  },
-  {
-    label: "ZIP_CODE",
-    pattern: /\b\d{5}(?:-\d{4})?\b/g,
-  },
-];
-
-export class RegexEngine {
-  scan(text: string): Entity[] {
-    const entities: Entity[] = [];
-
-    for (const { label, pattern } of PATTERNS) {
-      // Reset lastIndex to avoid stale state from previous calls
-      pattern.lastIndex = 0;
-
-      let match: RegExpExecArray | null;
-      while ((match = pattern.exec(text)) !== null) {
-        entities.push({
-          text: match[0],
-          label,
-          start: match.index,
-          end: match.index + match[0].length,
-          confidence: 1.0,
-          source: "regex",
-        });
-      }
-    }
-
-    // Sort by start position for deterministic output
-    entities.sort((a, b) => a.start - b.start || a.end - b.end);
-
-    return entities;
-  }
-}

package/src/extract.ts

@@ -1,98 +0,0 @@
-/**
- * Utilities for extracting text from AgentMessage tool result payloads
- * and replacing text content after redaction.
- *
- * AgentMessage shapes handled:
- * - Plain string
- * - Object with `content: string`
- * - Object with `content: [{ type: "text", text: "..." }, ...]`
- *
- * When multiple text blocks exist in a content array, they are joined
- * with a null byte separator (\0) so entity offsets stay valid across
- * the concatenated string. replaceText splits on the same separator
- * to map redacted text back to individual blocks.
- */
-
-// Separator between text segments from content block arrays.
-// Null byte won't appear in regex PII patterns or normal text content.
-const SEGMENT_SEP = "\0";
-
-/**
- * Extract all text content from an AgentMessage tool result payload.
- * Returns an empty string if no text content is found.
- */
-export function extractText(message: unknown): string {
-  if (message == null) return "";
-  if (typeof message === "string") return message;
-  if (typeof message !== "object") return "";
-
-  const msg = message as Record<string, unknown>;
-  const content = msg.content;
-
-  if (content == null) return "";
-  if (typeof content === "string") return content;
-
-  if (Array.isArray(content)) {
-    const textParts: string[] = [];
-    for (const block of content) {
-      if (
-        block != null &&
-        typeof block === "object" &&
-        (block as Record<string, unknown>).type === "text" &&
-        typeof (block as Record<string, unknown>).text === "string"
-      ) {
-        textParts.push((block as Record<string, unknown>).text as string);
-      }
-    }
-    if (textParts.length === 0) return "";
-    return textParts.join(SEGMENT_SEP);
-  }
-
-  return "";
-}
-
-/**
- * Replace text content in an AgentMessage tool result payload with
- * the redacted version. Returns a shallow copy; does not mutate.
- *
- * If the message shape is not recognized or has no text, returns
- * the original message unchanged.
- */
-export function replaceText(message: unknown, redactedText: string): unknown {
-  if (message == null) return message;
-  if (typeof message === "string") return redactedText;
-  if (typeof message !== "object") return message;
-
-  const msg = message as Record<string, unknown>;
-  const content = msg.content;
-
-  if (content == null) return message;
-
-  if (typeof content === "string") {
-    return { ...msg, content: redactedText };
-  }
-
-  if (Array.isArray(content)) {
-    const segments = redactedText.split(SEGMENT_SEP);
-    let segmentIndex = 0;
-
-    const newContent = content.map((block) => {
-      if (
-        block != null &&
-        typeof block === "object" &&
-        (block as Record<string, unknown>).type === "text" &&
-        typeof (block as Record<string, unknown>).text === "string" &&
-        segmentIndex < segments.length
-      ) {
-        const replaced = { ...(block as Record<string, unknown>), text: segments[segmentIndex] };
-        segmentIndex++;
-        return replaced;
-      }
-      return block;
-    });
-
-    return { ...msg, content: newContent };
-  }
-
-  return message;
-}