openredaction 1.0.0 → 1.0.8

package/README.md

@@ -20,6 +20,62 @@ console.log(result.redacted);
 // "Email [EMAIL_9619] or call [PHONE_UK_MOBILE_9478]"
 ```
 
+## Optional AI Assist
+
+OpenRedaction supports an optional AI-assisted detection mode that enhances regex-based detection by calling a hosted AI endpoint. This feature is **OFF by default** and requires explicit configuration.
+
+### Configuration
+
+```typescript
+import { OpenRedaction } from 'openredaction';
+
+const detector = new OpenRedaction({
+  // ... other options ...
+  ai: {
+    enabled: true,
+    endpoint: 'https://your-api.example.com' // Optional: defaults to OPENREDACTION_AI_ENDPOINT env var
+  }
+});
+
+// detect() is now async when AI is enabled
+const result = await detector.detect('Contact John Doe at john@example.com');
+```
+
+### How It Works
+
+1. **Regex Detection First**: The library always runs regex detection first (existing behavior)
+2. **AI Enhancement**: If `ai.enabled === true` and an endpoint is configured, the library calls the `/ai-detect` endpoint
+3. **Smart Merging**: AI entities are merged with regex detections, with regex taking precedence on conflicts
+4. **Graceful Fallback**: If the AI endpoint fails or is unavailable, the library silently falls back to regex-only detection
+
+### Environment Variables
+
+In Node.js environments, you can set the endpoint via environment variable:
+
+```bash
+export OPENREDACTION_AI_ENDPOINT=https://your-api.example.com
+```
+
+### Important Notes
+
+- **AI is optional**: The library works exactly as before when `ai.enabled` is `false` or omitted
+- **Regex is primary**: AI only adds additional entities; regex detections always take precedence
+- **No breaking changes**: When AI is disabled, behavior is identical to previous versions
+- **Browser support**: In browsers, you must provide an explicit `ai.endpoint` (env vars not available)
+- **Network dependency**: AI mode requires network access to the endpoint
+
+### For Sensitive Workloads
+
+For maximum security and privacy, keep AI disabled and rely purely on regex detection:
+
+```typescript
+const detector = new OpenRedaction({
+  // AI not configured = pure regex detection
+  includeNames: true,
+  includeEmails: true
+});
+```
+
 ## Documentation
 
 Full documentation available at [GitHub](https://github.com/sam247/openredaction)
@@ -33,7 +89,7 @@ Full documentation available at [GitHub](https://github.com/sam247/openredaction
 - 🧠 **Semantic Detection** - Hybrid NER + regex with 40+ contextual rules
 - 🎨 **Multiple Redaction Modes** - Placeholder, mask-middle, mask-all, format-preserving, token-replace
 - ✅ **Built-in Validators** - Luhn, IBAN, NHS, National ID checksums
-- 🔒 **Compliance Presets** - GDPR, HIPAA, CCPA, PCI-DSS
+- 🔒 **Compliance Presets** - GDPR, HIPAA, CCPA plus finance, education, healthcare, and transport presets
 - 🎭 **Deterministic Placeholders** - Consistent redaction for same values
 - 🌍 **Global Coverage** - 50+ countries
 - 📄 **Structured Data Support** - JSON, CSV, XLSX with path/cell tracking

package/dist/index.d.ts

@@ -87,6 +87,7 @@ type RedactionMode = 'placeholder' | 'mask-middle' | 'mask-all' | 'format-preser
 /**
  * Configuration options for OpenRedaction
  */
+type PresetName = 'gdpr' | 'hipaa' | 'ccpa' | 'healthcare' | 'healthcare-provider' | 'healthcare-research' | 'finance' | 'financial-services' | 'education' | 'transport-logistics' | 'transportation' | 'logistics';
 interface OpenRedactionOptions {
     /** Include name detection (default: true) */
     includeNames?: boolean;
@@ -96,6 +97,8 @@ interface OpenRedactionOptions {
     includePhones?: boolean;
     /** Include email detection (default: true) */
     includeEmails?: boolean;
+    /** Pattern categories to include (e.g., ['personal', 'financial']) */
+    categories?: string[];
     /** Whitelist specific patterns only */
     patterns?: string[];
     /** Add custom patterns */
@@ -107,7 +110,7 @@ interface OpenRedactionOptions {
     /** Redaction mode (default: 'placeholder') */
     redactionMode?: RedactionMode;
     /** Compliance preset */
-    preset?: 'gdpr' | 'hipaa' | 'ccpa';
+    preset?: PresetName;
     /** Enable context-aware detection (default: true) */
     enableContextAnalysis?: boolean;
     /** Minimum confidence threshold for detections (0-1, default: 0.5) */
@@ -146,6 +149,17 @@ interface OpenRedactionOptions {
     rbacManager?: IRBACManager;
     /** Predefined role name (admin, analyst, operator, viewer) */
     role?: RoleName;
+    /** Optional AI assist configuration */
+    ai?: AIOptions;
+}
+/**
+ * AI assist configuration options
+ */
+interface AIOptions {
+    /** Enable AI assist mode (default: false) */
+    enabled?: boolean;
+    /** AI endpoint URL (defaults to OPENREDACTION_AI_ENDPOINT env var if available) */
+    endpoint?: string;
 }
 /**
  * Validator function type
@@ -775,29 +789,30 @@ declare class ExplainAPI {
     /**
      * Explain why text was or wasn't detected as PII
      */
-    explain(text: string): TextExplanation;
+    explain(text: string): Promise<TextExplanation>;
     /**
      * Explain a specific detection
      */
-    explainDetection(detection: PIIDetection, text: string): {
+    explainDetection(detection: PIIDetection, text: string): Promise<{
         detection: PIIDetection;
         pattern?: PIIPattern;
         contextAnalysis?: ContextAnalysis;
         reasoning: string[];
-    };
+        suggestions: string[];
+    }>;
     /**
      * Suggest why text wasn't detected
      */
-    suggestWhy(text: string, expectedType: string): {
+    suggestWhy(text: string, expectedType: string): Promise<{
         text: string;
         expectedType: string;
         suggestions: string[];
         similarPatterns: PIIPattern[];
-    };
+    }>;
     /**
      * Get debugging information for entire detection process
      */
-    debug(text: string): {
+    debug(text: string): Promise<{
         text: string;
         textLength: number;
         enabledFeatures: string[];
@@ -806,7 +821,7 @@ declare class ExplainAPI {
         performance: {
             estimatedTime: string;
         };
-    };
+    }>;
 }
 /**
  * Helper to create explain API from detector
@@ -996,8 +1011,9 @@ declare class OpenRedaction {
     private processPatterns;
     /**
      * Detect PII in text
+     * Now async to support optional AI assist
      */
-    detect(text: string): DetectionResult;
+    detect(text: string): Promise<DetectionResult>;
     /**
      * Restore redacted text using redaction map
      */
@@ -1021,12 +1037,12 @@ declare class OpenRedaction {
     /**
      * Get severity-based scan results
      */
-    scan(text: string): {
+    scan(text: string): Promise<{
         high: PIIDetection[];
         medium: PIIDetection[];
         low: PIIDetection[];
         total: number;
-    };
+    }>;
     /**
      * Record a false positive (incorrectly detected as PII)
      */
@@ -1792,7 +1808,7 @@ declare class JsonProcessor {
     /**
      * Detect PII in JSON data
      */
-    detect(data: any, detector: OpenRedaction, options?: JsonProcessorOptions): JsonDetectionResult;
+    detect(data: any, detector: OpenRedaction, options?: JsonProcessorOptions): Promise<JsonDetectionResult>;
     /**
      * Redact PII in JSON data
      */
@@ -1840,7 +1856,7 @@ declare class JsonProcessor {
     /**
      * Detect PII in JSON Lines format
      */
-    detectJsonLines(input: Buffer | string, detector: OpenRedaction, options?: JsonProcessorOptions): JsonDetectionResult[];
+    detectJsonLines(input: Buffer | string, detector: OpenRedaction, options?: JsonProcessorOptions): Promise<JsonDetectionResult[]>;
 }
 /**
  * Create a JSON processor instance
@@ -1957,7 +1973,7 @@ declare class CsvProcessor {
     /**
      * Detect PII in CSV data
      */
-    detect(input: Buffer | string, detector: OpenRedaction, options?: CsvProcessorOptions): CsvDetectionResult;
+    detect(input: Buffer | string, detector: OpenRedaction, options?: CsvProcessorOptions): Promise<CsvDetectionResult>;
     /**
      * Redact PII in CSV data
      */
@@ -2114,7 +2130,7 @@ declare class XlsxProcessor {
     /**
      * Detect PII in XLSX data
      */
-    detect(buffer: Buffer, detector: OpenRedaction, options?: XlsxProcessorOptions): XlsxDetectionResult;
+    detect(buffer: Buffer, detector: OpenRedaction, options?: XlsxProcessorOptions): Promise<XlsxDetectionResult>;
     /**
      * Detect PII in a single sheet
      */
@@ -2370,6 +2386,26 @@ declare const hipaaPreset: Partial<OpenRedactionOptions>;
  * CCPA compliance preset - California consumer privacy
  */
 declare const ccpaPreset: Partial<OpenRedactionOptions>;
+/**
+ * Healthcare operations preset - provider-centric coverage
+ */
+declare const healthcarePreset: Partial<OpenRedactionOptions>;
+/**
+ * Healthcare research preset - clinical research and trials
+ */
+declare const healthcareResearchPreset: Partial<OpenRedactionOptions>;
+/**
+ * Financial services preset - banking, trading, and payments
+ */
+declare const financePreset: Partial<OpenRedactionOptions>;
+/**
+ * Education preset - FERPA-style coverage for schools and universities
+ */
+declare const educationPreset: Partial<OpenRedactionOptions>;
+/**
+ * Transportation and logistics preset - fleet, shipping, and mobility
+ */
+declare const transportLogisticsPreset: Partial<OpenRedactionOptions>;
 /**
  * Get preset configuration by name
  */
@@ -2770,6 +2806,7 @@ declare class StreamingDetector {
         overlap: number;
         estimatedMemory: number;
     };
+    private escapeRegex;
 }
 /**
  * Helper to create a streaming detector from OpenRedaction instance
@@ -2937,7 +2974,7 @@ declare class BatchProcessor {
     /**
      * Process multiple documents sequentially
      */
-    processSequential(documents: string[], options?: BatchOptions): BatchResult;
+    processSequential(documents: string[], options?: BatchOptions): Promise<BatchResult>;
     /**
      * Process multiple documents in parallel
      */
@@ -3004,15 +3041,15 @@ interface OpenRedactionRequest extends Request {
 /**
  * Create Express middleware for PII detection
  */
-declare function openredactionMiddleware(options?: OpenRedactionMiddlewareOptions): (req: Request, res: Response, next: NextFunction) => void | Response<any, Record<string, any>>;
+declare function openredactionMiddleware(options?: OpenRedactionMiddlewareOptions): (req: Request, res: Response, next: NextFunction) => Promise<void | Response<any, Record<string, any>>>;
 /**
  * Express route handler for PII detection
  */
-declare function detectPII(options?: OpenRedactionOptions): (req: Request, res: Response) => void;
+declare function detectPII(options?: OpenRedactionOptions): (req: Request, res: Response) => Promise<void>;
 /**
  * Express route handler for generating reports
  */
-declare function generateReport(options?: OpenRedactionOptions): (req: Request, res: Response) => void;
+declare function generateReport(options?: OpenRedactionOptions): (req: Request, res: Response) => Promise<void>;
 
 /**
  * Hook for PII detection in React components
@@ -3032,7 +3069,7 @@ declare function generateReport(options?: OpenRedactionOptions): (req: Request,
  * ```
  */
 declare function useOpenRedaction(options?: OpenRedactionOptions): {
-    detect: (text: string) => DetectionResult;
+    detect: (text: string) => Promise<DetectionResult>;
     result: DetectionResult | null;
     isDetecting: boolean;
     hasPII: boolean;
@@ -3095,7 +3132,7 @@ declare function useFormFieldValidator(options?: OpenRedactionOptions & {
     value: string;
     error: string | null;
     result: DetectionResult | null;
-    validate: (inputValue: string) => boolean;
+    validate: (inputValue: string) => Promise<boolean>;
     getFieldProps: () => {
         value: string;
         'aria-invalid': string;
@@ -4108,4 +4145,57 @@ declare function validatePattern(pattern: string | RegExp): void;
  */
 declare function compileSafeRegex(pattern: string | RegExp, flags?: string): RegExp;
 
-export { ADMIN_ROLE, ALL_PERMISSIONS, ANALYST_ROLE, type APIRequest, type APIResponse, APIServer, type APIServerConfig, type AuditBackend, type AuditDatabaseConfig, type AuditLogEntry, type AuditQueryFilter, type AuditStats, type BatchOptions, BatchProcessor, type BatchResult, type CellMatch$1 as CellMatch, type ChunkResult, type ColumnStats$1 as ColumnStats, ConfigExporter, ConfigLoader, ConsoleAuditLogger, type ContextAnalysis, type ContextFeatures, type ContextRulesConfig, ContextRulesEngine, type CsvDetectionResult, CsvProcessor, type CsvProcessorOptions, DEFAULT_DOMAIN_VOCABULARIES, DEFAULT_PROXIMITY_RULES, DEFAULT_SEVERITY_MAP, DEFAULT_TIER_QUOTAS, type DetectTask, type DetectionPass, type DetectionResult, type DocumentFormat, type DocumentMetadata, type DocumentOptions, DocumentProcessor, type DocumentResult, type DocumentTask, type DomainVocabulary, type ErrorSuggestion, ExplainAPI, type ExportedConfig, type FalsePositiveRule, GRAFANA_DASHBOARD_TEMPLATE, type HashedAuditLogEntry, type HealthCheckOptions, type HealthCheckResult, type HealthCheckStatus, HealthChecker, type HybridMatch, type IAuditDatabaseAdapter, type IAuditLogger, type IDocumentProcessor, type IMetricsCollector, type IMetricsExporter, type IOCRProcessor, type IRBACManager, type ImageFormat, InMemoryAuditLogger, InMemoryMetricsCollector, type JsonDetectionResult, JsonProcessor, type JsonProcessorOptions, type LearningData, type LearningStats, LocalLearningStore, type MultiPassStats, NERDetector, type NEREntityType, type NERMatch, type OCRLanguage, type OCROptions, OCRProcessor, type OCRResult, OPERATOR_ROLE, OpenRedaction, type OpenRedactionConfig, OpenRedactionError, type OpenRedactionMiddlewareOptions, type OpenRedactionOptions, type OpenRedactionRequest, type OptimizerOptions, type PIIDetection, type PIIMatch, type PIIPattern, type PatternAdjustment, type PatternMatchResult, type PatternStats, type Permission, PersistentAuditLogger, type PersistentAuditLoggerOptions, PriorityOptimizer, PrometheusServer, type PrometheusServerOptions, type ProximityRule, RBACManager, type RedactionMetrics, type RedactionMode, RegexMaxMatchesError, RegexTimeoutError, type ReportFormat, ReportGenerator, type ReportOptions, type ReportType, type RetentionPolicy, type RiskScore, type Role, type RoleName, SEVERITY_SCORES, type SafeRegexOptions, type SeverityClassification, SeverityClassifier, type SeverityLevel, type SheetDetectionResult, StreamingDetector, type StreamingOptions, type TenantConfig, TenantManager, TenantNotFoundError, TenantQuotaExceededError, type TenantQuotas, TenantSuspendedError, type TenantUsage, type TextExplanation, VIEWER_ROLE, type Validator, type WebhookConfig, type WebhookDelivery, type WebhookDeliveryStatus, type WebhookEvent, type WebhookEventType, WebhookManager, type WebhookStats, type WhitelistEntry, WorkerPool, type WorkerPoolConfig, type WorkerPoolStats, type WorkerResult, type WorkerTask, type XlsxDetectionResult, XlsxProcessor, type XlsxProcessorOptions, allPatterns, analyzeContextFeatures, analyzeFullContext, calculateContextConfidence, calculateRisk, ccpaPreset, commonFalsePositives, compileSafeRegex, contactPatterns, createAPIServer, createBatchProcessor, createCacheDisabledError, createConfigLoadError, createConfigPreset, createContextRulesEngine, createCsvProcessor, createCustomRole, createDocumentProcessor, createExplainAPI, createHealthChecker, createHighMemoryError, createInvalidPatternError, createJsonProcessor, createLearningDisabledError, createMultiPassDisabledError, createNERDetector, createOCRProcessor, createOptimizationDisabledError, createPersistentAuditLogger, createPriorityOptimizer, createPrometheusServer, createRBACManager, createReportGenerator, createSeverityClassifier, createSimpleMultiPass, createStreamingDetector, createTenantManager, createValidationError, createWebhookManager, createWorkerPool, createXlsxProcessor, defaultPasses, detectPII, exportForVersionControl, extractContext, filterFalsePositives, financialPatterns, gdprPreset, generateReport, getPatternsByCategory, getPredefinedRole, getPreset, getSeverity, governmentPatterns, groupPatternsByPass, healthCheckMiddleware, hipaaPreset, inferDocumentType, isFalsePositive, isUnsafePattern, mergePassDetections, networkPatterns, openredactionMiddleware, personalPatterns, safeExec, safeExecAll, useAutoRedact, useBatchDetector, useFormFieldValidator, useOpenRedaction, usePIIDetector, validateEmail, validateIBAN, validateLuhn, validateNHS, validateNINO, validateName, validatePattern, validateSSN, validateSortCode, validateUKPassport, verifyWebhookSignature };
+/**
+ * AI Assist utilities for enhanced PII detection
+ * Calls the hosted OpenRedaction AI endpoint to get additional entities
+ */
+
+/**
+ * AI endpoint response entity structure
+ */
+interface AIEntity {
+    type: string;
+    value: string;
+    start: number;
+    end: number;
+    confidence?: number;
+}
+/**
+ * AI endpoint response structure
+ */
+interface AIResponse {
+    entities: AIEntity[];
+    aiUsed: boolean;
+}
+/**
+ * Get the AI endpoint URL from options or environment
+ */
+declare function getAIEndpoint(aiOptions?: {
+    enabled?: boolean;
+    endpoint?: string;
+}): string | null;
+/**
+ * Call the AI endpoint to get additional PII entities
+ * Returns null if AI is disabled, endpoint unavailable, or on error
+ */
+declare function callAIDetect(text: string, endpoint: string, debug?: boolean): Promise<AIEntity[] | null>;
+/**
+ * Validate an AI entity
+ */
+declare function validateAIEntity(entity: AIEntity, textLength: number): boolean;
+/**
+ * Check if two detections overlap significantly
+ * Returns true if they overlap by more than 50% of the shorter detection
+ */
+declare function detectionsOverlap(det1: PIIDetection, det2: PIIDetection): boolean;
+/**
+ * Convert AI entity to PIIDetection format
+ */
+declare function convertAIEntityToDetection(entity: AIEntity, text: string): PIIDetection | null;
+/**
+ * Merge AI entities with regex detections
+ * Prefers regex detections on conflicts
+ */
+declare function mergeAIEntities(regexDetections: PIIDetection[], aiEntities: AIEntity[], text: string): PIIDetection[];
+
+export { ADMIN_ROLE, type AIEntity, type AIOptions, type AIResponse, ALL_PERMISSIONS, ANALYST_ROLE, type APIRequest, type APIResponse, APIServer, type APIServerConfig, type AuditBackend, type AuditDatabaseConfig, type AuditLogEntry, type AuditQueryFilter, type AuditStats, type BatchOptions, BatchProcessor, type BatchResult, type CellMatch$1 as CellMatch, type ChunkResult, type ColumnStats$1 as ColumnStats, ConfigExporter, ConfigLoader, ConsoleAuditLogger, type ContextAnalysis, type ContextFeatures, type ContextRulesConfig, ContextRulesEngine, type CsvDetectionResult, CsvProcessor, type CsvProcessorOptions, DEFAULT_DOMAIN_VOCABULARIES, DEFAULT_PROXIMITY_RULES, DEFAULT_SEVERITY_MAP, DEFAULT_TIER_QUOTAS, type DetectTask, type DetectionPass, type DetectionResult, type DocumentFormat, type DocumentMetadata, type DocumentOptions, DocumentProcessor, type DocumentResult, type DocumentTask, type DomainVocabulary, type ErrorSuggestion, ExplainAPI, type ExportedConfig, type FalsePositiveRule, GRAFANA_DASHBOARD_TEMPLATE, type HashedAuditLogEntry, type HealthCheckOptions, type HealthCheckResult, type HealthCheckStatus, HealthChecker, type HybridMatch, type IAuditDatabaseAdapter, type IAuditLogger, type IDocumentProcessor, type IMetricsCollector, type IMetricsExporter, type IOCRProcessor, type IRBACManager, type ImageFormat, InMemoryAuditLogger, InMemoryMetricsCollector, type JsonDetectionResult, JsonProcessor, type JsonProcessorOptions, type LearningData, type LearningStats, LocalLearningStore, type MultiPassStats, NERDetector, type NEREntityType, type NERMatch, type OCRLanguage, type OCROptions, OCRProcessor, type OCRResult, OPERATOR_ROLE, OpenRedaction, type OpenRedactionConfig, OpenRedactionError, type OpenRedactionMiddlewareOptions, type OpenRedactionOptions, type OpenRedactionRequest, type OptimizerOptions, type PIIDetection, type PIIMatch, type PIIPattern, type PatternAdjustment, type PatternMatchResult, type PatternStats, type Permission, PersistentAuditLogger, type PersistentAuditLoggerOptions, type PresetName, PriorityOptimizer, PrometheusServer, type PrometheusServerOptions, type ProximityRule, RBACManager, type RedactionMetrics, type RedactionMode, RegexMaxMatchesError, RegexTimeoutError, type ReportFormat, ReportGenerator, type ReportOptions, type ReportType, type RetentionPolicy, type RiskScore, type Role, type RoleName, SEVERITY_SCORES, type SafeRegexOptions, type SeverityClassification, SeverityClassifier, type SeverityLevel, type SheetDetectionResult, StreamingDetector, type StreamingOptions, type TenantConfig, TenantManager, TenantNotFoundError, TenantQuotaExceededError, type TenantQuotas, TenantSuspendedError, type TenantUsage, type TextExplanation, VIEWER_ROLE, type Validator, type WebhookConfig, type WebhookDelivery, type WebhookDeliveryStatus, type WebhookEvent, type WebhookEventType, WebhookManager, type WebhookStats, type WhitelistEntry, WorkerPool, type WorkerPoolConfig, type WorkerPoolStats, type WorkerResult, type WorkerTask, type XlsxDetectionResult, XlsxProcessor, type XlsxProcessorOptions, allPatterns, analyzeContextFeatures, analyzeFullContext, calculateContextConfidence, calculateRisk, callAIDetect, ccpaPreset, commonFalsePositives, compileSafeRegex, contactPatterns, convertAIEntityToDetection, createAPIServer, createBatchProcessor, createCacheDisabledError, createConfigLoadError, createConfigPreset, createContextRulesEngine, createCsvProcessor, createCustomRole, createDocumentProcessor, createExplainAPI, createHealthChecker, createHighMemoryError, createInvalidPatternError, createJsonProcessor, createLearningDisabledError, createMultiPassDisabledError, createNERDetector, createOCRProcessor, createOptimizationDisabledError, createPersistentAuditLogger, createPriorityOptimizer, createPrometheusServer, createRBACManager, createReportGenerator, createSeverityClassifier, createSimpleMultiPass, createStreamingDetector, createTenantManager, createValidationError, createWebhookManager, createWorkerPool, createXlsxProcessor, defaultPasses, detectPII, detectionsOverlap, educationPreset, exportForVersionControl, extractContext, filterFalsePositives, financePreset, financialPatterns, gdprPreset, generateReport, getAIEndpoint, getPatternsByCategory, getPredefinedRole, getPreset, getSeverity, governmentPatterns, groupPatternsByPass, healthCheckMiddleware, healthcarePreset, healthcareResearchPreset, hipaaPreset, inferDocumentType, isFalsePositive, isUnsafePattern, mergeAIEntities, mergePassDetections, networkPatterns, openredactionMiddleware, personalPatterns, safeExec, safeExecAll, transportLogisticsPreset, useAutoRedact, useBatchDetector, useFormFieldValidator, useOpenRedaction, usePIIDetector, validateAIEntity, validateEmail, validateIBAN, validateLuhn, validateNHS, validateNINO, validateName, validatePattern, validateSSN, validateSortCode, validateUKPassport, verifyWebhookSignature };