od-temp 1.0.4

package/README.md

@@ -0,0 +1,130 @@
+# OpenRedaction
+
+Production-ready PII detection and redaction library with 571+ built-in patterns, multiple redaction modes, compliance presets, enterprise SaaS features, and zero dependencies.
+
+## Installation
+
+```bash
+npm install openredaction
+```
+
+## Quick Start
+
+```typescript
+import { OpenRedaction } from 'openredaction';
+
+const shield = new OpenRedaction();
+const result = shield.detect("Email john@example.com or call 07700900123");
+
+console.log(result.redacted);
+// "Email [EMAIL_9619] or call [PHONE_UK_MOBILE_9478]"
+```
+
+## Optional AI Assist
+
+OpenRedaction supports an optional AI-assisted detection mode that enhances regex-based detection by calling a hosted AI endpoint. This feature is **OFF by default** and requires explicit configuration.
+
+### Configuration
+
+```typescript
+import { OpenRedaction } from 'openredaction';
+
+const detector = new OpenRedaction({
+  // ... other options ...
+  ai: {
+    enabled: true,
+    endpoint: 'https://your-api.example.com' // Optional: defaults to OPENREDACTION_AI_ENDPOINT env var
+  }
+});
+
+// detect() is now async when AI is enabled
+const result = await detector.detect('Contact John Doe at john@example.com');
+```
+
+### How It Works
+
+1. **Regex Detection First**: The library always runs regex detection first (existing behavior)
+2. **AI Enhancement**: If `ai.enabled === true` and an endpoint is configured, the library calls the `/ai-detect` endpoint
+3. **Smart Merging**: AI entities are merged with regex detections, with regex taking precedence on conflicts
+4. **Graceful Fallback**: If the AI endpoint fails or is unavailable, the library silently falls back to regex-only detection
+
+### Environment Variables
+
+In Node.js environments, you can set the endpoint via environment variable:
+
+```bash
+export OPENREDACTION_AI_ENDPOINT=https://your-api.example.com
+```
+
+### Important Notes
+
+- **AI is optional**: The library works exactly as before when `ai.enabled` is `false` or omitted
+- **Regex is primary**: AI only adds additional entities; regex detections always take precedence
+- **No breaking changes**: When AI is disabled, behavior is identical to previous versions
+- **Browser support**: In browsers, you must provide an explicit `ai.endpoint` (env vars not available)
+- **Network dependency**: AI mode requires network access to the endpoint
+
+### For Sensitive Workloads
+
+For maximum security and privacy, keep AI disabled and rely purely on regex detection:
+
+```typescript
+const detector = new OpenRedaction({
+  // AI not configured = pure regex detection
+  includeNames: true,
+  includeEmails: true
+});
+```
+
+## Documentation
+
+Full documentation available at [GitHub](https://github.com/sam247/openredaction)
+
+## Features
+
+- 🚀 **Fast & Accurate** - 10-20ms for 2-3KB text
+- 🎯 **571+ PII Patterns** - Comprehensive coverage across multiple categories
+- 🔐 **Enterprise SaaS Ready** - Multi-tenancy, persistent audit logging, webhooks, REST API
+- 📊 **Production Monitoring** - Prometheus metrics, Grafana dashboards, health checks
+- 🧠 **Semantic Detection** - Hybrid NER + regex with 40+ contextual rules
+- 🎨 **Multiple Redaction Modes** - Placeholder, mask-middle, mask-all, format-preserving, token-replace
+- ✅ **Built-in Validators** - Luhn, IBAN, NHS, National ID checksums
+- 🔒 **Compliance Presets** - GDPR, HIPAA, CCPA plus finance, education, healthcare, and transport presets
+- 🎭 **Deterministic Placeholders** - Consistent redaction for same values
+- 🌍 **Global Coverage** - 50+ countries
+- 📄 **Structured Data Support** - JSON, CSV, XLSX with path/cell tracking
+- 🌳 **Zero Dependencies** - No external packages required (core)
+- 📝 **TypeScript Native** - Full type safety and IntelliSense
+- 🧪 **Battle Tested** - 276+ passing tests
+
+## Pattern Categories
+
+### Personal Information
+Email, Phone Numbers (US, UK, International), Names, Social Security Numbers, Passports, Driver's Licenses
+
+### Financial (13 patterns)
+Credit Cards, IBANs, Bank Accounts, Swift Codes, Routing Numbers, IFSC, CLABE, BSB, ISIN, CUSIP, SEDOL, LEI, Cryptocurrencies
+
+### Government IDs (50+ countries)
+SSN, NINO, NHS, Passports, Tax IDs, UTR, VAT, Company Numbers, ITIN, SIN, and more
+
+### Healthcare
+Medical Record Numbers, NHS Numbers, CHI, EHIC, Health Insurance, Prescription Numbers, DEA Numbers, Biometric Data
+
+### Digital Identity
+API Keys, OAuth Tokens, JWT, Bearer Tokens, Discord, Steam, Social Media IDs
+
+### Industries (25+)
+Retail, Legal, Real Estate, Logistics, Insurance, Healthcare, Emergency Response, Hospitality, Professional Certifications, and more
+
+## Enterprise Features
+
+- **Persistent Audit Logging** - SQLite/PostgreSQL with cryptographic hashing
+- **Multi-Tenancy** - Tenant isolation, quotas, usage tracking
+- **Prometheus Metrics** - HTTP server with Grafana dashboards
+- **Webhook System** - Event-driven alerts with retry logic
+- **REST API** - Production-ready HTTP API with authentication
+
+## License
+
+MIT

package/dist/cli/test-pattern.cjs

@@ -0,0 +1,378 @@
+#!/usr/bin/env node
+#!/usr/bin/env node
+
+//#region src/utils/safe-regex.ts
+/**
+* Test if a regex pattern is potentially unsafe (basic static analysis)
+* Detects common ReDoS patterns
+*
+* Note: This is a very basic heuristic check. The real protection comes from
+* the execution timeout in safeExec(). This just catches obvious mistakes.
+*/
+function isUnsafePattern(pattern) {
+	if (/\*\+|\+\*|\+\+|\*\*/.test(pattern)) return true;
+	if (/\(a\+\)\+|\(b\*\)\*|\(c\+\)\+/.test(pattern)) return true;
+	return false;
+}
+/**
+* Validate a regex pattern before use
+* Throws error if pattern is potentially unsafe
+*/
+function validatePattern(pattern) {
+	const patternStr = typeof pattern === "string" ? pattern : pattern.source;
+	if (patternStr.length > 5e3) throw new Error(`Regex pattern too long: ${patternStr.length} chars (max 5000)`);
+	if (isUnsafePattern(patternStr)) throw new Error(`Potentially unsafe regex pattern detected: ${patternStr.substring(0, 100)}...`);
+	try {
+		new RegExp(patternStr);
+	} catch (error) {
+		throw new Error(`Invalid regex pattern: ${error.message}`);
+	}
+}
+
+//#endregion
+//#region src/cli/test-pattern.ts
+const args = process.argv.slice(2);
+function printHelp() {
+	console.log(`
+OpenRedaction Pattern Testing Tool
+
+Test custom patterns before deployment to prevent ReDoS vulnerabilities and validate functionality.
+
+Usage:
+  openredaction-test-pattern validate <pattern>           Validate pattern safety
+  openredaction-test-pattern test <pattern> <text>        Test pattern against sample text
+  openredaction-test-pattern check <pattern> [flags]      Check pattern with optional flags
+  openredaction-test-pattern benchmark <pattern> <text>   Benchmark pattern performance
+  openredaction-test-pattern --help                       Show this help message
+
+Commands:
+  validate <pattern>
+    Checks if a regex pattern is safe (no ReDoS vulnerabilities)
+    Returns: SAFE or UNSAFE with explanation
+
+  test <pattern> <text>
+    Tests a pattern against sample text and shows all matches
+    Returns: List of matches with positions
+
+  check <pattern> [flags]
+    Validates pattern syntax and compiles with optional flags
+    Returns: Pattern info and any warnings
+
+  benchmark <pattern> <text>
+    Measures pattern execution time and match count
+    Returns: Performance metrics
+
+Options:
+  --flags <flags>         Regex flags (g, i, m, etc.)
+  --timeout <ms>          Regex timeout in milliseconds (default: 100)
+  --json                  Output results as JSON
+  --verbose               Show detailed output
+
+Examples:
+  # Validate a pattern for ReDoS
+  openredaction-test-pattern validate "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
+
+  # Test pattern against sample text
+  openredaction-test-pattern test "\\b\\d{3}-\\d{2}-\\d{4}\\b" "SSN: 123-45-6789"
+
+  # Check pattern with flags
+  openredaction-test-pattern check "[a-z]+" --flags gi
+
+  # Benchmark pattern performance
+  openredaction-test-pattern benchmark "\\b[A-Z][a-z]+ [A-Z][a-z]+\\b" "John Smith and Jane Doe"
+
+  # Test a custom pattern as JSON
+  openredaction-test-pattern test "\\b\\d{16}\\b" "Card: 4111111111111111" --json
+
+Safety Checks:
+  ✓ Nested quantifiers (e.g., (a+)+)
+  ✓ Overlapping alternation (e.g., (a|ab)+)
+  ✓ Consecutive quantifiers (e.g., a*+)
+  ✓ Dangerous backreferences (e.g., \\1+)
+  ✓ Excessive pattern length (>5000 chars)
+  ✓ Pattern compilation errors
+`);
+}
+function parseOptions(args) {
+	const options = {};
+	for (let i = 0; i < args.length; i++) if (args[i] === "--flags" && args[i + 1]) {
+		options.flags = args[i + 1];
+		i++;
+	} else if (args[i] === "--timeout" && args[i + 1]) {
+		options.timeout = parseInt(args[i + 1], 10);
+		i++;
+	} else if (args[i] === "--json") options.json = true;
+	else if (args[i] === "--verbose") options.verbose = true;
+	return options;
+}
+function validatePatternCommand(pattern, options) {
+	const result = {
+		pattern,
+		safe: true,
+		warnings: [],
+		errors: []
+	};
+	try {
+		if (isUnsafePattern(pattern)) {
+			result.safe = false;
+			result.errors.push("Pattern contains potentially unsafe constructs (ReDoS risk)");
+			if (/(\([^)]*[*+{][^)]*\)[*+{])/.test(pattern)) result.warnings.push("Nested quantifiers detected: (a+)+ or (a*)*");
+			if (/\([^)]*\|[^)]*\)[*+{]/.test(pattern)) result.warnings.push("Overlapping alternation with quantifier: (a|ab)+");
+			if (/[*+?{][*+?{]/.test(pattern)) result.warnings.push("Consecutive quantifiers: a*+ or a+*");
+			if (/\\\d[*+{]/.test(pattern)) result.warnings.push("Backreference with quantifier: \\1+");
+		}
+		if (pattern.length > 5e3) {
+			result.safe = false;
+			result.errors.push(`Pattern too long: ${pattern.length} chars (max 5000)`);
+		}
+		validatePattern(pattern);
+		if (result.safe) result.message = "✓ Pattern is SAFE";
+	} catch (error) {
+		result.safe = false;
+		result.errors.push(error.message);
+	}
+	if (options.json) console.log(JSON.stringify(result, null, 2));
+	else {
+		console.log("\nPattern Validation Result:");
+		console.log("─".repeat(50));
+		console.log(`Pattern: ${pattern}`);
+		console.log(`Status: ${result.safe ? "✓ SAFE" : "✗ UNSAFE"}`);
+		if (result.warnings.length > 0) {
+			console.log("\nWarnings:");
+			result.warnings.forEach((w) => console.log(`  ⚠ ${w}`));
+		}
+		if (result.errors.length > 0) {
+			console.log("\nErrors:");
+			result.errors.forEach((e) => console.log(`  ✗ ${e}`));
+		}
+		if (result.safe) console.log("\n✓ Pattern is safe to use");
+		else console.log("\n✗ Pattern is NOT safe - please revise before use");
+	}
+	process.exit(result.safe ? 0 : 1);
+}
+function testPatternCommand(pattern, text, options) {
+	const result = {
+		pattern,
+		text,
+		matches: [],
+		matchCount: 0
+	};
+	try {
+		validatePattern(pattern);
+		const flags = options.flags || "g";
+		const regex = new RegExp(pattern, flags);
+		let match;
+		while ((match = regex.exec(text)) !== null) {
+			result.matches.push({
+				value: match[0],
+				captureGroups: match.slice(1),
+				index: match.index,
+				length: match[0].length
+			});
+			result.matchCount++;
+			if (result.matchCount >= 1e3) {
+				result.warning = "Stopped after 1000 matches";
+				break;
+			}
+			if (match.index === regex.lastIndex) regex.lastIndex++;
+		}
+		result.success = true;
+	} catch (error) {
+		result.success = false;
+		result.error = error.message;
+	}
+	if (options.json) console.log(JSON.stringify(result, null, 2));
+	else {
+		console.log("\nPattern Test Result:");
+		console.log("─".repeat(50));
+		console.log(`Pattern: ${pattern}`);
+		console.log(`Flags: ${options.flags || "g"}`);
+		console.log(`Text: ${text}`);
+		console.log(`Matches: ${result.matchCount}`);
+		if (result.matchCount > 0) {
+			console.log("\nMatches Found:");
+			result.matches.forEach((m, i) => {
+				console.log(`  ${i + 1}. "${m.value}" at position ${m.index}`);
+				if (m.captureGroups.length > 0 && m.captureGroups.some((g) => g)) console.log(`     Capture groups: [${m.captureGroups.join(", ")}]`);
+			});
+		} else console.log("\n⚠ No matches found");
+		if (result.warning) console.log(`\n⚠ ${result.warning}`);
+		if (result.error) console.log(`\n✗ Error: ${result.error}`);
+	}
+	process.exit(result.success ? 0 : 1);
+}
+function checkPatternCommand(pattern, options) {
+	const result = {
+		pattern,
+		valid: false,
+		info: {},
+		warnings: []
+	};
+	try {
+		validatePattern(pattern);
+		const flags = options.flags || "";
+		const regex = new RegExp(pattern, flags);
+		result.valid = true;
+		result.info = {
+			source: regex.source,
+			flags: regex.flags,
+			length: pattern.length,
+			hasGroups: /\([^)]*\)/.test(pattern),
+			hasQuantifiers: /[*+?{]/.test(pattern),
+			hasAnchors: /[\^$]/.test(pattern),
+			hasLookahead: /\(\?[=!]/.test(pattern),
+			hasLookbehind: /\(\?<[=!]/.test(pattern)
+		};
+		if (isUnsafePattern(pattern)) result.warnings.push("Pattern may be vulnerable to ReDoS attacks");
+		if (pattern.length > 1e3) result.warnings.push("Pattern is very long, may impact performance");
+		if (!flags.includes("g") && /[*+{]/.test(pattern)) result.warnings.push("Pattern has quantifiers but no global flag - will only match once");
+	} catch (error) {
+		result.valid = false;
+		result.error = error.message;
+	}
+	if (options.json) console.log(JSON.stringify(result, null, 2));
+	else {
+		console.log("\nPattern Check Result:");
+		console.log("─".repeat(50));
+		console.log(`Pattern: ${pattern}`);
+		console.log(`Flags: ${options.flags || "(none)"}`);
+		console.log(`Valid: ${result.valid ? "✓ Yes" : "✗ No"}`);
+		if (result.valid) {
+			console.log("\nPattern Info:");
+			console.log(`  Length: ${result.info.length} characters`);
+			console.log(`  Has capture groups: ${result.info.hasGroups ? "Yes" : "No"}`);
+			console.log(`  Has quantifiers: ${result.info.hasQuantifiers ? "Yes" : "No"}`);
+			console.log(`  Has anchors (^/$): ${result.info.hasAnchors ? "Yes" : "No"}`);
+			console.log(`  Has lookahead: ${result.info.hasLookahead ? "Yes" : "No"}`);
+			console.log(`  Has lookbehind: ${result.info.hasLookbehind ? "Yes" : "No"}`);
+		}
+		if (result.warnings.length > 0) {
+			console.log("\nWarnings:");
+			result.warnings.forEach((w) => console.log(`  ⚠ ${w}`));
+		}
+		if (result.error) console.log(`\n✗ Error: ${result.error}`);
+	}
+	process.exit(result.valid ? 0 : 1);
+}
+function benchmarkPatternCommand(pattern, text, options) {
+	const result = {
+		pattern,
+		text,
+		textLength: text.length,
+		metrics: {}
+	};
+	try {
+		validatePattern(pattern);
+		const flags = options.flags || "g";
+		const regex = new RegExp(pattern, flags);
+		const startTime = performance.now();
+		let matchCount = 0;
+		let match;
+		while ((match = regex.exec(text)) !== null) {
+			matchCount++;
+			if (matchCount >= 1e4) {
+				result.warning = "Stopped after 10000 matches";
+				break;
+			}
+			if (match.index === regex.lastIndex) regex.lastIndex++;
+		}
+		const executionTime = performance.now() - startTime;
+		result.metrics = {
+			executionTime: `${executionTime.toFixed(3)}ms`,
+			matchCount,
+			matchesPerMs: matchCount > 0 ? (matchCount / executionTime).toFixed(2) : "0",
+			charsPerMs: (text.length / executionTime).toFixed(0)
+		};
+		result.success = true;
+	} catch (error) {
+		result.success = false;
+		result.error = error.message;
+	}
+	if (options.json) console.log(JSON.stringify(result, null, 2));
+	else {
+		console.log("\nPattern Benchmark Result:");
+		console.log("─".repeat(50));
+		console.log(`Pattern: ${pattern}`);
+		console.log(`Text length: ${result.textLength} characters`);
+		console.log("\nPerformance Metrics:");
+		console.log(`  Execution time: ${result.metrics.executionTime}`);
+		console.log(`  Matches found: ${result.metrics.matchCount}`);
+		console.log(`  Throughput: ${result.metrics.charsPerMs} chars/ms`);
+		if (result.warning) console.log(`\n⚠ ${result.warning}`);
+		if (result.error) console.log(`\n✗ Error: ${result.error}`);
+		const execTime = parseFloat(result.metrics.executionTime);
+		console.log("\nPerformance Assessment:");
+		if (execTime < 1) console.log("  ✓ Excellent - Very fast execution");
+		else if (execTime < 10) console.log("  ✓ Good - Acceptable performance");
+		else if (execTime < 50) console.log("  ⚠ Fair - May be slow on large texts");
+		else console.log("  ✗ Poor - Pattern needs optimization");
+	}
+	process.exit(result.success ? 0 : 1);
+}
+async function main() {
+	if (args.length === 0 || args.includes("--help") || args.includes("-h")) {
+		printHelp();
+		process.exit(0);
+	}
+	const command = args[0];
+	const options = parseOptions(args);
+	try {
+		switch (command) {
+			case "validate": {
+				const pattern = args[1];
+				if (!pattern) {
+					console.error("Error: Pattern is required");
+					console.log("Usage: openredaction-test-pattern validate <pattern>");
+					process.exit(1);
+				}
+				validatePatternCommand(pattern, options);
+				break;
+			}
+			case "test": {
+				const pattern = args[1];
+				const text = args[2];
+				if (!pattern || !text) {
+					console.error("Error: Pattern and text are required");
+					console.log("Usage: openredaction-test-pattern test <pattern> <text>");
+					process.exit(1);
+				}
+				testPatternCommand(pattern, text, options);
+				break;
+			}
+			case "check": {
+				const pattern = args[1];
+				if (!pattern) {
+					console.error("Error: Pattern is required");
+					console.log("Usage: openredaction-test-pattern check <pattern> [--flags <flags>]");
+					process.exit(1);
+				}
+				checkPatternCommand(pattern, options);
+				break;
+			}
+			case "benchmark": {
+				const pattern = args[1];
+				const text = args[2];
+				if (!pattern || !text) {
+					console.error("Error: Pattern and text are required");
+					console.log("Usage: openredaction-test-pattern benchmark <pattern> <text>");
+					process.exit(1);
+				}
+				benchmarkPatternCommand(pattern, text, options);
+				break;
+			}
+			default:
+				console.error(`Unknown command: ${command}`);
+				console.log("Run with --help for usage information");
+				process.exit(1);
+		}
+	} catch (error) {
+		console.error("Error:", error.message);
+		process.exit(1);
+	}
+}
+main().catch((error) => {
+	console.error("Fatal error:", error);
+	process.exit(1);
+});
+
+//#endregion