data_redactor 0.8.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aae84ce43ab8d2ad6751ade655397480057d687bdff7a6b857ee2821dffeb91b
-  data.tar.gz: 6e01ebe9d76e64ac3a93c31f14f7089ddaff4645e0819dec675d7585e37c4078
+  metadata.gz: 59ae814186478e16f6ba16f66aa1dfa8f3fd63d088cd5c837221d7530c6a0c73
+  data.tar.gz: 631c2a6f5198d7c2e741f9a283263ffbadfb49053bf6767ba57cce67b33e381f
 SHA512:
-  metadata.gz: a80b34b6e35fdf97cca2d9fecf1cb136b0e0c676ca1e3080c3680aaeb41b442cb2400c371e38417703910fc023ad01a3cf61f2f4a8f8dc5d4bd681174420d2b4
-  data.tar.gz: 4dbf049d027385c21a721044ac6651f4b24b33500af98a0c4c88d7860c06eb2721ea5aab4b8793f6fcd5614cac0cc645c1f67e73b5ebf8d7ff04ead58b67a244
+  metadata.gz: a5fdcc1bf088c9065f7e0c458fa4cf210917d688cb9b2b17e0824e59e9757f2f9c0491ef53c0e91f7ff29ac971b1ef6cc2d434a1d80505206e2d9f5b36893ca9
+  data.tar.gz: 1457805dc7599d1655ebb8bc569607be3380290f99f65cdf15cdc17fc7431932e4ff4250f967ff0d7459350d1d836c4d63857bf0b90ad408c2fcd7a969ab453f

data/CHANGELOG.md

@@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.10.0] - 2026-06-09
+
+### Changed
+- **Engine rewrite (v19 hybrid)** — `redact` and `scan` now run through a
+  Thompson NFA → bytecode → lazy-DFA multi-pattern engine (v19) for all 88
+  built-in patterns, replacing the previous per-pattern POSIX `regexec` loop.
+  Custom patterns (`add_pattern`) continue to use the glibc path (hybrid split
+  — required for correct UTF-8 multibyte character-class matching in user regex).
+- Throughput on a 1 MB log: **~8.4× faster** than the previous C engine
+  (0.87 i/s → 7.27 i/s); **2.25× faster** than pure-Ruby `gsub` (was 4×
+  slower). Small per-call strings: 1.7–2.3× faster (was 3–4.6× slower).
+- Overlap resolution: built-in matches are now resolved by an index-order
+  greedy claim (`mm_resolve`) that reproduces today's sequential per-pattern
+  rewrite semantics exactly. The one accepted divergence (rewrite-created
+  boundary when two secrets abut with no separator) is documented in
+  `TODO.md §1d` and pinned by `DIVERGENCE` specs.
+- `rb_data_redactor_scan`: coordinate mapping (`repl_log` / `WORKING_TO_ORIG`)
+  replaced by direct original-frame offset emission from the v19 engine; custom
+  patterns use a lightweight offset-walk over the built-in event list.
+
+### Fixed
+- **Swiss AHV false-negative** — boundary-wrapped patterns with a
+  start-anchored required literal now correctly set `max_back = 1` (not 0) so
+  the literal-skip does not overshoot the boundary byte. `756.1234.5678.90`
+  now matches as expected. (Pre-existing bug in the old engine, caught by
+  going live.)
+
+## [0.9.0] - 2026-05-22
+
+### Added
+- `DataRedactor.name_pattern(first, last, middle:)` — generates a POSIX ERE that matches a person's name across common written variations (case-insensitivity, First/Last order swaps, `Last, First`, initials, diacritics, and interchangeable space/hyphen separators). Returns a String ready to pass to `add_pattern`. The pattern is boundary-wrapped, so `"Mario"` matches as a word but not inside `"Mariolino"`. When `middle:` is given, both the no-middle and with-middle forms match.
+
+## [0.8.0] - 2026-05-21
+
 ### Added
 - `DataRedactor.redact_deep(data, only:, except:, placeholder:)` — recursively redacts every String value in a nested Hash/Array structure. Non-string scalars (Integer, Float, nil, Boolean) and Hash keys are passed through unchanged. Returns a deep copy; never mutates the input. Raises `ArgumentError` on circular references.
 - `DataRedactor.redact_json(json_string, only:, except:, placeholder:)` — parses JSON, redacts via `redact_deep`, and returns valid JSON. Raises `JSON::ParserError` on invalid input.
@@ -170,7 +204,9 @@ features as 0.7.1 plus the pipeline fix.
 - `DataRedactor.redact(text)` module function returning the input with every match replaced by `[REDACTED]`.
 - RSpec suite with one example per pattern.
 
-[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.2...HEAD
+[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.9.0...HEAD
+[0.9.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.8.0...v0.9.0
+[0.8.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.2...v0.8.0
 [0.7.2]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.1...v0.7.2
 [0.7.1]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.0...v0.7.1
 [0.7.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.6.1...v0.7.0

data/ext/data_redactor/custom_patterns.c

@@ -3,6 +3,11 @@
 #include <string.h>
 #include <stdlib.h>
 
+/* Custom patterns deliberately do NOT use the v19 engine: they keep the glibc
+ * regexec path (replace_all_matches), because user regex can contain multibyte
+ * UTF-8 inside character classes (e.g. name_pattern's [oOòóô…]) that the
+ * byte-level v19 engine cannot match the way glibc's locale-aware matcher does.
+ * See TODO.md §1d Gap 2 for the hybrid-split rationale. */
 custom_pattern_t *custom_patterns = NULL;
 int custom_count = 0;
 int custom_cap   = 0;

data/ext/data_redactor/data_redactor.c

@@ -7,8 +7,12 @@
 #include "redact.h"
 #include "scan.h"
 #include "custom_patterns.h"
+#include "matcher.h"
 
 void Init_data_redactor(void) {
+    /* Build the v19 multi-pattern engines from the gem's pattern arrays. */
+    mm_init();
+
     /* Compile all built-in regex patterns at load time. */
     for (int i = 0; i < NUM_PATTERNS; i++) {
         const char *pat;
@@ -43,15 +47,25 @@ void Init_data_redactor(void) {
     rb_define_module_function(mDataRedactor, "_clear_custom_patterns", rb_clear_custom_patterns,   0);
     rb_define_module_function(mDataRedactor, "_custom_patterns",       rb_custom_patterns,         0);
 
-    /* Frozen array of built-in pattern names, for introspection and only:/except: validation. */
+    /* Frozen introspection arrays, parallel to the pattern table (same index = same
+     * pattern). NAMES/TAG_BITS back only:/except: validation; SOURCES/BOUNDARY are
+     * internal aids (the benchmark suite rebuilds the patterns in pure Ruby from
+     * them). SOURCES holds the unwrapped POSIX ERE — boundary wrapping is applied
+     * above at compile time for patterns with BOUNDARY[i] == true. */
     VALUE builtin_names = rb_ary_new_capa(NUM_PATTERNS);
     VALUE builtin_tag_bits = rb_ary_new_capa(NUM_PATTERNS);
+    VALUE builtin_sources = rb_ary_new_capa(NUM_PATTERNS);
+    VALUE builtin_boundary = rb_ary_new_capa(NUM_PATTERNS);
     for (int i = 0; i < NUM_PATTERNS; i++) {
         rb_ary_push(builtin_names, rb_str_new_frozen(rb_str_new_cstr(pattern_names[i])));
         rb_ary_push(builtin_tag_bits, INT2NUM(pattern_tags[i]));
+        rb_ary_push(builtin_sources, rb_str_new_frozen(rb_str_new_cstr(pattern_strings[i])));
+        rb_ary_push(builtin_boundary, boundary_wrapped[i] ? Qtrue : Qfalse);
     }
     rb_define_const(mDataRedactor, "BUILTIN_PATTERN_NAMES",    rb_ary_freeze(builtin_names));
     rb_define_const(mDataRedactor, "BUILTIN_PATTERN_TAG_BITS", rb_ary_freeze(builtin_tag_bits));
+    rb_define_const(mDataRedactor, "BUILTIN_PATTERN_SOURCES",  rb_ary_freeze(builtin_sources));
+    rb_define_const(mDataRedactor, "BUILTIN_PATTERN_BOUNDARY", rb_ary_freeze(builtin_boundary));
 
     /* Placeholder mode constants. */
     rb_define_const(mDataRedactor, "PH_MODE_PLAIN",  INT2NUM(PLACEHOLDER_MODE_PLAIN));