data_redactor 0.9.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 007d59e430d1675a13b84670f6c34c300f8b72fd7ee4744aa191f846bb89b072
-  data.tar.gz: a23f3b99c3ead341d2c9415a1b4b2eb32a45ee002f052a8e58d928eb1ce03919
+  metadata.gz: e744f5e18d5ce2311c21197b6d47b20d6032a38a72c8bfb2287a80219b8e2d77
+  data.tar.gz: cfd314afec1d018175a8424d81c12958582d5ddd9c16d05baa6048c17bba8ca8
 SHA512:
-  metadata.gz: ccd4f6f97a0110585e4f43f9402eac2a1f57b2aef01a3c6870f0e57ea578377291a7367ee924585d9d11e92af98f4178bb0b9488c1a24a2338f6a41936efad30
-  data.tar.gz: 5281171119b4892167a6b1d55e0996db47408c8a6d334656998f8f2ca50794a3a7b5c987132369ca32965da0943f954eab61f34f5a97c683b8a14851e9beca1e
+  metadata.gz: f7ea267c8927f9852621180d77530818980af4fc5c089b7db95e6b0f980c1f3cb09c11806e9459241ca363b5c06203f34ede5e2b1473b1a2046a9a4e37e63fbb
+  data.tar.gz: 4081b898b339423bb5dd06f557e32bea5c7242869fc61a6b0a2447053ddcec28a751768b83d1ba66a2fcb82755ceb654c6e95052bc01e2cb23e1fac4b44b8d37

data/CHANGELOG.md

@@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.10.1] - 2026-06-10
+
+### Fixed
+- **musl/Alpine load failure** — the `hashicorp_vault_batch_token` pattern used a
+  `{138,300}` interval whose upper bound exceeds POSIX `RE_DUP_MAX` (255). glibc
+  accepts it, but musl's `regcomp` rejects it ("Invalid contents of {}"), so the
+  native musl gem raised at load (`require "data_redactor"`) on Alpine. Capped the
+  bound at 255; tokens are still neutralized (prefix + 251+ chars redacted).
+
+## [0.10.0] - 2026-06-09
+
+### Changed
+- **Engine rewrite (v19 hybrid)** — `redact` and `scan` now run through a
+  Thompson NFA → bytecode → lazy-DFA multi-pattern engine (v19) for all 88
+  built-in patterns, replacing the previous per-pattern POSIX `regexec` loop.
+  Custom patterns (`add_pattern`) continue to use the glibc path (hybrid split
+  — required for correct UTF-8 multibyte character-class matching in user regex).
+- Throughput on a 1 MB log: **~8.4× faster** than the previous C engine
+  (0.87 i/s → 7.27 i/s); **2.25× faster** than pure-Ruby `gsub` (was 4×
+  slower). Small per-call strings: 1.7–2.3× faster (was 3–4.6× slower).
+- Overlap resolution: built-in matches are now resolved by an index-order
+  greedy claim (`mm_resolve`) that reproduces today's sequential per-pattern
+  rewrite semantics exactly. The one accepted divergence (rewrite-created
+  boundary when two secrets abut with no separator) is documented in
+  `TODO.md §1d` and pinned by `DIVERGENCE` specs.
+- `rb_data_redactor_scan`: coordinate mapping (`repl_log` / `WORKING_TO_ORIG`)
+  replaced by direct original-frame offset emission from the v19 engine; custom
+  patterns use a lightweight offset-walk over the built-in event list.
+
+### Fixed
+- **Swiss AHV false-negative** — boundary-wrapped patterns with a
+  start-anchored required literal now correctly set `max_back = 1` (not 0) so
+  the literal-skip does not overshoot the boundary byte. `756.1234.5678.90`
+  now matches as expected. (Pre-existing bug in the old engine, caught by
+  going live.)
+
 ## [0.9.0] - 2026-05-22
 
 ### Added
@@ -177,7 +213,9 @@ features as 0.7.1 plus the pipeline fix.
 - `DataRedactor.redact(text)` module function returning the input with every match replaced by `[REDACTED]`.
 - RSpec suite with one example per pattern.
 
-[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.9.0...HEAD
+[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.10.1...HEAD
+[0.10.1]: https://github.com/danielefrisanco/data_redactor/compare/v0.10.0...v0.10.1
+[0.10.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.9.0...v0.10.0
 [0.9.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.8.0...v0.9.0
 [0.8.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.2...v0.8.0
 [0.7.2]: https://github.com/danielefrisanco/data_redactor/compare/v0.7.1...v0.7.2

data/{readme.md → README.md}

@@ -10,9 +10,10 @@ A Ruby gem with a C extension for high-performance regex-based redaction of sens
 
 DataRedactor scans text for sensitive data — API keys and cloud secrets, IBANs,
 credit cards, national IDs, emails, phone numbers, IPs, and more — and replaces
-each match with a placeholder. The scanning runs in a C extension backed by POSIX
-`regex.h`, so the heavy lifting happens outside the Ruby VM and stays fast enough
-to run inline on large payloads.
+each match with a placeholder. The scanning runs in a C extension backed by a
+zero-dependency Thompson NFA → lazy-DFA multi-pattern engine (v19) that scans
+all 88 built-in patterns in a single pass — 2–2.5× faster than pure-Ruby `gsub`
+on large payloads, with no external library dependencies.
 
 It ships **88 built-in patterns** across 15+ countries, grouped into tags
 (`:credentials`, `:financial`, `:contact`, ...) so you can redact only what you
@@ -384,8 +385,18 @@ redactor/
 │       ├── scan.{c,h}            # _scan + byte-offset replacement-log macros
 │       ├── custom_patterns.{c,h} # Dynamic registry: add/remove/clear/list
 │       └── tags.h                # TAG_* bit constants
-└── spec/
-    └── data_redactor_spec.rb     # RSpec tests — at least one example per pattern, plus filter / placeholder / custom-pattern coverage
+├── spec/
+│   └── data_redactor_spec.rb     # RSpec tests — at least one example per pattern, plus filter / placeholder / custom-pattern coverage
+├── benchmark/                    # Repo-only perf scripts (not packaged in the gem)
+│   ├── README.md                 # How to run, what each script measures
+│   ├── support/corpus.rb         # Shared payload builders + pure-Ruby baseline redactor
+│   ├── throughput.rb             # MB/s on representative payloads
+│   ├── vs_pure_ruby.rb           # C extension vs pure-Ruby gsub (same 88 patterns)
+│   ├── scaling.rb                # Runtime vs input size 1KB → 50MB
+│   └── per_pattern.rb            # Per-pattern scan cost
+└── docs/                         # Design and execution docs for future work
+    ├── standalone_matcher_design.md
+    └── combined_matcher_plan.md
 ```
 
 ## Requirements
@@ -460,6 +471,45 @@ Or compile and test in one step:
 bundle exec rake
 ```
 
+## Benchmarks
+
+The `benchmark/` directory holds four scripts that measure the C engine under
+different angles. They are **not** packaged with the gem.
+
+```bash
+bundle install                                   # pulls benchmark-ips, benchmark-memory (dev deps)
+bundle exec rake compile
+bundle exec ruby benchmark/vs_pure_ruby.rb       # head-to-head vs pure-Ruby gsub, same 88 patterns
+bundle exec ruby benchmark/throughput.rb         # MB/s on a log line, JSON, 1MB and 10MB log files
+bundle exec ruby benchmark/scaling.rb            # runtime vs input size (1KB → 50MB), confirms linear scaling
+bundle exec ruby benchmark/per_pattern.rb        # per-pattern scan cost over a 1MB payload
+```
+
+See [`benchmark/README.md`](benchmark/README.md) for what each script measures
+and how the pure-Ruby baseline is kept honest (it reads the same patterns the
+C engine uses, via `DataRedactor::BUILTIN_PATTERN_SOURCES`).
+
+### Performance (0.10.0 — v19 multi-pattern engine)
+
+As of 0.10.0 the C extension runs a **Thompson NFA → lazy-DFA multi-pattern
+engine** (v19) that scans the input once across all 88 built-in patterns,
+with two selective-merge passes (pure-digit group + IBAN union) that further
+reduce work for the most common pattern classes. Custom patterns (`add_pattern`)
+still use the glibc path (required for correct UTF-8 diacritic matching).
+
+| Payload               | v19 engine (0.10.0) | Pure-Ruby `gsub` | Ratio           |
+|-----------------------|---------------------|------------------|-----------------|
+| log line (168 B)      | 41 µs / call        | 71 µs / call     | **1.7× faster** |
+| JSON blob (~580 B)    | 81 µs / call        | 132 µs / call    | **1.6× faster** |
+| 8 log lines (1.3 KB)  | 175 µs / call       | 399 µs / call    | **2.3× faster** |
+| 100 log lines (17 KB) | 2.0 ms / call       | 4.6 ms / call    | **2.3× faster** |
+| 1 MB log              | 138 ms / call       | 294 ms / call    | **2.1× faster** |
+| 10 MB log             | 1.44 s / call       | —                | 6.9 MB/s        |
+
+All payload sizes pass a correctness check (redaction count matches pure-Ruby `gsub`).
+The previous engine (per-pattern `regexec`) was **4.25× slower** than pure Ruby on the
+1 MB payload — a ~9× swing. Old numbers are in git history (`CHANGELOG.md` [0.9.0]).
+
 ## How it works
 
 1. At load time, `Init_data_redactor` compiles all 85 regex patterns once using `regcomp` (POSIX ERE) and stores them as static `regex_t` structs. Patterns marked as boundary-wrapped are expanded with `wrap_boundary()` before compilation.
@@ -473,7 +523,7 @@ All C-side buffers are heap-allocated with `malloc`/`strdup` and freed before th
 
 ## Thread safety
 
-`DataRedactor.redact` and `DataRedactor.scan` are safe to call concurrently from multiple threads. Built-in patterns are compiled into a static `regex_t` array at load time and never mutated afterward, and each call allocates its own working buffers. POSIX `regexec` is documented as thread-safe.
+`DataRedactor.redact` and `DataRedactor.scan` are safe to call concurrently from multiple threads. The v19 engine holds MRI's GVL for the duration of each call (no `rb_thread_call_without_gvl`), so concurrent calls are serialised by the GVL. Each call allocates its own working buffers; built-in engine state is read-only after `mm_init()` at load time.
 
 `DataRedactor.add_pattern`, `remove_pattern`, and `clear_custom_patterns!` mutate a shared dynamic array and are **not** thread-safe. Register custom patterns once at boot — before spawning worker threads or forking — and they will be visible (read-only) to every subsequent `redact`/`scan` call.
 
@@ -490,3 +540,4 @@ Released under the [MIT License](LICENSE).
 - **Pattern ordering matters** — patterns run sequentially. An early broad pattern (e.g. the 9-digit passport) may consume digits that a later pattern (e.g. credit card) depends on. Boundary wrapping mitigates this for pure-digit patterns.
 - **AWS Secret Key (pattern 1)** — 40 consecutive base64 characters is a broad match. It can produce false positives in base64-encoded content such as embedded images or binary blobs.
 - **Duplicate digit patterns** — several national ID formats share the same digit-length (11 digits: PESEL, Norwegian Fødselsnummer, Belgian National Number). They are kept as separate slots for clarity but the practical effect is that any 11-digit boundary-delimited number will be redacted.
+- **Single-pass overlap semantics** — built-in patterns are resolved by an index-order greedy claim: the lower-index pattern wins any region it matches. When two secrets abut with no separator, a rewrite-created word boundary can cause the second to be missed. This is rare in real text (secrets are almost always separator-delimited) and will be fixed by the upcoming longest-match-wins resolver in 1.0.

data/ext/data_redactor/custom_patterns.c

@@ -3,6 +3,11 @@
 #include <string.h>
 #include <stdlib.h>
 
+/* Custom patterns deliberately do NOT use the v19 engine: they keep the glibc
+ * regexec path (replace_all_matches), because user regex can contain multibyte
+ * UTF-8 inside character classes (e.g. name_pattern's [oOòóô…]) that the
+ * byte-level v19 engine cannot match the way glibc's locale-aware matcher does.
+ * See TODO.md §1d Gap 2 for the hybrid-split rationale. */
 custom_pattern_t *custom_patterns = NULL;
 int custom_count = 0;
 int custom_cap   = 0;

data/ext/data_redactor/data_redactor.c

@@ -7,8 +7,12 @@
 #include "redact.h"
 #include "scan.h"
 #include "custom_patterns.h"
+#include "matcher.h"
 
 void Init_data_redactor(void) {
+    /* Build the v19 multi-pattern engines from the gem's pattern arrays. */
+    mm_init();
+
     /* Compile all built-in regex patterns at load time. */
     for (int i = 0; i < NUM_PATTERNS; i++) {
         const char *pat;
@@ -43,15 +47,25 @@ void Init_data_redactor(void) {
     rb_define_module_function(mDataRedactor, "_clear_custom_patterns", rb_clear_custom_patterns,   0);
     rb_define_module_function(mDataRedactor, "_custom_patterns",       rb_custom_patterns,         0);
 
-    /* Frozen array of built-in pattern names, for introspection and only:/except: validation. */
+    /* Frozen introspection arrays, parallel to the pattern table (same index = same
+     * pattern). NAMES/TAG_BITS back only:/except: validation; SOURCES/BOUNDARY are
+     * internal aids (the benchmark suite rebuilds the patterns in pure Ruby from
+     * them). SOURCES holds the unwrapped POSIX ERE — boundary wrapping is applied
+     * above at compile time for patterns with BOUNDARY[i] == true. */
     VALUE builtin_names = rb_ary_new_capa(NUM_PATTERNS);
     VALUE builtin_tag_bits = rb_ary_new_capa(NUM_PATTERNS);
+    VALUE builtin_sources = rb_ary_new_capa(NUM_PATTERNS);
+    VALUE builtin_boundary = rb_ary_new_capa(NUM_PATTERNS);
     for (int i = 0; i < NUM_PATTERNS; i++) {
         rb_ary_push(builtin_names, rb_str_new_frozen(rb_str_new_cstr(pattern_names[i])));
         rb_ary_push(builtin_tag_bits, INT2NUM(pattern_tags[i]));
+        rb_ary_push(builtin_sources, rb_str_new_frozen(rb_str_new_cstr(pattern_strings[i])));
+        rb_ary_push(builtin_boundary, boundary_wrapped[i] ? Qtrue : Qfalse);
     }
     rb_define_const(mDataRedactor, "BUILTIN_PATTERN_NAMES",    rb_ary_freeze(builtin_names));
     rb_define_const(mDataRedactor, "BUILTIN_PATTERN_TAG_BITS", rb_ary_freeze(builtin_tag_bits));
+    rb_define_const(mDataRedactor, "BUILTIN_PATTERN_SOURCES",  rb_ary_freeze(builtin_sources));
+    rb_define_const(mDataRedactor, "BUILTIN_PATTERN_BOUNDARY", rb_ary_freeze(builtin_boundary));
 
     /* Placeholder mode constants. */
     rb_define_const(mDataRedactor, "PH_MODE_PLAIN",  INT2NUM(PLACEHOLDER_MODE_PLAIN));