data_redactor 0.11.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bde76b717309c4a58b5fbe2cd85c7986622291a9e8f6a7c1e7423ae804ef0246
-  data.tar.gz: 497e5edb87745db889ab2900ef9a275335357311b747327d036d91a687008b94
+  metadata.gz: fa978ec8daa0c8285f48283bc251a512c9263202046100ac03ad39ef4889e070
+  data.tar.gz: 6b5ff39107b5948bcf499fa122e2af4c342d46fb520e94c6d0d3d1f128f30781
 SHA512:
-  metadata.gz: 4bc80d118f450b5798d8c7932fe39e1513c897842e40bf370f0d8b66d37ab180a3c4aae52ee1c85628a81a57c5612d67b969cd060b0023ab5d549be14c128bf7
-  data.tar.gz: af61f49db51ddc2d0806cb392700c93c4fde3a869e9e089f615ae26acabb6b777e38792d0fb061c53e5a01bc69355d8b95a9c5f60946e3a393df3f2ef958f066
+  metadata.gz: 4169cf320312e05e77d5c6fe699f1516f031f6ba097fb9a327e9c7686c462d0c96759b58f03eb2aab024178e01998f6c7607f2ac758eb773e6681b68f8b0717e
+  data.tar.gz: 92a9d114b28305d2da038571259ff3d819371e1276aa66aea436d820e4e540084a0e0c2e555013e53fc454abe50029058e9ca7fcdc239591b86685700cf29d62

data/CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.13.0] - 2026-06-13
+
+### Changed
+- **Custom-pattern registration is now thread-safe.** `add_pattern`,
+  `remove_pattern`, and `clear_custom_patterns!` are guarded by a mutex shared
+  with the `redact`/`scan` custom-pattern loop, so patterns may be registered,
+  removed, or cleared from any thread at any time — including at runtime from a
+  request handler — without coordinating with in-flight redactions. The previous
+  "register custom patterns at boot only" caveat is lifted. (The C extension now
+  links `-lpthread` on glibc; no-op on musl and macOS where pthread is in libc.)
+- **`redact` releases the GVL for large inputs.** The v19 engine's per-scan
+  mutable state (NFA scratch and the lazy DFA cache) moved into per-thread
+  storage, making the engine re-entrant. `redact` now releases the GVL
+  (`rb_thread_call_without_gvl`) around the built-in scan for inputs above a few
+  KB, so a large redaction on one thread no longer blocks other Ruby threads.
+  Small inputs keep the GVL. No public API change; output is byte-for-byte
+  identical (verified by a differential gate over ~6000 inputs). The per-thread
+  DFA cache's allocation floor was tuned so this adds ~0.86 MB per scanning
+  thread (down from a naive ~3.2 MB), with no throughput change. Per-thread scan
+  state is freed at thread exit (via a `pthread_key` destructor), so processes
+  that churn many short-lived scanning threads do not accumulate dead caches —
+  RSS stays flat across thousands of threads.
+
 ## [0.11.0] - 2026-06-10
 
 ### Added
@@ -232,7 +255,8 @@ features as 0.7.1 plus the pipeline fix.
 - `DataRedactor.redact(text)` module function returning the input with every match replaced by `[REDACTED]`.
 - RSpec suite with one example per pattern.
 
-[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.11.0...HEAD
+[Unreleased]: https://github.com/danielefrisanco/data_redactor/compare/v0.13.0...HEAD
+[0.13.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.11.0...v0.13.0
 [0.11.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.10.1...v0.11.0
 [0.10.1]: https://github.com/danielefrisanco/data_redactor/compare/v0.10.0...v0.10.1
 [0.10.0]: https://github.com/danielefrisanco/data_redactor/compare/v0.9.0...v0.10.0

data/README.md

@@ -19,7 +19,7 @@ It ships **88 built-in patterns** across 15+ countries, grouped into tags
 (`:credentials`, `:financial`, `:contact`, ...) so you can redact only what you
 care about. Beyond plain strings it can walk nested Hashes, Arrays, and JSON,
 audit a payload without mutating it (`scan`), and plug into Logger, Rails, and
-Rack. You can also register your own patterns at boot.
+Rack. You can also register your own patterns — at boot or at runtime from any thread.
 
 ### Use cases
 
@@ -161,7 +161,7 @@ DataRedactor.redact_json("not json")  # => JSON::ParserError
 
 ### Custom patterns
 
-Teams often have internal IDs that the gem can't ship. Register them at boot:
+Teams often have internal IDs that the gem can't ship. Register them at boot — or at runtime from any thread (registration is thread-safe, see [Thread safety](#thread-safety)):
 
 ```ruby
 # String (POSIX ERE) or Regexp — both accepted
@@ -571,9 +571,9 @@ All C-side buffers are heap-allocated with `malloc`/`strdup` and freed before th
 
 ## Thread safety
 
-`DataRedactor.redact` and `DataRedactor.scan` are safe to call concurrently from multiple threads. The v19 engine holds MRI's GVL for the duration of each call (no `rb_thread_call_without_gvl`), so concurrent calls are serialised by the GVL. Each call allocates its own working buffers; built-in engine state is read-only after `mm_init()` at load time.
+`DataRedactor.redact` and `DataRedactor.scan` are safe to call concurrently from multiple threads. The v19 engine keeps its compiled patterns immutable and shared (read-only after `mm_init()` at load time) and all per-scan mutable state — NFA scratch and the lazy DFA cache — in per-thread storage, so concurrent scans never touch each other's state. For inputs above a few KB, `redact` **releases the GVL** (`rb_thread_call_without_gvl`) around the built-in scan, so a large redaction on one thread no longer blocks other Ruby threads from running. Small inputs keep the GVL (the release bookkeeping would cost more than the scan). Each call allocates its own working buffers. A thread's per-thread state is freed automatically when the thread exits, so processes that spawn many short-lived scanning threads do not accumulate memory.
 
-`DataRedactor.add_pattern`, `remove_pattern`, and `clear_custom_patterns!` mutate a shared dynamic array and are **not** thread-safe. Register custom patterns once at boot — before spawning worker threads or forking — and they will be visible (read-only) to every subsequent `redact`/`scan` call.
+`DataRedactor.add_pattern`, `remove_pattern`, and `clear_custom_patterns!` are also thread-safe: the shared custom-pattern array is guarded by a mutex that writers take around the mutation and `redact`/`scan` take around their custom-pattern loop. You can register, remove, or clear custom patterns from any thread at any time — including from request handlers in a running server — without coordinating with in-flight redactions. (Registration is still a rare operation; the lock is uncontended in practice.)
 
 ## Versioning
 

data/ext/data_redactor/custom_patterns.c

@@ -2,6 +2,7 @@
 #include "redact.h" /* wrap_boundary */
 #include <string.h>
 #include <stdlib.h>
+#include <pthread.h>
 
 /* Custom patterns deliberately do NOT use the v19 engine: they keep the glibc
  * regexec path (replace_all_matches), because user regex can contain multibyte
@@ -12,6 +13,11 @@ custom_pattern_t *custom_patterns = NULL;
 int custom_count = 0;
 int custom_cap   = 0;
 
+static pthread_mutex_t custom_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+void custom_patterns_lock(void)   { pthread_mutex_lock(&custom_mutex); }
+void custom_patterns_unlock(void) { pthread_mutex_unlock(&custom_mutex); }
+
 static int find_custom_by_name(const char *name) {
     for (int i = 0; i < custom_count; i++) {
         if (strcmp(custom_patterns[i].name, name) == 0) return i;
@@ -58,6 +64,13 @@ VALUE rb_add_pattern(VALUE self, VALUE rb_name, VALUE rb_source,
         rb_raise(eClass, "%s", errbuf);
     }
 
+    /* regcomp succeeded above (no array access yet); now mutate the shared array
+     * under the lock. Keep the critical section rb_raise-free: on failure, record
+     * it, unlock, then raise outside the lock so the mutex can't leak via longjmp. */
+    custom_patterns_lock();
+
+    const char *err = NULL;
+    int stored = 0; /* 1 once `compiled` is owned by a slot (don't regfree it) */
     int idx = find_custom_by_name(name);
     if (idx >= 0) {
         free_custom_at(idx);
@@ -67,8 +80,8 @@ VALUE rb_add_pattern(VALUE self, VALUE rb_name, VALUE rb_source,
             custom_pattern_t *tmp = (custom_pattern_t *)realloc(
                 custom_patterns, sizeof(custom_pattern_t) * new_cap);
             if (!tmp) {
-                regfree(&compiled);
-                rb_raise(rb_eNoMemError, "custom_patterns realloc failed");
+                err = "custom_patterns realloc failed";
+                goto unlock;
             }
             custom_patterns = tmp;
             custom_cap = new_cap;
@@ -81,9 +94,17 @@ VALUE rb_add_pattern(VALUE self, VALUE rb_name, VALUE rb_source,
     custom_patterns[idx].compiled = compiled;
     custom_patterns[idx].tag      = tag_bit;
     custom_patterns[idx].boundary = boundary;
+    stored = 1;
 
     if (!custom_patterns[idx].name || !custom_patterns[idx].source) {
-        rb_raise(rb_eNoMemError, "strdup failed");
+        err = "strdup failed";
+    }
+
+unlock:
+    custom_patterns_unlock();
+    if (err) {
+        if (!stored) regfree(&compiled);
+        rb_raise(rb_eNoMemError, "%s", err);
     }
 
     return Qnil;
@@ -93,8 +114,12 @@ VALUE rb_remove_pattern(VALUE self, VALUE rb_name) {
     Check_Type(rb_name, T_STRING);
     const char *name = StringValueCStr(rb_name);
 
+    custom_patterns_lock();
     int idx = find_custom_by_name(name);
-    if (idx < 0) return Qfalse;
+    if (idx < 0) {
+        custom_patterns_unlock();
+        return Qfalse;
+    }
 
     free_custom_at(idx);
 
@@ -102,19 +127,23 @@ VALUE rb_remove_pattern(VALUE self, VALUE rb_name) {
         custom_patterns[i] = custom_patterns[i + 1];
     }
     custom_count--;
+    custom_patterns_unlock();
 
     return Qtrue;
 }
 
 VALUE rb_clear_custom_patterns(VALUE self) {
+    custom_patterns_lock();
     for (int i = 0; i < custom_count; i++) {
         free_custom_at(i);
     }
     custom_count = 0;
+    custom_patterns_unlock();
     return Qnil;
 }
 
 VALUE rb_custom_patterns(VALUE self) {
+    custom_patterns_lock();
     VALUE arr = rb_ary_new_capa(custom_count);
     for (int i = 0; i < custom_count; i++) {
         VALUE h = rb_hash_new();
@@ -124,5 +153,6 @@ VALUE rb_custom_patterns(VALUE self) {
         rb_hash_aset(h, ID2SYM(rb_intern("boundary")), custom_patterns[i].boundary ? Qtrue : Qfalse);
         rb_ary_push(arr, h);
     }
+    custom_patterns_unlock();
     return arr;
 }

data/ext/data_redactor/custom_patterns.h

@@ -16,6 +16,16 @@ extern custom_pattern_t *custom_patterns;
 extern int custom_count;
 extern int custom_cap;
 
+/* Guards the custom_patterns array against concurrent mutation. redact/scan
+ * take it for the duration of their custom-pattern loop (readers); add/remove/
+ * clear take it around the array mutation (writers). A plain mutex is enough:
+ * contention is low (registration is rare relative to redaction) and the GVL
+ * already serialises everything else, so the only race this closes is a writer
+ * realloc/shift running against a reader's iteration. Lock is always taken
+ * inside the GVL, never the reverse, so there is no lock-ordering hazard. */
+void custom_patterns_lock(void);
+void custom_patterns_unlock(void);
+
 VALUE rb_add_pattern(VALUE self, VALUE rb_name, VALUE rb_source,
                      VALUE rb_tag_bit, VALUE rb_boundary);
 VALUE rb_remove_pattern(VALUE self, VALUE rb_name);

data/ext/data_redactor/extconf.rb

@@ -4,6 +4,11 @@ abort "Missing C compiler or stdio.h" unless have_header("stdio.h")
 abort "Missing regex.h"               unless have_header("regex.h")
 abort "Missing stdlib.h"              unless have_header("stdlib.h")
 abort "Missing string.h"              unless have_header("string.h")
+abort "Missing pthread.h"             unless have_header("pthread.h")
+
+# pthread_mutex_* needs -lpthread on glibc; on musl and macOS it lives in libc
+# and have_library is a harmless no-op.
+have_library("pthread")
 
 # Compile every .c file in this directory. Order doesn't matter; mkmf
 # generates per-object rules.

data/ext/data_redactor/matcher.c

@@ -44,6 +44,7 @@
 #include <stdint.h>
 #include <ctype.h>
 #include <limits.h>
+#include <pthread.h>
 
 /* ========================================================================
  * 0. Utilities
@@ -386,8 +387,13 @@ typedef struct {
     int       matched;
 } tlist_t;
 
+/* engine_t holds ONLY immutable, compiled state — built once at mm_init()/mm_add()
+ * and never written during a scan, so it is safe to share read-only across
+ * threads. All per-scan mutable state (NFA scratch, merge cursors) and the lazy
+ * DFA cache live in scan_state_t, which is per-thread (t_block below). This
+ * split is what lets redact/scan release the GVL: with no shared writes during a
+ * scan, concurrent scans on distinct threads cannot race. */
 typedef struct {
-    /* compiled, immutable after build (safe to share across scans) */
     prog_t      prog;
     size_t      min_len;
     const char *req_literal;    /* points into a heap copy owned by this engine */
@@ -405,17 +411,21 @@ typedef struct {
     /* selective-merge membership (built-ins only; customs never join a merge) */
     int         digit_member, digit_lo, digit_hi;
     int         iban_member;
-    /* lazy DFA + per-scan scratch (mutable; GVL-guarded in Phase 1) */
+} engine_t;
+
+/* Per-engine MUTABLE scan state. One per engine, owned per-thread (t_state).
+ * The DFA cache warms lazily across this thread's scans; the rest is reset each
+ * scan. seen_cap==0 / dfa.n_states==0 means "not yet warmed" for this thread. */
+typedef struct {
     dfa_t       dfa;
     int        *seen;
     int         seen_cap;
     tlist_t     clist, nlist;
     int        *estack;
     int         gen;
-    /* per-scan non-overlapping cursors used by the selective-merge passes */
-    int         digit_last_end;
+    int         digit_last_end;   /* selective-merge non-overlap cursors */
     size_t      iban_last_end;
-} engine_t;
+} scan_state_t;
 
 static engine_t *g_eng    = NULL;
 static int       g_eng_n  = 0;    /* engines built (NUM_PATTERNS + custom_n) */
@@ -423,6 +433,33 @@ static int       g_eng_cap= 0;
 static int       g_custom_n = 0;
 static int       g_initialized = 0;
 
+/* Bumped whenever the pattern set changes (mm_add/mm_remove/mm_clear_custom).
+ * A thread whose cached t_gen lags this value drops its whole scan-state cache
+ * and rebuilds — the simplest safe invalidation (slot p may now hold a
+ * different pattern after mm_remove compacts g_eng). Registration is rare, so
+ * the full rebuild is cheap; a surgical per-slot invalidation is a possible
+ * future refinement (see TODO §"Full thread safety"). */
+static unsigned g_pattern_gen = 0;
+
+/* Per-thread mutable scan state: one scan_state_t per engine, lazily grown to
+ * g_eng_n. Held in a heap block whose header carries the element count, so the
+ * pthread_key destructor (which frees the block at thread exit) is fully
+ * self-contained — it must NOT read __thread storage, which may already be torn
+ * down when key destructors run. The __thread pointer is the fast hot-path
+ * handle; the key holds the same pointer purely so it can be reclaimed on exit.
+ * This bounds memory for processes that churn many short-lived scanning threads;
+ * fixed pools (Puma/Sidekiq) just reuse the block for the thread's lifetime. */
+typedef struct {
+    int          n;          /* number of scan_state_t entries in states[] */
+    scan_state_t states[];   /* flexible array member */
+} thread_block_t;
+
+static __thread thread_block_t *t_block = NULL;
+static __thread unsigned        t_gen   = 0;
+
+static pthread_key_t  t_block_key;
+static pthread_once_t t_block_key_once = PTHREAD_ONCE_INIT;
+
 /* IBAN union-pass dispatch (built-ins only): unique 2-byte country prefixes. */
 static int g_iban_first[256];
 static int g_iban_pair[256][256];
@@ -606,14 +643,64 @@ static void engine_set_literal(engine_t *eng, const char *lit, int at_start) {
 static void engine_free(engine_t *eng) {
     free(eng->prog.code);
     free(eng->req_literal_own);
-    free(eng->seen);
-    free(eng->clist.list);
-    free(eng->nlist.list);
-    free(eng->estack);
-    dfa_t *d = &eng->dfa;
+    memset(eng, 0, sizeof(*eng));
+}
+
+/* Free one thread's mutable scan state for an engine (scratch + DFA cache).
+ * Used when a thread drops its cache on a pattern-set generation change. */
+static void free_scan_state(scan_state_t *st) {
+    free(st->seen);
+    free(st->clist.list);
+    free(st->nlist.list);
+    free(st->estack);
+    dfa_t *d = &st->dfa;
     free(d->set_pool); free(d->set_off); free(d->set_len);
     free(d->matched);  free(d->trans);   free(d->hash);
-    memset(eng, 0, sizeof(*eng));
+    memset(st, 0, sizeof(*st));
+}
+
+/* pthread_key destructor: free a thread's whole block at thread exit. Reads only
+ * the passed-in pointer + its header count — no __thread access (unsafe here). */
+static void free_thread_block(void *p) {
+    thread_block_t *b = (thread_block_t *)p;
+    if (!b) return;
+    for (int i = 0; i < b->n; i++) free_scan_state(&b->states[i]);
+    free(b);
+}
+
+static void make_t_block_key(void) {
+    if (pthread_key_create(&t_block_key, free_thread_block) != 0) {
+        perror("pthread_key_create"); exit(1);
+    }
+}
+
+/* Return this thread's scan_state_t array, synced to the current pattern set.
+ * Drops the whole cache if the pattern set changed (generation guard), then
+ * lazily grows (zero-initialised) to cover every engine. Called under the
+ * custom-pattern mutex during a scan, so g_pattern_gen / g_eng_n are stable.
+ * The owning block is registered with t_block_key so it is freed at thread exit;
+ * the key value is re-set after any (re)allocation since the block may move. */
+static scan_state_t *thread_state(void) {
+    pthread_once(&t_block_key_once, make_t_block_key);
+
+    if (t_gen != g_pattern_gen) {
+        free_thread_block(t_block);
+        t_block = NULL;
+        pthread_setspecific(t_block_key, NULL);
+        t_gen = g_pattern_gen;
+    }
+    int have = t_block ? t_block->n : 0;
+    if (have < g_eng_n) {
+        thread_block_t *nb = realloc(t_block,
+            sizeof(thread_block_t) + (size_t)g_eng_n * sizeof(scan_state_t));
+        if (!nb) { perror("realloc"); exit(1); }
+        memset(&nb->states[have], 0,
+               (size_t)(g_eng_n - have) * sizeof(scan_state_t));
+        nb->n = g_eng_n;
+        t_block = nb;
+        pthread_setspecific(t_block_key, nb);
+    }
+    return t_block->states;
 }
 
 /* ========================================================================
@@ -692,7 +779,13 @@ static void dfa_hash_insert(dfa_t *d, int sid);
 
 static void dfa_grow_states(dfa_t *d) {
     if (d->n_states < d->states_cap) return;
-    int newcap = d->states_cap ? d->states_cap * 2 : 64;
+    /* Start small (8) and double. Each state owns a 1 KB transition row, and the
+     * DFA cache is now per-thread, so the initial cap is the per-thread memory
+     * floor multiplied across every engine. Most patterns settle at 1-14 states
+     * (max 45), so a floor of 8 fits the common case in 8 KB instead of 64 KB
+     * (~4x less per-thread memory across 79 DFA engines); the few larger DFAs
+     * just do a couple extra doublings during warmup, off the hot path. */
+    int newcap = d->states_cap ? d->states_cap * 2 : 8;
     d->set_off = realloc(d->set_off, (size_t)newcap * sizeof(int));
     d->set_len = realloc(d->set_len, (size_t)newcap * sizeof(int));
     d->matched = realloc(d->matched, (size_t)newcap * sizeof(int));
@@ -748,28 +841,28 @@ static int dfa_intern(dfa_t *d, const int *set, int n, int matched) {
     return sid;
 }
 
-static void ensure_scratch(engine_t *eng) {
+static void ensure_scratch(engine_t *eng, scan_state_t *st) {
     prog_t *pr = &eng->prog;
-    if (eng->seen_cap >= pr->n) return;
-    eng->seen       = realloc(eng->seen,       pr->n * sizeof(int));
-    eng->clist.list = realloc(eng->clist.list, pr->n * sizeof(int));
-    eng->nlist.list = realloc(eng->nlist.list, pr->n * sizeof(int));
-    eng->estack     = realloc(eng->estack,     (2 * pr->n + 1) * sizeof(int));
-    if (!eng->seen || !eng->clist.list || !eng->nlist.list || !eng->estack) {
+    if (st->seen_cap >= pr->n) return;
+    st->seen       = realloc(st->seen,       pr->n * sizeof(int));
+    st->clist.list = realloc(st->clist.list, pr->n * sizeof(int));
+    st->nlist.list = realloc(st->nlist.list, pr->n * sizeof(int));
+    st->estack     = realloc(st->estack,     (2 * pr->n + 1) * sizeof(int));
+    if (!st->seen || !st->clist.list || !st->nlist.list || !st->estack) {
         perror("realloc"); exit(1);
     }
-    memset(eng->seen, 0, pr->n * sizeof(int));
-    eng->seen_cap = pr->n;
+    memset(st->seen, 0, pr->n * sizeof(int));
+    st->seen_cap = pr->n;
 }
 
-static int dfa_compute_trans(engine_t *eng, int sid, unsigned char c) {
+static int dfa_compute_trans(engine_t *eng, scan_state_t *st, int sid, unsigned char c) {
     prog_t  *pr  = &eng->prog;
-    dfa_t   *d   = &eng->dfa;
-    int     *seen = eng->seen;
-    int     *estk = eng->estack;
-    tlist_t *nl   = &eng->nlist;
+    dfa_t   *d   = &st->dfa;
+    int     *seen = st->seen;
+    int     *estk = st->estack;
+    tlist_t *nl   = &st->nlist;
 
-    int gen = ++eng->gen;
+    int gen = ++st->gen;
     nl->n = 0; nl->matched = 0;
 
     const int *set = &d->set_pool[d->set_off[sid]];
@@ -800,14 +893,14 @@ static int dfa_compute_trans(engine_t *eng, int sid, unsigned char c) {
     return next;
 }
 
-static void dfa_build_start(engine_t *eng) {
+static void dfa_build_start(engine_t *eng, scan_state_t *st) {
     prog_t  *pr  = &eng->prog;
-    dfa_t   *d   = &eng->dfa;
-    int     *seen = eng->seen;
-    int     *estk = eng->estack;
-    tlist_t *cl   = &eng->clist;
+    dfa_t   *d   = &st->dfa;
+    int     *seen = st->seen;
+    int     *estk = st->estack;
+    tlist_t *cl   = &st->clist;
 
-    int gen = ++eng->gen;
+    int gen = ++st->gen;
     cl->n = 0; cl->matched = 0;
     addthread_dfa(pr, cl, seen, gen, estk, 0);
     qsort(cl->list, (size_t)cl->n, sizeof(int), int_cmp);
@@ -821,23 +914,24 @@ static void dfa_build_start(engine_t *eng) {
  * 9. Per-pattern scan (scan_one) — identical logic to the prototype
  * ======================================================================== */
 
-static size_t scan_one(int p, const char *input, size_t len,
+static size_t scan_one(int p, scan_state_t *state, const char *input, size_t len,
                        mm_match_t *out, size_t max, size_t count) {
-    engine_t *eng = &g_eng[p];
-    prog_t   *pr  = &eng->prog;
+    engine_t     *eng = &g_eng[p];
+    scan_state_t *sst = &state[p];
+    prog_t       *pr  = &eng->prog;
 
-    ensure_scratch(eng);
-    int     *seen = eng->seen;
-    int     *estk = eng->estack;
-    tlist_t *cl   = &eng->clist, *nl = &eng->nlist;
+    ensure_scratch(eng, sst);
+    int     *seen = sst->seen;
+    int     *estk = sst->estack;
+    tlist_t *cl   = &sst->clist, *nl = &sst->nlist;
 
-    if (eng->gen > INT_MAX - (int)(2 * (len + 2))) {
+    if (sst->gen > INT_MAX - (int)(2 * (len + 2))) {
         memset(seen, 0, pr->n * sizeof(int));
-        eng->gen = 0;
+        sst->gen = 0;
     }
 
-    dfa_t *d = &eng->dfa;
-    if (eng->use_dfa && d->n_states == 0) dfa_build_start(eng);
+    dfa_t *d = &sst->dfa;
+    if (eng->use_dfa && d->n_states == 0) dfa_build_start(eng, sst);
 
     size_t pos = 0;
     while (pos <= len) {
@@ -874,19 +968,19 @@ static size_t scan_one(int p, const char *input, size_t len,
                 if (sp == len) break;
                 int next = d->trans[st * 256 + (unsigned char)input[sp]];
                 if (next == TRANS_UNFILLED)
-                    next = dfa_compute_trans(eng, st, (unsigned char)input[sp]);
+                    next = dfa_compute_trans(eng, sst, st, (unsigned char)input[sp]);
                 st = next;
                 sp++;
             }
         } else {
-            int gen = ++eng->gen;
+            int gen = ++sst->gen;
             cl->n = 0; cl->matched = 0;
             addthread(pr, cl, seen, gen, estk, 0, input, len, pos);
             while (cl->n > 0 || cl->matched) {
                 if (cl->matched && sp - pos >= eng->min_len) match_end = sp;
                 if (cl->n == 0 || sp == len) break;
                 unsigned char c = (unsigned char)input[sp];
-                gen = ++eng->gen;
+                gen = ++sst->gen;
                 nl->n = 0; nl->matched = 0;
                 for (int i = 0; i < cl->n; i++) {
                     inst_t *in = &pr->code[cl->list[i]];
@@ -935,11 +1029,11 @@ static size_t scan_one(int p, const char *input, size_t len,
  * 10. Selective merges (digit run pass + IBAN union pass)
  * ======================================================================== */
 
-static size_t scan_digit_group(const char *input, size_t len,
+static size_t scan_digit_group(scan_state_t *state, const char *input, size_t len,
                                const int *enable_bits, size_t n_bits,
                                mm_match_t *out, size_t max, size_t count) {
     for (int p = 0; p < g_eng_n; p++)
-        if (g_eng[p].digit_member) g_eng[p].digit_last_end = 0;
+        if (g_eng[p].digit_member) state[p].digit_last_end = 0;
 
     size_t i = 0;
     while (i < len) {
@@ -963,7 +1057,7 @@ static size_t scan_digit_group(const char *input, size_t len,
 
             size_t start;
             if (rs > 0 && !isalnum((unsigned char)input[rs-1]) &&
-                rs - 1 >= (size_t)eng->digit_last_end) {
+                rs - 1 >= (size_t)state[p].digit_last_end) {
                 start = rs - 1;
             } else if (rs == 0 || input[rs-1] == '\n') {
                 start = rs;
@@ -978,22 +1072,22 @@ static size_t scan_digit_group(const char *input, size_t len,
              * separator are resolved exactly as gsub would. */
             (void)start;
             out[count++] = (mm_match_t){p, rs, re - rs};
-            eng->digit_last_end = (int)end;
+            state[p].digit_last_end = (int)end;
         }
         if (count >= max) break;
     }
     return count;
 }
 
-static size_t scan_iban_group(const char *input, size_t len,
+static size_t scan_iban_group(scan_state_t *state, const char *input, size_t len,
                               const int *enable_bits, size_t n_bits,
                               mm_match_t *out, size_t max, size_t count) {
     for (int p = 0; p < g_eng_n; p++)
         if (g_eng[p].iban_member) {
-            g_eng[p].iban_last_end = 0;
+            state[p].iban_last_end = 0;
             engine_t *eng = &g_eng[p];
-            if (eng->use_dfa && eng->dfa.n_states == 0) {
-                ensure_scratch(eng); dfa_build_start(eng);
+            if (eng->use_dfa && state[p].dfa.n_states == 0) {
+                ensure_scratch(eng, &state[p]); dfa_build_start(eng, &state[p]);
             }
         }
 
@@ -1004,10 +1098,11 @@ static size_t scan_iban_group(const char *input, size_t len,
         int p = g_iban_pair[c0][(unsigned char)input[i + 1]];
         if (p < 0) { i++; continue; }
         if ((size_t)p < n_bits && !enable_bits[p]) { i++; continue; }
-        if (i < g_eng[p].iban_last_end) { i++; continue; }
+        if (i < state[p].iban_last_end) { i++; continue; }
 
-        engine_t *eng = &g_eng[p];
-        dfa_t *d = &eng->dfa;
+        engine_t     *eng = &g_eng[p];
+        scan_state_t *sst = &state[p];
+        dfa_t *d = &sst->dfa;
         size_t match_end = (size_t)-1, sp = i;
         int st = 0;
         while (st != DFA_DEAD) {
@@ -1015,7 +1110,7 @@ static size_t scan_iban_group(const char *input, size_t len,
             if (sp == len) break;
             int next = d->trans[st * 256 + (unsigned char)input[sp]];
             if (next == TRANS_UNFILLED)
-                next = dfa_compute_trans(eng, st, (unsigned char)input[sp]);
+                next = dfa_compute_trans(eng, sst, st, (unsigned char)input[sp]);
             st = next;
             sp++;
         }
@@ -1023,7 +1118,7 @@ static size_t scan_iban_group(const char *input, size_t len,
         if (match_end != (size_t)-1) {
             size_t span = match_end - i;
             out[count++] = (mm_match_t){p, i, span};
-            eng->iban_last_end = match_end;
+            sst->iban_last_end = match_end;
             i = (span == 0) ? i + 1 : match_end;
         } else {
             i++;
@@ -1089,6 +1184,7 @@ int mm_add(const char *regex, int boundary) {
     /* Custom patterns never join the selective merges (TODO §1d Gap 4): they keep
      * the per-pattern path. No digit/IBAN membership, no literal-skip hint. */
     g_custom_n++;
+    g_pattern_gen++;   /* invalidate every thread's cached scan state */
     return 0;
 }
 
@@ -1102,12 +1198,16 @@ void mm_remove(int idx) {
         g_eng[s] = g_eng[s + 1];
     g_eng_n--;
     g_custom_n--;
+    /* slot p now holds a DIFFERENT pattern (compaction), so every thread's
+     * scan-state cache indexed by p is stale — invalidate. */
+    g_pattern_gen++;
 }
 
 void mm_clear_custom(void) {
     for (int s = NUM_PATTERNS; s < g_eng_n; s++) engine_free(&g_eng[s]);
     g_eng_n = NUM_PATTERNS;
     g_custom_n = 0;
+    g_pattern_gen++;
 }
 
 /* ========================================================================
@@ -1124,18 +1224,19 @@ size_t mm_scan(const char *input, size_t len,
                const int *enable_bits, size_t n_bits,
                mm_match_t *out, size_t max) {
     if (!g_initialized) mm_init();
+    scan_state_t *state = thread_state();
     size_t count = 0;
 
     for (int p = 0; p < g_eng_n && count < max; p++) {
         if (g_eng[p].digit_member) continue;
         if (g_eng[p].iban_member)  continue;
         if (!enabled(enable_bits, n_bits, p)) continue;
-        count = scan_one(p, input, len, out, max, count);
+        count = scan_one(p, state, input, len, out, max, count);
     }
     if (g_have_iban_group && count < max)
-        count = scan_iban_group(input, len, enable_bits, n_bits, out, max, count);
+        count = scan_iban_group(state, input, len, enable_bits, n_bits, out, max, count);
     if (g_have_digit_group && count < max)
-        count = scan_digit_group(input, len, enable_bits, n_bits, out, max, count);
+        count = scan_digit_group(state, input, len, enable_bits, n_bits, out, max, count);
     return count;
 }
 

data/ext/data_redactor/redact.c

@@ -4,10 +4,19 @@
 #include "custom_patterns.h"
 #include "matcher.h"
 #include "tags.h"
+#include <ruby/thread.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
 
+/* Inputs at or above this byte size release the GVL around the built-in v19
+ * pass so other Ruby threads can run during the scan. Below it, the
+ * rb_thread_call_without_gvl bookkeeping costs more than the scan, so we keep
+ * the GVL. The Ruby layer chunks inputs > CHUNK_SIZE (64 KB) before calling
+ * _redact, so the practical ceiling per call is one chunk; 4 KB cleanly
+ * separates per-leaf/log-line calls (keep GVL) from chunk-sized work (release). */
+#define GVL_RELEASE_THRESHOLD (4 * 1024)
+
 char *wrap_boundary(const char *core) {
     const char *prefix = "(^|[^0-9A-Za-z])(";
     const char *suffix = ")([^0-9A-Za-z]|$)";
@@ -179,6 +188,41 @@ static char *redact_builtins(const char *input, size_t in_len, const int *bits,
     return output;
 }
 
+/* Trampoline for running redact_builtins() with the GVL released. Everything it
+ * touches is plain C (raw char* in/out, the per-thread engine state); no Ruby
+ * VALUE or Ruby API call happens inside, which is the contract for
+ * rb_thread_call_without_gvl. */
+typedef struct {
+    const char  *input;
+    size_t       in_len;
+    const int   *bits;
+    int          ph_mode;
+    const char  *ph_str_plain;
+    size_t       out_len;
+    char        *result;
+} builtins_args_t;
+
+static void *redact_builtins_nogvl(void *p) {
+    builtins_args_t *a = (builtins_args_t *)p;
+    a->result = redact_builtins(a->input, a->in_len, a->bits,
+                                a->ph_mode, a->ph_str_plain, &a->out_len);
+    return NULL;
+}
+
+/* Run the built-in v19 pass, releasing the GVL for inputs large enough that the
+ * scan dominates the release bookkeeping. Small inputs run inline under the GVL. */
+static char *redact_builtins_maybe_nogvl(const char *input, size_t in_len,
+                                         const int *bits, int ph_mode,
+                                         const char *ph_str_plain, size_t *out_len_p) {
+    if (in_len < GVL_RELEASE_THRESHOLD)
+        return redact_builtins(input, in_len, bits, ph_mode, ph_str_plain, out_len_p);
+
+    builtins_args_t a = { input, in_len, bits, ph_mode, ph_str_plain, 0, NULL };
+    rb_thread_call_without_gvl(redact_builtins_nogvl, &a, RUBY_UBF_IO, NULL);
+    *out_len_p = a.out_len;
+    return a.result;
+}
+
 VALUE rb_data_redactor_redact(VALUE self, VALUE rb_text,
                               VALUE rb_ph_mode, VALUE rb_ph_str,
                               VALUE rb_enable_bits) {
@@ -197,7 +241,7 @@ VALUE rb_data_redactor_redact(VALUE self, VALUE rb_text,
     int *bits = builtin_enable_bits(rb_enable_bits);
     if (!bits) rb_raise(rb_eNoMemError, "enable_bits allocation failed");
     size_t work_len = 0;
-    char *working = redact_builtins(input, in_len, bits, ph_mode, ph_str_plain, &work_len);
+    char *working = redact_builtins_maybe_nogvl(input, in_len, bits, ph_mode, ph_str_plain, &work_len);
     free(bits);
     if (!working) rb_raise(rb_eNoMemError, "built-in redaction allocation failed");
 
@@ -208,6 +252,8 @@ VALUE rb_data_redactor_redact(VALUE self, VALUE rb_text,
      * incidentally beyond what today already did. */
     placeholder_t ph;
     ph.mode = ph_mode;
+    custom_patterns_lock();
+    int oom = 0;
     for (int i = 0; i < custom_count; i++) {
         if (!enable_bit(rb_enable_bits, NUM_PATTERNS + i)) continue;
         ph.str = (ph_mode == PLACEHOLDER_MODE_PLAIN)
@@ -216,9 +262,11 @@ VALUE rb_data_redactor_redact(VALUE self, VALUE rb_text,
         char *result = replace_all_matches(&custom_patterns[i].compiled, working,
                                            custom_patterns[i].boundary, &ph);
         free(working);
-        if (!result) rb_raise(rb_eNoMemError, "replace_all_matches allocation failed (custom)");
+        if (!result) { working = NULL; oom = 1; break; }
         working = result;
     }
+    custom_patterns_unlock();
+    if (oom) rb_raise(rb_eNoMemError, "replace_all_matches allocation failed (custom)");
 
     VALUE rb_result = rb_str_new_cstr(working);
     free(working);

data/ext/data_redactor/scan.c

@@ -126,6 +126,8 @@ VALUE rb_data_redactor_scan(VALUE self, VALUE rb_text, VALUE rb_enable_bits) {
     /* Stage 2: custom patterns via glibc on the rewritten buffer.         */
     /* Original coords recovered via working_to_orig() using ev[].         */
     /* ------------------------------------------------------------------ */
+    custom_patterns_lock();
+    int oom = 0;
     for (int i = 0; i < custom_count; i++) {
         if (!scan_enable_bit(rb_enable_bits, NUM_PATTERNS + i)) continue;
 
@@ -163,9 +165,11 @@ VALUE rb_data_redactor_scan(VALUE self, VALUE rb_text, VALUE rb_enable_bits) {
         char *next = replace_all_matches(&custom_patterns[i].compiled, working,
                                          custom_patterns[i].boundary, &ph_plain);
         free(working);
-        if (!next) { free(ev); rb_raise(rb_eNoMemError, "replace_all_matches failed in scan"); }
+        if (!next) { working = NULL; oom = 1; break; }
         working = next;
     }
+    custom_patterns_unlock();
+    if (oom) { free(ev); rb_raise(rb_eNoMemError, "replace_all_matches failed in scan"); }
 
     free(ev);
 

data/lib/data_redactor/version.rb

@@ -1,4 +1,4 @@
 module DataRedactor
   # Current gem version. Follows {https://semver.org Semantic Versioning 2.0.0}.
-  VERSION = "0.11.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_redactor
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Daniele Frisanco