data_redactor 0.9.0 → 0.10.1

data/ext/data_redactor/scan.c

@@ -3,22 +3,12 @@
 #include "placeholder.h"
 #include "custom_patterns.h"
 #include "redact.h"
+#include "matcher.h"
+#include "tags.h"
 #include <regex.h>
 #include <string.h>
 #include <stdlib.h>
 
-/*
- * To map working-buffer positions back to original-string positions we
- * maintain a log of every replacement already applied. Each entry records
- * where in the *working* buffer the replacement started (after all prior
- * replacements) and how many bytes were removed (orig_len) vs. inserted
- * (always 10, the length of "[REDACTED]").
- *
- * For a new match at working position W:
- *   cumulative_shift_before_W = sum of (10 - orig_len) for all prior
- *                               replacements whose working_pos <= W
- *   original_pos = W - cumulative_shift_before_W
- */
 /* Look up the i-th entry of the enable_bits Array. Out-of-bounds → 0 (skip). */
 static inline int scan_enable_bit(VALUE rb_enable_bits, long i) {
     if (i < 0 || i >= RARRAY_LEN(rb_enable_bits)) return 0;
@@ -26,105 +16,164 @@ static inline int scan_enable_bit(VALUE rb_enable_bits, long i) {
     return RTEST(v) && NUM2INT(v) != 0;
 }
 
+/*
+ * Map a working-buffer position (after built-in redaction) back to the
+ * original-input position.
+ *
+ * After the built-in pass, the working buffer contains the original input
+ * with each matched CORE span replaced by "[REDACTED]" (10 bytes). The
+ * ev[] array (sorted by start, non-overlapping) records every replacement
+ * in original-frame coordinates. Walking ev[] we can find which verbatim
+ * segment or replacement a working position falls in and recover the
+ * original position.
+ *
+ * For a match that lands inside a "[REDACTED]" span we return the start of
+ * the corresponding original CORE (can only happen if a custom pattern
+ * matches the literal "[REDACTED]" itself, which is a degenerate case).
+ */
+static long working_to_orig(long wpos, const mm_match_t *ev, size_t n,
+                             size_t ph_len) {
+    long cum_orig = 0;
+    long cum_work = 0;
+    for (size_t i = 0; i < n; i++) {
+        long seg = (long)ev[i].start - cum_orig;
+        if (wpos < cum_work + seg)
+            return cum_orig + (wpos - cum_work);
+        cum_orig += seg + (long)ev[i].length;
+        cum_work += seg + (long)ph_len;
+    }
+    return cum_orig + (wpos - cum_work);
+}
+
 VALUE rb_data_redactor_scan(VALUE self, VALUE rb_text, VALUE rb_enable_bits) {
     Check_Type(rb_text,        T_STRING);
     Check_Type(rb_enable_bits, T_ARRAY);
 
-    const char *input = StringValueCStr(rb_text);
+    const char *input  = RSTRING_PTR(rb_text);
+    size_t      in_len = (size_t)RSTRING_LEN(rb_text);
+
+    static const placeholder_t ph_plain = { PLACEHOLDER_MODE_PLAIN, "[REDACTED]" };
+
+    /* ------------------------------------------------------------------ */
+    /* Stage 1: built-ins through v19 (original-frame coords, no rewrite  */
+    /* coordinate mapping needed).                                         */
+    /* ------------------------------------------------------------------ */
 
-    static const placeholder_t ph_default = { PLACEHOLDER_MODE_PLAIN, "[REDACTED]" };
+    /* Build enable-bits array for built-ins. */
+    int *bits = (int *)malloc((size_t)NUM_PATTERNS * sizeof(int));
+    if (!bits) rb_raise(rb_eNoMemError, "enable_bits allocation failed");
+    long alen = RARRAY_LEN(rb_enable_bits);
+    for (int i = 0; i < NUM_PATTERNS; i++) {
+        if (i < alen) {
+            VALUE v = rb_ary_entry(rb_enable_bits, i);
+            bits[i] = (RTEST(v) && NUM2INT(v) != 0) ? 1 : 0;
+        } else {
+            bits[i] = 0;
+        }
+    }
 
-    char *working = strdup(input);
-    if (!working) rb_raise(rb_eNoMemError, "strdup failed");
+    /* Scan + resolve, growing buffer if needed. */
+    size_t cap = in_len / 4 + 16;
+    mm_match_t *ev = NULL;
+    size_t n_ev;
+    for (;;) {
+        mm_match_t *grown = (mm_match_t *)realloc(ev, cap * sizeof(mm_match_t));
+        if (!grown) { free(ev); free(bits); rb_raise(rb_eNoMemError, "mm_scan alloc"); }
+        ev = grown;
+        n_ev = mm_scan(input, in_len, bits, (size_t)NUM_PATTERNS, ev, cap);
+        if (n_ev < cap) break;
+        cap *= 2;
+    }
+    free(bits);
+    n_ev = mm_resolve(ev, n_ev);
 
+    /* Collect built-in match hashes. */
     VALUE matches_arr = rb_ary_new();
+    for (size_t i = 0; i < n_ev; i++) {
+        int   pid = ev[i].pattern_id;
+        VALUE h   = rb_hash_new();
+        rb_hash_aset(h, ID2SYM(rb_intern("tag")),
+                     ID2SYM(rb_intern(tag_name_for_bit(pattern_tags[pid]))));
+        rb_hash_aset(h, ID2SYM(rb_intern("name")),
+                     rb_str_new_cstr(pattern_names[pid]));
+        rb_hash_aset(h, ID2SYM(rb_intern("value")),
+                     rb_str_new(input + ev[i].start, ev[i].length));
+        rb_hash_aset(h, ID2SYM(rb_intern("start")),
+                     LONG2NUM((long)ev[i].start));
+        rb_hash_aset(h, ID2SYM(rb_intern("length")),
+                     LONG2NUM((long)ev[i].length));
+        rb_ary_push(matches_arr, h);
+    }
 
-    typedef struct { long wpos; long orig_len; } repl_t;
-    repl_t *repl_log = NULL;
-    int     repl_count = 0;
-    int     repl_cap   = 0;
-
-    #define REPL_LOG_PUSH(_wpos, _olen) do {                                  \
-        if (repl_count >= repl_cap) {                                         \
-            int _nc = repl_cap == 0 ? 16 : repl_cap * 2;                     \
-            repl_t *_t = (repl_t *)realloc(repl_log, sizeof(repl_t) * _nc);  \
-            if (!_t) { free(repl_log); free(working); rb_raise(rb_eNoMemError, "repl_log"); } \
-            repl_log = _t; repl_cap = _nc;                                    \
-        }                                                                     \
-        repl_log[repl_count].wpos     = (_wpos);                              \
-        repl_log[repl_count].orig_len = (_olen);                              \
-        repl_count++;                                                         \
-    } while (0)
-
-    #define WORKING_TO_ORIG(_wpos) ({                                         \
-        long _shift = 0;                                                      \
-        for (int _ri = 0; _ri < repl_count; _ri++) {                         \
-            if (repl_log[_ri].wpos <= (_wpos))                                \
-                _shift += 10 - repl_log[_ri].orig_len;                       \
-        }                                                                     \
-        (_wpos) - _shift;                                                     \
-    })
-
-    #define COLLECT_AND_REPLACE(pat, use_bnd, tag_bit, pat_name) do {        \
-        const char *_cur = working;                                           \
-        regmatch_t _m[4];                                                     \
-        while (regexec((pat), _cur, 4, _m, 0) == 0) {                        \
-            regoff_t _fso = _m[0].rm_so, _feo = _m[0].rm_eo;                 \
-            if (_fso < 0 || _feo < _fso) break;                               \
-            regoff_t _cso = _fso, _ceo = _feo;                                \
-            if (use_bnd) {                                                    \
-                if (_m[1].rm_so >= 0 && _m[1].rm_eo > _m[1].rm_so)          \
-                    _cso = _m[1].rm_eo;                                       \
-                if (_m[3].rm_so >= 0 && _m[3].rm_eo > _m[3].rm_so)          \
-                    _ceo = _m[3].rm_so;                                       \
-            }                                                                 \
-            size_t _vlen = (size_t)(_ceo - _cso);                             \
-            long _wpos   = (long)(_cur - working) + (long)_cso;              \
-            long _orig   = WORKING_TO_ORIG(_wpos);                            \
-            VALUE _match = rb_hash_new();                                     \
-            rb_hash_aset(_match, ID2SYM(rb_intern("tag")),                    \
-                         ID2SYM(rb_intern(tag_name_for_bit(tag_bit))));       \
-            rb_hash_aset(_match, ID2SYM(rb_intern("name")),                  \
-                         rb_str_new_cstr(pat_name));                          \
-            rb_hash_aset(_match, ID2SYM(rb_intern("value")),                 \
-                         rb_str_new(_cur + _cso, _vlen));                     \
-            rb_hash_aset(_match, ID2SYM(rb_intern("start")),                 \
-                         LONG2NUM(_orig));                                    \
-            rb_hash_aset(_match, ID2SYM(rb_intern("length")),                \
-                         LONG2NUM((long)_vlen));                              \
-            rb_ary_push(matches_arr, _match);                                 \
-            REPL_LOG_PUSH(_wpos, (long)_vlen);                                \
-            if (_feo == _fso) { if (*_cur) _cur++; else break; }             \
-            else _cur += _feo;                                                \
-        }                                                                     \
-        char *_next = replace_all_matches((pat), working, (use_bnd), &ph_default); \
-        free(working);                                                        \
-        if (!_next) { free(repl_log); rb_raise(rb_eNoMemError, "replace_all_matches failed in scan"); } \
-        working = _next;                                                      \
-    } while (0)
+    /* Build the redacted working buffer (same logic as redact_builtins). */
+    size_t ph_len  = strlen(ph_plain.str); /* "[REDACTED]" = 10 */
+    size_t out_cap = in_len + n_ev * ph_len + 1;
+    char *working  = (char *)malloc(out_cap);
+    if (!working) { free(ev); rb_raise(rb_eNoMemError, "scan working buffer alloc"); }
 
-    for (int i = 0; i < NUM_PATTERNS; i++) {
-        if (!scan_enable_bit(rb_enable_bits, i)) continue;
-        COLLECT_AND_REPLACE(&compiled_patterns[i], boundary_wrapped[i],
-                            pattern_tags[i], pattern_names[i]);
+    size_t out_len = 0, cur = 0;
+    for (size_t i = 0; i < n_ev; i++) {
+        size_t s = ev[i].start, l = ev[i].length;
+        if (s > cur) { memcpy(working + out_len, input + cur, s - cur); out_len += s - cur; }
+        memcpy(working + out_len, ph_plain.str, ph_len);
+        out_len += ph_len;
+        cur = s + l;
     }
+    if (cur < in_len) { memcpy(working + out_len, input + cur, in_len - cur); out_len += in_len - cur; }
+    working[out_len] = '\0';
 
+    /* ------------------------------------------------------------------ */
+    /* Stage 2: custom patterns via glibc on the rewritten buffer.         */
+    /* Original coords recovered via working_to_orig() using ev[].         */
+    /* ------------------------------------------------------------------ */
     for (int i = 0; i < custom_count; i++) {
         if (!scan_enable_bit(rb_enable_bits, NUM_PATTERNS + i)) continue;
-        COLLECT_AND_REPLACE(&custom_patterns[i].compiled,
-                            custom_patterns[i].boundary,
-                            custom_patterns[i].tag, custom_patterns[i].name);
-    }
 
-    #undef COLLECT_AND_REPLACE
-    #undef WORKING_TO_ORIG
-    #undef REPL_LOG_PUSH
+        const char *cur_ptr = working;
+        regmatch_t  m[4];
+        while (regexec(&custom_patterns[i].compiled, cur_ptr, 4, m, 0) == 0) {
+            regoff_t fso = m[0].rm_so, feo = m[0].rm_eo;
+            if (fso < 0 || feo < fso) break;
+
+            regoff_t cso = fso, ceo = feo;
+            if (custom_patterns[i].boundary) {
+                if (m[1].rm_so >= 0 && m[1].rm_eo > m[1].rm_so) cso = m[1].rm_eo;
+                if (m[3].rm_so >= 0 && m[3].rm_eo > m[3].rm_so) ceo = m[3].rm_so;
+            }
+
+            long wpos_core  = (long)(cur_ptr - working) + (long)cso;
+            long orig_start = working_to_orig(wpos_core, ev, n_ev, ph_len);
+            long core_len   = (long)(ceo - cso);
+
+            VALUE h = rb_hash_new();
+            rb_hash_aset(h, ID2SYM(rb_intern("tag")),
+                         ID2SYM(rb_intern(tag_name_for_bit(custom_patterns[i].tag))));
+            rb_hash_aset(h, ID2SYM(rb_intern("name")),
+                         rb_str_new_cstr(custom_patterns[i].name));
+            rb_hash_aset(h, ID2SYM(rb_intern("value")),
+                         rb_str_new(cur_ptr + cso, (size_t)core_len));
+            rb_hash_aset(h, ID2SYM(rb_intern("start")),  LONG2NUM(orig_start));
+            rb_hash_aset(h, ID2SYM(rb_intern("length")), LONG2NUM(core_len));
+            rb_ary_push(matches_arr, h);
+
+            if (feo == fso) { if (*cur_ptr) cur_ptr++; else break; }
+            else cur_ptr += feo;
+        }
+
+        char *next = replace_all_matches(&custom_patterns[i].compiled, working,
+                                         custom_patterns[i].boundary, &ph_plain);
+        free(working);
+        if (!next) { free(ev); rb_raise(rb_eNoMemError, "replace_all_matches failed in scan"); }
+        working = next;
+    }
 
-    free(repl_log);
+    free(ev);
 
-    VALUE result = rb_hash_new();
+    VALUE result      = rb_hash_new();
     VALUE rb_redacted = rb_str_new_cstr(working);
     free(working);
+    rb_funcall(rb_redacted, rb_intern("force_encoding"), 1,
+               rb_funcall(rb_text, rb_intern("encoding"), 0));
     rb_hash_aset(result, ID2SYM(rb_intern("redacted")), rb_redacted);
     rb_hash_aset(result, ID2SYM(rb_intern("matches")),  matches_arr);
     return result;

data/lib/data_redactor/version.rb

@@ -1,4 +1,4 @@
 module DataRedactor
   # Current gem version. Follows {https://semver.org Semantic Versioning 2.0.0}.
-  VERSION = "0.9.0"
+  VERSION = "0.10.1"
 end

data/lib/data_redactor.rb

@@ -74,6 +74,15 @@ module DataRedactor
   # Default placeholder used when +placeholder:+ is not given to {redact}.
   PLACEHOLDER_DEFAULT = "[REDACTED]"
 
+  # @api private
+  # Inputs larger than this (bytes) are split into newline-bounded chunks before
+  # being handed to the C engine. Bounds the per-call O(N) cost glibc regexec
+  # pays for state-log allocation, turning total redaction cost from O(N²) (one
+  # giant pass) into O(N × CHUNK_SIZE) (many bounded passes). 64 KB is a
+  # compromise: small enough to keep per-call cost low, large enough that
+  # typical log/JSON inputs use few chunks. See option G in TODO.md.
+  CHUNK_SIZE = 64 * 1024
+
   module_function
 
   # List of supported tag symbols.
@@ -132,6 +141,11 @@ module DataRedactor
   def redact(text, only: nil, except: nil, placeholder: PLACEHOLDER_DEFAULT)
     enable_bits = build_enable_bits(only, except)
     ph_mode, ph_str = resolve_placeholder(placeholder)
+    # Defer to the C layer's TypeError for non-Strings; only chunk if the input
+    # is a String big enough to benefit (avoid bytesize on non-Strings).
+    if text.is_a?(String) && text.bytesize > CHUNK_SIZE
+      return _chunk_bytes(text).map { |c| _redact(c, ph_mode, ph_str, enable_bits) }.join
+    end
     _redact(text, ph_mode, ph_str, enable_bits)
   end
 
@@ -157,7 +171,12 @@ module DataRedactor
   #   #                 value: "user@example.com", start: 0, length: 16}] }
   def scan(text, only: nil, except: nil)
     enable_bits = build_enable_bits(only, except)
-    result = _scan(text, enable_bits)
+    result =
+      if text.is_a?(String) && text.bytesize > CHUNK_SIZE
+        _chunked_scan(text, enable_bits)
+      else
+        _scan(text, enable_bits)
+      end
     # Normalise: convert tag string from C (uppercase) back to the Symbol used in TAGS
     result[:matches].each { |m| m[:tag] = m[:tag].to_s.downcase.to_sym }
     result
@@ -419,4 +438,59 @@ module DataRedactor
         "placeholder must be a String, :tagged, or :hash — got #{placeholder.inspect}"
     end
   end
+
+  # @api private
+  # Split +text+ into byte-bounded chunks for the chunked redact/scan path.
+  # Chunks end at a +\n+ when possible so no match straddles a boundary; if a
+  # single line exceeds {CHUNK_SIZE} (rare in real inputs), it becomes one
+  # oversized chunk and pays the per-pattern O(N) cost — documented limitation.
+  # Returns an Array of byte-Strings whose concatenation equals +text+ exactly
+  # (including the original newline separators).
+  #
+  # @param text [String]
+  # @return [Array<String>]
+  def _chunk_bytes(text)
+    chunks = []
+    pos = 0
+    len = text.bytesize
+    while pos < len
+      remaining = len - pos
+      if remaining <= CHUNK_SIZE
+        chunks << text.byteslice(pos, remaining)
+        break
+      end
+      # Find the last \n in [pos, pos+CHUNK_SIZE). If none, chunk is one long
+      # line — take CHUNK_SIZE bytes as a fallback (boundary-split risk).
+      window = text.byteslice(pos, CHUNK_SIZE)
+      nl = window.rindex("\n")
+      take = nl ? nl + 1 : CHUNK_SIZE
+      chunks << text.byteslice(pos, take)
+      pos += take
+    end
+    chunks
+  end
+
+  # @api private
+  # Chunked variant of +_scan+: runs the C scanner on each chunk, then offsets
+  # each match's +:start+ by the chunk's base byte-position in the original
+  # input so the byteslice invariant holds end-to-end.
+  #
+  # @param text [String]
+  # @param enable_bits [Array<Integer>]
+  # @return [Hash{Symbol => Object}] +{ redacted: String, matches: Array<Hash> }+
+  def _chunked_scan(text, enable_bits)
+    redacted = +""
+    matches = []
+    base = 0
+    _chunk_bytes(text).each do |chunk|
+      part = _scan(chunk, enable_bits)
+      redacted << part[:redacted]
+      part[:matches].each do |m|
+        m[:start] += base
+        matches << m
+      end
+      base += chunk.bytesize
+    end
+    { redacted: redacted, matches: matches }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_redactor
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Daniele Frisanco
@@ -79,6 +79,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: benchmark-memory
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 description: A Ruby gem with a C extension for high-performance scanning and redaction
   of 85 sensitive patterns — API keys, tokens, credentials, IBANs, national IDs, emails,
   phone numbers, and PII from 15+ countries. Optional Logger formatter, Rails filter_parameters
@@ -93,10 +121,13 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE
+- README.md
 - ext/data_redactor/custom_patterns.c
 - ext/data_redactor/custom_patterns.h
 - ext/data_redactor/data_redactor.c
 - ext/data_redactor/extconf.rb
+- ext/data_redactor/matcher.c
+- ext/data_redactor/matcher.h
 - ext/data_redactor/patterns.c
 - ext/data_redactor/patterns.h
 - ext/data_redactor/placeholder.c
@@ -112,7 +143,6 @@ files:
 - lib/data_redactor/integrations/rails.rb
 - lib/data_redactor/name_pattern.rb
 - lib/data_redactor/version.rb
-- readme.md
 homepage: https://github.com/danielefrisanco/data_redactor
 licenses:
 - MIT