character_set 1.3.0-java → 1.6.0-java

data/ext/character_set/character_set.c

@@ -82,7 +82,11 @@ static const rb_data_type_t cs_type = {
         .dsize = cs_memsize,
     },
     .data = NULL,
+#ifdef RUBY_TYPED_FROZEN_SHAREABLE
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE,
+#else
     .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+#endif
 };
 
 static inline VALUE
@@ -216,6 +220,7 @@ cs_method_hash(VALUE self)
   cs_cp cp, len, hash, four_byte_value;
   cs_ar *cps;
   cps = cs_fetch_cps(self, &len);
+  four_byte_value = 0;
 
   hash = 17;
   for (cp = 0; cp < len; cp++)
@@ -314,9 +319,9 @@ cs_method_minmax(VALUE self)
     cs_cp cp, alen, blen;                                        \
     cs_ar *acps, *bcps;                                          \
     struct cs_data *new_data;                                    \
-    new_cs = cs_alloc(RBASIC(self)->klass, &new_data);           \
     acps = cs_fetch_cps(cs_a, &alen);                            \
     bcps = cs_fetch_cps(cs_b, &blen);                            \
+    new_cs = cs_alloc(RBASIC(self)->klass, &new_data);           \
     for (cp = 0; cp < UNICODE_CP_COUNT; cp++)                    \
     {                                                            \
       if (tst_cp(acps, alen, cp) comp_op tst_cp(bcps, blen, cp)) \
@@ -1045,13 +1050,14 @@ raise_arg_err_unless_string(VALUE val)
 }
 
 static VALUE
-cs_class_method_of(VALUE self, VALUE str)
+cs_class_method_of_string(VALUE self, VALUE string)
 {
   VALUE new_cs;
   struct cs_data *new_data;
+
+  raise_arg_err_unless_string(string);
   new_cs = cs_alloc(self, &new_data);
-  raise_arg_err_unless_string(str);
-  each_cp(str, add_str_cp_to_arr, 0, 0, new_data, 0);
+  each_cp(string, add_str_cp_to_arr, 0, 0, new_data, 0);
   return new_cs;
 }
 
@@ -1074,7 +1080,7 @@ cs_method_count_in(VALUE self, VALUE str)
   data = cs_fetch_data(self);
   count = 0;
   each_cp(str, count_str_cp, data->cps, data->len, data, &count);
-  return INT2NUM(count);
+  return INT2NUM((int)count);
 }
 
 static inline int
@@ -1132,118 +1138,76 @@ cs_method_used_by_p(VALUE self, VALUE str)
   return only_uses_other_cps == Qfalse ? Qtrue : Qfalse;
 }
 
-static void
-cs_str_buf_cat(VALUE str, const char *ptr, long len)
-{
-  long total, olen;
-  char *sptr;
-
-  RSTRING_GETMEM(str, sptr, olen);
-  sptr = RSTRING(str)->as.heap.ptr;
-  olen = RSTRING(str)->as.heap.len;
-  total = olen + len;
-  memcpy(sptr + olen, ptr, len);
-  RSTRING(str)->as.heap.len = total;
-}
-
-#ifndef TERM_FILL
-#define TERM_FILL(ptr, termlen)                     \
-  do                                                \
-  {                                                 \
-    char *const term_fill_ptr = (ptr);              \
-    const int term_fill_len = (termlen);            \
-    *term_fill_ptr = '\0';                          \
-    if (__builtin_expect(!!(term_fill_len > 1), 0)) \
-      memset(term_fill_ptr, 0, term_fill_len);      \
-  } while (0)
-#endif
-
-static void
-cs_str_buf_terminate(VALUE str, rb_encoding *enc)
-{
-  char *ptr;
-  long len;
-
-  ptr = RSTRING(str)->as.heap.ptr;
-  len = RSTRING(str)->as.heap.len;
-  TERM_FILL(ptr + len, rb_enc_mbminlen(enc));
-}
-
+// partially based on rb_str_delete_bang
 static inline VALUE
 cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
 {
   cs_ar *cps;
-  cs_cp len;
-  rb_encoding *str_enc;
-  VALUE orig_len, new_str_buf;
-  int cp_len;
-  unsigned int str_cp;
-  const char *ptr, *end;
+  cs_cp cs_len;
+  VALUE orig_str_len;
+
+  rb_encoding *enc;
+  char *s, *send, *t;
+  int ascompat, cr;
 
   raise_arg_err_unless_string(str);
 
-  cps = cs_fetch_cps(set, &len);
+  orig_str_len = RSTRING_LEN(str);
 
-  orig_len = RSTRING_LEN(str);
-  if (orig_len < 1) // empty string, will never change
+  if (orig_str_len == 0)
   {
-    if (bang)
-    {
-      return Qnil;
-    }
-    return rb_str_dup(str);
+    return bang ? Qnil : str;
   }
 
-  new_str_buf = rb_str_buf_new(orig_len);
-  str_enc = rb_enc_get(str);
-  rb_enc_associate(new_str_buf, str_enc);
-  rb_str_modify(new_str_buf);
-  ENC_CODERANGE_SET(new_str_buf, rb_enc_asciicompat(str_enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID);
-
-  ptr = RSTRING_PTR(str);
-  end = RSTRING_END(str);
+  if (!bang)
+  {
+    str = rb_str_dup(str);
+  }
 
-  if (single_byte_optimizable(str))
+  cps = cs_fetch_cps(set, &cs_len);
+  rb_str_modify(str);
+  enc = rb_enc_get(str);
+  ascompat = rb_enc_asciicompat(enc);
+  s = t = RSTRING_PTR(str);
+  send = RSTRING_END(str);
+  cr = ascompat ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID;
+  while (s < send)
   {
-    while (ptr < end)
+    unsigned int c;
+    int clen;
+
+    if (ascompat && (c = *(unsigned char *)s) < 0x80)
     {
-      str_cp = *ptr & 0xff;
-      if ((!tst_cp(cps, len, str_cp)) == delete)
+      if (tst_cp(cps, cs_len, c) != delete)
       {
-        cs_str_buf_cat(new_str_buf, ptr, 1);
+        if (t != s)
+          *t = c;
+        t++;
       }
-      ptr++;
+      s++;
     }
-  }
-  else // likely to be multibyte string
-  {
-    while (ptr < end)
+    else
     {
-      str_cp = rb_enc_codepoint_len(ptr, end, &cp_len, str_enc);
-      if ((!tst_cp(cps, len, str_cp)) == delete)
+      c = rb_enc_codepoint_len(s, send, &clen, enc);
+
+      if (tst_cp(cps, cs_len, c) != delete)
       {
-        cs_str_buf_cat(new_str_buf, ptr, cp_len);
+        if (t != s)
+          rb_enc_mbcput(c, t, enc);
+        t += clen;
+        if (cr == ENC_CODERANGE_7BIT)
+          cr = ENC_CODERANGE_VALID;
       }
-      ptr += cp_len;
+      s += clen;
     }
   }
 
-  cs_str_buf_terminate(new_str_buf, str_enc);
+  rb_str_set_len(str, t - RSTRING_PTR(str));
+  ENC_CODERANGE_SET(str, cr);
 
-  if (bang)
-  {
-    if (RSTRING_LEN(new_str_buf) == (long)orig_len) // string unchanged
-    {
-      return Qnil;
-    }
-    rb_str_shared_replace(str, new_str_buf);
-  }
-  else
+  if (bang && (RSTRING_LEN(str) == (long)orig_str_len)) // string unchanged
   {
-    RB_OBJ_WRITE(new_str_buf, &(RBASIC(new_str_buf))->klass, rb_obj_class(str));
-    // slightly cumbersome approach needed for compatibility with Ruby < 2.3:
-    RBASIC(new_str_buf)->flags |= (RBASIC(str)->flags & (FL_TAINT));
-    str = new_str_buf;
+    return Qnil;
   }
 
   return str;
@@ -1285,6 +1249,10 @@ cs_method_allocated_length(VALUE self)
 
 void Init_character_set()
 {
+#ifdef HAVE_RB_EXT_RACTOR_SAFE
+  rb_ext_ractor_safe(true);
+#endif
+
   VALUE cs = rb_define_class("CharacterSet", rb_cObject);
 
   rb_define_alloc_func(cs, cs_method_allocate);
@@ -1339,7 +1307,7 @@ void Init_character_set()
   // `CharacterSet`-specific methods
 
   rb_define_singleton_method(cs, "from_ranges", cs_class_method_from_ranges, -2);
-  rb_define_singleton_method(cs, "of", cs_class_method_of, 1);
+  rb_define_singleton_method(cs, "of_string", cs_class_method_of_string, 1);
 
   rb_define_method(cs, "ranges", cs_method_ranges, 0);
   rb_define_method(cs, "sample", cs_method_sample, -1);

data/ext/character_set/unicode_casefold_table.h

@@ -6,7 +6,7 @@ typedef struct casefold_mapping {
   unsigned long to;
 } casefold_mapping;
 
-#define CASEFOLD_COUNT 1383
+#define CASEFOLD_COUNT 1426
 
 static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x0041,0x0061},
@@ -564,6 +564,41 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x104D1,0x104F9},
 {0x104D2,0x104FA},
 {0x104D3,0x104FB},
+{0x10570,0x10597},
+{0x10571,0x10598},
+{0x10572,0x10599},
+{0x10573,0x1059A},
+{0x10574,0x1059B},
+{0x10575,0x1059C},
+{0x10576,0x1059D},
+{0x10577,0x1059E},
+{0x10578,0x1059F},
+{0x10579,0x105A0},
+{0x1057A,0x105A1},
+{0x1057C,0x105A3},
+{0x1057D,0x105A4},
+{0x1057E,0x105A5},
+{0x1057F,0x105A6},
+{0x10580,0x105A7},
+{0x10581,0x105A8},
+{0x10582,0x105A9},
+{0x10583,0x105AA},
+{0x10584,0x105AB},
+{0x10585,0x105AC},
+{0x10586,0x105AD},
+{0x10587,0x105AE},
+{0x10588,0x105AF},
+{0x10589,0x105B0},
+{0x1058A,0x105B1},
+{0x1058C,0x105B3},
+{0x1058D,0x105B4},
+{0x1058E,0x105B5},
+{0x1058F,0x105B6},
+{0x10590,0x105B7},
+{0x10591,0x105B8},
+{0x10592,0x105B9},
+{0x10594,0x105BB},
+{0x10595,0x105BC},
 {0x10A0,0x2D00},
 {0x10A1,0x2D01},
 {0x10A2,0x2D02},
@@ -1102,6 +1137,7 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0x2C2C,0x2C5C},
 {0x2C2D,0x2C5D},
 {0x2C2E,0x2C5E},
+{0x2C2F,0x2C5F},
 {0x2C60,0x2C61},
 {0x2C62,0x026B},
 {0x2C63,0x1D7D},
@@ -1282,10 +1318,17 @@ static const casefold_mapping unicode_casefold_table[CASEFOLD_COUNT] = {
 {0xA7BA,0xA7BB},
 {0xA7BC,0xA7BD},
 {0xA7BE,0xA7BF},
+{0xA7C0,0xA7C1},
 {0xA7C2,0xA7C3},
 {0xA7C4,0xA794},
 {0xA7C5,0x0282},
 {0xA7C6,0x1D8E},
+{0xA7C7,0xA7C8},
+{0xA7C9,0xA7CA},
+{0xA7D0,0xA7D1},
+{0xA7D6,0xA7D7},
+{0xA7D8,0xA7D9},
+{0xA7F5,0xA7F6},
 {0xAB70,0x13A0},
 {0xAB71,0x13A1},
 {0xAB72,0x13A2},

data/lib/character_set/character.rb

@@ -1,7 +1,7 @@
 class CharacterSet
   class Character
     ENCODING = 'utf-8'.freeze
-    SAFELY_PRINTABLE = (0x21..0x7E).to_a - ['-', '[', '\\', ']', '^'].map(&:ord)
+    SAFELY_PRINTABLE = (0x21..0x7E).to_a - %w(- / [ \\ ] ^).map(&:ord)
 
     attr_accessor :codepoint
 

data/lib/character_set/core_ext/regexp_ext.rb

@@ -8,4 +8,4 @@ class CharacterSet
   end
 end
 
-::Regexp.send(:include, CharacterSet::CoreExt::RegexpExt)
+::Regexp.instance_eval { include CharacterSet::CoreExt::RegexpExt }

data/lib/character_set/core_ext/string_ext.rb

@@ -2,7 +2,7 @@ class CharacterSet
   module CoreExt
     module StringExt
       def character_set
-        CharacterSet.of(self)
+        CharacterSet.of_string(self)
       end
 
       {
@@ -29,4 +29,4 @@ class CharacterSet
   end
 end
 
-::String.send(:include, CharacterSet::CoreExt::StringExt)
+::String.instance_eval { include CharacterSet::CoreExt::StringExt }

data/lib/character_set/expression_converter.rb

@@ -4,56 +4,57 @@ class CharacterSet
 
     Error = Class.new(ArgumentError)
 
-    def convert(expression)
-      CharacterSet.require_optional_dependency('regexp_parser')
+    def convert(expression, to = CharacterSet)
+      CharacterSet.require_optional_dependency('regexp_parser', __method__)
 
       case expression
       when Regexp::Expression::Root
         if expression.count != 1
           raise Error, 'Pass a Regexp with exactly one expression, e.g. /[a-z]/'
         end
-        convert(expression[0])
+        convert(expression[0], to)
 
       when Regexp::Expression::CharacterSet
-        content = expression.map { |subexp| convert(subexp) }.reduce(:+)
+        content = expression.map { |subexp| convert(subexp, to) }.reduce(:+)
+        content ||= to[]
         expression.negative? ? content.inversion : content
 
       when Regexp::Expression::CharacterSet::Intersection
-        expression.map { |subexp| convert(subexp) }.reduce(:&)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:&)
 
       when Regexp::Expression::CharacterSet::IntersectedSequence
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+) || to[]
 
       when Regexp::Expression::CharacterSet::Range
-        start, finish = expression.map { |subexp| convert(subexp) }
-        CharacterSet.new((start.min)..(finish.max))
+        start, finish = expression.map { |subexp| convert(subexp, to) }
+        to.new((start.min)..(finish.max))
 
       when Regexp::Expression::CharacterType::Any
-        CharacterSet.unicode
+        to.unicode
 
       when Regexp::Expression::CharacterType::Base
         /(?<negative>non)?(?<base_name>.+)/ =~ expression.token
         content =
           if expression.unicode_classes?
             # in u-mode, type shortcuts match the same as \p{<long type name>}
-            CharacterSet.of_property(base_name)
+            to.of_property(base_name)
           else
             # in normal mode, types match only ascii chars
             case base_name.to_sym
-            when :digit then CharacterSet.from_ranges(48..57)
-            when :hex   then CharacterSet.from_ranges(48..57, 65..70, 97..102)
-            when :space then CharacterSet.from_ranges(9..13, 32..32)
-            when :word  then CharacterSet.from_ranges(48..57, 65..90, 95..95, 97..122)
+            when :digit then to.from_ranges(48..57)
+            when :hex   then to.from_ranges(48..57, 65..70, 97..102)
+            when :space then to.from_ranges(9..13, 32..32)
+            when :word  then to.from_ranges(48..57, 65..90, 95..95, 97..122)
             else raise Error, "Unsupported CharacterType #{base_name}"
             end
           end
         negative ? content.inversion : content
 
       when Regexp::Expression::EscapeSequence::CodepointList
-        CharacterSet.new(expression.codepoints)
+        to.new(expression.codepoints)
 
       when Regexp::Expression::EscapeSequence::Base
-        CharacterSet[expression.codepoint]
+        to[expression.codepoint]
 
       when Regexp::Expression::Group::Capture,
            Regexp::Expression::Group::Passive,
@@ -61,19 +62,19 @@ class CharacterSet
            Regexp::Expression::Group::Atomic,
            Regexp::Expression::Group::Options
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Groups must contain exactly one expression, e.g. ([a-z])'
         end
 
-      when Regexp::Expression::Alternation
-        expression.map { |subexp| convert(subexp) }.reduce(:+)
+      when Regexp::Expression::Alternation # rubocop:disable Lint/DuplicateBranch
+        expression.map { |subexp| convert(subexp, to) }.reduce(:+)
 
       when Regexp::Expression::Alternative
         case expression.count
-        when 0 then CharacterSet[]
-        when 1 then convert(expression.first)
+        when 0 then to[]
+        when 1 then convert(expression.first, to)
         else
           raise Error, 'Alternatives must contain exactly one expression'
         end
@@ -82,11 +83,11 @@ class CharacterSet
         if expression.set_level == 0 && expression.text.size != 1
           raise Error, 'Literal runs outside of sets are codepoint *sequences*'
         end
-        CharacterSet[expression.text.ord]
+        to[expression.text.ord]
 
       when Regexp::Expression::UnicodeProperty::Base,
            Regexp::Expression::PosixClass
-        content = CharacterSet.of_property(expression.token)
+        content = to.of_property(expression.token)
         if expression.type == :posixclass && expression.ascii_classes?
           content = content.ascii_part
         end

data/lib/character_set/parser.rb

@@ -5,7 +5,7 @@ class CharacterSet
     def codepoints_from_enumerable(object)
       raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
       # Use #each to check first element (only this works for all Enumerables)
-      object.each do |e|
+      object.each do |e| # rubocop:disable Lint/UnreachableLoop
         return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
         return object.map(&:ord) if e.is_a?(String)  && e.length == 1
         raise ArgumentError, "#{e.inspect} is not valid as a codepoint"