character_set 1.5.0-java → 1.7.0-java

data/ext/character_set/character_set.c

@@ -82,7 +82,11 @@ static const rb_data_type_t cs_type = {
         .dsize = cs_memsize,
     },
     .data = NULL,
+#ifdef RUBY_TYPED_FROZEN_SHAREABLE
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_FROZEN_SHAREABLE,
+#else
     .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+#endif
 };
 
 static inline VALUE
@@ -315,9 +319,9 @@ cs_method_minmax(VALUE self)
     cs_cp cp, alen, blen;                                        \
     cs_ar *acps, *bcps;                                          \
     struct cs_data *new_data;                                    \
-    new_cs = cs_alloc(RBASIC(self)->klass, &new_data);           \
     acps = cs_fetch_cps(cs_a, &alen);                            \
     bcps = cs_fetch_cps(cs_b, &blen);                            \
+    new_cs = cs_alloc(RBASIC(self)->klass, &new_data);           \
     for (cp = 0; cp < UNICODE_CP_COUNT; cp++)                    \
     {                                                            \
       if (tst_cp(acps, alen, cp) comp_op tst_cp(bcps, blen, cp)) \
@@ -372,22 +376,20 @@ cs_toggle_codepoint(VALUE cs, VALUE cp_num, int on, int return_nil_if_noop)
   cps = data->cps;
   len = data->len;
   cp = FIX2ULONG(cp_num);
-  if (return_nil_if_noop && (!tst_cp(cps, len, cp) == !on))
+  if (return_nil_if_noop && tst_cp(cps, len, cp) == on)
   {
     return Qnil;
   }
+
+  if (on)
+  {
+    set_cp(data, cp);
+  }
   else
   {
-    if (on)
-    {
-      set_cp(data, cp);
-    }
-    else
-    {
-      clr_cp(cps, len, cp);
-    }
-    return cs;
+    clr_cp(cps, len, cp);
   }
+  return cs;
 }
 
 static VALUE
@@ -571,7 +573,7 @@ cs_method_merge(VALUE self, VALUE other)
   {
     return cs_merge_cs(self, other);
   }
-  else if (TYPE(other) == T_ARRAY)
+  if (TYPE(other) == T_ARRAY)
   {
     return cs_merge_rb_array(self, other);
   }
@@ -705,8 +707,7 @@ cs_method_ranges(VALUE self)
 
       if (!previous_cp_num) {
         current_start = cp_num;
-      } else if (previous_cp_num + 2 != cp_num)
-      {
+      } else if (previous_cp_num + 2 != cp_num) {
         // gap found, finalize previous range
         rb_ary_push(ranges, rb_range_new(current_start, current_end, 0));
         current_start = cp_num;
@@ -914,10 +915,10 @@ cs_method_ext_inversion(int argc, VALUE *argv, VALUE self)
   return new_cs;
 }
 
-typedef int (*str_cp_handler)(unsigned int, cs_ar *, cs_cp len, struct cs_data *data, VALUE *memo);
+typedef int (*str_cp_handler)(unsigned int, cs_ar *, cs_cp len, struct cs_data *data, VALUE memo);
 
 static inline int
-add_str_cp_to_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+add_str_cp_to_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   set_cp(data, str_cp);
   return 1;
@@ -964,7 +965,7 @@ cs_method_case_insensitive(VALUE self)
 }
 
 static inline VALUE
-each_sb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+each_sb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   long i, str_len;
   unsigned int str_cp;
@@ -983,21 +984,29 @@ each_sb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_d
 }
 
 static inline VALUE
-each_mb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+each_mb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   int n;
   unsigned int str_cp;
   const char *ptr, *end;
-  rb_encoding *enc;
+  rb_encoding *utf8;
+
+  utf8 = rb_utf8_encoding();
+  if (rb_enc_get(str) == utf8)
+  {
+    str = rb_str_new_frozen(str);
+  }
+  else
+  {
+    str = rb_str_encode(str, rb_enc_from_encoding(utf8), 0, Qnil);
+  }
 
-  str = rb_str_new_frozen(str);
   ptr = RSTRING_PTR(str);
   end = RSTRING_END(str);
-  enc = rb_enc_get(str);
 
   while (ptr < end)
   {
-    str_cp = rb_enc_codepoint_len(ptr, end, &n, enc);
+    str_cp = rb_enc_codepoint_len(ptr, end, &n, utf8);
     if (!(*func)(str_cp, cp_arr, len, data, memo))
     {
       return Qfalse;
@@ -1028,12 +1037,13 @@ single_byte_optimizable(VALUE str)
 }
 
 static inline VALUE
-each_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+each_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   if (single_byte_optimizable(str))
   {
     return each_sb_cp(str, func, cp_arr, len, data, memo);
   }
+
   return each_mb_cp(str, func, cp_arr, len, data, memo);
 }
 
@@ -1047,26 +1057,23 @@ raise_arg_err_unless_string(VALUE val)
 }
 
 static VALUE
-cs_class_method_of(int argc, VALUE *argv, VALUE self)
+cs_class_method_of_string(VALUE self, VALUE string)
 {
   VALUE new_cs;
   struct cs_data *new_data;
-  int i;
+
+  raise_arg_err_unless_string(string);
   new_cs = cs_alloc(self, &new_data);
-  for (i = 0; i < argc; i++)
-  {
-    raise_arg_err_unless_string(argv[i]);
-    each_cp(argv[i], add_str_cp_to_arr, 0, 0, new_data, 0);
-  }
+  each_cp(string, add_str_cp_to_arr, 0, 0, new_data, 0);
   return new_cs;
 }
 
 static inline int
-count_str_cp(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+count_str_cp(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   if (tst_cp(cp_arr, len, str_cp))
   {
-    *memo += 1;
+    *((VALUE *)memo) += 1;
   }
   return 1;
 }
@@ -1074,17 +1081,17 @@ count_str_cp(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data
 static VALUE
 cs_method_count_in(VALUE self, VALUE str)
 {
-  VALUE count;
+  long count;
   struct cs_data *data;
   raise_arg_err_unless_string(str);
   data = cs_fetch_data(self);
   count = 0;
-  each_cp(str, count_str_cp, data->cps, data->len, data, &count);
-  return INT2NUM((int)count);
+  each_cp(str, count_str_cp, data->cps, data->len, data, (VALUE)&count);
+  return LONG2FIX(count);
 }
 
 static inline int
-str_cp_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+str_cp_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   return tst_cp(cp_arr, len, str_cp);
 }
@@ -1099,11 +1106,11 @@ cs_method_cover_p(VALUE self, VALUE str)
 }
 
 static inline int
-add_str_cp_to_str_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+add_str_cp_to_str_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   if (tst_cp(cp_arr, len, str_cp))
   {
-    rb_ary_push(memo[0], rb_enc_uint_chr((int)str_cp, (rb_encoding *)memo[1]));
+    rb_ary_push(memo, rb_enc_uint_chr((int)str_cp, rb_utf8_encoding()));
   }
   return 1;
 }
@@ -1111,18 +1118,17 @@ add_str_cp_to_str_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_d
 static VALUE
 cs_method_scan(VALUE self, VALUE str)
 {
-  VALUE memo[2];
+  VALUE memo;
   struct cs_data *data;
   raise_arg_err_unless_string(str);
   data = cs_fetch_data(self);
-  memo[0] = rb_ary_new();
-  memo[1] = (VALUE)rb_enc_get(str);
+  memo = rb_ary_new();
   each_cp(str, add_str_cp_to_str_arr, data->cps, data->len, data, memo);
-  return memo[0];
+  return memo;
 }
 
 static inline int
-str_cp_not_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+str_cp_not_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE memo)
 {
   return !tst_cp(cp_arr, len, str_cp);
 }
@@ -1138,116 +1144,91 @@ cs_method_used_by_p(VALUE self, VALUE str)
   return only_uses_other_cps == Qfalse ? Qtrue : Qfalse;
 }
 
-static void
-cs_str_buf_cat(VALUE str, const char *ptr, long len)
-{
-  long total, olen;
-  char *sptr;
-
-  RSTRING_GETMEM(str, sptr, olen);
-  sptr = RSTRING(str)->as.heap.ptr;
-  olen = RSTRING(str)->as.heap.len;
-  total = olen + len;
-  memcpy(sptr + olen, ptr, len);
-  RSTRING(str)->as.heap.len = total;
-}
-
-#ifndef TERM_FILL
-#define TERM_FILL(ptr, termlen)                     \
-  do                                                \
-  {                                                 \
-    char *const term_fill_ptr = (ptr);              \
-    const int term_fill_len = (termlen);            \
-    *term_fill_ptr = '\0';                          \
-    if (__builtin_expect(!!(term_fill_len > 1), 0)) \
-      memset(term_fill_ptr, 0, term_fill_len);      \
-  } while (0)
-#endif
-
-static void
-cs_str_buf_terminate(VALUE str, rb_encoding *enc)
-{
-  char *ptr;
-  long len;
-
-  ptr = RSTRING(str)->as.heap.ptr;
-  len = RSTRING(str)->as.heap.len;
-  TERM_FILL(ptr + len, rb_enc_mbminlen(enc));
-}
-
+// partially based on rb_str_delete_bang
 static inline VALUE
 cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
 {
   cs_ar *cps;
-  cs_cp len;
-  rb_encoding *str_enc;
-  VALUE orig_len, new_str_buf;
-  int cp_len;
-  unsigned int str_cp;
-  const char *ptr, *end;
+  cs_cp cs_len;
+  VALUE orig_str_len;
+
+  rb_encoding *orig_enc, *utf8;
+  char *s, *send, *t;
+  int orig_was_utf8, cr;
 
   raise_arg_err_unless_string(str);
 
-  cps = cs_fetch_cps(set, &len);
+  orig_str_len = RSTRING_LEN(str);
+
+  if (orig_str_len == 0)
+  {
+    return bang ? Qnil : str;
+  }
 
-  orig_len = RSTRING_LEN(str);
-  if (orig_len < 1) // empty string, will never change
+  orig_enc = rb_enc_get(str);
+  utf8 = rb_utf8_encoding();
+  orig_was_utf8 = orig_enc == utf8;
+
+  if (!orig_was_utf8 && orig_enc != rb_usascii_encoding())
+  {
+    str = rb_str_encode(str, rb_enc_from_encoding(utf8), 0, Qnil);
+  }
+  else
   {
-    if (bang)
+    if (!bang)
     {
-      return Qnil;
+      str = rb_str_dup(str);
     }
-    return rb_str_dup(str);
   }
 
-  new_str_buf = rb_str_buf_new(orig_len + 30); // len + margin
-  str_enc = rb_enc_get(str);
-  rb_enc_associate(new_str_buf, str_enc);
-  rb_str_modify(new_str_buf);
-  ENC_CODERANGE_SET(new_str_buf, rb_enc_asciicompat(str_enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID);
+  cps = cs_fetch_cps(set, &cs_len);
+  rb_str_modify(str);
+  s = t = RSTRING_PTR(str);
+  send = RSTRING_END(str);
+  cr = ENC_CODERANGE_7BIT;
 
-  ptr = RSTRING_PTR(str);
-  end = RSTRING_END(str);
-
-  if (single_byte_optimizable(str))
+  while (s < send)
   {
-    while (ptr < end)
+    unsigned int c;
+    int clen;
+
+    if ((c = *(unsigned char *)s) < 0x80)
     {
-      str_cp = *ptr & 0xff;
-      if ((!tst_cp(cps, len, str_cp)) == delete)
+      if (tst_cp(cps, cs_len, c) != delete)
       {
-        cs_str_buf_cat(new_str_buf, ptr, 1);
+        if (t != s)
+          *t = c;
+        t++;
       }
-      ptr++;
+      s++;
     }
-  }
-  else // likely to be multibyte string
-  {
-    while (ptr < end)
+    else
     {
-      str_cp = rb_enc_codepoint_len(ptr, end, &cp_len, str_enc);
-      if ((!tst_cp(cps, len, str_cp)) == delete)
+      c = rb_enc_codepoint_len(s, send, &clen, utf8);
+
+      if (tst_cp(cps, cs_len, c) != delete)
       {
-        cs_str_buf_cat(new_str_buf, ptr, cp_len);
+        if (t != s)
+          rb_enc_mbcput(c, t, utf8);
+        t += clen;
+        if (cr == ENC_CODERANGE_7BIT)
+          cr = ENC_CODERANGE_VALID;
       }
-      ptr += cp_len;
+      s += clen;
     }
   }
 
-  cs_str_buf_terminate(new_str_buf, str_enc);
+  rb_str_set_len(str, t - RSTRING_PTR(str));
+  ENC_CODERANGE_SET(str, cr);
 
-  if (bang)
+  if (bang && (RSTRING_LEN(str) == (long)orig_str_len)) // string unchanged
   {
-    if (RSTRING_LEN(new_str_buf) == (long)orig_len) // string unchanged
-    {
-      return Qnil;
-    }
-    rb_str_shared_replace(str, new_str_buf);
+    return Qnil;
   }
-  else
+
+  if (!orig_was_utf8)
   {
-    RB_OBJ_WRITE(new_str_buf, &(RBASIC(new_str_buf))->klass, rb_obj_class(str));
-    str = new_str_buf;
+    return rb_str_encode(str, rb_enc_from_encoding(orig_enc), 0, Qnil);
   }
 
   return str;
@@ -1289,6 +1270,10 @@ cs_method_allocated_length(VALUE self)
 
 void Init_character_set()
 {
+#ifdef HAVE_RB_EXT_RACTOR_SAFE
+  rb_ext_ractor_safe(true);
+#endif
+
   VALUE cs = rb_define_class("CharacterSet", rb_cObject);
 
   rb_define_alloc_func(cs, cs_method_allocate);
@@ -1343,7 +1328,7 @@ void Init_character_set()
   // `CharacterSet`-specific methods
 
   rb_define_singleton_method(cs, "from_ranges", cs_class_method_from_ranges, -2);
-  rb_define_singleton_method(cs, "of", cs_class_method_of, -1);
+  rb_define_singleton_method(cs, "of_string", cs_class_method_of_string, 1);
 
   rb_define_method(cs, "ranges", cs_method_ranges, 0);
   rb_define_method(cs, "sample", cs_method_sample, -1);

data/lib/character_set/core_ext/string_ext.rb

@@ -2,7 +2,7 @@ class CharacterSet
   module CoreExt
     module StringExt
       def character_set
-        CharacterSet.of(self)
+        CharacterSet.of_string(self)
       end
 
       {

data/lib/character_set/parser.rb

@@ -4,11 +4,15 @@ class CharacterSet
 
     def codepoints_from_enumerable(object)
       raise ArgumentError, 'pass an Enumerable' unless object.respond_to?(:each)
+
       # Use #each to check first element (only this works for all Enumerables)
-      object.each do |e| # rubocop:disable Lint/UnreachableLoop
-        return object            if e.is_a?(Integer) && e >= 0 && e < 0x110000
-        return object.map(&:ord) if e.is_a?(String)  && e.length == 1
-        raise ArgumentError, "#{e.inspect} is not valid as a codepoint"
+      object.each do |el| # rubocop:disable Lint/UnreachableLoop
+        if el.is_a?(Integer) && el >= 0 && el < 0x110000
+          return object
+        elsif el.is_a?(String) && el.length == 1
+          return object.to_a.join.encode('utf-8').codepoints
+        end
+        raise ArgumentError, "#{el.inspect} is not valid as a codepoint"
       end
     end