character_set 1.1.1 → 1.4.1

data/benchmarks/count_in.rb

@@ -0,0 +1,13 @@
+require_relative './shared'
+
+str = 'Lorem ipsum et dolorem'
+tr = '^A-Za-z'
+cs = CharacterSet.non_ascii_letter
+
+benchmark(
+  caption: 'Counting non-letters',
+  cases: {
+    'String#count'          => -> { str.count(tr) },
+    'CharacterSet#count_in' => -> { cs.count_in(str) },
+  }
+)

data/benchmarks/delete_in.rb

@@ -14,7 +14,7 @@ benchmark(
 
 str = 'Lörem ipsüm ⛷ et dölörem'
 rx = /[\s\p{emoji}äüö]/
-cs = CharacterSet.whitespace + CharacterSet.emoji + CS['ä', 'ü', 'ö']
+cs = CharacterSet.whitespace + CharacterSet.emoji + CharacterSet['ä', 'ö', 'ü']
 
 benchmark(
   caption: 'Removing whitespace, emoji and umlauts',

data/benchmarks/scan.rb

@@ -0,0 +1,13 @@
+require_relative './shared'
+
+str = 'Lorem ipsum ⛷ et dolorem'
+rx = /\p{emoji}/
+cs = CharacterSet.emoji
+
+benchmark(
+  caption: 'Extracting emoji to an Array',
+  cases: {
+    'String#scan'       => -> { str.scan(rx) },
+    'CharacterSet#scan' => -> { cs.scan(str) },
+  }
+)

data/benchmarks/shared.rb

@@ -3,6 +3,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require 'benchmark/ips'
 require 'character_set'
+if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+  require 'sorted_set'
+else
+  require 'set'
+end
 
 def benchmark(caption: nil, cases: {})
   puts caption

data/benchmarks/z_add.rb

@@ -0,0 +1,12 @@
+require_relative './shared'
+
+cs = CharacterSet[]
+ss = SortedSet[]
+
+benchmark(
+  caption: 'Adding entries',
+  cases: {
+    'CharacterSet#add' => -> { cs.add(rand(0x10FFFF)) },
+    'SortedSet#add'    => -> { ss.add(rand(0x10FFFF)) },
+  }
+)

data/benchmarks/z_delete.rb

@@ -0,0 +1,12 @@
+require_relative './shared'
+
+cs = CharacterSet.new(0..0x10FFFF)
+ss = SortedSet.new(0..0x10FFFF)
+
+benchmark(
+  caption: 'Removing entries',
+  cases: {
+    'CharacterSet#delete' => -> { cs.delete(rand(0x10FFFF)) },
+    'SortedSet#delete'    => -> { ss.delete(rand(0x10FFFF)) },
+  }
+)

data/benchmarks/z_merge.rb

@@ -0,0 +1,15 @@
+require_relative './shared'
+
+cs1 = CharacterSet.new(0...0x88000)
+cs2 = CharacterSet.new(0x88000..0x10FFFF)
+
+ss1 = SortedSet.new(0...0x88000)
+ss2 = SortedSet.new(0x88000..0x10FFFF)
+
+benchmark(
+  caption: 'Merging entries',
+  cases: {
+    'CharacterSet#merge' => -> { cs1.merge(cs2) },
+    'SortedSet#merge'    => -> { ss1.merge(ss2) },
+  }
+)

data/benchmarks/z_minmax.rb

@@ -0,0 +1,12 @@
+require_relative './shared'
+
+cs = CharacterSet.new(0..0xFFFF)
+ss = SortedSet.new(0..0xFFFF)
+
+benchmark(
+  caption: 'Getting the min and max',
+  cases: {
+    'CharacterSet#minmax' => -> { cs.minmax },
+    'SortedSet#minmax'    => -> { ss.minmax },
+  }
+)

data/bin/console

@@ -2,6 +2,8 @@
 
 require 'bundler/setup'
 
+`bundle exec rake compile`
+
 require 'character_set'
 require 'character_set/core_ext'
 require 'character_set/pure'

data/character_set.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.email         = ['janosch84@gmail.com']
 
   s.summary       = 'Build, read, write and compare sets of Unicode codepoints.'
-  s.homepage      = 'https://github.com/janosch-x/character_set'
+  s.homepage      = 'https://github.com/jaynetics/character_set'
   s.license       = 'MIT'
 
   s.files         = `git ls-files -z`.split("\x0").reject do |f|
@@ -22,12 +22,23 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = '>= 2.1.0'
 
+  # SortedSet, needed for RubyFallback, was moved to a gem in Ruby 3.
+  # This dependency is only used if the C extension is unavailable.
+  # JRuby has it in the stdlib.
+  if RUBY_VERSION.to_f >= 3.0 && !RUBY_PLATFORM[/java/i]
+    s.add_dependency 'sorted_set', '~> 1.0'
+  end
+
   s.add_development_dependency 'benchmark-ips', '~> 2.7'
-  s.add_development_dependency 'bundler', '~> 1.16'
-  s.add_development_dependency 'rake', '~> 12.0'
-  s.add_development_dependency 'rake-compiler', '~> 1.0'
+  s.add_development_dependency 'get_process_mem', '~> 0.2.3'
+  s.add_development_dependency 'rake', '~> 13.0'
+  s.add_development_dependency 'rake-compiler', '~> 1.1'
   s.add_development_dependency 'range_compressor', '~> 1.0'
-  s.add_development_dependency 'regexp_parser', '~> 1.1'
-  s.add_development_dependency 'regexp_property_values', '~> 0.3.4'
+  s.add_development_dependency 'regexp_parser', '~> 1.6'
+  s.add_development_dependency 'regexp_property_values', '~> 1.0'
   s.add_development_dependency 'rspec', '~> 3.8'
+  if RUBY_VERSION.to_f >= 2.7
+    s.add_development_dependency 'codecov', '~> 0.2.12'
+    s.add_development_dependency 'rubocop', '~> 1.8'
+  end
 end

data/ext/character_set/character_set.c

@@ -2,81 +2,180 @@
 #include "ruby/encoding.h"
 #include "unicode_casefold_table.h"
 
-#define SETBIT(byte_arr, bit) (byte_arr[bit >> 3] |=  (1 << (bit & 0x07)))
-#define CLRBIT(byte_arr, bit) (byte_arr[bit >> 3] &= ~(1 << (bit & 0x07)))
-#define TSTBIT(byte_arr, bit) (byte_arr[bit >> 3] &   (1 << (bit & 0x07)))
+#define UNICODE_PLANE_SIZE 0x10000
+#define UNICODE_PLANE_COUNT 17
+#define UNICODE_CP_COUNT (UNICODE_PLANE_SIZE * UNICODE_PLANE_COUNT)
 
-typedef char cp_byte;
-typedef unsigned long cp_index;
+// start at ascii size
+#define CS_DEFAULT_INITIAL_LEN 128
 
-#define UNICODE_CP_COUNT    0x110000
-#define UNICODE_BYTES       UNICODE_CP_COUNT / 8
-#define UNICODE_PLANE_SIZE  0x10000
-#define UNICODE_PLANE_COUNT UNICODE_CP_COUNT / UNICODE_PLANE_SIZE
+typedef char cs_ar;
+typedef unsigned long cs_cp;
+
+struct cs_data
+{
+  cs_ar *cps;
+  cs_cp len;
+};
+
+#define CS_MSIZE(len) (sizeof(cs_ar) * (len / 8))
+
+static inline void
+add_memspace_for_another_plane(struct cs_data *data)
+{
+  data->cps = ruby_xrealloc(data->cps, CS_MSIZE(data->len + UNICODE_PLANE_SIZE));
+  memset(data->cps + CS_MSIZE(data->len), 0, CS_MSIZE(UNICODE_PLANE_SIZE));
+  data->len += UNICODE_PLANE_SIZE;
+}
+
+static inline void
+ensure_memsize_fits(struct cs_data *data, cs_cp target_cp)
+{
+  while (target_cp >= data->len)
+  {
+    add_memspace_for_another_plane(data);
+  }
+}
+
+static inline void
+set_cp(struct cs_data *data, cs_cp cp)
+{
+  ensure_memsize_fits(data, cp);
+  data->cps[cp >> 3] |= (1 << (cp & 0x07));
+}
+
+static inline int
+tst_cp(cs_ar *cps, cs_cp len, cs_cp cp)
+{
+  return ((cp < len) && cps[cp >> 3] & (1 << (cp & 0x07)));
+}
+
+static inline void
+clr_cp(cs_ar *cps, cs_cp len, cs_cp cp)
+{
+  if (cp < len)
+  {
+    cps[cp >> 3] &= ~(1 << (cp & 0x07));
+  }
+}
 
 static void
-free_character_set(void* codepoints) {
-  free(codepoints);
+cs_free(void *ptr)
+{
+  struct cs_data *data = ptr;
+  ruby_xfree(data->cps);
+  ruby_xfree(data);
 }
 
 static size_t
-memsize_character_set(const void* codepoints) {
-  return sizeof(cp_byte) * UNICODE_BYTES;
-}
-
-static const rb_data_type_t
-character_set_type = {
-  .wrap_struct_name = "character_set",
-  .function = {
-    .dmark = NULL,
-    .dfree = free_character_set,
-    .dsize = memsize_character_set,
-  },
-  .data = NULL,
-  .flags = RUBY_TYPED_FREE_IMMEDIATELY,
+cs_memsize(const void *ptr)
+{
+  const struct cs_data *data = ptr;
+  return sizeof(*data) + CS_MSIZE(data->len);
+}
+
+static const rb_data_type_t cs_type = {
+    .wrap_struct_name = "character_set",
+    .function = {
+        .dmark = NULL,
+        .dfree = cs_free,
+        .dsize = cs_memsize,
+    },
+    .data = NULL,
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY,
 };
 
-#define FETCH_CODEPOINTS(set, cps)\
-  TypedData_Get_Struct(set, cp_byte, &character_set_type, cps)
+static inline VALUE
+cs_alloc_len(VALUE klass, struct cs_data **data_ptr, cs_cp len)
+{
+  VALUE cs;
+  struct cs_data *data;
+  cs = TypedData_Make_Struct(klass, struct cs_data, &cs_type, data);
+  data->cps = ruby_xmalloc(CS_MSIZE(len));
+  memset(data->cps, 0, CS_MSIZE(len));
+  data->len = len;
+
+  if (data_ptr)
+  {
+    *data_ptr = data;
+  }
 
-#define NEW_CHARACTER_SET(klass, cps)\
-  TypedData_Wrap_Struct(klass, &character_set_type, cps)
+  return cs;
+}
 
-static VALUE
-method_allocate(VALUE self) {
-  cp_byte *cp_arr;
-  cp_arr = calloc(UNICODE_BYTES, sizeof(cp_byte));
-  return NEW_CHARACTER_SET(self, cp_arr);
+static inline VALUE
+cs_alloc(VALUE klass, struct cs_data **data_ptr)
+{
+  return cs_alloc_len(klass, data_ptr, CS_DEFAULT_INITIAL_LEN);
 }
 
-#define FOR_EACH_ACTIVE_CODEPOINT(action)\
-  cp_index cp;\
-  cp_byte *cps;\
-  FETCH_CODEPOINTS(self, cps);\
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) {\
-    if (TSTBIT(cps, cp)) { action; }\
-  }
+static inline struct cs_data *
+cs_fetch_data(VALUE cs)
+{
+  struct cs_data *data;
+  TypedData_Get_Struct(cs, struct cs_data, &cs_type, data);
+  return data;
+}
+
+static inline cs_ar *
+cs_fetch_cps(VALUE cs, cs_cp *len_ptr)
+{
+  struct cs_data *data;
+  data = cs_fetch_data(cs);
+  *len_ptr = data->len;
+  return data->cps;
+}
+
+static VALUE
+cs_method_allocate(VALUE self)
+{
+  return cs_alloc(self, 0);
+}
+
+#define FOR_EACH_ACTIVE_CODEPOINT(action) \
+  do                                      \
+  {                                       \
+    cs_cp cp, len;                        \
+    cs_ar *cps;                           \
+    cps = cs_fetch_cps(self, &len);       \
+    for (cp = 0; cp < len; cp++)          \
+    {                                     \
+      if (tst_cp(cps, len, cp))           \
+      {                                   \
+        action;                           \
+      }                                   \
+    }                                     \
+  } while (0)
 
 // ***************************
 // `Set` compatibility methods
 // ***************************
 
-static inline VALUE
-enumerator_length(VALUE self, VALUE args, VALUE eobj) {
-  cp_index count;
+static inline cs_cp
+cs_active_cp_count(VALUE self)
+{
+  cs_cp count;
   count = 0;
   FOR_EACH_ACTIVE_CODEPOINT(count++);
-  return LONG2FIX(count);
+  return count;
 }
 
 static VALUE
-method_length(VALUE self) {
-  return enumerator_length(self, 0, 0);
+cs_method_length(VALUE self)
+{
+  return LONG2FIX(cs_active_cp_count(self));
+}
+
+static inline VALUE
+cs_enumerator_length(VALUE self, VALUE args, VALUE eobj)
+{
+  return LONG2FIX(cs_active_cp_count(self));
 }
 
 static VALUE
-method_each(VALUE self) {
-  RETURN_SIZED_ENUMERATOR(self, 0, 0, enumerator_length);
+cs_method_each(VALUE self)
+{
+  RETURN_SIZED_ENUMERATOR(self, 0, 0, cs_enumerator_length);
   FOR_EACH_ACTIVE_CODEPOINT(rb_yield(LONG2FIX(cp)));
   return self;
 }
@@ -84,16 +183,19 @@ method_each(VALUE self) {
 // returns an Array of codepoint Integers by default.
 // returns an Array of Strings of length 1 if passed `true`.
 static VALUE
-method_to_a(int argc, VALUE *argv, VALUE self) {
+cs_method_to_a(int argc, VALUE *argv, VALUE self)
+{
   VALUE arr;
   rb_encoding *enc;
   rb_check_arity(argc, 0, 1);
 
   arr = rb_ary_new();
-  if (!argc || NIL_P(argv[0]) || argv[0] == Qfalse) {
+  if (!argc || NIL_P(argv[0]) || argv[0] == Qfalse)
+  {
     FOR_EACH_ACTIVE_CODEPOINT(rb_ary_push(arr, LONG2FIX(cp)));
   }
-  else {
+  else
+  {
     enc = rb_utf8_encoding();
     FOR_EACH_ACTIVE_CODEPOINT(rb_ary_push(arr, rb_enc_uint_chr((int)cp, enc)));
   }
@@ -102,302 +204,473 @@ method_to_a(int argc, VALUE *argv, VALUE self) {
 }
 
 static VALUE
-method_empty_p(VALUE self) {
+cs_method_empty_p(VALUE self)
+{
   FOR_EACH_ACTIVE_CODEPOINT(return Qfalse);
   return Qtrue;
 }
 
 static VALUE
-method_hash(VALUE self) {
-  cp_index cp, hash, four_byte_value;
-  cp_byte *cps;
-  FETCH_CODEPOINTS(self, cps);
+cs_method_hash(VALUE self)
+{
+  cs_cp cp, len, hash, four_byte_value;
+  cs_ar *cps;
+  cps = cs_fetch_cps(self, &len);
+  four_byte_value = 0;
 
   hash = 17;
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) {
-    if (cp % 32 == 0) {
-      if (cp != 0) { hash = hash * 23 + four_byte_value; }
+  for (cp = 0; cp < len; cp++)
+  {
+    if (cp % 32 == 0)
+    {
+      if (cp != 0)
+      {
+        hash = hash * 23 + four_byte_value;
+      }
       four_byte_value = 0;
     }
-    if (TSTBIT(cps, cp)) four_byte_value++;
+    if (tst_cp(cps, len, cp))
+    {
+      four_byte_value++;
+    }
   }
 
   return LONG2FIX(hash);
 }
 
 static inline VALUE
-delete_if_block_result(VALUE self, int truthy) {
+cs_delete_if_block_result(VALUE self, int truthy)
+{
   VALUE result;
   rb_need_block();
   rb_check_frozen(self);
   FOR_EACH_ACTIVE_CODEPOINT(
-    result = rb_yield(LONG2FIX(cp));
-    if ((NIL_P(result) || result == Qfalse) != truthy) CLRBIT(cps, cp);
-  );
+      result = rb_yield(LONG2FIX(cp));
+      if ((NIL_P(result) || result == Qfalse) != truthy) clr_cp(cps, len, cp););
   return self;
 }
 
 static VALUE
-method_delete_if(VALUE self) {
-  RETURN_SIZED_ENUMERATOR(self, 0, 0, enumerator_length);
-  return delete_if_block_result(self, 1);
+cs_method_delete_if(VALUE self)
+{
+  RETURN_SIZED_ENUMERATOR(self, 0, 0, cs_enumerator_length);
+  return cs_delete_if_block_result(self, 1);
 }
 
 static VALUE
-method_keep_if(VALUE self) {
-  RETURN_SIZED_ENUMERATOR(self, 0, 0, enumerator_length);
-  return delete_if_block_result(self, 0);
+cs_method_keep_if(VALUE self)
+{
+  RETURN_SIZED_ENUMERATOR(self, 0, 0, cs_enumerator_length);
+  return cs_delete_if_block_result(self, 0);
 }
 
 static VALUE
-method_clear(VALUE self) {
-  cp_index cp;
-  cp_byte *cps;
+cs_method_clear(VALUE self)
+{
+  struct cs_data *data;
   rb_check_frozen(self);
-  FETCH_CODEPOINTS(self, cps);
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) {
-    CLRBIT(cps, cp);
-  }
+  data = cs_fetch_data(self);
+  memset(data->cps, 0, CS_MSIZE(data->len));
   return self;
 }
 
-#define RETURN_NEW_SET_BASED_ON(condition)\
-  cp_index cp;\
-  cp_byte *a, *b, *new_cps;\
-  FETCH_CODEPOINTS(self, a);\
-  if (other) FETCH_CODEPOINTS(other, b);\
-  new_cps = calloc(UNICODE_BYTES, sizeof(cp_byte));\
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) {\
-    if (condition) SETBIT(new_cps, cp);\
-  }\
-  return NEW_CHARACTER_SET(RBASIC(self)->klass, new_cps);\
+static VALUE
+cs_method_min(VALUE self)
+{
+  FOR_EACH_ACTIVE_CODEPOINT(return LONG2FIX(cp));
+  return Qnil;
+}
+
+static VALUE
+cs_method_max(VALUE self)
+{
+  cs_cp len;
+  long reverse_idx;
+  cs_ar *cps;
+  cps = cs_fetch_cps(self, &len);
+  for (reverse_idx = len; reverse_idx >= 0; reverse_idx--)
+  {
+    if (tst_cp(cps, len, reverse_idx))
+    {
+      return LONG2FIX(reverse_idx);
+    }
+  }
+  return Qnil;
+}
+
+static VALUE
+cs_method_minmax(VALUE self)
+{
+  VALUE arr;
+  arr = rb_ary_new2(2);
+  rb_ary_push(arr, cs_method_min(self));
+  rb_ary_push(arr, cs_method_max(self));
+  return arr;
+}
+
+#define RETURN_COMBINED_CS(cs_a, cs_b, comp_op)                  \
+  do                                                             \
+  {                                                              \
+    VALUE new_cs;                                                \
+    cs_cp cp, alen, blen;                                        \
+    cs_ar *acps, *bcps;                                          \
+    struct cs_data *new_data;                                    \
+    new_cs = cs_alloc(RBASIC(self)->klass, &new_data);           \
+    acps = cs_fetch_cps(cs_a, &alen);                            \
+    bcps = cs_fetch_cps(cs_b, &blen);                            \
+    for (cp = 0; cp < UNICODE_CP_COUNT; cp++)                    \
+    {                                                            \
+      if (tst_cp(acps, alen, cp) comp_op tst_cp(bcps, blen, cp)) \
+      {                                                          \
+        set_cp(new_data, cp);                                    \
+      }                                                          \
+    }                                                            \
+    return new_cs;                                               \
+  } while (0)
 
 static VALUE
-method_intersection(VALUE self, VALUE other) {
-  RETURN_NEW_SET_BASED_ON(TSTBIT(a, cp) && TSTBIT(b, cp));
+cs_method_intersection(VALUE self, VALUE other)
+{
+  RETURN_COMBINED_CS(self, other, &&);
 }
 
 static VALUE
-method_exclusion(VALUE self, VALUE other) {
-  RETURN_NEW_SET_BASED_ON(TSTBIT(a, cp) ^ TSTBIT(b, cp));
+cs_method_exclusion(VALUE self, VALUE other)
+{
+  RETURN_COMBINED_CS(self, other, ^);
 }
 
 static VALUE
-method_union(VALUE self, VALUE other) {
-  RETURN_NEW_SET_BASED_ON(TSTBIT(a, cp) || TSTBIT(b, cp));
+cs_method_union(VALUE self, VALUE other)
+{
+  RETURN_COMBINED_CS(self, other, ||);
 }
 
 static VALUE
-method_difference(VALUE self, VALUE other) {
-  RETURN_NEW_SET_BASED_ON(TSTBIT(a, cp) > TSTBIT(b, cp));
+cs_method_difference(VALUE self, VALUE other)
+{
+  RETURN_COMBINED_CS(self, other, >);
 }
 
 static VALUE
-method_include_p(VALUE self, VALUE num) {
-  cp_byte *cps;
-  FETCH_CODEPOINTS(self, cps);
-  return (TSTBIT(cps, FIX2ULONG(num)) ? Qtrue : Qfalse);
+cs_method_include_p(VALUE self, VALUE num)
+{
+  cs_ar *cps;
+  cs_cp len;
+  cps = cs_fetch_cps(self, &len);
+  return (tst_cp(cps, len, FIX2ULONG(num)) ? Qtrue : Qfalse);
 }
 
-static inline int
-toggle_codepoint(VALUE set, VALUE cp_num, unsigned int on, int check_if_noop) {
-  cp_index cp;
-  cp_byte *cps;
-  rb_check_frozen(set);
-  FETCH_CODEPOINTS(set, cps);
+static inline VALUE
+cs_toggle_codepoint(VALUE cs, VALUE cp_num, int on, int return_nil_if_noop)
+{
+  cs_cp cp, len;
+  cs_ar *cps;
+  struct cs_data *data;
+  rb_check_frozen(cs);
+  data = cs_fetch_data(cs);
+  cps = data->cps;
+  len = data->len;
   cp = FIX2ULONG(cp_num);
-  if (check_if_noop && (!TSTBIT(cps, cp) == !on)) {
-    return 0;
+  if (return_nil_if_noop && (!tst_cp(cps, len, cp) == !on))
+  {
+    return Qnil;
   }
-  else {
-    if (on) { SETBIT(cps, cp); }
-    else    { CLRBIT(cps, cp); }
-    return 1;
+  else
+  {
+    if (on)
+    {
+      set_cp(data, cp);
+    }
+    else
+    {
+      clr_cp(cps, len, cp);
+    }
+    return cs;
   }
 }
 
 static VALUE
-method_add(VALUE self, VALUE cp_num) {
-  return toggle_codepoint(self, cp_num, 1, 0) ? self : Qnil;
+cs_method_add(VALUE self, VALUE cp_num)
+{
+  return cs_toggle_codepoint(self, cp_num, 1, 0);
 }
 
 static VALUE
-method_add_p(VALUE self, VALUE cp_num) {
-  return toggle_codepoint(self, cp_num, 1, 1) ? self : Qnil;
+cs_method_add_p(VALUE self, VALUE cp_num)
+{
+  return cs_toggle_codepoint(self, cp_num, 1, 1);
 }
 
 static VALUE
-method_delete(VALUE self, VALUE cp_num) {
-  return toggle_codepoint(self, cp_num, 0, 0) ? self : Qnil;
+cs_method_delete(VALUE self, VALUE cp_num)
+{
+  return cs_toggle_codepoint(self, cp_num, 0, 0);
 }
 
 static VALUE
-method_delete_p(VALUE self, VALUE cp_num) {
-  return toggle_codepoint(self, cp_num, 0, 1) ? self : Qnil;
+cs_method_delete_p(VALUE self, VALUE cp_num)
+{
+  return cs_toggle_codepoint(self, cp_num, 0, 1);
 }
 
-#define COMPARE_SETS(action)\
-  cp_index cp;\
-  cp_byte *cps, *other_cps;\
-  FETCH_CODEPOINTS(self, cps);\
-  FETCH_CODEPOINTS(other, other_cps);\
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) { action; }\
-
 static VALUE
-method_intersect_p(VALUE self, VALUE other) {
-  COMPARE_SETS(if (TSTBIT(cps, cp) && TSTBIT(other_cps, cp)) return Qtrue);
+cs_method_intersect_p(VALUE self, VALUE other)
+{
+  cs_cp cp, alen, blen;
+  cs_ar *acps, *bcps;
+  acps = cs_fetch_cps(self, &alen);
+  bcps = cs_fetch_cps(other, &blen);
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (tst_cp(acps, alen, cp) && tst_cp(bcps, blen, cp))
+    {
+      return Qtrue;
+    }
+  }
   return Qfalse;
 }
 
 static VALUE
-method_disjoint_p(VALUE self, VALUE other) {
-  return method_intersect_p(self, other) ? Qfalse : Qtrue;
+cs_method_disjoint_p(VALUE self, VALUE other)
+{
+  return cs_method_intersect_p(self, other) ? Qfalse : Qtrue;
 }
 
 static inline int
-is_character_set(VALUE obj) {
-  return rb_typeddata_is_kind_of(obj, &character_set_type);
+cs_check_type(VALUE obj)
+{
+  return rb_typeddata_is_kind_of(obj, &cs_type);
 }
 
 static VALUE
-method_eql_p(VALUE self, VALUE other) {
-  if (!is_character_set(other)) return Qfalse;
-  if (self == other) return Qtrue; // same object_id
-
-  COMPARE_SETS(if (TSTBIT(cps, cp) != TSTBIT(other_cps, cp)) return Qfalse);
-
+cs_cps_eql(VALUE cs_a, VALUE cs_b)
+{
+  cs_cp cp, alen, blen;
+  cs_ar *acps, *bcps;
+  acps = cs_fetch_cps(cs_a, &alen);
+  bcps = cs_fetch_cps(cs_b, &blen);
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (tst_cp(acps, alen, cp) != tst_cp(bcps, blen, cp))
+    {
+      return Qfalse;
+    }
+  }
   return Qtrue;
 }
 
+static VALUE
+cs_method_eql_p(VALUE self, VALUE other)
+{
+  if (!cs_check_type(other))
+  {
+    return Qfalse;
+  }
+  if (self == other) // same object_id
+  {
+    return Qtrue;
+  }
+  return cs_cps_eql(self, other);
+}
+
 static inline VALUE
-merge_character_set(VALUE self, VALUE other) {
-  COMPARE_SETS(if (TSTBIT(other_cps, cp)) SETBIT(cps, cp));
-  return self;
+cs_merge_cs(VALUE recipient, VALUE source)
+{
+  cs_cp cp, source_len;
+  struct cs_data *data;
+  cs_ar *source_cps;
+  data = cs_fetch_data(recipient);
+  source_cps = cs_fetch_cps(source, &source_len);
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (tst_cp(source_cps, source_len, cp))
+    {
+      set_cp(data, cp);
+    }
+  }
+  return recipient;
 }
 
-static inline void
-raise_arg_err_unless_valid_as_cp(VALUE object_id) {
-  if (FIXNUM_P(object_id) && object_id > 0 && object_id < 0x220001) return;
+static inline cs_cp
+cs_checked_cp(VALUE object_id)
+{
+  if (FIXNUM_P(object_id) && object_id > 0 && object_id < 0x220001)
+  {
+    return FIX2ULONG(object_id);
+  }
   rb_raise(rb_eArgError, "CharacterSet members must be between 0 and 0x10FFFF");
 }
 
 static inline VALUE
-merge_rb_range(VALUE self, VALUE rb_range) {
+cs_merge_rb_range(VALUE self, VALUE rb_range)
+{
   VALUE from_id, upto_id;
+  cs_cp from_cp, upto_cp, cont_len, rem;
   int excl;
-  cp_index cp;
-  cp_byte *cps;
-  FETCH_CODEPOINTS(self, cps);
+  struct cs_data *data;
+  data = cs_fetch_data(self);
 
-  if (!RTEST(rb_range_values(rb_range, &from_id, &upto_id, &excl))) {
+  if (!RTEST(rb_range_values(rb_range, &from_id, &upto_id, &excl)))
+  {
     rb_raise(rb_eArgError, "pass a Range");
   }
-  if (excl) upto_id -= 2;
+  if (excl)
+  {
+    upto_id -= 2;
+  }
+
+  from_cp = cs_checked_cp(from_id);
+  upto_cp = cs_checked_cp(upto_id);
 
-  raise_arg_err_unless_valid_as_cp(from_id);
-  raise_arg_err_unless_valid_as_cp(upto_id);
+  if (upto_cp > from_cp && (upto_cp - from_cp > 6))
+  {
+    // set bits in preceding partially toggled bytes individually
+    for (/* */; (from_cp <= upto_cp) && (from_cp % 8); from_cp++)
+    {
+      set_cp(data, from_cp);
+    }
+    // memset contiguous bits directly
+    cont_len = upto_cp - from_cp + 1;
+    rem = cont_len % 8;
+    ensure_memsize_fits(data, upto_cp);
+    memset(data->cps + CS_MSIZE(from_cp), 0xFF, CS_MSIZE(cont_len - rem) / 8);
+    from_cp = upto_cp - rem + 1;
+  }
 
-  for (/* */; from_id <= upto_id; from_id += 2) {
-    cp = FIX2ULONG(from_id);
-    SETBIT(cps, cp);
+  // set bits in partially toggled bytes individually
+  for (/* */; from_cp <= upto_cp; from_cp++)
+  {
+    set_cp(data, from_cp);
   }
+
   return self;
 }
 
 static inline VALUE
-merge_rb_array(VALUE self, VALUE rb_array) {
-  VALUE el;
-  cp_byte *cps;
-  VALUE array_length, i;
-  FETCH_CODEPOINTS(self, cps);
+cs_merge_rb_array(VALUE self, VALUE rb_array)
+{
+  VALUE el, array_length, i;
+  struct cs_data *data;
   Check_Type(rb_array, T_ARRAY);
+  data = cs_fetch_data(self);
   array_length = RARRAY_LEN(rb_array);
-  for (i = 0; i < array_length; i++) {
+  for (i = 0; i < array_length; i++)
+  {
     el = RARRAY_AREF(rb_array, i);
-    raise_arg_err_unless_valid_as_cp(el);
-    SETBIT(cps, FIX2ULONG(el));
+    set_cp(data, cs_checked_cp(el));
   }
   return self;
 }
 
 static VALUE
-method_merge(VALUE self, VALUE other) {
+cs_method_merge(VALUE self, VALUE other)
+{
   rb_check_frozen(self);
-  if (is_character_set(other)) {
-    return merge_character_set(self, other);
+  if (cs_check_type(other))
+  {
+    return cs_merge_cs(self, other);
   }
-  else if (TYPE(other) == T_ARRAY) {
-    return merge_rb_array(self, other);
+  else if (TYPE(other) == T_ARRAY)
+  {
+    return cs_merge_rb_array(self, other);
   }
-  return merge_rb_range(self, other);
+  return cs_merge_rb_range(self, other);
 }
 
 static VALUE
-method_initialize_copy(VALUE self, VALUE other) {
-  merge_character_set(self, other);
-  return other;
+cs_method_initialize_copy(VALUE self, VALUE orig)
+{
+  cs_merge_cs(self, orig);
+  return self;
 }
 
 static VALUE
-method_subtract(VALUE self, VALUE other) {
+cs_method_subtract(VALUE self, VALUE other)
+{
+  cs_cp cp, len, other_len;
+  cs_ar *cps, *other_cps;
   rb_check_frozen(self);
-  COMPARE_SETS(if (TSTBIT(other_cps, cp)) CLRBIT(cps, cp));
+  cps = cs_fetch_cps(self, &len);
+  other_cps = cs_fetch_cps(other, &other_len);
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (tst_cp(other_cps, other_len, cp))
+    {
+      clr_cp(cps, len, cp);
+    }
+  }
   return self;
 }
 
 static inline int
-a_subset_of_b(VALUE set_a, VALUE set_b, int *is_proper) {
-  cp_byte *cps_a, *cps_b;
-  cp_index cp, size_a, size_b;
+cs_a_subset_of_b(VALUE cs_a, VALUE cs_b, int *is_proper_ptr)
+{
+  cs_ar *a, *b;
+  cs_cp cp, alen, blen, count_a, count_b;
 
-  if (!is_character_set(set_a) || !is_character_set(set_b)) {
+  if (!cs_check_type(cs_a) || !cs_check_type(cs_b))
+  {
     rb_raise(rb_eArgError, "pass a CharacterSet");
   }
 
-  FETCH_CODEPOINTS(set_a, cps_a);
-  FETCH_CODEPOINTS(set_b, cps_b);
-
-  *is_proper = 0;
-  size_a = 0;
-  size_b = 0;
-
-  for (cp = 0; cp < UNICODE_CP_COUNT; cp++) {
-    if (TSTBIT(cps_a, cp)) {
-      if (!TSTBIT(cps_b, cp)) return 0;
-      size_a++;
-      size_b++;
+  a = cs_fetch_cps(cs_a, &alen);
+  b = cs_fetch_cps(cs_b, &blen);
+
+  count_a = 0;
+  count_b = 0;
+
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (tst_cp(a, alen, cp))
+    {
+      if (!tst_cp(b, blen, cp))
+      {
+        return 0;
+      }
+      count_a++;
+      count_b++;
+    }
+    else if (tst_cp(b, blen, cp))
+    {
+      count_b++;
     }
-    else if (TSTBIT(cps_b, cp)) size_b++;
   }
 
-  if (size_b > size_a) *is_proper = 1;
+  if (is_proper_ptr)
+  {
+    *is_proper_ptr = count_b > count_a;
+  }
+
   return 1;
 }
 
 static VALUE
-method_subset_p(VALUE self, VALUE other) {
-  int is_proper;
-  return a_subset_of_b(self, other, &is_proper) ? Qtrue : Qfalse;
+cs_method_subset_p(VALUE self, VALUE other)
+{
+  return cs_a_subset_of_b(self, other, NULL) ? Qtrue : Qfalse;
 }
 
 static VALUE
-method_proper_subset_p(VALUE self, VALUE other) {
-  int is, is_proper;
-  is = a_subset_of_b(self, other, &is_proper);
-  return (is && is_proper) ? Qtrue : Qfalse;
+cs_method_proper_subset_p(VALUE self, VALUE other)
+{
+  int is_subset, is_proper;
+  is_subset = cs_a_subset_of_b(self, other, &is_proper);
+  return (is_subset && is_proper) ? Qtrue : Qfalse;
 }
 
 static VALUE
-method_superset_p(VALUE self, VALUE other) {
-  int is_proper;
-  return a_subset_of_b(other, self, &is_proper) ? Qtrue : Qfalse;
+cs_method_superset_p(VALUE self, VALUE other)
+{
+  return cs_a_subset_of_b(other, self, NULL) ? Qtrue : Qfalse;
 }
 
 static VALUE
-method_proper_superset_p(VALUE self, VALUE other) {
-  int is, is_proper;
-  is = a_subset_of_b(other, self, &is_proper);
-  return (is && is_proper) ? Qtrue : Qfalse;
+cs_method_proper_superset_p(VALUE self, VALUE other)
+{
+  int is_superset, is_proper;
+  is_superset = cs_a_subset_of_b(other, self, &is_proper);
+  return (is_superset && is_proper) ? Qtrue : Qfalse;
 }
 
 // *******************************
@@ -405,42 +678,43 @@ method_proper_superset_p(VALUE self, VALUE other) {
 // *******************************
 
 static VALUE
-class_method_from_ranges(VALUE self, VALUE ranges) {
-  VALUE new_set, range_count, i;
-  new_set = rb_class_new_instance(0, 0, self);
+cs_class_method_from_ranges(VALUE self, VALUE ranges)
+{
+  VALUE new_cs, range_count, i;
+  new_cs = rb_class_new_instance(0, 0, self);
   range_count = RARRAY_LEN(ranges);
-  for (i = 0; i < range_count; i++) {
-    merge_rb_range(new_set, RARRAY_AREF(ranges, i));
+  for (i = 0; i < range_count; i++)
+  {
+    cs_merge_rb_range(new_cs, RARRAY_AREF(ranges, i));
   }
-  return new_set;
+  return new_cs;
 }
 
 static VALUE
-method_ranges(VALUE self) {
-  VALUE ranges, codepoint, previous_codepoint, current_start, current_end;
+cs_method_ranges(VALUE self)
+{
+  VALUE ranges, cp_num, previous_cp_num, current_start, current_end;
 
   ranges = rb_ary_new();
-  previous_codepoint = 0;
+  previous_cp_num = 0;
   current_start = 0;
   current_end = 0;
 
   FOR_EACH_ACTIVE_CODEPOINT(
-    codepoint = LONG2FIX(cp);
+      cp_num = LONG2FIX(cp);
 
-    if (!previous_codepoint) {
-      current_start = codepoint;
-    }
-    else if (previous_codepoint + 2 != codepoint) {
-      // gap found, finalize previous range
-      rb_ary_push(ranges, rb_range_new(current_start, current_end, 0));
-      current_start = codepoint;
-    }
-    current_end = codepoint;
-    previous_codepoint = codepoint;
-  );
+      if (!previous_cp_num) {
+        current_start = cp_num;
+      } else if (previous_cp_num + 2 != cp_num) {
+        // gap found, finalize previous range
+        rb_ary_push(ranges, rb_range_new(current_start, current_end, 0));
+        current_start = cp_num;
+      } current_end = cp_num;
+      previous_cp_num = cp_num;);
 
   // add final range
-  if (current_start) {
+  if (current_start)
+  {
     rb_ary_push(ranges, rb_range_new(current_start, current_end, 0));
   }
 
@@ -448,117 +722,233 @@ method_ranges(VALUE self) {
 }
 
 static VALUE
-method_sample(int argc, VALUE *argv, VALUE self) {
-  VALUE to_a_args[1], array;
+cs_method_sample(int argc, VALUE *argv, VALUE self)
+{
+  VALUE array, to_a_args[1] = {Qtrue};
   rb_check_arity(argc, 0, 1);
-  to_a_args[0] = Qtrue;
-  array = method_to_a(1, to_a_args, self);
+  array = cs_method_to_a(1, to_a_args, self);
   return rb_funcall(array, rb_intern("sample"), argc, argc ? argv[0] : 0);
 }
 
 static inline VALUE
-new_set_from_section(VALUE set, cp_index from, cp_index upto) {
-  cp_byte *cps, *new_cps;
-  cp_index cp;
-  FETCH_CODEPOINTS(set, cps);
-  new_cps = calloc(UNICODE_BYTES, sizeof(cp_byte));
-  for (cp = from; cp <= upto; cp++) {
-    if (TSTBIT(cps, cp)) SETBIT(new_cps, cp);
+cs_from_section(VALUE set, cs_cp from, cs_cp upto)
+{
+  VALUE new_cs;
+  cs_ar *cps;
+  cs_cp cp, len;
+  struct cs_data *new_data;
+  new_cs = cs_alloc(RBASIC(set)->klass, &new_data);
+  cps = cs_fetch_cps(set, &len);
+  for (cp = from; cp <= upto; cp++)
+  {
+    if (tst_cp(cps, len, cp))
+    {
+      set_cp(new_data, cp);
+    }
   }
-  return NEW_CHARACTER_SET(RBASIC(set)->klass, new_cps);
+  return new_cs;
 }
 
 static VALUE
-method_bmp_part(VALUE self) {
-  return new_set_from_section(self, 0, UNICODE_PLANE_SIZE - 1);
+cs_method_ext_section(VALUE self, VALUE from, VALUE upto)
+{
+  return cs_from_section(self, FIX2ULONG(from), FIX2ULONG(upto));
+}
+
+static inline cs_cp
+cs_active_cp_count_in_section(VALUE set, cs_cp from, cs_cp upto)
+{
+  cs_ar *cps;
+  cs_cp cp, count, len;
+  cps = cs_fetch_cps(set, &len);
+  for (count = 0, cp = from; cp <= upto; cp++)
+  {
+    if (tst_cp(cps, len, cp))
+    {
+      count++;
+    }
+  }
+  return count;
 }
 
 static VALUE
-method_astral_part(VALUE self) {
-  return new_set_from_section(self, UNICODE_PLANE_SIZE, UNICODE_CP_COUNT - 1);
+cs_method_ext_count_in_section(VALUE self, VALUE from, VALUE upto)
+{
+  cs_cp count;
+  count = cs_active_cp_count_in_section(self, FIX2ULONG(from), FIX2ULONG(upto));
+  return LONG2FIX(count);
 }
 
 static inline VALUE
-set_has_member_in_plane(VALUE set, unsigned int plane) {
-  cp_byte *cps;
-  cp_index cp, max_cp;
-  FETCH_CODEPOINTS(set, cps);
-  cp     = plane * UNICODE_PLANE_SIZE;
-  max_cp = (plane + 1) * UNICODE_PLANE_SIZE - 1;
-  for (/* */; cp <= max_cp; cp++) {
-    if (TSTBIT(cps, cp)) return Qtrue;
+cs_has_cp_in_section(cs_ar *cps, cs_cp len, cs_cp from, cs_cp upto)
+{
+  cs_cp cp;
+  for (cp = from; cp <= upto; cp++)
+  {
+    if (tst_cp(cps, len, cp))
+    {
+      return Qtrue;
+    }
   }
   return Qfalse;
 }
 
 static VALUE
-method_planes(VALUE self) {
+cs_method_ext_section_p(VALUE self, VALUE from, VALUE upto)
+{
+  cs_ar *cps;
+  cs_cp len;
+  cps = cs_fetch_cps(self, &len);
+  return cs_has_cp_in_section(cps, len, FIX2ULONG(from), FIX2ULONG(upto));
+}
+
+static inline VALUE
+cs_ratio_of_section(VALUE set, cs_cp from, cs_cp upto)
+{
+  double section_count, total_count;
+  section_count = (double)cs_active_cp_count_in_section(set, from, upto);
+  total_count = (double)cs_active_cp_count(set);
+  return DBL2NUM(section_count / total_count);
+}
+
+static VALUE
+cs_method_ext_section_ratio(VALUE self, VALUE from, VALUE upto)
+{
+  return cs_ratio_of_section(self, FIX2ULONG(from), FIX2ULONG(upto));
+}
+
+#define MAX_CP 0x10FFFF
+#define MAX_ASCII_CP 0x7F
+#define MAX_BMP_CP 0xFFFF
+#define MIN_ASTRAL_CP 0x10000
+
+static inline VALUE
+cs_has_cp_in_plane(cs_ar *cps, cs_cp len, unsigned int plane)
+{
+  cs_cp plane_beg, plane_end;
+  plane_beg = plane * UNICODE_PLANE_SIZE;
+  plane_end = (plane + 1) * MAX_BMP_CP;
+  return cs_has_cp_in_section(cps, len, plane_beg, plane_end);
+}
+
+static VALUE
+cs_method_planes(VALUE self)
+{
+  cs_ar *cps;
+  cs_cp len;
   unsigned int i;
   VALUE planes;
+  cps = cs_fetch_cps(self, &len);
   planes = rb_ary_new();
-  for (i = 0; i < UNICODE_PLANE_COUNT; i++) {
-    if (set_has_member_in_plane(self, i)) rb_ary_push(planes, INT2FIX(i));
+  for (i = 0; i < UNICODE_PLANE_COUNT; i++)
+  {
+    if (cs_has_cp_in_plane(cps, len, i))
+    {
+      rb_ary_push(planes, INT2FIX(i));
+    }
   }
   return planes;
 }
 
-static VALUE
-method_member_in_plane_p(VALUE self, VALUE plane_num) {
+static inline int
+cs_valid_plane_num(VALUE num)
+{
   int plane;
-  Check_Type(plane_num, T_FIXNUM);
-  plane = FIX2INT(plane_num);
-  if (plane < 0 || plane >= UNICODE_PLANE_COUNT) {
-    rb_raise(rb_eArgError, "plane must be between 0 and 16");
+  Check_Type(num, T_FIXNUM);
+  plane = FIX2INT(num);
+  if (plane < 0 || plane >= UNICODE_PLANE_COUNT)
+  {
+    rb_raise(rb_eArgError, "plane must be between 0 and %d", UNICODE_PLANE_COUNT - 1);
   }
-  return set_has_member_in_plane(self, plane);
+  return plane;
+}
+
+static VALUE
+cs_method_plane(VALUE self, VALUE plane_num)
+{
+  cs_cp plane, plane_beg, plane_end;
+  plane = cs_valid_plane_num(plane_num);
+  plane_beg = plane * UNICODE_PLANE_SIZE;
+  plane_end = (plane + 1) * MAX_BMP_CP;
+  return cs_from_section(self, plane_beg, plane_end);
+}
+
+static VALUE
+cs_method_member_in_plane_p(VALUE self, VALUE plane_num)
+{
+  cs_ar *cps;
+  cs_cp len;
+  unsigned int plane;
+  plane = cs_valid_plane_num(plane_num);
+  cps = cs_fetch_cps(self, &len);
+  return cs_has_cp_in_plane(cps, len, plane);
 }
 
 #define NON_SURROGATE(cp) (cp > 0xDFFF || cp < 0xD800)
 
 static VALUE
-method_ext_inversion(int argc, VALUE *argv, VALUE self) {
-  int include_surrogates;
-  cp_index upto;
-  VALUE other;
-  other = 0;
+cs_method_ext_inversion(int argc, VALUE *argv, VALUE self)
+{
+  int inc_surr;
+  cs_cp upto, cp, len;
+  cs_ar *cps;
+  VALUE new_cs;
+  struct cs_data *new_data;
+
   rb_check_arity(argc, 0, 2);
-  include_surrogates = ((argc > 0) && (argv[0] == Qtrue));
-  if ((argc > 1) && FIXNUM_P(argv[1])) {
-    upto = FIX2ULONG(argv[1]);
-    RETURN_NEW_SET_BASED_ON(
-      cp <= upto && !TSTBIT(a, cp) && (include_surrogates || NON_SURROGATE(cp))
-    );
+
+  cps = cs_fetch_cps(self, &len);
+  inc_surr = argc && argv[0] == Qtrue;
+  new_cs = cs_alloc(RBASIC(self)->klass, &new_data);
+  upto = argc > 1 && FIXNUM_P(argv[1]) ? FIX2ULONG(argv[1]) : UNICODE_CP_COUNT;
+
+  for (cp = 0; cp < UNICODE_CP_COUNT; cp++)
+  {
+    if (cp <= upto && !tst_cp(cps, len, cp) && (inc_surr || NON_SURROGATE(cp)))
+    {
+      set_cp(new_data, cp);
+    }
   }
-  RETURN_NEW_SET_BASED_ON(
-    !TSTBIT(a, cp) && (include_surrogates || NON_SURROGATE(cp))
-  );
+
+  return new_cs;
 }
 
-typedef int(*str_cp_handler)(unsigned int, cp_byte*);
+typedef int (*str_cp_handler)(unsigned int, cs_ar *, cs_cp len, struct cs_data *data, VALUE *memo);
 
 static inline int
-add_str_cp_to_arr(unsigned int str_cp, cp_byte *cp_arr) {
-  SETBIT(cp_arr, str_cp);
+add_str_cp_to_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  set_cp(data, str_cp);
   return 1;
 }
 
 static VALUE
-method_case_insensitive(VALUE self) {
-  cp_index i;
-  cp_byte *new_cps;
-
-  new_cps = calloc(UNICODE_BYTES, sizeof(cp_byte));
+cs_method_case_insensitive(VALUE self)
+{
+  cs_cp i, len;
+  cs_ar *cps;
+  VALUE new_cs;
+  struct cs_data *new_data;
 
-  FOR_EACH_ACTIVE_CODEPOINT(SETBIT(new_cps, cp));
+  cps = cs_fetch_cps(self, &len);
+  new_cs = cs_alloc(RBASIC(self)->klass, &new_data);
+  cs_merge_cs(new_cs, self);
 
-  for (i = 0; i < CASEFOLD_COUNT; i++) {
+  for (i = 0; i < CASEFOLD_COUNT; i++)
+  {
     casefold_mapping m = unicode_casefold_table[i];
 
-    if      (TSTBIT(cps, m.from)) { SETBIT(new_cps, m.to); }
-    else if (TSTBIT(cps, m.to)) { SETBIT(new_cps, m.from); }
+    if (tst_cp(cps, len, m.from))
+    {
+      set_cp(new_data, m.to);
+    }
+    else if (tst_cp(cps, len, m.to))
+    {
+      set_cp(new_data, m.from);
+    }
   }
 
-  return NEW_CHARACTER_SET(RBASIC(self)->klass, new_cps);
+  return new_cs;
 
   // OnigCaseFoldType flags;
   // rb_encoding *enc;
@@ -573,20 +963,27 @@ method_case_insensitive(VALUE self) {
 }
 
 static inline VALUE
-each_sb_cp(VALUE str, str_cp_handler func, cp_byte *cp_arr) {
-  long i;
+each_sb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  long i, str_len;
   unsigned int str_cp;
+  str_len = RSTRING_LEN(str);
 
-  for (i = 0; i < RSTRING_LEN(str); i++) {
+  for (i = 0; i < str_len; i++)
+  {
     str_cp = (RSTRING_PTR(str)[i] & 0xff);
-    if (!(*func)(str_cp, cp_arr)) return Qfalse;
+    if (!(*func)(str_cp, cp_arr, len, data, memo))
+    {
+      return Qfalse;
+    }
   }
 
   return Qtrue;
 }
 
 static inline VALUE
-each_mb_cp(VALUE str, str_cp_handler func, cp_byte *cp_arr) {
+each_mb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
   int n;
   unsigned int str_cp;
   const char *ptr, *end;
@@ -597,9 +994,13 @@ each_mb_cp(VALUE str, str_cp_handler func, cp_byte *cp_arr) {
   end = RSTRING_END(str);
   enc = rb_enc_get(str);
 
-  while (ptr < end) {
+  while (ptr < end)
+  {
     str_cp = rb_enc_codepoint_len(ptr, end, &n, enc);
-    if (!(*func)(str_cp, cp_arr)) return Qfalse;
+    if (!(*func)(str_cp, cp_arr, len, data, memo))
+    {
+      return Qfalse;
+    }
     ptr += n;
   }
 
@@ -611,105 +1012,236 @@ static inline int
 single_byte_optimizable(VALUE str)
 {
   rb_encoding *enc;
-  if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) return 1;
+  if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT)
+  {
+    return 1;
+  }
 
   enc = rb_enc_get(str);
-  if (rb_enc_mbmaxlen(enc) == 1) return 1;
+  if (rb_enc_mbmaxlen(enc) == 1)
+  {
+    return 1;
+  }
 
   return 0;
 }
 
 static inline VALUE
-each_cp(VALUE str, str_cp_handler func, cp_byte *cp_arr) {
-  if (single_byte_optimizable(str)) {
-    return each_sb_cp(str, func, cp_arr);
+each_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  if (single_byte_optimizable(str))
+  {
+    return each_sb_cp(str, func, cp_arr, len, data, memo);
   }
-  return each_mb_cp(str, func, cp_arr);
+  return each_mb_cp(str, func, cp_arr, len, data, memo);
 }
 
 static inline void
-raise_arg_err_unless_string(VALUE val) {
-  if (!RB_TYPE_P(val, T_STRING)) rb_raise(rb_eArgError, "pass a String");
+raise_arg_err_unless_string(VALUE val)
+{
+  if (!RB_TYPE_P(val, T_STRING))
+  {
+    rb_raise(rb_eArgError, "pass a String");
+  }
 }
 
 static VALUE
-class_method_of(VALUE self, VALUE str) {
-  cp_byte *cp_arr;
+cs_class_method_of(VALUE self, VALUE str)
+{
+  VALUE new_cs;
+  struct cs_data *new_data;
+  new_cs = cs_alloc(self, &new_data);
   raise_arg_err_unless_string(str);
-  cp_arr = calloc(UNICODE_BYTES, sizeof(cp_byte));
-  each_cp(str, add_str_cp_to_arr, cp_arr);
-  return NEW_CHARACTER_SET(self, cp_arr);
+  each_cp(str, add_str_cp_to_arr, 0, 0, new_data, 0);
+  return new_cs;
 }
 
 static inline int
-str_cp_not_in_arr(unsigned int str_cp, cp_byte *cp_arr) {
-  return !TSTBIT(cp_arr, str_cp);
+count_str_cp(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  if (tst_cp(cp_arr, len, str_cp))
+  {
+    *memo += 1;
+  }
+  return 1;
 }
 
 static VALUE
-method_used_by_p(VALUE self, VALUE str) {
-  cp_byte *cps;
-  VALUE only_uses_other_cps;
+cs_method_count_in(VALUE self, VALUE str)
+{
+  VALUE count;
+  struct cs_data *data;
   raise_arg_err_unless_string(str);
-  FETCH_CODEPOINTS(self, cps);
-  only_uses_other_cps = each_cp(str, str_cp_not_in_arr, cps);
-  return only_uses_other_cps == Qfalse ? Qtrue : Qfalse;
+  data = cs_fetch_data(self);
+  count = 0;
+  each_cp(str, count_str_cp, data->cps, data->len, data, &count);
+  return INT2NUM((int)count);
+}
+
+static inline int
+str_cp_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  return tst_cp(cp_arr, len, str_cp);
+}
+
+static VALUE
+cs_method_cover_p(VALUE self, VALUE str)
+{
+  struct cs_data *data;
+  raise_arg_err_unless_string(str);
+  data = cs_fetch_data(self);
+  return each_cp(str, str_cp_in_arr, data->cps, data->len, data, 0);
+}
+
+static inline int
+add_str_cp_to_str_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  if (tst_cp(cp_arr, len, str_cp))
+  {
+    rb_ary_push(memo[0], rb_enc_uint_chr((int)str_cp, (rb_encoding *)memo[1]));
+  }
+  return 1;
+}
+
+static VALUE
+cs_method_scan(VALUE self, VALUE str)
+{
+  VALUE memo[2];
+  struct cs_data *data;
+  raise_arg_err_unless_string(str);
+  data = cs_fetch_data(self);
+  memo[0] = rb_ary_new();
+  memo[1] = (VALUE)rb_enc_get(str);
+  each_cp(str, add_str_cp_to_str_arr, data->cps, data->len, data, memo);
+  return memo[0];
 }
 
 static inline int
-str_cp_in_arr(unsigned int str_cp, cp_byte *cp_arr) {
-  return TSTBIT(cp_arr, str_cp);
+str_cp_not_in_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data *data, VALUE *memo)
+{
+  return !tst_cp(cp_arr, len, str_cp);
 }
 
 static VALUE
-method_cover_p(VALUE self, VALUE str) {
-  cp_byte *cps;
+cs_method_used_by_p(VALUE self, VALUE str)
+{
+  VALUE only_uses_other_cps;
+  struct cs_data *data;
   raise_arg_err_unless_string(str);
-  FETCH_CODEPOINTS(self, cps);
-  return each_cp(str, str_cp_in_arr, cps);
+  data = cs_fetch_data(self);
+  only_uses_other_cps = each_cp(str, str_cp_not_in_arr, data->cps, data->len, data, 0);
+  return only_uses_other_cps == Qfalse ? Qtrue : Qfalse;
+}
+
+static void
+cs_str_buf_cat(VALUE str, const char *ptr, long len)
+{
+  long total, olen;
+  char *sptr;
+
+  RSTRING_GETMEM(str, sptr, olen);
+  sptr = RSTRING(str)->as.heap.ptr;
+  olen = RSTRING(str)->as.heap.len;
+  total = olen + len;
+  memcpy(sptr + olen, ptr, len);
+  RSTRING(str)->as.heap.len = total;
+}
+
+#ifndef TERM_FILL
+#define TERM_FILL(ptr, termlen)                     \
+  do                                                \
+  {                                                 \
+    char *const term_fill_ptr = (ptr);              \
+    const int term_fill_len = (termlen);            \
+    *term_fill_ptr = '\0';                          \
+    if (__builtin_expect(!!(term_fill_len > 1), 0)) \
+      memset(term_fill_ptr, 0, term_fill_len);      \
+  } while (0)
+#endif
+
+static void
+cs_str_buf_terminate(VALUE str, rb_encoding *enc)
+{
+  char *ptr;
+  long len;
+
+  ptr = RSTRING(str)->as.heap.ptr;
+  len = RSTRING(str)->as.heap.len;
+  TERM_FILL(ptr + len, rb_enc_mbminlen(enc));
 }
 
 static inline VALUE
-apply_to_str(VALUE set, VALUE str, int delete, int bang) {
-  cp_byte *cps;
+cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
+{
+  cs_ar *cps;
+  cs_cp len;
   rb_encoding *str_enc;
-  VALUE orig_len, blen, new_str_buf, chr;
-  int n;
+  VALUE orig_len, new_str_buf;
+  int cp_len;
   unsigned int str_cp;
   const char *ptr, *end;
 
   raise_arg_err_unless_string(str);
 
-  FETCH_CODEPOINTS(set, cps);
+  cps = cs_fetch_cps(set, &len);
 
   orig_len = RSTRING_LEN(str);
-  blen = orig_len + 30; /* len + margin */ // not sure why, copied from string.c
-  new_str_buf = rb_str_buf_new(blen);
+  if (orig_len < 1) // empty string, will never change
+  {
+    if (bang)
+    {
+      return Qnil;
+    }
+    return rb_str_dup(str);
+  }
+
+  new_str_buf = rb_str_buf_new(orig_len + 30); // len + margin
   str_enc = rb_enc_get(str);
   rb_enc_associate(new_str_buf, str_enc);
-  ENC_CODERANGE_SET(new_str_buf, rb_enc_asciicompat(str_enc) ?
-                    ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID);
+  rb_str_modify(new_str_buf);
+  ENC_CODERANGE_SET(new_str_buf, rb_enc_asciicompat(str_enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID);
 
   ptr = RSTRING_PTR(str);
   end = RSTRING_END(str);
 
-  while (ptr < end) {
-    str_cp = rb_enc_codepoint_len(ptr, end, &n, str_enc);
-    if (!TSTBIT(cps, str_cp) != !delete) {
-      chr = rb_enc_uint_chr(str_cp, str_enc);
-      rb_enc_str_buf_cat(new_str_buf, RSTRING_PTR(chr), n, str_enc);
+  if (single_byte_optimizable(str))
+  {
+    while (ptr < end)
+    {
+      str_cp = *ptr & 0xff;
+      if ((!tst_cp(cps, len, str_cp)) == delete)
+      {
+        cs_str_buf_cat(new_str_buf, ptr, 1);
+      }
+      ptr++;
+    }
+  }
+  else // likely to be multibyte string
+  {
+    while (ptr < end)
+    {
+      str_cp = rb_enc_codepoint_len(ptr, end, &cp_len, str_enc);
+      if ((!tst_cp(cps, len, str_cp)) == delete)
+      {
+        cs_str_buf_cat(new_str_buf, ptr, cp_len);
+      }
+      ptr += cp_len;
     }
-    ptr += n;
   }
 
-  if (bang) {
-    if (RSTRING_LEN(new_str_buf) == (long)orig_len) return Qnil; // unchanged
+  cs_str_buf_terminate(new_str_buf, str_enc);
+
+  if (bang)
+  {
+    if (RSTRING_LEN(new_str_buf) == (long)orig_len) // string unchanged
+    {
+      return Qnil;
+    }
     rb_str_shared_replace(str, new_str_buf);
   }
-  else {
+  else
+  {
     RB_OBJ_WRITE(new_str_buf, &(RBASIC(new_str_buf))->klass, rb_obj_class(str));
-    // slightly cumbersome approach needed for compatibility with Ruby < 2.3:
-    RBASIC(new_str_buf)->flags |= (RBASIC(str)->flags&(FL_TAINT));
     str = new_str_buf;
   }
 
@@ -717,98 +1249,115 @@ apply_to_str(VALUE set, VALUE str, int delete, int bang) {
 }
 
 static VALUE
-method_delete_in(VALUE self, VALUE str) {
-  return apply_to_str(self, str, 1, 0);
+cs_method_delete_in(VALUE self, VALUE str)
+{
+  return cs_apply_to_str(self, str, 1, 0);
+}
+
+static VALUE
+cs_method_delete_in_bang(VALUE self, VALUE str)
+{
+  return cs_apply_to_str(self, str, 1, 1);
 }
 
 static VALUE
-method_delete_in_bang(VALUE self, VALUE str) {
-  return apply_to_str(self, str, 1, 1);
+cs_method_keep_in(VALUE self, VALUE str)
+{
+  return cs_apply_to_str(self, str, 0, 0);
 }
 
 static VALUE
-method_keep_in(VALUE self, VALUE str) {
-  return apply_to_str(self, str, 0, 0);
+cs_method_keep_in_bang(VALUE self, VALUE str)
+{
+  return cs_apply_to_str(self, str, 0, 1);
 }
 
 static VALUE
-method_keep_in_bang(VALUE self, VALUE str) {
-  return apply_to_str(self, str, 0, 1);
+cs_method_allocated_length(VALUE self)
+{
+  return LONG2FIX(cs_fetch_data(self)->len);
 }
 
 // ****
 // init
 // ****
 
-void
-Init_character_set()
+void Init_character_set()
 {
   VALUE cs = rb_define_class("CharacterSet", rb_cObject);
 
-  rb_define_alloc_func(cs, method_allocate);
+  rb_define_alloc_func(cs, cs_method_allocate);
 
   // `Set` compatibility methods
 
-  rb_define_method(cs, "each",             method_each, 0);
-  rb_define_method(cs, "to_a",             method_to_a, -1);
-  rb_define_method(cs, "length",           method_length, 0);
-  rb_define_method(cs, "size",             method_length, 0);
-  rb_define_method(cs, "count",            method_length, 0);
-  rb_define_method(cs, "empty?",           method_empty_p, 0);
-  rb_define_method(cs, "hash",             method_hash, 0);
-  rb_define_method(cs, "keep_if",          method_keep_if, 0);
-  rb_define_method(cs, "delete_if",        method_delete_if, 0);
-  rb_define_method(cs, "clear",            method_clear, 0);
-  rb_define_method(cs, "intersection",     method_intersection, 1);
-  rb_define_method(cs, "&",                method_intersection, 1);
-  rb_define_method(cs, "union",            method_union, 1);
-  rb_define_method(cs, "+",                method_union, 1);
-  rb_define_method(cs, "|",                method_union, 1);
-  rb_define_method(cs, "difference",       method_difference, 1);
-  rb_define_method(cs, "-",                method_difference, 1);
-  rb_define_method(cs, "^",                method_exclusion, 1);
-  rb_define_method(cs, "include?",         method_include_p, 1);
-  rb_define_method(cs, "member?",          method_include_p, 1);
-  rb_define_method(cs, "===",              method_include_p, 1);
-  rb_define_method(cs, "add",              method_add, 1);
-  rb_define_method(cs, "<<",               method_add, 1);
-  rb_define_method(cs, "add?",             method_add_p, 1);
-  rb_define_method(cs, "delete",           method_delete, 1);
-  rb_define_method(cs, "delete?",          method_delete_p, 1);
-  rb_define_method(cs, "intersect?",       method_intersect_p, 1);
-  rb_define_method(cs, "disjoint?",        method_disjoint_p, 1);
-  rb_define_method(cs, "eql?",             method_eql_p, 1);
-  rb_define_method(cs, "==",               method_eql_p, 1);
-  rb_define_method(cs, "merge",            method_merge, 1);
-  rb_define_method(cs, "initialize_clone", method_initialize_copy, 1);
-  rb_define_method(cs, "initialize_dup",   method_initialize_copy, 1);
-  rb_define_method(cs, "subtract",         method_subtract, 1);
-  rb_define_method(cs, "subset?",          method_subset_p, 1);
-  rb_define_method(cs, "<=",               method_subset_p, 1);
-  rb_define_method(cs, "proper_subset?",   method_proper_subset_p, 1);
-  rb_define_method(cs, "<",                method_proper_subset_p, 1);
-  rb_define_method(cs, "superset?",        method_superset_p, 1);
-  rb_define_method(cs, ">=",               method_superset_p, 1);
-  rb_define_method(cs, "proper_superset?", method_proper_superset_p, 1);
-  rb_define_method(cs, ">",                method_proper_superset_p, 1);
+  rb_define_method(cs, "each", cs_method_each, 0);
+  rb_define_method(cs, "to_a", cs_method_to_a, -1);
+  rb_define_method(cs, "length", cs_method_length, 0);
+  rb_define_method(cs, "size", cs_method_length, 0);
+  rb_define_method(cs, "empty?", cs_method_empty_p, 0);
+  rb_define_method(cs, "hash", cs_method_hash, 0);
+  rb_define_method(cs, "keep_if", cs_method_keep_if, 0);
+  rb_define_method(cs, "delete_if", cs_method_delete_if, 0);
+  rb_define_method(cs, "clear", cs_method_clear, 0);
+  rb_define_method(cs, "min", cs_method_min, 0);
+  rb_define_method(cs, "max", cs_method_max, 0);
+  rb_define_method(cs, "minmax", cs_method_minmax, 0);
+  rb_define_method(cs, "intersection", cs_method_intersection, 1);
+  rb_define_method(cs, "&", cs_method_intersection, 1);
+  rb_define_method(cs, "union", cs_method_union, 1);
+  rb_define_method(cs, "+", cs_method_union, 1);
+  rb_define_method(cs, "|", cs_method_union, 1);
+  rb_define_method(cs, "difference", cs_method_difference, 1);
+  rb_define_method(cs, "-", cs_method_difference, 1);
+  rb_define_method(cs, "^", cs_method_exclusion, 1);
+  rb_define_method(cs, "include?", cs_method_include_p, 1);
+  rb_define_method(cs, "member?", cs_method_include_p, 1);
+  rb_define_method(cs, "===", cs_method_include_p, 1);
+  rb_define_method(cs, "add", cs_method_add, 1);
+  rb_define_method(cs, "<<", cs_method_add, 1);
+  rb_define_method(cs, "add?", cs_method_add_p, 1);
+  rb_define_method(cs, "delete", cs_method_delete, 1);
+  rb_define_method(cs, "delete?", cs_method_delete_p, 1);
+  rb_define_method(cs, "intersect?", cs_method_intersect_p, 1);
+  rb_define_method(cs, "disjoint?", cs_method_disjoint_p, 1);
+  rb_define_method(cs, "eql?", cs_method_eql_p, 1);
+  rb_define_method(cs, "==", cs_method_eql_p, 1);
+  rb_define_method(cs, "merge", cs_method_merge, 1);
+  rb_define_method(cs, "initialize_clone", cs_method_initialize_copy, 1);
+  rb_define_method(cs, "initialize_dup", cs_method_initialize_copy, 1);
+  rb_define_method(cs, "subtract", cs_method_subtract, 1);
+  rb_define_method(cs, "subset?", cs_method_subset_p, 1);
+  rb_define_method(cs, "<=", cs_method_subset_p, 1);
+  rb_define_method(cs, "proper_subset?", cs_method_proper_subset_p, 1);
+  rb_define_method(cs, "<", cs_method_proper_subset_p, 1);
+  rb_define_method(cs, "superset?", cs_method_superset_p, 1);
+  rb_define_method(cs, ">=", cs_method_superset_p, 1);
+  rb_define_method(cs, "proper_superset?", cs_method_proper_superset_p, 1);
+  rb_define_method(cs, ">", cs_method_proper_superset_p, 1);
 
   // `CharacterSet`-specific methods
 
-  rb_define_singleton_method(cs, "from_ranges", class_method_from_ranges, -2);
-  rb_define_singleton_method(cs, "of",          class_method_of, 1);
-
-  rb_define_method(cs, "ranges",           method_ranges, 0);
-  rb_define_method(cs, "sample",           method_sample, -1);
-  rb_define_method(cs, "bmp_part",         method_bmp_part, 0);
-  rb_define_method(cs, "astral_part",      method_astral_part, 0);
-  rb_define_method(cs, "planes",           method_planes, 0);
-  rb_define_method(cs, "member_in_plane?", method_member_in_plane_p, 1);
-  rb_define_method(cs, "ext_inversion",    method_ext_inversion, -1);
-  rb_define_method(cs, "case_insensitive", method_case_insensitive, 0);
-  rb_define_method(cs, "used_by?",         method_used_by_p, 1);
-  rb_define_method(cs, "cover?",           method_cover_p, 1);
-  rb_define_method(cs, "delete_in",        method_delete_in, 1);
-  rb_define_method(cs, "delete_in!",       method_delete_in_bang, 1);
-  rb_define_method(cs, "keep_in",          method_keep_in, 1);
-  rb_define_method(cs, "keep_in!",         method_keep_in_bang, 1);
+  rb_define_singleton_method(cs, "from_ranges", cs_class_method_from_ranges, -2);
+  rb_define_singleton_method(cs, "of", cs_class_method_of, 1);
+
+  rb_define_method(cs, "ranges", cs_method_ranges, 0);
+  rb_define_method(cs, "sample", cs_method_sample, -1);
+  rb_define_method(cs, "ext_section", cs_method_ext_section, 2);
+  rb_define_method(cs, "ext_count_in_section", cs_method_ext_count_in_section, 2);
+  rb_define_method(cs, "ext_section?", cs_method_ext_section_p, 2);
+  rb_define_method(cs, "ext_section_ratio", cs_method_ext_section_ratio, 2);
+  rb_define_method(cs, "planes", cs_method_planes, 0);
+  rb_define_method(cs, "plane", cs_method_plane, 1);
+  rb_define_method(cs, "member_in_plane?", cs_method_member_in_plane_p, 1);
+  rb_define_method(cs, "ext_inversion", cs_method_ext_inversion, -1);
+  rb_define_method(cs, "case_insensitive", cs_method_case_insensitive, 0);
+  rb_define_method(cs, "count_in", cs_method_count_in, 1);
+  rb_define_method(cs, "cover?", cs_method_cover_p, 1);
+  rb_define_method(cs, "delete_in", cs_method_delete_in, 1);
+  rb_define_method(cs, "delete_in!", cs_method_delete_in_bang, 1);
+  rb_define_method(cs, "keep_in", cs_method_keep_in, 1);
+  rb_define_method(cs, "keep_in!", cs_method_keep_in_bang, 1);
+  rb_define_method(cs, "scan", cs_method_scan, 1);
+  rb_define_method(cs, "used_by?", cs_method_used_by_p, 1);
+  rb_define_method(cs, "allocated_length", cs_method_allocated_length, 0);
 }