character_set 1.7.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 778cea0208adb290e09f454e3f88c021531decede99604cf2b67f48c9ab3bcd8
-  data.tar.gz: 43672f0afce2846bec846791e7445c9614605194967e061b1d9f0be305298be4
+  metadata.gz: da6d7ab7c6a15344346659ce9681d1fc1b34de097964d78f424d8de524c5a268
+  data.tar.gz: 438b26ea6414b47536f7b10ad6b257d5b8f07c1d7609e222d2bb67f9de895443
 SHA512:
-  metadata.gz: 635f9fb21c973b03b9a0556f2b6cf2c608753acb73616aa8681a5bada3418f955ec887164a03fd2add4edae8a60292eb6e4b681d11a8cf33d1083499afe83815
-  data.tar.gz: 6a80f4f7f3f6c2357d84dc71bb5086f273471a8ae5af4e09abc502dc9893da90962d49527b40564de664af0a910f2e99720062dd41ba53427882ecb36e1a40a0
+  metadata.gz: '094d9fd765cba5ac6625af56fe76b6c0ef1c0c5987456a213dc82bef01dde5ceed4379149b681eed6d3f7156b9fa56c1b90bc3e24cacf570a24dc057797a8876'
+  data.tar.gz: 60cac5a36346957f73bee186003d1154fb7281823128d0b16104300025dbe07704724bd87cd166de8f0259e3a513bb9a525356b591078377fa21dd3cddffe369

data/.rubocop.yml

@@ -8,10 +8,13 @@ AllCops:
   RubyInterpreters:
     - ruby
     - rake
-  TargetRubyVersion: 2.5 # really 2.1, but 2.5 is lowest supported by rubocop
+  TargetRubyVersion: 3.0
 
 Lint/AmbiguousOperatorPrecedence:
   Enabled: false
 
 Lint/AmbiguousRegexpLiteral:
   Enabled: false
+
+Metrics:
+  Enabled: false

data/CHANGELOG.md

@@ -6,6 +6,25 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [1.9.0] - 2026-10-28
+
+The minimum ruby version is now 3.0 (3.1 in jruby).
+
+### Added
+
+- fast Java backend based on `BitSet` operations (#4)
+- updated predefined "assigned" and "emoji" sets
+- complete casefold mapping (e.g. `k` => `K`, `K`)
+
+## [1.8.0] - 2024-01-07
+
+### Added
+
+- support for `#<=>` and `#join`, which were added to `set` in the meantime
+- support for getting the (overall) character set of a Regexp with multiple expressions
+- support for global and local case-insensitivity in Regexp inputs
+- `Regexp#{covered_by_character_set?,uses_character_set?}` methods (if core ext is used)
+
 ## [1.7.0] - 2023-05-12
 
 ### Added

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Janosch Müller
+Copyright (c) 2018-2023 Janosch Müller
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -3,9 +3,9 @@
 [![Gem Version](https://badge.fury.io/rb/character_set.svg)](http://badge.fury.io/rb/character_set)
 [![Build Status](https://github.com/jaynetics/character_set/workflows/tests/badge.svg)](https://github.com/jaynetics/character_set/actions)
 [![Build Status](https://github.com/jaynetics/character_set/workflows/gouteur/badge.svg)](https://github.com/jaynetics/character_set/actions)
-[![codecov](https://codecov.io/gh/jaynetics/character_set/branch/master/graph/badge.svg)](https://codecov.io/gh/jaynetics/character_set)
+[![Coverage](https://codecov.io/gh/jaynetics/character_set/branch/main/graph/badge.svg?token=oY7gcWNbIN)](https://codecov.io/gh/jaynetics/character_set)
 
-This is a C-extended Ruby gem to work with sets of Unicode codepoints.
+This is a Ruby gem with C and Java extensions that allows fast operations on sets of Unicode codepoints.
 
 It can [read](#parseinitialize) and [write](#write) sets of codepoints in various formats and it implements the stdlib `Set` interface for them.
 
@@ -43,7 +43,7 @@ CharacterSet.parse('[a-c]')
 CharacterSet.parse('\U00000061-\U00000063')
 ```
 
-If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `Regexp` and unicode property names can also be read. Regexp intersections, negations, and set nesting are covered, but the `i`-flag is ignored; call `#case_insensitive` on the result if needed.
+If the gems [`regexp_parser`](https://github.com/ammar/regexp_parser) and [`regexp_property_values`](https://github.com/jaynetics/regexp_property_values) are installed, `Regexp` instances and unicode property names can also be read.
 
 ```ruby
 CharacterSet.of(/./) # => #<CharacterSet (size: 1112064)>
@@ -144,6 +144,7 @@ CharacterSet['a'].inversion(include_surrogates: true)
 
 ```ruby
 CharacterSet['1', 'A'].case_insensitive # => CharacterSet['1', 'A', 'a']
+CharacterSet['s'].case_insensitive # => CharacterSet["S", "s", "ſ"]
 ```
 
 ### Write

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: false
+
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'rubygems/package_task'
@@ -11,7 +13,7 @@ task default: :spec
 
 namespace :spec do
   task :quick do
-    ENV['SKIP_MEMSAFETY_SPECS'] = '1'
+    ENV['QUICK_SPEC_RUN'] = '1'
     Rake::Task[:spec].invoke
   end
 end
@@ -21,11 +23,11 @@ Rake::ExtensionTask.new('character_set') do |ext|
 end
 
 namespace :java do
-  java_gemspec = eval File.read('./character_set.gemspec')
+  java_gemspec = Gem::Specification.load('./character_set.gemspec')
   java_gemspec.platform = 'java'
   java_gemspec.extensions = []
-
-  java_gemspec.add_runtime_dependency 'range_compressor', '~> 1.0'
+  java_gemspec.required_ruby_version = '>= 3.1'
+  java_gemspec.files |= ['lib/character_set/java.rb', 'lib/character_set/character_set.jar']
 
   Gem::PackageTask.new(java_gemspec) do |pkg|
     pkg.need_zip = true
@@ -34,9 +36,14 @@ namespace :java do
   end
 end
 
+Rake::Task['java:gem'].enhance(['java:compile'])
+
 task package: 'java:gem'
 
-unless RUBY_PLATFORM =~ /java/
+if RUBY_ENGINE == 'jruby'
+  task(:benchmark).enhance(['java:compile'])
+  task(:spec).enhance(['java:compile', 'java:verify_backend'])
+else
   # recompile before benchmarking or running specs
   task(:benchmark).enhance([:compile])
   task(:spec).enhance([:compile])

data/ext/character_set/character_set.c

@@ -1,6 +1,5 @@
 #include "ruby.h"
 #include "ruby/encoding.h"
-#include "unicode_casefold_table.h"
 
 #define UNICODE_PLANE_SIZE 0x10000
 #define UNICODE_PLANE_COUNT 17
@@ -305,11 +304,7 @@ cs_method_max(VALUE self)
 static VALUE
 cs_method_minmax(VALUE self)
 {
-  VALUE arr;
-  arr = rb_ary_new2(2);
-  rb_ary_push(arr, cs_method_min(self));
-  rb_ary_push(arr, cs_method_max(self));
-  return arr;
+  return rb_ary_new_from_args(2, cs_method_min(self), cs_method_max(self));
 }
 
 #define RETURN_COMBINED_CS(cs_a, cs_b, comp_op)                  \
@@ -462,18 +457,33 @@ cs_cps_eql(VALUE cs_a, VALUE cs_b)
   return Qtrue;
 }
 
+static inline VALUE
+cs_is_pure_cs(VALUE obj)
+{
+  VALUE klass;
+  ID pure_id;
+  klass = rb_const_get(rb_cObject, rb_intern("CharacterSet"));
+  pure_id = rb_intern("Pure");
+  return rb_const_defined_at(klass, pure_id) &&
+         RTEST(rb_obj_is_kind_of(obj, rb_const_get_at(klass, pure_id)));
+}
+
 static VALUE
 cs_method_eql_p(VALUE self, VALUE other)
 {
-  if (!cs_check_type(other))
+  if (cs_check_type(other))
   {
-    return Qfalse;
+    if (self == other) // same object_id
+    {
+      return Qtrue;
+    }
+    return cs_cps_eql(self, other);
   }
-  if (self == other) // same object_id
+  if (cs_is_pure_cs(other))
   {
-    return Qtrue;
+    return rb_funcall(other, rb_intern("=="), 1, self);
   }
-  return cs_cps_eql(self, other);
+  return Qfalse;
 }
 
 static inline VALUE
@@ -568,7 +578,9 @@ cs_merge_rb_array(VALUE self, VALUE rb_array)
 static VALUE
 cs_method_merge(VALUE self, VALUE other)
 {
+
   rb_check_frozen(self);
+
   if (cs_check_type(other))
   {
     return cs_merge_cs(self, other);
@@ -577,12 +589,22 @@ cs_method_merge(VALUE self, VALUE other)
   {
     return cs_merge_rb_array(self, other);
   }
-  return cs_merge_rb_range(self, other);
+  if (rb_obj_is_kind_of(other, rb_cRange))
+  {
+    return cs_merge_rb_range(self, other);
+  }
+  if (cs_is_pure_cs(other))
+  {
+    return cs_merge_rb_array(self, rb_funcall(other, rb_intern("to_a"), 0, 0));
+  }
+
+  rb_raise(rb_eArgError, "pass a CharacterSet, Array, or Range");
 }
 
 static VALUE
 cs_method_initialize_copy(VALUE self, VALUE orig)
 {
+  rb_obj_init_copy(self, orig);
   cs_merge_cs(self, orig);
   return self;
 }
@@ -675,6 +697,18 @@ cs_method_proper_superset_p(VALUE self, VALUE other)
   return (is_superset && is_proper) ? Qtrue : Qfalse;
 }
 
+static VALUE
+cs_method_spaceship_operator(VALUE self, VALUE other)
+{
+  if (cs_method_eql_p(self, other))
+    return INT2FIX(0);
+  if (cs_method_proper_subset_p(self, other))
+    return INT2FIX(-1);
+  if (cs_method_proper_superset_p(self, other))
+    return INT2FIX(1);
+  return Qnil;
+}
+
 // *******************************
 // `CharacterSet`-specific methods
 // *******************************
@@ -729,7 +763,7 @@ cs_method_sample(int argc, VALUE *argv, VALUE self)
   VALUE array, to_a_args[1] = {Qtrue};
   rb_check_arity(argc, 0, 1);
   array = cs_method_to_a(1, to_a_args, self);
-  return rb_funcall(array, rb_intern("sample"), argc, argc ? argv[0] : 0);
+  return rb_funcallv(array, rb_intern("sample"), argc, argv);
 }
 
 static inline VALUE
@@ -821,7 +855,6 @@ cs_method_ext_section_ratio(VALUE self, VALUE from, VALUE upto)
 
 #define MAX_CP 0x10FFFF
 #define MAX_ASCII_CP 0x7F
-#define MAX_BMP_CP 0xFFFF
 #define MIN_ASTRAL_CP 0x10000
 
 static inline VALUE
@@ -829,7 +862,7 @@ cs_has_cp_in_plane(cs_ar *cps, cs_cp len, unsigned int plane)
 {
   cs_cp plane_beg, plane_end;
   plane_beg = plane * UNICODE_PLANE_SIZE;
-  plane_end = (plane + 1) * MAX_BMP_CP;
+  plane_end = (plane + 1) * UNICODE_PLANE_SIZE - 1;
   return cs_has_cp_in_section(cps, len, plane_beg, plane_end);
 }
 
@@ -871,7 +904,7 @@ cs_method_plane(VALUE self, VALUE plane_num)
   cs_cp plane, plane_beg, plane_end;
   plane = cs_valid_plane_num(plane_num);
   plane_beg = plane * UNICODE_PLANE_SIZE;
-  plane_end = (plane + 1) * MAX_BMP_CP;
+  plane_end = (plane + 1) * UNICODE_PLANE_SIZE - 1;
   return cs_from_section(self, plane_beg, plane_end);
 }
 
@@ -924,44 +957,41 @@ add_str_cp_to_arr(unsigned int str_cp, cs_ar *cp_arr, cs_cp len, struct cs_data
   return 1;
 }
 
+struct cs_case_fold_data
+{
+  cs_ar *cps;
+  cs_cp len;
+  struct cs_data *data;
+};
+
+static int
+cs_apply_case_fold(OnigCodePoint from, OnigCodePoint *to, int to_len, void *arg)
+{
+  struct cs_case_fold_data *fold = arg;
+
+  if (to_len == 1 && tst_cp(fold->cps, fold->len, from))
+  {
+    set_cp(fold->data, to[0]);
+  }
+
+  return 0;
+}
+
 static VALUE
 cs_method_case_insensitive(VALUE self)
 {
-  cs_cp i, len;
-  cs_ar *cps;
   VALUE new_cs;
-  struct cs_data *new_data;
+  struct cs_case_fold_data fold;
 
-  cps = cs_fetch_cps(self, &len);
-  new_cs = cs_alloc(RBASIC(self)->klass, &new_data);
+  fold.cps = cs_fetch_cps(self, &fold.len);
+  new_cs = cs_alloc(RBASIC(self)->klass, &fold.data);
   cs_merge_cs(new_cs, self);
 
-  for (i = 0; i < CASEFOLD_COUNT; i++)
-  {
-    casefold_mapping m = unicode_casefold_table[i];
-
-    if (tst_cp(cps, len, m.from))
-    {
-      set_cp(new_data, m.to);
-    }
-    else if (tst_cp(cps, len, m.to))
-    {
-      set_cp(new_data, m.from);
-    }
-  }
+  // Flags 0 exclude multi-character folds; the API supplies both directions.
+  ONIGENC_APPLY_ALL_CASE_FOLD(rb_utf8_encoding(), 0, cs_apply_case_fold, &fold);
+  RB_GC_GUARD(self);
 
   return new_cs;
-
-  // OnigCaseFoldType flags;
-  // rb_encoding *enc;
-  //
-  // enc = rb_utf8_encoding();
-  //
-  // ONIGENC_CASE_UPCASE | ONIGENC_CASE_DOWNCASE (not public on ruby < 2.4)
-  // flags = (1<<13) | (1<<14);
-  //
-  // // case_map args: flags, pp, end, to, to_end, enc
-  // enc->case_map(flags, (const OnigUChar**)&cp, ?, ?, ?, enc);
 }
 
 static inline VALUE
@@ -1017,23 +1047,13 @@ each_mb_cp(VALUE str, str_cp_handler func, cs_ar *cp_arr, cs_cp len, struct cs_d
   return Qtrue;
 }
 
-// single_byte_optimizable - copied from string.c
+// Keep byte iteration for ASCII text and binary input.
 static inline int
 single_byte_optimizable(VALUE str)
 {
-  rb_encoding *enc;
-  if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT)
-  {
-    return 1;
-  }
-
-  enc = rb_enc_get(str);
-  if (rb_enc_mbmaxlen(enc) == 1)
-  {
-    return 1;
-  }
-
-  return 0;
+  rb_encoding *enc = rb_enc_get(str);
+  return enc == rb_ascii8bit_encoding() ||
+         (rb_enc_asciicompat(enc) && ENC_CODERANGE(str) == ENC_CODERANGE_7BIT);
 }
 
 static inline VALUE
@@ -1150,7 +1170,8 @@ cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
 {
   cs_ar *cps;
   cs_cp cs_len;
-  VALUE orig_str_len;
+  VALUE original = str;
+  long working_str_len;
 
   rb_encoding *orig_enc, *utf8;
   char *s, *send, *t;
@@ -1158,9 +1179,12 @@ cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
 
   raise_arg_err_unless_string(str);
 
-  orig_str_len = RSTRING_LEN(str);
+  if (bang)
+    rb_check_frozen(original);
+
+  working_str_len = RSTRING_LEN(str);
 
-  if (orig_str_len == 0)
+  if (working_str_len == 0)
   {
     return bang ? Qnil : str;
   }
@@ -1221,17 +1245,17 @@ cs_apply_to_str(VALUE set, VALUE str, int delete, int bang)
   rb_str_set_len(str, t - RSTRING_PTR(str));
   ENC_CODERANGE_SET(str, cr);
 
-  if (bang && (RSTRING_LEN(str) == (long)orig_str_len)) // string unchanged
+  if (bang && (RSTRING_LEN(str) == working_str_len)) // string unchanged
   {
     return Qnil;
   }
 
   if (!orig_was_utf8)
   {
-    return rb_str_encode(str, rb_enc_from_encoding(orig_enc), 0, Qnil);
+    str = rb_str_encode(str, rb_enc_from_encoding(orig_enc), 0, Qnil);
   }
 
-  return str;
+  return bang ? rb_str_replace(original, str) : str;
 }
 
 static VALUE
@@ -1313,8 +1337,7 @@ void Init_character_set()
   rb_define_method(cs, "eql?", cs_method_eql_p, 1);
   rb_define_method(cs, "==", cs_method_eql_p, 1);
   rb_define_method(cs, "merge", cs_method_merge, 1);
-  rb_define_method(cs, "initialize_clone", cs_method_initialize_copy, 1);
-  rb_define_method(cs, "initialize_dup", cs_method_initialize_copy, 1);
+  rb_define_method(cs, "initialize_copy", cs_method_initialize_copy, 1);
   rb_define_method(cs, "subtract", cs_method_subtract, 1);
   rb_define_method(cs, "subset?", cs_method_subset_p, 1);
   rb_define_method(cs, "<=", cs_method_subset_p, 1);
@@ -1324,6 +1347,7 @@ void Init_character_set()
   rb_define_method(cs, ">=", cs_method_superset_p, 1);
   rb_define_method(cs, "proper_superset?", cs_method_proper_superset_p, 1);
   rb_define_method(cs, ">", cs_method_proper_superset_p, 1);
+  rb_define_method(cs, "<=>", cs_method_spaceship_operator, 1);
 
   // `CharacterSet`-specific methods
 

data/ext/character_set/java/src/main/java/org/jaynetics/characterset/CharacterSetService.java

@@ -0,0 +1,15 @@
+package org.jaynetics.characterset;
+
+import org.jruby.Ruby;
+import org.jruby.RubyClass;
+import org.jruby.runtime.load.BasicLibraryService;
+
+public final class CharacterSetService implements BasicLibraryService {
+    @Override
+    public boolean basicLoad(Ruby runtime) {
+        RubyClass klass = runtime.getClass("CharacterSet");
+        klass.setAllocator(RubyCharacterSet::new);
+        klass.defineAnnotatedMethods(RubyCharacterSet.class);
+        return true;
+    }
+}