cld3 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e12ff58a2cc0242896e307f33cb1adb2e210a1889244861e2c751b0399cb6415
-  data.tar.gz: 3b0c348b00126c8ee825a76eed0919e074a7f97e7dc7e647614f7a754c69d716
+  metadata.gz: 730f1cc6022bc381311f03ee67a87cdcaab01bcfc6c9bc5d1b056871acef197c
+  data.tar.gz: d1f640c24df2a95cdd2605e4078c5b820ec15ddbe775b7c281004d6f3772df8f
 SHA512:
-  metadata.gz: 690c6e6f0cc8f0cc369b05a611daeed74adc273cd9c02f780233921ed0c220a425552e7020c127ff0503e88d473f57088cafd3b0d2143f3f86527d886d3eaf63
-  data.tar.gz: 6c3b7c105fa799fb918a077d9d1f47c948ec29bad86e57f452ea9eae82fc4f5bfc2686654a2a68ebc7102dfcb34e2a1ca55648d75a1f6bcf97b532de482de76e
+  metadata.gz: 41aedd4699c653dcfcc97f34adde401918928c7bf27cfe48fd5df4c108476be837e5c43add574823577afa35066c0386b31ee97f28c83660d35c753103955b20
+  data.tar.gz: 6f3b389ac6b334980e5899fd1a40664ee67011bb998deef5ccef4794f67d1613b4777243b97c7e87b8aab6118332907712700799437e3e261e0eb6ce85cd6952

data/README.md

@@ -35,19 +35,11 @@ FreeBSD port is available as `rubygem-cld3` in `textproc` category.
 
 https://svnweb.freebsd.org/ports/head/textproc/rubygem-cld3/
 
-#### JRuby
-JRuby has a bug which prevents the feature detection. Apply the following
-change:
-https://github.com/jruby/jruby/pull/4118/commits/edad375ef4dcf195b19ce0afe4befac66468c736
-
 ### Troubleshooting
 `gem install cld3` triggers native library building. If it fails, it is likely
 that some required facilities are missing. Make sure C++ compiler is installed.
 I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler.
 
-Runtime errors are likely to be issues of [FFI](https://github.com/ffi/ffi) or
-programming errors. Make sure they are all correct.
-
 If you cannot identify the cause of your problem, run spec of this library and
 see whether the problem is reproducible with it or not. Spec is not included in
 the gem, so clone the source code repository and then run `rake spec`.

data/cld3.gemspec

@@ -16,7 +16,7 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.5.0"
+  gem.version = "3.5.1"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"
@@ -24,13 +24,13 @@ Gem::Specification.new do |gem|
   gem.author = "Akihiko Odaki"
   gem.email = "akihiko.odaki@gmail.com"
   gem.required_ruby_version = [ ">= 2.7.0", "< 3.3.0" ]
-  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.16.0" ]
-  gem.add_development_dependency "rbs", [ ">= 2.6.0", "< 2.7.0" ]
-  gem.add_development_dependency "rspec", [ ">= 3.11.0", "< 3.12.0" ]
-  gem.add_development_dependency "steep", [ ">= 1.0.0", "< 1.1.0" ]
+  gem.add_development_dependency "rbs", [ ">= 2.8.0", "< 2.9.0" ]
+  gem.add_development_dependency "rspec", [ ">= 3.12.0", "< 3.13.0" ]
+  gem.add_development_dependency "steep", [ ">= 1.3.0", "< 1.4.0" ]
   gem.files = Dir[
     "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
-    "cld3.gemspec", "ext/**/*", "lib/**/*", "sig/**/*"
+    "cld3.gemspec", "ext/**/*.c", "ext/**/*.cc", "ext/**/*.h",
+    "lib/**/*.rb", "sig/**/*"
   ]
   gem.require_paths = [ "lib" ]
   gem.extensions = [ "ext/cld3/extconf.rb" ]

data/ext/cld3/extconf.rb

@@ -47,5 +47,4 @@ FileUtils.mkdir_p("script_span")
 }
 
 $CXXFLAGS += " -fvisibility=hidden -std=c++17"
-$LIBRUBYARG = ""
-create_makefile("libcld3")
+create_makefile("cld3_ext")

data/ext/cld3/nnet_language_identifier_c.cc

@@ -18,6 +18,7 @@ limitations under the License.
 #include <iostream>
 #include <string>
 #include <utility>
+#include <ruby.h>
 #include "nnet_language_identifier.h"
 
 #if defined _WIN32 || defined __CYGWIN__
@@ -27,89 +28,180 @@ limitations under the License.
 #endif
 
 struct Result {
-  struct {
-    const char *data;
-    std::size_t size;
-  } language;
-  struct {
-    const chrome_lang_id::NNetLanguageIdentifier::SpanInfo *data;
-    std::size_t size;
-  } byte_ranges;
-  float probability;
-  float proportion;
-  bool is_reliable;
+  VALUE result_klass;
+  VALUE span_info_klass;
+  const chrome_lang_id::NNetLanguageIdentifier::Result& data;
+
+  VALUE convert() const {
+    if (data.language == chrome_lang_id::NNetLanguageIdentifier::kUnknown)
+      return Qnil;
+
+    VALUE byte_ranges = rb_ary_new2(data.byte_ranges.size());
+    for (auto& byte_range_data : data.byte_ranges) {
+      VALUE argv[] = {
+        INT2NUM(byte_range_data.start_index),
+        INT2NUM(byte_range_data.end_index),
+        DBL2NUM(byte_range_data.probability),
+      };
+
+      VALUE byte_range = rb_class_new_instance(sizeof(argv) / sizeof(*argv),
+                                               argv,
+                                               span_info_klass);
+      rb_ary_push(byte_ranges, byte_range);
+    }
+
+    VALUE argv[] = {
+      ID2SYM(rb_intern2(data.language.data(), data.language.size())),
+      DBL2NUM(data.probability),
+      data.is_reliable ? Qtrue : Qfalse,
+      DBL2NUM(data.proportion),
+      byte_ranges,
+    };
+
+    return rb_class_new_instance(sizeof(argv) / sizeof(*argv), argv,
+                                 result_klass);
+  }
 };
 
-struct OwningResult {
-  OwningResult(chrome_lang_id::NNetLanguageIdentifier::Result&& result) {
-    references.language = std::move(result.language);
-    references.byte_ranges = std::move(result.byte_ranges);
-    plain.language.data = references.language.data();
-    plain.language.size = references.language.size();
-    plain.byte_ranges.data = references.byte_ranges.data();
-    plain.byte_ranges.size = references.byte_ranges.size();
-    plain.probability = result.probability;
-    plain.proportion = result.proportion;
-    plain.is_reliable = result.is_reliable;
+struct ResultVector {
+  VALUE result_klass;
+  VALUE span_info_klass;
+  VALUE buffer;
+  const std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>& data;
+
+  VALUE convert() const {
+    for (auto& element_data : data) {
+      Result result { result_klass, span_info_klass, element_data };
+      VALUE element = result.convert();
+      if (element == Qnil)
+        break;
+
+      rb_ary_push(buffer, element);
+    }
+
+    return buffer;
   }
+};
 
-  Result plain;
-  struct {
-    std::string language;
-    std::vector<chrome_lang_id::NNetLanguageIdentifier::SpanInfo> byte_ranges;
-  } references;
+template<typename T>
+VALUE convert_protected(VALUE arg)
+{
+    auto result = reinterpret_cast<const T *>(arg);
+    return result->convert();
+}
+
+static void dfree(void *arg) {
+  auto data = static_cast<chrome_lang_id::NNetLanguageIdentifier *>(arg);
+  data->~NNetLanguageIdentifier();
+  xfree(arg);
+}
+
+static size_t dsize(const void *data) {
+  return sizeof(chrome_lang_id::NNetLanguageIdentifier);
+}
+
+static const rb_data_type_t data_type = {
+  .wrap_struct_name = "CLD3::NNetLanguageIdentifier",
+  .function = {
+    .dfree = dfree,
+    .dsize = dsize,
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
 };
 
-extern "C" {
-  EXPORT OwningResult *NNetLanguageIdentifier_find_language(
-      chrome_lang_id::NNetLanguageIdentifier *instance,
-      const char *data,
-      std::size_t size) {
-    return new OwningResult(instance->FindLanguage(std::string(data, size)));
+static VALUE find_language(VALUE obj,
+                           VALUE result_klass, VALUE span_info_klass,
+                           VALUE text) {
+  int state;
+  VALUE converted;
+
+  {
+    chrome_lang_id::NNetLanguageIdentifier *data;
+    TypedData_Get_Struct(obj, chrome_lang_id::NNetLanguageIdentifier,
+                         &data_type, data);
+    std::string text_string = std::string(RSTRING_PTR(text), RSTRING_LEN(text));
+    auto result_data = data->FindLanguage(text_string);
+    Result result { result_klass, span_info_klass, result_data };
+
+    converted = rb_protect(convert_protected<Result>,
+                           reinterpret_cast<VALUE>(&result),
+                           &state);
   }
 
-  EXPORT std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>*
-  NNetLanguageIdentifier_find_top_n_most_freq_langs(
-      chrome_lang_id::NNetLanguageIdentifier *instance,
-      const char *data, std::size_t size, int num_langs) {
-    std::string text(data, size);
-    return new auto(instance->FindTopNMostFreqLangs(text, num_langs));
-  }
+  if (state)
+    rb_jump_tag(state);
 
-  EXPORT void delete_NNetLanguageIdentifier(
-      chrome_lang_id::NNetLanguageIdentifier *pointer) {
-    delete pointer;
-  }
+  return converted;
+}
 
-  EXPORT void delete_result(OwningResult *pointer) {
-    delete pointer;
+static VALUE find_top_n_most_freq_langs(VALUE obj,
+                                        VALUE result_klass,
+                                        VALUE span_info_klass,
+                                        VALUE text,
+                                        VALUE num_langs) {
+  int state;
+  VALUE converted;
+
+  {
+    chrome_lang_id::NNetLanguageIdentifier *data;
+    TypedData_Get_Struct(obj, chrome_lang_id::NNetLanguageIdentifier,
+                         &data_type, data);
+    VALUE buffer = rb_ary_new2(NUM2INT(num_langs));
+    std::string text_string = std::string(RSTRING_PTR(text), RSTRING_LEN(text));
+    auto result_data = data->FindTopNMostFreqLangs(text_string, num_langs);
+    ResultVector result { result_klass, span_info_klass, buffer, result_data };
+
+    converted = rb_protect(convert_protected<ResultVector>,
+                           reinterpret_cast<VALUE>(&result),
+                           &state);
   }
 
-  EXPORT void delete_results(
-      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *pointer) {
-    delete pointer;
-  }
+  if (state)
+    rb_jump_tag(state);
 
-  EXPORT chrome_lang_id::NNetLanguageIdentifier *new_NNetLanguageIdentifier(
-      int min_num_bytes, int max_num_bytes) {
-    return new chrome_lang_id::NNetLanguageIdentifier(
-        min_num_bytes, max_num_bytes);
-  }
+  return converted;
+}
+
+static VALUE make(VALUE klass, VALUE min_num_bytes, VALUE max_num_bytes) {
+  int min_num_bytes_int = NUM2INT(min_num_bytes);
+  int max_num_bytes_int = NUM2INT(max_num_bytes);
+  chrome_lang_id::NNetLanguageIdentifier *data;
+  VALUE value = TypedData_Make_Struct(klass,
+                                      chrome_lang_id::NNetLanguageIdentifier,
+                                      &data_type, data);
+  new (data) chrome_lang_id::NNetLanguageIdentifier(min_num_bytes_int, max_num_bytes_int);
+  return value;
+}
 
-  EXPORT Result refer_to_nth_result(
-      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *results,
-      std::size_t index) {
-    Result c;
-    auto& cc = (*results)[index];
-
-    c.language.data = cc.language.data();
-    c.language.size = cc.language.size();
-    c.byte_ranges.data = cc.byte_ranges.data();
-    c.byte_ranges.size = cc.byte_ranges.size();
-    c.probability = cc.probability;
-    c.proportion = cc.proportion;
-    c.is_reliable = cc.is_reliable;
-
-    return c;
+extern "C" EXPORT void Init_cld3_ext() {
+  VALUE cld3 = rb_const_get(rb_cObject, rb_intern("CLD3"));
+  VALUE identifier = 
+    rb_const_get(cld3, rb_intern("NNetLanguageIdentifier"));
+  VALUE unstable = rb_const_get(identifier, rb_intern("Unstable"));
+  VALUE params = rb_const_get(cld3, rb_intern("TaskContextParams"));
+  VALUE language_names = rb_const_get(params, rb_intern("LANGUAGE_NAMES"));
+
+  rb_define_const(identifier, "MIN_NUM_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMinNumBytesToConsider));
+  rb_define_const(identifier, "MAX_NUM_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMaxNumBytesToConsider));
+  rb_define_const(identifier, "MAX_NUM_INPUT_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMaxNumInputBytesToConsider));
+  rb_define_const(identifier, "RELIABILITY_THRESHOLD",
+                  DBL2NUM(chrome_lang_id::NNetLanguageIdentifier::kReliabilityThreshold));
+  rb_define_const(identifier, "RELIABILITY_HR_BS_THRESHOLD",
+                  DBL2NUM(chrome_lang_id::NNetLanguageIdentifier::kReliabilityHrBsThreshold));
+
+  rb_define_singleton_method(unstable, "make", make, 2);
+  rb_define_method(unstable, "find_language", find_language, 3);
+  rb_define_method(unstable, "find_top_n_most_freq_langs",
+                   find_top_n_most_freq_langs, 4);
+
+  for (int i = 0; ; i++) {
+    const char *name = chrome_lang_id::TaskContextParams::language_names(i);
+    if (!name)
+      break;
+
+    rb_ary_push(language_names, ID2SYM(rb_intern(name)));
   }
 }

data/lib/cld3.rb

@@ -17,39 +17,10 @@
 # limitations under the License.
 # ==============================================================================
 
-require "ffi"
-require "rbconfig"
-require "cld3/unstable"
-
 # Module providing an interface for Compact Language Detector v3 (CLD3)
 module CLD3
   # Class for detecting the language of a document.
   class NNetLanguageIdentifier
-    # Min number of bytes needed to make a prediction if the construcotr is
-    # called without the corresponding parameter.
-    # This is Numeric object.
-    MIN_NUM_BYTES_TO_CONSIDER = 140
-
-    # Max number of bytes needed to make a prediction if the construcotr is
-    # called without the corresponding parameter.
-    # This is Numeric object.
-    MAX_NUM_BYTES_TO_CONSIDER = 700
-
-    # Max number of input bytes to process.
-    # This is Numeric object.
-    MAX_NUM_INPUT_BYTES_TO_CONSIDER = 10000
-
-    # Predictions with probability greater than or equal to this threshold are
-    # marked as reliable. This threshold was optimized on a set of text segments
-    # extracted from wikipedia, and results in an overall precision, recall,
-    # and f1 equal to 0.9760, 0.9624, and 0.9692, respectively.
-    # This is Numeric object.
-    RELIABILITY_THRESHOLD = 0.7
-
-    # Reliability threshold for the languages hr and bs.
-    # This is Numeric object.
-    RELIABILITY_HR_BS_THRESHOLD = 0.5
-
     # Holds probability that Span, specified by start/end indices, is a given
     # language. The langauge is not stored here; it can be found in Result, which
     # holds an Array of SpanInfo.
@@ -76,8 +47,10 @@ module CLD3
 
     # The arguments are two Numeric objects.
     def initialize(min_num_bytes = MIN_NUM_BYTES_TO_CONSIDER, max_num_bytes = MAX_NUM_BYTES_TO_CONSIDER)
+      min_num_bytes = min_num_bytes.ceil
+      max_num_bytes = max_num_bytes.floor
       raise ArgumentError if min_num_bytes < 0 || min_num_bytes >= max_num_bytes
-      @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(min_num_bytes, max_num_bytes))
+      @cc = Unstable.make(min_num_bytes, max_num_bytes)
     end
 
     # Finds the most likely language for the given text, along with additional
@@ -88,23 +61,7 @@ module CLD3
     # The argument is a String object.
     # The returned value of this function is an instance of Result.
     def find_language(text)
-      # @type const FFI: untyped
-
-      text_utf8 = text.encode(Encoding::UTF_8)
-      pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
-
-      begin
-        pointer.put_bytes(0, text_utf8)
-
-        result = Unstable.NNetLanguageIdentifier_find_language(@cc, pointer, text_utf8.bytesize)
-        begin
-          convert_result Unstable::NNetLanguageIdentifier::Result.new(result)
-        ensure
-          Unstable.delete_result result
-        end
-      ensure
-        pointer.free
-      end
+      @cc.find_language(Result, SpanInfo, text.encode(Encoding::UTF_8))
     end
 
     # Splits the input text (up to the first byte, if any, that is not
@@ -121,52 +78,15 @@ module CLD3
     # The second argument is Numeric object.
     # The returned value of this functions is an Array of Result instances.
     def find_top_n_most_freq_langs(text, num_langs)
-      # @type const FFI: untyped
-      # @type var a: untyped
-
-      text_utf8 = text.encode(Encoding::UTF_8)
-      pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
-
-      begin
-        pointer.put_bytes(0, text_utf8)
-
-        results = Unstable.NNetLanguageIdentifier_find_top_n_most_freq_langs(@cc, pointer, text_utf8.bytesize, num_langs)
-        begin
-          a = num_langs.times
-            .lazy
-            .map { |index| convert_result Unstable.refer_to_nth_result(results, index) }
-            .take_while { |result| !result.nil? }
-            .to_a
-
-          a
-        ensure
-          Unstable.delete_results results
-        end
-      ensure
-        pointer.free
-      end
+      @cc.find_top_n_most_freq_langs(Result, SpanInfo,
+                                     text.encode(Encoding::UTF_8),
+                                     num_langs)
     end
 
-    private
-
-    def convert_result(result)
-      language = result[:language_data].read_bytes(result[:language_size])
-      return nil if language == "und"
-
-      cursor = result[:byte_ranges_data]
-      byte_ranges = result[:byte_ranges_size].times.map do
-        info = Unstable::NNetLanguageIdentifier::SpanInfo.new(cursor)
-        cursor += Unstable::NNetLanguageIdentifier::SpanInfo.size
-        SpanInfo.new(info[:start_index], info[:end_index], info[:probability])
-      end
-
-      Result.new(
-          language.to_sym,
-          result[:probability],
-          result[:reliable?],
-          result[:proportion],
-          byte_ranges)
+    class Unstable
     end
+  
+    private_constant :Unstable
   end
 
   # Encapsulates the TaskContext specifying only the parameters for the model.
@@ -174,17 +94,9 @@ module CLD3
   module TaskContextParams
     # This is an frozen Array object containing symbols.
     # @type const LANGUAGE_NAMES: untyped
-    LANGUAGE_NAMES = [
-      :eo, :co, :eu, :ta, :de, :mt, :ps, :te, :su, :uz, :'zh-Latn', :ne,
-      :nl, :sw, :sq, :hmn, :ja, :no, :mn, :so, :ko, :kk, :sl, :ig,
-      :mr, :th, :zu, :ml, :hr, :bs, :lo, :sd, :cy, :hy, :uk, :pt,
-      :lv, :iw, :cs, :vi, :jv, :be, :km, :mk, :tr, :fy, :am, :zh,
-      :da, :sv, :fi, :ht, :af, :la, :id, :fil, :sm, :ca, :el, :ka,
-      :sr, :it, :sk, :ru, :'ru-Latn', :bg, :ny, :fa, :haw, :gl, :et,
-      :ms, :gd, :'bg-Latn', :ha, :is, :ur, :mi, :hi, :bn, :'hi-Latn', :fr,
-      :yi, :hu, :xh, :my, :tg, :ro, :ar, :lb, :'el-Latn', :st, :ceb,
-      :kn, :az, :si, :ky, :mg, :en, :gu, :es, :pl, :'ja-Latn', :ga, :lt,
-      :sn, :yo, :pa, :ku,
-    ].freeze
+    LANGUAGE_NAMES = []
   end
 end
+
+require "cld3_ext"
+CLD3::TaskContextParams::LANGUAGE_NAMES.freeze

data/sig/cld3.rbs

@@ -36,6 +36,8 @@ module CLD3
       attr_accessor byte_ranges(): Array[SpanInfo]
     end
 
+    @cc: untyped
+
     def initialize: (?Integer, ?Integer) -> void
     def find_language: (String) -> Result?
     def find_top_n_most_freq_langs: (String, Integer) -> Array[Result]