cld3 3.4.4 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f40e4947fea97543686caceba0082bdba30b5ae0485a25b41004ad048057b0ad
-  data.tar.gz: e45c60300550caf513fdde6bcbc05e68e1063bf9ad8074626bf5f88f4a6f77bd
+  metadata.gz: 730f1cc6022bc381311f03ee67a87cdcaab01bcfc6c9bc5d1b056871acef197c
+  data.tar.gz: d1f640c24df2a95cdd2605e4078c5b820ec15ddbe775b7c281004d6f3772df8f
 SHA512:
-  metadata.gz: 393fc138a279ee42c3de90c49bcc982e55860f74e2796d4c895d0f2f175894bcb1ec1bbe796811f896a16be9cc97943e1309cbe175bc029a510b4c51b2f700da
-  data.tar.gz: d16e8c87e7d12cc90cc1a4babb4873df8f553d9527e1d69a548a250ae0b240f79a6338070bbc88cbb0e23db48c23ef0393cd4b62e0ac673722ace81ce1564895
+  metadata.gz: 41aedd4699c653dcfcc97f34adde401918928c7bf27cfe48fd5df4c108476be837e5c43add574823577afa35066c0386b31ee97f28c83660d35c753103955b20
+  data.tar.gz: 6f3b389ac6b334980e5899fd1a40664ee67011bb998deef5ccef4794f67d1613b4777243b97c7e87b8aab6118332907712700799437e3e261e0eb6ce85cd6952

data/Gemfile

@@ -15,5 +15,4 @@
 #==============================================================================
 
 source 'https://rubygems.org'
-gem 'steep', github: 'akihikodaki/steep', branch: 'cld3'
 gemspec

data/README.md

@@ -19,7 +19,6 @@ cld3.find_language("здравствуйте") # => #<struct Struct::Result lang
 ### Prerequisites
 * [Bundler](http://bundler.io/)
 * C++ compiler
-* [Protocol buffers](https://developers.google.com/protocol-buffers/)
 * [Rake](https://ruby.github.io/rake/)
 * [RubyGems](https://rubygems.org/)
 
@@ -36,23 +35,13 @@ FreeBSD port is available as `rubygem-cld3` in `textproc` category.
 
 https://svnweb.freebsd.org/ports/head/textproc/rubygem-cld3/
 
-#### JRuby
-JRuby has a bug which prevents the feature detection. Apply the following
-change:
-https://github.com/jruby/jruby/pull/4118/commits/edad375ef4dcf195b19ce0afe4befac66468c736
-
 ### Troubleshooting
-`gem install cld3` triggers native library building. If it fails, you are likely
-to missing required facilities. Make sure C++ compiler and protocol buffers
-is installed. I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler. Ruby is
-likely to need [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
-as well.
-
-Runtime errors are likely to be issues of [FFI](https://github.com/ffi/ffi) or
-programming errors. Make sure they are all correct.
+`gem install cld3` triggers native library building. If it fails, it is likely
+that some required facilities are missing. Make sure C++ compiler is installed.
+I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler.
 
 If you cannot identify the cause of your problem, run spec of this library and
-see whether the problem is reproducable with it or not. Spec is not included in
+see whether the problem is reproducible with it or not. Spec is not included in
 the gem, so clone the source code repository and then run `rake spec`.
 The source code repository is at
 https://github.com/akihikodaki/cld3-ruby.

data/cld3.gemspec

@@ -16,21 +16,21 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.4.4"
+  gem.version = "3.5.1"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"
   gem.homepage = "https://github.com/akihikodaki/cld3-ruby"
   gem.author = "Akihiko Odaki"
   gem.email = "akihiko.odaki@gmail.com"
-  gem.required_ruby_version = [ ">= 2.6.0", "< 3.2.0" ]
-  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.16.0" ]
-  gem.add_development_dependency "rbs", [ ">= 1.7.0", "< 1.8.0" ]
-  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.11.0" ]
-  gem.add_development_dependency "steep", [ ">= 0.47.0", "< 0.48.0" ]
+  gem.required_ruby_version = [ ">= 2.7.0", "< 3.3.0" ]
+  gem.add_development_dependency "rbs", [ ">= 2.8.0", "< 2.9.0" ]
+  gem.add_development_dependency "rspec", [ ">= 3.12.0", "< 3.13.0" ]
+  gem.add_development_dependency "steep", [ ">= 1.3.0", "< 1.4.0" ]
   gem.files = Dir[
     "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
-    "cld3.gemspec", "ext/**/*", "lib/**/*", "sig/**/*"
+    "cld3.gemspec", "ext/**/*.c", "ext/**/*.cc", "ext/**/*.h",
+    "lib/**/*.rb", "sig/**/*"
   ]
   gem.require_paths = [ "lib" ]
   gem.extensions = [ "ext/cld3/extconf.rb" ]

data/ext/cld3/cld_3/protos/feature_extractor.pb.h

@@ -0,0 +1,100 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef FEATURE_EXTRACTOR_PB_H_
+#define FEATURE_EXTRACTOR_PB_H_
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace chrome_lang_id {
+
+class Parameter {
+ public:
+  const std::string& name() const { return name_; }
+  void set_name(std::string value) { name_ = std::move(value); }
+  const std::string& value() const { return value_; }
+  void set_value(std::string value) { value_ = std::move(value); }
+
+ private:
+  std::string name_;
+  std::string value_;
+};
+
+class FeatureFunctionDescriptor {
+ public:
+  const std::string& type() const { return type_; }
+
+  void set_type(std::string value) { type_ = std::move(value); }
+
+  const std::string& name() const { return name_; }
+
+  void set_name(std::string value) { name_ = std::move(value); }
+
+  bool has_argument() const { return true; }
+
+  std::int32_t argument() const { return argument_; }
+
+  void set_argument(int32_t value) { argument_ = value; }
+
+  int parameter_size() const { return parameter_.size(); }
+
+  const Parameter& parameter(int index) const { return parameter_[index]; }
+
+  Parameter* add_parameter() { return &parameter_.emplace_back(); }
+
+  int feature_size() const { return feature_.size(); }
+
+  FeatureFunctionDescriptor* mutable_feature(int index) {
+    return &feature_[index];
+  }
+
+  const FeatureFunctionDescriptor& feature(int index) const {
+    return feature_[index];
+  }
+
+  FeatureFunctionDescriptor* add_feature() { return &feature_.emplace_back(); }
+
+ private:
+  std::string type_;
+  std::string name_;
+  std::int32_t argument_;
+  std::vector<Parameter> parameter_;
+  std::vector<FeatureFunctionDescriptor> feature_;
+};
+
+class FeatureExtractorDescriptor {
+ public:
+  int feature_size() const { return feature_.size(); }
+
+  FeatureFunctionDescriptor* mutable_feature(int index) {
+    return &feature_[index];
+  }
+
+  const FeatureFunctionDescriptor& feature(int index) const {
+    return feature_[index];
+  }
+
+  FeatureFunctionDescriptor* add_feature() { return &feature_.emplace_back(); }
+
+ private:
+  std::vector<FeatureFunctionDescriptor> feature_;
+};
+
+}
+
+#endif

data/ext/cld3/cld_3/protos/sentence.pb.h

@@ -0,0 +1,35 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef SENTENCE_PB_H_
+#define SENTENCE_PB_H_
+
+#include <string>
+
+namespace chrome_lang_id {
+
+class Sentence {
+ public:
+  const std::string& text() const { return text_; }
+  void set_text(std::string value) { text_ = std::move(value); }
+
+ private:
+  std::string text_;
+};
+
+}
+
+#endif

data/ext/cld3/cld_3/protos/task_spec.pb.h

@@ -0,0 +1,106 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TASK_SPEC_PB_H_
+#define TASK_SPEC_PB_H_
+
+#include <string>
+#include <vector>
+
+namespace chrome_lang_id {
+
+class TaskInput {
+ public:
+  class Part {
+   public:
+    const std::string& file_pattern() const { return file_pattern_; }
+
+   private:
+    std::string file_pattern_;
+  };
+
+  const std::string& name() const { return name_; }
+
+  void set_name(std::string value) { name_ = value; }
+
+  int file_format_size() const { return file_format_.size(); }
+
+  const std::string& file_format(int index) const {
+    return file_format_[index];
+  }
+
+  void add_file_format(std::string value) {
+    file_format_.push_back(std::move(value));
+  }
+
+  int record_format_size() const { return record_format_.size(); }
+
+  const std::string& record_format(int index) const {
+    return record_format_[index];
+  }
+
+  void add_record_format(std::string value) {
+    record_format_.push_back(std::move(value));
+  }
+
+  int part_size() const { return part_.size(); }
+  const Part& part(int index) const { return part_[index]; }
+
+ private:
+  std::string name_;
+  std::vector<std::string> file_format_;
+  std::vector<std::string> record_format_;
+  std::vector<Part> part_;
+};
+
+class TaskSpec {
+ public:
+  class Parameter {
+   public:
+    const std::string& name() const { return name_; }
+    void set_name(std::string value) { name_ = std::move(value); }
+    const std::string& value() const { return value_; }
+    void set_value(std::string value) { value_ = std::move(value); }
+
+   private:
+    std::string name_;
+    std::string value_;
+  };
+
+  int parameter_size() const { return parameter_.size(); }
+
+  Parameter* mutable_parameter(int index) { return &parameter_[index]; }
+
+  const Parameter& parameter(int index) const { return parameter_[index]; }
+
+  Parameter* add_parameter() { return &parameter_.emplace_back(); }
+
+  int input_size() const { return input_.size(); }
+
+  TaskInput* mutable_input(int index) { return &input_[index]; }
+
+  const TaskInput& input(int index) const { return input_[index]; }
+
+  TaskInput* add_input() { return &input_.emplace_back(); }
+
+ private:
+  std::vector<Parameter> parameter_;
+  std::vector<TaskInput> input_;
+};
+
+}
+
+#endif

data/ext/cld3/extconf.rb

@@ -26,17 +26,8 @@ rescue
   end
 end
 
-# Check pkg-config first to inform the library is missing if so.
-pkg_config("protobuf") or abort "Failed to locate protobuf"
-
-FileUtils.mkdir_p("cld_3/protos")
 FileUtils.mkdir_p("script_span")
 
-[ "feature_extractor", "sentence", "task_spec" ].each {|name|
-  system "protoc", "#{name}.proto", "--cpp_out=.", exception: true
-  ln_fallback("#{name}.pb.h", "cld_3/protos/#{name}.pb.h")
-}
-
 [
   "fixunicodevalue.h",
   "generated_ulscript.h",
@@ -55,6 +46,5 @@ FileUtils.mkdir_p("script_span")
   ln_fallback("#{name}", "script_span/#{name}")
 }
 
-$CXXFLAGS += " -fvisibility=hidden -std=c++11"
-$LIBRUBYARG = ""
-create_makefile("libcld3")
+$CXXFLAGS += " -fvisibility=hidden -std=c++17"
+create_makefile("cld3_ext")

data/ext/cld3/getonescriptspan.cc

@@ -878,7 +878,6 @@ bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
   // copying letters to buffer with single spaces for each run of non-letters
   while (take < byte_length_) {
     // Copy run of letters in same script (&LS | LS)*
-    int letter_count = 0;              // Keep track of word length
     bool need_break = false;
 
     while (take < byte_length_) {
@@ -963,7 +962,6 @@ bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
         map2original_.Delete(tlen - plen);
       }
 
-      ++letter_count;
       if (put >= kMaxScriptBytes) {
         // Buffer is full
         span->truncated = true;

data/ext/cld3/nnet_language_identifier_c.cc

@@ -18,6 +18,7 @@ limitations under the License.
 #include <iostream>
 #include <string>
 #include <utility>
+#include <ruby.h>
 #include "nnet_language_identifier.h"
 
 #if defined _WIN32 || defined __CYGWIN__
@@ -27,89 +28,180 @@ limitations under the License.
 #endif
 
 struct Result {
-  struct {
-    const char *data;
-    std::size_t size;
-  } language;
-  struct {
-    const chrome_lang_id::NNetLanguageIdentifier::SpanInfo *data;
-    std::size_t size;
-  } byte_ranges;
-  float probability;
-  float proportion;
-  bool is_reliable;
+  VALUE result_klass;
+  VALUE span_info_klass;
+  const chrome_lang_id::NNetLanguageIdentifier::Result& data;
+
+  VALUE convert() const {
+    if (data.language == chrome_lang_id::NNetLanguageIdentifier::kUnknown)
+      return Qnil;
+
+    VALUE byte_ranges = rb_ary_new2(data.byte_ranges.size());
+    for (auto& byte_range_data : data.byte_ranges) {
+      VALUE argv[] = {
+        INT2NUM(byte_range_data.start_index),
+        INT2NUM(byte_range_data.end_index),
+        DBL2NUM(byte_range_data.probability),
+      };
+
+      VALUE byte_range = rb_class_new_instance(sizeof(argv) / sizeof(*argv),
+                                               argv,
+                                               span_info_klass);
+      rb_ary_push(byte_ranges, byte_range);
+    }
+
+    VALUE argv[] = {
+      ID2SYM(rb_intern2(data.language.data(), data.language.size())),
+      DBL2NUM(data.probability),
+      data.is_reliable ? Qtrue : Qfalse,
+      DBL2NUM(data.proportion),
+      byte_ranges,
+    };
+
+    return rb_class_new_instance(sizeof(argv) / sizeof(*argv), argv,
+                                 result_klass);
+  }
 };
 
-struct OwningResult {
-  OwningResult(chrome_lang_id::NNetLanguageIdentifier::Result&& result) {
-    references.language = std::move(result.language);
-    references.byte_ranges = std::move(result.byte_ranges);
-    plain.language.data = references.language.data();
-    plain.language.size = references.language.size();
-    plain.byte_ranges.data = references.byte_ranges.data();
-    plain.byte_ranges.size = references.byte_ranges.size();
-    plain.probability = result.probability;
-    plain.proportion = result.proportion;
-    plain.is_reliable = result.is_reliable;
+struct ResultVector {
+  VALUE result_klass;
+  VALUE span_info_klass;
+  VALUE buffer;
+  const std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>& data;
+
+  VALUE convert() const {
+    for (auto& element_data : data) {
+      Result result { result_klass, span_info_klass, element_data };
+      VALUE element = result.convert();
+      if (element == Qnil)
+        break;
+
+      rb_ary_push(buffer, element);
+    }
+
+    return buffer;
   }
+};
 
-  Result plain;
-  struct {
-    std::string language;
-    std::vector<chrome_lang_id::NNetLanguageIdentifier::SpanInfo> byte_ranges;
-  } references;
+template<typename T>
+VALUE convert_protected(VALUE arg)
+{
+    auto result = reinterpret_cast<const T *>(arg);
+    return result->convert();
+}
+
+static void dfree(void *arg) {
+  auto data = static_cast<chrome_lang_id::NNetLanguageIdentifier *>(arg);
+  data->~NNetLanguageIdentifier();
+  xfree(arg);
+}
+
+static size_t dsize(const void *data) {
+  return sizeof(chrome_lang_id::NNetLanguageIdentifier);
+}
+
+static const rb_data_type_t data_type = {
+  .wrap_struct_name = "CLD3::NNetLanguageIdentifier",
+  .function = {
+    .dfree = dfree,
+    .dsize = dsize,
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
 };
 
-extern "C" {
-  EXPORT OwningResult *NNetLanguageIdentifier_find_language(
-      chrome_lang_id::NNetLanguageIdentifier *instance,
-      const char *data,
-      std::size_t size) {
-    return new OwningResult(instance->FindLanguage(std::string(data, size)));
+static VALUE find_language(VALUE obj,
+                           VALUE result_klass, VALUE span_info_klass,
+                           VALUE text) {
+  int state;
+  VALUE converted;
+
+  {
+    chrome_lang_id::NNetLanguageIdentifier *data;
+    TypedData_Get_Struct(obj, chrome_lang_id::NNetLanguageIdentifier,
+                         &data_type, data);
+    std::string text_string = std::string(RSTRING_PTR(text), RSTRING_LEN(text));
+    auto result_data = data->FindLanguage(text_string);
+    Result result { result_klass, span_info_klass, result_data };
+
+    converted = rb_protect(convert_protected<Result>,
+                           reinterpret_cast<VALUE>(&result),
+                           &state);
   }
 
-  EXPORT std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>*
-  NNetLanguageIdentifier_find_top_n_most_freq_langs(
-      chrome_lang_id::NNetLanguageIdentifier *instance,
-      const char *data, std::size_t size, int num_langs) {
-    std::string text(data, size);
-    return new auto(instance->FindTopNMostFreqLangs(text, num_langs));
-  }
+  if (state)
+    rb_jump_tag(state);
 
-  EXPORT void delete_NNetLanguageIdentifier(
-      chrome_lang_id::NNetLanguageIdentifier *pointer) {
-    delete pointer;
-  }
+  return converted;
+}
 
-  EXPORT void delete_result(OwningResult *pointer) {
-    delete pointer;
+static VALUE find_top_n_most_freq_langs(VALUE obj,
+                                        VALUE result_klass,
+                                        VALUE span_info_klass,
+                                        VALUE text,
+                                        VALUE num_langs) {
+  int state;
+  VALUE converted;
+
+  {
+    chrome_lang_id::NNetLanguageIdentifier *data;
+    TypedData_Get_Struct(obj, chrome_lang_id::NNetLanguageIdentifier,
+                         &data_type, data);
+    VALUE buffer = rb_ary_new2(NUM2INT(num_langs));
+    std::string text_string = std::string(RSTRING_PTR(text), RSTRING_LEN(text));
+    auto result_data = data->FindTopNMostFreqLangs(text_string, num_langs);
+    ResultVector result { result_klass, span_info_klass, buffer, result_data };
+
+    converted = rb_protect(convert_protected<ResultVector>,
+                           reinterpret_cast<VALUE>(&result),
+                           &state);
   }
 
-  EXPORT void delete_results(
-      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *pointer) {
-    delete pointer;
-  }
+  if (state)
+    rb_jump_tag(state);
 
-  EXPORT chrome_lang_id::NNetLanguageIdentifier *new_NNetLanguageIdentifier(
-      int min_num_bytes, int max_num_bytes) {
-    return new chrome_lang_id::NNetLanguageIdentifier(
-        min_num_bytes, max_num_bytes);
-  }
+  return converted;
+}
+
+static VALUE make(VALUE klass, VALUE min_num_bytes, VALUE max_num_bytes) {
+  int min_num_bytes_int = NUM2INT(min_num_bytes);
+  int max_num_bytes_int = NUM2INT(max_num_bytes);
+  chrome_lang_id::NNetLanguageIdentifier *data;
+  VALUE value = TypedData_Make_Struct(klass,
+                                      chrome_lang_id::NNetLanguageIdentifier,
+                                      &data_type, data);
+  new (data) chrome_lang_id::NNetLanguageIdentifier(min_num_bytes_int, max_num_bytes_int);
+  return value;
+}
 
-  EXPORT Result refer_to_nth_result(
-      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *results,
-      std::size_t index) {
-    Result c;
-    auto& cc = (*results)[index];
-
-    c.language.data = cc.language.data();
-    c.language.size = cc.language.size();
-    c.byte_ranges.data = cc.byte_ranges.data();
-    c.byte_ranges.size = cc.byte_ranges.size();
-    c.probability = cc.probability;
-    c.proportion = cc.proportion;
-    c.is_reliable = cc.is_reliable;
-
-    return c;
+extern "C" EXPORT void Init_cld3_ext() {
+  VALUE cld3 = rb_const_get(rb_cObject, rb_intern("CLD3"));
+  VALUE identifier = 
+    rb_const_get(cld3, rb_intern("NNetLanguageIdentifier"));
+  VALUE unstable = rb_const_get(identifier, rb_intern("Unstable"));
+  VALUE params = rb_const_get(cld3, rb_intern("TaskContextParams"));
+  VALUE language_names = rb_const_get(params, rb_intern("LANGUAGE_NAMES"));
+
+  rb_define_const(identifier, "MIN_NUM_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMinNumBytesToConsider));
+  rb_define_const(identifier, "MAX_NUM_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMaxNumBytesToConsider));
+  rb_define_const(identifier, "MAX_NUM_INPUT_BYTES_TO_CONSIDER",
+                  INT2NUM(chrome_lang_id::NNetLanguageIdentifier::kMaxNumInputBytesToConsider));
+  rb_define_const(identifier, "RELIABILITY_THRESHOLD",
+                  DBL2NUM(chrome_lang_id::NNetLanguageIdentifier::kReliabilityThreshold));
+  rb_define_const(identifier, "RELIABILITY_HR_BS_THRESHOLD",
+                  DBL2NUM(chrome_lang_id::NNetLanguageIdentifier::kReliabilityHrBsThreshold));
+
+  rb_define_singleton_method(unstable, "make", make, 2);
+  rb_define_method(unstable, "find_language", find_language, 3);
+  rb_define_method(unstable, "find_top_n_most_freq_langs",
+                   find_top_n_most_freq_langs, 4);
+
+  for (int i = 0; ; i++) {
+    const char *name = chrome_lang_id::TaskContextParams::language_names(i);
+    if (!name)
+      break;
+
+    rb_ary_push(language_names, ID2SYM(rb_intern(name)));
   }
 }