cld3 3.4.2 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f5b3cc203abda97cb85d5dee0983b7f63c626397b8af8b90e2110bb5fedbbdec
-  data.tar.gz: 197f66798925404ded7af722d0194a705018d6953b11f4576c4e180ea093675d
+  metadata.gz: e12ff58a2cc0242896e307f33cb1adb2e210a1889244861e2c751b0399cb6415
+  data.tar.gz: 3b0c348b00126c8ee825a76eed0919e074a7f97e7dc7e647614f7a754c69d716
 SHA512:
-  metadata.gz: 855e8ee464a2842906bfef211e2afb21820fe9a7449b58d91b9ab1908c997966b9dd4c2d5d51f82ceb84b65b5a118736a5aa4eff6ea9548b9a9abc61b297a9d0
-  data.tar.gz: e38ddfd81489aeb83bccc7b509dd17ea79c56ba641de37cac2d800d3428ed31e5ac57066016bd118e9e71c30c78d31b4c38a266abe012065495558adf07e68f5
+  metadata.gz: 690c6e6f0cc8f0cc369b05a611daeed74adc273cd9c02f780233921ed0c220a425552e7020c127ff0503e88d473f57088cafd3b0d2143f3f86527d886d3eaf63
+  data.tar.gz: 6c3b7c105fa799fb918a077d9d1f47c948ec29bad86e57f452ea9eae82fc4f5bfc2686654a2a68ebc7102dfcb34e2a1ca55648d75a1f6bcf97b532de482de76e

data/README.md

@@ -19,7 +19,6 @@ cld3.find_language("здравствуйте") # => #<struct Struct::Result lang
 ### Prerequisites
 * [Bundler](http://bundler.io/)
 * C++ compiler
-* [Protocol buffers](https://developers.google.com/protocol-buffers/)
 * [Rake](https://ruby.github.io/rake/)
 * [RubyGems](https://rubygems.org/)
 
@@ -41,36 +40,16 @@ JRuby has a bug which prevents the feature detection. Apply the following
 change:
 https://github.com/jruby/jruby/pull/4118/commits/edad375ef4dcf195b19ce0afe4befac66468c736
 
-#### OpenBSD
-Ruby has a bug which recognizes non-fatal linker warnings as fatal. Apply the
-following patch to Ruby to workaround the bug.
-
-```diff
---- a/lib/mkmf.rb
-+++ b/lib/mkmf.rb
-@@ -657,7 +657,7 @@ def with_ldflags(flags)
-   end
- 
-   def try_ldflags(flags, opts = {})
--    try_link(MAIN_DOES_NOTHING, flags, {:werror => true}.update(opts))
-+    try_link(MAIN_DOES_NOTHING, flags, {:werror => false}.update(opts))
-   end
- 
-   def append_ldflags(flags, *opts)
-```
-
 ### Troubleshooting
-`gem install cld3` triggers native library building. If it fails, you are likely
-to missing required facilities. Make sure C++ compiler and protocol buffers
-is installed. I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler. Ruby is
-likely to need [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
-as well.
+`gem install cld3` triggers native library building. If it fails, it is likely
+that some required facilities are missing. Make sure C++ compiler is installed.
+I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler.
 
 Runtime errors are likely to be issues of [FFI](https://github.com/ffi/ffi) or
 programming errors. Make sure they are all correct.
 
 If you cannot identify the cause of your problem, run spec of this library and
-see whether the problem is reproducable with it or not. Spec is not included in
+see whether the problem is reproducible with it or not. Spec is not included in
 the gem, so clone the source code repository and then run `rake spec`.
 The source code repository is at
 https://github.com/akihikodaki/cld3-ruby.

data/cld3.gemspec

@@ -16,19 +16,21 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.4.2"
+  gem.version = "3.5.0"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"
   gem.homepage = "https://github.com/akihikodaki/cld3-ruby"
   gem.author = "Akihiko Odaki"
   gem.email = "akihiko.odaki@gmail.com"
-  gem.required_ruby_version = [ ">= 2.6.0", "< 3.1.0" ]
+  gem.required_ruby_version = [ ">= 2.7.0", "< 3.3.0" ]
   gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.16.0" ]
-  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.11.0" ]
+  gem.add_development_dependency "rbs", [ ">= 2.6.0", "< 2.7.0" ]
+  gem.add_development_dependency "rspec", [ ">= 3.11.0", "< 3.12.0" ]
+  gem.add_development_dependency "steep", [ ">= 1.0.0", "< 1.1.0" ]
   gem.files = Dir[
     "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
-    "cld3.gemspec", "ext/**/*", "lib/**/*"
+    "cld3.gemspec", "ext/**/*", "lib/**/*", "sig/**/*"
   ]
   gem.require_paths = [ "lib" ]
   gem.extensions = [ "ext/cld3/extconf.rb" ]

data/ext/cld3/Makefile

@@ -3,6 +3,7 @@ SHELL = /bin/sh
 
 # V=0 quiet, V=1 verbose.  other values don't work.
 V = 1
+V0 = $(V:0=)
 Q1 = $(V:1=)
 Q = $(Q1:0=@)
 ECHO1 = $(V:1=@ :)
@@ -52,7 +53,8 @@ htmldir = $(docdir)
 infodir = $(DESTDIR)/usr/share/info
 docdir = $(datarootdir)/doc/$(PACKAGE)
 oldincludedir = $(DESTDIR)/usr/include
-includedir = $(DESTDIR)/usr/include
+includedir = $(exec_prefix)/include
+runstatedir = $(localstatedir)/run
 localstatedir = $(DESTDIR)/var
 sharedstatedir = $(DESTDIR)/var/lib
 sysconfdir = $(DESTDIR)/etc
@@ -79,23 +81,23 @@ CSRCFLAG = $(empty)
 RUBY_EXTCONF_H = 
 cflags   = $(optflags) $(debugflags) $(warnflags)
 cxxflags = 
-optflags = -O3
+optflags = -O3 -fno-fast-math
 debugflags = -ggdb3
-warnflags = -Wall -Wextra -Wdeprecated-declarations -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wwrite-strings -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable
+warnflags = -Wall -Wextra -Wdeprecated-declarations -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wwrite-strings -Wold-style-definition -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable -Wundef
 cppflags = 
 CCDLFLAGS = -fPIC
-CFLAGS   = $(CCDLFLAGS) -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC  $(ARCH_FLAG)
-INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir) 
+CFLAGS   = $(CCDLFLAGS) -O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fPIC $(ARCH_FLAG)
+INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
 DEFS     = 
 CPPFLAGS =   $(DEFS) $(cppflags)
-CXXFLAGS = $(CCDLFLAGS) -O2  -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection  -fvisibility=hidden -std=c++11 $(ARCH_FLAG)
-ldflags  = -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  -fstack-protector-strong -rdynamic -Wl,-export-dynamic 
-dldflags = -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld  
+CXXFLAGS = $(CCDLFLAGS) -O2 -flto=auto -ffat-lto-objects -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -fstack-protector-strong -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -mbranch-protection=standard -fasynchronous-unwind-tables -fstack-clash-protection -fvisibility=hidden -std=c++17 $(ARCH_FLAG)
+ldflags  = -L. -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  -fstack-protector-strong -rdynamic -Wl,-export-dynamic -Wl,--no-as-needed
+dldflags = -Wl,-z,relro -Wl,--as-needed  -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1  -Wl,--build-id=sha1  
 ARCH_FLAG = 
 DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
 LDSHARED = $(CC) -shared
 LDSHAREDXX = $(CXX) -shared
-AR = ar
+AR = gcc-ar
 EXEEXT = 
 
 RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
@@ -107,13 +109,13 @@ RUBY_BASE_NAME = ruby
 
 arch = aarch64-linux
 sitearch = $(arch)
-ruby_version = 2.7.0
+ruby_version = 3.1.0
 ruby = $(bindir)/$(RUBY_BASE_NAME)
 RUBY = $(ruby)
 ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
 
 RM = rm -f
-RM_RF = $(RUBY) -run -e rm -- -rf
+RM_RF = rm -fr
 RMDIRS = rmdir --ignore-fail-on-non-empty -p
 MAKEDIRS = /usr/bin/mkdir -p
 INSTALL = /usr/bin/install -c
@@ -137,11 +139,11 @@ extout =
 extout_prefix = 
 target_prefix = 
 LOCAL_LIBS = 
-LIBS =   -lprotobuf -lpthread -lm   -lc
-ORIG_SRCS = base.cc embedding_feature_extractor.cc embedding_network.cc feature_extractor.cc feature_extractor.pb.cc feature_types.cc fixunicodevalue.cc fml_parser.cc generated_entities.cc generated_ulscript.cc getonescriptspan.cc lang_id_nn_params.cc language_identifier_features.cc nnet_language_identifier.cc nnet_language_identifier_c.cc offsetmap.cc registry.cc relevant_script_feature.cc sentence.pb.cc sentence_features.cc task_context.cc task_context_params.cc task_spec.pb.cc text_processing.cc unicodetext.cc utf8statetable.cc utils.cc workspace.cc
+LIBS =   -lm  -lc
+ORIG_SRCS = base.cc embedding_feature_extractor.cc embedding_network.cc feature_extractor.cc feature_types.cc fixunicodevalue.cc fml_parser.cc generated_entities.cc generated_ulscript.cc getonescriptspan.cc lang_id_nn_params.cc language_identifier_features.cc nnet_language_identifier.cc nnet_language_identifier_c.cc offsetmap.cc registry.cc relevant_script_feature.cc sentence_features.cc task_context.cc task_context_params.cc text_processing.cc unicodetext.cc utf8statetable.cc utils.cc workspace.cc
 SRCS = $(ORIG_SRCS) 
-OBJS = base.o embedding_feature_extractor.o embedding_network.o feature_extractor.o feature_extractor.pb.o feature_types.o fixunicodevalue.o fml_parser.o generated_entities.o generated_ulscript.o getonescriptspan.o lang_id_nn_params.o language_identifier_features.o nnet_language_identifier.o nnet_language_identifier_c.o offsetmap.o registry.o relevant_script_feature.o sentence.pb.o sentence_features.o task_context.o task_context_params.o task_spec.pb.o text_processing.o unicodetext.o utf8statetable.o utils.o workspace.o
-HDRS = $(srcdir)/base.h $(srcdir)/casts.h $(srcdir)/embedding_feature_extractor.h $(srcdir)/embedding_network.h $(srcdir)/embedding_network_params.h $(srcdir)/feature_extractor.h $(srcdir)/feature_types.h $(srcdir)/float16.h $(srcdir)/fml_parser.h $(srcdir)/language_identifier_features.h $(srcdir)/lang_id_nn_params.h $(srcdir)/nnet_language_identifier.h $(srcdir)/registry.h $(srcdir)/relevant_script_feature.h $(srcdir)/script_detector.h $(srcdir)/sentence_features.h $(srcdir)/simple_adder.h $(srcdir)/fixunicodevalue.h $(srcdir)/generated_ulscript.h $(srcdir)/getonescriptspan.h $(srcdir)/integral_types.h $(srcdir)/offsetmap.h $(srcdir)/port.h $(srcdir)/stringpiece.h $(srcdir)/text_processing.h $(srcdir)/utf8acceptinterchange.h $(srcdir)/utf8prop_lettermarkscriptnum.h $(srcdir)/utf8repl_lettermarklower.h $(srcdir)/utf8scannot_lettermarkspecial.h $(srcdir)/utf8statetable.h $(srcdir)/task_context.h $(srcdir)/task_context_params.h $(srcdir)/unicodetext.h $(srcdir)/utils.h $(srcdir)/workspace.h $(srcdir)/feature_extractor.pb.h $(srcdir)/sentence.pb.h $(srcdir)/task_spec.pb.h
+OBJS = base.o embedding_feature_extractor.o embedding_network.o feature_extractor.o feature_types.o fixunicodevalue.o fml_parser.o generated_entities.o generated_ulscript.o getonescriptspan.o lang_id_nn_params.o language_identifier_features.o nnet_language_identifier.o nnet_language_identifier_c.o offsetmap.o registry.o relevant_script_feature.o sentence_features.o task_context.o task_context_params.o text_processing.o unicodetext.o utf8statetable.o utils.o workspace.o
+HDRS = $(srcdir)/base.h $(srcdir)/casts.h $(srcdir)/embedding_feature_extractor.h $(srcdir)/embedding_network.h $(srcdir)/embedding_network_params.h $(srcdir)/feature_extractor.h $(srcdir)/feature_types.h $(srcdir)/fixunicodevalue.h $(srcdir)/float16.h $(srcdir)/fml_parser.h $(srcdir)/generated_ulscript.h $(srcdir)/getonescriptspan.h $(srcdir)/integral_types.h $(srcdir)/lang_id_nn_params.h $(srcdir)/language_identifier_features.h $(srcdir)/nnet_language_identifier.h $(srcdir)/offsetmap.h $(srcdir)/port.h $(srcdir)/registry.h $(srcdir)/relevant_script_feature.h $(srcdir)/script_detector.h $(srcdir)/sentence_features.h $(srcdir)/simple_adder.h $(srcdir)/stringpiece.h $(srcdir)/task_context.h $(srcdir)/task_context_params.h $(srcdir)/text_processing.h $(srcdir)/unicodetext.h $(srcdir)/utf8acceptinterchange.h $(srcdir)/utf8prop_lettermarkscriptnum.h $(srcdir)/utf8repl_lettermarklower.h $(srcdir)/utf8scannot_lettermarkspecial.h $(srcdir)/utf8statetable.h $(srcdir)/utils.h $(srcdir)/workspace.h
 LOCAL_HDRS = 
 TARGET = libcld3
 TARGET_NAME = libcld3
@@ -155,11 +157,11 @@ BINDIR        = $(bindir)
 RUBYCOMMONDIR = $(sitedir)$(target_prefix)
 RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
 RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
-HDRDIR        = $(rubyhdrdir)/ruby$(target_prefix)
-ARCHHDRDIR    = $(rubyhdrdir)/$(arch)/ruby$(target_prefix)
+HDRDIR        = $(sitehdrdir)$(target_prefix)
+ARCHHDRDIR    = $(sitearchhdrdir)$(target_prefix)
 TARGET_SO_DIR =
 TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
-CLEANLIBS     = $(TARGET_SO) 
+CLEANLIBS     = $(TARGET_SO) false
 CLEANOBJS     = *.o  *.bak
 
 all:    $(DLLIB)
@@ -172,7 +174,7 @@ clean-rb-default::
 clean-rb::
 clean-so::
 clean: clean-so clean-static clean-rb-default clean-rb
-		-$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
+		-$(Q)$(RM_RF) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
 
 distclean-rb-default::
 distclean-rb::

data/ext/cld3/cld_3/protos/feature_extractor.pb.h

@@ -0,0 +1,100 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef FEATURE_EXTRACTOR_PB_H_
+#define FEATURE_EXTRACTOR_PB_H_
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace chrome_lang_id {
+
+class Parameter {
+ public:
+  const std::string& name() const { return name_; }
+  void set_name(std::string value) { name_ = std::move(value); }
+  const std::string& value() const { return value_; }
+  void set_value(std::string value) { value_ = std::move(value); }
+
+ private:
+  std::string name_;
+  std::string value_;
+};
+
+class FeatureFunctionDescriptor {
+ public:
+  const std::string& type() const { return type_; }
+
+  void set_type(std::string value) { type_ = std::move(value); }
+
+  const std::string& name() const { return name_; }
+
+  void set_name(std::string value) { name_ = std::move(value); }
+
+  bool has_argument() const { return true; }
+
+  std::int32_t argument() const { return argument_; }
+
+  void set_argument(int32_t value) { argument_ = value; }
+
+  int parameter_size() const { return parameter_.size(); }
+
+  const Parameter& parameter(int index) const { return parameter_[index]; }
+
+  Parameter* add_parameter() { return &parameter_.emplace_back(); }
+
+  int feature_size() const { return feature_.size(); }
+
+  FeatureFunctionDescriptor* mutable_feature(int index) {
+    return &feature_[index];
+  }
+
+  const FeatureFunctionDescriptor& feature(int index) const {
+    return feature_[index];
+  }
+
+  FeatureFunctionDescriptor* add_feature() { return &feature_.emplace_back(); }
+
+ private:
+  std::string type_;
+  std::string name_;
+  std::int32_t argument_;
+  std::vector<Parameter> parameter_;
+  std::vector<FeatureFunctionDescriptor> feature_;
+};
+
+class FeatureExtractorDescriptor {
+ public:
+  int feature_size() const { return feature_.size(); }
+
+  FeatureFunctionDescriptor* mutable_feature(int index) {
+    return &feature_[index];
+  }
+
+  const FeatureFunctionDescriptor& feature(int index) const {
+    return feature_[index];
+  }
+
+  FeatureFunctionDescriptor* add_feature() { return &feature_.emplace_back(); }
+
+ private:
+  std::vector<FeatureFunctionDescriptor> feature_;
+};
+
+}
+
+#endif

data/ext/cld3/cld_3/protos/sentence.pb.h

@@ -0,0 +1,35 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef SENTENCE_PB_H_
+#define SENTENCE_PB_H_
+
+#include <string>
+
+namespace chrome_lang_id {
+
+class Sentence {
+ public:
+  const std::string& text() const { return text_; }
+  void set_text(std::string value) { text_ = std::move(value); }
+
+ private:
+  std::string text_;
+};
+
+}
+
+#endif

data/ext/cld3/cld_3/protos/task_spec.pb.h

@@ -0,0 +1,106 @@
+/* Copyright 2022 Akihiko Odaki <akihiko.odaki@gmail.com>
+All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TASK_SPEC_PB_H_
+#define TASK_SPEC_PB_H_
+
+#include <string>
+#include <vector>
+
+namespace chrome_lang_id {
+
+class TaskInput {
+ public:
+  class Part {
+   public:
+    const std::string& file_pattern() const { return file_pattern_; }
+
+   private:
+    std::string file_pattern_;
+  };
+
+  const std::string& name() const { return name_; }
+
+  void set_name(std::string value) { name_ = value; }
+
+  int file_format_size() const { return file_format_.size(); }
+
+  const std::string& file_format(int index) const {
+    return file_format_[index];
+  }
+
+  void add_file_format(std::string value) {
+    file_format_.push_back(std::move(value));
+  }
+
+  int record_format_size() const { return record_format_.size(); }
+
+  const std::string& record_format(int index) const {
+    return record_format_[index];
+  }
+
+  void add_record_format(std::string value) {
+    record_format_.push_back(std::move(value));
+  }
+
+  int part_size() const { return part_.size(); }
+  const Part& part(int index) const { return part_[index]; }
+
+ private:
+  std::string name_;
+  std::vector<std::string> file_format_;
+  std::vector<std::string> record_format_;
+  std::vector<Part> part_;
+};
+
+class TaskSpec {
+ public:
+  class Parameter {
+   public:
+    const std::string& name() const { return name_; }
+    void set_name(std::string value) { name_ = std::move(value); }
+    const std::string& value() const { return value_; }
+    void set_value(std::string value) { value_ = std::move(value); }
+
+   private:
+    std::string name_;
+    std::string value_;
+  };
+
+  int parameter_size() const { return parameter_.size(); }
+
+  Parameter* mutable_parameter(int index) { return &parameter_[index]; }
+
+  const Parameter& parameter(int index) const { return parameter_[index]; }
+
+  Parameter* add_parameter() { return &parameter_.emplace_back(); }
+
+  int input_size() const { return input_.size(); }
+
+  TaskInput* mutable_input(int index) { return &input_[index]; }
+
+  const TaskInput& input(int index) const { return input_[index]; }
+
+  TaskInput* add_input() { return &input_.emplace_back(); }
+
+ private:
+  std::vector<Parameter> parameter_;
+  std::vector<TaskInput> input_;
+};
+
+}
+
+#endif

data/ext/cld3/embedding_network.cc

@@ -167,6 +167,7 @@ EmbeddingNetwork::EmbeddingNetwork(const EmbeddingNetworkParams *model)
   for (int i = 0; i < model_->embedding_dim_size(); ++i) {
     CLD3_DCHECK(offset_sum == model_->concat_offset(i));
     offset_sum += model_->embedding_dim(i) * model_->embedding_num_features(i);
+    (void)offset_sum;  // Avoid compiler warning for "unused" variable.
     embedding_matrices_.emplace_back(model_->GetEmbeddingMatrix(i));
   }
 

data/ext/cld3/extconf.rb

@@ -26,17 +26,8 @@ rescue
   end
 end
 
-# Check pkg-config first to inform the library is missing if so.
-pkg_config("protobuf") or abort "Failed to locate protobuf"
-
-FileUtils.mkdir_p("cld_3/protos")
 FileUtils.mkdir_p("script_span")
 
-[ "feature_extractor", "sentence", "task_spec" ].each {|name|
-  system "protoc", "#{name}.proto", "--cpp_out=.", exception: true
-  ln_fallback("#{name}.pb.h", "cld_3/protos/#{name}.pb.h")
-}
-
 [
   "fixunicodevalue.h",
   "generated_ulscript.h",
@@ -55,6 +46,6 @@ FileUtils.mkdir_p("script_span")
   ln_fallback("#{name}", "script_span/#{name}")
 }
 
-$CXXFLAGS += " -fvisibility=hidden -std=c++11"
+$CXXFLAGS += " -fvisibility=hidden -std=c++17"
 $LIBRUBYARG = ""
 create_makefile("libcld3")

data/ext/cld3/getonescriptspan.cc

@@ -878,7 +878,6 @@ bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
   // copying letters to buffer with single spaces for each run of non-letters
   while (take < byte_length_) {
     // Copy run of letters in same script (&LS | LS)*
-    int letter_count = 0;              // Keep track of word length
     bool need_break = false;
 
     while (take < byte_length_) {
@@ -963,7 +962,6 @@ bool ScriptScanner::GetOneScriptSpan(LangSpan* span) {
         map2original_.Delete(tlen - plen);
       }
 
-      ++letter_count;
       if (put >= kMaxScriptBytes) {
         // Buffer is full
         span->truncated = true;

data/ext/cld3/getonescriptspan.h

@@ -33,14 +33,14 @@ static const int kMaxScriptBytes = kMaxScriptBuffer - 32;   // Leave some room
 static const int kWithinScriptTail = 32;    // Stop at word space in last
                                             // N bytes of script buffer
 
-typedef struct {
+struct LangSpan {
   char* text = nullptr;   // Pointer to the span, somewhere
   int text_bytes = 0;     // Number of bytes of text in the span
   int offset = 0;         // Offset of start of span in original input buffer
   ULScript ulscript = UNKNOWN_ULSCRIPT; // Unicode Letters Script of this span
   bool truncated = false; // true if buffer filled up before a
                           // different script or EOF was found
-} LangSpan;
+};
 
 static inline bool IsContinuationByte(char c) {
   return static_cast<signed char>(c) < -64;

data/ext/cld3/nnet_language_identifier.cc

@@ -284,8 +284,6 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
   CLD2::LangSpan script_span;
   std::unordered_map<string, LangChunksStats> lang_stats;
   int total_num_bytes = 0;
-  Result result;
-  string language;
   int chunk_size = 0;  // Use the default.
   while (ss.GetOneScriptSpanLower(&script_span)) {
     const int num_original_span_bytes = script_span.text_bytes;
@@ -302,8 +300,8 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
 
     const string selected_text = SelectTextGivenScriptSpan(script_span);
 
-    result = FindLanguageOfValidUTF8(selected_text);
-    language = result.language;
+    Result result = FindLanguageOfValidUTF8(selected_text);
+    string language = result.language;
     lang_stats[language].byte_sum += num_original_span_bytes;
     lang_stats[language].prob_sum +=
         result.probability * num_original_span_bytes;
@@ -356,7 +354,7 @@ string NNetLanguageIdentifier::SelectTextGivenBeginAndSize(
     const char *text_begin, int text_size) {
   string output_text;
 
-  // If the size of the input is greater than the maxium number of bytes needed
+  // If the size of the input is greater than the maximum number of bytes needed
   // for a prediction, then concatenate snippets that are equally spread out
   // throughout the input.
   if (text_size > max_num_bytes_) {

data/ext/cld3/script_span/fixunicodevalue.h

@@ -0,0 +1,69 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Routine that maps a Unicode code point to an interchange-valid one
+//
+// Table that maps MS CP1252 bytes 00-FF to their corresponding Unicode
+// code points. C0 and C1 control codes that are not interchange-valid
+// are mapped to spaces.
+
+
+#ifndef SCRIPT_SPAN_FIXUNICODEVALUE_H_
+#define SCRIPT_SPAN_FIXUNICODEVALUE_H_
+
+#include "integral_types.h" // for char32
+#include "port.h"
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+// Map byte value 0000-00FF to char32
+// Maps C0 control codes (other than CR LF HT FF) to space [29 instances including DEL=0x7F]
+// Maps C1 control codes to CP1252 [27 instances] or space [5 instances]
+static const char32 kMapFullMicrosoft1252OrSpace[256] = {
+  0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x09,0x0a,0x20, 0x0c,0x0d,0x20,0x20, // 00
+  0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20,
+  0x20,0x21,0x22,0x23, 0x24,0x25,0x26,0x27, 0x28,0x29,0x2a,0x2b, 0x2c,0x2d,0x2e,0x2f,
+  0x30,0x31,0x32,0x33, 0x34,0x35,0x36,0x37, 0x38,0x39,0x3a,0x3b, 0x3c,0x3d,0x3e,0x3f,
+
+  0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47, 0x48,0x49,0x4a,0x4b, 0x4c,0x4d,0x4e,0x4f, // 40
+  0x50,0x51,0x52,0x53, 0x54,0x55,0x56,0x57, 0x58,0x59,0x5a,0x5b, 0x5c,0x5d,0x5e,0x5f,
+  0x60,0x61,0x62,0x63, 0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b, 0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73, 0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x7b, 0x7c,0x7d,0x7e,0x20,
+
+  0x20ac,0x20,0x201a,0x0192, 0x201e,0x2026,0x2020,0x2021,                             // 80
+  0x02c6,0x2030,0x0160,0x2039, 0x0152,0x20,0x017d,0x20,
+  0x20,0x2018,0x2019,0x201c, 0x201d,0x2022,0x2013,0x2014,
+  0x02dc,0x2122,0x0161,0x203a, 0x0153,0x20,0x017e,0x0178,
+  0xa0,0xa1,0xa2,0xa3, 0xa4,0xa5,0xa6,0xa7, 0xa8,0xa9,0xaa,0xab, 0xac,0xad,0xae,0xaf, // A0
+  0xb0,0xb1,0xb2,0xb3, 0xb4,0xb5,0xb6,0xb7, 0xb8,0xb9,0xba,0xbb, 0xbc,0xbd,0xbe,0xbf,
+
+  0xc0,0xc1,0xc2,0xc3, 0xc4,0xc5,0xc6,0xc7, 0xc8,0xc9,0xca,0xcb, 0xcc,0xcd,0xce,0xcf, // C0
+  0xd0,0xd1,0xd2,0xd3, 0xd4,0xd5,0xd6,0xd7, 0xd8,0xd9,0xda,0xdb, 0xdc,0xdd,0xde,0xdf,
+  0xe0,0xe1,0xe2,0xe3, 0xe4,0xe5,0xe6,0xe7, 0xe8,0xe9,0xea,0xeb, 0xec,0xed,0xee,0xef,
+  0xf0,0xf1,0xf2,0xf3, 0xf4,0xf5,0xf6,0xf7, 0xf8,0xf9,0xfa,0xfb, 0xfc,0xfd,0xfe,0xff,
+};
+
+// Guarantees that the resulting output value is interchange valid
+//  00-FF; map to spaces or MS CP1252
+//  D800-DFFF; surrogates
+//  FDD0-FDEF; non-characters
+//  xxFFFE-xxFFFF; non-characters
+char32 FixUnicodeValue(char32 uv);
+
+}       // End namespace CLD2
+}       // End namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_FIXUNICODEVALUE_H_