cld3 3.2.6 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ee3c6166247aaf958310ffa9976400fcfa5050eb1969dd17e186e3500dd06d9
-  data.tar.gz: bfa75958e205683dfa2429d388bb40d18d237ef5c2b5877a9fd718489f95b7bd
+  metadata.gz: 44d2292a62861aa8551a46b69ebf0d55f518bf07ab7b20605ec7db61cd58d6c4
+  data.tar.gz: e706b2bc83c2f4915c95c0f9a9d60b8f4728626ca47f705cd3bd0dbc2c555c11
 SHA512:
-  metadata.gz: ac1fb08ebf438995878bb7c992bc2e2a71adf9d0f06f01316121b6d8d48f5b8f2f1ea9a3f68f501dad6682168b7a3e16b0137be16cae7ad876d0bce9f6d866e7
-  data.tar.gz: 40e1036c1c7e08af1caed6efd187a04bb8883f9ff427c61824193f72f58e696279611575f7a63b8cbd80fff2c9f20da644807e71be2ea281a8d870e3721410bd
+  metadata.gz: 1f70a575dbb1c18ceb0c9f79588e1cdd1d15a09fc2b0ea8cd0ea6dbc24837d2a2d9619bc555349552b64e9b3ec29b51e0dd062384a2f798ae27aa2ddd3803cb7
+  data.tar.gz: 59952aaf853da6e5acc5a1043f1ccf1777ed708848d9be7ba51e64fc0f1667a4f9319faaa118e07c1fb1ef2eac80f8cc0112fe5d9224b0c3fa733bb33221498c

data/README.md

@@ -8,11 +8,11 @@ require 'cld3'
 
 cld3 = CLD3::NNetLanguageIdentifier.new(0, 1000)
 
-cld3.find_language("こんにちは") # => #<struct Struct::Result language=:ja, probability=1.0, reliable?=true, proportion=1.0>
+cld3.find_language("こんにちは") # => #<struct Struct::Result language=:ja, probability=1.0, reliable?=true, proportion=1.0, byte_ranges=[]>
 
-cld3.find_language("This is a pen.") # => #<struct Struct::Result language=:en, probability=0.9999408721923828, reliable?=true, proportion=1.0>
+cld3.find_language("This is a pen.") # => #<struct Struct::Result language=:en, probability=0.9999408721923828, reliable?=true, proportion=1.0, byte_ranges=[]>
 
-cld3.find_language("здравствуйте") # => #<struct Struct::Result language=:ru, probability=0.3140212297439575, reliable?=false, proportion=1.0>
+cld3.find_language("здравствуйте") # => #<struct Struct::Result language=:ru, probability=0.3140212297439575, reliable?=false, proportion=1.0, byte_ranges=[]>
 ```
 
 ## Installation

data/cld3.gemspec

@@ -16,7 +16,7 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.2.6"
+  gem.version = "3.3.0"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"

data/ext/cld3/nnet_language_identifier_c.cc

@@ -26,42 +26,90 @@ limitations under the License.
   #define EXPORT __attribute__ ((visibility ("default")))
 #endif
 
-struct NNetLanguageIdentifier {
-  chrome_lang_id::NNetLanguageIdentifier context;
-  std::string language;
-};
-
 struct Result {
   struct {
     const char *data;
     std::size_t size;
   } language;
+  struct {
+    const chrome_lang_id::NNetLanguageIdentifier::SpanInfo *data;
+    std::size_t size;
+  } byte_ranges;
   float probability;
   float proportion;
   bool is_reliable;
 };
 
+struct OwningResult {
+  OwningResult(chrome_lang_id::NNetLanguageIdentifier::Result&& result) {
+    references.language = std::move(result.language);
+    references.byte_ranges = std::move(result.byte_ranges);
+    plain.language.data = references.language.data();
+    plain.language.size = references.language.size();
+    plain.byte_ranges.data = references.byte_ranges.data();
+    plain.byte_ranges.size = references.byte_ranges.size();
+    plain.probability = result.probability;
+    plain.proportion = result.proportion;
+    plain.is_reliable = result.is_reliable;
+  }
+
+  Result plain;
+  struct {
+    std::string language;
+    std::vector<chrome_lang_id::NNetLanguageIdentifier::SpanInfo> byte_ranges;
+  } references;
+};
+
 extern "C" {
-  EXPORT Result NNetLanguageIdentifier_find_language(void *pointer,
-                                                     const char *data,
-                                                     std::size_t size) {
-    auto instance = static_cast<NNetLanguageIdentifier *>(pointer);
-    auto result = instance->context.FindLanguage(std::string(data, size));
-    instance->language = std::move(result.language);
-
-    return Result {
-        { instance->language.data(), instance->language.size() },
-        result.probability,
-        result.proportion,
-        result.is_reliable
-    };
+  EXPORT OwningResult *NNetLanguageIdentifier_find_language(
+      chrome_lang_id::NNetLanguageIdentifier *instance,
+      const char *data,
+      std::size_t size) {
+    return new OwningResult(instance->FindLanguage(std::string(data, size)));
   }
 
-  EXPORT void delete_NNetLanguageIdentifier(void *pointer) {
-    delete static_cast<NNetLanguageIdentifier *>(pointer);
+  EXPORT std::vector<chrome_lang_id::NNetLanguageIdentifier::Result>*
+  NNetLanguageIdentifier_find_top_n_most_freq_langs(
+      chrome_lang_id::NNetLanguageIdentifier *instance,
+      const char *data, std::size_t size, int num_langs) {
+    std::string text(data, size);
+    return new auto(instance->FindTopNMostFreqLangs(text, num_langs));
   }
 
-  EXPORT void *new_NNetLanguageIdentifier(int min_num_bytes, int max_num_bytes) {
-    return new NNetLanguageIdentifier{{min_num_bytes, max_num_bytes}, {}};
+  EXPORT void delete_NNetLanguageIdentifier(
+      chrome_lang_id::NNetLanguageIdentifier *pointer) {
+    delete pointer;
+  }
+
+  EXPORT void delete_result(OwningResult *pointer) {
+    delete pointer;
+  }
+
+  EXPORT void delete_results(
+      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *pointer) {
+    delete pointer;
+  }
+
+  EXPORT chrome_lang_id::NNetLanguageIdentifier *new_NNetLanguageIdentifier(
+      int min_num_bytes, int max_num_bytes) {
+    return new chrome_lang_id::NNetLanguageIdentifier(
+        min_num_bytes, max_num_bytes);
+  }
+
+  EXPORT Result refer_to_nth_result(
+      std::vector<chrome_lang_id::NNetLanguageIdentifier::Result> *results,
+      std::size_t index) {
+    Result c;
+    auto& cc = (*results)[index];
+
+    c.language.data = cc.language.data();
+    c.language.size = cc.language.size();
+    c.byte_ranges.data = cc.byte_ranges.data();
+    c.byte_ranges.size = cc.byte_ranges.size();
+    c.probability = cc.probability;
+    c.proportion = cc.proportion;
+    c.is_reliable = cc.is_reliable;
+
+    return c;
   }
 }

data/lib/cld3.rb

@@ -49,10 +49,15 @@ module CLD3
     # This is Numeric object.
     RELIABILITY_HR_BS_THRESHOLD = 0.5
 
+    # Holds probability that Span, specified by start/end indices, is a given
+    # language. The langauge is not stored here; it can be found in Result, which
+    # holds an Array of SpanInfo.
+    SpanInfo = Struct.new(:start_index, :end_index, :probability)
+
     # Information about a predicted language.
     # This is an instance of Struct with the following members:
     #
-    # [language]    This is symbol or nil.
+    # [language]    This is symbol.
     #
     # [probability] Language probability. This is Numeric object.
     #
@@ -61,33 +66,95 @@ module CLD3
     # [proportion]  Proportion of bytes associated with the language. If
     #               #find_language is called, this variable is set to 1.
     #               This is Numeric object.
-    Result = Struct.new("Result", :language, :probability, :reliable?, :proportion)
+    #
+    # [byte_ranges] Specifies the byte ranges in UTF-8 that |language| applies to.
+    #               This is an Array of SpanInfo.
+    Result = Struct.new(:language, :probability, :reliable?, :proportion, :byte_ranges)
 
     # The arguments are two String objects.
-    def initialize(minNumBytes = MIN_NUM_BYTES_TO_CONSIDER, maxNumBytes = MAX_NUM_BYTES_TO_CONSIDER)
-      @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(minNumBytes, maxNumBytes))
+    def initialize(min_num_bytes = MIN_NUM_BYTES_TO_CONSIDER, max_num_bytes = MAX_NUM_BYTES_TO_CONSIDER)
+      @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(min_num_bytes, max_num_bytes))
     end
 
     # Finds the most likely language for the given text, along with additional
     # information (e.g., probability). The prediction is based on the first N
     # bytes where N is the minumum between the number of interchange valid UTF8
     # bytes and +max_num_bytes_+. If N is less than +min_num_bytes_+ long, then
-    # this function returns nil as language.
+    # this function returns nil.
     # The argument is a String object.
     # The returned value of this function is an instance of Result.
     def find_language(text)
       text_utf8 = text.encode(Encoding::UTF_8)
       pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
-      pointer.put_bytes(0, text_utf8)
 
-      cc_result = Unstable.NNetLanguageIdentifier_find_language(@cc, pointer, text_utf8.bytesize)
-      language = cc_result[:language_data].read_bytes(cc_result[:language_size])
+      begin
+        pointer.put_bytes(0, text_utf8)
+
+        result = Unstable.NNetLanguageIdentifier_find_language(@cc, pointer, text_utf8.bytesize)
+        begin
+          convert_result Unstable::NNetLanguageIdentifier::Result.new(result)
+        ensure
+          Unstable.delete_result result
+        end
+      ensure
+        pointer.free
+      end
+    end
+
+    # Splits the input text (up to the first byte, if any, that is not
+    # interchange valid UTF8) into spans based on the script, predicts a language
+    # for each span, and returns a vector storing the top num_langs most frequent
+    # languages along with additional information (e.g., proportions). The number
+    # of bytes considered for each span is the minimum between the size of the
+    # span and +max_num_bytes_+. If more languages are requested than what is
+    # available in the input, then the number of the returned elements will be
+    # the number of the latter. Also, if the size of the span is less than
+    # +min_num_bytes_+ long, then the span is skipped. If the input text is too
+    # long, only the first +MAX_NUM_INPUT_BYTES_TO_CONSIDER+ bytes are processed.
+    # The first argument is a String object.
+    # The second argument is Numeric object.
+    # The returned value of this functions is an Array of Result instances.
+    def find_top_n_most_freq_langs(text, num_langs)
+      text_utf8 = text.encode(Encoding::UTF_8)
+      pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
+
+      begin
+        pointer.put_bytes(0, text_utf8)
+
+        results = Unstable.NNetLanguageIdentifier_find_top_n_most_freq_langs(@cc, pointer, text_utf8.bytesize, num_langs)
+        begin
+          num_langs.times
+            .lazy
+            .map { |index| convert_result Unstable.refer_to_nth_result(results, index) }
+            .take_while { |result| !result.nil? }
+            .to_a
+        ensure
+          Unstable.delete_results results
+        end
+      ensure
+        pointer.free
+      end
+    end
+
+    private
+
+    def convert_result(result)
+      language = result[:language_data].read_bytes(result[:language_size])
+      return nil if language == "und"
+
+      cursor = result[:byte_ranges_data]
+      byte_ranges = result[:byte_ranges_size].times.map do
+        info = Unstable::NNetLanguageIdentifier::SpanInfo.new(cursor)
+        cursor += Unstable::NNetLanguageIdentifier::SpanInfo.size
+        SpanInfo.new(info[:start_index], info[:end_index], info[:probability])
+      end
 
       Result.new(
-          language == "und" ? nil : language.to_sym,
-          cc_result[:probability],
-          cc_result[:reliable?],
-          cc_result[:proportion])
+          language.to_sym,
+          result[:probability],
+          result[:reliable?],
+          result[:proportion],
+          byte_ranges)
     end
   end
 
@@ -121,17 +188,30 @@ module CLD3
         end
       end
 
+      class SpanInfo < FFI::Struct
+        layout :start_index, :int, :end_index, :int, :probability, :float
+      end
+
       class Result < FFI::Struct
-        layout :language_data, :pointer, :language_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
+        layout :language_data, :pointer, :language_size, :size_t, :byte_ranges_data, :pointer, :byte_ranges_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
       end
     end
 
     attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
 
+    attach_function :delete_result, [ :pointer ], :void
+
+    attach_function :delete_results, [ :pointer ], :void
+
     attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
 
+    attach_function :refer_to_nth_result, [ :pointer, :size_t ], NNetLanguageIdentifier::Result.by_value
+
     attach_function :NNetLanguageIdentifier_find_language,
-        [ :pointer, :buffer_in, :size_t ], NNetLanguageIdentifier::Result.by_value
+        [ :pointer, :buffer_in, :size_t ], :pointer
+
+    attach_function :NNetLanguageIdentifier_find_top_n_most_freq_langs,
+        [ :pointer, :buffer_in, :size_t, :int ], :pointer
   end
 
   private_constant :Unstable

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.2.6
+  version: 3.3.0
 platform: ruby
 authors:
 - Akihiko Odaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-04 00:00:00.000000000 Z
+date: 2020-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -63,101 +63,70 @@ files:
 - LICENSE_CLD3
 - README.md
 - cld3.gemspec
-- ext/cld3/Makefile
 - ext/cld3/base.cc
 - ext/cld3/base.h
-- ext/cld3/base.o
 - ext/cld3/casts.h
 - ext/cld3/embedding_feature_extractor.cc
 - ext/cld3/embedding_feature_extractor.h
-- ext/cld3/embedding_feature_extractor.o
 - ext/cld3/embedding_network.cc
 - ext/cld3/embedding_network.h
-- ext/cld3/embedding_network.o
 - ext/cld3/embedding_network_params.h
 - ext/cld3/extconf.rb
 - ext/cld3/feature_extractor.cc
 - ext/cld3/feature_extractor.h
-- ext/cld3/feature_extractor.o
-- ext/cld3/feature_extractor.pb.o
 - ext/cld3/feature_extractor.proto
 - ext/cld3/feature_types.cc
 - ext/cld3/feature_types.h
-- ext/cld3/feature_types.o
 - ext/cld3/fixunicodevalue.cc
 - ext/cld3/fixunicodevalue.h
-- ext/cld3/fixunicodevalue.o
 - ext/cld3/float16.h
 - ext/cld3/fml_parser.cc
 - ext/cld3/fml_parser.h
-- ext/cld3/fml_parser.o
 - ext/cld3/generated_entities.cc
-- ext/cld3/generated_entities.o
 - ext/cld3/generated_ulscript.cc
 - ext/cld3/generated_ulscript.h
-- ext/cld3/generated_ulscript.o
 - ext/cld3/getonescriptspan.cc
 - ext/cld3/getonescriptspan.h
-- ext/cld3/getonescriptspan.o
 - ext/cld3/integral_types.h
 - ext/cld3/lang_id_nn_params.cc
 - ext/cld3/lang_id_nn_params.h
-- ext/cld3/lang_id_nn_params.o
 - ext/cld3/language_identifier_features.cc
 - ext/cld3/language_identifier_features.h
-- ext/cld3/language_identifier_features.o
-- ext/cld3/libcld3.so
-- ext/cld3/mkmf.log
 - ext/cld3/nnet_language_identifier.cc
 - ext/cld3/nnet_language_identifier.h
-- ext/cld3/nnet_language_identifier.o
 - ext/cld3/nnet_language_identifier_c.cc
-- ext/cld3/nnet_language_identifier_c.o
 - ext/cld3/offsetmap.cc
 - ext/cld3/offsetmap.h
-- ext/cld3/offsetmap.o
 - ext/cld3/port.h
 - ext/cld3/registry.cc
 - ext/cld3/registry.h
-- ext/cld3/registry.o
 - ext/cld3/relevant_script_feature.cc
 - ext/cld3/relevant_script_feature.h
-- ext/cld3/relevant_script_feature.o
 - ext/cld3/script_detector.h
-- ext/cld3/sentence.pb.o
 - ext/cld3/sentence.proto
 - ext/cld3/sentence_features.cc
 - ext/cld3/sentence_features.h
-- ext/cld3/sentence_features.o
 - ext/cld3/simple_adder.h
 - ext/cld3/stringpiece.h
 - ext/cld3/task_context.cc
 - ext/cld3/task_context.h
-- ext/cld3/task_context.o
 - ext/cld3/task_context_params.cc
 - ext/cld3/task_context_params.h
-- ext/cld3/task_context_params.o
-- ext/cld3/task_spec.pb.o
 - ext/cld3/task_spec.proto
 - ext/cld3/text_processing.cc
 - ext/cld3/text_processing.h
-- ext/cld3/text_processing.o
 - ext/cld3/unicodetext.cc
 - ext/cld3/unicodetext.h
-- ext/cld3/unicodetext.o
 - ext/cld3/utf8acceptinterchange.h
 - ext/cld3/utf8prop_lettermarkscriptnum.h
 - ext/cld3/utf8repl_lettermarklower.h
 - ext/cld3/utf8scannot_lettermarkspecial.h
 - ext/cld3/utf8statetable.cc
 - ext/cld3/utf8statetable.h
-- ext/cld3/utf8statetable.o
 - ext/cld3/utils.cc
 - ext/cld3/utils.h
-- ext/cld3/utils.o
 - ext/cld3/workspace.cc
 - ext/cld3/workspace.h
-- ext/cld3/workspace.o
 - lib/cld3.rb
 homepage: https://github.com/akihikodaki/cld3-ruby
 licenses:
@@ -181,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Compact Language Detector v3 (CLD3)

data/ext/cld3/Makefile

@@ -1,266 +0,0 @@
-
-SHELL = /bin/sh
-
-# V=0 quiet, V=1 verbose.  other values don't work.
-V = 0
-Q1 = $(V:1=)
-Q = $(Q1:0=@)
-ECHO1 = $(V:1=@ :)
-ECHO = $(ECHO1:0=@ echo)
-NULLCMD = :
-
-#### Start of system configuration section. ####
-
-srcdir = .
-topdir = /usr/include/ruby-2.6.0
-hdrdir = $(topdir)
-arch_hdrdir = /usr/include/ruby-2.6.0/x86_64-linux
-PATH_SEPARATOR = :
-VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
-prefix = $(DESTDIR)/usr
-rubysitearchprefix = $(rubylibprefix)/$(sitearch)
-rubyarchprefix = $(rubylibprefix)/$(arch)
-rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
-exec_prefix = $(prefix)
-vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
-sitearchhdrdir = $(sitehdrdir)/$(sitearch)
-rubyarchhdrdir = $(rubyhdrdir)/$(arch)
-vendorhdrdir = $(rubyhdrdir)/vendor_ruby
-sitehdrdir = $(rubyhdrdir)/site_ruby
-rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
-vendorarchdir = $(vendorlibdir)/$(sitearch)
-vendorlibdir = $(vendordir)/$(ruby_version)
-vendordir = $(rubylibprefix)/vendor_ruby
-sitearchdir = $(sitelibdir)/$(sitearch)
-sitelibdir = $(sitedir)/$(ruby_version)
-sitedir = $(rubylibprefix)/site_ruby
-rubyarchdir = $(rubylibdir)/$(arch)
-rubylibdir = $(rubylibprefix)/$(ruby_version)
-sitearchincludedir = $(includedir)/$(sitearch)
-archincludedir = $(includedir)/$(arch)
-sitearchlibdir = $(libdir)/$(sitearch)
-archlibdir = $(libdir)/$(arch)
-ridir = $(datarootdir)/$(RI_BASE_NAME)
-mandir = $(datarootdir)/man
-localedir = $(datarootdir)/locale
-libdir = $(exec_prefix)/lib
-psdir = $(docdir)
-pdfdir = $(docdir)
-dvidir = $(docdir)
-htmldir = $(docdir)
-infodir = $(datarootdir)/info
-docdir = $(datarootdir)/doc/$(PACKAGE)
-oldincludedir = $(DESTDIR)/usr/include
-includedir = $(prefix)/include
-runstatedir = $(localstatedir)/run
-localstatedir = $(DESTDIR)/var
-sharedstatedir = $(DESTDIR)/var/lib
-sysconfdir = $(DESTDIR)/etc
-datadir = $(datarootdir)
-datarootdir = $(prefix)/share
-libexecdir = $(DESTDIR)/usr/lib/ruby
-sbindir = $(exec_prefix)/sbin
-bindir = $(exec_prefix)/bin
-archdir = $(rubyarchdir)
-
-
-CC_WRAPPER = 
-CC = gcc
-CXX = g++
-LIBRUBY = $(LIBRUBY_SO)
-LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
-LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
-LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static $(MAINLIBS)
-empty =
-OUTFLAG = -o $(empty)
-COUTFLAG = -o $(empty)
-CSRCFLAG = $(empty)
-
-RUBY_EXTCONF_H = 
-cflags   = $(optflags) $(debugflags) $(warnflags)
-cxxflags = $(optflags) $(debugflags) $(warnflags)
-optflags = -O3
-debugflags = -ggdb3
-warnflags = -Wall -Wextra -Wdeclaration-after-statement -Wdeprecated-declarations -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wrestrict -Wwrite-strings -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable
-cppflags = 
-CCDLFLAGS = -fPIC
-CFLAGS   = $(CCDLFLAGS) -march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fPIC -pthread $(ARCH_FLAG)
-INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir) 
-DEFS     = 
-CPPFLAGS =  -D_FORTIFY_SOURCE=2 $(DEFS) $(cppflags)
-CXXFLAGS = $(CCDLFLAGS) -march=x86-64 -mtune=generic -O2 -pipe -fno-plt -pthread -fvisibility=hidden -std=c++11 $(ARCH_FLAG)
-ldflags  = -L. -Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now -fstack-protector-strong -rdynamic -Wl,-export-dynamic 
-dldflags = -Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now -Wl,--compress-debug-sections=zlib 
-ARCH_FLAG = 
-DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
-LDSHARED = $(CC) -shared
-LDSHAREDXX = $(CXX) -shared
-AR = ar
-EXEEXT = 
-
-RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
-RUBY_SO_NAME = ruby
-RUBYW_INSTALL_NAME = 
-RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
-RUBYW_BASE_NAME = rubyw
-RUBY_BASE_NAME = ruby
-
-arch = x86_64-linux
-sitearch = $(arch)
-ruby_version = 2.6.0
-ruby = $(bindir)/$(RUBY_BASE_NAME)
-RUBY = $(ruby)
-ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
-
-RM = rm -f
-RM_RF = $(RUBY) -run -e rm -- -rf
-RMDIRS = rmdir --ignore-fail-on-non-empty -p
-MAKEDIRS = /usr/bin/mkdir -p
-INSTALL = /usr/bin/install -c
-INSTALL_PROG = $(INSTALL) -m 0755
-INSTALL_DATA = $(INSTALL) -m 644
-COPY = cp
-TOUCH = exit >
-
-#### End of system configuration section. ####
-
-preload = 
-libpath = . $(libdir)
-LIBPATH =  -L. -L$(libdir)
-DEFFILE = 
-
-CLEANFILES = mkmf.log
-DISTCLEANFILES = 
-DISTCLEANDIRS = 
-
-extout = 
-extout_prefix = 
-target_prefix = 
-LOCAL_LIBS = 
-LIBS = $(LIBRUBYARG_SHARED)  -lprotobuf -lm   -lc
-ORIG_SRCS = base.cc embedding_feature_extractor.cc embedding_network.cc feature_extractor.cc feature_extractor.pb.cc feature_types.cc fixunicodevalue.cc fml_parser.cc generated_entities.cc generated_ulscript.cc getonescriptspan.cc lang_id_nn_params.cc language_identifier_features.cc nnet_language_identifier.cc nnet_language_identifier_c.cc offsetmap.cc registry.cc relevant_script_feature.cc sentence.pb.cc sentence_features.cc task_context.cc task_context_params.cc task_spec.pb.cc text_processing.cc unicodetext.cc utf8statetable.cc utils.cc workspace.cc
-SRCS = $(ORIG_SRCS) 
-OBJS = base.o embedding_feature_extractor.o embedding_network.o feature_extractor.o feature_extractor.pb.o feature_types.o fixunicodevalue.o fml_parser.o generated_entities.o generated_ulscript.o getonescriptspan.o lang_id_nn_params.o language_identifier_features.o nnet_language_identifier.o nnet_language_identifier_c.o offsetmap.o registry.o relevant_script_feature.o sentence.pb.o sentence_features.o task_context.o task_context_params.o task_spec.pb.o text_processing.o unicodetext.o utf8statetable.o utils.o workspace.o
-HDRS = $(srcdir)/base.h $(srcdir)/casts.h $(srcdir)/embedding_feature_extractor.h $(srcdir)/embedding_network.h $(srcdir)/embedding_network_params.h $(srcdir)/feature_extractor.h $(srcdir)/feature_types.h $(srcdir)/float16.h $(srcdir)/fml_parser.h $(srcdir)/language_identifier_features.h $(srcdir)/lang_id_nn_params.h $(srcdir)/nnet_language_identifier.h $(srcdir)/registry.h $(srcdir)/relevant_script_feature.h $(srcdir)/script_detector.h $(srcdir)/sentence_features.h $(srcdir)/simple_adder.h $(srcdir)/fixunicodevalue.h $(srcdir)/generated_ulscript.h $(srcdir)/getonescriptspan.h $(srcdir)/integral_types.h $(srcdir)/offsetmap.h $(srcdir)/port.h $(srcdir)/stringpiece.h $(srcdir)/text_processing.h $(srcdir)/utf8acceptinterchange.h $(srcdir)/utf8prop_lettermarkscriptnum.h $(srcdir)/utf8repl_lettermarklower.h $(srcdir)/utf8scannot_lettermarkspecial.h $(srcdir)/utf8statetable.h $(srcdir)/task_context.h $(srcdir)/task_context_params.h $(srcdir)/unicodetext.h $(srcdir)/utils.h $(srcdir)/workspace.h $(srcdir)/feature_extractor.pb.h $(srcdir)/sentence.pb.h $(srcdir)/task_spec.pb.h
-LOCAL_HDRS = 
-TARGET = libcld3
-TARGET_NAME = libcld3
-TARGET_ENTRY = Init_$(TARGET_NAME)
-DLLIB = $(TARGET).so
-EXTSTATIC = 
-STATIC_LIB = 
-
-TIMESTAMP_DIR = .
-BINDIR        = $(bindir)
-RUBYCOMMONDIR = $(sitedir)$(target_prefix)
-RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
-RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
-HDRDIR        = $(rubyhdrdir)/ruby$(target_prefix)
-ARCHHDRDIR    = $(rubyhdrdir)/$(arch)/ruby$(target_prefix)
-TARGET_SO_DIR =
-TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
-CLEANLIBS     = $(TARGET_SO) 
-CLEANOBJS     = *.o  *.bak
-
-all:    $(DLLIB)
-static: $(STATIC_LIB)
-.PHONY: all install static install-so install-rb
-.PHONY: clean clean-so clean-static clean-rb
-
-clean-static::
-clean-rb-default::
-clean-rb::
-clean-so::
-clean: clean-so clean-static clean-rb-default clean-rb
-		-$(Q)$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
-
-distclean-rb-default::
-distclean-rb::
-distclean-so::
-distclean-static::
-distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
-		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
-		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
-		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
-
-realclean: distclean
-install: install-so install-rb
-
-install-so: $(DLLIB) $(TIMESTAMP_DIR)/.sitearchdir.time
-	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
-clean-static::
-	-$(Q)$(RM) $(STATIC_LIB)
-install-rb: pre-install-rb do-install-rb install-rb-default
-install-rb-default: pre-install-rb-default do-install-rb-default
-pre-install-rb: Makefile
-pre-install-rb-default: Makefile
-do-install-rb:
-do-install-rb-default:
-pre-install-rb-default:
-	@$(NULLCMD)
-$(TIMESTAMP_DIR)/.sitearchdir.time:
-	$(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
-	$(Q) $(TOUCH) $@
-
-site-install: site-install-so site-install-rb
-site-install-so: install-so
-site-install-rb: install-rb
-
-.SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
-
-.cc.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cc.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.mm.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.mm.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.cxx.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cxx.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.cpp.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cpp.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.c.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.c.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.m.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.m.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-$(TARGET_SO): $(OBJS) Makefile
-	$(ECHO) linking shared-object $(DLLIB)
-	-$(Q)$(RM) $(@)
-	$(Q) $(LDSHAREDXX) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
-
-
-
-$(OBJS): $(HDRS) $(ruby_headers)

data/ext/cld3/mkmf.log

@@ -1,36 +0,0 @@
-"pkg-config --exists protobuf"
-| pkg-config --libs protobuf
-=> "-lprotobuf \n"
-"gcc -o conftest -I/usr/include/ruby-2.6.0/x86_64-linux -I/usr/include/ruby-2.6.0/ruby/backward -I/usr/include/ruby-2.6.0 -I. -D_FORTIFY_SOURCE=2   -march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fPIC conftest.c  -L. -L/usr/lib -L. -Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby  -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return 0;
-6: }
-/* end */
-
-"gcc -o conftest -I/usr/include/ruby-2.6.0/x86_64-linux -I/usr/include/ruby-2.6.0/ruby/backward -I/usr/include/ruby-2.6.0 -I. -D_FORTIFY_SOURCE=2   -march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fPIC conftest.c  -L. -L/usr/lib -L. -Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now -fstack-protector-strong -rdynamic -Wl,-export-dynamic     -lruby -lprotobuf -lm   -lc"
-checked program was:
-/* begin */
-1: #include "ruby.h"
-2: 
-3: int main(int argc, char **argv)
-4: {
-5:   return 0;
-6: }
-/* end */
-
-| pkg-config --cflags-only-I protobuf
-=> "\n"
-| pkg-config --cflags-only-other protobuf
-=> "-pthread \n"
-| pkg-config --libs-only-l protobuf
-=> "-lprotobuf \n"
-package configuration for protobuf
-cflags: -pthread
-ldflags: 
-libs: -lprotobuf
-