cld3 3.1.3 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 628760134e7e969a546b80c31ac97fd1d3417171
-  data.tar.gz: 04a7d9c49a6752625075a5c863fc1c624021655e
+  metadata.gz: 63df114521c24a9f75617d88ca7bcd2f62643884
+  data.tar.gz: e1c9bff9042b97ec6974506065ca6ccb634a4702
 SHA512:
-  metadata.gz: 4b8e553de162e7c8802989dd25bedf999888fad4127ae090afc505231f0033d14c89a2ea0fc670c1e6ba8081a1fbe5973bb78704e3e44383706a89ac2adaa28c
-  data.tar.gz: 84308a74f754ade6a012e329ab86030b96990cde0329fc9f712d538e3d0967d03643fd75f40e90f8915feddda082b7f1c74be0909f8e69b9de0f4f055d680235
+  metadata.gz: 4c653d6888313436fc2a90c2cd8c7e39c92bc1e8cf9ce09520cac466139b16e2dac285bf7fc587a39fa5313ee0c28da86963588bf39433c655a335d46e5ce419
+  data.tar.gz: d21670ff37a9404f2c07d8d2e17385dfe271f26eb436e76766aec5c6b372c0547bd975918f7d6a1d8d98ce811f218353e24d85fb07050b382c06f252a5b64b8c

data/README.md

@@ -13,6 +13,44 @@ cld3-ruby is an interface of Compact Language Detector v3 (CLD3) for Ruby.
 `Rakefile` includes a Rake task to put this code into files buildable as a gem.
 Build a gem with `rake` command.
 
+## Troubleshooting Setup Problems
+I (Akihiko Odaki) recommend to setup this library installing via `gem`.
+
+`gem install cld3` triggers native library building. If it fails, you are likely
+to missing required facilities. Make sure C++ compiler and protocol buffers
+is installed. I recommend [GCC](https://gcc.gnu.org/) as a C++ compiler. Ruby is
+likely to need [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
+as well.
+
+Runtime errors are likely to be issues of [FFI](https://github.com/ffi/ffi) or
+programming errors. Make sure they are all correct.
+
+If you cannot identify the cause of your problem, run spec of this library and
+see whether the problem is reproducable with it or not. Spec is not included in
+the gem, so clone the source code repository and then run `rake spec`.
+The source code repository is at
+https://github.com/akihikodaki/cld3-ruby.
+
+In case you cannot solve your problem by yourself and cannot help abandoning or
+find an issue in this library, please open an issue at
+https://github.com/akihikodaki/cld3-ruby/issues.
+
+If you found an issue and managed to fix it and agree to share the fix under
+[Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0), please open a pull
+request at https://github.com/akihikodaki/cld3-ruby/pulls. Your contribution
+would be appreciated by other users and recorded with Git.
+
+## Versioning
+
+The version has 3 parts: major, minor, and patch. They are joined with . as
+delimiters in the order.
+
+The increment of the major version and the minor version indicates it can involve
+any change.
+
+The increment of the patch version indicates there is no change of the supported
+languages and no change of the existing APIs except `CLD3::Unstable`.
+
 ## Contact
 
 To ask questions or report issues please open issues at

data/cld3.gemspec

@@ -16,7 +16,7 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.1.3"
+  gem.version = "3.2.0"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"

data/lib/cld3.rb

@@ -26,36 +26,55 @@ module CLD3
   class NNetLanguageIdentifier
     # Min number of bytes needed to make a prediction if the construcotr is
     # called without the corresponding parameter.
+    # This is Numeric object.
     MIN_NUM_BYTES_TO_CONSIDER = 140
 
     # Max number of bytes needed to make a prediction if the construcotr is
     # called without the corresponding parameter.
+    # This is Numeric object.
     MAX_NUM_BYTES_TO_CONSIDER = 700
 
     # Max number of input bytes to process.
+    # This is Numeric object.
     MAX_NUM_INPUT_BYTES_TO_CONSIDER = 10000
 
     # Predictions with probability greater than or equal to this threshold are
     # marked as reliable. This threshold was optimized on a set of text segments
     # extracted from wikipedia, and results in an overall precision, recall,
     # and f1 equal to 0.9760, 0.9624, and 0.9692, respectively.
+    # This is Numeric object.
     RELIABILITY_THRESHOLD = 0.7
 
     # Reliability threshold for the languages hr and bs.
+    # This is Numeric object.
     RELIABILITY_HR_BS_THRESHOLD = 0.5
 
     # Information about a predicted language.
+    # This is an instance of Struct with the following members:
+    #
+    # [language]    This is symbol or nil.
+    #
+    # [probability] Language probability. This is Numeric object.
+    #
+    # [reliable?]   Whether the prediction is reliable. This is true or false.
+    #
+    # [proportion]  Proportion of bytes associated with the language. If
+    #               #find_language is called, this variable is set to 1.
+    #               This is Numeric object.
     Result = Struct.new("Result", :language, :probability, :reliable?, :proportion)
 
+    # The arguments are two String objects.
     def initialize(minNumBytes = MIN_NUM_BYTES_TO_CONSIDER, maxNumBytes = MAX_NUM_BYTES_TO_CONSIDER)
-      @cc = Pointer.new(CLD3::Unstable.new_NNetLanguageIdentifier(minNumBytes, maxNumBytes))
+      @cc = CLD3::Unstable::NNetLanguageIdentifier::Pointer.new(CLD3::Unstable.new_NNetLanguageIdentifier(minNumBytes, maxNumBytes))
     end
 
     # Finds the most likely language for the given text, along with additional
     # information (e.g., probability). The prediction is based on the first N
     # bytes where N is the minumum between the number of interchange valid UTF8
-    # bytes and max_num_bytes_. If N is less than min_num_bytes_ long, then this
-    # function returns nil.
+    # bytes and +max_num_bytes_+. If N is less than +min_num_bytes_+ long, then
+    # this function returns nil as language.
+    # The argument is a String object.
+    # The returned value of this function is an instance of Result.
     def find_language(text)
       text_utf8 = text.encode(Encoding::UTF_8)
       pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
@@ -65,29 +84,48 @@ module CLD3
       language = cc_result[:language_data].read_bytes(cc_result[:language_size])
 
       Result.new(
-          language == "und" ? nil : language,
+          language == "und" ? nil : language.to_sym,
           cc_result[:probability],
           cc_result[:reliable?],
           cc_result[:proportion])
     end
+  end
 
-    private
-
-    class Pointer < FFI::AutoPointer
-      def self.release(pointer)
-        CLD3::Unstable.delete_NNetLanguageIdentifier(pointer)
-      end
-    end
+  # Encapsulates the TaskContext specifying only the parameters for the model.
+  # The model weights are loaded statically.
+  module TaskContextParams
+    # This is an frozen Array object containing symbols.
+    LANGUAGE_NAMES = [
+      :eo, :co, :eu, :ta, :de, :mt, :ps, :te, :su, :uz, :'zh-Latn', :ne,
+      :nl, :sw, :sq, :hmn, :ja, :no, :mn, :so, :ko, :kk, :sl, :ig,
+      :mr, :th, :zu, :ml, :hr, :bs, :lo, :sd, :cy, :hy, :uk, :pt,
+      :lv, :iw, :cs, :vi, :jv, :be, :km, :mk, :tr, :fy, :am, :zh,
+      :da, :sv, :fi, :ht, :af, :la, :id, :fil, :sm, :ca, :el, :ka,
+      :sr, :it, :sk, :ru, :'ru-Latn', :bg, :ny, :fa, :haw, :gl, :et,
+      :ms, :gd, :'bg-Latn', :ha, :is, :ur, :mi, :hi, :bn, :'hi-Latn', :fr,
+      :yi, :hu, :xh, :my, :tg, :ro, :ar, :lb, :'el-Latn', :st, :ceb,
+      :kn, :az, :si, :ky, :mg, :en, :gu, :es, :pl, :'ja-Latn', :ga, :lt,
+      :sn, :yo, :pa, :ku,
+    ].freeze
   end
 
+  # :nodoc: all
   # Do NOT use this module from outside.
   module Unstable
     extend FFI::Library
 
     ffi_lib File.join(File.expand_path(File.dirname(__FILE__)), "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
 
-    class NNetLanguageIdentifierResult < FFI::Struct
-      layout :language_data, :pointer, :language_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
+    module NNetLanguageIdentifier
+      class Pointer < FFI::AutoPointer
+        def self.release(pointer)
+          CLD3::Unstable.delete_NNetLanguageIdentifier(pointer)
+        end
+      end
+
+      class Result < FFI::Struct
+        layout :language_data, :pointer, :language_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
+      end
     end
 
     attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
@@ -95,6 +133,6 @@ module CLD3
     attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
 
     attach_function :NNetLanguageIdentifier_find_language,
-        [ :pointer, :buffer_in, :size_t ], NNetLanguageIdentifierResult.by_value
+        [ :pointer, :buffer_in, :size_t ], NNetLanguageIdentifier::Result.by_value
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.2.0
 platform: ruby
 authors:
 - Akihiko Odaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-13 00:00:00.000000000 Z
+date: 2017-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi