cld3 3.4.2 → 3.5.0

data/ext/cld3/script_span/utf8statetable.h

@@ -0,0 +1,285 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// State Table follower for scanning UTF-8 strings without converting to
+// 32- or 16-bit Unicode values.
+//
+// Author: dsites@google.com (Dick Sites)
+//
+
+#ifndef SCRIPT_SPAN_UTF8STATETABLE_H_
+#define SCRIPT_SPAN_UTF8STATETABLE_H_
+
+#include <string>
+
+#include "integral_types.h" // for uint8, uint32, uint16
+#include "stringpiece.h"
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+class OffsetMap;
+
+
+// These four-byte entries compactly encode how many bytes 0..255 to delete
+// in making a string replacement, how many bytes to add 0..255, and the offset
+// 0..64k-1 of the replacement string in remap_string.
+struct RemapEntry {
+  uint8 delete_bytes;
+  uint8 add_bytes;
+  uint16 bytes_offset;
+};
+
+// Exit type codes for state tables. All but the first get stuffed into
+// signed one-byte entries. The first is only generated by executable code.
+// To distinguish from next-state entries, these must be contiguous and
+// all <= kExitNone
+typedef enum {
+  kExitDstSpaceFull = 239,
+  kExitIllegalStructure,  // 240
+  kExitOK,                // 241
+  kExitReject,            // ...
+  kExitReplace1,
+  kExitReplace2,
+  kExitReplace3,
+  kExitReplace21,
+  kExitReplace31,
+  kExitReplace32,
+  kExitReplaceOffset1,
+  kExitReplaceOffset2,
+  kExitReplace1S0,
+  kExitSpecial,
+  kExitDoAgain,
+  kExitRejectAlt,
+  kExitNone               // 255
+} ExitReason;
+
+typedef enum {
+  kExitDstSpaceFull_2 = 32767,       // 0x7fff
+  kExitIllegalStructure_2,  // 32768    0x8000
+  kExitOK_2,                // 32769    0x8001
+  kExitReject_2,            // ...
+  kExitReplace1_2,
+  kExitReplace2_2,
+  kExitReplace3_2,
+  kExitReplace21_2,
+  kExitReplace31_2,
+  kExitReplace32_2,
+  kExitReplaceOffset1_2,
+  kExitReplaceOffset2_2,
+  kExitReplace1S0_2,
+  kExitSpecial_2,
+  kExitDoAgain_2,
+  kExitRejectAlt_2,
+  kExitNone_2               // 32783    0x800f
+} ExitReason_2;
+
+
+// This struct represents one entire state table. The three initialized byte
+// areas are state_table, remap_base, and remap_string. state0 and state0_size
+// give the byte offset and length within state_table of the initial state --
+// table lookups are expected to start and end in this state, but for
+// truncated UTF-8 strings, may end in a different state. These allow a quick
+// test for that condition. entry_shift is 8 for tables subscripted by a full
+// byte value and 6 for space-optimized tables subscripted by only six
+// significant bits in UTF-8 continuation bytes.
+typedef struct {
+  const uint32 state0;
+  const uint32 state0_size;
+  const uint32 total_size;
+  const int max_expand;
+  const int entry_shift;
+  const int bytes_per_entry;
+  const uint32 losub;
+  const uint32 hiadd;
+  const uint8* state_table;
+  const RemapEntry* remap_base;
+  const uint8* remap_string;
+  const uint8* fast_state;
+} UTF8StateMachineObj;
+
+// Near-duplicate declaration for tables with two-byte entries
+typedef struct {
+  const uint32 state0;
+  const uint32 state0_size;
+  const uint32 total_size;
+  const int max_expand;
+  const int entry_shift;
+  const int bytes_per_entry;
+  const uint32 losub;
+  const uint32 hiadd;
+  const unsigned short* state_table;
+  const RemapEntry* remap_base;
+  const uint8* remap_string;
+  const uint8* fast_state;
+} UTF8StateMachineObj_2;
+
+
+typedef UTF8StateMachineObj UTF8PropObj;
+typedef UTF8StateMachineObj UTF8ScanObj;
+typedef UTF8StateMachineObj UTF8ReplaceObj;
+typedef UTF8StateMachineObj_2 UTF8PropObj_2;
+typedef UTF8StateMachineObj_2 UTF8ReplaceObj_2;
+// NOT IMPLEMENTED typedef UTF8StateMachineObj_2 UTF8ScanObj_2;
+
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericProperty(const UTF8PropObj* st,
+                          const uint8** src,
+                          int* srclen);
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericProperty(const UTF8PropObj& st, const char* src);
+
+
+// BigOneByte versions are needed for tables > 240 states, but most
+// won't need the TwoByte versions.
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericPropertyBigOneByte(const UTF8PropObj* st,
+                          const uint8** src,
+                          int* srclen);
+
+
+// TwoByte versions are needed for tables > 240 states that don't fit onto
+// BigOneByte -- rare ultimate fallback
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericPropertyBigOneByte(const UTF8PropObj& st, const char* src);
+
+// Look up property of one UTF-8 character and advance over it
+// Return 0 if input length is zero
+// Return 0 and advance one byte if input is ill-formed
+uint8 UTF8GenericPropertyTwoByte(const UTF8PropObj_2* st,
+                          const uint8** src,
+                          int* srclen);
+
+// Look up property of one UTF-8 character (assumed to be valid).
+// (This is a faster version of UTF8GenericProperty.)
+bool UTF8HasGenericPropertyTwoByte(const UTF8PropObj_2& st, const char* src);
+
+// Scan a UTF-8 stringpiece based on a state table.
+// Always scan complete UTF-8 characters
+// Set number of bytes scanned. Return reason for exiting
+int UTF8GenericScan(const UTF8ScanObj* st,
+                    const StringPiece& str,
+                    int* bytes_consumed);
+
+
+
+// Scan a UTF-8 stringpiece based on state table, copying to output stringpiece
+//   and doing text replacements.
+// Always scan complete UTF-8 characters
+// Set number of bytes consumed from input, number filled to output.
+// Return reason for exiting
+// Also writes an optional OffsetMap. Pass NULL to skip writing one.
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    bool is_plain_text,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed,
+                    OffsetMap* offsetmap);
+
+// Older version without offsetmap
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    bool is_plain_text,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed);
+
+// Older version without is_plain_text or offsetmap
+int UTF8GenericReplace(const UTF8ReplaceObj* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed);
+
+
+// TwoByte version is needed for tables > about 256 states, such
+// as the table for full Unicode 4.1 canonical + compatibility mapping
+
+// Scan a UTF-8 stringpiece based on state table with two-byte entries,
+//   copying to output stringpiece
+//   and doing text replacements.
+// Always scan complete UTF-8 characters
+// Set number of bytes consumed from input, number filled to output.
+// Return reason for exiting
+// Also writes an optional OffsetMap. Pass NULL to skip writing one.
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    bool is_plain_text,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed,
+                    OffsetMap* offsetmap);
+
+// Older version without offsetmap
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    bool is_plain_text,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed);
+
+// Older version without is_plain_text or offsetmap
+int UTF8GenericReplaceTwoByte(const UTF8ReplaceObj_2* st,
+                    const StringPiece& istr,
+                    StringPiece& ostr,
+                    int* bytes_consumed,
+                    int* bytes_filled,
+                    int* chars_changed);
+
+
+static const unsigned char kUTF8LenTbl[256] = {
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+  1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,
+  2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,
+  3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4
+};
+
+inline int UTF8OneCharLen(const char* in) {
+  return kUTF8LenTbl[*reinterpret_cast<const uint8*>(in)];
+}
+
+// Adjust a stringpiece to encompass complete UTF-8 characters.
+// The data pointer will be increased by 0..3 bytes to get to a character
+// boundary, and the length will then be decreased by 0..3 bytes
+// to encompass the last complete character.
+// This is useful especially when a UTF-8 string must be put into a fixed-
+// maximum-size buffer cleanly, such as a MySQL buffer.
+void UTF8TrimToChars(StringPiece* istr);
+
+}       // End namespace CLD2
+}       // End namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_UTF8STATETABLE_H_

data/ext/cld3/sentence_features.cc

@@ -19,11 +19,11 @@ limitations under the License.
 
 namespace chrome_lang_id {
 
-// Declare registry for the whole Sentence feature functions.  NOTE: this is not
+// Define registry for the whole Sentence feature functions.  NOTE: this is not
 // yet set to anything meaningful.  It will be set so in NNetLanguageIdentifier
 // constructor, *before* we use any feature.
 template <>
-WholeSentenceFeature::Registry
-    *RegisterableClass<WholeSentenceFeature>::registry_ = nullptr;
+WholeSentenceFeature::Registry*
+    RegisterableClass<WholeSentenceFeature>::registry_ = nullptr;
 
-}  // namespace chrome_lang_id
+}  // namespace chrome_lang_id

data/ext/cld3/sentence_features.h

@@ -26,9 +26,19 @@ limitations under the License.
 namespace chrome_lang_id {
 
 // Feature function that extracts features for the full Sentence.
-typedef FeatureFunction<Sentence> WholeSentenceFeature;
-
-typedef FeatureExtractor<Sentence> WholeSentenceExtractor;
+using WholeSentenceFeature = FeatureFunction<Sentence>;
+
+using WholeSentenceExtractor = FeatureExtractor<Sentence>;
+
+// Declare registry for the whole Sentence feature functions.  This is required
+// for clang's -Wundefined-var-template.  However, MSVC has a bug which treats
+// this declaration as a definition, leading to multiple definition errors, so
+// omit this on MSVC.
+#if !defined(COMPILER_MSVC)
+template <>
+WholeSentenceFeature::Registry
+    *RegisterableClass<WholeSentenceFeature>::registry_;
+#endif
 
 }  // namespace chrome_lang_id
 

data/lib/cld3/unstable.rb

@@ -0,0 +1,58 @@
+
+# Copyright 2021 Akihiko Odaki <akihiko.odaki@gmail.com>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+module CLD3
+  module Unstable
+    extend FFI::Library
+
+    ffi_lib File.join(__dir__, "..", "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
+
+    module NNetLanguageIdentifier
+      class Pointer < FFI::AutoPointer
+        def self.release(pointer)
+          Unstable.delete_NNetLanguageIdentifier(pointer)
+        end
+      end
+
+      class SpanInfo < FFI::Struct
+        layout :start_index, :int, :end_index, :int, :probability, :float
+      end
+
+      class Result < FFI::Struct
+        layout :language_data, :pointer, :language_size, :size_t, :byte_ranges_data, :pointer, :byte_ranges_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
+      end
+    end
+
+    attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
+
+    attach_function :delete_result, [ :pointer ], :void
+
+    attach_function :delete_results, [ :pointer ], :void
+
+    attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
+
+    attach_function :refer_to_nth_result, [ :pointer, :size_t ], NNetLanguageIdentifier::Result.by_value
+
+    attach_function :NNetLanguageIdentifier_find_language,
+        [ :pointer, :buffer_in, :size_t ], :pointer
+
+    attach_function :NNetLanguageIdentifier_find_top_n_most_freq_langs,
+        [ :pointer, :buffer_in, :size_t, :int ], :pointer
+  end
+
+  private_constant :Unstable
+end

data/lib/cld3.rb

@@ -19,6 +19,7 @@
 
 require "ffi"
 require "rbconfig"
+require "cld3/unstable"
 
 # Module providing an interface for Compact Language Detector v3 (CLD3)
 module CLD3
@@ -52,6 +53,7 @@ module CLD3
     # Holds probability that Span, specified by start/end indices, is a given
     # language. The langauge is not stored here; it can be found in Result, which
     # holds an Array of SpanInfo.
+    # @type const SpanInfo: untyped
     SpanInfo = Struct.new(:start_index, :end_index, :probability)
 
     # Information about a predicted language.
@@ -69,21 +71,25 @@ module CLD3
     #
     # [byte_ranges] Specifies the byte ranges in UTF-8 that |language| applies to.
     #               This is an Array of SpanInfo.
+    # @type const Result: untyped
     Result = Struct.new(:language, :probability, :reliable?, :proportion, :byte_ranges)
 
-    # The arguments are two String objects.
+    # The arguments are two Numeric objects.
     def initialize(min_num_bytes = MIN_NUM_BYTES_TO_CONSIDER, max_num_bytes = MAX_NUM_BYTES_TO_CONSIDER)
+      raise ArgumentError if min_num_bytes < 0 || min_num_bytes >= max_num_bytes
       @cc = Unstable::NNetLanguageIdentifier::Pointer.new(Unstable.new_NNetLanguageIdentifier(min_num_bytes, max_num_bytes))
     end
 
     # Finds the most likely language for the given text, along with additional
     # information (e.g., probability). The prediction is based on the first N
-    # bytes where N is the minumum between the number of interchange valid UTF8
+    # bytes where N is the minimum between the number of interchange valid UTF8
     # bytes and +max_num_bytes_+. If N is less than +min_num_bytes_+ long, then
     # this function returns nil.
     # The argument is a String object.
     # The returned value of this function is an instance of Result.
     def find_language(text)
+      # @type const FFI: untyped
+
       text_utf8 = text.encode(Encoding::UTF_8)
       pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
 
@@ -115,6 +121,9 @@ module CLD3
     # The second argument is Numeric object.
     # The returned value of this functions is an Array of Result instances.
     def find_top_n_most_freq_langs(text, num_langs)
+      # @type const FFI: untyped
+      # @type var a: untyped
+
       text_utf8 = text.encode(Encoding::UTF_8)
       pointer = FFI::MemoryPointer.new(:char, text_utf8.bytesize)
 
@@ -123,11 +132,13 @@ module CLD3
 
         results = Unstable.NNetLanguageIdentifier_find_top_n_most_freq_langs(@cc, pointer, text_utf8.bytesize, num_langs)
         begin
-          num_langs.times
+          a = num_langs.times
             .lazy
             .map { |index| convert_result Unstable.refer_to_nth_result(results, index) }
             .take_while { |result| !result.nil? }
             .to_a
+
+          a
         ensure
           Unstable.delete_results results
         end
@@ -162,6 +173,7 @@ module CLD3
   # The model weights are loaded statically.
   module TaskContextParams
     # This is an frozen Array object containing symbols.
+    # @type const LANGUAGE_NAMES: untyped
     LANGUAGE_NAMES = [
       :eo, :co, :eu, :ta, :de, :mt, :ps, :te, :su, :uz, :'zh-Latn', :ne,
       :nl, :sw, :sq, :hmn, :ja, :no, :mn, :so, :ko, :kk, :sl, :ig,
@@ -175,44 +187,4 @@ module CLD3
       :sn, :yo, :pa, :ku,
     ].freeze
   end
-
-  module Unstable
-    extend FFI::Library
-
-    ffi_lib File.join(__dir__, "..", "ext", "cld3", "libcld3." + RbConfig::CONFIG["DLEXT"])
-
-    module NNetLanguageIdentifier
-      class Pointer < FFI::AutoPointer
-        def self.release(pointer)
-          Unstable.delete_NNetLanguageIdentifier(pointer)
-        end
-      end
-
-      class SpanInfo < FFI::Struct
-        layout :start_index, :int, :end_index, :int, :probability, :float
-      end
-
-      class Result < FFI::Struct
-        layout :language_data, :pointer, :language_size, :size_t, :byte_ranges_data, :pointer, :byte_ranges_size, :size_t, :probability, :float, :proportion, :float, :reliable?, :bool
-      end
-    end
-
-    attach_function :delete_NNetLanguageIdentifier, [ :pointer ], :void
-
-    attach_function :delete_result, [ :pointer ], :void
-
-    attach_function :delete_results, [ :pointer ], :void
-
-    attach_function :new_NNetLanguageIdentifier, [ :int, :int ], :pointer
-
-    attach_function :refer_to_nth_result, [ :pointer, :size_t ], NNetLanguageIdentifier::Result.by_value
-
-    attach_function :NNetLanguageIdentifier_find_language,
-        [ :pointer, :buffer_in, :size_t ], :pointer
-
-    attach_function :NNetLanguageIdentifier_find_top_n_most_freq_langs,
-        [ :pointer, :buffer_in, :size_t, :int ], :pointer
-  end
-
-  private_constant :Unstable
 end

data/sig/cld3.rbs

@@ -0,0 +1,65 @@
+# Copyright 2021 Akihiko Odaki <akihiko.odaki@gmail.com>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+module CLD3
+  class NNetLanguageIdentifier
+    MIN_NUM_BYTES_TO_CONSIDER: Integer
+    MAX_NUM_BYTES_TO_CONSIDER: Integer
+    MAX_NUM_INPUT_BYTES_TO_CONSIDER: Integer
+    RELIABILITY_THRESHOLD: Float
+    RELIABILITY_HR_BS_THRESHOLD: Float
+
+    class SpanInfo < Struct[Float | Integer]
+      attr_accessor start_index(): Integer
+      attr_accessor end_index(): Integer
+      attr_accessor probability(): Float
+    end
+
+    class Result < Struct[Array[SpanInfo] | Float | TaskContextParams::language_names | bool]
+      attr_accessor language(): TaskContextParams::language_names
+      attr_accessor probability(): Float
+      attr_accessor reliable?(): bool
+      attr_accessor proportion(): Float
+      attr_accessor byte_ranges(): Array[SpanInfo]
+    end
+
+    def initialize: (?Integer, ?Integer) -> void
+    def find_language: (String) -> Result?
+    def find_top_n_most_freq_langs: (String, Integer) -> Array[Result]
+
+    private
+
+    def convert_result: (untyped) -> Result?
+  end
+
+  module TaskContextParams
+    type language_names = 
+      :eo | :co | :eu | :ta | :de | :mt | :ps | :te | :su | :uz | :'zh-Latn' | :ne |
+      :nl | :sw | :sq | :hmn | :ja | :no | :mn | :so | :ko | :kk | :sl | :ig |
+      :mr | :th | :zu | :ml | :hr | :bs | :lo | :sd | :cy | :hy | :uk | :pt |
+      :lv | :iw | :cs | :vi | :jv | :be | :km | :mk | :tr | :fy | :am | :zh |
+      :da | :sv | :fi | :ht | :af | :la | :id | :fil | :sm | :ca | :el | :ka |
+      :sr | :it | :sk | :ru | :'ru-Latn' | :bg | :ny | :fa | :haw | :gl | :et |
+      :ms | :gd | :'bg-Latn' | :ha | :is | :ur | :mi | :hi | :bn | :'hi-Latn' | :fr |
+      :yi | :hu | :xh | :my | :tg | :ro | :ar | :lb | :'el-Latn' | :st | :ceb |
+      :kn | :az | :si | :ky | :mg | :en | :gu | :es | :pl | :'ja-Latn' | :ga | :lt |
+      :sn | :yo | :pa | :ku
+
+    LANGUAGE_NAMES: Array[language_names]
+  end
+
+  Unstable: untyped
+end