cld3 3.1.0

data/ext/cld3/lang_id_nn_params.h

@@ -0,0 +1,178 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef LANG_ID_NN_PARAMS_H_
+#define LANG_ID_NN_PARAMS_H_
+
+#include "base.h"
+#include "embedding_network_params.h"
+#include "float16.h"
+
+namespace chrome_lang_id {
+
+class LangIdNNParams : public EmbeddingNetworkParams {
+ public:
+  ~LangIdNNParams() override {}
+
+  // Access methods for embeddings:
+  int embeddings_size() const override { return 6; }
+  int embeddings_num_rows(int i) const override {
+    return kEmbeddingsNumRows[i];
+  }
+  int embeddings_num_cols(int i) const override {
+    return kEmbeddingsNumCols[i];
+  }
+  const void *embeddings_weights(int i) const override {
+    return embeddings_weights_[i];
+  }
+  QuantizationType embeddings_quant_type(int i) const override {
+    return QuantizationType::UINT8;
+  }
+  const float16 *embeddings_quant_scales(int i) const override {
+    return embeddings_quant_scales_[i];
+  }
+
+  // Access methods for hidden:
+  int hidden_size() const override { return 1; }
+  int hidden_num_rows(int i) const override { return kHiddenNumRows[i]; }
+  int hidden_num_cols(int i) const override { return kHiddenNumCols[i]; }
+  const void *hidden_weights(int i) const override {
+    return hidden_weights_[i];
+  }
+
+  // Access methods for hidden_bias:
+  int hidden_bias_size() const override { return 1; }
+  int hidden_bias_num_rows(int i) const override {
+    return kHiddenBiasNumRows[i];
+  }
+  int hidden_bias_num_cols(int i) const override {
+    return kHiddenBiasNumCols[i];
+  }
+  const void *hidden_bias_weights(int i) const override {
+    return hidden_bias_weights_[i];
+  }
+
+  // Access methods for softmax:
+  int softmax_size() const override { return 1; }
+  int softmax_num_rows(int i) const override { return kSoftmaxNumRows[i]; }
+  int softmax_num_cols(int i) const override { return kSoftmaxNumCols[i]; }
+  const void *softmax_weights(int i) const override {
+    return softmax_weights_[i];
+  }
+
+  // Access methods for softmax_bias:
+  int softmax_bias_size() const override { return 1; }
+  int softmax_bias_num_rows(int i) const override {
+    return kSoftmaxBiasNumRows[i];
+  }
+  int softmax_bias_num_cols(int i) const override {
+    return kSoftmaxBiasNumCols[i];
+  }
+  const void *softmax_bias_weights(int i) const override {
+    return softmax_bias_weights_[i];
+  }
+
+  // Access methods for embedding_dim:
+  int embedding_dim_size() const override { return 6; }
+  int32 embedding_dim(int i) const override { return kEmbeddingDimValues[i]; }
+
+  // Access methods for embedding_num_features:
+  int embedding_num_features_size() const override { return 6; }
+  int32 embedding_num_features(int i) const override {
+    return kEmbeddingNumFeaturesValues[i];
+  }
+
+  // Access methods for embedding_features_domain_size:
+  int embedding_features_domain_size_size() const override { return 6; }
+  int32 embedding_features_domain_size(int i) const override {
+    return kEmbeddingFeaturesDomainSizeValues[i];
+  }
+
+  // Access methods for concat_offset:
+  int concat_offset_size() const override { return 6; }
+  int32 concat_offset(int i) const override { return kConcatOffsetValues[i]; }
+
+  // Access methods for concat_layer_size:
+  bool has_concat_layer_size() const override { return true; }
+  int32 concat_layer_size() const override { return 80; }
+
+  // Access methods for is_precomputed:
+  bool has_is_precomputed() const override { return false; }
+  bool is_precomputed() const override { return false; }
+
+ private:
+  // Private fields for embeddings:
+  static const int kEmbeddingsNumRows[];
+  static const int kEmbeddingsNumCols[];
+  static const uint8 kEmbeddingsWeights0[];
+  static const uint8 kEmbeddingsWeights1[];
+  static const uint8 kEmbeddingsWeights2[];
+  static const uint8 kEmbeddingsWeights3[];
+  static const uint8 kEmbeddingsWeights4[];
+  static const uint8 kEmbeddingsWeights5[];
+  const void *embeddings_weights_[6] = {
+      kEmbeddingsWeights0, kEmbeddingsWeights1, kEmbeddingsWeights2,
+      kEmbeddingsWeights3, kEmbeddingsWeights4, kEmbeddingsWeights5};
+  static const float16 kEmbeddingsQuantScales0[];
+  static const float16 kEmbeddingsQuantScales1[];
+  static const float16 kEmbeddingsQuantScales2[];
+  static const float16 kEmbeddingsQuantScales3[];
+  static const float16 kEmbeddingsQuantScales4[];
+  static const float16 kEmbeddingsQuantScales5[];
+  const float16 *embeddings_quant_scales_[6] = {
+      kEmbeddingsQuantScales0, kEmbeddingsQuantScales1,
+      kEmbeddingsQuantScales2, kEmbeddingsQuantScales3,
+      kEmbeddingsQuantScales4, kEmbeddingsQuantScales5};
+
+  // Private fields for hidden:
+  static const int kHiddenNumRows[];
+  static const int kHiddenNumCols[];
+  static const float kHiddenWeights0[];
+  const void *hidden_weights_[1] = {kHiddenWeights0};
+
+  // Private fields for hidden_bias:
+  static const int kHiddenBiasNumRows[];
+  static const int kHiddenBiasNumCols[];
+  static const float kHiddenBiasWeights0[];
+  const void *hidden_bias_weights_[1] = {kHiddenBiasWeights0};
+
+  // Private fields for softmax:
+  static const int kSoftmaxNumRows[];
+  static const int kSoftmaxNumCols[];
+  static const float kSoftmaxWeights0[];
+  const void *softmax_weights_[1] = {kSoftmaxWeights0};
+
+  // Private fields for softmax_bias:
+  static const int kSoftmaxBiasNumRows[];
+  static const int kSoftmaxBiasNumCols[];
+  static const float kSoftmaxBiasWeights0[];
+  const void *softmax_bias_weights_[1] = {kSoftmaxBiasWeights0};
+
+  // Private fields for embedding_dim:
+  static const int32 kEmbeddingDimValues[];
+
+  // Private fields for embedding_num_features:
+  static const int32 kEmbeddingNumFeaturesValues[];
+
+  // Private fields for embedding_features_domain_size:
+  static const int32 kEmbeddingFeaturesDomainSizeValues[];
+
+  // Private fields for concat_offset:
+  static const int32 kConcatOffsetValues[];
+};  // class LangIdNNParams
+
+}  // namespace chrome_lang_id
+
+#endif  // LANG_ID_NN_PARAMS_H_

data/ext/cld3/language_identifier_features.cc

@@ -0,0 +1,165 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "language_identifier_features.h"
+
+#include <sstream>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "base.h"
+#include "feature_extractor.h"
+#include "feature_types.h"
+#include "script_span/generated_ulscript.h"
+#include "script_span/getonescriptspan.h"
+#include "sentence_features.h"
+#include "task_context.h"
+#include "unicodetext.h"
+#include "utils.h"
+
+namespace chrome_lang_id {
+NumericFeatureType::NumericFeatureType(const string &name, FeatureValue size)
+    : FeatureType(name), size_(size) {}
+
+string NumericFeatureType::GetFeatureValueName(FeatureValue value) const {
+  return value < 0 ? "" : Int64ToString(value);
+}
+
+FeatureValue NumericFeatureType::GetDomainSize() const { return size_; }
+
+void ContinuousBagOfNgramsFunction::Setup(TaskContext *context) {
+  // Parameters in the feature function descriptor.
+  include_terminators_ = GetBoolParameter("include_terminators", false);
+  include_spaces_ = GetBoolParameter("include_spaces", false);
+  use_equal_ngram_weight_ = GetBoolParameter("use_equal_weight", false);
+  ngram_id_dimension_ = GetIntParameter("id_dim", 10000);
+  ngram_size_ = GetIntParameter("size", 3);
+}
+
+void ContinuousBagOfNgramsFunction::Init(TaskContext *context) {
+  set_feature_type(new NumericFeatureType(name(), ngram_id_dimension_));
+}
+
+void ContinuousBagOfNgramsFunction::Evaluate(const WorkspaceSet &workspaces,
+                                             const Sentence &sentence,
+                                             FeatureVector *result) const {
+  // Include terminators for each token. Tokens are discovered by splitting the
+  // text on spaces.
+  std::vector<string> chars;
+  utils::GetUTF8Chars(sentence.text(), &chars);
+  if (include_terminators_) {
+    std::vector<string> new_chars{"^"};
+    for (size_t index = 0; index < chars.size(); ++index) {
+      if (chars.at(index) == " ") {
+        new_chars.push_back("$");
+        new_chars.push_back(" ");
+        new_chars.push_back("^");
+      } else {
+        new_chars.push_back(chars.at(index));
+      }
+    }
+    new_chars.push_back("$");
+    chars.swap(new_chars);
+  }
+
+  // Find the char ngram counts.
+  std::unordered_map<string, int> char_ngram_counts;
+  int count_sum = 0;
+  for (int start = 0; start <= static_cast<int>(chars.size()) - ngram_size_;
+       ++start) {
+    string char_ngram;
+    int index;
+    for (index = 0; index < ngram_size_; ++index) {
+      const string &current_char = chars.at(start + index);
+      if (current_char == " " && !include_spaces_) {
+        break;
+      }
+      char_ngram.append(current_char);
+    }
+    if (index == ngram_size_) {
+      char_ngram_counts[char_ngram]++;
+      ++count_sum;
+    }
+  }
+
+  // Populate the feature vector.
+  const float equal_weight = 1.0 / char_ngram_counts.size();
+  const float norm = static_cast<float>(count_sum);
+  for (const auto &ngram_and_count : char_ngram_counts) {
+    const float weight =
+        use_equal_ngram_weight_ ? equal_weight : ngram_and_count.second / norm;
+    FloatFeatureValue value(
+        utils::Hash32WithDefaultSeed(ngram_and_count.first) %
+            ngram_id_dimension_,
+        weight);
+    result->add(feature_type(), value.discrete_value);
+  }
+}
+
+FeatureValue ScriptFeature::Compute(const WorkspaceSet &workspaces,
+                                    const Sentence &sentence,
+                                    const FeatureVector *result) const {
+  const string &text = sentence.text();
+  CLD2::ScriptScanner ss(text.c_str(), text.size(),
+                         /*is_plain_text=*/true);
+
+  // GetOneScriptSpan() is called only once because of the assumption that the
+  // input contains one script. This function also cleans up the input (e.g.,
+  // removes digits, punctuation).
+  // TODO(abakalov): Extract the clean-up and script detection code out of
+  // GetOneScriptSpan() because we don't have to iterate over the whole text,
+  // just look at the first codepoint after clean-up.
+  CLD2::LangSpan script_span;
+  ss.GetOneScriptSpan(&script_span);
+  const CLD2::ULScript ulscript = script_span.ulscript;
+  if (ulscript != CLD2::ULScript_Hani) {
+    return ulscript;
+  } else {
+    // Out of the codepoints captured by ULScript_Hani, separately count those
+    // in Hangul (Korean script) and those in a script other than Hangul.
+    int num_hangul = 0;
+    int num_non_hangul = 0;
+    UnicodeText unicode_text;
+    unicode_text.PointToUTF8(script_span.text, script_span.text_bytes);
+    for (chrome_lang_id::char32 codepoint : unicode_text) {
+      // If the current codepoint is space, continue.
+      if (codepoint == 0x20) {
+        continue;
+      }
+
+      // Check if the current codepoint is within the ranges associated with
+      // Hangul.
+      if ((codepoint >= 0x1100 && codepoint <= 0x11FF) ||  // Hangul Jamo
+          (codepoint >= 0xA960 && codepoint <= 0xA97F) ||  // Jamo Extended A
+          (codepoint >= 0xD7B0 && codepoint <= 0xD7FF) ||  // Jamo Extended B
+          (codepoint >= 0x3130 && codepoint <= 0x318F) ||  // Compatibility Jamo
+          (codepoint >= 0xFFA0 && codepoint <= 0xFFDC) ||  // Halfwidth Jamo
+          (codepoint >= 0xAC00 && codepoint <= 0xD7AF)) {  // Hangul Syllables
+        num_hangul++;
+      } else {
+        num_non_hangul++;
+      }
+    }
+
+    if (num_hangul > num_non_hangul) {
+      return static_cast<FeatureValue>(CLD2::NUM_ULSCRIPTS);
+    } else {
+      return static_cast<FeatureValue>(CLD2::ULScript_Hani);
+    }
+  }
+}
+
+}  // namespace chrome_lang_id

data/ext/cld3/language_identifier_features.h

@@ -0,0 +1,116 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef LANGUAGE_IDENTIFIER_FEATURES_H_
+#define LANGUAGE_IDENTIFIER_FEATURES_H_
+
+#include <string>
+
+#include "feature_extractor.h"
+#include "feature_types.h"
+#include "script_span/generated_ulscript.h"
+#include "cld_3/protos/sentence.pb.h"
+#include "sentence_features.h"
+#include "task_context.h"
+#include "workspace.h"
+
+namespace chrome_lang_id {
+
+// Feature type for numeric features.
+class NumericFeatureType : public FeatureType {
+ public:
+  // Initializes numeric feature.
+  NumericFeatureType(const string &name, FeatureValue size);
+
+  // Returns numeric feature value.
+  string GetFeatureValueName(FeatureValue value) const override;
+
+  // Returns the number of feature values.
+  FeatureValue GetDomainSize() const override;
+
+ private:
+  FeatureValue size_;
+};
+
+// Class for computing continuous char ngram features.
+// Feature function descriptor parameters:
+//   include_terminators(bool, false):
+//     If 'true', then splits the text based on spaces to get tokens, adds "^"
+//     to the beginning of each token, and adds "$" to the end of each token.
+//   include_spaces(bool, false):
+//     If 'true', then includes char ngrams containing spaces.
+//   use_equal_weight(bool, false):
+//     If 'true', then weighs each unique ngram by 1.0 / (number of unique
+//     ngrams in the input). Otherwise, weighs each unique ngram by (ngram
+//     count) / (total number of ngrams).
+//   id_dim(int, 10000):
+//     The integer id of each char ngram is computed as follows:
+//     Hash32WithDefaultSeed(char ngram) % id_dim.
+//   size(int, 3):
+//     Only ngrams of this size will be extracted.
+class ContinuousBagOfNgramsFunction : public WholeSentenceFeature {
+ public:
+  void Setup(TaskContext *context) override;
+  void Init(TaskContext *context) override;
+
+  // Appends the features computed from the focus to the feature vector.
+  void Evaluate(const WorkspaceSet &workspaces, const Sentence &sentence,
+                FeatureVector *result) const override;
+
+ private:
+  // If 'true', then splits the text based on spaces to get tokens, adds "^" to
+  // the beginning of each token, and adds "$" to the end of each token.
+  bool include_terminators_;
+
+  // If 'true', then includes char ngrams containing spaces.
+  bool include_spaces_;
+
+  // If 'true', then weighs each unique ngram by 1.0 / (number of unique ngrams
+  // in the input). Otherwise, weighs each unique ngram by (ngram count) /
+  // (total number of ngrams).
+  bool use_equal_ngram_weight_;
+
+  // The integer id of each char ngram is computed as follows:
+  // Hash32WithDefaultSeed(char_ngram) % ngram_id_dimension_.
+  int ngram_id_dimension_;
+
+  // Only ngrams of size ngram_size_ will be extracted.
+  int ngram_size_;
+};
+
+// Class for detecting the script of a piece of text. The list of supported
+// scripts is chrome_lang_id::CLD2::ULScript. This class uses the script
+// recognition code ported from CLD2. ULScript_Hani is split into non-Korean
+// script and Korean script (Hangul). In the former case, the function emits
+// ULScript_Hani. In the latter case, the function emits NUM_ULSCRIPTS. The
+// class assumes that the input is (1) interchange valid UTF8, and (2) contains
+// only one chrome_lang_id::CLD2::ULScript.
+class ScriptFeature : public WholeSentenceFeature {
+ public:
+  void Init(TaskContext *context) override {
+    // The dimension is incremented by 1 because ULScript_Hani is split into two
+    // as mentioned in the class description.
+    set_feature_type(new NumericFeatureType(
+        name(), chrome_lang_id::CLD2::NUM_ULSCRIPTS + 1));
+  }
+
+  // Computes the feature and saves it in the feature vector.
+  FeatureValue Compute(const WorkspaceSet &workspaces, const Sentence &sentence,
+                       const FeatureVector *result) const override;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // LANGUAGE_IDENTIFIER_FEATURES_H_