cld3 3.2.4 → 3.2.5

data/ext/cld3/ext/src/nnet_lang_id_test_data.h

@@ -0,0 +1,117 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef NNET_LANG_ID_TEST_DATA_H_
+#define NNET_LANG_ID_TEST_DATA_H_
+
+namespace chrome_lang_id {
+
+class NNetLangIdTestData {
+ public:
+  // Pieces of text in different languages.
+  static const char *const kTestStrAF;
+  static const char *const kTestStrAR;
+  static const char *const kTestStrAZ;
+  static const char *const kTestStrBE;
+  static const char *const kTestStrBG;
+  static const char *const kTestStrBN;
+  static const char *const kTestStrBS;
+  static const char *const kTestStrCA;
+  static const char *const kTestStrCEB;
+  static const char *const kTestStrCS;
+  static const char *const kTestStrCY;
+  static const char *const kTestStrDA;
+  static const char *const kTestStrDE;
+  static const char *const kTestStrEL;
+  static const char *const kTestStrEN;
+  static const char *const kTestStrEO;
+  static const char *const kTestStrES;
+  static const char *const kTestStrET;
+  static const char *const kTestStrEU;
+  static const char *const kTestStrFA;
+  static const char *const kTestStrFI;
+  static const char *const kTestStrFIL;
+  static const char *const kTestStrFR;
+  static const char *const kTestStrGA;
+  static const char *const kTestStrGL;
+  static const char *const kTestStrGU;
+  static const char *const kTestStrHA;
+  static const char *const kTestStrHI;
+  static const char *const kTestStrHMN;
+  static const char *const kTestStrHR;
+  static const char *const kTestStrHT;
+  static const char *const kTestStrHU;
+  static const char *const kTestStrHY;
+  static const char *const kTestStrID;
+  static const char *const kTestStrIG;
+  static const char *const kTestStrIS;
+  static const char *const kTestStrIT;
+  static const char *const kTestStrIW;
+  static const char *const kTestStrJA;
+  static const char *const kTestStrJV;
+  static const char *const kTestStrKA;
+  static const char *const kTestStrKK;
+  static const char *const kTestStrKM;
+  static const char *const kTestStrKN;
+  static const char *const kTestStrKO;
+  static const char *const kTestStrLA;
+  static const char *const kTestStrLO;
+  static const char *const kTestStrLT;
+  static const char *const kTestStrLV;
+  static const char *const kTestStrMG;
+  static const char *const kTestStrMI;
+  static const char *const kTestStrMK;
+  static const char *const kTestStrML;
+  static const char *const kTestStrMN;
+  static const char *const kTestStrMR;
+  static const char *const kTestStrMS;
+  static const char *const kTestStrMT;
+  static const char *const kTestStrMY;
+  static const char *const kTestStrNE;
+  static const char *const kTestStrNL;
+  static const char *const kTestStrNO;
+  static const char *const kTestStrNY;
+  static const char *const kTestStrPA;
+  static const char *const kTestStrPL;
+  static const char *const kTestStrPT;
+  static const char *const kTestStrRO;
+  static const char *const kTestStrRU;
+  static const char *const kTestStrSI;
+  static const char *const kTestStrSK;
+  static const char *const kTestStrSL;
+  static const char *const kTestStrSO;
+  static const char *const kTestStrSQ;
+  static const char *const kTestStrSR;
+  static const char *const kTestStrST;
+  static const char *const kTestStrSU;
+  static const char *const kTestStrSV;
+  static const char *const kTestStrSW;
+  static const char *const kTestStrTA;
+  static const char *const kTestStrTE;
+  static const char *const kTestStrTG;
+  static const char *const kTestStrTH;
+  static const char *const kTestStrTR;
+  static const char *const kTestStrUK;
+  static const char *const kTestStrUR;
+  static const char *const kTestStrUZ;
+  static const char *const kTestStrVI;
+  static const char *const kTestStrYI;
+  static const char *const kTestStrYO;
+  static const char *const kTestStrZH;
+  static const char *const kTestStrZU;
+};
+}  // namespace chrome_lang_id
+
+#endif  // NNET_LANG_ID_TEST_DATA_H_

data/ext/cld3/{nnet_language_identifier.cc → ext/src/nnet_language_identifier.cc}

@@ -47,6 +47,9 @@ struct LangChunksStats {
 
   // Number chunks corresponding to the language.
   int num_chunks = 0;
+
+  // Specifies the byte ranges that language applies to.
+  std::vector<NNetLanguageIdentifier::SpanInfo> byte_ranges;
 };
 
 // Compares two pairs based on their values.
@@ -298,12 +301,16 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
     total_num_bytes += num_original_span_bytes;
 
     const string selected_text = SelectTextGivenScriptSpan(script_span);
+
     result = FindLanguageOfValidUTF8(selected_text);
     language = result.language;
     lang_stats[language].byte_sum += num_original_span_bytes;
     lang_stats[language].prob_sum +=
         result.probability * num_original_span_bytes;
     lang_stats[language].num_chunks++;
+    // Add SpanInfo. Start and end indices are relative to original input.
+    lang_stats[language].byte_ranges.push_back(SpanInfo(
+        ss.MapBack(0), ss.MapBack(script_span.text_bytes), result.probability));
   }
 
   // Sort the languages based on the number of bytes associated with them.
@@ -329,6 +336,7 @@ NNetLanguageIdentifier::FindTopNMostFreqLangs(const string &text,
     result.probability = stats.prob_sum / stats.byte_sum;
     result.proportion = stats.byte_sum / byte_sum;
     result.is_reliable = ResultIsReliable(language, result.probability);
+    result.byte_ranges = stats.byte_ranges;
     results.push_back(result);
   }
 

data/ext/cld3/{nnet_language_identifier.h → ext/src/nnet_language_identifier.h}

@@ -44,6 +44,19 @@ class LanguageIdEmbeddingFeatureExtractor
 // Class for detecting the language of a document.
 class NNetLanguageIdentifier {
  public:
+  // Holds probability that Span, specified by start/end indices, is a given
+  // language. The langauge is not stored here; it can be found in Result, which
+  // holds a vector of SpanInfo.
+  struct SpanInfo {
+    SpanInfo(int start_index_val, int end_index_val, float probability_val)
+        : start_index(start_index_val),
+          end_index(end_index_val),
+          probability(probability_val) {}
+    int start_index = -1;
+    int end_index = -1;
+    float probability = 0.0;
+  };
+
   // Information about a predicted language.
   struct Result {
     string language = kUnknown;
@@ -53,6 +66,9 @@ class NNetLanguageIdentifier {
     // Proportion of bytes associated with the language. If FindLanguage is
     // called, this variable is set to 1.
     float proportion = 0.0;
+
+    // Specifies the byte ranges that |language| applies to.
+    std::vector<SpanInfo> byte_ranges;
   };
 
   NNetLanguageIdentifier();

data/ext/cld3/ext/src/relevant_script_feature_test.cc

@@ -0,0 +1,259 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <cmath>
+#include <iostream>
+#include <memory>
+
+#include "feature_extractor.h"
+#include "feature_types.h"
+#include "relevant_script_feature.h"
+#include "script_detector.h"
+#include "cld_3/protos/sentence.pb.h"
+#include "sentence_features.h"
+#include "task_context.h"
+#include "utils.h"
+#include "workspace.h"
+
+namespace chrome_lang_id {
+namespace relevant_script_feature_test {
+namespace {
+// Checks whether the expected and actual float feature values are within 0.0001
+// of each other.
+bool FeatureValuesNear(float expected_value, float actual_value) {
+  return std::abs(expected_value - actual_value) < 0.0001;
+}
+
+// Checks whether two sets of feature values are within an acceptable amount of
+// each other.
+bool FeaturesNear(const string &test_input,
+                  const std::map<int, float> &expected_features,
+                  const std::map<int, float> &actual_features) {
+  if (expected_features.size() != actual_features.size()) {
+    std::cout << "  Failure for input: " << test_input << std::endl;
+    return false;
+  }
+
+  for (const auto &id_and_value : expected_features) {
+    const int id = id_and_value.first;
+    if (actual_features.count(id) == 0 ||
+        !FeatureValuesNear(expected_features.at(id), actual_features.at(id))) {
+      std::cout << "  Failure for input: " << test_input << std::endl;
+      return false;
+    }
+  }
+  std::cout << "  Success for input: " << test_input << std::endl;
+  return true;
+}
+
+// Checks whether the set of features is empty.
+bool CheckFeaturesEmpty(const string &input,
+                        const std::map<int, float> &actual_features) {
+  if (!actual_features.empty()) {
+    std::cout << "  Failure for input: " << input << std::endl;
+    return false;
+  } else {
+    std::cout << "  Success for input: " << input << std::endl;
+    return true;
+  }
+}
+}  // namespace
+
+static WholeSentenceFeature *rsf_factory() { return new RelevantScriptFeature; }
+
+class RelevantScriptFeatureExtractor {
+ public:
+  RelevantScriptFeatureExtractor() {
+    if (WholeSentenceFeature::registry() == nullptr) {
+      // Create registry for our WholeSentenceFeature(s).
+      RegisterableClass<WholeSentenceFeature>::CreateRegistry(
+          "sentence feature function", "WholeSentenceFeature", __FILE__,
+          __LINE__);
+    }
+
+    // Register our WholeSentenceFeature(s).
+    // Register RelevantScriptFeature feature function.
+    static WholeSentenceFeature::Registry::Registrar rsf_registrar(
+        WholeSentenceFeature::registry(), "continuous-bag-of-relevant-scripts",
+        "RelevantScriptFeature", __FILE__, __LINE__, rsf_factory);
+
+    feature_extractor_.Parse("continuous-bag-of-relevant-scripts");
+    TaskContext context;
+    feature_extractor_.Setup(&context);
+    feature_extractor_.Init(&context);
+    feature_extractor_.RequestWorkspaces(&workspace_registry_);
+  }
+
+  // Returns "true" if feature extraction is successful, and "false" otherwise.
+  bool Extract(const string &text, std::map<int, float> *float_features) {
+    float_features->clear();
+    if (text.empty()) {
+      return true;
+    }
+    Sentence sentence;
+    sentence.set_text(text);
+    workspace_.Reset(workspace_registry_);
+    feature_extractor_.Preprocess(&workspace_, &sentence);
+    FeatureVector feature_vector;
+    feature_extractor_.ExtractFeatures(workspace_, sentence, &feature_vector);
+
+    for (int index = 0; index < feature_vector.size(); ++index) {
+      const FloatFeatureValue value =
+          FloatFeatureValue(feature_vector.value(index));
+      if (float_features->count(value.value.id) != 0) {
+        std::cout << "  Failure: duplicate feature" << std::endl;
+        return false;
+      }
+      float_features->emplace(value.value.id, value.value.weight);
+    }
+    return true;
+  }
+
+ private:
+  WorkspaceSet workspace_;
+  WholeSentenceExtractor feature_extractor_;
+
+  // The registry of shared workspaces in the feature extractor.
+  WorkspaceRegistry workspace_registry_;
+};
+
+bool TestCommonCases() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+
+  RelevantScriptFeatureExtractor extractor;
+  std::map<int, float> float_features;
+  bool test_successful = true;
+
+  string input = "just some plain text";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  input = "ヸヂ゠ヂ";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptKatakana, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  // 4 Latin letters mixed with 4 Katakana letters.
+  input = "ヸtヂe゠xtヂ";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 0.5},
+                            {chrome_lang_id::kScriptKatakana, 0.5}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  input = "just some 121212%^^( ヸヂ゠ヂ   text";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 0.75},
+                            {chrome_lang_id::kScriptKatakana, 0.25}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  return test_successful;
+}
+
+bool TestCornerCases() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+
+  RelevantScriptFeatureExtractor extractor;
+  std::map<int, float> float_features;
+  bool test_successful = true;
+
+  // Empty string.
+  string input = "";
+  if (!extractor.Extract(input, &float_features) ||
+      !CheckFeaturesEmpty(input, float_features)) {
+    test_successful = false;
+  }
+
+  // Only whitespaces.
+  input = "   ";
+  if (!extractor.Extract(input, &float_features) ||
+      !CheckFeaturesEmpty(input, float_features)) {
+    test_successful = false;
+  }
+
+  // Only numbers and punctuation.
+  input = "12----)(";
+  if (!extractor.Extract(input, &float_features) ||
+      !CheckFeaturesEmpty(input, float_features)) {
+    test_successful = false;
+  }
+
+  // Only numbers, punctuation, and spaces.
+  input = "12--- - ) ( ";
+  if (!extractor.Extract(input, &float_features) ||
+      !CheckFeaturesEmpty(input, float_features)) {
+    test_successful = false;
+  }
+
+  // One UTF8 character by itself.
+  input = "ゟ";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  input = "ה";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptHebrew, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  // One UTF8 character with some numbers / punctuation / spaces: character at
+  // one extremity or in the middle.
+  input = "1234ゟ";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  input = "ゟ12-(";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  input = "8*1ゟ12----";
+  if (!extractor.Extract(input, &float_features) ||
+      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
+                    float_features)) {
+    test_successful = false;
+  }
+
+  return test_successful;
+}
+
+}  // namespace relevant_script_feature_test
+}  // namespace chrome_lang_id
+
+// Runs the feature extraction tests.
+int main(int argc, char **argv) {
+  const bool tests_successful =
+      chrome_lang_id::relevant_script_feature_test::TestCommonCases() &&
+      chrome_lang_id::relevant_script_feature_test::TestCornerCases();
+  return tests_successful ? 0 : 1;
+}