cld3 3.2.5 → 3.2.6

data/ext/cld3/ext/src/nnet_lang_id_test_data.h

@@ -1,117 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef NNET_LANG_ID_TEST_DATA_H_
-#define NNET_LANG_ID_TEST_DATA_H_
-
-namespace chrome_lang_id {
-
-class NNetLangIdTestData {
- public:
-  // Pieces of text in different languages.
-  static const char *const kTestStrAF;
-  static const char *const kTestStrAR;
-  static const char *const kTestStrAZ;
-  static const char *const kTestStrBE;
-  static const char *const kTestStrBG;
-  static const char *const kTestStrBN;
-  static const char *const kTestStrBS;
-  static const char *const kTestStrCA;
-  static const char *const kTestStrCEB;
-  static const char *const kTestStrCS;
-  static const char *const kTestStrCY;
-  static const char *const kTestStrDA;
-  static const char *const kTestStrDE;
-  static const char *const kTestStrEL;
-  static const char *const kTestStrEN;
-  static const char *const kTestStrEO;
-  static const char *const kTestStrES;
-  static const char *const kTestStrET;
-  static const char *const kTestStrEU;
-  static const char *const kTestStrFA;
-  static const char *const kTestStrFI;
-  static const char *const kTestStrFIL;
-  static const char *const kTestStrFR;
-  static const char *const kTestStrGA;
-  static const char *const kTestStrGL;
-  static const char *const kTestStrGU;
-  static const char *const kTestStrHA;
-  static const char *const kTestStrHI;
-  static const char *const kTestStrHMN;
-  static const char *const kTestStrHR;
-  static const char *const kTestStrHT;
-  static const char *const kTestStrHU;
-  static const char *const kTestStrHY;
-  static const char *const kTestStrID;
-  static const char *const kTestStrIG;
-  static const char *const kTestStrIS;
-  static const char *const kTestStrIT;
-  static const char *const kTestStrIW;
-  static const char *const kTestStrJA;
-  static const char *const kTestStrJV;
-  static const char *const kTestStrKA;
-  static const char *const kTestStrKK;
-  static const char *const kTestStrKM;
-  static const char *const kTestStrKN;
-  static const char *const kTestStrKO;
-  static const char *const kTestStrLA;
-  static const char *const kTestStrLO;
-  static const char *const kTestStrLT;
-  static const char *const kTestStrLV;
-  static const char *const kTestStrMG;
-  static const char *const kTestStrMI;
-  static const char *const kTestStrMK;
-  static const char *const kTestStrML;
-  static const char *const kTestStrMN;
-  static const char *const kTestStrMR;
-  static const char *const kTestStrMS;
-  static const char *const kTestStrMT;
-  static const char *const kTestStrMY;
-  static const char *const kTestStrNE;
-  static const char *const kTestStrNL;
-  static const char *const kTestStrNO;
-  static const char *const kTestStrNY;
-  static const char *const kTestStrPA;
-  static const char *const kTestStrPL;
-  static const char *const kTestStrPT;
-  static const char *const kTestStrRO;
-  static const char *const kTestStrRU;
-  static const char *const kTestStrSI;
-  static const char *const kTestStrSK;
-  static const char *const kTestStrSL;
-  static const char *const kTestStrSO;
-  static const char *const kTestStrSQ;
-  static const char *const kTestStrSR;
-  static const char *const kTestStrST;
-  static const char *const kTestStrSU;
-  static const char *const kTestStrSV;
-  static const char *const kTestStrSW;
-  static const char *const kTestStrTA;
-  static const char *const kTestStrTE;
-  static const char *const kTestStrTG;
-  static const char *const kTestStrTH;
-  static const char *const kTestStrTR;
-  static const char *const kTestStrUK;
-  static const char *const kTestStrUR;
-  static const char *const kTestStrUZ;
-  static const char *const kTestStrVI;
-  static const char *const kTestStrYI;
-  static const char *const kTestStrYO;
-  static const char *const kTestStrZH;
-  static const char *const kTestStrZU;
-};
-}  // namespace chrome_lang_id
-
-#endif  // NNET_LANG_ID_TEST_DATA_H_

data/ext/cld3/ext/src/relevant_script_feature_test.cc

@@ -1,259 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <algorithm>
-#include <cmath>
-#include <iostream>
-#include <memory>
-
-#include "feature_extractor.h"
-#include "feature_types.h"
-#include "relevant_script_feature.h"
-#include "script_detector.h"
-#include "cld_3/protos/sentence.pb.h"
-#include "sentence_features.h"
-#include "task_context.h"
-#include "utils.h"
-#include "workspace.h"
-
-namespace chrome_lang_id {
-namespace relevant_script_feature_test {
-namespace {
-// Checks whether the expected and actual float feature values are within 0.0001
-// of each other.
-bool FeatureValuesNear(float expected_value, float actual_value) {
-  return std::abs(expected_value - actual_value) < 0.0001;
-}
-
-// Checks whether two sets of feature values are within an acceptable amount of
-// each other.
-bool FeaturesNear(const string &test_input,
-                  const std::map<int, float> &expected_features,
-                  const std::map<int, float> &actual_features) {
-  if (expected_features.size() != actual_features.size()) {
-    std::cout << "  Failure for input: " << test_input << std::endl;
-    return false;
-  }
-
-  for (const auto &id_and_value : expected_features) {
-    const int id = id_and_value.first;
-    if (actual_features.count(id) == 0 ||
-        !FeatureValuesNear(expected_features.at(id), actual_features.at(id))) {
-      std::cout << "  Failure for input: " << test_input << std::endl;
-      return false;
-    }
-  }
-  std::cout << "  Success for input: " << test_input << std::endl;
-  return true;
-}
-
-// Checks whether the set of features is empty.
-bool CheckFeaturesEmpty(const string &input,
-                        const std::map<int, float> &actual_features) {
-  if (!actual_features.empty()) {
-    std::cout << "  Failure for input: " << input << std::endl;
-    return false;
-  } else {
-    std::cout << "  Success for input: " << input << std::endl;
-    return true;
-  }
-}
-}  // namespace
-
-static WholeSentenceFeature *rsf_factory() { return new RelevantScriptFeature; }
-
-class RelevantScriptFeatureExtractor {
- public:
-  RelevantScriptFeatureExtractor() {
-    if (WholeSentenceFeature::registry() == nullptr) {
-      // Create registry for our WholeSentenceFeature(s).
-      RegisterableClass<WholeSentenceFeature>::CreateRegistry(
-          "sentence feature function", "WholeSentenceFeature", __FILE__,
-          __LINE__);
-    }
-
-    // Register our WholeSentenceFeature(s).
-    // Register RelevantScriptFeature feature function.
-    static WholeSentenceFeature::Registry::Registrar rsf_registrar(
-        WholeSentenceFeature::registry(), "continuous-bag-of-relevant-scripts",
-        "RelevantScriptFeature", __FILE__, __LINE__, rsf_factory);
-
-    feature_extractor_.Parse("continuous-bag-of-relevant-scripts");
-    TaskContext context;
-    feature_extractor_.Setup(&context);
-    feature_extractor_.Init(&context);
-    feature_extractor_.RequestWorkspaces(&workspace_registry_);
-  }
-
-  // Returns "true" if feature extraction is successful, and "false" otherwise.
-  bool Extract(const string &text, std::map<int, float> *float_features) {
-    float_features->clear();
-    if (text.empty()) {
-      return true;
-    }
-    Sentence sentence;
-    sentence.set_text(text);
-    workspace_.Reset(workspace_registry_);
-    feature_extractor_.Preprocess(&workspace_, &sentence);
-    FeatureVector feature_vector;
-    feature_extractor_.ExtractFeatures(workspace_, sentence, &feature_vector);
-
-    for (int index = 0; index < feature_vector.size(); ++index) {
-      const FloatFeatureValue value =
-          FloatFeatureValue(feature_vector.value(index));
-      if (float_features->count(value.value.id) != 0) {
-        std::cout << "  Failure: duplicate feature" << std::endl;
-        return false;
-      }
-      float_features->emplace(value.value.id, value.value.weight);
-    }
-    return true;
-  }
-
- private:
-  WorkspaceSet workspace_;
-  WholeSentenceExtractor feature_extractor_;
-
-  // The registry of shared workspaces in the feature extractor.
-  WorkspaceRegistry workspace_registry_;
-};
-
-bool TestCommonCases() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-
-  RelevantScriptFeatureExtractor extractor;
-  std::map<int, float> float_features;
-  bool test_successful = true;
-
-  string input = "just some plain text";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  input = "ヸヂ゠ヂ";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptKatakana, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  // 4 Latin letters mixed with 4 Katakana letters.
-  input = "ヸtヂe゠xtヂ";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 0.5},
-                            {chrome_lang_id::kScriptKatakana, 0.5}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  input = "just some 121212%^^( ヸヂ゠ヂ   text";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptOtherUtf8OneByte, 0.75},
-                            {chrome_lang_id::kScriptKatakana, 0.25}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  return test_successful;
-}
-
-bool TestCornerCases() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-
-  RelevantScriptFeatureExtractor extractor;
-  std::map<int, float> float_features;
-  bool test_successful = true;
-
-  // Empty string.
-  string input = "";
-  if (!extractor.Extract(input, &float_features) ||
-      !CheckFeaturesEmpty(input, float_features)) {
-    test_successful = false;
-  }
-
-  // Only whitespaces.
-  input = "   ";
-  if (!extractor.Extract(input, &float_features) ||
-      !CheckFeaturesEmpty(input, float_features)) {
-    test_successful = false;
-  }
-
-  // Only numbers and punctuation.
-  input = "12----)(";
-  if (!extractor.Extract(input, &float_features) ||
-      !CheckFeaturesEmpty(input, float_features)) {
-    test_successful = false;
-  }
-
-  // Only numbers, punctuation, and spaces.
-  input = "12--- - ) ( ";
-  if (!extractor.Extract(input, &float_features) ||
-      !CheckFeaturesEmpty(input, float_features)) {
-    test_successful = false;
-  }
-
-  // One UTF8 character by itself.
-  input = "ゟ";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  input = "ה";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptHebrew, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  // One UTF8 character with some numbers / punctuation / spaces: character at
-  // one extremity or in the middle.
-  input = "1234ゟ";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  input = "ゟ12-(";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  input = "8*1ゟ12----";
-  if (!extractor.Extract(input, &float_features) ||
-      !FeaturesNear(input, {{chrome_lang_id::kScriptHiragana, 1.00}},
-                    float_features)) {
-    test_successful = false;
-  }
-
-  return test_successful;
-}
-
-}  // namespace relevant_script_feature_test
-}  // namespace chrome_lang_id
-
-// Runs the feature extraction tests.
-int main(int argc, char **argv) {
-  const bool tests_successful =
-      chrome_lang_id::relevant_script_feature_test::TestCommonCases() &&
-      chrome_lang_id::relevant_script_feature_test::TestCornerCases();
-  return tests_successful ? 0 : 1;
-}

data/ext/cld3/ext/src/script_detector_test.cc

@@ -1,161 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "script_detector.h"
-
-#include <iostream>
-
-#include "utils.h"
-
-namespace chrome_lang_id {
-namespace script_detector_test {
-
-Script GetScript(const char *p) {
-  const int num_bytes = utils::OneCharLen(p);
-  return chrome_lang_id::GetScript(p, num_bytes);
-}
-
-bool PrintAndReturnStatus(bool status) {
-  if (status) {
-    std::cout << "  Success" << std::endl;
-    return true;
-  } else {
-    std::cout << "  Failure" << std::endl;
-    return false;
-  }
-}
-
-bool TestGreekScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-
-  // The first two conditions check first / last character from the Greek and
-  // Coptic script. The last two ones are negative tests.
-  return PrintAndReturnStatus(
-      kScriptGreek == GetScript("Ͱ") && kScriptGreek == GetScript("Ͽ") &&
-      kScriptGreek == GetScript("δ") && kScriptGreek == GetScript("Θ") &&
-      kScriptGreek == GetScript("Δ") && kScriptGreek != GetScript("a") &&
-      kScriptGreek != GetScript("0"));
-}
-
-bool TestCyrillicScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(
-      kScriptCyrillic == GetScript("Ѐ") && kScriptCyrillic == GetScript("ӿ") &&
-      kScriptCyrillic == GetScript("ш") && kScriptCyrillic == GetScript("Б") &&
-      kScriptCyrillic == GetScript("Ӱ"));
-}
-
-bool TestHebrewScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(
-      kScriptHebrew == GetScript("֑") && kScriptHebrew == GetScript("״") &&
-      kScriptHebrew == GetScript("ד") && kScriptHebrew == GetScript("ה") &&
-      kScriptHebrew == GetScript("צ"));
-}
-
-bool TestArabicScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(kScriptArabic == GetScript("م") &&
-                              kScriptArabic == GetScript("خ"));
-}
-
-bool TestHangulJamoScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(kScriptHangulJamo == GetScript("ᄀ") &&
-                              kScriptHangulJamo == GetScript("ᇿ") &&
-                              kScriptHangulJamo == GetScript("ᄡ") &&
-                              kScriptHangulJamo == GetScript("ᆅ") &&
-                              kScriptHangulJamo == GetScript("ᅘ"));
-}
-
-bool TestHiraganaScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(kScriptHiragana == GetScript("ぁ") &&
-                              kScriptHiragana == GetScript("ゟ") &&
-                              kScriptHiragana == GetScript("こ") &&
-                              kScriptHiragana == GetScript("や") &&
-                              kScriptHiragana == GetScript("ぜ"));
-}
-
-bool TestKatakanaScript() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  return PrintAndReturnStatus(kScriptKatakana == GetScript("゠") &&
-                              kScriptKatakana == GetScript("ヿ") &&
-                              kScriptKatakana == GetScript("ヂ") &&
-                              kScriptKatakana == GetScript("ザ") &&
-                              kScriptKatakana == GetScript("ヸ"));
-}
-
-bool TestOtherScripts() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  bool test_successful = true;
-
-  if (kScriptOtherUtf8OneByte != GetScript("^") ||
-      kScriptOtherUtf8OneByte != GetScript("$")) {
-    test_successful = false;
-  }
-
-  // Unrecognized 2-byte scripts.  For info on the scripts mentioned below, see
-  // http://www.unicode.org/charts/#scripts Note: the scripts below are uniquely
-  // associated with a language.  Still, the number of queries in those
-  // languages is small and we didn't want to increase the code size and
-  // latency, so (at least for now) we do not treat them specially.
-  // The following three tests are, respectively, for Armenian, Syriac and
-  // Thaana.
-  if (kScriptOtherUtf8TwoBytes != GetScript("Ձ") ||
-      kScriptOtherUtf8TwoBytes != GetScript("ܔ") ||
-      kScriptOtherUtf8TwoBytes != GetScript("ށ")) {
-    test_successful = false;
-  }
-
-  // Unrecognized 3-byte script: CJK Unified Ideographs: not uniquely associated
-  // with a language.
-  if (kScriptOtherUtf8ThreeBytes != GetScript("万") ||
-      kScriptOtherUtf8ThreeBytes != GetScript("両")) {
-    test_successful = false;
-  }
-
-  // Unrecognized 4-byte script: CJK Unified Ideographs Extension C.  Note:
-  // there is a nice UTF-8 encoder / decoder at https://mothereff.in/utf-8
-  if (kScriptOtherUtf8FourBytes != GetScript("\xF0\xAA\x9C\x94")) {
-    test_successful = false;
-  }
-
-  // Unrecognized 4-byte script: CJK Unified Ideographs Extension E
-  if (kScriptOtherUtf8FourBytes != GetScript("\xF0\xAB\xA0\xB5") ||
-      kScriptOtherUtf8FourBytes != GetScript("\xF0\xAC\xBA\xA1")) {
-    test_successful = false;
-  }
-
-  return PrintAndReturnStatus(test_successful);
-}
-
-}  // namespace script_detector_test
-}  // namespace chrome_lang_id
-
-// Runs the feature extraction tests.
-int main(int argc, char **argv) {
-  const bool tests_successful =
-      chrome_lang_id::script_detector_test::TestGreekScript() &&
-      chrome_lang_id::script_detector_test::TestCyrillicScript() &&
-      chrome_lang_id::script_detector_test::TestHebrewScript() &&
-      chrome_lang_id::script_detector_test::TestArabicScript() &&
-      chrome_lang_id::script_detector_test::TestHangulJamoScript() &&
-      chrome_lang_id::script_detector_test::TestHiraganaScript() &&
-      chrome_lang_id::script_detector_test::TestKatakanaScript() &&
-      chrome_lang_id::script_detector_test::TestOtherScripts();
-
-  return tests_successful ? 0 : 1;
-}