cld3 3.2.5 → 3.2.6

data/ext/cld3/ext/src/script_span/README.md

@@ -1,11 +0,0 @@
-The code in this directory identifies the scripts present in a given piece of
-text along with the corresponding spans. The code was copied from
-[CLD2](https://github.com/CLD2Owners/cld2) and was slightly refactored. It can
-be further simplified and cleaned up.
-
-
-
-
-
-
-

data/ext/cld3/ext/src/script_span/getonescriptspan_test.cc

@@ -1,135 +0,0 @@
-/* Copyright 2016 Google Inc. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "getonescriptspan.h"
-
-#include <iostream>
-#include <vector>
-
-namespace chrome_lang_id {
-namespace CLD2 {
-namespace getonescriptspan_test {
-
-// Tests invalid and interchange-invalid input. Returns "true" if the test is
-// successful and "false" otherwise.
-bool TestInvalidUTF8Input() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-  const std::vector<std::string> invalid_strings{"\xC0\xA9",
-                                                 "\377\377\377\377"};
-  const std::string gold_valid_prefix = "Some valid bytes followed by ";
-
-  // Iterates over the invalid strings, inserts each of them in the middle of a
-  // piece of text, and checks whether these strings are correctly identified.
-  bool test_successful = true;
-  for (size_t i = 0; i < invalid_strings.size(); ++i) {
-    const std::string text = "Some valid bytes followed by " +
-                             invalid_strings.at(i) +
-                             " and then valid ones again.";
-
-    const int num_valid_bytes = SpanInterchangeValid(text.c_str(), text.size());
-    const std::string detected_valid_prefix(text.c_str(), num_valid_bytes);
-    std::cout << "  Testing input string at position " << i << std::endl;
-    if (detected_valid_prefix == gold_valid_prefix) {
-      std::cout << "    Success!" << std::endl;
-    } else {
-      std::cout << "    Failure" << std::endl;
-      std::cout << "    Gold: " << gold_valid_prefix << std::endl;
-      std::cout << "    Detected: " << detected_valid_prefix << std::endl;
-      test_successful = false;
-    }
-  }
-  return test_successful;
-}
-
-// Tests whether different scripts are correctly detected. Returns "true" if the
-// test is successful and "false" otherwise.
-bool TestScriptDetection() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-
-  // Text containing a snippet in English, a snippet in Bulgarian, and a snippet
-  // in English again.
-  const std::string text =
-      "Text in English. Текст на Български. Also text in English.";
-  const std::vector<std::string> gold_script_spans{
-      " Text in English ", " Текст на Български ", " Also text in English "};
-
-  std::vector<std::string> detected_script_spans;
-  ScriptScanner ss(text.c_str(), text.size(), /*is_plain_text=*/true);
-  LangSpan script_span;
-  while (ss.GetOneScriptSpan(&script_span)) {
-    detected_script_spans.emplace_back(script_span.text,
-                                       script_span.text_bytes);
-  }
-
-  if (detected_script_spans.size() != gold_script_spans.size()) {
-    std::cout << "  Failure" << std::endl;
-    std::cout << "  Number of gold spans " << gold_script_spans.size()
-              << std::endl;
-    std::cout << "  Number of detected spans " << detected_script_spans.size()
-              << std::endl;
-    return false;
-  }
-  for (size_t i = 0; i < detected_script_spans.size(); ++i) {
-    if (detected_script_spans.at(i) != gold_script_spans.at(i)) {
-      std::cout << "  Failure" << std::endl;
-      std::cout << "  Gold span: " << gold_script_spans.at(i) << std::endl;
-      std::cout << "  Detected span: " << detected_script_spans.at(i)
-                << std::endl;
-      return false;
-    }
-  }
-  std::cout << "  Success!" << std::endl;
-  return true;
-}
-
-// Tests the case when the input string is truncated in such a way that a
-// character is split in two pieces. Returns "true" if the test is successful
-// and "false" otherwise.
-bool TestStringCut() {
-  std::cout << "Running " << __FUNCTION__ << std::endl;
-
-  // Text in Bulgarian (Cyrillic script).
-  const std::string text = "Текст на Български";
-
-  // The size of the first two words ("Текст на ") is 16, and size of the first
-  // two words plus the first char of the third word ("Текст на Б") is 18, so a
-  // threshold of 17 results in slicing the first char of the third word.
-  const int first_two_words_size = 16;
-  const int span_size = 17;
-  const int num_valid_bytes = SpanInterchangeValid(text.c_str(), span_size);
-  if (num_valid_bytes == first_two_words_size) {
-    std::cout << "  Success!" << std::endl;
-    return true;
-  } else {
-    std::cout << "  Failure" << std::endl;
-    std::cout << "  Size of gold interchange-valid span: "
-              << first_two_words_size << std::endl;
-    std::cout << "  Size of detected span: " << num_valid_bytes << std::endl;
-    return false;
-  }
-}
-
-}  // namespace getonescriptspan_test
-}  // namespace CLD2
-}  // namespace chrome_lang_id
-
-// Runs the functions above.
-int main(int argc, char **argv) {
-  const bool tests_successful =
-      chrome_lang_id::CLD2::getonescriptspan_test::TestInvalidUTF8Input() &&
-      chrome_lang_id::CLD2::getonescriptspan_test::TestScriptDetection() &&
-      chrome_lang_id::CLD2::getonescriptspan_test::TestStringCut();
-  return tests_successful ? 0 : 1;
-}