cld3 3.2.4 → 3.2.5

data/ext/cld3/ext/src/script_detector_test.cc

@@ -0,0 +1,161 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "script_detector.h"
+
+#include <iostream>
+
+#include "utils.h"
+
+namespace chrome_lang_id {
+namespace script_detector_test {
+
+Script GetScript(const char *p) {
+  const int num_bytes = utils::OneCharLen(p);
+  return chrome_lang_id::GetScript(p, num_bytes);
+}
+
+bool PrintAndReturnStatus(bool status) {
+  if (status) {
+    std::cout << "  Success" << std::endl;
+    return true;
+  } else {
+    std::cout << "  Failure" << std::endl;
+    return false;
+  }
+}
+
+bool TestGreekScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+
+  // The first two conditions check first / last character from the Greek and
+  // Coptic script. The last two ones are negative tests.
+  return PrintAndReturnStatus(
+      kScriptGreek == GetScript("Ͱ") && kScriptGreek == GetScript("Ͽ") &&
+      kScriptGreek == GetScript("δ") && kScriptGreek == GetScript("Θ") &&
+      kScriptGreek == GetScript("Δ") && kScriptGreek != GetScript("a") &&
+      kScriptGreek != GetScript("0"));
+}
+
+bool TestCyrillicScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(
+      kScriptCyrillic == GetScript("Ѐ") && kScriptCyrillic == GetScript("ӿ") &&
+      kScriptCyrillic == GetScript("ш") && kScriptCyrillic == GetScript("Б") &&
+      kScriptCyrillic == GetScript("Ӱ"));
+}
+
+bool TestHebrewScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(
+      kScriptHebrew == GetScript("֑") && kScriptHebrew == GetScript("״") &&
+      kScriptHebrew == GetScript("ד") && kScriptHebrew == GetScript("ה") &&
+      kScriptHebrew == GetScript("צ"));
+}
+
+bool TestArabicScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(kScriptArabic == GetScript("م") &&
+                              kScriptArabic == GetScript("خ"));
+}
+
+bool TestHangulJamoScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(kScriptHangulJamo == GetScript("ᄀ") &&
+                              kScriptHangulJamo == GetScript("ᇿ") &&
+                              kScriptHangulJamo == GetScript("ᄡ") &&
+                              kScriptHangulJamo == GetScript("ᆅ") &&
+                              kScriptHangulJamo == GetScript("ᅘ"));
+}
+
+bool TestHiraganaScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(kScriptHiragana == GetScript("ぁ") &&
+                              kScriptHiragana == GetScript("ゟ") &&
+                              kScriptHiragana == GetScript("こ") &&
+                              kScriptHiragana == GetScript("や") &&
+                              kScriptHiragana == GetScript("ぜ"));
+}
+
+bool TestKatakanaScript() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  return PrintAndReturnStatus(kScriptKatakana == GetScript("゠") &&
+                              kScriptKatakana == GetScript("ヿ") &&
+                              kScriptKatakana == GetScript("ヂ") &&
+                              kScriptKatakana == GetScript("ザ") &&
+                              kScriptKatakana == GetScript("ヸ"));
+}
+
+bool TestOtherScripts() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  bool test_successful = true;
+
+  if (kScriptOtherUtf8OneByte != GetScript("^") ||
+      kScriptOtherUtf8OneByte != GetScript("$")) {
+    test_successful = false;
+  }
+
+  // Unrecognized 2-byte scripts.  For info on the scripts mentioned below, see
+  // http://www.unicode.org/charts/#scripts Note: the scripts below are uniquely
+  // associated with a language.  Still, the number of queries in those
+  // languages is small and we didn't want to increase the code size and
+  // latency, so (at least for now) we do not treat them specially.
+  // The following three tests are, respectively, for Armenian, Syriac and
+  // Thaana.
+  if (kScriptOtherUtf8TwoBytes != GetScript("Ձ") ||
+      kScriptOtherUtf8TwoBytes != GetScript("ܔ") ||
+      kScriptOtherUtf8TwoBytes != GetScript("ށ")) {
+    test_successful = false;
+  }
+
+  // Unrecognized 3-byte script: CJK Unified Ideographs: not uniquely associated
+  // with a language.
+  if (kScriptOtherUtf8ThreeBytes != GetScript("万") ||
+      kScriptOtherUtf8ThreeBytes != GetScript("両")) {
+    test_successful = false;
+  }
+
+  // Unrecognized 4-byte script: CJK Unified Ideographs Extension C.  Note:
+  // there is a nice UTF-8 encoder / decoder at https://mothereff.in/utf-8
+  if (kScriptOtherUtf8FourBytes != GetScript("\xF0\xAA\x9C\x94")) {
+    test_successful = false;
+  }
+
+  // Unrecognized 4-byte script: CJK Unified Ideographs Extension E
+  if (kScriptOtherUtf8FourBytes != GetScript("\xF0\xAB\xA0\xB5") ||
+      kScriptOtherUtf8FourBytes != GetScript("\xF0\xAC\xBA\xA1")) {
+    test_successful = false;
+  }
+
+  return PrintAndReturnStatus(test_successful);
+}
+
+}  // namespace script_detector_test
+}  // namespace chrome_lang_id
+
+// Runs the feature extraction tests.
+int main(int argc, char **argv) {
+  const bool tests_successful =
+      chrome_lang_id::script_detector_test::TestGreekScript() &&
+      chrome_lang_id::script_detector_test::TestCyrillicScript() &&
+      chrome_lang_id::script_detector_test::TestHebrewScript() &&
+      chrome_lang_id::script_detector_test::TestArabicScript() &&
+      chrome_lang_id::script_detector_test::TestHangulJamoScript() &&
+      chrome_lang_id::script_detector_test::TestHiraganaScript() &&
+      chrome_lang_id::script_detector_test::TestKatakanaScript() &&
+      chrome_lang_id::script_detector_test::TestOtherScripts();
+
+  return tests_successful ? 0 : 1;
+}

data/ext/cld3/ext/src/script_span/README.md

@@ -0,0 +1,11 @@
+The code in this directory identifies the scripts present in a given piece of
+text along with the corresponding spans. The code was copied from
+[CLD2](https://github.com/CLD2Owners/cld2) and was slightly refactored. It can
+be further simplified and cleaned up.
+
+
+
+
+
+
+

data/ext/cld3/{getonescriptspan.h → ext/src/script_span/getonescriptspan.h}

@@ -93,7 +93,7 @@ class ScriptScanner {
   // again with the first byte of the following range.
   int MapBack(int text_offset);
 
-  const char* GetBufferStart() {return start_byte_;};
+  const char* GetBufferStart() {return start_byte_;}
 
  private:
   // Skip over tags and non-letters

data/ext/cld3/ext/src/script_span/getonescriptspan_test.cc

@@ -0,0 +1,135 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "getonescriptspan.h"
+
+#include <iostream>
+#include <vector>
+
+namespace chrome_lang_id {
+namespace CLD2 {
+namespace getonescriptspan_test {
+
+// Tests invalid and interchange-invalid input. Returns "true" if the test is
+// successful and "false" otherwise.
+bool TestInvalidUTF8Input() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+  const std::vector<std::string> invalid_strings{"\xC0\xA9",
+                                                 "\377\377\377\377"};
+  const std::string gold_valid_prefix = "Some valid bytes followed by ";
+
+  // Iterates over the invalid strings, inserts each of them in the middle of a
+  // piece of text, and checks whether these strings are correctly identified.
+  bool test_successful = true;
+  for (size_t i = 0; i < invalid_strings.size(); ++i) {
+    const std::string text = "Some valid bytes followed by " +
+                             invalid_strings.at(i) +
+                             " and then valid ones again.";
+
+    const int num_valid_bytes = SpanInterchangeValid(text.c_str(), text.size());
+    const std::string detected_valid_prefix(text.c_str(), num_valid_bytes);
+    std::cout << "  Testing input string at position " << i << std::endl;
+    if (detected_valid_prefix == gold_valid_prefix) {
+      std::cout << "    Success!" << std::endl;
+    } else {
+      std::cout << "    Failure" << std::endl;
+      std::cout << "    Gold: " << gold_valid_prefix << std::endl;
+      std::cout << "    Detected: " << detected_valid_prefix << std::endl;
+      test_successful = false;
+    }
+  }
+  return test_successful;
+}
+
+// Tests whether different scripts are correctly detected. Returns "true" if the
+// test is successful and "false" otherwise.
+bool TestScriptDetection() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+
+  // Text containing a snippet in English, a snippet in Bulgarian, and a snippet
+  // in English again.
+  const std::string text =
+      "Text in English. Текст на Български. Also text in English.";
+  const std::vector<std::string> gold_script_spans{
+      " Text in English ", " Текст на Български ", " Also text in English "};
+
+  std::vector<std::string> detected_script_spans;
+  ScriptScanner ss(text.c_str(), text.size(), /*is_plain_text=*/true);
+  LangSpan script_span;
+  while (ss.GetOneScriptSpan(&script_span)) {
+    detected_script_spans.emplace_back(script_span.text,
+                                       script_span.text_bytes);
+  }
+
+  if (detected_script_spans.size() != gold_script_spans.size()) {
+    std::cout << "  Failure" << std::endl;
+    std::cout << "  Number of gold spans " << gold_script_spans.size()
+              << std::endl;
+    std::cout << "  Number of detected spans " << detected_script_spans.size()
+              << std::endl;
+    return false;
+  }
+  for (size_t i = 0; i < detected_script_spans.size(); ++i) {
+    if (detected_script_spans.at(i) != gold_script_spans.at(i)) {
+      std::cout << "  Failure" << std::endl;
+      std::cout << "  Gold span: " << gold_script_spans.at(i) << std::endl;
+      std::cout << "  Detected span: " << detected_script_spans.at(i)
+                << std::endl;
+      return false;
+    }
+  }
+  std::cout << "  Success!" << std::endl;
+  return true;
+}
+
+// Tests the case when the input string is truncated in such a way that a
+// character is split in two pieces. Returns "true" if the test is successful
+// and "false" otherwise.
+bool TestStringCut() {
+  std::cout << "Running " << __FUNCTION__ << std::endl;
+
+  // Text in Bulgarian (Cyrillic script).
+  const std::string text = "Текст на Български";
+
+  // The size of the first two words ("Текст на ") is 16, and size of the first
+  // two words plus the first char of the third word ("Текст на Б") is 18, so a
+  // threshold of 17 results in slicing the first char of the third word.
+  const int first_two_words_size = 16;
+  const int span_size = 17;
+  const int num_valid_bytes = SpanInterchangeValid(text.c_str(), span_size);
+  if (num_valid_bytes == first_two_words_size) {
+    std::cout << "  Success!" << std::endl;
+    return true;
+  } else {
+    std::cout << "  Failure" << std::endl;
+    std::cout << "  Size of gold interchange-valid span: "
+              << first_two_words_size << std::endl;
+    std::cout << "  Size of detected span: " << num_valid_bytes << std::endl;
+    return false;
+  }
+}
+
+}  // namespace getonescriptspan_test
+}  // namespace CLD2
+}  // namespace chrome_lang_id
+
+// Runs the functions above.
+int main(int argc, char **argv) {
+  const bool tests_successful =
+      chrome_lang_id::CLD2::getonescriptspan_test::TestInvalidUTF8Input() &&
+      chrome_lang_id::CLD2::getonescriptspan_test::TestScriptDetection() &&
+      chrome_lang_id::CLD2::getonescriptspan_test::TestStringCut();
+  return tests_successful ? 0 : 1;
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.2.4
+  version: 3.2.5
 platform: ruby
 authors:
 - Akihiko Odaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-20 00:00:00.000000000 Z
+date: 2020-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -19,7 +19,7 @@ dependencies:
         version: 1.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 1.12.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 1.1.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.11.0
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +39,7 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.10.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ dependencies:
         version: 3.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: 3.10.0
 description: Compact Language Detector v3 (CLD3) is a neural network model for language
   identification.
 email: akihiko.odaki.4i@stu.hosei.ac.jp
@@ -60,73 +60,89 @@ extra_rdoc_files: []
 files:
 - Gemfile
 - LICENSE
-- LICENSE_CLD3
 - README.md
 - cld3.gemspec
-- ext/cld3/base.cc
-- ext/cld3/base.h
-- ext/cld3/casts.h
-- ext/cld3/embedding_feature_extractor.cc
-- ext/cld3/embedding_feature_extractor.h
-- ext/cld3/embedding_network.cc
-- ext/cld3/embedding_network.h
-- ext/cld3/embedding_network_params.h
+- ext/cld3/ext/CMakeLists.txt
+- ext/cld3/ext/CONTRIBUTING.md
+- ext/cld3/ext/LICENSE
+- ext/cld3/ext/README.md
+- ext/cld3/ext/misc/myprotobuf.cmake
+- ext/cld3/ext/model.png
+- ext/cld3/ext/src/BUILD.gn
+- ext/cld3/ext/src/DEPS
+- ext/cld3/ext/src/base.cc
+- ext/cld3/ext/src/base.h
+- ext/cld3/ext/src/casts.h
+- ext/cld3/ext/src/embedding_feature_extractor.cc
+- ext/cld3/ext/src/embedding_feature_extractor.h
+- ext/cld3/ext/src/embedding_network.cc
+- ext/cld3/ext/src/embedding_network.h
+- ext/cld3/ext/src/embedding_network_params.h
+- ext/cld3/ext/src/feature_extractor.cc
+- ext/cld3/ext/src/feature_extractor.h
+- ext/cld3/ext/src/feature_extractor.proto
+- ext/cld3/ext/src/feature_types.cc
+- ext/cld3/ext/src/feature_types.h
+- ext/cld3/ext/src/float16.h
+- ext/cld3/ext/src/fml_parser.cc
+- ext/cld3/ext/src/fml_parser.h
+- ext/cld3/ext/src/lang_id_nn_params.cc
+- ext/cld3/ext/src/lang_id_nn_params.h
+- ext/cld3/ext/src/language_identifier_features.cc
+- ext/cld3/ext/src/language_identifier_features.h
+- ext/cld3/ext/src/language_identifier_features_test.cc
+- ext/cld3/ext/src/language_identifier_main.cc
+- ext/cld3/ext/src/nnet_lang_id_test.cc
+- ext/cld3/ext/src/nnet_lang_id_test_data.cc
+- ext/cld3/ext/src/nnet_lang_id_test_data.h
+- ext/cld3/ext/src/nnet_language_identifier.cc
+- ext/cld3/ext/src/nnet_language_identifier.h
+- ext/cld3/ext/src/registry.cc
+- ext/cld3/ext/src/registry.h
+- ext/cld3/ext/src/relevant_script_feature.cc
+- ext/cld3/ext/src/relevant_script_feature.h
+- ext/cld3/ext/src/relevant_script_feature_test.cc
+- ext/cld3/ext/src/script_detector.h
+- ext/cld3/ext/src/script_detector_test.cc
+- ext/cld3/ext/src/script_span/README.md
+- ext/cld3/ext/src/script_span/fixunicodevalue.cc
+- ext/cld3/ext/src/script_span/fixunicodevalue.h
+- ext/cld3/ext/src/script_span/generated_entities.cc
+- ext/cld3/ext/src/script_span/generated_ulscript.cc
+- ext/cld3/ext/src/script_span/generated_ulscript.h
+- ext/cld3/ext/src/script_span/getonescriptspan.cc
+- ext/cld3/ext/src/script_span/getonescriptspan.h
+- ext/cld3/ext/src/script_span/getonescriptspan_test.cc
+- ext/cld3/ext/src/script_span/integral_types.h
+- ext/cld3/ext/src/script_span/offsetmap.cc
+- ext/cld3/ext/src/script_span/offsetmap.h
+- ext/cld3/ext/src/script_span/port.h
+- ext/cld3/ext/src/script_span/stringpiece.h
+- ext/cld3/ext/src/script_span/text_processing.cc
+- ext/cld3/ext/src/script_span/text_processing.h
+- ext/cld3/ext/src/script_span/utf8acceptinterchange.h
+- ext/cld3/ext/src/script_span/utf8prop_lettermarkscriptnum.h
+- ext/cld3/ext/src/script_span/utf8repl_lettermarklower.h
+- ext/cld3/ext/src/script_span/utf8scannot_lettermarkspecial.h
+- ext/cld3/ext/src/script_span/utf8statetable.cc
+- ext/cld3/ext/src/script_span/utf8statetable.h
+- ext/cld3/ext/src/sentence.proto
+- ext/cld3/ext/src/sentence_features.cc
+- ext/cld3/ext/src/sentence_features.h
+- ext/cld3/ext/src/simple_adder.h
+- ext/cld3/ext/src/task_context.cc
+- ext/cld3/ext/src/task_context.h
+- ext/cld3/ext/src/task_context_params.cc
+- ext/cld3/ext/src/task_context_params.h
+- ext/cld3/ext/src/task_spec.proto
+- ext/cld3/ext/src/unicodetext.cc
+- ext/cld3/ext/src/unicodetext.h
+- ext/cld3/ext/src/utils.cc
+- ext/cld3/ext/src/utils.h
+- ext/cld3/ext/src/workspace.cc
+- ext/cld3/ext/src/workspace.h
 - ext/cld3/extconf.rb
-- ext/cld3/feature_extractor.cc
-- ext/cld3/feature_extractor.h
-- ext/cld3/feature_extractor.proto
-- ext/cld3/feature_types.cc
-- ext/cld3/feature_types.h
-- ext/cld3/fixunicodevalue.cc
-- ext/cld3/fixunicodevalue.h
-- ext/cld3/float16.h
-- ext/cld3/fml_parser.cc
-- ext/cld3/fml_parser.h
-- ext/cld3/generated_entities.cc
-- ext/cld3/generated_ulscript.cc
-- ext/cld3/generated_ulscript.h
-- ext/cld3/getonescriptspan.cc
-- ext/cld3/getonescriptspan.h
-- ext/cld3/integral_types.h
-- ext/cld3/lang_id_nn_params.cc
-- ext/cld3/lang_id_nn_params.h
-- ext/cld3/language_identifier_features.cc
-- ext/cld3/language_identifier_features.h
-- ext/cld3/nnet_language_identifier.cc
-- ext/cld3/nnet_language_identifier.h
 - ext/cld3/nnet_language_identifier_c.cc
-- ext/cld3/offsetmap.cc
-- ext/cld3/offsetmap.h
-- ext/cld3/port.h
-- ext/cld3/registry.cc
-- ext/cld3/registry.h
-- ext/cld3/relevant_script_feature.cc
-- ext/cld3/relevant_script_feature.h
-- ext/cld3/script_detector.h
-- ext/cld3/sentence.proto
-- ext/cld3/sentence_features.cc
-- ext/cld3/sentence_features.h
-- ext/cld3/simple_adder.h
-- ext/cld3/stringpiece.h
-- ext/cld3/task_context.cc
-- ext/cld3/task_context.h
-- ext/cld3/task_context_params.cc
-- ext/cld3/task_context_params.h
-- ext/cld3/task_spec.proto
-- ext/cld3/text_processing.cc
-- ext/cld3/text_processing.h
-- ext/cld3/unicodetext.cc
-- ext/cld3/unicodetext.h
-- ext/cld3/utf8acceptinterchange.h
-- ext/cld3/utf8prop_lettermarkscriptnum.h
-- ext/cld3/utf8repl_lettermarklower.h
-- ext/cld3/utf8scannot_lettermarkspecial.h
-- ext/cld3/utf8statetable.cc
-- ext/cld3/utf8statetable.h
-- ext/cld3/utils.cc
-- ext/cld3/utils.h
-- ext/cld3/workspace.cc
-- ext/cld3/workspace.h
 - lib/cld3.rb
 homepage: https://github.com/akihikodaki/cld3-ruby
 licenses:
@@ -143,14 +159,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
   - - "<"
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 2.8.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Compact Language Detector v3 (CLD3)