cld3 3.2.4 → 3.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91da77d7c9125f43c76c2645b04701b4c72a955859cd5cef23b9e846119e1112
-  data.tar.gz: 2090d645b5ae57212b496c41565031624e0987992b1cb616bdbfa5991244f55a
+  metadata.gz: 3c11344b17ebf6afb1108da299e132c5dda6a032c3d2e038ea4a476ff27d312f
+  data.tar.gz: 60c5d15ae88e10cb40629624e634960e1556c8032dcd48921c9d23b0514fb456
 SHA512:
-  metadata.gz: e9dd9783d699ee4653b1f808405b655425d1286dd6be663a965e4e448874dfd811eab73dd82f2a0c5e0ef91de8d4df62ffb2228bddb1642a57f9e1d01172115c
-  data.tar.gz: 2d4e703a21feba6b6ef95d5afbeb2f1748e5135dd120ba6a7976383b650659db4501045b2ffded8ff8575312a2a3f6cf092552d5833c5f435a122ae43e7c4a4e
+  metadata.gz: cfe48d2f1992204154b4d3a162f6c4a2ecf358d01f09eb0925be9a41fb968287544c84581458f11abd8d01ff54c74afb8b8a75d555da781c5b843868ca19e887
+  data.tar.gz: cc603eeaee27183aac35d223a01fc09342c314091d8ca0f25d54f051e4071853a8ba02fbd471ac575a3273919f8fc08497c539023e7be4fe72b416cb629b4ea5

data/cld3.gemspec

@@ -16,16 +16,16 @@
 
 Gem::Specification.new do |gem|
   gem.name = "cld3"
-  gem.version = "3.2.4"
+  gem.version = "3.2.5"
   gem.summary = "Compact Language Detector v3 (CLD3)"
   gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
   gem.license = "Apache-2.0"
   gem.homepage = "https://github.com/akihikodaki/cld3-ruby"
   gem.author = "Akihiko Odaki"
   gem.email = "akihiko.odaki.4i@stu.hosei.ac.jp"
-  gem.required_ruby_version = [ ">= 2.3.0", "< 2.7.0" ]
-  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.11.0" ]
-  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.9.0" ]
+  gem.required_ruby_version = [ ">= 2.3.0", "< 2.8.0" ]
+  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.12.0" ]
+  gem.add_development_dependency "rspec", [ ">=3.0.0", "< 3.10.0" ]
   gem.files = Dir[
     "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
     "cld3.gemspec", "ext/**/*", "lib/**/*"

data/ext/cld3/ext/CMakeLists.txt

@@ -0,0 +1,69 @@
+# This cmake scripts only builds a static cld3 lib and the unittests.
+
+project(cld3)
+
+# Old versions of cmake dont search/find protobuf lite
+cmake_minimum_required(VERSION 3.9)
+
+find_package(Protobuf REQUIRED)
+message(STATUS "Protobuf_FOUND= ${Protobuf_FOUND}")
+message(STATUS "Protobuf_VERSION= ${Protobuf_VERSION}")
+message(WARNING "Protobuf 2.5 and CLD3 seems happy together. This script does NOT check if your verison of protobuf is compatible.")
+message(STATUS "Protobuf_LIBRARIES= ${Protobuf_LIBRARIES}")
+message(STATUS "Protobuf_LITE_LIBRARIES= ${Protobuf_LITE_LIBRARIES}") # Usually /usr/lib64/libprotobuf-lite.so
+
+# By default, protobuf_generate_cpp generates pb.* files directy in the cmake build dir. 
+# But CLD3 sources have been coded using hard coded pathes to cld_3/protos/*.pb.h. 
+# So *.pb.h must be output to cld_3/protos.
+# For that, let's use a custom my_protobuf_generate_cpp:
+include(${CMAKE_CURRENT_SOURCE_DIR}/misc/myprotobuf.cmake)
+my_protobuf_generate_cpp(cld_3/protos PROTO_SRCS PROTO_HDRS src/feature_extractor.proto src/sentence.proto src/task_spec.proto)
+message(STATUS "PROTO_HDRS= ${PROTO_HDRS}")
+
+add_definitions(-fPIC) # Position Independant Code
+add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+add_definitions(-std=c++11) # Needed for std::to_string(), ...
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR}) # needed to include generated pb headers
+
+add_library(${PROJECT_NAME} 
+	${PROTO_SRCS} ${PROTO_HDRS}
+	src/base.cc
+	src/embedding_feature_extractor.cc
+	src/embedding_network.cc
+	src/feature_extractor.cc
+	src/feature_extractor.h
+	src/feature_types.cc
+	src/fml_parser.cc
+	src/language_identifier_features.cc
+	src/lang_id_nn_params.cc 
+	src/nnet_language_identifier.cc
+	src/registry.cc
+	src/relevant_script_feature.cc
+	src/sentence_features.cc
+	src/task_context.cc
+	src/task_context_params.cc
+	src/unicodetext.cc
+	src/utils.cc
+	src/workspace.cc
+	
+	src/script_span/generated_entities.cc
+	src/script_span/getonescriptspan.cc
+	src/script_span/getonescriptspan.h
+	src/script_span/getonescriptspan_test.cc
+	src/script_span/utf8statetable.cc
+	src/script_span/offsetmap.cc
+	src/script_span/text_processing.cc
+	src/script_span/text_processing.h
+	src/script_span/fixunicodevalue.cc
+	)
+
+# unit tests exec:
+add_executable(language_identifier_main src/language_identifier_main.cc)
+target_link_libraries(language_identifier_main cld3 ${Protobuf_LITE_LIBRARIES})
+
+add_executable(getonescriptspan_test src/script_span/getonescriptspan_test.cc)
+target_link_libraries(getonescriptspan_test cld3 ${Protobuf_LITE_LIBRARIES})
+
+add_executable(language_identifier_features_test src/language_identifier_features_test.cc)
+target_link_libraries(language_identifier_features_test cld3 ${Protobuf_LITE_LIBRARIES})

data/ext/cld3/ext/CONTRIBUTING.md

@@ -0,0 +1,26 @@
+Want to contribute? Great! First, read this page (including the small print at
+the end).
+
+### Before you contribute
+Before we can use your code, you must sign the
+[Google Individual Contributor License Agreement](https://cla.developers.google.com/about/google-individual)
+(CLA), which you can do online. The CLA is necessary mainly because you own the
+copyright to your changes, even after your contribution becomes part of our
+codebase, so we need your permission to use and distribute your code. We also
+need to be sure of various other things—for instance that you'll tell us if you
+know that your code infringes on other people's patents. You don't have to sign
+the CLA until after you've submitted your code for review and a member has
+approved it, but you must do it before we can put your code into our codebase.
+Before you start working on a larger contribution, you should get in touch with
+us first through the issue tracker with your idea so that we can help out and
+possibly guide you. Coordinating up front makes it much easier to avoid
+frustration later on.
+
+### Code reviews
+All submissions, including submissions by project members, require review. We
+use Github pull requests for this purpose.
+
+### The small print
+Contributions made by corporations are covered by a different agreement than
+the one above, the
+[Software Grant and Corporate Contributor License Agreement](https://cla.developers.google.com/about/google-corporate).

data/ext/cld3/ext/README.md

@@ -0,0 +1,73 @@
+# Compact Language Detector v3 (CLD3)
+
+* [Model](#model)
+* [Installation](#installation)
+* [Contact](#contact)
+* [Credits](#credits)
+
+### Model
+
+CLD3 is a neural network model for language identification. This package
+ contains the inference code and a trained model. The inference code
+ extracts character ngrams from the input text and computes the fraction
+ of times each of them appears. For example, as shown in the figure below,
+ if the input text is "banana", then one of the extracted trigrams is "ana"
+ and the corresponding fraction is 2/4. The ngrams are hashed down to an id
+ within a small range, and each id is represented by a dense embedding vector
+ estimated during training.
+
+The model averages the embeddings corresponding to each ngram type according
+ to the fractions, and the averaged embeddings are concatenated to produce
+ the embedding layer. The remaining components of the network are a hidden
+ (Rectified linear) layer and a softmax layer.
+
+To get a language prediction for the input text, we simply perform a forward
+ pass through the network.
+
+![Figure](model.png "CLD3")
+
+### Installation
+CLD3 is designed to run in the Chrome browser, so it relies on code in
+[Chromium](http://www.chromium.org/).
+The steps for building and running the demo of the language detection model are:
+
+- [check out](http://www.chromium.org/developers/how-tos/get-the-code) the
+  Chromium repository.
+- copy the code to `//third_party/cld_3`
+- Uncomment `language_identifier_main` executable in `src/BUILD.gn`.
+- build and run the model using the commands:
+
+```shell
+gn gen out/Default
+ninja -C out/Default third_party/cld_3/src/src:language_identifier_main
+out/Default/language_identifier_main
+```
+### Bugs and Feature Requests
+
+Open a [GitHub issue](https://github.com/google/cld3/issues) for this repository to file bugs and feature requests.
+
+### Announcements and Discussion
+
+For announcements regarding major updates as well as general discussion list, please subscribe to:
+[cld3-users@googlegroups.com](https://groups.google.com/forum/#!forum/cld3-users)
+
+### Credits
+
+Original authors of the code in this package include (in alphabetical order):
+
+* Alex Salcianu
+* Andy Golding
+* Anton Bakalov
+* Chris Alberti
+* Daniel Andor
+* David Weiss
+* Emily Pitler
+* Greg Coppola
+* Jason Riesa
+* Kuzman Ganchev
+* Michael Ringgaard
+* Nan Hua
+* Ryan McDonald
+* Slav Petrov
+* Stefan Istrate
+* Terry Koo

data/ext/cld3/ext/misc/myprotobuf.cmake

@@ -0,0 +1,58 @@
+# Special PROTOBUF_GENERATE_CPP which allows to set the output folder:
+# From https://stackoverflow.com/users/1600278/akira-okumura
+
+function(MY_PROTOBUF_GENERATE_CPP PATH SRCS HDRS)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS)
+    foreach(DIR ${PROTOBUF_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.h")
+
+    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${PATH})
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.h"
+      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS --cpp_out  ${CMAKE_CURRENT_BINARY_DIR}/${PATH} ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()

data/ext/cld3/ext/src/BUILD.gn

@@ -0,0 +1,133 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#==============================================================================
+
+import("//third_party/protobuf/proto_library.gni")
+
+proto_library("protos") {
+  sources = [
+    "feature_extractor.proto",
+    "sentence.proto",
+    "task_spec.proto",
+  ]
+  proto_out_dir = "cld_3/protos"
+}
+
+static_library("cld_3") {
+  sources = [
+    "base.cc",
+    "base.h",
+    "casts.h",
+    "embedding_feature_extractor.cc",
+    "embedding_feature_extractor.h",
+    "embedding_network.cc",
+    "embedding_network.h",
+    "embedding_network_params.h",
+    "feature_extractor.cc",
+    "feature_extractor.h",
+    "feature_types.cc",
+    "feature_types.h",
+    "float16.h",
+    "fml_parser.cc",
+    "fml_parser.h",
+    "language_identifier_features.cc",
+    "language_identifier_features.h",
+    "lang_id_nn_params.cc",
+    "lang_id_nn_params.h",
+    "nnet_language_identifier.cc",
+    "nnet_language_identifier.h",
+    "registry.cc",
+    "registry.h",
+    "relevant_script_feature.cc",
+    "relevant_script_feature.h",
+    "script_detector.h",
+    "sentence_features.cc",
+    "sentence_features.h",
+    "simple_adder.h",
+    "script_span/fixunicodevalue.cc",
+    "script_span/fixunicodevalue.h",
+    "script_span/generated_entities.cc",
+    "script_span/generated_ulscript.cc",
+    "script_span/generated_ulscript.h",
+    "script_span/getonescriptspan.cc",
+    "script_span/getonescriptspan.h",
+    "script_span/integral_types.h",
+    "script_span/offsetmap.cc",
+    "script_span/offsetmap.h",
+    "script_span/port.h",
+    "script_span/stringpiece.h",
+    "script_span/text_processing.cc",
+    "script_span/text_processing.h",
+    "script_span/utf8acceptinterchange.h",
+    "script_span/utf8prop_lettermarkscriptnum.h",
+    "script_span/utf8repl_lettermarklower.h",
+    "script_span/utf8scannot_lettermarkspecial.h",
+    "script_span/utf8statetable.cc",
+    "script_span/utf8statetable.h",
+    "task_context.cc",
+    "task_context.h",
+    "task_context_params.cc",
+    "task_context_params.h",
+    "unicodetext.cc",
+    "unicodetext.h",
+    "utils.cc",
+    "utils.h",
+    "workspace.cc",
+    "workspace.h",
+  ]
+  public_deps = [
+    "//third_party/protobuf:protobuf_lite",
+    ":protos",
+  ]
+}
+
+# The executables below are functional. Uncomment to use.
+
+#executable("language_identifier_main") {
+#  sources = [
+#    "language_identifier_main.cc",
+#  ]
+#  deps = [
+#    ":cld_3",
+#  ]
+#}
+
+#executable("getonescriptspan_test") {
+#  sources = [
+#    "script_span/getonescriptspan_test.cc",
+#  ]
+#  deps = [
+#    ":cld_3",
+#  ]
+#}
+
+#executable("language_identifier_features_test") {
+#  sources = [
+#    "language_identifier_features_test.cc",
+#  ]
+#  deps = [
+#    ":cld_3",
+#  ]
+#}
+
+#executable("nnet_lang_id_test") {
+#  sources = [
+#    "nnet_lang_id_test.cc",
+#    "nnet_lang_id_test_data.cc",
+#    "nnet_lang_id_test_data.h",
+#  ]
+#  deps = [
+#    ":cld_3",
+#  ]
+#}

data/ext/cld3/ext/src/DEPS

@@ -0,0 +1,4 @@
+include_rules = [
+  '+cld_3',
+  '+script_span',
+]