cld3 3.2.5 → 3.2.6

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cld3
 version: !ruby/object:Gem::Version
-  version: 3.2.5
+  version: 3.2.6
 platform: ruby
 authors:
 - Akihiko Odaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-03 00:00:00.000000000 Z
+date: 2020-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -60,89 +60,104 @@ extra_rdoc_files: []
 files:
 - Gemfile
 - LICENSE
+- LICENSE_CLD3
 - README.md
 - cld3.gemspec
-- ext/cld3/ext/CMakeLists.txt
-- ext/cld3/ext/CONTRIBUTING.md
-- ext/cld3/ext/LICENSE
-- ext/cld3/ext/README.md
-- ext/cld3/ext/misc/myprotobuf.cmake
-- ext/cld3/ext/model.png
-- ext/cld3/ext/src/BUILD.gn
-- ext/cld3/ext/src/DEPS
-- ext/cld3/ext/src/base.cc
-- ext/cld3/ext/src/base.h
-- ext/cld3/ext/src/casts.h
-- ext/cld3/ext/src/embedding_feature_extractor.cc
-- ext/cld3/ext/src/embedding_feature_extractor.h
-- ext/cld3/ext/src/embedding_network.cc
-- ext/cld3/ext/src/embedding_network.h
-- ext/cld3/ext/src/embedding_network_params.h
-- ext/cld3/ext/src/feature_extractor.cc
-- ext/cld3/ext/src/feature_extractor.h
-- ext/cld3/ext/src/feature_extractor.proto
-- ext/cld3/ext/src/feature_types.cc
-- ext/cld3/ext/src/feature_types.h
-- ext/cld3/ext/src/float16.h
-- ext/cld3/ext/src/fml_parser.cc
-- ext/cld3/ext/src/fml_parser.h
-- ext/cld3/ext/src/lang_id_nn_params.cc
-- ext/cld3/ext/src/lang_id_nn_params.h
-- ext/cld3/ext/src/language_identifier_features.cc
-- ext/cld3/ext/src/language_identifier_features.h
-- ext/cld3/ext/src/language_identifier_features_test.cc
-- ext/cld3/ext/src/language_identifier_main.cc
-- ext/cld3/ext/src/nnet_lang_id_test.cc
-- ext/cld3/ext/src/nnet_lang_id_test_data.cc
-- ext/cld3/ext/src/nnet_lang_id_test_data.h
-- ext/cld3/ext/src/nnet_language_identifier.cc
-- ext/cld3/ext/src/nnet_language_identifier.h
-- ext/cld3/ext/src/registry.cc
-- ext/cld3/ext/src/registry.h
-- ext/cld3/ext/src/relevant_script_feature.cc
-- ext/cld3/ext/src/relevant_script_feature.h
-- ext/cld3/ext/src/relevant_script_feature_test.cc
-- ext/cld3/ext/src/script_detector.h
-- ext/cld3/ext/src/script_detector_test.cc
-- ext/cld3/ext/src/script_span/README.md
-- ext/cld3/ext/src/script_span/fixunicodevalue.cc
-- ext/cld3/ext/src/script_span/fixunicodevalue.h
-- ext/cld3/ext/src/script_span/generated_entities.cc
-- ext/cld3/ext/src/script_span/generated_ulscript.cc
-- ext/cld3/ext/src/script_span/generated_ulscript.h
-- ext/cld3/ext/src/script_span/getonescriptspan.cc
-- ext/cld3/ext/src/script_span/getonescriptspan.h
-- ext/cld3/ext/src/script_span/getonescriptspan_test.cc
-- ext/cld3/ext/src/script_span/integral_types.h
-- ext/cld3/ext/src/script_span/offsetmap.cc
-- ext/cld3/ext/src/script_span/offsetmap.h
-- ext/cld3/ext/src/script_span/port.h
-- ext/cld3/ext/src/script_span/stringpiece.h
-- ext/cld3/ext/src/script_span/text_processing.cc
-- ext/cld3/ext/src/script_span/text_processing.h
-- ext/cld3/ext/src/script_span/utf8acceptinterchange.h
-- ext/cld3/ext/src/script_span/utf8prop_lettermarkscriptnum.h
-- ext/cld3/ext/src/script_span/utf8repl_lettermarklower.h
-- ext/cld3/ext/src/script_span/utf8scannot_lettermarkspecial.h
-- ext/cld3/ext/src/script_span/utf8statetable.cc
-- ext/cld3/ext/src/script_span/utf8statetable.h
-- ext/cld3/ext/src/sentence.proto
-- ext/cld3/ext/src/sentence_features.cc
-- ext/cld3/ext/src/sentence_features.h
-- ext/cld3/ext/src/simple_adder.h
-- ext/cld3/ext/src/task_context.cc
-- ext/cld3/ext/src/task_context.h
-- ext/cld3/ext/src/task_context_params.cc
-- ext/cld3/ext/src/task_context_params.h
-- ext/cld3/ext/src/task_spec.proto
-- ext/cld3/ext/src/unicodetext.cc
-- ext/cld3/ext/src/unicodetext.h
-- ext/cld3/ext/src/utils.cc
-- ext/cld3/ext/src/utils.h
-- ext/cld3/ext/src/workspace.cc
-- ext/cld3/ext/src/workspace.h
+- ext/cld3/Makefile
+- ext/cld3/base.cc
+- ext/cld3/base.h
+- ext/cld3/base.o
+- ext/cld3/casts.h
+- ext/cld3/embedding_feature_extractor.cc
+- ext/cld3/embedding_feature_extractor.h
+- ext/cld3/embedding_feature_extractor.o
+- ext/cld3/embedding_network.cc
+- ext/cld3/embedding_network.h
+- ext/cld3/embedding_network.o
+- ext/cld3/embedding_network_params.h
 - ext/cld3/extconf.rb
+- ext/cld3/feature_extractor.cc
+- ext/cld3/feature_extractor.h
+- ext/cld3/feature_extractor.o
+- ext/cld3/feature_extractor.pb.o
+- ext/cld3/feature_extractor.proto
+- ext/cld3/feature_types.cc
+- ext/cld3/feature_types.h
+- ext/cld3/feature_types.o
+- ext/cld3/fixunicodevalue.cc
+- ext/cld3/fixunicodevalue.h
+- ext/cld3/fixunicodevalue.o
+- ext/cld3/float16.h
+- ext/cld3/fml_parser.cc
+- ext/cld3/fml_parser.h
+- ext/cld3/fml_parser.o
+- ext/cld3/generated_entities.cc
+- ext/cld3/generated_entities.o
+- ext/cld3/generated_ulscript.cc
+- ext/cld3/generated_ulscript.h
+- ext/cld3/generated_ulscript.o
+- ext/cld3/getonescriptspan.cc
+- ext/cld3/getonescriptspan.h
+- ext/cld3/getonescriptspan.o
+- ext/cld3/integral_types.h
+- ext/cld3/lang_id_nn_params.cc
+- ext/cld3/lang_id_nn_params.h
+- ext/cld3/lang_id_nn_params.o
+- ext/cld3/language_identifier_features.cc
+- ext/cld3/language_identifier_features.h
+- ext/cld3/language_identifier_features.o
+- ext/cld3/libcld3.so
+- ext/cld3/mkmf.log
+- ext/cld3/nnet_language_identifier.cc
+- ext/cld3/nnet_language_identifier.h
+- ext/cld3/nnet_language_identifier.o
 - ext/cld3/nnet_language_identifier_c.cc
+- ext/cld3/nnet_language_identifier_c.o
+- ext/cld3/offsetmap.cc
+- ext/cld3/offsetmap.h
+- ext/cld3/offsetmap.o
+- ext/cld3/port.h
+- ext/cld3/registry.cc
+- ext/cld3/registry.h
+- ext/cld3/registry.o
+- ext/cld3/relevant_script_feature.cc
+- ext/cld3/relevant_script_feature.h
+- ext/cld3/relevant_script_feature.o
+- ext/cld3/script_detector.h
+- ext/cld3/sentence.pb.o
+- ext/cld3/sentence.proto
+- ext/cld3/sentence_features.cc
+- ext/cld3/sentence_features.h
+- ext/cld3/sentence_features.o
+- ext/cld3/simple_adder.h
+- ext/cld3/stringpiece.h
+- ext/cld3/task_context.cc
+- ext/cld3/task_context.h
+- ext/cld3/task_context.o
+- ext/cld3/task_context_params.cc
+- ext/cld3/task_context_params.h
+- ext/cld3/task_context_params.o
+- ext/cld3/task_spec.pb.o
+- ext/cld3/task_spec.proto
+- ext/cld3/text_processing.cc
+- ext/cld3/text_processing.h
+- ext/cld3/text_processing.o
+- ext/cld3/unicodetext.cc
+- ext/cld3/unicodetext.h
+- ext/cld3/unicodetext.o
+- ext/cld3/utf8acceptinterchange.h
+- ext/cld3/utf8prop_lettermarkscriptnum.h
+- ext/cld3/utf8repl_lettermarklower.h
+- ext/cld3/utf8scannot_lettermarkspecial.h
+- ext/cld3/utf8statetable.cc
+- ext/cld3/utf8statetable.h
+- ext/cld3/utf8statetable.o
+- ext/cld3/utils.cc
+- ext/cld3/utils.h
+- ext/cld3/utils.o
+- ext/cld3/workspace.cc
+- ext/cld3/workspace.h
+- ext/cld3/workspace.o
 - lib/cld3.rb
 homepage: https://github.com/akihikodaki/cld3-ruby
 licenses:

data/ext/cld3/ext/CMakeLists.txt

@@ -1,69 +0,0 @@
-# This cmake scripts only builds a static cld3 lib and the unittests.
-
-project(cld3)
-
-# Old versions of cmake dont search/find protobuf lite
-cmake_minimum_required(VERSION 3.9)
-
-find_package(Protobuf REQUIRED)
-message(STATUS "Protobuf_FOUND= ${Protobuf_FOUND}")
-message(STATUS "Protobuf_VERSION= ${Protobuf_VERSION}")
-message(WARNING "Protobuf 2.5 and CLD3 seems happy together. This script does NOT check if your verison of protobuf is compatible.")
-message(STATUS "Protobuf_LIBRARIES= ${Protobuf_LIBRARIES}")
-message(STATUS "Protobuf_LITE_LIBRARIES= ${Protobuf_LITE_LIBRARIES}") # Usually /usr/lib64/libprotobuf-lite.so
-
-# By default, protobuf_generate_cpp generates pb.* files directy in the cmake build dir. 
-# But CLD3 sources have been coded using hard coded pathes to cld_3/protos/*.pb.h. 
-# So *.pb.h must be output to cld_3/protos.
-# For that, let's use a custom my_protobuf_generate_cpp:
-include(${CMAKE_CURRENT_SOURCE_DIR}/misc/myprotobuf.cmake)
-my_protobuf_generate_cpp(cld_3/protos PROTO_SRCS PROTO_HDRS src/feature_extractor.proto src/sentence.proto src/task_spec.proto)
-message(STATUS "PROTO_HDRS= ${PROTO_HDRS}")
-
-add_definitions(-fPIC) # Position Independant Code
-add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
-add_definitions(-std=c++11) # Needed for std::to_string(), ...
-
-include_directories(${CMAKE_CURRENT_BINARY_DIR}) # needed to include generated pb headers
-
-add_library(${PROJECT_NAME} 
-	${PROTO_SRCS} ${PROTO_HDRS}
-	src/base.cc
-	src/embedding_feature_extractor.cc
-	src/embedding_network.cc
-	src/feature_extractor.cc
-	src/feature_extractor.h
-	src/feature_types.cc
-	src/fml_parser.cc
-	src/language_identifier_features.cc
-	src/lang_id_nn_params.cc 
-	src/nnet_language_identifier.cc
-	src/registry.cc
-	src/relevant_script_feature.cc
-	src/sentence_features.cc
-	src/task_context.cc
-	src/task_context_params.cc
-	src/unicodetext.cc
-	src/utils.cc
-	src/workspace.cc
-	
-	src/script_span/generated_entities.cc
-	src/script_span/getonescriptspan.cc
-	src/script_span/getonescriptspan.h
-	src/script_span/getonescriptspan_test.cc
-	src/script_span/utf8statetable.cc
-	src/script_span/offsetmap.cc
-	src/script_span/text_processing.cc
-	src/script_span/text_processing.h
-	src/script_span/fixunicodevalue.cc
-	)
-
-# unit tests exec:
-add_executable(language_identifier_main src/language_identifier_main.cc)
-target_link_libraries(language_identifier_main cld3 ${Protobuf_LITE_LIBRARIES})
-
-add_executable(getonescriptspan_test src/script_span/getonescriptspan_test.cc)
-target_link_libraries(getonescriptspan_test cld3 ${Protobuf_LITE_LIBRARIES})
-
-add_executable(language_identifier_features_test src/language_identifier_features_test.cc)
-target_link_libraries(language_identifier_features_test cld3 ${Protobuf_LITE_LIBRARIES})

data/ext/cld3/ext/CONTRIBUTING.md

@@ -1,26 +0,0 @@
-Want to contribute? Great! First, read this page (including the small print at
-the end).
-
-### Before you contribute
-Before we can use your code, you must sign the
-[Google Individual Contributor License Agreement](https://cla.developers.google.com/about/google-individual)
-(CLA), which you can do online. The CLA is necessary mainly because you own the
-copyright to your changes, even after your contribution becomes part of our
-codebase, so we need your permission to use and distribute your code. We also
-need to be sure of various other things—for instance that you'll tell us if you
-know that your code infringes on other people's patents. You don't have to sign
-the CLA until after you've submitted your code for review and a member has
-approved it, but you must do it before we can put your code into our codebase.
-Before you start working on a larger contribution, you should get in touch with
-us first through the issue tracker with your idea so that we can help out and
-possibly guide you. Coordinating up front makes it much easier to avoid
-frustration later on.
-
-### Code reviews
-All submissions, including submissions by project members, require review. We
-use Github pull requests for this purpose.
-
-### The small print
-Contributions made by corporations are covered by a different agreement than
-the one above, the
-[Software Grant and Corporate Contributor License Agreement](https://cla.developers.google.com/about/google-corporate).

data/ext/cld3/ext/README.md

@@ -1,73 +0,0 @@
-# Compact Language Detector v3 (CLD3)
-
-* [Model](#model)
-* [Installation](#installation)
-* [Contact](#contact)
-* [Credits](#credits)
-
-### Model
-
-CLD3 is a neural network model for language identification. This package
- contains the inference code and a trained model. The inference code
- extracts character ngrams from the input text and computes the fraction
- of times each of them appears. For example, as shown in the figure below,
- if the input text is "banana", then one of the extracted trigrams is "ana"
- and the corresponding fraction is 2/4. The ngrams are hashed down to an id
- within a small range, and each id is represented by a dense embedding vector
- estimated during training.
-
-The model averages the embeddings corresponding to each ngram type according
- to the fractions, and the averaged embeddings are concatenated to produce
- the embedding layer. The remaining components of the network are a hidden
- (Rectified linear) layer and a softmax layer.
-
-To get a language prediction for the input text, we simply perform a forward
- pass through the network.
-
-![Figure](model.png "CLD3")
-
-### Installation
-CLD3 is designed to run in the Chrome browser, so it relies on code in
-[Chromium](http://www.chromium.org/).
-The steps for building and running the demo of the language detection model are:
-
-- [check out](http://www.chromium.org/developers/how-tos/get-the-code) the
-  Chromium repository.
-- copy the code to `//third_party/cld_3`
-- Uncomment `language_identifier_main` executable in `src/BUILD.gn`.
-- build and run the model using the commands:
-
-```shell
-gn gen out/Default
-ninja -C out/Default third_party/cld_3/src/src:language_identifier_main
-out/Default/language_identifier_main
-```
-### Bugs and Feature Requests
-
-Open a [GitHub issue](https://github.com/google/cld3/issues) for this repository to file bugs and feature requests.
-
-### Announcements and Discussion
-
-For announcements regarding major updates as well as general discussion list, please subscribe to:
-[cld3-users@googlegroups.com](https://groups.google.com/forum/#!forum/cld3-users)
-
-### Credits
-
-Original authors of the code in this package include (in alphabetical order):
-
-* Alex Salcianu
-* Andy Golding
-* Anton Bakalov
-* Chris Alberti
-* Daniel Andor
-* David Weiss
-* Emily Pitler
-* Greg Coppola
-* Jason Riesa
-* Kuzman Ganchev
-* Michael Ringgaard
-* Nan Hua
-* Ryan McDonald
-* Slav Petrov
-* Stefan Istrate
-* Terry Koo

data/ext/cld3/ext/misc/myprotobuf.cmake

@@ -1,58 +0,0 @@
-# Special PROTOBUF_GENERATE_CPP which allows to set the output folder:
-# From https://stackoverflow.com/users/1600278/akira-okumura
-
-function(MY_PROTOBUF_GENERATE_CPP PATH SRCS HDRS)
-  if(NOT ARGN)
-    message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
-    return()
-  endif()
-
-  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
-    # Create an include path for each file specified
-    foreach(FIL ${ARGN})
-      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
-      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
-      if(${_contains_already} EQUAL -1)
-          list(APPEND _protobuf_include_path -I ${ABS_PATH})
-      endif()
-    endforeach()
-  else()
-    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
-  endif()
-
-  if(DEFINED PROTOBUF_IMPORT_DIRS)
-    foreach(DIR ${PROTOBUF_IMPORT_DIRS})
-      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
-      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
-      if(${_contains_already} EQUAL -1)
-          list(APPEND _protobuf_include_path -I ${ABS_PATH})
-      endif()
-    endforeach()
-  endif()
-
-  set(${SRCS})
-  set(${HDRS})
-  foreach(FIL ${ARGN})
-    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
-    get_filename_component(FIL_WE ${FIL} NAME_WE)
-
-    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.cc")
-    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.h")
-
-    execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${PATH})
-
-    add_custom_command(
-      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.cc"
-             "${CMAKE_CURRENT_BINARY_DIR}/${PATH}/${FIL_WE}.pb.h"
-      COMMAND  ${PROTOBUF_PROTOC_EXECUTABLE}
-      ARGS --cpp_out  ${CMAKE_CURRENT_BINARY_DIR}/${PATH} ${_protobuf_include_path} ${ABS_FIL}
-      DEPENDS ${ABS_FIL}
-      COMMENT "Running C++ protocol buffer compiler on ${FIL}"
-      VERBATIM )
-  endforeach()
-
-  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
-  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
-  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
-endfunction()