cld3 3.1.0

data/cld3.gemspec

@@ -0,0 +1,35 @@
+# Copyright 2017 Akihiko Odaki <akihiko.odaki.4i@stu.hosei.ac.jp>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#==============================================================================
+
+Gem::Specification.new do |gem|
+  gem.name = "cld3"
+  gem.version = "3.1.0"
+  gem.summary = "Compact Language Detector v3 (CLD3)"
+  gem.description = "Compact Language Detector v3 (CLD3) is a neural network model for language identification."
+  gem.license = "Apache-2.0"
+  gem.homepage = "https://github.com/akihikodaki/cld3-ruby"
+  gem.author = "Akihiko Odaki"
+  gem.email = "akihiko.odaki.4i@stu.hosei.ac.jp"
+  gem.required_ruby_version = [ ">= 2.3.0", "< 2.5.0" ]
+  gem.add_dependency "ffi", [ ">= 1.1.0", "< 1.10.0" ]
+  gem.add_development_dependency "rspec", [ ">=2.11.0", "< 3.7.0" ]
+  gem.files = Dir[
+    "Gemfile", "LICENSE", "LICENSE_CLD3", "README.md",
+    "cld3.gemspec", "ext/**/*", "lib/**/*"
+  ]
+  gem.require_paths = [ "lib" ]
+  gem.extensions = [ "ext/cld3/extconf.rb" ]
+end

data/ext/cld3/base.cc

@@ -0,0 +1,36 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "base.h"
+
+#include <string>
+#ifdef COMPILER_MSVC
+#include <sstream>
+#endif  // COMPILER_MSVC
+
+namespace chrome_lang_id {
+
+// TODO(abakalov): Pick the most efficient approach.
+#ifdef COMPILER_MSVC
+std::string Int64ToString(int64 input) {
+  std::stringstream stream;
+  stream << input;
+  return stream.str();
+}
+#else
+std::string Int64ToString(int64 input) { return std::to_string(input); }
+#endif  // COMPILER_MSVC
+
+}  // namespace chrome_lang_id

data/ext/cld3/base.h

@@ -0,0 +1,106 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef BASE_H_
+#define BASE_H_
+
+#include <cassert>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace chrome_lang_id {
+
+using std::vector;
+using std::string;
+using std::map;
+using std::pair;
+typedef unsigned int uint32;
+
+#if LANG_CXX11
+#define CLD3_DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName &) = delete;          \
+  TypeName &operator=(const TypeName &) = delete
+#else  // C++98 case follows
+
+// Note that these C++98 implementations cannot completely disallow copying,
+// as members and friends can still accidentally make elided copies without
+// triggering a linker error.
+#define CLD3_DISALLOW_COPY_AND_ASSIGN(TypeName) \
+  TypeName(const TypeName &);                   \
+  TypeName &operator=(const TypeName &)
+#endif  // LANG_CXX11
+
+#ifndef CLD3_IMMEDIATE_CRASH
+#if defined(__GNUC__) || defined(__clang__)
+#define CLD3_IMMEDIATE_CRASH() __builtin_trap()
+#else
+#define CLD3_IMMEDIATE_CRASH() ((void)(*(volatile char *)0 = 0))
+#endif
+#endif  // CLD3_IMMEDIATE_CRASH
+
+#define CLD3_CHECK(f) (!(f) ? CLD3_IMMEDIATE_CRASH() : (void)0)
+
+#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON)
+#define CLD3_DCHECK(f) ((void)0)
+#else
+#define CLD3_DCHECK(f) CLD3_CHECK(f)
+#endif
+
+#ifndef SWIG
+typedef int int32;
+typedef unsigned char uint8;    // NOLINT
+typedef unsigned short uint16;  // NOLINT
+
+// A type to represent a Unicode code-point value. As of Unicode 4.0,
+// such values require up to 21 bits.
+// (For type-checking on pointers, make this explicitly signed,
+// and it should always be the signed version of whatever int32 is.)
+typedef signed int char32;
+#endif  // SWIG
+
+#ifdef COMPILER_MSVC
+typedef __int64 int64;
+#else
+typedef long long int64;  // NOLINT
+#endif  // COMPILER_MSVC
+
+#if defined(__GNUC__) && \
+    (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))
+
+// For functions we want to force inline.
+// Introduced in gcc 3.1.
+#define CLD3_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline))
+
+#elif defined(_MSC_VER)
+#define CLD3_ATTRIBUTE_ALWAYS_INLINE __forceinline
+#else
+
+// Other compilers will have to figure it out for themselves.
+#define CLD3_ATTRIBUTE_ALWAYS_INLINE
+#endif
+
+#ifdef INTERNAL_BUILD
+typedef basic_string<char> bstring;
+#else
+typedef std::basic_string<char> bstring;
+#endif  // INTERNAL_BUILD
+
+// Converts int64 to string.
+std::string Int64ToString(int64 input);
+
+}  // namespace chrome_lang_id
+
+#endif  // BASE_H_

data/ext/cld3/casts.h

@@ -0,0 +1,98 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This code is compiled directly on many platforms, including client
+// platforms like Windows, Mac, and embedded systems.  Before making
+// any changes here, make sure that you're not breaking any platforms.
+//
+
+#ifndef CASTS_H_
+#define CASTS_H_
+
+#include <string.h>  // for memcpy
+
+namespace chrome_lang_id {
+
+// lang_id_bit_cast<Dest,Source> is a template function that implements the
+// equivalent of "*reinterpret_cast<Dest*>(&source)".  We need this in
+// very low-level functions like the protobuf library and fast math
+// support.
+//
+//   float f = 3.14159265358979;
+//   int i = lang_id_bit_cast<int32>(f);
+//   // i = 0x40490fdb
+//
+// The classical address-casting method is:
+//
+//   // WRONG
+//   float f = 3.14159265358979;            // WRONG
+//   int i = * reinterpret_cast<int*>(&f);  // WRONG
+//
+// The address-casting method actually produces undefined behavior
+// according to ISO C++ specification section 3.10 -15 -.  Roughly, this
+// section says: if an object in memory has one type, and a program
+// accesses it with a different type, then the result is undefined
+// behavior for most values of "different type".
+//
+// This is true for any cast syntax, either *(int*)&f or
+// *reinterpret_cast<int*>(&f).  And it is particularly true for
+// conversions between integral lvalues and floating-point lvalues.
+//
+// The purpose of 3.10 -15- is to allow optimizing compilers to assume
+// that expressions with different types refer to different memory.  gcc
+// 4.0.1 has an optimizer that takes advantage of this.  So a
+// non-conforming program quietly produces wildly incorrect output.
+//
+// The problem is not the use of reinterpret_cast.  The problem is type
+// punning: holding an object in memory of one type and reading its bits
+// back using a different type.
+//
+// The C++ standard is more subtle and complex than this, but that
+// is the basic idea.
+//
+// Anyways ...
+//
+// lang_id_bit_cast<> calls memcpy() which is blessed by the standard,
+// especially by the example in section 3.9 .  Also, of course,
+// lang_id_bit_cast<> wraps up the nasty logic in one place.
+//
+// Fortunately memcpy() is very fast.  In optimized mode, with a
+// constant size, gcc 2.95.3, gcc 4.0.1, and msvc 7.1 produce inline
+// code with the minimal amount of data movement.  On a 32-bit system,
+// memcpy(d,s,4) compiles to one load and one store, and memcpy(d,s,8)
+// compiles to two loads and two stores.
+//
+// I tested this code with gcc 2.95.3, gcc 4.0.1, icc 8.1, and msvc 7.1.
+//
+// WARNING: if Dest or Source is a non-POD type, the result of the memcpy
+// is likely to surprise you.
+//
+// Props to Bill Gibbons for the compile time assertion technique and
+// Art Komninos and Igor Tandetnik for the msvc experiments.
+//
+// -- mec 2005-10-17
+
+template <class Dest, class Source>
+inline Dest lang_id_bit_cast(const Source &source) {
+  static_assert(sizeof(Dest) == sizeof(Source), "Sizes do not match");
+
+  Dest dest;
+  memcpy(&dest, &source, sizeof(dest));
+  return dest;
+}
+
+}  // namespace chrome_lang_id
+
+#endif  // CASTS_H_

data/ext/cld3/embedding_feature_extractor.cc

@@ -0,0 +1,51 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "embedding_feature_extractor.h"
+
+#include <stddef.h>
+#include <vector>
+
+#include "feature_extractor.h"
+#include "feature_types.h"
+#include "task_context.h"
+#include "utils.h"
+
+namespace chrome_lang_id {
+
+GenericEmbeddingFeatureExtractor::GenericEmbeddingFeatureExtractor() {}
+
+GenericEmbeddingFeatureExtractor::~GenericEmbeddingFeatureExtractor() {}
+
+void GenericEmbeddingFeatureExtractor::Setup(TaskContext *context) {
+  // Don't use version to determine how to get feature FML.
+  string features_param = ArgPrefix();
+  features_param += "_features";
+  const string features = context->Get(features_param, "");
+  const string embedding_names =
+      context->Get(GetParamName("embedding_names"), "");
+  const string embedding_dims =
+      context->Get(GetParamName("embedding_dims"), "");
+  embedding_fml_ = utils::Split(features, ';');
+  add_strings_ = context->Get(GetParamName("add_varlen_strings"), false);
+  embedding_names_ = utils::Split(embedding_names, ';');
+  for (const string &dim : utils::Split(embedding_dims, ';')) {
+    embedding_dims_.push_back(utils::ParseUsing<int>(dim, utils::ParseInt32));
+  }
+}
+
+void GenericEmbeddingFeatureExtractor::Init(TaskContext *context) {}
+
+}  // namespace chrome_lang_id

data/ext/cld3/embedding_feature_extractor.h

@@ -0,0 +1,182 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef EMBEDDING_FEATURE_EXTRACTOR_H_
+#define EMBEDDING_FEATURE_EXTRACTOR_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "feature_extractor.h"
+#include "task_context.h"
+#include "workspace.h"
+
+namespace chrome_lang_id {
+
+// An EmbeddingFeatureExtractor manages the extraction of features for
+// embedding-based models. It wraps a sequence of underlying classes of feature
+// extractors, along with associated predicate maps. Each class of feature
+// extractors is associated with a name, e.g., "unigrams", "bigrams".
+//
+// The class is split between a generic abstract version,
+// GenericEmbeddingFeatureExtractor (that can be initialized without knowing the
+// signature of the ExtractFeatures method) and a typed version.
+//
+// The predicate maps must be initialized before use: they can be loaded using
+// Read() or updated via UpdateMapsForExample.
+class GenericEmbeddingFeatureExtractor {
+ public:
+  GenericEmbeddingFeatureExtractor();
+  virtual ~GenericEmbeddingFeatureExtractor();
+
+  // Get the prefix string to put in front of all arguments, so they don't
+  // conflict with other embedding models.
+  virtual const string ArgPrefix() const = 0;
+
+  // Sets up predicate maps and embedding space names that are common for all
+  // embedding based feature extractors.
+  virtual void Setup(TaskContext *context);
+  virtual void Init(TaskContext *context);
+
+  // Requests workspace for the underlying feature extractors. This is
+  // implemented in the typed class.
+  virtual void RequestWorkspaces(WorkspaceRegistry *registry) = 0;
+
+  // Number of predicates for the embedding at a given index (vocabulary size.)
+  int EmbeddingSize(int index) const {
+    return generic_feature_extractor(index).GetDomainSize();
+  }
+
+  // Returns number of embedding spaces.
+  int NumEmbeddings() const { return embedding_dims_.size(); }
+
+  // Returns the number of features in the embedding space.
+  int FeatureSize(int idx) const {
+    return generic_feature_extractor(idx).feature_types();
+  }
+
+  // Returns the dimensionality of the embedding space.
+  int EmbeddingDims(int index) const { return embedding_dims_[index]; }
+
+  // Accessor for embedding dims (dimensions of the embedding spaces).
+  const std::vector<int> &embedding_dims() const { return embedding_dims_; }
+
+  const std::vector<string> &embedding_fml() const { return embedding_fml_; }
+
+  // Get parameter name by concatenating the prefix and the original name.
+  string GetParamName(const string &param_name) const {
+    string name = ArgPrefix();
+    name += "_";
+    name += param_name;
+    return name;
+  }
+
+ protected:
+  // Provides the generic class with access to the templated extractors. This is
+  // used to get the type information out of the feature extractor without
+  // knowing the specific calling arguments of the extractor itself.
+  virtual const GenericFeatureExtractor &generic_feature_extractor(
+      int idx) const = 0;
+
+ private:
+  // Embedding space names for parameter sharing.
+  std::vector<string> embedding_names_;
+
+  // FML strings for each feature extractor.
+  std::vector<string> embedding_fml_;
+
+  // Size of each of the embedding spaces (maximum predicate id).
+  std::vector<int> embedding_sizes_;
+
+  // Embedding dimensions of the embedding spaces (i.e. 32, 64 etc.)
+  std::vector<int> embedding_dims_;
+
+  // Whether or not to add string descriptions to converted examples.
+  bool add_strings_;
+};
+
+// Templated, object-specific implementation of the
+// EmbeddingFeatureExtractor. EXTRACTOR should be a FeatureExtractor<OBJ,
+// ARGS...> class that has the appropriate FeatureTraits() to ensure that
+// locator type features work.
+//
+// Note: for backwards compatibility purposes, this always reads the FML spec
+// from "<prefix>_features".
+template <class EXTRACTOR, class OBJ, class... ARGS>
+class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
+ public:
+  // Sets up all predicate maps, feature extractors, and flags.
+  void Setup(TaskContext *context) override {
+    GenericEmbeddingFeatureExtractor::Setup(context);
+    feature_extractors_.resize(embedding_fml().size());
+    for (size_t i = 0; i < embedding_fml().size(); ++i) {
+      feature_extractors_[i].Parse(embedding_fml()[i]);
+      feature_extractors_[i].Setup(context);
+    }
+  }
+
+  // Initializes resources needed by the feature extractors.
+  void Init(TaskContext *context) override {
+    GenericEmbeddingFeatureExtractor::Init(context);
+    for (auto &feature_extractor : feature_extractors_) {
+      feature_extractor.Init(context);
+    }
+  }
+
+  // Requests workspaces from the registry. Must be called after Init(), and
+  // before Preprocess().
+  void RequestWorkspaces(WorkspaceRegistry *registry) override {
+    for (auto &feature_extractor : feature_extractors_) {
+      feature_extractor.RequestWorkspaces(registry);
+    }
+  }
+
+  // Must be called on the object one state for each sentence, before any
+  // feature extraction (e.g., UpdateMapsForExample, ExtractSparseFeatures).
+  void Preprocess(WorkspaceSet *workspaces, OBJ *obj) const {
+    for (auto &feature_extractor : feature_extractors_) {
+      feature_extractor.Preprocess(workspaces, obj);
+    }
+  }
+
+  // Extracts features using the extractors. Note that features must already
+  // be initialized to the correct number of feature extractors. No predicate
+  // mapping is applied.
+  void ExtractFeatures(const WorkspaceSet &workspaces, const OBJ &obj,
+                       ARGS... args,
+                       std::vector<FeatureVector> *features) const {
+    for (size_t i = 0; i < feature_extractors_.size(); ++i) {
+      features->at(i).clear();
+      feature_extractors_.at(i).ExtractFeatures(workspaces, obj, args...,
+                                                &features->at(i));
+    }
+  }
+
+ protected:
+  // Provides generic access to the feature extractors.
+  const GenericFeatureExtractor &generic_feature_extractor(
+      int idx) const override {
+    return feature_extractors_.at(idx);
+  }
+
+ private:
+  // Templated feature extractor class.
+  std::vector<EXTRACTOR> feature_extractors_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // EMBEDDING_FEATURE_EXTRACTOR_H_