cld3 3.1.0

data/ext/cld3/stringpiece.h

@@ -0,0 +1,81 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// A StringPiece points to part or all of a string, double-quoted string
+// literal, or other string-like object.  A StringPiece does *not* own the
+// string to which it points.  A StringPiece is not null-terminated. [subset]
+//
+
+#ifndef SCRIPT_SPAN_STRINGPIECE_H_
+#define SCRIPT_SPAN_STRINGPIECE_H_
+
+#include <string.h>
+#include <string>
+
+namespace chrome_lang_id {
+
+typedef int stringpiece_ssize_type;
+
+class StringPiece {
+ private:
+  const char* ptr_;
+  stringpiece_ssize_type length_;
+
+ public:
+  // We provide non-explicit singleton constructors so users can pass
+  // in a "const char*" or a "string" wherever a "StringPiece" is
+  // expected.
+  StringPiece() : ptr_(NULL), length_(0) {}
+
+  StringPiece(const char* str)  // NOLINT(runtime/explicit)
+      : ptr_(str), length_(0) {
+    if (str != NULL) {
+      length_ = static_cast<stringpiece_ssize_type>(strlen(str));
+    }
+  }
+
+  StringPiece(const std::string& str)  // NOLINT(runtime/explicit)
+      : ptr_(str.data()), length_(0) {
+    length_ = static_cast<stringpiece_ssize_type>(str.size());
+  }
+
+  StringPiece(const char* offset, stringpiece_ssize_type len)
+      : ptr_(offset), length_(len) {
+  }
+
+  void remove_prefix(stringpiece_ssize_type n) {
+    ptr_ += n;
+    length_ -= n;
+  }
+
+  void remove_suffix(stringpiece_ssize_type n) {
+    length_ -= n;
+  }
+
+  // data() may return a pointer to a buffer with embedded NULs, and the
+  // returned buffer may or may not be null terminated.  Therefore it is
+  // typically a mistake to pass data() to a routine that expects a NUL
+  // terminated string.
+  const char* data() const { return ptr_; }
+  stringpiece_ssize_type size() const { return length_; }
+  stringpiece_ssize_type length() const { return length_; }
+  bool empty() const { return length_ == 0; }
+};
+
+class StringPiece;
+
+}  // namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_STRINGPIECE_H__

data/ext/cld3/task_context.cc

@@ -0,0 +1,161 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "task_context.h"
+
+#include "utils.h"
+
+namespace chrome_lang_id {
+
+TaskContext::TaskContext() {}
+
+TaskContext::~TaskContext() {}
+
+TaskInput *TaskContext::GetInput(const string &name) {
+  // Return existing input if it exists.
+  for (int i = 0; i < spec_.input_size(); ++i) {
+    if (spec_.input(i).name() == name) return spec_.mutable_input(i);
+  }
+
+  // Create new input.
+  TaskInput *input = spec_.add_input();
+  input->set_name(name);
+  return input;
+}
+
+TaskInput *TaskContext::GetInput(const string &name, const string &file_format,
+                                 const string &record_format) {
+  TaskInput *input = GetInput(name);
+  if (!file_format.empty()) {
+    bool found = false;
+    for (int i = 0; i < input->file_format_size(); ++i) {
+      if (input->file_format(i) == file_format) found = true;
+    }
+    if (!found) input->add_file_format(file_format);
+  }
+  if (!record_format.empty()) {
+    bool found = false;
+    for (int i = 0; i < input->record_format_size(); ++i) {
+      if (input->record_format(i) == record_format) found = true;
+    }
+    if (!found) input->add_record_format(record_format);
+  }
+  return input;
+}
+
+void TaskContext::SetParameter(const string &name, const string &value) {
+  // If the parameter already exists update the value.
+  for (int i = 0; i < spec_.parameter_size(); ++i) {
+    if (spec_.parameter(i).name() == name) {
+      spec_.mutable_parameter(i)->set_value(value);
+      return;
+    }
+  }
+
+  // Add new parameter.
+  TaskSpec::Parameter *param = spec_.add_parameter();
+  param->set_name(name);
+  param->set_value(value);
+}
+
+string TaskContext::GetParameter(const string &name) const {
+  // First try to find parameter in task specification.
+  for (int i = 0; i < spec_.parameter_size(); ++i) {
+    if (spec_.parameter(i).name() == name) return spec_.parameter(i).value();
+  }
+
+  // Parameter not found, return empty string.
+  return "";
+}
+
+int TaskContext::GetIntParameter(const string &name) const {
+  string value = GetParameter(name);
+  return utils::ParseUsing<int>(value, 0, utils::ParseInt32);
+}
+
+bool TaskContext::GetBoolParameter(const string &name) const {
+  string value = GetParameter(name);
+  return value == "true";
+}
+
+double TaskContext::GetFloatParameter(const string &name) const {
+  string value = GetParameter(name);
+  return utils::ParseUsing<double>(value, .0, utils::ParseDouble);
+}
+
+string TaskContext::Get(const string &name, const char *defval) const {
+  // First try to find parameter in task specification.
+  for (int i = 0; i < spec_.parameter_size(); ++i) {
+    if (spec_.parameter(i).name() == name) return spec_.parameter(i).value();
+  }
+
+  // Parameter not found, return default value.
+  return defval;
+}
+
+string TaskContext::Get(const string &name, const string &defval) const {
+  return Get(name, defval.c_str());
+}
+
+int TaskContext::Get(const string &name, int defval) const {
+  string value = Get(name, "");
+  return utils::ParseUsing<int>(value, defval, utils::ParseInt32);
+}
+
+double TaskContext::Get(const string &name, double defval) const {
+  string value = Get(name, "");
+  return utils::ParseUsing<double>(value, defval, utils::ParseDouble);
+}
+
+bool TaskContext::Get(const string &name, bool defval) const {
+  string value = Get(name, "");
+  return value.empty() ? defval : value == "true";
+}
+
+string TaskContext::InputFile(const TaskInput &input) {
+  CLD3_CHECK(input.part_size() == 1);
+  return input.part(0).file_pattern();
+}
+
+bool TaskContext::Supports(const TaskInput &input, const string &file_format,
+                           const string &record_format) {
+  // Check file format.
+  if (input.file_format_size() > 0) {
+    bool found = false;
+    for (int i = 0; i < input.file_format_size(); ++i) {
+      if (input.file_format(i) == file_format) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) return false;
+  }
+
+  // Check record format.
+  if (input.record_format_size() > 0) {
+    bool found = false;
+    for (int i = 0; i < input.record_format_size(); ++i) {
+      if (input.record_format(i) == record_format) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) return false;
+  }
+
+  return true;
+}
+
+}  // namespace chrome_lang_id

data/ext/cld3/task_context.h

@@ -0,0 +1,81 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TASK_CONTEXT_H_
+#define TASK_CONTEXT_H_
+
+#include <string>
+#include <vector>
+
+#include "base.h"
+#include "cld_3/protos/task_spec.pb.h"
+
+namespace chrome_lang_id {
+
+// A task context holds configuration information for a task. It is basically a
+// wrapper around a TaskSpec protocol buffer.
+class TaskContext {
+ public:
+  TaskContext();
+  ~TaskContext();
+
+  // Returns the underlying task specification protocol buffer for the context.
+  const TaskSpec &spec() const { return spec_; }
+  TaskSpec *mutable_spec() { return &spec_; }
+
+  // Returns a named input descriptor for the task. A new input  is created if
+  // the task context does not already have an input with that name.
+  TaskInput *GetInput(const string &name);
+  TaskInput *GetInput(const string &name, const string &file_format,
+                      const string &record_format);
+
+  // Sets task parameter.
+  void SetParameter(const string &name, const string &value);
+
+  // Returns task parameter. If the parameter is not in the task configuration
+  // the (default) value of the corresponding command line flag is returned.
+  string GetParameter(const string &name) const;
+  int GetIntParameter(const string &name) const;
+  bool GetBoolParameter(const string &name) const;
+  double GetFloatParameter(const string &name) const;
+
+  // Returns task parameter. If the parameter is not in the task configuration
+  // the default value is returned. Parameters retrieved using these methods
+  // don't need to be defined with a DEFINE_*() macro.
+  string Get(const string &name, const string &defval) const;
+  string Get(const string &name, const char *defval) const;
+  int Get(const string &name, int defval) const;
+  double Get(const string &name, double defval) const;
+  bool Get(const string &name, bool defval) const;
+
+  // Returns input file name for a single-file task input.
+  static string InputFile(const TaskInput &input);
+
+  // Returns true if task input supports the file and record format.
+  static bool Supports(const TaskInput &input, const string &file_format,
+                       const string &record_format);
+
+ private:
+  // Underlying task specification protocol buffer.
+  TaskSpec spec_;
+
+  // Vector of parameters required by this task.  These must be specified in the
+  // task rather than relying on default values.
+  std::vector<string> required_parameters_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // TASK_CONTEXT_H_

data/ext/cld3/task_context_params.cc

@@ -0,0 +1,74 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// This file contains the hard-coded parameters from the training workflow. If
+// you update the binary model, you may need to update the variables below as
+// well.
+
+#include "task_context_params.h"
+
+#include "task_context.h"
+
+namespace chrome_lang_id {
+
+void TaskContextParams::ToTaskContext(TaskContext *context) {
+  context->SetParameter("language_identifier_features",
+                        kLanguageIdentifierFeatures);
+  context->SetParameter("language_identifier_embedding_names",
+                        kLanguageIdentifierEmbeddingNames);
+  context->SetParameter("language_identifier_embedding_dims",
+                        kLanguageIdentifierEmbeddingDims);
+}
+
+int TaskContextParams::GetNumLanguages() {
+  int i = 0;
+  while (kLanguageNames[i] != nullptr) {
+    i++;
+  }
+  return i;
+}
+
+const char *const TaskContextParams::kLanguageNames[] = {
+    "eo", "co", "eu", "ta", "de", "mt", "ps", "te", "su", "uz", "zh-Latn", "ne",
+    "nl", "sw", "sq", "hmn", "ja", "no", "mn", "so", "ko", "kk", "sl", "ig",
+    "mr", "th", "zu", "ml", "hr", "bs", "lo", "sd", "cy", "hy", "uk", "pt",
+    "lv", "iw", "cs", "vi", "jv", "be", "km", "mk", "tr", "fy", "am", "zh",
+    "da", "sv", "fi", "ht", "af", "la", "id", "fil", "sm", "ca", "el", "ka",
+    "sr", "it", "sk", "ru", "ru-Latn", "bg", "ny", "fa", "haw", "gl", "et",
+    "ms", "gd", "bg-Latn", "ha", "is", "ur", "mi", "hi", "bn", "hi-Latn", "fr",
+    "yi", "hu", "xh", "my", "tg", "ro", "ar", "lb", "el-Latn", "st", "ceb",
+    "kn", "az", "si", "ky", "mg", "en", "gu", "es", "pl", "ja-Latn", "ga", "lt",
+    "sn", "yo", "pa", "ku",
+
+    // last element must be nullptr
+    nullptr,
+};
+
+const char TaskContextParams::kLanguageIdentifierFeatures[] =
+    "continuous-bag-of-ngrams(include_terminators=true,include_spaces=false,"
+    "use_equal_weight=false,id_dim=1000,size=2);continuous-bag-of-ngrams("
+    "include_terminators=true,include_spaces=false,use_equal_weight=false,id_"
+    "dim=5000,size=4);continuous-bag-of-relevant-scripts;script;continuous-bag-"
+    "of-ngrams(include_terminators=true,include_spaces=false,use_equal_weight="
+    "false,id_dim=5000,size=3);continuous-bag-of-ngrams(include_terminators="
+    "true,include_spaces=false,use_equal_weight=false,id_dim=100,size=1)";
+
+const char TaskContextParams::kLanguageIdentifierEmbeddingNames[] =
+    "bigrams;quadgrams;relevant-scripts;text-script;trigrams;unigrams";
+
+const char TaskContextParams::kLanguageIdentifierEmbeddingDims[] =
+    "16;16;8;8;16;16";
+
+}  // namespace chrome_lang_id

data/ext/cld3/task_context_params.h

@@ -0,0 +1,54 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TASK_CONTEXT_PARAMS_H_
+#define TASK_CONTEXT_PARAMS_H_
+
+#include <string>
+
+#include "base.h"
+#include "task_context.h"
+
+namespace chrome_lang_id {
+
+// Encapsulates the TaskContext specifying only the parameters for the model.
+// The model weights are loaded statically.
+class TaskContextParams {
+ public:
+  // Gets the name of the i'th language.
+  static const char *language_names(int i) { return kLanguageNames[i]; }
+
+  // Saves the parameters to the given TaskContext.
+  static void ToTaskContext(TaskContext *context);
+
+  // Gets the number of languages.
+  static int GetNumLanguages();
+
+ private:
+  // Names of all the languages.
+  static const char *const kLanguageNames[];
+
+  // Features in FML format.
+  static const char kLanguageIdentifierFeatures[];
+
+  // Names of the embedding spaces.
+  static const char kLanguageIdentifierEmbeddingNames[];
+
+  // Dimensions of the embedding spaces.
+  static const char kLanguageIdentifierEmbeddingDims[];
+};
+}  // namespace chrome_lang_id
+
+#endif  // TASK_CONTEXT_PARAMS_H_