cld3 3.1.0

data/ext/cld3/feature_extractor.proto

@@ -0,0 +1,50 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Protocol buffers for feature extractor.
+
+syntax = "proto2";
+option optimize_for = LITE_RUNTIME;
+
+package chrome_lang_id;
+
+message Parameter {
+  optional string name = 1;
+  optional string value = 2;
+}
+
+// Descriptor for feature function.
+message FeatureFunctionDescriptor {
+  // Feature function type.
+  required string type = 1;
+
+  // Feature function name.
+  optional string name = 2;
+
+  // Default argument for feature function.
+  optional int32 argument = 3 [default = 0];
+
+  // Named parameters for feature descriptor.
+  repeated Parameter parameter = 4;
+
+  // Nested sub-feature function descriptors.
+  repeated FeatureFunctionDescriptor feature = 7;
+};
+
+// Descriptor for feature extractor.
+message FeatureExtractorDescriptor {
+  // Top-level feature function for extractor.
+  repeated FeatureFunctionDescriptor feature = 1;
+};

data/ext/cld3/feature_types.cc

@@ -0,0 +1,72 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "feature_types.h"
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "base.h"
+
+namespace chrome_lang_id {
+
+FeatureType::FeatureType(const string &name)
+    : name_(name),
+      base_(0),
+      is_continuous_(name.find("continuous") != string::npos) {}
+
+FeatureType::~FeatureType() {}
+
+template <class Resource>
+ResourceBasedFeatureType<Resource>::ResourceBasedFeatureType(
+    const string &name, const Resource *resource,
+    const std::map<FeatureValue, string> &values)
+    : FeatureType(name), resource_(resource), values_(values) {
+  max_value_ = resource->NumValues() - 1;
+  for (const auto &pair : values) {
+    CLD3_DCHECK(pair.first >= resource->NumValues());
+    max_value_ = pair.first > max_value_ ? pair.first : max_value_;
+  }
+}
+
+template <class Resource>
+ResourceBasedFeatureType<Resource>::ResourceBasedFeatureType(
+    const string &name, const Resource *resource)
+    : ResourceBasedFeatureType(name, resource, {}) {}
+
+EnumFeatureType::EnumFeatureType(
+    const string &name, const std::map<FeatureValue, string> &value_names)
+    : FeatureType(name), value_names_(value_names) {
+  for (const auto &pair : value_names) {
+    CLD3_DCHECK(pair.first >= 0);
+    domain_size_ = std::max(domain_size_, pair.first + 1);
+  }
+}
+
+EnumFeatureType::~EnumFeatureType() {}
+
+string EnumFeatureType::GetFeatureValueName(FeatureValue value) const {
+  auto it = value_names_.find(value);
+  if (it == value_names_.end()) {
+    return "<INVALID>";
+  }
+  return it->second;
+}
+
+FeatureValue EnumFeatureType::GetDomainSize() const { return domain_size_; }
+
+}  // namespace chrome_lang_id

data/ext/cld3/feature_types.h

@@ -0,0 +1,158 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Common feature types for parser components.
+
+#ifndef FEATURE_TYPES_H_
+#define FEATURE_TYPES_H_
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+
+#include "base.h"
+
+namespace chrome_lang_id {
+
+// TODO(djweiss) Clean this up as well.
+// Use the same type for feature values as is used for predicated.
+typedef int64 Predicate;
+typedef Predicate FeatureValue;
+
+// Each feature value in a feature vector has a feature type. The feature type
+// is used for converting feature type and value pairs to predicate values. The
+// feature type can also return names for feature values and calculate the size
+// of the feature value domain. The FeatureType class is abstract and must be
+// specialized for the concrete feature types.
+class FeatureType {
+ public:
+  // Initializes a feature type.
+  explicit FeatureType(const string &name);
+
+  virtual ~FeatureType();
+
+  // Converts a feature value to a name.
+  virtual string GetFeatureValueName(FeatureValue value) const = 0;
+
+  // Returns the size of the feature values domain.
+  virtual int64 GetDomainSize() const = 0;
+
+  // Returns the feature type name.
+  const string &name() const { return name_; }
+
+  Predicate base() const { return base_; }
+  void set_base(Predicate base) { base_ = base; }
+
+  // Returns true iff this feature is continuous; see FloatFeatureValue.
+  bool is_continuous() const { return is_continuous_; }
+
+ private:
+  // Feature type name.
+  string name_;
+
+  // "Base" feature value: i.e. a "slot" in a global ordering of features.
+  Predicate base_;
+
+  // See doc for is_continuous().
+  bool is_continuous_;
+};
+
+// Templated generic resource based feature type. This feature type delegates
+// look up of feature value names to an unknown resource class, which is not
+// owned. Optionally, this type can also store a mapping of extra values which
+// are not in the resource.
+//
+// Note: this class assumes that Resource->GetFeatureValueName() will return
+// successfully for values ONLY in the range [0, Resource->NumValues()) Any
+// feature value not in the extra value map and not in the above range of
+// Resource will result in a ERROR and return of "<INVALID>".
+template <class Resource>
+class ResourceBasedFeatureType : public FeatureType {
+ public:
+  // Creates a new type with given name, resource object, and a mapping of
+  // special values. The values must be greater or equal to
+  // resource->NumValues() so as to avoid collisions; this is verified with
+  // CHECK at creation.
+  ResourceBasedFeatureType(const string &name, const Resource *resource,
+                           const std::map<FeatureValue, string> &values);
+
+  // Creates a new type with no special values.
+  ResourceBasedFeatureType(const string &name, const Resource *resource);
+
+  // Returns the feature name for a given feature value. First checks the values
+  // map, then checks the resource to look up the name.
+  string GetFeatureValueName(FeatureValue value) const override {
+    if (values_.find(value) != values_.end()) {
+      return values_.find(value)->second;
+    }
+    if (value >= 0 && value < resource_->NumValues()) {
+      return resource_->GetFeatureValueName(value);
+    } else {
+      return "<INVALID>";
+    }
+  }
+
+  // Returns the number of possible values for this feature type. This is the
+  // based on the largest value that was observed in the extra values.
+  FeatureValue GetDomainSize() const override { return max_value_ + 1; }
+
+ protected:
+  // Shared resource. Not owned.
+  const Resource *resource_ = nullptr;
+
+  // Maximum possible value this feature could take.
+  FeatureValue max_value_;
+
+  // Mapping for extra feature values not in the resource.
+  std::map<FeatureValue, string> values_;
+};
+
+// Feature type that is defined using an explicit map from FeatureValue to
+// string values.  This can reduce some of the boilerplate when defining
+// features that generate enum values.  Example usage:
+//
+//   class BeverageSizeFeature : public FeatureFunction<Beverage>
+//     enum FeatureValue { SMALL, MEDIUM, LARGE };  // values for this feature
+//     void Init(TaskContext *context) override {
+//       set_feature_type(new EnumFeatureType("beverage_size",
+//           {{SMALL, "SMALL"}, {MEDIUM, "MEDIUM"}, {LARGE, "LARGE"}});
+//     }
+//     [...]
+//   };
+class EnumFeatureType : public FeatureType {
+ public:
+  EnumFeatureType(const string &name,
+                  const std::map<FeatureValue, string> &value_names);
+  ~EnumFeatureType() override;
+
+  // Returns the feature name for a given feature value.
+  string GetFeatureValueName(FeatureValue value) const override;
+
+  // Returns the number of possible values for this feature type. This is one
+  // greater than the largest value in the value_names map.
+  FeatureValue GetDomainSize() const override;
+
+ protected:
+  // Maximum possible value this feature could take.
+  FeatureValue domain_size_ = 0;
+
+  // Names of feature values.
+  std::map<FeatureValue, string> value_names_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // FEATURE_TYPES_H_

data/ext/cld3/fixunicodevalue.cc

@@ -0,0 +1,55 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Routine that maps a Unicode code point to an interchange-valid one
+//
+
+#include "fixunicodevalue.h"
+#include "integral_types.h"
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+// Guarantees that the resulting output value is interchange valid
+//  00-FF; map to spaces or MS CP1252
+//  D800-DFFF; surrogates
+//  FDD0-FDEF; non-characters
+//  xxFFFE-xxFFFF; non-characters
+char32 FixUnicodeValue(char32 uv) {
+  uint32 uuv = static_cast<uint32>(uv);
+  if (uuv < 0x0100) {
+    return kMapFullMicrosoft1252OrSpace[uuv];
+  }
+  if (uuv < 0xD800) {
+    return uv;
+  }
+  if ((uuv & ~0x0F) == 0xFDD0) {              // non-characters
+    return 0xFFFD;
+  }
+  if ((uuv & ~0x0F) == 0xFDE0) {              // non-characters
+    return 0xFFFD;
+  }
+  if ((uuv & 0x00FFFE) == 0xFFFE) {           // non-characters
+    return 0xFFFD;
+  }
+  if ((0xE000 <= uuv) && (uuv <= 0x10FFFF))  {
+    return uv;
+  }
+  // surrogates and negative and > 0x10FFFF all land here
+  return 0xFFFD;
+}
+
+}       // End namespace CLD2
+}       // End namespace chrome_lang_id

data/ext/cld3/fixunicodevalue.h

@@ -0,0 +1,69 @@
+// Copyright 2013 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Routine that maps a Unicode code point to an interchange-valid one
+//
+// Table that maps MS CP1252 bytes 00-FF to their corresponding Unicode
+// code points. C0 and C1 control codes that are not interchange-valid
+// are mapped to spaces.
+
+
+#ifndef SCRIPT_SPAN_FIXUNICODEVALUE_H_
+#define SCRIPT_SPAN_FIXUNICODEVALUE_H_
+
+#include "integral_types.h" // for char32
+#include "port.h"
+
+namespace chrome_lang_id {
+namespace CLD2 {
+
+// Map byte value 0000-00FF to char32
+// Maps C0 control codes (other than CR LF HT FF) to space [29 instances including DEL=0x7F]
+// Maps C1 control codes to CP1252 [27 instances] or space [5 instances]
+static const char32 kMapFullMicrosoft1252OrSpace[256] = {
+  0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x09,0x0a,0x20, 0x0c,0x0d,0x20,0x20, // 00
+  0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20, 0x20,0x20,0x20,0x20,
+  0x20,0x21,0x22,0x23, 0x24,0x25,0x26,0x27, 0x28,0x29,0x2a,0x2b, 0x2c,0x2d,0x2e,0x2f,
+  0x30,0x31,0x32,0x33, 0x34,0x35,0x36,0x37, 0x38,0x39,0x3a,0x3b, 0x3c,0x3d,0x3e,0x3f,
+
+  0x40,0x41,0x42,0x43, 0x44,0x45,0x46,0x47, 0x48,0x49,0x4a,0x4b, 0x4c,0x4d,0x4e,0x4f, // 40
+  0x50,0x51,0x52,0x53, 0x54,0x55,0x56,0x57, 0x58,0x59,0x5a,0x5b, 0x5c,0x5d,0x5e,0x5f,
+  0x60,0x61,0x62,0x63, 0x64,0x65,0x66,0x67, 0x68,0x69,0x6a,0x6b, 0x6c,0x6d,0x6e,0x6f,
+  0x70,0x71,0x72,0x73, 0x74,0x75,0x76,0x77, 0x78,0x79,0x7a,0x7b, 0x7c,0x7d,0x7e,0x20,
+
+  0x20ac,0x20,0x201a,0x0192, 0x201e,0x2026,0x2020,0x2021,                             // 80
+  0x02c6,0x2030,0x0160,0x2039, 0x0152,0x20,0x017d,0x20,
+  0x20,0x2018,0x2019,0x201c, 0x201d,0x2022,0x2013,0x2014,
+  0x02dc,0x2122,0x0161,0x203a, 0x0153,0x20,0x017e,0x0178,
+  0xa0,0xa1,0xa2,0xa3, 0xa4,0xa5,0xa6,0xa7, 0xa8,0xa9,0xaa,0xab, 0xac,0xad,0xae,0xaf, // A0
+  0xb0,0xb1,0xb2,0xb3, 0xb4,0xb5,0xb6,0xb7, 0xb8,0xb9,0xba,0xbb, 0xbc,0xbd,0xbe,0xbf,
+
+  0xc0,0xc1,0xc2,0xc3, 0xc4,0xc5,0xc6,0xc7, 0xc8,0xc9,0xca,0xcb, 0xcc,0xcd,0xce,0xcf, // C0
+  0xd0,0xd1,0xd2,0xd3, 0xd4,0xd5,0xd6,0xd7, 0xd8,0xd9,0xda,0xdb, 0xdc,0xdd,0xde,0xdf,
+  0xe0,0xe1,0xe2,0xe3, 0xe4,0xe5,0xe6,0xe7, 0xe8,0xe9,0xea,0xeb, 0xec,0xed,0xee,0xef,
+  0xf0,0xf1,0xf2,0xf3, 0xf4,0xf5,0xf6,0xf7, 0xf8,0xf9,0xfa,0xfb, 0xfc,0xfd,0xfe,0xff,
+};
+
+// Guarantees that the resulting output value is interchange valid
+//  00-FF; map to spaces or MS CP1252
+//  D800-DFFF; surrogates
+//  FDD0-FDEF; non-characters
+//  xxFFFE-xxFFFF; non-characters
+char32 FixUnicodeValue(char32 uv);
+
+}       // End namespace CLD2
+}       // End namespace chrome_lang_id
+
+#endif  // SCRIPT_SPAN_FIXUNICODEVALUE_H_

data/ext/cld3/float16.h

@@ -0,0 +1,58 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef FLOAT16_H_
+#define FLOAT16_H_
+
+#include <string.h>  // for memcpy
+
+#include "base.h"
+#include "casts.h"
+
+namespace chrome_lang_id {
+
+// Compact 16-bit encoding of floating point numbers. This
+// representation uses 1 bit for the sign, 8 bits for the exponent and
+// 7 bits for the mantissa.  It is assumed that floats are in IEEE 754
+// format so a float16 is just bits 16-31 of a single precision float.
+//
+// NOTE: The IEEE floating point standard defines a float16 format that
+// is different than this format (it has fewer bits of exponent and more
+// bits of mantissa).  We don't use that format here because conversion
+// to/from 32-bit floats is more complex for that format, and the
+// conversion for this format is very simple.
+//
+// <---------float16------------>
+// s e e e e e e e e f f f f f f f f f f f f f f f f f f f f f f f
+// <------------------------------float-------------------------->
+// 3 3             2 2             1 1                           0
+// 1 0             3 2             5 4                           0
+
+typedef uint16 float16;
+
+static inline float16 Float32To16(float f) {
+  // Note that we just truncate the mantissa bits: we make no effort to
+  // do any smarter rounding.
+  return (lang_id_bit_cast<uint32>(f) >> 16) & 0xffff;
+}
+
+static inline float Float16To32(float16 f) {
+  // We fill in the new mantissa bits with 0, and don't do anything smarter.
+  return lang_id_bit_cast<float>(f << 16);
+}
+
+}  // namespace chrome_lang_id
+
+#endif  // FLOAT16_H_