cld3 3.1.0

data/ext/cld3/embedding_network.cc

@@ -0,0 +1,196 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "embedding_network.h"
+
+#include "base.h"
+#include "embedding_network_params.h"
+#include "float16.h"
+#include "simple_adder.h"
+
+namespace chrome_lang_id {
+namespace {
+
+using VectorWrapper = EmbeddingNetwork::VectorWrapper;
+
+void CheckNoQuantization(const EmbeddingNetworkParams::Matrix matrix) {
+  // Quantization not allowed here.
+  CLD3_DCHECK(static_cast<int>(QuantizationType::NONE) ==
+              static_cast<int>(matrix.quant_type));
+}
+
+// Fills a Matrix object with the parameters in the given MatrixParams.  This
+// function is used to initialize weight matrices that are *not* embedding
+// matrices.
+void FillMatrixParams(const EmbeddingNetworkParams::Matrix source_matrix,
+                      EmbeddingNetwork::Matrix *mat) {
+  mat->resize(source_matrix.rows);
+  CheckNoQuantization(source_matrix);
+  const float *weights =
+      reinterpret_cast<const float *>(source_matrix.elements);
+  for (int r = 0; r < source_matrix.rows; ++r) {
+    (*mat)[r] = EmbeddingNetwork::VectorWrapper(weights, source_matrix.cols);
+    weights += source_matrix.cols;
+  }
+}
+
+// Computes y = weights * Relu(x) + b where Relu is optionally applied.
+template <typename ScaleAdderClass>
+void SparseReluProductPlusBias(bool apply_relu,
+                               const EmbeddingNetwork::Matrix &weights,
+                               const EmbeddingNetwork::VectorWrapper &b,
+                               const EmbeddingNetwork::Vector &x,
+                               EmbeddingNetwork::Vector *y) {
+  y->assign(b.data(), b.data() + b.size());
+  ScaleAdderClass adder(y->data(), y->size());
+
+  const int x_size = x.size();
+  for (int i = 0; i < x_size; ++i) {
+    const float &scale = x[i];
+    if (apply_relu) {
+      if (scale > 0) {
+        adder.LazyScaleAdd(weights[i].data(), scale);
+      }
+    } else {
+      adder.LazyScaleAdd(weights[i].data(), scale);
+    }
+  }
+  adder.Finalize();
+}
+}  // namespace
+
+void EmbeddingNetwork::ConcatEmbeddings(
+    const std::vector<FeatureVector> &feature_vectors, Vector *concat) const {
+  concat->resize(model_->concat_layer_size());
+
+  // "es_index" stands for "embedding space index".
+  for (size_t es_index = 0; es_index < feature_vectors.size(); ++es_index) {
+    const int concat_offset = model_->concat_offset(es_index);
+    const int embedding_dim = model_->embedding_dim(es_index);
+
+    const EmbeddingMatrix &embedding_matrix = embedding_matrices_[es_index];
+    CLD3_DCHECK(embedding_matrix.dim() == embedding_dim);
+
+    const bool is_quantized =
+        embedding_matrix.quant_type() != QuantizationType::NONE;
+
+    const FeatureVector &feature_vector = feature_vectors[es_index];
+    const int num_features = feature_vector.size();
+    for (int fi = 0; fi < num_features; ++fi) {
+      const FeatureType *feature_type = feature_vector.type(fi);
+      int feature_offset = concat_offset + feature_type->base() * embedding_dim;
+      CLD3_DCHECK(feature_offset + embedding_dim <=
+                  static_cast<int>(concat->size()));
+
+      // Weighted embeddings will be added starting from this address.
+      float *concat_ptr = concat->data() + feature_offset;
+
+      // Pointer to float / uint8 weights for relevant embedding.
+      const void *embedding_data;
+
+      // Multiplier for each embedding weight.
+      float multiplier;
+      const FeatureValue feature_value = feature_vector.value(fi);
+      if (feature_type->is_continuous()) {
+        // Continuous features (encoded as FloatFeatureValue).
+        FloatFeatureValue float_feature_value(feature_value);
+        const int id = float_feature_value.value.id;
+        embedding_matrix.get_embedding(id, &embedding_data, &multiplier);
+        multiplier *= float_feature_value.value.weight;
+      } else {
+        // Discrete features: every present feature has implicit value 1.0.
+        embedding_matrix.get_embedding(feature_value, &embedding_data,
+                                       &multiplier);
+      }
+
+      if (is_quantized) {
+        const uint8 *quant_weights =
+            reinterpret_cast<const uint8 *>(embedding_data);
+        for (int i = 0; i < embedding_dim; ++i, ++quant_weights, ++concat_ptr) {
+          // 128 is bias for UINT8 quantization, only one we currently support.
+          *concat_ptr += (static_cast<int>(*quant_weights) - 128) * multiplier;
+        }
+      } else {
+        const float *weights = reinterpret_cast<const float *>(embedding_data);
+        for (int i = 0; i < embedding_dim; ++i, ++weights, ++concat_ptr) {
+          *concat_ptr += *weights * multiplier;
+        }
+      }
+    }
+  }
+}
+
+template <typename ScaleAdderClass>
+void EmbeddingNetwork::FinishComputeFinalScores(const Vector &concat,
+                                                Vector *scores) const {
+  Vector h0(hidden_bias_[0].size());
+  SparseReluProductPlusBias<ScaleAdderClass>(false, hidden_weights_[0],
+                                             hidden_bias_[0], concat, &h0);
+
+  CLD3_DCHECK((hidden_weights_.size() == 1) || (hidden_weights_.size() == 2));
+  if (hidden_weights_.size() == 1) {  // 1 hidden layer
+    SparseReluProductPlusBias<ScaleAdderClass>(true, softmax_weights_,
+                                               softmax_bias_, h0, scores);
+  } else if (hidden_weights_.size() == 2) {  // 2 hidden layers
+    Vector h1(hidden_bias_[1].size());
+    SparseReluProductPlusBias<ScaleAdderClass>(true, hidden_weights_[1],
+                                               hidden_bias_[1], h0, &h1);
+    SparseReluProductPlusBias<ScaleAdderClass>(true, softmax_weights_,
+                                               softmax_bias_, h1, scores);
+  }
+}
+
+void EmbeddingNetwork::ComputeFinalScores(
+    const std::vector<FeatureVector> &features, Vector *scores) const {
+  Vector concat;
+  ConcatEmbeddings(features, &concat);
+
+  scores->resize(softmax_bias_.size());
+  FinishComputeFinalScores<SimpleAdder>(concat, scores);
+}
+
+EmbeddingNetwork::EmbeddingNetwork(const EmbeddingNetworkParams *model)
+    : model_(model) {
+  int offset_sum = 0;
+  for (int i = 0; i < model_->embedding_dim_size(); ++i) {
+    CLD3_DCHECK(offset_sum == model_->concat_offset(i));
+    offset_sum += model_->embedding_dim(i) * model_->embedding_num_features(i);
+    embedding_matrices_.emplace_back(model_->GetEmbeddingMatrix(i));
+  }
+
+  CLD3_DCHECK(model_->hidden_size() == model_->hidden_bias_size());
+  hidden_weights_.resize(model_->hidden_size());
+  hidden_bias_.resize(model_->hidden_size());
+  for (int i = 0; i < model_->hidden_size(); ++i) {
+    FillMatrixParams(model_->GetHiddenLayerMatrix(i), &hidden_weights_[i]);
+    EmbeddingNetworkParams::Matrix bias = model_->GetHiddenLayerBias(i);
+    CLD3_DCHECK(1 == bias.cols);
+    CheckNoQuantization(bias);
+    hidden_bias_[i] = VectorWrapper(
+        reinterpret_cast<const float *>(bias.elements), bias.rows);
+  }
+
+  CLD3_DCHECK(model_->HasSoftmax());
+  FillMatrixParams(model_->GetSoftmaxMatrix(), &softmax_weights_);
+
+  EmbeddingNetworkParams::Matrix softmax_bias = model_->GetSoftmaxBias();
+  CLD3_DCHECK(1 == softmax_bias.cols);
+  CheckNoQuantization(softmax_bias);
+  softmax_bias_ =
+      VectorWrapper(reinterpret_cast<const float *>(softmax_bias.elements),
+                    softmax_bias.rows);
+}
+
+}  // namespace chrome_lang_id

data/ext/cld3/embedding_network.h

@@ -0,0 +1,186 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef EMBEDDING_NETWORK_H_
+#define EMBEDDING_NETWORK_H_
+
+#include <vector>
+
+#include "embedding_network_params.h"
+#include "feature_extractor.h"
+#include "float16.h"
+
+namespace chrome_lang_id {
+
+// Classifier using a hand-coded feed-forward neural network.
+//
+// No gradient computation, just inference.
+//
+// Based on the more general nlp_saft::EmbeddingNetwork.
+//
+// Classification works as follows:
+//
+// Discrete features -> Embeddings -> Concatenation -> Hidden+ -> Softmax
+//
+// In words: given some discrete features, this class extracts the embeddings
+// for these features, concatenates them, passes them through one or two hidden
+// layers (each layer uses Relu) and next through a softmax layer that computes
+// an unnormalized score for each possible class.  Note: there is always a
+// softmax layer.
+//
+// NOTE(salcianu): current code can easily be changed to allow more than two
+// hidden layers.  Feel free to do so if you have a genuine need for that.
+class EmbeddingNetwork {
+ public:
+  // Class used to represent an embedding matrix.  Each row is the embedding on
+  // a vocabulary element.  Number of columns = number of embedding dimensions.
+  class EmbeddingMatrix {
+   public:
+    explicit EmbeddingMatrix(const EmbeddingNetworkParams::Matrix source_matrix)
+        : rows_(source_matrix.rows),
+          cols_(source_matrix.cols),
+          quant_type_(source_matrix.quant_type),
+          data_(source_matrix.elements),
+          row_size_in_bytes_(GetRowSizeInBytes(cols_, quant_type_)),
+          quant_scales_(source_matrix.quant_scales) {}
+
+    // Returns vocabulary size; one embedding for each vocabulary element.
+    int size() const { return rows_; }
+
+    // Returns number of weights in embedding of each vocabulary element.
+    int dim() const { return cols_; }
+
+    // Returns quantization type for this embedding matrix.
+    QuantizationType quant_type() const { return quant_type_; }
+
+    // Gets embedding for k-th vocabulary element: on return, sets *data to
+    // point to the embedding weights and *scale to the quantization scale (1.0
+    // if no quantization).
+    void get_embedding(int k, const void **data, float *scale) const {
+      CLD3_CHECK(k >= 0);
+      CLD3_CHECK(k < size());
+      *data = reinterpret_cast<const char *>(data_) + k * row_size_in_bytes_;
+      if (quant_type_ == QuantizationType::NONE) {
+        *scale = 1.0;
+      } else {
+        *scale = Float16To32(quant_scales_[k]);
+      }
+    }
+
+   private:
+    static int GetRowSizeInBytes(int cols, QuantizationType quant_type) {
+      CLD3_DCHECK((quant_type == QuantizationType::NONE) ||
+                  (quant_type == QuantizationType::UINT8));
+      if (quant_type == QuantizationType::NONE) {
+        return cols * sizeof(float);
+      } else {  // QuantizationType::UINT8
+        return cols * sizeof(uint8);
+      }
+    }
+
+    // Vocabulary size.
+    int rows_;
+
+    // Number of elements in each embedding.
+    int cols_;
+
+    QuantizationType quant_type_;
+
+    // Pointer to the embedding weights, in row-major order.  This is a pointer
+    // to an array of floats / uint8, depending on the quantization type.
+    // Not owned.
+    const void *data_;
+
+    // Number of bytes for one row.  Used to jump to next row in data_.
+    int row_size_in_bytes_;
+
+    // Pointer to quantization scales.  nullptr if no quantization.  Otherwise,
+    // quant_scales_[i] is scale for embedding of i-th vocabulary element.
+    const float16 *quant_scales_;
+  };
+
+  // An immutable vector that doesn't own the memory that stores the underlying
+  // floats.  Can be used e.g., as a wrapper around model weights stored in the
+  // static memory.
+  class VectorWrapper {
+   public:
+    VectorWrapper() : VectorWrapper(nullptr, 0) {}
+
+    // Constructs a vector wrapper around the size consecutive floats that start
+    // at address data.  Note: the underlying data should be alive for at least
+    // the lifetime of this VectorWrapper object.  That's trivially true if data
+    // points to statically allocated data :)
+    VectorWrapper(const float *data, int size) : data_(data), size_(size) {}
+
+    int size() const { return size_; }
+
+    const float *data() const { return data_; }
+
+   private:
+    const float *data_;  // Not owned.
+    int size_;
+
+    // Doesn't own anything, so it can be copied and assigned at will :)
+  };
+
+  typedef std::vector<VectorWrapper> Matrix;
+  typedef std::vector<float> Vector;
+
+  // Constructs an embedding network using the parameters from model.
+  //
+  // Note: model should stay alive for at least the lifetime of this
+  // EmbeddingNetwork object.  TODO(salcianu): remove this constraint: we should
+  // copy all necessary data (except, of course, the static weights) at
+  // construction time and use that, instead of relying on model.
+  explicit EmbeddingNetwork(const EmbeddingNetworkParams *model);
+
+  virtual ~EmbeddingNetwork() {}
+
+  // Runs forward computation to fill scores with unnormalized output unit
+  // scores. This is useful for making predictions.
+  void ComputeFinalScores(const std::vector<FeatureVector> &features,
+                          Vector *scores) const;
+
+ private:
+  // Computes the softmax scores (prior to normalization) from the concatenated
+  // representation.
+  template <typename ScaleAdderClass>
+  void FinishComputeFinalScores(const Vector &concat, Vector *scores) const;
+
+  // Constructs the concatenated input embedding vector in place in output
+  // vector concat.
+  void ConcatEmbeddings(const std::vector<FeatureVector> &features,
+                        Vector *concat) const;
+
+  // Pointer to the model object passed to the constructor.  Not owned.
+  const EmbeddingNetworkParams *model_;
+
+  // Network parameters.
+
+  // One weight matrix for each embedding.
+  std::vector<EmbeddingMatrix> embedding_matrices_;
+
+  // One weight matrix and one vector of bias weights for each hiden layer.
+  std::vector<Matrix> hidden_weights_;
+  std::vector<VectorWrapper> hidden_bias_;
+
+  // Weight matrix and bias vector for the softmax layer.
+  Matrix softmax_weights_;
+  VectorWrapper softmax_bias_;
+};
+
+}  // namespace chrome_lang_id
+
+#endif  // EMBEDDING_NETWORK_H_

data/ext/cld3/embedding_network_params.h

@@ -0,0 +1,285 @@
+/* Copyright 2016 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef EMBEDDING_NETWORK_PARAMS_H_
+#define EMBEDDING_NETWORK_PARAMS_H_
+
+#include <string>
+
+#include "base.h"
+#include "float16.h"
+
+namespace chrome_lang_id {
+
+enum class QuantizationType { NONE = 0, UINT8 };
+
+// API for accessing parameters from a statically-linked EmbeddingNetworkProto.
+class EmbeddingNetworkParams {
+ public:
+  virtual ~EmbeddingNetworkParams() {}
+
+  // **** High-level API.
+
+  // Simple representation of a matrix.  This small struct that doesn't own any
+  // resource intentionally supports copy / assign, to simplify our APIs.
+  struct Matrix {
+    // Number of rows.
+    int rows;
+
+    // Number of columns.
+    int cols;
+
+    QuantizationType quant_type;
+
+    // Pointer to matrix elements, in row-major order
+    // (https://en.wikipedia.org/wiki/Row-major_order) Not owned.
+    const void *elements;
+
+    // Quantization scales: one scale for each row.
+    const float16 *quant_scales;
+  };
+
+  // Returns i-th embedding matrix.  Crashes on out of bounds indices.
+  //
+  // This is the transpose of the corresponding matrix from the original proto.
+  Matrix GetEmbeddingMatrix(int i) const {
+    CheckMatrixRange(i, embeddings_size(), "embedding matrix");
+    Matrix matrix;
+    matrix.rows = embeddings_num_rows(i);
+    matrix.cols = embeddings_num_cols(i);
+    matrix.elements = embeddings_weights(i);
+    matrix.quant_type = embeddings_quant_type(i);
+    matrix.quant_scales = embeddings_quant_scales(i);
+    return matrix;
+  }
+
+  // Returns weight matrix for i-th hidden layer.  Crashes on out of bounds
+  // indices.
+  //
+  // This is the transpose of the corresponding matrix from the original proto.
+  Matrix GetHiddenLayerMatrix(int i) const {
+    CheckMatrixRange(i, hidden_size(), "hidden layer");
+    Matrix matrix;
+    matrix.rows = hidden_num_rows(i);
+    matrix.cols = hidden_num_cols(i);
+
+    // Quantization not supported here.
+    matrix.quant_type = QuantizationType::NONE;
+    matrix.elements = hidden_weights(i);
+    return matrix;
+  }
+
+  // Returns bias for i-th hidden layer.  Technically a Matrix, but we expect it
+  // to be a row/column vector (i.e., num rows or num cols is 1).  However, we
+  // don't CHECK for that: we just provide access to underlying data.  Crashes
+  // on out of bounds indices.
+  Matrix GetHiddenLayerBias(int i) const {
+    CheckMatrixRange(i, hidden_bias_size(), "hidden layer bias");
+    Matrix matrix;
+    matrix.rows = hidden_bias_num_rows(i);
+    matrix.cols = hidden_bias_num_cols(i);
+
+    // Quantization not supported here.
+    matrix.quant_type = QuantizationType::NONE;
+    matrix.elements = hidden_bias_weights(i);
+    return matrix;
+  }
+
+  // Returns true if a softmax layer exists.
+  bool HasSoftmax() const { return softmax_size() == 1; }
+
+  // Returns weight matrix for the softmax layer.  Note: should be called only
+  // if HasSoftmax() is true.
+  //
+  // This is the transpose of the corresponding matrix from the original proto.
+  Matrix GetSoftmaxMatrix() const {
+    CLD3_DCHECK(HasSoftmax());
+    Matrix matrix;
+    matrix.rows = softmax_num_rows(0);
+    matrix.cols = softmax_num_cols(0);
+
+    // Quantization not supported here.
+    matrix.quant_type = QuantizationType::NONE;
+    matrix.elements = softmax_weights(0);
+    return matrix;
+  }
+
+  // Returns bias for the softmax layer.  Technically a Matrix, but we expect it
+  // to be a row/column vector (i.e., num rows or num cols is 1).  However, we
+  // don't CHECK for that: we just provide access to underlying data.
+  Matrix GetSoftmaxBias() const {
+    CLD3_DCHECK(HasSoftmax());
+    Matrix matrix;
+    matrix.rows = softmax_bias_num_rows(0);
+    matrix.cols = softmax_bias_num_cols(0);
+
+    // Quantization not supported here.
+    matrix.quant_type = QuantizationType::NONE;
+    matrix.elements = softmax_bias_weights(0);
+    return matrix;
+  }
+
+  // **** Low-level API.
+  //
+  // * Most low-level API methods are documented by giving an equivalent
+  //   function call on proto, the original proto (of type
+  //   EmbeddingNetworkProto) which was used to generate the C++ code.
+  //
+  // * To simplify our generation code, optional proto fields of message type
+  //   are treated as repeated fields with 0 or 1 instances.  As such, we have
+  //   *_size() methods for such optional fields: they return 0 or 1.
+  //
+  // * "transpose(M)" denotes the transpose of a matrix M.
+
+  // ** Access methods for repeated MatrixParams embeddings.
+  //
+  // Returns proto.embeddings_size().
+  virtual int embeddings_size() const = 0;
+
+  // Returns number of rows of transpose(proto.embeddings(i)).
+  virtual int embeddings_num_rows(int i) const = 0;
+
+  // Returns number of columns of transpose(proto.embeddings(i)).
+  virtual int embeddings_num_cols(int i) const = 0;
+
+  // Returns pointer to elements of transpose(proto.embeddings(i)), in row-major
+  // order.
+  virtual const void *embeddings_weights(int i) const = 0;
+
+  virtual QuantizationType embeddings_quant_type(int i) const {
+    return QuantizationType::NONE;
+  }
+
+  virtual const float16 *embeddings_quant_scales(int i) const {
+    return nullptr;
+  }
+
+  // ** Access methods for repeated MatrixParams hidden.
+  //
+  // Returns embedding_network_proto.hidden_size().
+  virtual int hidden_size() const = 0;
+
+  // Returns embedding_network_proto.hidden(i).rows().
+  virtual int hidden_num_rows(int i) const = 0;
+
+  // Returns embedding_network_proto.hidden(i).rows().
+  virtual int hidden_num_cols(int i) const = 0;
+
+  // Returns pointer to beginning of array of floats with all values from
+  // embedding_network_proto.hidden(i).
+  virtual const void *hidden_weights(int i) const = 0;
+
+  // ** Access methods for repeated MatrixParams hidden_bias.
+  //
+  // Returns proto.hidden_bias_size().
+  virtual int hidden_bias_size() const = 0;
+
+  // Returns number of rows of proto.hidden_bias(i).
+  virtual int hidden_bias_num_rows(int i) const = 0;
+
+  // Returns number of columns of proto.hidden_bias(i).
+  virtual int hidden_bias_num_cols(int i) const = 0;
+
+  // Returns pointer to elements of proto.hidden_bias(i), in row-major order.
+  virtual const void *hidden_bias_weights(int i) const = 0;
+
+  // ** Access methods for optional MatrixParams softmax.
+  //
+  // Returns 1 if proto has optional field softmax, 0 otherwise.
+  virtual int softmax_size() const = 0;
+
+  // Returns number of rows of transpose(proto.softmax()).
+  virtual int softmax_num_rows(int i) const = 0;
+
+  // Returns number of columns of transpose(proto.softmax()).
+  virtual int softmax_num_cols(int i) const = 0;
+
+  // Returns pointer to elements of transpose(proto.softmax()), in row-major
+  // order.
+  virtual const void *softmax_weights(int i) const = 0;
+
+  // ** Access methods for optional MatrixParams softmax_bias.
+  //
+  // Returns 1 if proto has optional field softmax_bias, 0 otherwise.
+  virtual int softmax_bias_size() const = 0;
+
+  // Returns number of rows of proto.softmax_bias().
+  virtual int softmax_bias_num_rows(int i) const = 0;
+
+  // Returns number of columns of proto.softmax_bias().
+  virtual int softmax_bias_num_cols(int i) const = 0;
+
+  // Returns pointer to elements of proto.softmax_bias(), in row-major order.
+  virtual const void *softmax_bias_weights(int i) const = 0;
+
+  // ** Access methods for repeated int32 embedding_dim.
+  //
+  // Returns proto.embedding_dim_size().
+  virtual int embedding_dim_size() const = 0;
+
+  // Returns proto.embedding_dim(i).
+  virtual int embedding_dim(int i) const = 0;
+
+  // ** Access methods for repeated int32 embedding_num_features.
+  //
+  // Returns proto.embedding_num_features_size().
+  virtual int embedding_num_features_size() const = 0;
+
+  // Returns proto.embedding_num_features(i).
+  virtual int embedding_num_features(int i) const = 0;
+
+  // ** Access methods for repeated int32 embedding_features_domain_size.
+  //
+  // Returns proto.embedding_features_domain_size_size().
+  virtual int embedding_features_domain_size_size() const = 0;
+
+  // Returns proto.embedding_features_domain_size(i).
+  virtual int embedding_features_domain_size(int i) const = 0;
+
+  // ** Access methods for repeated int32 concat_offset.
+  //
+  // Returns proto.concat_offset_size().
+  virtual int concat_offset(int i) const = 0;
+
+  // Returns proto.concat_offset(i).
+  virtual int concat_offset_size() const = 0;
+
+  // ** Access methods for concat_layer_size.
+  //
+  // Returns proto.has_concat_layer_size().
+  virtual bool has_concat_layer_size() const = 0;
+
+  // Returns proto.concat_layer_size().
+  virtual int concat_layer_size() const = 0;
+
+  // ** Access methods for is_precomputed
+  //
+  // Returns proto.has_is_precomputed().
+  virtual bool has_is_precomputed() const = 0;
+
+  // Returns proto.is_precomputed().
+  virtual bool is_precomputed() const = 0;
+
+ private:
+  void CheckMatrixRange(int index, int num_matrices,
+                        const string &description) const {
+    CLD3_DCHECK(index >= 0);
+    CLD3_DCHECK(index < num_matrices);
+  }
+};  // class EmbeddingNetworkParams
+
+}  // namespace chrome_lang_id
+
+#endif  // EMBEDDING_NETWORK_PARAMS_H_