transformers-rb 0.1.0

data/lib/transformers/models/auto/auto_factory.rb

@@ -0,0 +1,138 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  class BaseAutoModelClass
+    extend ClassAttribute
+
+    class_attribute :_model_mapping
+
+    class << self
+      def from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)
+        config = kwargs.delete(:config)
+        trust_remote_code = kwargs.delete(:trust_remote_code)
+        hub_kwargs_names = [
+          :cache_dir,
+          :force_download,
+          :local_files_only,
+          :proxies,
+          :resume_download,
+          :revision,
+          :subfolder,
+          :use_auth_token,
+          :token
+        ]
+        hub_kwargs = hub_kwargs_names.select { |k| kwargs.key?(k) }.to_h { |name| [name, kwargs.delete(name)] }
+        code_revision = kwargs.delete(:code_revision)
+        commit_hash = kwargs.delete(:_commit_hash)
+
+        if commit_hash.nil?
+          if !config.is_a?(PretrainedConfig)
+            # We make a call to the config file first (which may be absent) to get the commit hash as soon as possible
+            resolved_config_file = Utils::Hub.cached_file(
+              pretrained_model_name_or_path,
+              CONFIG_NAME,
+              _raise_exceptions_for_gated_repo: false,
+              _raise_exceptions_for_missing_entries: false,
+              _raise_exceptions_for_connection_errors: false,
+              **hub_kwargs
+            )
+            commit_hash = Utils::Hub.extract_commit_hash(resolved_config_file, commit_hash)
+          else
+            raise Todo
+          end
+        end
+
+        if !config.is_a?(PretrainedConfig)
+          config, kwargs =
+            AutoConfig.from_pretrained(
+              pretrained_model_name_or_path,
+              return_unused_kwargs: true,
+              trust_remote_code: trust_remote_code,
+              code_revision: code_revision,
+              _commit_hash: commit_hash,
+              **hub_kwargs,
+              **kwargs
+            )
+        end
+
+        model_class = _get_model_class(config, _model_mapping)
+        model_class.from_pretrained(
+          pretrained_model_name_or_path, *model_args, config: config, **hub_kwargs, **kwargs
+        )
+      end
+
+      private
+
+      def _get_model_class(config, model_mapping)
+        supported_models = model_mapping[config.class.name.split("::").last]
+        if !supported_models.is_a?(Array)
+          return supported_models
+        end
+
+        raise Todo
+      end
+    end
+  end
+
+  class LazyAutoMapping
+    def initialize(config_mapping, model_mapping)
+      @config_mapping = config_mapping
+      @reverse_config_mapping = config_mapping.invert
+      @model_mapping = model_mapping
+      @modules = {}
+    end
+
+    def [](key)
+      model_type = @reverse_config_mapping[key]
+      if @model_mapping[model_type]
+        model_name = @model_mapping[model_type]
+        return _load_attr_from_module(model_type, model_name)
+      end
+
+      raise KeyError, key
+    end
+
+    def include?(key)
+      self[key]
+      true
+    rescue KeyError
+      false
+    end
+
+    private
+
+    def _load_attr_from_module(model_type, attr)
+      module_name = model_type_to_module_name(model_type)
+      if !@modules.include?(module_name)
+        @modules[module_name] = Transformers.const_get(module_name.capitalize)
+      end
+      getattribute_from_module(@modules[module_name], attr)
+    end
+
+    def getattribute_from_module(mod, attr)
+      if attr.nil?
+        nil
+      elsif attr.is_a?(Array)
+        attr.map { |a| mod.const_get(a) }
+      else
+        mod.const_get(attr)
+      end
+    end
+
+    def model_type_to_module_name(key)
+      key
+    end
+  end
+end

data/lib/transformers/models/auto/configuration_auto.rb

@@ -0,0 +1,61 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  CONFIG_MAPPING_NAMES = {
+    "bert" => "BertConfig",
+    "distilbert" => "DistilBertConfig",
+    "vit" => "ViTConfig"
+  }
+
+  class LazyConfigMapping
+    def initialize(mapping)
+      @mapping = mapping
+      @extra_content = {}
+      @modules = {}
+    end
+
+    def [](key)
+      value = @mapping.fetch(key)
+      module_name = model_type_to_module_name(key)
+      if !@modules.include?(module_name)
+        @modules[module_name] = Transformers.const_get(module_name.capitalize)
+      end
+      @modules[module_name].const_get(value)
+    end
+
+    def model_type_to_module_name(key)
+      key
+    end
+  end
+
+  CONFIG_MAPPING = LazyConfigMapping.new(CONFIG_MAPPING_NAMES)
+
+  class AutoConfig
+    def self.from_pretrained(pretrained_model_name_or_path, **kwargs)
+      kwargs[:_from_auto] = true
+      kwargs[:name_or_path] = pretrained_model_name_or_path
+      _trust_remote_code = kwargs.delete(:trust_remote_code)
+      _code_revision = kwargs.delete(:code_revision)
+
+      config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
+      if config_dict[:model_type]
+        config_class = CONFIG_MAPPING[config_dict[:model_type]]
+        return config_class.from_dict(config_dict, **unused_kwargs)
+      else
+        raise Todo
+      end
+    end
+  end
+end

data/lib/transformers/models/auto/feature_extraction_auto.rb

@@ -0,0 +1,20 @@
+# Copyright 2021 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  FEATURE_EXTRACTOR_MAPPING_NAMES = {
+  }
+
+  FEATURE_EXTRACTOR_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, FEATURE_EXTRACTOR_MAPPING_NAMES)
+end

data/lib/transformers/models/auto/image_processing_auto.rb

@@ -0,0 +1,104 @@
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  IMAGE_PROCESSOR_MAPPING_NAMES = {
+    "vit" => ["ViTImageProcessor"]
+  }
+
+  IMAGE_PROCESSOR_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, IMAGE_PROCESSOR_MAPPING_NAMES)
+
+  class AutoImageProcessor
+    def self.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+      config = kwargs.delete(:config)
+      use_fast = kwargs.delete(:use_fast)
+      trust_remote_code = kwargs.delete(:trust_remote_code)
+      kwargs[:_from_auto] = true
+
+      config_dict, _ = ImageProcessingMixin.get_image_processor_dict(pretrained_model_name_or_path, **kwargs)
+      image_processor_class = config_dict[:image_processor_type]
+      image_processor_auto_map = nil
+      if (config_dict[:auto_map] || {}).include?("AutoImageProcessor")
+        image_processor_auto_map = config_dict[:auto_map]["AutoImageProcessor"]
+      end
+
+      # If we still don't have the image processor class, check if we're loading from a previous feature extractor config
+      # and if so, infer the image processor class from there.
+      if image_processor_class.nil? && image_processor_auto_map.nil?
+        feature_extractor_class = config_dict.delete(:feature_extractor_type)
+        if !feature_extractor_class.nil?
+          image_processor_class = feature_extractor_class.sub("FeatureExtractor", "ImageProcessor")
+        end
+        if (config_dict[:auto_map] || {}).include?("AutoFeatureExtractor")
+          feature_extractor_auto_map = config_dict[:auto_map]["AutoFeatureExtractor"]
+          image_processor_auto_map = feature_extractor_auto_map.sub("FeatureExtractor", "ImageProcessor")
+        end
+      end
+
+      # If we don't find the image processor class in the image processor config, let's try the model config.
+      if image_processor_class.nil? && image_processor_auto_map.nil?
+        if !config.is_a?(PretrainedConfig)
+          config = AutoConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+        end
+        # It could be in `config.image_processor_type``
+        image_processor_class = config.instance_variable_get(:@image_processor_type)
+      end
+
+      if !image_processor_class.nil?
+        raise Todo
+      end
+
+      has_remote_code = !image_processor_auto_map.nil?
+      has_local_code = !image_processor_class.nil? || IMAGE_PROCESSOR_MAPPING.include?(config.class.name.split("::").last)
+      trust_remote_code = DynamicModuleUtils.resolve_trust_remote_code(
+        trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code
+      )
+
+      if !image_processor_auto_map.nil? && !image_processor_auto_map.is_a?(Array)
+        raise Todo
+      end
+
+      if has_remote_code && trust_remote_code
+        raise Todo
+      elsif !image_processor_class.nil?
+        return image_processor_class.from_dict(config_dict, **kwargs)
+      # Last try: we use the IMAGE_PROCESSOR_MAPPING.
+      elsif IMAGE_PROCESSOR_MAPPING.include?(config.class.name.split("::").last)
+        image_processor_tuple = IMAGE_PROCESSOR_MAPPING[config.class.name.split("::").last]
+
+        image_processor_class_py, image_processor_class_fast = image_processor_tuple
+
+        if !use_fast && !image_processor_class_fast.nil?
+          _warning_fast_image_processor_available(image_processor_class_fast)
+        end
+
+        if image_processor_class_fast && (use_fast || image_processor_class_py.nil?)
+          return image_processor_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+        else
+          if !image_processor_class_py.nil?
+            return image_processor_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+          else
+            raise ArgumentError,
+              "This image processor cannot be instantiated. Please make sure you have `Pillow` installed."
+          end
+        end
+      end
+
+      raise ArgumentError,
+        "Unrecognized image processor in #{pretrained_model_name_or_path}. Should have a " +
+        "`image_processor_type` key in its #{IMAGE_PROCESSOR_NAME} of #{CONFIG_NAME}, or one of the following " +
+        "`model_type` keys in its #{CONFIG_NAME}: #{IMAGE_PROCESSOR_MAPPING_NAMES.keys.join(", ")}"
+    end
+  end
+end

data/lib/transformers/models/auto/modeling_auto.rb

@@ -0,0 +1,80 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  MODEL_MAPPING_NAMES = {
+    "bert" => "BertModel",
+    "distilbert" => "DistilBertModel",
+    "vit" => "ViTModel"
+  }
+
+  MODEL_FOR_MASKED_LM_MAPPING_NAMES = {
+    "bert" => "BertForMaskedLM"
+  }
+
+  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES = {
+    "distilbert" => "DistilBertForSequenceClassification"
+  }
+
+  MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES = {
+    "distilbert" => "DistilBertForQuestionAnswering"
+  }
+
+  MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES = {
+    "vit" => "ViTForImageClassification"
+  }
+
+  MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES = {
+    "bert" => "BertForTokenClassification"
+  }
+
+  MODEL_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, MODEL_MAPPING_NAMES)
+  MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING = LazyAutoMapping.new(
+    CONFIG_MAPPING_NAMES, MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES
+  )
+  MODEL_FOR_MASKED_LM_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, MODEL_FOR_MASKED_LM_MAPPING_NAMES)
+  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING = LazyAutoMapping.new(
+    CONFIG_MAPPING_NAMES, MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES
+  )
+  MODEL_FOR_QUESTION_ANSWERING_MAPPING = LazyAutoMapping.new(
+    CONFIG_MAPPING_NAMES, MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES
+  )
+  MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING = LazyAutoMapping.new(
+    CONFIG_MAPPING_NAMES, MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES
+  )
+
+  class AutoModel < BaseAutoModelClass
+    self._model_mapping = MODEL_MAPPING
+  end
+
+  class AutoModelForMaskedLM < BaseAutoModelClass
+    self._model_mapping = MODEL_FOR_MASKED_LM_MAPPING
+  end
+
+  class AutoModelForSequenceClassification < BaseAutoModelClass
+    self._model_mapping = MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
+  end
+
+  class AutoModelForQuestionAnswering < BaseAutoModelClass
+    self._model_mapping = MODEL_FOR_QUESTION_ANSWERING_MAPPING
+  end
+
+  class AutoModelForTokenClassification < BaseAutoModelClass
+    self._model_mapping = MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING
+  end
+
+  class AutoModelForImageClassification < BaseAutoModelClass
+    self._model_mapping = MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
+  end
+end

data/lib/transformers/models/auto/tokenization_auto.rb

@@ -0,0 +1,160 @@
+# Copyright 2018 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  TOKENIZER_MAPPING_NAMES = {
+    "bert" => ["BertTokenizer", "BertTokenizerFast"],
+    "distilbert" => ["DistilBertTokenizer", "DistilBertTokenizerFast"]
+  }
+
+  TOKENIZER_MAPPING = LazyAutoMapping.new(CONFIG_MAPPING_NAMES, TOKENIZER_MAPPING_NAMES)
+
+  class AutoTokenizer
+    class << self
+      def from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+        config = kwargs.delete(:config)
+        kwargs[:_from_auto] = true
+
+        use_fast = kwargs.delete(:use_fast) { true }
+        tokenizer_type = kwargs.delete(:tokenizer_type) { nil }
+        trust_remote_code = kwargs.delete(:trust_remote_code)
+
+        if !tokenizer_type.nil?
+          raise Todo
+        end
+
+        tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
+        if tokenizer_config.include?("_commit_hash")
+          kwargs[:_commit_hash] = tokenizer_config["_commit_hash"]
+        end
+        config_tokenizer_class = tokenizer_config["tokenizer_class"]
+        _tokenizer_auto_map = nil
+        if tokenizer_config["auto_map"]
+          raise Todo
+        end
+
+        # If that did not work, let's try to use the config.
+        if config_tokenizer_class.nil?
+          if !config.is_a?(PretrainedConfig)
+            config = AutoConfig.from_pretrained(
+              pretrained_model_name_or_path, trust_remote_code: trust_remote_code, **kwargs
+            )
+            config_tokenizer_class = config.tokenizer_class
+            # if hasattr(config, "auto_map") and "AutoTokenizer" in config.auto_map:
+            #     tokenizer_auto_map = config.auto_map["AutoTokenizer"]
+          end
+        end
+
+        if !config_tokenizer_class.nil?
+          tokenizer_class = nil
+          if use_fast && !config_tokenizer_class.end_with?("Fast")
+            tokenizer_class_candidate = "#{config_tokenizer_class}Fast"
+            tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
+          end
+          if tokenizer_class.nil?
+            tokenizer_class_candidate = config_tokenizer_class
+            tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
+          end
+          if tokenizer_class.nil?
+            raise ArgumentError, "Tokenizer class #{tokenizer_class_candidate} does not exist or is not currently imported."
+          end
+          return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+        end
+
+        model_type = config_class_to_model_type(config.class.name.split("::").last)
+        if !model_type.nil?
+          tokenizer_class_py, tokenizer_class_fast = TOKENIZER_MAPPING[config.class.name.split("::").last]
+          if tokenizer_class_fast && (use_fast || tokenizer_class_py.nil?)
+            return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+          else
+            if !tokenizer_class_py.nil?
+              return tokenizer_class_py.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+            else
+              raise ArgumentError, "This tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer."
+            end
+          end
+        end
+
+        raise ArgumentError, "Unrecognized configuration class #{config.class.name} to build an AutoTokenizer."
+      end
+
+      private
+
+      def tokenizer_class_from_name(class_name)
+        if class_name == "PreTrainedTokenizerFast"
+          return PreTrainedTokenizerFast
+        end
+
+        TOKENIZER_MAPPING_NAMES.each do |module_name, tokenizers|
+          if tokenizers.include?(class_name)
+            cls = Transformers.const_get(module_name.capitalize).const_get(class_name)
+            raise Error, "Invalid tokenizer class: #{class_name}" unless cls < PreTrainedTokenizer || cls < PreTrainedTokenizerFast
+            return cls
+          end
+        end
+
+        raise Todo
+      end
+
+      def get_tokenizer_config(
+        pretrained_model_name_or_path,
+        cache_dir: nil,
+        force_download: false,
+        resume_download: false,
+        proxies: nil,
+        token: nil,
+        revision: nil,
+        local_files_only: false,
+        subfolder: "",
+        **kwargs
+      )
+        commit_hash = kwargs[:_commit_hash]
+        resolved_config_file = Utils::Hub.cached_file(
+          pretrained_model_name_or_path,
+          TOKENIZER_CONFIG_FILE,
+          cache_dir: cache_dir,
+          force_download: force_download,
+          resume_download: resume_download,
+          proxies: proxies,
+          token: token,
+          revision: revision,
+          local_files_only: local_files_only,
+          subfolder: subfolder,
+          _raise_exceptions_for_gated_repo: false,
+          _raise_exceptions_for_missing_entries: false,
+          _raise_exceptions_for_connection_errors: false,
+          _commit_hash: commit_hash
+        )
+        if resolved_config_file.nil?
+          Transformers.logger.info("Could not locate the tokenizer configuration file, will try to use the model config instead.")
+          return {}
+        end
+        commit_hash = Utils::Hub.extract_commit_hash(resolved_config_file, commit_hash)
+
+        result = JSON.load_file(resolved_config_file)
+        result["_commit_hash"] = commit_hash
+        result
+      end
+
+      def config_class_to_model_type(config)
+        CONFIG_MAPPING_NAMES.each do |key, cls|
+          if cls == config
+            return key
+          end
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/transformers/models/bert/configuration_bert.rb

@@ -0,0 +1,65 @@
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Bert
+    class BertConfig < PretrainedConfig
+      self.model_type = "bert"
+
+      attr_reader :vocab_size, :hidden_size, :num_hidden_layers, :num_attention_heads,
+        :intermediate_size, :hidden_act, :hidden_dropout_prob, :attention_probs_dropout_prob,
+        :max_position_embeddings, :type_vocab_size, :initializer_range, :layer_norm_eps,
+        :position_embedding_type, :use_cache, :classifier_dropout
+
+      def initialize(
+        vocab_size: 30522,
+        hidden_size: 768,
+        num_hidden_layers: 12,
+        num_attention_heads: 12,
+        intermediate_size: 3072,
+        hidden_act: "gelu",
+        hidden_dropout_prob: 0.1,
+        attention_probs_dropout_prob: 0.1,
+        max_position_embeddings: 512,
+        type_vocab_size: 2,
+        initializer_range: 0.02,
+        layer_norm_eps: 1e-12,
+        pad_token_id: 0,
+        position_embedding_type: "absolute",
+        use_cache: true,
+        classifier_dropout: nil,
+        **kwargs
+      )
+        super(pad_token_id: pad_token_id, **kwargs)
+
+        @vocab_size = vocab_size
+        @hidden_size = hidden_size
+        @num_hidden_layers = num_hidden_layers
+        @num_attention_heads = num_attention_heads
+        @hidden_act = hidden_act
+        @intermediate_size = intermediate_size
+        @hidden_dropout_prob = hidden_dropout_prob
+        @attention_probs_dropout_prob = attention_probs_dropout_prob
+        @max_position_embeddings = max_position_embeddings
+        @type_vocab_size = type_vocab_size
+        @initializer_range = initializer_range
+        @layer_norm_eps = layer_norm_eps
+        @position_embedding_type = position_embedding_type
+        @use_cache = use_cache
+        @classifier_dropout = classifier_dropout
+      end
+    end
+  end
+end