transformers-rb 0.1.0

data/lib/transformers/utils/hub.rb

@@ -0,0 +1,209 @@
+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Utils
+    module Hub
+      IS_OFFLINE_MODE = HfHub::HF_HUB_OFFLINE
+
+      PYTORCH_PRETRAINED_BERT_CACHE = ENV.fetch("PYTORCH_PRETRAINED_BERT_CACHE", HfHub::HF_HUB_CACHE)
+      PYTORCH_TRANSFORMERS_CACHE = ENV.fetch("PYTORCH_TRANSFORMERS_CACHE", PYTORCH_PRETRAINED_BERT_CACHE)
+      TRANSFORMERS_CACHE = ENV.fetch("TRANSFORMERS_CACHE", PYTORCH_TRANSFORMERS_CACHE)
+
+      DEFAULT_ENDPOINT = "https://huggingface.co"
+      HUGGINGFACE_CO_RESOLVE_ENDPOINT = ENV.fetch("HF_ENDPOINT", DEFAULT_ENDPOINT)
+
+      class << self
+        def is_offline_mode
+          IS_OFFLINE_MODE
+        end
+
+        def is_remote_url(url_or_filename)
+          url_or_filename.is_a?(URI)
+        end
+
+        def http_user_agent(user_agent = nil)
+          ua = "transformers.rb/#{Transformers::VERSION}; ruby/#{RUBY_VERSION.to_f}"
+          if user_agent.is_a?(Hash)
+            ua += "; " + user_agent.map { |k, v| "#{k}/#{v}" }.join("; ")
+          elsif user_agent.is_a?(String)
+            ua += "; " + user_agent
+          end
+          ua
+        end
+
+        def extract_commit_hash(resolved_file, commit_hash)
+          if resolved_file.nil? || !commit_hash.nil?
+            return commit_hash
+          end
+          search = /snapshots\/([^\/]+)/.match(resolved_file)
+          if search.nil?
+            return nil
+          end
+          commit_hash = search[1]
+          HfHub::REGEX_COMMIT_HASH.match(commit_hash) ? commit_hash : nil
+        end
+
+        def cached_file(
+          path_or_repo_id,
+          filename,
+          cache_dir: nil,
+          force_download: false,
+          resume_download: false,
+          proxies: nil,
+          token: nil,
+          revision: nil,
+          local_files_only: false,
+          subfolder: "",
+          repo_type: nil,
+          user_agent: nil,
+          _raise_exceptions_for_gated_repo: true,
+          _raise_exceptions_for_missing_entries: true,
+          _raise_exceptions_for_connection_errors: true,
+          _commit_hash: nil,
+          **deprecated_kwargs
+        )
+          if is_offline_mode && !local_files_only
+            Transformers.logger.info "Offline mode: forcing local_files_only: true"
+            local_files_only = true
+          end
+          if subfolder.nil?
+            subfolder = ""
+          end
+
+          path_or_repo_id = path_or_repo_id.to_s
+          full_filename = File.join(subfolder, filename)
+          if Dir.exist?(path_or_repo_id)
+            raise Todo
+          end
+
+          if cache_dir.nil?
+            cache_dir = TRANSFORMERS_CACHE
+          end
+          if cache_dir.is_a?(Pathname)
+            cache_dir = cache_dir.to_s
+          end
+
+          if !_commit_hash.nil? && !force_download
+            # If the file is cached under that commit hash, we return it directly.
+            resolved_file =
+              HfHub.try_to_load_from_cache(
+                path_or_repo_id, full_filename, cache_dir: cache_dir, revision: _commit_hash, repo_type: repo_type
+              )
+            if !resolved_file.nil?
+              if resolved_file != HfHub::CACHED_NO_EXIST
+                return resolved_file
+              elsif !_raise_exceptions_for_missing_entries
+                return nil
+              else
+                raise EnvironmentError, "Could not locate #{full_filename} inside #{path_or_repo_id}."
+              end
+            end
+          end
+
+          user_agent = http_user_agent(user_agent)
+
+          resolved_file = nil
+          begin
+            resolved_file =
+              HfHub.hf_hub_download(
+                path_or_repo_id,
+                filename,
+                subfolder: subfolder.length == 0 ? nil : subfolder,
+                repo_type: repo_type,
+                revision: revision,
+                cache_dir: cache_dir,
+                user_agent: user_agent,
+                force_download: force_download,
+                proxies: proxies,
+                resume_download: resume_download,
+                token: token,
+                local_files_only: local_files_only
+              )
+          rescue => e
+            raise e if _raise_exceptions_for_missing_entries
+          end
+          resolved_file
+        end
+
+        def has_file(
+          path_or_repo,
+          filename,
+          revision: nil,
+          proxies: nil,
+          token: nil,
+          local_files_only: false,
+          cache_dir: nil,
+          repo_type: nil,
+          **deprecated_kwargs
+        )
+          # If path to local directory, check if the file exists
+          if Dir.exist?(path_or_repo)
+            return File.exist?(File.join(path_or_repo, filename))
+          end
+
+          # Else it's a repo => let's check if the file exists in local cache or on the Hub
+
+          # Check if file exists in cache
+          # This information might be outdated so it's best to also make a HEAD call (if allowed).
+          cached_path = HfHub.try_to_load_from_cache(
+            path_or_repo,
+            filename,
+            revision: revision,
+            repo_type: repo_type,
+            cache_dir: cache_dir
+          )
+          has_file_in_cache = cached_path.is_a?(String)
+
+          # If local_files_only, don't try the HEAD call
+          if local_files_only
+            return has_file_in_cache
+          end
+
+          # Check if the file exists
+          begin
+            HfHub._request_wrapper(
+              "HEAD",
+              HfHub.hf_hub_url(path_or_repo, filename, revision: revision, repo_type: repo_type),
+              headers: HfHub.build_hf_headers(token: token, user_agent: http_user_agent),
+              allow_redirects: false,
+              proxies: proxies,
+              timeout: 10
+            )
+            return true
+          rescue HfHub::OfflineModeIsEnabled
+            return has_file_in_cache
+          rescue HfHub::GatedRepoError => e
+            Transformers.logger.error(e)
+            raise EnvironmentError,
+              "#{path_or_repo} is a gated repository. Make sure to request access at " +
+              "https://huggingface.co/#{path_or_repo} and pass a token having permission to this repo either by " +
+              "logging in with `huggingface-cli login` or by passing `token=<your_token>`."
+          rescue HfHub::RepositoryNotFoundError => e
+            Transformers.logger.error(e)
+            raise EnvironmentError,
+              "#{path_or_repo} is not a local folder or a valid repository name on 'https://hf.co'."
+          rescue HfHub::RevisionNotFoundError => e
+            Transformers.logger.error(e)
+            raise EnvironmentError,
+              "#{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for this " +
+              "model name. Check the model page at 'https://huggingface.co/#{path_or_repo}' for available revisions."
+          rescue HfHub::EntryNotFoundError
+            return false  # File does not exist
+          end
+        end
+      end
+    end
+  end
+end

data/lib/transformers/utils/import_utils.rb

@@ -0,0 +1,45 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  module Utils
+    ENV_VARS_TRUE_VALUES = ["1", "ON", "YES", "TRUE"]
+
+    def self.requires_backends(obj, backends)
+      if !backends.is_a?(Array)
+        backends = [backends]
+      end
+
+      name = obj.is_a?(Symbol) ? obj : obj.class.name
+
+      checks = backends.map { |backend| BACKENDS_MAPPING.fetch(backend) }
+      failed = checks.filter_map { |available, msg| format(msg, name) if !available.call }
+      if failed.any?
+        raise Error, failed.join("")
+      end
+    end
+
+    def self.is_vision_available
+      defined?(Vips)
+    end
+
+    VISION_IMPORT_ERROR = <<~MSG
+    %s requires the `ruby-vips` gem
+    MSG
+
+    BACKENDS_MAPPING = {
+      "vision" => [singleton_method(:is_vision_available), VISION_IMPORT_ERROR]
+    }
+  end
+end

data/lib/transformers/utils/logging.rb

@@ -0,0 +1,52 @@
+# Copyright 2020 Optuna, Hugging Face
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Transformers
+  class << self
+    attr_accessor :logger
+  end
+
+  # TODO add detail
+  LOG_LEVELS = {
+    "debug" => Logger::DEBUG,
+    "info" => Logger::INFO,
+    "warning" => Logger::WARN,
+    "error" => Logger::ERROR,
+    "critical" => Logger::FATAL
+  }
+
+  DEFAULT_LOG_LEVEL = Logger::WARN
+
+  def self._get_default_logging_level
+    env_level_str = ENV["TRANSFORMERS_VERBOSITY"]
+    if env_level_str
+      if LOG_LEVELS.include?(env_level_str)
+        return LOG_LEVELS[env_level_str]
+      else
+        warn(
+          "Unknown option TRANSFORMERS_VERBOSITY=#{env_level_str}, " +
+          "has to be one of: #{LOG_LEVELS.keys.join(", ")}"
+        )
+      end
+    end
+    DEFAULT_LOG_LEVEL
+  end
+
+  self.logger = begin
+    logger = Logger.new(STDERR)
+    logger.level = _get_default_logging_level
+    logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+    logger
+  end
+end

data/lib/transformers/version.rb

@@ -0,0 +1,3 @@
+module Transformers
+  VERSION = "0.1.0"
+end

data/lib/transformers-rb.rb

@@ -0,0 +1 @@
+require_relative "transformers"

data/lib/transformers.rb

@@ -0,0 +1,100 @@
+# dependencies
+require "numo/narray"
+require "safetensors"
+require "tokenizers"
+require "torch-rb"
+
+# stdlib
+require "cgi"
+require "fileutils"
+require "io/console"
+require "json"
+require "logger"
+require "net/http"
+require "pathname"
+require "securerandom"
+require "set"
+require "uri"
+
+# modules
+require_relative "transformers/ruby_utils"
+require_relative "transformers/utils/generic"
+require_relative "transformers/activations"
+require_relative "transformers/dynamic_module_utils"
+require_relative "transformers/configuration_utils"
+require_relative "transformers/convert_slow_tokenizer"
+require_relative "transformers/feature_extraction_utils"
+require_relative "transformers/image_utils"
+require_relative "transformers/image_processing_base"
+require_relative "transformers/image_processing_utils"
+require_relative "transformers/image_transforms"
+require_relative "transformers/modeling_outputs"
+require_relative "transformers/modeling_utils"
+require_relative "transformers/sentence_transformer"
+require_relative "transformers/tokenization_utils_base"
+require_relative "transformers/tokenization_utils"
+require_relative "transformers/tokenization_utils_fast"
+require_relative "transformers/torch_utils"
+require_relative "transformers/version"
+
+# data
+require_relative "transformers/data/processors/squad"
+
+# hub
+require_relative "transformers/hf_hub/constants"
+require_relative "transformers/hf_hub/errors"
+require_relative "transformers/hf_hub/file_download"
+require_relative "transformers/hf_hub/utils/_errors"
+require_relative "transformers/hf_hub/utils/_headers"
+
+# models auto
+require_relative "transformers/models/auto/auto_factory"
+require_relative "transformers/models/auto/configuration_auto"
+require_relative "transformers/models/auto/feature_extraction_auto"
+require_relative "transformers/models/auto/image_processing_auto"
+require_relative "transformers/models/auto/modeling_auto"
+require_relative "transformers/models/auto/tokenization_auto"
+
+# models bert
+require_relative "transformers/models/bert/configuration_bert"
+require_relative "transformers/models/bert/modeling_bert"
+require_relative "transformers/models/bert/tokenization_bert"
+require_relative "transformers/models/bert/tokenization_bert_fast"
+
+# models distilbert
+require_relative "transformers/models/distilbert/configuration_distilbert"
+require_relative "transformers/models/distilbert/modeling_distilbert"
+require_relative "transformers/models/distilbert/tokenization_distilbert"
+require_relative "transformers/models/distilbert/tokenization_distilbert_fast"
+
+# models vit
+require_relative "transformers/models/vit/configuration_vit"
+require_relative "transformers/models/vit/image_processing_vit"
+require_relative "transformers/models/vit/modeling_vit"
+
+# pipelines
+require_relative "transformers/pipelines/base"
+require_relative "transformers/pipelines/feature_extraction"
+require_relative "transformers/pipelines/image_classification"
+require_relative "transformers/pipelines/image_feature_extraction"
+require_relative "transformers/pipelines/pt_utils"
+require_relative "transformers/pipelines/question_answering"
+require_relative "transformers/pipelines/text_classification"
+require_relative "transformers/pipelines/token_classification"
+require_relative "transformers/pipelines/_init"
+
+# utils
+require_relative "transformers/utils/_init"
+require_relative "transformers/utils/import_utils"
+require_relative "transformers/utils/hub"
+require_relative "transformers/utils/logging"
+
+module Transformers
+  class Error < StandardError; end
+
+  class Todo < Error
+    def message
+      "not implemented yet"
+    end
+  end
+end

data/licenses/LICENSE-huggingface-hub.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.