google-cloud-gemserver 0.1.0

data/lib/google/cloud/gemserver/gcs.rb

@@ -0,0 +1,171 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/storage"
+require "google/cloud/gemserver"
+require "fileutils"
+
+module Google
+  module Cloud
+    module Gemserver
+      ##
+      # # Google Cloud Storage
+      #
+      # Interacts with Google Cloud Storage by providing methods that upload
+      # and download files to and from Google Cloud Storage.
+      #
+      module GCS
+        ##
+        # @private Fetches the project ID of the Google Cloud Platform project
+        # the gemserver was deployed to.
+        #
+        # @return [String]
+        def self.proj_id
+          Google::Cloud::Gemserver::Configuration.new[:proj_id]
+        end
+
+        ##
+        # @private Creates a Google::Cloud::Storage::Project object with the
+        # current project ID.
+        #
+        # @return [Google::Cloud::Storage::Project]
+        def self.cs
+          return unless proj_id
+          Google::Cloud::Storage.new project: proj_id, keyfile: ENV["GOOGLE_APPLICATION_CREDENTIALS"]
+        end
+
+        ##
+        # @private Fetches the bucket used to store gem files for the gemserver.
+        # If it does not exist a bucket is created.
+        #
+        # @return [Google::Cloud::Storage::Bucket]
+        def self.bucket
+          return unless proj_id
+          bucket = cs.bucket proj_id
+          bucket ? bucket : cs.create_bucket(proj_id)
+        end
+
+        ##
+        # Retrieves a file from Google Cloud Storage from a project's
+        # corresponding bucket.
+        #
+        # @param [String] file Name of the file to be retrieved.
+        #
+        # @return [Google::Cloud::Storage::File]
+        def self.get_file file
+          return unless proj_id
+          bucket.file file
+        end
+
+        ##
+        # Uploads a given file to a project's corresponding bucket on Google
+        # Cloud Storage. A destination path of the file can be provided.
+        # By default the path of the file is the same on Google Cloud Storage.
+        #
+        # @param [String] file Path to the file to be uploaded.
+        # @param [String] dest Destination path of the file on Google Cloud
+        # Storage. Optional.
+        #
+        # @return [Google::Cloud::Storage::File]
+        def self.upload file, dest = nil
+          return unless proj_id
+          bucket.create_file file, dest
+        end
+
+        ##
+        # Deletes a given file from Google Cloud Storage.
+        #
+        # @param [String] file Name of the file to be deleted.
+        def self.delete_file file
+          return unless proj_id
+          get_file(file).delete
+        end
+
+        ##
+        # @private Retrieves all files in the bucket corresponding to the
+        # project the gemserver was deployed. If specified, only files with a
+        # certain prefix will be retrieved.
+        #
+        # @param [String] prefix Prefix of the file name. Optional
+        #
+        # @return [Google::Cloud::Storage::File::List]
+        def self.files prefix = nil
+          return unless proj_id
+          bucket.files prefix: prefix
+        end
+
+        ##
+        # @private Checks if a file exists on both Google Cloud Storage and the
+        # local file system. If the file is on Cloud Storage, but missing on
+        # the file system it will be downloaded.
+        #
+        # @param [String] file_path File path of the file to be synced.
+        #
+        # @return [Boolean]
+        def self.sync file_path
+          return true unless proj_id
+          on_cloud = on_gcs? file_path
+          on_host = File.exist? file_path
+
+          if on_cloud && !on_host
+            copy_to_host file_path
+            true
+          elsif on_cloud && on_host
+            true
+          else
+            false
+          end
+        end
+
+        ##
+        # @private Checks if a given file exists on Google Cloud Storage.
+        #
+        # @param [String] file_path Path of the file on Google Cloud Storage.
+        #
+        # @return [Boolean]
+        def self.on_gcs? file_path
+          return false unless proj_id
+          get_file(file_path) != nil
+        end
+
+        ##
+        # @private Downloads a given file from Google Cloud Storage.
+        #
+        # @param [String] path Path to the file.
+        def self.copy_to_host path
+          return unless proj_id
+          file = get_file path
+          folder = extract_dir path
+          begin
+            FileUtils.mkpath(folder) unless Dir.exist?(folder)
+            file.download path
+          rescue
+            puts "Could not download #{file.name}." if file
+          end
+        end
+
+        ##
+        # @private Extracts the parent directory from a file path
+        #
+        # @param [String] path Path of the file.
+        #
+        # @return [String]
+        def self.extract_dir path
+          parts = path.split("/")
+          parts.map { |x| x != parts.last ? x : nil }.join("/")
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/gemserver/version.rb

@@ -0,0 +1,23 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Google
+  module Cloud
+    module Gemserver
+      ##
+      # The version of the google-cloud-gemserver gem.
+      VERSION = "0.1.0".freeze
+    end
+  end
+end

data/lib/patched/configuration.rb

@@ -0,0 +1,29 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+
+module PatchedConfiguration
+  ##
+  # Monkeypatch to support Cloud SQL by returning the necessary settings.
+  def database_connection_config
+    if self[:db_adapter] == "cloud_sql"
+      { adapter: "mysql2" }.merge(self[:db_connection_options])
+    else
+      super
+    end
+  end
+end
+
+Gemstash::Configuration.send :prepend, PatchedConfiguration

data/lib/patched/dependencies.rb

@@ -0,0 +1,33 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "google/cloud/gemserver"
+require "active_support/core_ext/module/aliasing"
+
+module PatchedDependencies
+  ##
+  # Monkeypatch to run a gem syncing service in the background after fetches.
+  #
+  # @param [Array] gems An array of gems to fetch.
+  #
+  # @return [Array]
+  def fetch gems
+    fetched = super gems
+    Google::Cloud::Gemserver::Backend::StorageSync.run
+    fetched
+  end
+end
+
+Gemstash::Dependencies.send :prepend, PatchedDependencies

data/lib/patched/env.rb

@@ -0,0 +1,42 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "active_support/core_ext/file/atomic"
+require "active_support/core_ext/module/aliasing"
+
+module PatchedEnv
+  ##
+  # Monkey patch to support Cloud SQL as an adapter
+  def db
+    return @db if @db
+
+    @db = if config[:db_adapter] == "cloud_sql"
+        connection = Sequel.connect config.database_connection_config
+        migrate_cloud_sql connection
+        connection
+      else
+        super
+      end
+  end
+
+  def migrate_cloud_sql database
+    Sequel.extension :migration
+    lib_dir = Gem::Specification.find_by_name("gemstash").lib_dirs_glob
+    m_dir = "#{lib_dir}/gemstash/migrations"
+    Sequel::Migrator.run database, m_dir, use_transactions: false
+  end
+end
+
+Gemstash::Env.send :prepend, PatchedEnv

data/lib/patched/gem_pusher.rb

@@ -0,0 +1,28 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "google/cloud/gemserver"
+
+module PatchedGemPusher
+  ##
+  # Monkeypatch to run a gem syncing service in the background after pushing a
+  # gem to the gemserver.
+  def serve
+    super
+    Google::Cloud::Gemserver::Backend::StorageSync.run
+  end
+end
+
+Gemstash::GemPusher.send :prepend, PatchedGemPusher

data/lib/patched/gem_yanker.rb

@@ -0,0 +1,28 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "google/cloud/gemserver"
+
+module PatchedGemYanker
+  ##
+  # Monkeypatch to run a gem syncing service in the background after yanking
+  # a gem from the gemserver.
+  def serve
+    super
+    Google::Cloud::Gemserver::Backend::StorageSync.run
+  end
+end
+
+Gemstash::GemYanker.send :prepend, PatchedGemYanker

data/lib/patched/storage.rb

@@ -0,0 +1,37 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "google/cloud/gemserver"
+require "filelock"
+
+module PatchedResource
+  ##
+  # Monkeypatch to delete a file from both the local file system and Google
+  # Cloud Storage. Done atomically to prevent circular file syncing where
+  # files never get deleted.
+  #
+  # @param [String] key Name of the gem to delete.
+  def delete key
+    file = content_filename key
+    return unless File.exist?(file) && File.exist?(properties_filename)
+    Filelock file do
+      super
+      Google::Cloud::Gemserver::GCS.delete_file file
+      Google::Cloud::Gemserver::GCS.delete_file properties_filename
+    end
+  end
+end
+
+Gemstash::Resource.send :prepend, PatchedResource

data/lib/patched/web.rb

@@ -0,0 +1,64 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gemstash"
+require "google/cloud/gemserver"
+
+##
+# Monkeypatch gemstash to handle gemserver specific endpoints.
+Gemstash::Web.class_eval do
+  ##
+  # Displays statistics on the currently deployed gemserver such as private
+  # gems, cached gems, gemserver creation time, etc.
+  post "/api/v1/stats" do
+    auth = Google::Cloud::Gemserver::Authentication.new
+    if auth.validate_token request.env["HTTP_AUTHORIZATION"]
+      content_type "application/json;charset=UTF-8"
+      Google::Cloud::Gemserver::Backend::Stats.new.run
+    else
+      halt 401, "Unauthorized operation."
+    end
+  end
+
+  ##
+  # Creates a key used for installing or pushing gems to the gemserver
+  # with given permissions. By default, a key with all permissions is created.
+  post "/api/v1/key" do
+    auth = Google::Cloud::Gemserver::Authentication.new
+    if auth.validate_token request.env["HTTP_AUTHORIZATION"]
+      key = Google::Cloud::Gemserver::Backend::Key.create_key params["permissions"]
+      content_type "application/json;charset=UTF-8"
+      "Generated key: #{key}"
+    else
+      halt 401, "Unauthorized operation."
+    end
+  end
+
+  ##
+  # Deletes a key.
+  put "/api/v1/key" do
+    auth = Google::Cloud::Gemserver::Authentication.new
+    if auth.validate_token request.env["HTTP_AUTHORIZATION"]
+      res = Google::Cloud::Gemserver::Backend::Key.delete_key params["key"]
+      content_type "application/json;charset=UTF-8"
+      if res
+        "Deleted key #{params["key"]} successfully."
+      else
+        "Deleting key #{params["key"]} failed."
+      end
+    else
+      halt 401, "Unauthorized operation."
+    end
+  end
+end