google-cloud-gemserver 0.1.0

data/lib/google/cloud/gemserver/backend/stats.rb

@@ -0,0 +1,149 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/gemserver"
+require "yaml"
+
+module Google
+  module Cloud
+    module Gemserver
+      module Backend
+        ##
+        # # Stats
+        #
+        # Stats provides a set of methods that display detailed information
+        # about the deployed gemserver. It includes: general Google Cloud
+        # Platform project information, how long the gemserver has been running
+        # , what private gems are stored, and what gems have been cached.
+        #
+        class Stats
+
+          ##
+          # The project ID of the project on Google Cloud Platform the
+          # gemserver was deployed to.
+          # @return [String]
+          attr_accessor :proj
+
+          ##
+          # Initialize a Configuration object and project ID for the Stats
+          # object enabling it to fetch detailed information about the
+          # gemserver.
+          def initialize
+            @config = Google::Cloud::Gemserver::Configuration.new
+            @proj = (@config[:proj_id] || nil).freeze
+          end
+
+          ##
+          # Displays various sets of information about the gemserver such as
+          # how long it has been running, currently stored, private gems and
+          # their status, and cached gems.
+          def run
+            resp = ""
+            resp << log_uptime
+            resp << log_private_gems
+            resp << log_cached_gems
+          end
+
+          ##
+          # Displays information about the project on Google Cloud
+          # Platform the gemserver was deployed to.
+          def log_app_description
+            return "" if ENV["APP_ENV"] == "test"
+            puts "Project Information:"
+            cmd = "gcloud app describe --project #{@proj}"
+            puts run_cmd(cmd).gsub("\n", "\n\t").prepend "\t"
+          end
+
+          private
+
+          ##
+          # @private Displays the time of which the gemserver was deployed.
+          def log_uptime
+            return "" unless project
+            "The gemserver has been running since #{project.created_at}\n"
+          end
+
+          ##
+          # @private Displays the private gems stored on the gemserver and
+          # their status (currently indexed or not).
+          def log_private_gems
+            res = "Private Gems:\n"
+            versions = db :versions
+            format = "%35s\t%20s\n"
+            res << sprintf(format, "Gem Name - Version", "Available?")
+            versions.map do |gem|
+              res << sprintf(format, gem[:storage_id], gem[:indexed])
+            end
+            puts res
+            res
+          end
+
+          ##
+          # @private Displays the gems cached on the gemserver.
+          def log_cached_gems
+            res = "Cached Gem Dependencies:\n"
+            cached = db :cached_rubygems
+            format = "%35s\t%20s\n"
+            res << sprintf(format, "Gem Name - Version", "Date Cached")
+            cached.map do |gem|
+              res << sprintf(format, gem[:name], gem[:created_at])
+            end
+            puts res
+            res
+          end
+
+          ##
+          # @private Fetches the Google Cloud Platform project the gemserver
+          # was deployed to.
+          #
+          # @return [Project]
+          def project
+            if @proj.nil?
+              return nil if ENV["APP_ENV"] == "test"
+              raise ":proj_id not set in config file"
+            end
+            Google::Cloud::Gemserver::CLI::Project.new(@proj).send(:project)
+          end
+
+          ##
+          # @private Fetches the Environment object currently being used by the
+          # gemserver. It enables access to the database.
+          #
+          # @return [Gemstash::Env]
+          def env
+            GemstashServer.env @config.config_path
+          end
+
+          ##
+          # @private Retrieves all the rows in the database for a given table.
+          #
+          # @param [String] table The table to be read.
+          #
+          # @return [Array]
+          def db table
+            env.db[table].all
+          end
+
+          ##
+          # @private Runs a given command on the local machine.
+          #
+          # @param [String] cmd The command to be run.
+          def run_cmd cmd
+            `#{cmd}`
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/gemserver/backend/storage_sync.rb

@@ -0,0 +1,186 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/storage"
+require "google/cloud/gemserver"
+require "fileutils"
+require "filelock"
+require "digest/md5"
+require "singleton"
+require "concurrent"
+require "forwardable"
+
+module Google
+  module Cloud
+    module Gemserver
+      module Backend
+        ##
+        # # Storage Sync
+        #
+        # A set of methods that manage syncing files between the local file
+        # system that the gemserver runs on (a container on Google App Engine)
+        # and Google Cloud Storage. By doing so, when the gemserver is restarted
+        # , for whatever reason, the gems pushed to the gemserver will persist.
+        # Without such a system in place all gems / files on the gemserver will
+        # be lost as it runs on a container.
+        #
+        class StorageSync
+          include Concurrent::Async
+          include Singleton
+
+          ##
+          # A lock to ensure the .gemstash directory, used to store gem files, is
+          # created atomically.
+          DIR_LOCK = File.expand_path("~/gemstash_dir").freeze
+
+          ##
+          # Extend StorageSync such that it can be called without the
+          # .instance method.
+          class << self
+            extend Forwardable
+
+            ##
+            # Delegate the run and download_service methods to the Singleton
+            # via .instance.
+            def_delegators :instance, :run, :upload_service, :download_service,
+              :try_upload, :try_download, :file_changed?
+          end
+
+          ##
+          # Creates an instance of the Singleton StorageSync class with a
+          # background thread and asynchronous components to run methods
+          # asynchronously.
+          def initialize
+            super
+          end
+
+          ##
+          # Runs a background gem files syncing service to ensure they are up to
+          # date with respect to the files on Google Cloud Storage. This allows
+          # allow the gem metadata on the gemserver (in the container) to persist
+          # in case of situations where the gemserver goes down.
+          def run
+            async.sync
+          end
+
+          ##
+          # @private Runs the uploader to send updated gem files to Google Cloud
+          # Storage (source of truth) then updates the rest of the gem files by
+          # downloading them off Google Cloud Storage.
+          def sync
+            upload_service
+            download_service
+          end
+
+          ##
+          # @private The directory used to store gem data.
+          #
+          # @return [String]
+          def gemstash_dir
+            if ENV["APP_ENV"] == "production"
+              Configuration::GEMSTASH_DIR
+            else
+              File.expand_path("~/.gemstash")
+            end
+          end
+
+          ##
+          # @private Create the directory used to store gem data.
+          def prepare_dir
+            Filelock DIR_LOCK do
+              FileUtils.mkpath gemstash_dir
+            end
+          end
+
+          ##
+          # @private The uploading service that uploads gem files from the local
+          # file system to Google Cloud Storage. It does not upload any cached
+          # files.
+          def upload_service
+            puts "Running uploading service..."
+            prepare_dir
+
+            entries = Dir.glob("#{gemstash_dir}/**/*").reject do |e|
+              e.include? "gem_cache"
+            end
+            entries.each do |e|
+              try_upload e if File.file? e
+            end
+          end
+
+          ##
+          # @private Uploads a file to Google Cloud Storage only if the file
+          # has yet to be uploaded or has a different hash from the Cloud copy.
+          #
+          # @param [String] file The path to the file to be uploaded.
+          def try_upload file
+            GCS.upload(file) unless GCS.on_gcs?(file)
+            return unless file_changed?(file)
+            Filelock file do
+              GCS.upload file
+            end
+          end
+
+          ##
+          # @private The downloading service that downloads gem files from Google
+          # Cloud Storage to the local file system.
+          def download_service
+            puts "Running downloading service..."
+            prepare_dir
+
+            files = GCS.files
+            return unless files
+            files.each { |file| try_download file.name }
+          end
+
+          ##
+          # @private Downloads a file to the local file sytem from Google Cloud
+          # Storage only if there is sufficient space and the local copy's hash
+          # differs from the cloud copy's hash.
+          #
+          # @param [String] file Name of the file to download.
+          def try_download file
+            total = `df -k /`.split(" ")[11].to_f
+            used = `df -k /`.split(" ")[12].to_f
+            usage = used / total
+            if usage < 0.95
+              if File.exist? file
+                Filelock(file) { GCS.sync file if file_changed? file }
+              else
+                GCS.copy_to_host file
+              end
+            else
+              raise "Error downloading: disk usage at #{usage}! Increase disk space!"
+            end
+          end
+
+          ##
+          # @private Determines if a file on the local file system has changed
+          # from the corresponding file on Google Cloud Storage, if it exists.
+          #
+          # @param [String] file Name of the file
+          #
+          # @return [Boolean]
+          def file_changed? file
+            return true unless File.exist? file
+            return true unless GCS.get_file(file)
+            gcs_md5 = GCS.get_file(file).md5
+            local_md5 = Digest::MD5.file(file).base64digest
+            gcs_md5 != local_md5
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/gemserver/cli.rb

@@ -0,0 +1,174 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/gemserver"
+require "thor"
+
+module Google
+  module Cloud
+    module Gemserver
+      ##
+      #
+      # # CLI
+      #
+      # The command line interface which provides methods to interact with a
+      # gemserver and deploy it to a given Google Cloud Platform project.
+      #
+      class CLI < Thor
+        autoload :Project,         "google/cloud/gemserver/cli/project"
+        autoload :CloudSQL,        "google/cloud/gemserver/cli/cloud_sql"
+        autoload :Server,          "google/cloud/gemserver/cli/server"
+        autoload :Request,         "google/cloud/gemserver/cli/request"
+
+        # Error class thrown when a command that does not exist is run.
+        class Error < Thor::Error
+          def initialize cli, message
+            super cli.set_color(message, :red)
+          end
+        end
+
+        def self.start args = ARGV
+          Configuration.new.gen_config
+          super
+        end
+
+        ##
+        # Starts the gemserver by starting up gemstash.
+        desc "start", "Starts the gem server. This will be run automatically" \
+          " after a deploy. Running this locally will start the gemserver "\
+          "locally"
+        def start
+          Server.new.start
+        end
+
+        ##
+        # Creates a gemserver app and deploys it to a Google Cloud Platform
+        # project. An existing Google Cloud Platform project must be provided
+        # through the --use-proj option and an existing Cloud SQL instance may
+        # be provided through the --use-inst option, otherwise a new one will
+        # be created.
+        desc "create", "Creates and deploys the gem server then starts it"
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+          "Existing project to deploy gemserver to"
+        method_option :use_inst, type: :string, aliases: "-i", desc:
+          "Existing project to deploy gemserver to"
+        def create
+          prepare
+          Server.new.deploy
+        end
+
+        ##
+        # Retrieves a Google Cloud Platform instance and informs the user to
+        # enable necessary APIs for that project. Also creates a Cloud SQL
+        # instance if one was not provided with the --use-inst option.
+        desc "prepare", "Uses a project on Google Cloud Platform and deploys"\
+          " a gemserver to it."
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+          "Existing project to deploy gemserver to"
+        method_option :use_inst, type: :string, aliases: "-i", desc:
+          "Existing Cloud SQL instance to us"
+        def prepare
+          Project.new(options[:use_proj]).create
+          CloudSQL.new(options[:use_inst]).run
+        end
+
+        ##
+        # Updates the gemserver on Google Cloud Platform to the latest version
+        # of the gemserver installed on the user's system.
+        desc "update", "Redeploys the gemserver with the current config file" \
+          " and google-cloud-gemserver gem version (a deploy must have " \
+          "succeeded for 'update' to work."
+        def update
+          Server.new.update
+        end
+
+        ##
+        # Deletes a given gemserver provided by the --use-proj option.
+        # This deletes the Google Cloud Platform project, all associated
+        # Cloud SQL instances, and all Cloud Storage buckets.
+        desc "delete", "Delete a given gemserver"
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+        "Project id of GCP project the gemserver was deployed to. Warning:"\
+         " parent project and CloudSQL instance will also be deleted"
+        def delete
+          Server.new.delete options[:use_proj]
+        end
+
+        ##
+        # Creates a key used for installing or pushing gems to the given
+        # gemserver with given permissions provided with the --permissions
+        # option. By default, a key with all permissions is created.
+        desc "create_key", "Creates an authentication key"
+        method_option :permissions, type: :string, aliases: "-p", desc:
+          "Options: write, read, both. Default is both."
+        method_option :remote, type: :string, aliases: "-r", desc:
+          "The gemserver URL, i.e. gemserver.com"
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+          "The GCP project the gemserver was deployed to."
+        def create_key
+          if ENV["APP_ENV"] == "test"
+            return Backend::Key.create_key(options[:permissions])
+          end
+          puts Request.new(options[:remote], options[:use_proj]).create_key(options[:permissions]).body
+          Backend::Key.output_key_info
+        end
+
+        ##
+        # Deletes a given key provided by the --key option from the given
+        # gemserver.
+        desc "delete_key", "Deletes a given key"
+        method_option :key, type: :string, aliases: "-k", desc:
+          "The key to delete"
+        method_option :remote, type: :string, aliases: "-r", desc:
+          "The gemserver URL, i.e. gemserver.com"
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+          "The GCP project the gemserver was deployed to."
+        def delete_key
+          if ENV["APP_ENV"] == "test"
+            return Backend::Key.delete_key(options[:key])
+          end
+          puts Request.new(options[:remote], options[:use_proj]).delete_key(options[:key]).body
+        end
+
+        ##
+        # Displays the configuration used by the currently deployed gemserver.
+        desc "config", "Displays the config the current deployed gemserver is"\
+          " using (if one is running)"
+        def config
+          Configuration.display_config
+        end
+
+        ##
+        # Displays statistics on the given gemserver such as private gems,
+        # cached gems, gemserver creation time, etc.
+        desc "stats", "Displays statistics on the given gemserver"
+        method_option :remote, type: :string, aliases: "-r", desc:
+          "The gemserver URL, i.e. gemserver.com"
+        method_option :use_proj, type: :string, aliases: "-g", desc:
+          "The GCP project the gemserver was deployed to."
+        def stats
+          return Backend::Stats.new.run if ENV["APP_ENV"] == "test"
+          Backend::Stats.new.log_app_description
+          puts Request.new(options[:remote], options[:use_proj]).stats.body
+        end
+
+        desc "gen_config", "Generates configuration files with default" \
+          " values"
+        def gen_config
+          Configuration.new.gen_config
+        end
+      end
+    end
+  end
+end