google-cloud-gemserver 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,149 @@
1
+ # Copyright 2017 Google Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require "google/cloud/gemserver"
16
+ require "yaml"
17
+
18
+ module Google
19
+ module Cloud
20
+ module Gemserver
21
+ module Backend
22
+ ##
23
+ # # Stats
24
+ #
25
+ # Stats provides a set of methods that display detailed information
26
+ # about the deployed gemserver. It includes: general Google Cloud
27
+ # Platform project information, how long the gemserver has been running
28
+ # , what private gems are stored, and what gems have been cached.
29
+ #
30
+ class Stats
31
+
32
+ ##
33
+ # The project ID of the project on Google Cloud Platform the
34
+ # gemserver was deployed to.
35
+ # @return [String]
36
+ attr_accessor :proj
37
+
38
+ ##
39
+ # Initialize a Configuration object and project ID for the Stats
40
+ # object enabling it to fetch detailed information about the
41
+ # gemserver.
42
+ def initialize
43
+ @config = Google::Cloud::Gemserver::Configuration.new
44
+ @proj = (@config[:proj_id] || nil).freeze
45
+ end
46
+
47
+ ##
48
+ # Displays various sets of information about the gemserver such as
49
+ # how long it has been running, currently stored, private gems and
50
+ # their status, and cached gems.
51
+ def run
52
+ resp = ""
53
+ resp << log_uptime
54
+ resp << log_private_gems
55
+ resp << log_cached_gems
56
+ end
57
+
58
+ ##
59
+ # Displays information about the project on Google Cloud
60
+ # Platform the gemserver was deployed to.
61
+ def log_app_description
62
+ return "" if ENV["APP_ENV"] == "test"
63
+ puts "Project Information:"
64
+ cmd = "gcloud app describe --project #{@proj}"
65
+ puts run_cmd(cmd).gsub("\n", "\n\t").prepend "\t"
66
+ end
67
+
68
+ private
69
+
70
+ ##
71
+ # @private Displays the time of which the gemserver was deployed.
72
+ def log_uptime
73
+ return "" unless project
74
+ "The gemserver has been running since #{project.created_at}\n"
75
+ end
76
+
77
+ ##
78
+ # @private Displays the private gems stored on the gemserver and
79
+ # their status (currently indexed or not).
80
+ def log_private_gems
81
+ res = "Private Gems:\n"
82
+ versions = db :versions
83
+ format = "%35s\t%20s\n"
84
+ res << sprintf(format, "Gem Name - Version", "Available?")
85
+ versions.map do |gem|
86
+ res << sprintf(format, gem[:storage_id], gem[:indexed])
87
+ end
88
+ puts res
89
+ res
90
+ end
91
+
92
+ ##
93
+ # @private Displays the gems cached on the gemserver.
94
+ def log_cached_gems
95
+ res = "Cached Gem Dependencies:\n"
96
+ cached = db :cached_rubygems
97
+ format = "%35s\t%20s\n"
98
+ res << sprintf(format, "Gem Name - Version", "Date Cached")
99
+ cached.map do |gem|
100
+ res << sprintf(format, gem[:name], gem[:created_at])
101
+ end
102
+ puts res
103
+ res
104
+ end
105
+
106
+ ##
107
+ # @private Fetches the Google Cloud Platform project the gemserver
108
+ # was deployed to.
109
+ #
110
+ # @return [Project]
111
+ def project
112
+ if @proj.nil?
113
+ return nil if ENV["APP_ENV"] == "test"
114
+ raise ":proj_id not set in config file"
115
+ end
116
+ Google::Cloud::Gemserver::CLI::Project.new(@proj).send(:project)
117
+ end
118
+
119
+ ##
120
+ # @private Fetches the Environment object currently being used by the
121
+ # gemserver. It enables access to the database.
122
+ #
123
+ # @return [Gemstash::Env]
124
+ def env
125
+ GemstashServer.env @config.config_path
126
+ end
127
+
128
+ ##
129
+ # @private Retrieves all the rows in the database for a given table.
130
+ #
131
+ # @param [String] table The table to be read.
132
+ #
133
+ # @return [Array]
134
+ def db table
135
+ env.db[table].all
136
+ end
137
+
138
+ ##
139
+ # @private Runs a given command on the local machine.
140
+ #
141
+ # @param [String] cmd The command to be run.
142
+ def run_cmd cmd
143
+ `#{cmd}`
144
+ end
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,186 @@
1
+ # Copyright 2017 Google Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require "google/cloud/storage"
16
+ require "google/cloud/gemserver"
17
+ require "fileutils"
18
+ require "filelock"
19
+ require "digest/md5"
20
+ require "singleton"
21
+ require "concurrent"
22
+ require "forwardable"
23
+
24
+ module Google
25
+ module Cloud
26
+ module Gemserver
27
+ module Backend
28
+ ##
29
+ # # Storage Sync
30
+ #
31
+ # A set of methods that manage syncing files between the local file
32
+ # system that the gemserver runs on (a container on Google App Engine)
33
+ # and Google Cloud Storage. By doing so, when the gemserver is restarted
34
+ # , for whatever reason, the gems pushed to the gemserver will persist.
35
+ # Without such a system in place all gems / files on the gemserver will
36
+ # be lost as it runs on a container.
37
+ #
38
+ class StorageSync
39
+ include Concurrent::Async
40
+ include Singleton
41
+
42
+ ##
43
+ # A lock to ensure the .gemstash directory, used to store gem files, is
44
+ # created atomically.
45
+ DIR_LOCK = File.expand_path("~/gemstash_dir").freeze
46
+
47
+ ##
48
+ # Extend StorageSync such that it can be called without the
49
+ # .instance method.
50
+ class << self
51
+ extend Forwardable
52
+
53
+ ##
54
+ # Delegate the run and download_service methods to the Singleton
55
+ # via .instance.
56
+ def_delegators :instance, :run, :upload_service, :download_service,
57
+ :try_upload, :try_download, :file_changed?
58
+ end
59
+
60
+ ##
61
+ # Creates an instance of the Singleton StorageSync class with a
62
+ # background thread and asynchronous components to run methods
63
+ # asynchronously.
64
+ def initialize
65
+ super
66
+ end
67
+
68
+ ##
69
+ # Runs a background gem files syncing service to ensure they are up to
70
+ # date with respect to the files on Google Cloud Storage. This allows
71
+ # allow the gem metadata on the gemserver (in the container) to persist
72
+ # in case of situations where the gemserver goes down.
73
+ def run
74
+ async.sync
75
+ end
76
+
77
+ ##
78
+ # @private Runs the uploader to send updated gem files to Google Cloud
79
+ # Storage (source of truth) then updates the rest of the gem files by
80
+ # downloading them off Google Cloud Storage.
81
+ def sync
82
+ upload_service
83
+ download_service
84
+ end
85
+
86
+ ##
87
+ # @private The directory used to store gem data.
88
+ #
89
+ # @return [String]
90
+ def gemstash_dir
91
+ if ENV["APP_ENV"] == "production"
92
+ Configuration::GEMSTASH_DIR
93
+ else
94
+ File.expand_path("~/.gemstash")
95
+ end
96
+ end
97
+
98
+ ##
99
+ # @private Create the directory used to store gem data.
100
+ def prepare_dir
101
+ Filelock DIR_LOCK do
102
+ FileUtils.mkpath gemstash_dir
103
+ end
104
+ end
105
+
106
+ ##
107
+ # @private The uploading service that uploads gem files from the local
108
+ # file system to Google Cloud Storage. It does not upload any cached
109
+ # files.
110
+ def upload_service
111
+ puts "Running uploading service..."
112
+ prepare_dir
113
+
114
+ entries = Dir.glob("#{gemstash_dir}/**/*").reject do |e|
115
+ e.include? "gem_cache"
116
+ end
117
+ entries.each do |e|
118
+ try_upload e if File.file? e
119
+ end
120
+ end
121
+
122
+ ##
123
+ # @private Uploads a file to Google Cloud Storage only if the file
124
+ # has yet to be uploaded or has a different hash from the Cloud copy.
125
+ #
126
+ # @param [String] file The path to the file to be uploaded.
127
+ def try_upload file
128
+ GCS.upload(file) unless GCS.on_gcs?(file)
129
+ return unless file_changed?(file)
130
+ Filelock file do
131
+ GCS.upload file
132
+ end
133
+ end
134
+
135
+ ##
136
+ # @private The downloading service that downloads gem files from Google
137
+ # Cloud Storage to the local file system.
138
+ def download_service
139
+ puts "Running downloading service..."
140
+ prepare_dir
141
+
142
+ files = GCS.files
143
+ return unless files
144
+ files.each { |file| try_download file.name }
145
+ end
146
+
147
+ ##
148
+ # @private Downloads a file to the local file sytem from Google Cloud
149
+ # Storage only if there is sufficient space and the local copy's hash
150
+ # differs from the cloud copy's hash.
151
+ #
152
+ # @param [String] file Name of the file to download.
153
+ def try_download file
154
+ total = `df -k /`.split(" ")[11].to_f
155
+ used = `df -k /`.split(" ")[12].to_f
156
+ usage = used / total
157
+ if usage < 0.95
158
+ if File.exist? file
159
+ Filelock(file) { GCS.sync file if file_changed? file }
160
+ else
161
+ GCS.copy_to_host file
162
+ end
163
+ else
164
+ raise "Error downloading: disk usage at #{usage}! Increase disk space!"
165
+ end
166
+ end
167
+
168
+ ##
169
+ # @private Determines if a file on the local file system has changed
170
+ # from the corresponding file on Google Cloud Storage, if it exists.
171
+ #
172
+ # @param [String] file Name of the file
173
+ #
174
+ # @return [Boolean]
175
+ def file_changed? file
176
+ return true unless File.exist? file
177
+ return true unless GCS.get_file(file)
178
+ gcs_md5 = GCS.get_file(file).md5
179
+ local_md5 = Digest::MD5.file(file).base64digest
180
+ gcs_md5 != local_md5
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end
186
+ end
@@ -0,0 +1,174 @@
1
+ # Copyright 2017 Google Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require "google/cloud/gemserver"
16
+ require "thor"
17
+
18
+ module Google
19
+ module Cloud
20
+ module Gemserver
21
+ ##
22
+ #
23
+ # # CLI
24
+ #
25
+ # The command line interface which provides methods to interact with a
26
+ # gemserver and deploy it to a given Google Cloud Platform project.
27
+ #
28
+ class CLI < Thor
29
+ autoload :Project, "google/cloud/gemserver/cli/project"
30
+ autoload :CloudSQL, "google/cloud/gemserver/cli/cloud_sql"
31
+ autoload :Server, "google/cloud/gemserver/cli/server"
32
+ autoload :Request, "google/cloud/gemserver/cli/request"
33
+
34
+ # Error class thrown when a command that does not exist is run.
35
+ class Error < Thor::Error
36
+ def initialize cli, message
37
+ super cli.set_color(message, :red)
38
+ end
39
+ end
40
+
41
+ def self.start args = ARGV
42
+ Configuration.new.gen_config
43
+ super
44
+ end
45
+
46
+ ##
47
+ # Starts the gemserver by starting up gemstash.
48
+ desc "start", "Starts the gem server. This will be run automatically" \
49
+ " after a deploy. Running this locally will start the gemserver "\
50
+ "locally"
51
+ def start
52
+ Server.new.start
53
+ end
54
+
55
+ ##
56
+ # Creates a gemserver app and deploys it to a Google Cloud Platform
57
+ # project. An existing Google Cloud Platform project must be provided
58
+ # through the --use-proj option and an existing Cloud SQL instance may
59
+ # be provided through the --use-inst option, otherwise a new one will
60
+ # be created.
61
+ desc "create", "Creates and deploys the gem server then starts it"
62
+ method_option :use_proj, type: :string, aliases: "-g", desc:
63
+ "Existing project to deploy gemserver to"
64
+ method_option :use_inst, type: :string, aliases: "-i", desc:
65
+ "Existing project to deploy gemserver to"
66
+ def create
67
+ prepare
68
+ Server.new.deploy
69
+ end
70
+
71
+ ##
72
+ # Retrieves a Google Cloud Platform instance and informs the user to
73
+ # enable necessary APIs for that project. Also creates a Cloud SQL
74
+ # instance if one was not provided with the --use-inst option.
75
+ desc "prepare", "Uses a project on Google Cloud Platform and deploys"\
76
+ " a gemserver to it."
77
+ method_option :use_proj, type: :string, aliases: "-g", desc:
78
+ "Existing project to deploy gemserver to"
79
+ method_option :use_inst, type: :string, aliases: "-i", desc:
80
+ "Existing Cloud SQL instance to us"
81
+ def prepare
82
+ Project.new(options[:use_proj]).create
83
+ CloudSQL.new(options[:use_inst]).run
84
+ end
85
+
86
+ ##
87
+ # Updates the gemserver on Google Cloud Platform to the latest version
88
+ # of the gemserver installed on the user's system.
89
+ desc "update", "Redeploys the gemserver with the current config file" \
90
+ " and google-cloud-gemserver gem version (a deploy must have " \
91
+ "succeeded for 'update' to work."
92
+ def update
93
+ Server.new.update
94
+ end
95
+
96
+ ##
97
+ # Deletes a given gemserver provided by the --use-proj option.
98
+ # This deletes the Google Cloud Platform project, all associated
99
+ # Cloud SQL instances, and all Cloud Storage buckets.
100
+ desc "delete", "Delete a given gemserver"
101
+ method_option :use_proj, type: :string, aliases: "-g", desc:
102
+ "Project id of GCP project the gemserver was deployed to. Warning:"\
103
+ " parent project and CloudSQL instance will also be deleted"
104
+ def delete
105
+ Server.new.delete options[:use_proj]
106
+ end
107
+
108
+ ##
109
+ # Creates a key used for installing or pushing gems to the given
110
+ # gemserver with given permissions provided with the --permissions
111
+ # option. By default, a key with all permissions is created.
112
+ desc "create_key", "Creates an authentication key"
113
+ method_option :permissions, type: :string, aliases: "-p", desc:
114
+ "Options: write, read, both. Default is both."
115
+ method_option :remote, type: :string, aliases: "-r", desc:
116
+ "The gemserver URL, i.e. gemserver.com"
117
+ method_option :use_proj, type: :string, aliases: "-g", desc:
118
+ "The GCP project the gemserver was deployed to."
119
+ def create_key
120
+ if ENV["APP_ENV"] == "test"
121
+ return Backend::Key.create_key(options[:permissions])
122
+ end
123
+ puts Request.new(options[:remote], options[:use_proj]).create_key(options[:permissions]).body
124
+ Backend::Key.output_key_info
125
+ end
126
+
127
+ ##
128
+ # Deletes a given key provided by the --key option from the given
129
+ # gemserver.
130
+ desc "delete_key", "Deletes a given key"
131
+ method_option :key, type: :string, aliases: "-k", desc:
132
+ "The key to delete"
133
+ method_option :remote, type: :string, aliases: "-r", desc:
134
+ "The gemserver URL, i.e. gemserver.com"
135
+ method_option :use_proj, type: :string, aliases: "-g", desc:
136
+ "The GCP project the gemserver was deployed to."
137
+ def delete_key
138
+ if ENV["APP_ENV"] == "test"
139
+ return Backend::Key.delete_key(options[:key])
140
+ end
141
+ puts Request.new(options[:remote], options[:use_proj]).delete_key(options[:key]).body
142
+ end
143
+
144
+ ##
145
+ # Displays the configuration used by the currently deployed gemserver.
146
+ desc "config", "Displays the config the current deployed gemserver is"\
147
+ " using (if one is running)"
148
+ def config
149
+ Configuration.display_config
150
+ end
151
+
152
+ ##
153
+ # Displays statistics on the given gemserver such as private gems,
154
+ # cached gems, gemserver creation time, etc.
155
+ desc "stats", "Displays statistics on the given gemserver"
156
+ method_option :remote, type: :string, aliases: "-r", desc:
157
+ "The gemserver URL, i.e. gemserver.com"
158
+ method_option :use_proj, type: :string, aliases: "-g", desc:
159
+ "The GCP project the gemserver was deployed to."
160
+ def stats
161
+ return Backend::Stats.new.run if ENV["APP_ENV"] == "test"
162
+ Backend::Stats.new.log_app_description
163
+ puts Request.new(options[:remote], options[:use_proj]).stats.body
164
+ end
165
+
166
+ desc "gen_config", "Generates configuration files with default" \
167
+ " values"
168
+ def gen_config
169
+ Configuration.new.gen_config
170
+ end
171
+ end
172
+ end
173
+ end
174
+ end