google-cloud-gemserver 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +7 -0
- data/CONTRIBUTING.md +49 -0
- data/LICENSE +202 -0
- data/README.md +179 -0
- data/bin/console +7 -0
- data/bin/google-cloud-gemserver +5 -0
- data/bin/setup +6 -0
- data/lib/google/cloud/gemserver.rb +33 -0
- data/lib/google/cloud/gemserver/authentication.rb +254 -0
- data/lib/google/cloud/gemserver/backend.rb +33 -0
- data/lib/google/cloud/gemserver/backend/gemstash_server.rb +60 -0
- data/lib/google/cloud/gemserver/backend/key.rb +152 -0
- data/lib/google/cloud/gemserver/backend/stats.rb +149 -0
- data/lib/google/cloud/gemserver/backend/storage_sync.rb +186 -0
- data/lib/google/cloud/gemserver/cli.rb +174 -0
- data/lib/google/cloud/gemserver/cli/cloud_sql.rb +196 -0
- data/lib/google/cloud/gemserver/cli/project.rb +143 -0
- data/lib/google/cloud/gemserver/cli/request.rb +130 -0
- data/lib/google/cloud/gemserver/cli/server.rb +337 -0
- data/lib/google/cloud/gemserver/configuration.rb +505 -0
- data/lib/google/cloud/gemserver/gcs.rb +171 -0
- data/lib/google/cloud/gemserver/version.rb +23 -0
- data/lib/patched/configuration.rb +29 -0
- data/lib/patched/dependencies.rb +33 -0
- data/lib/patched/env.rb +42 -0
- data/lib/patched/gem_pusher.rb +28 -0
- data/lib/patched/gem_yanker.rb +28 -0
- data/lib/patched/storage.rb +37 -0
- data/lib/patched/web.rb +64 -0
- metadata +271 -0
@@ -0,0 +1,171 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "google/cloud/storage"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
require "fileutils"
|
18
|
+
|
19
|
+
module Google
|
20
|
+
module Cloud
|
21
|
+
module Gemserver
|
22
|
+
##
|
23
|
+
# # Google Cloud Storage
|
24
|
+
#
|
25
|
+
# Interacts with Google Cloud Storage by providing methods that upload
|
26
|
+
# and download files to and from Google Cloud Storage.
|
27
|
+
#
|
28
|
+
module GCS
|
29
|
+
##
|
30
|
+
# @private Fetches the project ID of the Google Cloud Platform project
|
31
|
+
# the gemserver was deployed to.
|
32
|
+
#
|
33
|
+
# @return [String]
|
34
|
+
def self.proj_id
|
35
|
+
Google::Cloud::Gemserver::Configuration.new[:proj_id]
|
36
|
+
end
|
37
|
+
|
38
|
+
##
|
39
|
+
# @private Creates a Google::Cloud::Storage::Project object with the
|
40
|
+
# current project ID.
|
41
|
+
#
|
42
|
+
# @return [Google::Cloud::Storage::Project]
|
43
|
+
def self.cs
|
44
|
+
return unless proj_id
|
45
|
+
Google::Cloud::Storage.new project: proj_id, keyfile: ENV["GOOGLE_APPLICATION_CREDENTIALS"]
|
46
|
+
end
|
47
|
+
|
48
|
+
##
|
49
|
+
# @private Fetches the bucket used to store gem files for the gemserver.
|
50
|
+
# If it does not exist a bucket is created.
|
51
|
+
#
|
52
|
+
# @return [Google::Cloud::Storage::Bucket]
|
53
|
+
def self.bucket
|
54
|
+
return unless proj_id
|
55
|
+
bucket = cs.bucket proj_id
|
56
|
+
bucket ? bucket : cs.create_bucket(proj_id)
|
57
|
+
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# Retrieves a file from Google Cloud Storage from a project's
|
61
|
+
# corresponding bucket.
|
62
|
+
#
|
63
|
+
# @param [String] file Name of the file to be retrieved.
|
64
|
+
#
|
65
|
+
# @return [Google::Cloud::Storage::File]
|
66
|
+
def self.get_file file
|
67
|
+
return unless proj_id
|
68
|
+
bucket.file file
|
69
|
+
end
|
70
|
+
|
71
|
+
##
|
72
|
+
# Uploads a given file to a project's corresponding bucket on Google
|
73
|
+
# Cloud Storage. A destination path of the file can be provided.
|
74
|
+
# By default the path of the file is the same on Google Cloud Storage.
|
75
|
+
#
|
76
|
+
# @param [String] file Path to the file to be uploaded.
|
77
|
+
# @param [String] dest Destination path of the file on Google Cloud
|
78
|
+
# Storage. Optional.
|
79
|
+
#
|
80
|
+
# @return [Google::Cloud::Storage::File]
|
81
|
+
def self.upload file, dest = nil
|
82
|
+
return unless proj_id
|
83
|
+
bucket.create_file file, dest
|
84
|
+
end
|
85
|
+
|
86
|
+
##
|
87
|
+
# Deletes a given file from Google Cloud Storage.
|
88
|
+
#
|
89
|
+
# @param [String] file Name of the file to be deleted.
|
90
|
+
def self.delete_file file
|
91
|
+
return unless proj_id
|
92
|
+
get_file(file).delete
|
93
|
+
end
|
94
|
+
|
95
|
+
##
|
96
|
+
# @private Retrieves all files in the bucket corresponding to the
|
97
|
+
# project the gemserver was deployed. If specified, only files with a
|
98
|
+
# certain prefix will be retrieved.
|
99
|
+
#
|
100
|
+
# @param [String] prefix Prefix of the file name. Optional
|
101
|
+
#
|
102
|
+
# @return [Google::Cloud::Storage::File::List]
|
103
|
+
def self.files prefix = nil
|
104
|
+
return unless proj_id
|
105
|
+
bucket.files prefix: prefix
|
106
|
+
end
|
107
|
+
|
108
|
+
##
|
109
|
+
# @private Checks if a file exists on both Google Cloud Storage and the
|
110
|
+
# local file system. If the file is on Cloud Storage, but missing on
|
111
|
+
# the file system it will be downloaded.
|
112
|
+
#
|
113
|
+
# @param [String] file_path File path of the file to be synced.
|
114
|
+
#
|
115
|
+
# @return [Boolean]
|
116
|
+
def self.sync file_path
|
117
|
+
return true unless proj_id
|
118
|
+
on_cloud = on_gcs? file_path
|
119
|
+
on_host = File.exist? file_path
|
120
|
+
|
121
|
+
if on_cloud && !on_host
|
122
|
+
copy_to_host file_path
|
123
|
+
true
|
124
|
+
elsif on_cloud && on_host
|
125
|
+
true
|
126
|
+
else
|
127
|
+
false
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
##
|
132
|
+
# @private Checks if a given file exists on Google Cloud Storage.
|
133
|
+
#
|
134
|
+
# @param [String] file_path Path of the file on Google Cloud Storage.
|
135
|
+
#
|
136
|
+
# @return [Boolean]
|
137
|
+
def self.on_gcs? file_path
|
138
|
+
return false unless proj_id
|
139
|
+
get_file(file_path) != nil
|
140
|
+
end
|
141
|
+
|
142
|
+
##
|
143
|
+
# @private Downloads a given file from Google Cloud Storage.
|
144
|
+
#
|
145
|
+
# @param [String] path Path to the file.
|
146
|
+
def self.copy_to_host path
|
147
|
+
return unless proj_id
|
148
|
+
file = get_file path
|
149
|
+
folder = extract_dir path
|
150
|
+
begin
|
151
|
+
FileUtils.mkpath(folder) unless Dir.exist?(folder)
|
152
|
+
file.download path
|
153
|
+
rescue
|
154
|
+
puts "Could not download #{file.name}." if file
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
##
|
159
|
+
# @private Extracts the parent directory from a file path
|
160
|
+
#
|
161
|
+
# @param [String] path Path of the file.
|
162
|
+
#
|
163
|
+
# @return [String]
|
164
|
+
def self.extract_dir path
|
165
|
+
parts = path.split("/")
|
166
|
+
parts.map { |x| x != parts.last ? x : nil }.join("/")
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module Google
|
16
|
+
module Cloud
|
17
|
+
module Gemserver
|
18
|
+
##
|
19
|
+
# The version of the google-cloud-gemserver gem.
|
20
|
+
VERSION = "0.1.0".freeze
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
|
17
|
+
module PatchedConfiguration
|
18
|
+
##
|
19
|
+
# Monkeypatch to support Cloud SQL by returning the necessary settings.
|
20
|
+
def database_connection_config
|
21
|
+
if self[:db_adapter] == "cloud_sql"
|
22
|
+
{ adapter: "mysql2" }.merge(self[:db_connection_options])
|
23
|
+
else
|
24
|
+
super
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
Gemstash::Configuration.send :prepend, PatchedConfiguration
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
require "active_support/core_ext/module/aliasing"
|
18
|
+
|
19
|
+
module PatchedDependencies
|
20
|
+
##
|
21
|
+
# Monkeypatch to run a gem syncing service in the background after fetches.
|
22
|
+
#
|
23
|
+
# @param [Array] gems An array of gems to fetch.
|
24
|
+
#
|
25
|
+
# @return [Array]
|
26
|
+
def fetch gems
|
27
|
+
fetched = super gems
|
28
|
+
Google::Cloud::Gemserver::Backend::StorageSync.run
|
29
|
+
fetched
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
Gemstash::Dependencies.send :prepend, PatchedDependencies
|
data/lib/patched/env.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "active_support/core_ext/file/atomic"
|
17
|
+
require "active_support/core_ext/module/aliasing"
|
18
|
+
|
19
|
+
module PatchedEnv
|
20
|
+
##
|
21
|
+
# Monkey patch to support Cloud SQL as an adapter
|
22
|
+
def db
|
23
|
+
return @db if @db
|
24
|
+
|
25
|
+
@db = if config[:db_adapter] == "cloud_sql"
|
26
|
+
connection = Sequel.connect config.database_connection_config
|
27
|
+
migrate_cloud_sql connection
|
28
|
+
connection
|
29
|
+
else
|
30
|
+
super
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def migrate_cloud_sql database
|
35
|
+
Sequel.extension :migration
|
36
|
+
lib_dir = Gem::Specification.find_by_name("gemstash").lib_dirs_glob
|
37
|
+
m_dir = "#{lib_dir}/gemstash/migrations"
|
38
|
+
Sequel::Migrator.run database, m_dir, use_transactions: false
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
Gemstash::Env.send :prepend, PatchedEnv
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
|
18
|
+
module PatchedGemPusher
|
19
|
+
##
|
20
|
+
# Monkeypatch to run a gem syncing service in the background after pushing a
|
21
|
+
# gem to the gemserver.
|
22
|
+
def serve
|
23
|
+
super
|
24
|
+
Google::Cloud::Gemserver::Backend::StorageSync.run
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Gemstash::GemPusher.send :prepend, PatchedGemPusher
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
|
18
|
+
module PatchedGemYanker
|
19
|
+
##
|
20
|
+
# Monkeypatch to run a gem syncing service in the background after yanking
|
21
|
+
# a gem from the gemserver.
|
22
|
+
def serve
|
23
|
+
super
|
24
|
+
Google::Cloud::Gemserver::Backend::StorageSync.run
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Gemstash::GemYanker.send :prepend, PatchedGemYanker
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
require "filelock"
|
18
|
+
|
19
|
+
module PatchedResource
|
20
|
+
##
|
21
|
+
# Monkeypatch to delete a file from both the local file system and Google
|
22
|
+
# Cloud Storage. Done atomically to prevent circular file syncing where
|
23
|
+
# files never get deleted.
|
24
|
+
#
|
25
|
+
# @param [String] key Name of the gem to delete.
|
26
|
+
def delete key
|
27
|
+
file = content_filename key
|
28
|
+
return unless File.exist?(file) && File.exist?(properties_filename)
|
29
|
+
Filelock file do
|
30
|
+
super
|
31
|
+
Google::Cloud::Gemserver::GCS.delete_file file
|
32
|
+
Google::Cloud::Gemserver::GCS.delete_file properties_filename
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
Gemstash::Resource.send :prepend, PatchedResource
|
data/lib/patched/web.rb
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# Copyright 2017 Google Inc.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "gemstash"
|
16
|
+
require "google/cloud/gemserver"
|
17
|
+
|
18
|
+
##
|
19
|
+
# Monkeypatch gemstash to handle gemserver specific endpoints.
|
20
|
+
Gemstash::Web.class_eval do
|
21
|
+
##
|
22
|
+
# Displays statistics on the currently deployed gemserver such as private
|
23
|
+
# gems, cached gems, gemserver creation time, etc.
|
24
|
+
post "/api/v1/stats" do
|
25
|
+
auth = Google::Cloud::Gemserver::Authentication.new
|
26
|
+
if auth.validate_token request.env["HTTP_AUTHORIZATION"]
|
27
|
+
content_type "application/json;charset=UTF-8"
|
28
|
+
Google::Cloud::Gemserver::Backend::Stats.new.run
|
29
|
+
else
|
30
|
+
halt 401, "Unauthorized operation."
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
##
|
35
|
+
# Creates a key used for installing or pushing gems to the gemserver
|
36
|
+
# with given permissions. By default, a key with all permissions is created.
|
37
|
+
post "/api/v1/key" do
|
38
|
+
auth = Google::Cloud::Gemserver::Authentication.new
|
39
|
+
if auth.validate_token request.env["HTTP_AUTHORIZATION"]
|
40
|
+
key = Google::Cloud::Gemserver::Backend::Key.create_key params["permissions"]
|
41
|
+
content_type "application/json;charset=UTF-8"
|
42
|
+
"Generated key: #{key}"
|
43
|
+
else
|
44
|
+
halt 401, "Unauthorized operation."
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
##
|
49
|
+
# Deletes a key.
|
50
|
+
put "/api/v1/key" do
|
51
|
+
auth = Google::Cloud::Gemserver::Authentication.new
|
52
|
+
if auth.validate_token request.env["HTTP_AUTHORIZATION"]
|
53
|
+
res = Google::Cloud::Gemserver::Backend::Key.delete_key params["key"]
|
54
|
+
content_type "application/json;charset=UTF-8"
|
55
|
+
if res
|
56
|
+
"Deleted key #{params["key"]} successfully."
|
57
|
+
else
|
58
|
+
"Deleting key #{params["key"]} failed."
|
59
|
+
end
|
60
|
+
else
|
61
|
+
halt 401, "Unauthorized operation."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|