gclouder 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +104 -0
  3. data/bin/gclouder +7 -0
  4. data/lib/gclouder/config/arguments.rb +35 -0
  5. data/lib/gclouder/config/cli_args.rb +77 -0
  6. data/lib/gclouder/config/cluster.rb +66 -0
  7. data/lib/gclouder/config/defaults.rb +35 -0
  8. data/lib/gclouder/config/files/project.rb +24 -0
  9. data/lib/gclouder/config/project.rb +34 -0
  10. data/lib/gclouder/config/resource_representations.rb +31 -0
  11. data/lib/gclouder/config_loader.rb +25 -0
  12. data/lib/gclouder/config_section.rb +26 -0
  13. data/lib/gclouder/dependencies.rb +11 -0
  14. data/lib/gclouder/gcloud.rb +39 -0
  15. data/lib/gclouder/gsutil.rb +25 -0
  16. data/lib/gclouder/header.rb +9 -0
  17. data/lib/gclouder/helpers.rb +77 -0
  18. data/lib/gclouder/logging.rb +181 -0
  19. data/lib/gclouder/mappings/argument.rb +31 -0
  20. data/lib/gclouder/mappings/file.rb +31 -0
  21. data/lib/gclouder/mappings/property.rb +31 -0
  22. data/lib/gclouder/mappings/resource_representation.rb +31 -0
  23. data/lib/gclouder/monkey_patches/array.rb +19 -0
  24. data/lib/gclouder/monkey_patches/boolean.rb +12 -0
  25. data/lib/gclouder/monkey_patches/hash.rb +44 -0
  26. data/lib/gclouder/monkey_patches/ipaddr.rb +10 -0
  27. data/lib/gclouder/monkey_patches/string.rb +30 -0
  28. data/lib/gclouder/resource.rb +63 -0
  29. data/lib/gclouder/resource_cleaner.rb +58 -0
  30. data/lib/gclouder/resources/compute/addresses.rb +108 -0
  31. data/lib/gclouder/resources/compute/bgp-vpns.rb +220 -0
  32. data/lib/gclouder/resources/compute/disks.rb +99 -0
  33. data/lib/gclouder/resources/compute/firewall_rules.rb +82 -0
  34. data/lib/gclouder/resources/compute/instances.rb +147 -0
  35. data/lib/gclouder/resources/compute/networks/subnets.rb +104 -0
  36. data/lib/gclouder/resources/compute/networks.rb +110 -0
  37. data/lib/gclouder/resources/compute/project_info/ssh_keys.rb +171 -0
  38. data/lib/gclouder/resources/compute/routers.rb +83 -0
  39. data/lib/gclouder/resources/compute/vpns.rb +199 -0
  40. data/lib/gclouder/resources/container/clusters.rb +257 -0
  41. data/lib/gclouder/resources/container/node_pools.rb +193 -0
  42. data/lib/gclouder/resources/dns.rb +390 -0
  43. data/lib/gclouder/resources/logging/sinks.rb +98 -0
  44. data/lib/gclouder/resources/project/iam_policy_binding.rb +293 -0
  45. data/lib/gclouder/resources/project.rb +85 -0
  46. data/lib/gclouder/resources/project_id.rb +71 -0
  47. data/lib/gclouder/resources/pubsub/subscriptions.rb +100 -0
  48. data/lib/gclouder/resources/pubsub/topics.rb +95 -0
  49. data/lib/gclouder/resources/storagebuckets.rb +103 -0
  50. data/lib/gclouder/resources/validate/global.rb +27 -0
  51. data/lib/gclouder/resources/validate/local.rb +68 -0
  52. data/lib/gclouder/resources/validate/region.rb +28 -0
  53. data/lib/gclouder/resources/validate/remote.rb +78 -0
  54. data/lib/gclouder/resources.rb +148 -0
  55. data/lib/gclouder/shell.rb +71 -0
  56. data/lib/gclouder/version.rb +5 -0
  57. data/lib/gclouder.rb +278 -0
  58. metadata +102 -0
@@ -0,0 +1,110 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ module GClouder
4
+ module Resources
5
+ module Compute
6
+ module Networks
7
+ include GClouder::Config::CLIArgs
8
+ include GClouder::Config::Project
9
+ include GClouder::Logging
10
+ include GClouder::Resource::Cleaner
11
+
12
+ def self.header(stage = :ensure)
13
+ info "[#{stage}] compute / network", indent: 1, title: true
14
+ end
15
+
16
+ def self.ensure
17
+ return if Local.list.empty?
18
+ header
19
+
20
+ Local.list.each do |region, networks|
21
+ info region, indent: 2, heading: true
22
+ info
23
+ networks.each do |network|
24
+ Network.ensure(network["name"])
25
+ end
26
+ end
27
+ end
28
+
29
+ def self.validate
30
+ return if Local.list.empty?
31
+ header :validate
32
+ Local.validate
33
+ end
34
+
35
+ module Local
36
+ include GClouder::Config::CLIArgs
37
+ include GClouder::Config::Project
38
+ include GClouder::Logging
39
+
40
+ def self.list
41
+ { "global" => resources }.delete_if { |_k, v| v.empty? }
42
+ end
43
+
44
+ def self.resources
45
+ merged = networks.merge(subnet_networks)["global"]
46
+ return [] unless merged
47
+ merged.uniq { |network| network["name"] }
48
+ end
49
+
50
+ def self.validate
51
+ return if list.empty?
52
+
53
+ failure = false
54
+
55
+ list.each do |region, networks|
56
+ info region, indent: 2, heading: true
57
+ networks.each do |network|
58
+ info network["name"], indent: 3, heading: true
59
+ if !network["name"].is_a?(String)
60
+ bad "#{network['name']} is incorrect type #{network['name'].class}, should be: String", indent: 4
61
+ failure = true
62
+ end
63
+
64
+ if cli_args[:debug] || !cli_args[:output_validation]
65
+ good "network is a String", indent: 4
66
+ end
67
+ end
68
+ end
69
+
70
+ fatal "\nerror: validation failure" if failure
71
+ end
72
+
73
+ def self.networks
74
+ GClouder::Resources::Global.instances(path: ["networks"])
75
+ end
76
+
77
+ def self.subnet_networks
78
+ GClouder::Resources::Compute::Networks::Subnets::Local.networks
79
+ end
80
+ end
81
+
82
+ module Remote
83
+ def self.list
84
+ { "global" => instances.fetch("global", []).map { |network| { "name" => network["name"] } } }.delete_if { |_k, v| v.empty? }
85
+ end
86
+
87
+ def self.instances
88
+ Resources::Remote.instances(
89
+ path: ["compute", "networks"],
90
+ ignore_keys: ["auto_create_subnetworks", "subnetworks", "x_gcloud_mode", "range"],
91
+ skip_instances: { "name" => /^default$/ },
92
+ )
93
+ end
94
+ end
95
+
96
+ module Network
97
+ include GClouder::GCloud
98
+
99
+ def self.ensure(network)
100
+ Resource.ensure :"compute networks", network, "--mode custom"
101
+ end
102
+
103
+ def self.purge(network)
104
+ Resource.purge :"compute networks", network
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,171 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ module GClouder
4
+ module Resources
5
+ module Compute
6
+ module ProjectInfo
7
+ module SSHKeys
8
+ include GClouder::Logging
9
+ include GClouder::Config::Project
10
+ include GClouder::Config::CLIArgs
11
+
12
+ def self.header(stage = :ensure)
13
+ info "[#{stage}] compute / metadata / ssh_keys", title: true
14
+ end
15
+
16
+ def self.clean
17
+ return unless project.key?("users")
18
+ header :clean
19
+ end
20
+
21
+ def self.check
22
+ end
23
+
24
+ def self.validate
25
+ return if Local.data.empty?
26
+ header :validate
27
+ Local.validate
28
+ end
29
+
30
+ def self.ensure
31
+ return unless project.key?("users")
32
+ header :ensure
33
+
34
+ info "global", heading: true, indent: 2
35
+ info
36
+
37
+ Local.data.each do |user_data|
38
+ description = user_data[:description]
39
+
40
+ user = Remote.data.find { |entry| entry[:description] == description }
41
+
42
+ # user doesn't exist, add it..
43
+ if user.nil?
44
+ add description
45
+ next
46
+ end
47
+
48
+ # user exists but has been modified
49
+ if user_data[:key] != user[:key]
50
+ change description
51
+ next
52
+ end
53
+
54
+ # user exists and is the same
55
+ good description
56
+ end
57
+
58
+ Remote.data.each do |user_data|
59
+ description = user_data[:description]
60
+
61
+ next if Local.data.find { |entry| entry[:description] == description }
62
+
63
+ # user isn't defined locally, remove it
64
+ remove description, indent: 3
65
+ end
66
+
67
+ return if Local.list == Remote.list
68
+ return if cli_args[:dry_run]
69
+
70
+ Key.ensure(Local.list)
71
+ end
72
+
73
+ module Local
74
+ include GClouder::Config::Project
75
+ include GClouder::Logging
76
+
77
+ def self.list
78
+ return [] unless project.key?("users")
79
+
80
+ project["users"].sort
81
+ end
82
+
83
+ def self.data
84
+ list.map do |line|
85
+ components = line.split
86
+ user, type = components[0].split(":")
87
+ key = components[1]
88
+ description = components.length >= 2 ? components[2] : components[0]
89
+
90
+ { key: key, type: type, user: user, description: description }
91
+ end
92
+ end
93
+
94
+ def self.validate
95
+ return if data.empty?
96
+
97
+ info "global", heading: true, indent: 2
98
+
99
+ data.each do |entry|
100
+ info
101
+ info entry[:description], indent: 3
102
+
103
+ if entry[:user].is_a?(String)
104
+ good "user is a String (#{entry[:user]})", indent: 4
105
+ else
106
+ bad "user is a String (#{entry[:user]})", indent: 4
107
+ end
108
+
109
+ if entry[:key].is_a?(String)
110
+ good "key is a String (#{entry[:key].reverse.truncate(20).reverse})", indent: 4
111
+ else
112
+ bad "key isn't a String (#{entry[:key]})", indent: 4
113
+ end
114
+
115
+ if entry[:type].is_a?(String)
116
+ good "type is a String (#{entry[:type]})", indent: 4
117
+ else
118
+ bad "type isn't a String (#{entry[:type]})", indent: 4
119
+ end
120
+
121
+ # check if description exists for key
122
+ # output useruser
123
+ # output key.truncate
124
+ # output description
125
+ end
126
+ end
127
+ end
128
+
129
+ module Remote
130
+ include GClouder::GCloud
131
+
132
+ def self.list
133
+ keys = metadata.dig("items")
134
+
135
+ return [] unless keys
136
+ return [] if keys.empty?
137
+
138
+ keys.delete_if { |h| h["key"] != "sshKeys" }
139
+ keys[0]["value"].split("\n").delete_if { |key| key =~ /^gke-/ }.sort
140
+ end
141
+
142
+ def self.data
143
+ list.map do |line|
144
+ components = line.split
145
+ user, type = components[0].split(":")
146
+ key = components[1]
147
+ description = components.length >= 2 ? components[2] : components[0]
148
+
149
+ { key: key, type: type, user: user, description: description }
150
+ end
151
+ end
152
+
153
+ def self.metadata
154
+ gcloud("compute project-info describe | jq -r '.commonInstanceMetadata'", force: true)
155
+ end
156
+ end
157
+ end
158
+
159
+ module Key
160
+ include GClouder::Config::Project
161
+ include GClouder::GCloud
162
+
163
+ def self.ensure(list)
164
+ list = list.join("\n") if list.is_a?(Array)
165
+ gcloud("compute project-info add-metadata --metadata sshKeys=\"#{list}\"")
166
+ end
167
+ end
168
+ end
169
+ end
170
+ end
171
+ end
@@ -0,0 +1,83 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ module GClouder
4
+ module Resources
5
+ module Compute
6
+ module Routers
7
+ include GClouder::Config::CLIArgs
8
+ include GClouder::Config::Project
9
+ include GClouder::Logging
10
+ include GClouder::Helpers
11
+ include GClouder::Config::Arguments
12
+ include GClouder::Resource::Cleaner
13
+
14
+ def self.header(stage = :ensure)
15
+ info "[#{stage}] compute / router", title: true
16
+ end
17
+
18
+ def self.validate
19
+ return if Local.list.empty?
20
+ header :validate
21
+ Local.validate
22
+ end
23
+
24
+ def self.ensure
25
+ return if Local.list.empty?
26
+ header
27
+
28
+ Local.list.each do |region, routers|
29
+ next if routers.empty?
30
+ info region, heading: true, indent: 2
31
+
32
+ routers.each do |router|
33
+ info
34
+ Router.ensure(region, router["name"], hash_to_args(router))
35
+ end
36
+ end
37
+ end
38
+
39
+ module Local
40
+ include GClouder::Logging
41
+
42
+ def self.section
43
+ ["compute", "routers"]
44
+ end
45
+
46
+ def self.list
47
+ Resources::Region.instances(path: ["routers"])
48
+ end
49
+
50
+ def self.validate
51
+ Resources::Validate::Region.instances(
52
+ list,
53
+ required_keys: GClouder::Config::Arguments.required(section),
54
+ permitted_keys: GClouder::Config::Arguments.permitted(section),
55
+ # ignore ASN until Fixnums are supported
56
+ ignore_keys: [ "asn" ],
57
+ )
58
+ end
59
+ end
60
+
61
+ module Remote
62
+ def self.list
63
+ Resources::Remote.instances(
64
+ path: ["compute", "routers"],
65
+ )
66
+ end
67
+ end
68
+
69
+ module Router
70
+ include GClouder::Resource
71
+
72
+ def self.ensure(region, router, args)
73
+ Resource.ensure :"compute routers", router, "--region #{region} #{args}"
74
+ end
75
+
76
+ def self.purge(region, router)
77
+ Resource.purge :"compute routers", router, "--region #{region}"
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,199 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ module GClouder
4
+ module Resources
5
+ module Compute
6
+ module VPNs
7
+ include GClouder::GCloud
8
+ include GClouder::Shell
9
+ include GClouder::Logging
10
+ include GClouder::Config::Project
11
+ include GClouder::Config::CLIArgs
12
+ include GClouder::Resource::Cleaner
13
+
14
+ def self.header(stage = :ensure)
15
+ info "[#{stage}] compute / vpns", indent: 1, title: true
16
+ end
17
+
18
+ def self.validate
19
+ return if Local.list.empty?
20
+ header :validate
21
+ Local.validate
22
+ end
23
+
24
+ def self.dir
25
+ cli_args[:keys_dir] || File.join(ENV["HOME"], "keys")
26
+ end
27
+
28
+ def self.ensure
29
+ return if Local.list.empty?
30
+
31
+ header
32
+
33
+ Local.list.each do |region, instances|
34
+ info region, indent: 2, heading: true
35
+ info
36
+
37
+ instances.each do |vpn|
38
+ skip_vpn = false
39
+
40
+ # if 'shared_secret' key is set, use it
41
+ # if not, fall back to trying to read the secret from an environment variable, the name
42
+ # of which is provided by the 'shared_secret_env_var' key
43
+ unless vpn.key?("shared_secret") || vpn.key?("shared_secret_env_var") || vpn.key?("shared_secret_file")
44
+ if cli_args[:dry_run]
45
+ warning "skipping resource since no shared secret found for VPN and this is a dry run"
46
+ skip_vpn = true
47
+ else
48
+ fatal "shared_secret_env_var or shared_secret must be set for region/vpn: #{region}/#{vpn["name"]}"
49
+ end
50
+ end
51
+
52
+ vpn["shared_secret"] = if vpn.key?("shared_secret") && !vpn["shared_secret"].empty? && !vpn["shared_secret"].nil?
53
+ vpn["shared_secret"]
54
+ else
55
+ ENV[vpn["shared_secret_env_var"]] if vpn["shared_secret_env_var"]
56
+ end
57
+
58
+ # this overrides the above for now..
59
+ if vpn.key?("shared_secret_file")
60
+ config_file = File.join(dir, vpn["shared_secret_file"])
61
+
62
+ if !File.exists?(config_file)
63
+ fatal "shared_secret_file specified for vpn but no file found for region/vpn: #{region}/#{vpn["name"]}"
64
+ end
65
+
66
+ vpn["shared_secret"] = File.read(config_file)
67
+ end
68
+
69
+ vpn.delete("shared_secret_env_var") if vpn.key?("shared_secret_env_var")
70
+ vpn.delete("shared_secret_file") if vpn.key?("shared_secret_file")
71
+
72
+ required_params = %w(peer_address shared_secret ike_version remote_traffic_selector
73
+ local_traffic_selector target_vpn_gateway network)
74
+
75
+ required_params.each do |param|
76
+ fatal "no #{param} defined for region/vpn: #{region}/#{vpn}" unless vpn.key?(param)
77
+
78
+ # FIXME: change once hashie has been ripped out
79
+ if vpn[param].nil?
80
+ if cli_args[:dry_run]
81
+ warning "no #{param} defined for region/vpn: #{vpn["name"]} [#{region}]"
82
+ skip_vpn = true
83
+ else
84
+ fatal "no #{param} defined for region/vpn: #{vpn["name"]} [#{region}]"
85
+ end
86
+ end
87
+
88
+ if vpn[param].is_a?(String)
89
+ if cli_args[:dry_run]
90
+ warning "no #{param} defined for region/vpn: #{vpn["name"]} [#{region}]" if vpn[param].empty?
91
+ skip_vpn = true
92
+ else
93
+ fatal "no #{param} defined for region/vpn: #{vpn["name"]} [#{region}]" if vpn[param].empty?
94
+ end
95
+ end
96
+ end
97
+
98
+ next if skip_vpn && !cli_args[:dry_run]
99
+
100
+ VPN.create(region, vpn["name"], vpn)
101
+ end
102
+ end
103
+ end
104
+
105
+ module Local
106
+ def self.section
107
+ ["vpns"]
108
+ end
109
+
110
+ def self.list
111
+ Resources::Region.instances(path: section).delete_if { |_k, v| v.empty? }
112
+ end
113
+
114
+ def self.validate
115
+ Resources::Validate::Region.instances(
116
+ list,
117
+ required_keys: GClouder::Config::Arguments.required(["compute", "vpn-tunnels"]),
118
+ permitted_keys: GClouder::Config::Arguments.permitted(["compute", "vpn-tunnels"]),
119
+ ignore_keys: ["ike_version", "shared_secret_file", "network"]
120
+ )
121
+ end
122
+ end
123
+
124
+ module Remote
125
+ def self.list
126
+ Resources::Remote.instances(path: %w(compute vpn-tunnels))
127
+ end
128
+ end
129
+
130
+ module VPN
131
+ include GClouder::GCloud
132
+ include GClouder::Shell
133
+ include GClouder::Logging
134
+ include GClouder::Helpers
135
+ include GClouder::Config::CLIArgs
136
+
137
+ def self.create(region, vpn, vpn_config)
138
+ network = vpn_config['network']
139
+ Resource.ensure :"compute target-vpn-gateways", vpn_config["target_vpn_gateway"],
140
+ "--network #{network} --region #{region}"
141
+
142
+ vpn_config.delete("network")
143
+
144
+ return if cli_args[:dry_run]
145
+
146
+ ip_data = gcloud("--format json compute addresses describe vpn-#{vpn} --region=#{region}", force: true)
147
+
148
+ unless ip_data.key?("address")
149
+ fatal "could not find address for static ip with key: vpn-#{vpn} (is key allocated in project config?)"
150
+ end
151
+
152
+ address = ip_data["address"]
153
+
154
+ Resource.ensure :"compute forwarding-rules",
155
+ "#{vpn}-esp",
156
+ "--region #{region} \
157
+ --ip-protocol ESP \
158
+ --address #{address} \
159
+ --target-vpn-gateway=#{vpn_config['target_vpn_gateway']}",
160
+ silent: true
161
+
162
+ Resource.ensure :"compute forwarding-rules",
163
+ "#{vpn}-udp500",
164
+ "--region #{region} \
165
+ --ip-protocol UDP \
166
+ --ports 500 \
167
+ --address #{address} \
168
+ --target-vpn-gateway=#{vpn_config['target_vpn_gateway']}",
169
+ silent: true
170
+
171
+ Resource.ensure :"compute forwarding-rules",
172
+ "#{vpn}-udp4500",
173
+ "--region #{region} --ip-protocol UDP --ports 4500 --address #{address} \
174
+ --target-vpn-gateway=#{vpn_config['target_vpn_gateway']}",
175
+ silent: true
176
+
177
+ Resource.ensure :"compute vpn-tunnels", vpn,
178
+ "--region=#{region} #{hash_to_args(vpn_config)}",
179
+ silent: true
180
+
181
+ vpn_config["remote_traffic_selector"].each_with_index do |range, index|
182
+ Resource.ensure :"compute routes",
183
+ "route-#{vpn}-#{index}",
184
+ "--network=#{network} --next-hop-vpn-tunnel=#{vpn} \
185
+ --next-hop-vpn-tunnel-region=#{region} --destination-range=#{range}",
186
+ silent: true
187
+ end
188
+
189
+ GClouder::Resources::Compute::FirewallRules::Rule.ensure("vpn-#{vpn}-icmp", {
190
+ "network" => network,
191
+ "source-ranges" => vpn_config["remote_traffic_selector"],
192
+ "allow" => "icmp"
193
+ }, silent: true)
194
+ end
195
+ end
196
+ end
197
+ end
198
+ end
199
+ end