gclouder 0.1.1

data/lib/gclouder/resources/compute/bgp-vpns.rb

@@ -0,0 +1,220 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Compute
+      module BGPVPNs
+        include GClouder::GCloud
+        include GClouder::Shell
+        include GClouder::Logging
+        include GClouder::Config::Project
+        include GClouder::Config::CLIArgs
+        include GClouder::Resource::Cleaner
+
+        module Cleaner
+          def self.custom
+            Proc.new do |local_resources, remote_resource|
+              local_resources.select { |r| "bgp-vpn-#{r['name']}" == remote_resource }.length > 0
+            end
+          end
+        end
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] compute / bgp-vpns", indent: 1, title: true
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+
+          header
+
+          Local.list.each do |region, instances|
+            info region, indent: 2, heading: true
+
+            instances.each do |vpn|
+              set_shared_secret(region, vpn)
+              info
+              BGPVPN.create(region, vpn)
+            end
+          end
+        end
+
+        def self.dir
+          cli_args[:keys_dir] || File.join(ENV["HOME"], "keys")
+        end
+
+        def self.set_shared_secret(region, vpn)
+          # if 'shared_secret' key is set, use it
+          # if not, fall back to trying to read the secret from an environment variable, the name
+          # of which is provided by the 'shared_secret_env_var' key
+          unless vpn.key?("shared_secret") || vpn.key?("shared_secret_env_var") || vpn.key?("shared_secret_file")
+            if cli_args[:dry_run]
+              warning "no shared secret found for VPN"
+            else
+              fatal "shared_secret_env_var or shared_secret must be set for region/vpn: #{region}/#{vpn["name"]}"
+              return false
+            end
+          end
+
+          vpn["shared_secret"] = if vpn.key?("shared_secret") && !vpn["shared_secret"].empty? && !vpn["shared_secret"].nil?
+            vpn["shared_secret"]
+          else
+            ENV[vpn["shared_secret_env_var"]] if vpn["shared_secret_env_var"]
+          end
+
+          # this overrides the above for now..
+          if vpn.key?("shared_secret_file")
+            config_file = File.join(dir, vpn["shared_secret_file"])
+
+            if !File.exists?(config_file)
+              fatal "shared_secret_file specified for vpn but no file found for region/vpn: #{region}/#{vpn["name"]}"
+            end
+
+            vpn["shared_secret"] = File.read(config_file)
+          end
+        end
+
+        module Local
+          def self.list
+            Resources::Region.instances(
+              path: %w{bgp-vpns}
+            ).delete_if { |_k, v| v.empty? }
+          end
+
+          def self.validate
+            # FIXME: better validation
+            Resources::Validate::Region.instances(
+              list,
+              required_keys:  GClouder::Config::Arguments.required(["compute", "vpn-tunnels"]),
+              permitted_keys: GClouder::Config::Arguments.permitted(["compute", "vpn-tunnels"]),
+              ignore_keys:    ["ike_version", "shared_secret", "address", "target_vpn_gateway", "bgp", "shared_secret_file", "network"]
+            )
+          end
+        end
+
+        module Remote
+          def self.list
+            # FIXME: should we be listing tunnels or gateways?
+            Resources::Remote.instances(path: %w(compute target-vpn-gateways))
+          end
+        end
+
+        module BGPVPN
+          include GClouder::GCloud
+          include GClouder::Shell
+          include GClouder::Logging
+          include GClouder::Helpers
+          include GClouder::Config::CLIArgs
+
+          def self.vpn_address(region, vpn)
+            response = gcloud("--format json compute addresses describe #{vpn['address']} --region=#{region}", force: true)
+
+            unless response.key?("address")
+              fatal "could not find address for static ip with key: #{vpn['address']} (is key allocated in project config?)"
+            end
+
+            response["address"]
+          end
+
+          def self.create(region, vpn)
+            network = vpn['network']
+
+            info "#{vpn['name']} (bgp-vpn-#{vpn['name']})", indent: 3
+
+            # check to see if router exists, if it doesn't then assume we need to create interface and bgp peer
+            configure_router = !Resource.resource?("compute routers", "bgp-vpn-#{vpn['name']}", silent: true)
+
+            # router
+            Resource.ensure :"compute routers",
+                            "bgp-vpn-#{vpn['name']}",
+                            "--region #{region} \
+                            --network #{network} \
+                            --asn #{vpn['bgp']['local_asn']}",
+                            extra_info: "(router)",
+                            indent: 4
+
+            # VPN gateway
+            Resource.ensure :"compute target-vpn-gateways",
+                            "bgp-vpn-#{vpn["name"]}",
+                            "--network #{network} \
+                            --region #{region}",
+                            extra_info: "(gateway)",
+                            indent: 4
+
+            address = cli_args[:dry_run] ? "<automatic>" : vpn_address(region, vpn)
+
+            # forwarding rules
+            Resource.ensure :"compute forwarding-rules",
+                            "bgp-vpn-#{vpn['name']}-esp",
+                            "--region #{region} \
+                            --ip-protocol ESP \
+                            --address #{address} \
+                            --target-vpn-gateway bgp-vpn-#{vpn['name']}",
+                            extra_info: "(forwarding-rule)",
+                            indent: 4
+
+            Resource.ensure :"compute forwarding-rules",
+                            "bgp-vpn-#{vpn['name']}-udp500",
+                            "--region #{region} \
+                            --ip-protocol UDP \
+                            --ports 500 \
+                            --address #{address} \
+                            --target-vpn-gateway bgp-vpn-#{vpn['name']}",
+                            extra_info: "(forwarding-rule)",
+                            indent: 4
+
+            Resource.ensure :"compute forwarding-rules",
+                            "bgp-vpn-#{vpn['name']}-udp4500",
+                            "--region #{region} --ip-protocol UDP --ports 4500 --address #{address} \
+                            --target-vpn-gateway bgp-vpn-#{vpn['name']}",
+                            extra_info: "(forwarding-rule)",
+                            indent: 4
+
+            # tunnel
+            Resource.ensure :"compute vpn-tunnels", "bgp-vpn-#{vpn['name']}",
+                            "--region #{region} \
+                            --peer-address #{vpn['peer_address']} \
+                            --ike-version #{vpn['ike_version']} \
+                            --router bgp-vpn-#{vpn['name']} \
+                            --target-vpn-gateway bgp-vpn-#{vpn['name']} \
+                            --shared-secret #{vpn['shared_secret']}",
+                            extra_info: "(tunnel)",
+                            indent: 4
+
+            if configure_router
+              # router interface
+              gcloud("compute routers add-interface bgp-vpn-#{vpn['name']} \
+                     --region #{region} \
+                     --interface-name bgp-vpn-interface-#{vpn['name']} \
+                     --vpn-tunnel bgp-vpn-#{vpn['name']} \
+                     --mask-length #{vpn['bgp']['mask']} \
+                     --ip-address #{vpn['bgp']['local_address']}",
+                     failure: false)
+              add "bgp-vpn-#{vpn['name']} (router interface)", indent: 4
+
+              # bgp peer
+              gcloud("compute routers add-bgp-peer bgp-vpn-#{vpn['name']} \
+                     --region #{region} \
+                     --interface bgp-vpn-interface-#{vpn['name']} \
+                     --advertised-route-priority #{vpn['bgp']['priority']} \
+                     --peer-asn #{vpn['bgp']['peer_asn']} \
+                     --peer-ip-address #{vpn['bgp']['peer_address']} \
+                     --peer-name #{vpn['name']}",
+                     failure: false)
+              add "bgp-vpn-#{vpn['name']} (bgp peer)", indent: 4
+            else
+              good "bgp-vpn-#{vpn['name']} (router interface)", indent: 4
+              good "bgp-vpn-#{vpn['name']} (bgp peer)", indent: 4
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/compute/disks.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Compute
+      module Disks
+        include GClouder::Config::CLIArgs
+        include GClouder::Config::Project
+        include GClouder::Logging
+        include GClouder::Config::Arguments
+        include GClouder::Resource::Cleaner
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] compute / disks", title: true, indent: 1
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+
+          Local.list.each do |region, disks|
+            next if disks.empty?
+            info region, indent: 2, heading: true
+            info
+            disks.each do |disk|
+              Disk.ensure(disk["name"], disk["zone"], disk["size"], disk["type"])
+            end
+          end
+        end
+
+        def self.check
+          Remote.list.each do |region, disks|
+            disks.each do |disk, config|
+              local_config = Local.list[region][disk]
+              next unless local_config
+              next if local_config == config
+              info "[compute disks] local resource definition differs from immutable remote resource: #{disk}"
+              info "# local config"
+              ap local_config
+              info "# remote config"
+              ap config
+              fatal ""
+            end
+          end
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        module Local
+          include GClouder::Logging
+
+          def self.section
+            ["compute", "disks"]
+          end
+
+          def self.list
+            Resources::Region.instances(path: %w(compute disks))
+          end
+
+          def self.validate
+            Resources::Validate::Region.instances(
+              list,
+              required_keys:  GClouder::Config::Arguments.required(section),
+              permitted_keys: GClouder::Config::Arguments.permitted(section),
+              ignore_keys:    ["size"]
+            )
+          end
+        end
+
+        module Remote
+          def self.list
+            vm_disk_pattern = GClouder::Resources::Compute::Instances::Local.instance_names.map{ |disk| "^#{disk}$" }.join("|")
+
+            Resources::Remote.instances(
+              path:           ["compute", "disks"],
+              skip_instances: { "name" => /^gke|#{vm_disk_pattern}/ },
+            )
+          end
+        end
+
+        module Disk
+          include GClouder::Resource
+
+          def self.ensure(disk, zone, size, type)
+            Resource.ensure :"compute disks", disk, "--zone #{zone} --size #{size} --type #{type}"
+          end
+
+          def self.purge(disk, args)
+            Resource.purge :"compute disks", disk, args
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/compute/firewall_rules.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Compute
+      module FirewallRules
+        include GClouder::Config::CLIArgs
+        include GClouder::Logging
+        include GClouder::GCloud
+        include GClouder::Resource::Cleaner
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] compute / firewall-rules", indent: 1, title: true
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+          Local.list.each do |region, rules|
+            info region, heading: true, indent: 2
+            info
+            rules.each do |rule|
+              Rule.ensure(rule["name"], rule)
+            end
+          end
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        def self.check
+        end
+
+        module Local
+          include GClouder::Config::Project
+          include GClouder::Logging
+
+          def self.validate
+            info "global", heading: true, indent: 2
+            Resources::Validate::Global.instances(
+              list,
+              required_keys:  GClouder::Config::Arguments.required(%w(compute firewall-rules)),
+              permitted_keys: GClouder::Config::Arguments.permitted(%w(compute firewall-rules)),
+              ignore_keys: ["internal-icmp"]
+            )
+          end
+
+          def self.list
+            Resources::Global.instances(path: %w(firewall rules))
+          end
+        end
+
+        module Remote
+          def self.list
+            Resources::Remote.instances(
+              path:           %w(compute firewall-rules),
+              ignore_keys:    %w(self_link creation_timestamp id kind self_link),
+              skip_instances: { "name" => /^default-.*|^gke-.*|^k8s-fw-.*/, "network" => /^default$/ }
+            )
+          end
+        end
+
+        module Rule
+          include GClouder::Logging
+          include GClouder::Helpers
+          include GClouder::GCloud
+
+          def self.ensure(rule, args = {}, silent: false)
+            Resource.ensure :"compute firewall-rules", rule, hash_to_args(args), silent: silent
+          end
+
+          def self.purge(rule, silent: false)
+            Resource.purge :"compute firewall-rules", rule, silent: silent
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/compute/instances.rb

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Compute
+      module Instances
+        include GClouder::Helpers
+        include GClouder::Logging
+        include GClouder::Config::CLIArgs
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] compute / instance", indent: 1, title: true
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+
+          Local.list.each do |region, instances|
+            next if instances.empty?
+            info region, indent: 2, heading: true
+            info
+            instances.each do |instance|
+              Instance.ensure(instance["name"], hash_to_args(instance))
+            end
+          end
+        end
+
+        def self.check
+          return if Remote.list.empty?
+          return if Local.list.empty?
+          header :check
+
+          Resources::Validate::Remote.instances(
+            Local.manipulated,
+            Remote.list,
+            skip_keys: [
+              "image",
+              "zone",
+              "network_interfaces"
+            ]
+          )
+        end
+
+        def self.clean
+          return if undefined.empty?
+          header :clean
+          undefined.each do |instance, zone|
+            info zone, heading: true, indent: 2
+            info
+            warning "#{instance['name']} (not defined locally)"
+            #Instance.purge(instance["name"], "--zone=#{zone}")
+          end
+        end
+
+        def self.undefined
+          Remote.list.map do |region, instances|
+            return instances.map do |instance|
+              next if Local.list.fetch(region, []).select {|i| i["name"] == instance["name"] }.length > 0
+              zone = Resource::Find.zone(:"compute instances", instance["name"], region)
+              [instance, zone]
+            end.clean
+          end
+        end
+
+        module Local
+          include GClouder::Resources
+
+          def self.section
+            %w(compute instances)
+          end
+
+          def self.list
+            Resources::Region.instances(path: section)
+          end
+
+          def self.instance_names
+            list.map { |region, instances| instances.each.map { |instance| instance["name"] } }.flatten
+          end
+
+          def self.validate
+            Resources::Validate::Region.instances(
+              list,
+              required_keys:  GClouder::Config::Arguments.required(section),
+              permitted_keys: GClouder::Config::Arguments.permitted(section),
+            )
+          end
+
+          def self.mappings
+            Mappings::Property.section(["compute::instances", "subnet"])
+          end
+
+          def self.create_from_mapping(mappings, value)
+            mappings.reverse.inject(value) { |obj, key| key.is_a?(Integer) ? [obj] : { key => obj } }
+          end
+
+          # manipulate local resources so they're comparable with remote..
+          #
+          # FIXME
+          # this could be automated:
+          # * iterate over compute::instaces
+          # * create key for each value
+          # * assign config[key] to newly made key
+          def self.manipulated
+            list.each do |_region, resources|
+              resources.each do |resource|
+                data_structure = create_from_mapping(mappings, resource["subnet"])
+                resource.merge! data_structure["compute"]["instances"]
+                resource.delete("subnet")
+                resource
+              end
+            end
+          end
+        end
+
+        module Remote
+          def self.list
+            get_arguments
+          end
+
+          def self.get_arguments
+            Resources::Remote.instances(
+              path:           ["compute", "instances"],
+              skip_instances: { "name" => /^gke/, "status" => /^TERMINATED$/ }
+            )
+          end
+        end
+
+        module Instance
+          def self.ensure(instance, args = nil)
+            Resource.ensure :"compute instances", instance, args
+          end
+
+          def self.purge(instance, args = nil)
+            Resource.purge :"compute instances", instance, args
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/compute/networks/subnets.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Compute
+      module Networks
+        module Subnets
+          include GClouder::Config::CLIArgs
+          include GClouder::Config::Project
+          include GClouder::Logging
+          include GClouder::Config::Arguments
+          include GClouder::Resource::Cleaner
+
+          def self.header(stage = :ensure)
+            info "[#{stage}] compute / network / subnet", title: true
+          end
+
+          def self.validate
+            return if Local.list.empty?
+            header :validate
+            Local.validate
+          end
+
+          def self.ensure
+            return if Local.list.empty?
+            header
+
+            Local.list.each do |region, subnets|
+              next if subnets.empty?
+              info region, heading: true, indent: 2
+              info
+              subnets.each do |subnet|
+                Subnet.ensure(region, subnet["network"], subnet["name"], subnet["range"])
+              end
+            end
+          end
+
+          def self.check
+            return if Remote.list.empty?
+            return if Local.list.empty?
+            header :check
+            Resources::Validate::Remote.instances(Local.list, Remote.list)
+          end
+
+          module Local
+            include GClouder::Logging
+
+            def self.section
+              ["compute", "networks", "subnets"]
+            end
+
+            def self.list
+              instances
+            end
+
+            def self.validate
+              Resources::Validate::Region.instances(
+                instances,
+                required_keys:  GClouder::Config::Arguments.required(section),
+                permitted_keys: GClouder::Config::Arguments.permitted(section)
+              )
+            end
+
+            def self.instances
+              Resources::Region.instances(path: ["subnets"])
+            end
+
+            def self.networks
+              collection = { "global" => [] }
+              list.each { |_region, subnets| subnets.each { |subnet| collection["global"].push({ "name" => subnet["network"] }) } }
+              collection.delete_if { |_k, v| v.empty? }
+            end
+          end
+
+          module Remote
+            def self.list
+              get_arguments
+            end
+
+            def self.get_arguments
+              Resources::Remote.instances(
+                path:           ["compute", "networks", "subnets"],
+                ignore_keys:    ["ip_cidr_range", "region"],
+                skip_instances: { "network" => /^default$/ }
+              )
+            end
+          end
+
+          module Subnet
+            include GClouder::Resource
+
+            def self.ensure(region, network, subnet, range)
+              Resource.ensure :"compute networks subnets", subnet, "--network #{network} --range #{range} --region #{region}"
+            end
+
+            def self.purge(region, subnet)
+              Resource.purge :"compute networks subnets", subnet, "--region #{region}", indent: 3
+            end
+          end
+        end
+      end
+    end
+  end
+end