gclouder 0.1.1

data/lib/gclouder/resources/storagebuckets.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module StorageBuckets
+      include GClouder::Logging
+      include GClouder::Shell
+      include GClouder::Resource::Cleaner
+
+      def self.header(stage = :ensure)
+        info "[#{stage}] storage / buckets", title: true, indent: 1
+      end
+
+      def self.validate
+        return if Local.list.empty?
+        header :validate
+        Local.validate
+      end
+
+      def self.ensure
+        return if Local.list.empty?
+        header
+
+        Local.list.each do |region, region_config|
+          info region, heading: true, indent: 2
+          region_config.each do |bucket|
+            info
+            StorageBucket.ensure(region, bucket)
+          end
+        end
+      end
+
+      module Local
+        def self.list
+          instances
+        end
+
+        def self.validate
+          # Validation knowledge included here because we don't have arguments parser for gsutil.
+          # We also don't support every key that gsutil does. See StorageBucket.ensure() below.
+          permitted_and_required_keys = {
+            "default_access"=>{"type"=>"String", "required"=>true}
+          }
+
+          Resources::Validate::Region.instances(
+            instances,
+            permitted_keys: permitted_and_required_keys
+          )
+        end
+
+        def self.instances
+          Resources::Region.instances(path: ["storage", "buckets"])
+        end
+      end
+
+      module Remote
+        include GClouder::GSUtil
+
+        # FIXME: make more robust(!)
+        def self.list
+          gsutil("ls", "-L").to_s.split("gs://").delete_if(&:empty?).each_with_object({}) do |data, collection|
+            normalized = data.split("\n").map! { |b| b.delete("\t") }
+            bucket_name = normalized[0].delete("/ :")
+            region = normalized.select { |e| e.match("^Location constraint:") }.last.split(":").last.downcase
+            collection[region] ||= []
+            collection[region] << { "name" => bucket_name }
+          end
+        end
+      end
+
+      module StorageBucket
+        include GClouder::GSUtil
+        include GClouder::Config::CLIArgs
+
+        def self.setDefaultAccess(bucket_name, default_access)
+          info "# gsutil defacl ch -u #{default_access} gs://#{bucket_name}" if cli_args[:debug]
+
+          return if cli_args[:dry_run]
+
+          # Just use shell, as -p flag is not valid for 'defacl ch'.
+          shell("gsutil defacl ch -u #{default_access} gs://#{bucket_name}")
+        end
+
+        def self.check_exists?(region, bucket_name)
+          gsutil_exec("ls", "gs://#{bucket_name} > /dev/null 2>&1 && echo 0 || echo 1").to_i == 0
+        end
+
+        def self.ensure(region, bucket)
+          if check_exists?(region, bucket["name"])
+            good bucket["name"]
+            return
+          end
+
+          add "#{bucket["name"]} [#{bucket["default_access"]}]"
+          gsutil "mb", "-l #{region} gs://#{bucket["name"]}"
+
+
+          setDefaultAccess bucket["name"], bucket["default_access"] if bucket.key?("default_access")
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/validate/global.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Validate
+      module Global
+        include GClouder::Logging
+        include GClouder::Config::Project
+        include GClouder::Config::CLIArgs
+        include Local
+
+        def self.instances(data, required_keys: {}, permitted_keys: {}, ignore_keys: [])
+          return unless data.key?("global")
+
+          data["global"].each do |instance|
+            info instance["name"], heading: true, indent: 3
+
+            next if !has_unknown_keys?(instance, permitted_keys, ignore_keys) &&
+              has_required_keys?(instance, required_keys, ignore_keys, indent: 3)
+
+            fatal "\nerror: validation failure"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/validate/local.rb

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Validate
+      module Local
+        include GClouder::Logging
+
+        def self.included(klass)
+          klass.extend Local
+        end
+
+        # FIXME: this should probably recurse
+        def has_required_keys?(instance, required_keys, ignore_keys, indent: 3)
+          success = true
+
+          required_keys["name"] = {
+            "type" => "String",
+            "required" => true
+          }
+
+          required_keys.each do |key, data|
+            next if ignore_keys.include?(key)
+
+            if !instance.key?(key)
+              bad "#{key} is missing", indent: indent
+              success = false
+            end
+          end
+
+          success
+        end
+
+        def has_unknown_keys?(instance, permitted_keys, ignore_keys, indent: 0)
+          success = false
+
+          # a name is required for every resources
+          permitted_keys["name"] = {
+            "type" => "String",
+            "required" => true
+          }
+
+          instance.each do |key, value|
+            next if ignore_keys.include?(key)
+
+            if !permitted_keys.key?(key)
+              bad "#{key} is an invalid key", indent: 4 + indent
+              success = true
+              next
+            end
+
+            required_type = Object.const_get(permitted_keys[key]["type"])
+
+            if !value.is_a?(required_type)
+              bad "#{key} invalid type: #{value.class} (should be: #{required_type})", indent: 4 + indent
+              success = true
+              next
+            end
+
+            good "#{key} is a #{required_type} (#{value})", indent: 4 + indent
+          end
+
+          success
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/validate/region.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Validate
+      module Region
+        include GClouder::Logging
+        include GClouder::Config::Project
+        include GClouder::Config::CLIArgs
+        extend Local
+
+        def self.instances(data, required_keys: {}, permitted_keys: {}, ignore_keys: [], skip_region: false, indent: 0)
+          data.each do |region, instances|
+            info region, indent: 2 + indent, heading: true unless skip_region
+            instances.each do |instance|
+              info instance["name"], indent: 3 + indent, heading: true
+
+              next if !has_unknown_keys?(instance, permitted_keys, ignore_keys, indent: indent) &&
+                has_required_keys?(instance, required_keys, ignore_keys, indent: 4 + indent)
+
+              fatal "\nerror: validation failure"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/validate/remote.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Validate
+      module Remote
+        include GClouder::Logging
+        include GClouder::Config::Project
+        include GClouder::Config::CLIArgs
+
+        def self.instances(local, remote, skip_keys: [])
+          remote.each do |region, resources|
+            info region, heading: true, indent: 2
+            resources.each do |resource|
+              #FIXME: This won't work with duplicate names
+              local_config = local.fetch(region, []).select {|s| s["name"] == resource["name"] }.first
+
+              failure = false
+
+              next unless local_config
+
+              info resource["name"], indent: 3, heading: true
+
+              local_config.each do |key, value|
+                skipped = false
+                skip_message = nil
+
+                # FIXME: we should recurse down into the data structure and check the values..
+                if value.is_a?(Hash) || value.is_a?(Array)
+                  skip_message ||= "(can't validate complex object)"
+                  skipped = true
+                end
+
+                if skip_keys.include?(key)
+                  skip_message ||= "(skip_keys in resource definition)"
+                  skipped = true
+                else
+                  if !resource.key?(key)
+                    bad "#{key} (missing key)", indent: 4
+
+                    failure = true
+                    next
+                  end
+
+                  if value != resource[key]
+                    bad "#{key} (\"#{value.to_s.truncate(30)}\" != \"#{resource[key].to_s.truncate(30)}\")", indent: 4
+
+                    failure = true
+                    next
+                  end
+                end
+
+                message = "#{key}" " (#{value.to_s.truncate(60)})"
+                message += " [skipped]" if skipped
+                message += " #{skip_message}" if skip_message
+
+                good message, indent: 4
+              end
+
+              next unless failure
+
+              info
+              info "local config:"
+              pp local_config.sort.to_h
+              info
+
+              info "remote config:"
+              pp resource.sort.to_h
+              info
+
+              fatal "error: immutable remote resource differs from local definition for resource: #{region}/#{resource["name"]}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources.rb

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Local
+      def self.included(klass)
+        klass.extend Local
+      end
+
+      # FIXME: error if path doesnt exist..
+      def get_section(data, path, silent: false)
+        path.each do |key|
+          return [] unless data.key?(key)
+          data = data[key]
+        end
+
+        data
+      end
+    end
+
+    module Global
+      include GClouder::Logging
+      include GClouder::Config::Project
+      include GClouder::Config::CLIArgs
+      include Local
+
+      def self.instances(path: [])
+        data = get_section(project, path)
+
+        return {} if data.empty?
+
+        { "global" => data }
+      end
+    end
+
+    module Region
+      include GClouder::Logging
+      include GClouder::Helpers
+      include GClouder::Config::Project
+      include GClouder::Config::CLIArgs
+      include Local
+
+      def self.instances(path: [])
+        return {} unless project.key?("regions")
+
+        data = project["regions"].each_with_object({}) do |(region, region_config), instances|
+          instances[region] ||= []
+
+          data = get_section(region_config, path, silent: true)
+
+          data.each do |instance|
+            if GClouder::Config::Defaults.section?(path)
+              defaults = to_deep_merge_hash(GClouder::Config::Defaults.section(path))
+              instance = defaults.deep_merge(instance)
+            end
+
+            instances[region] << instance
+          end
+        end
+
+        data.delete_if { |_k, v| v.empty? }
+      end
+    end
+
+    # FIXME: should this be split out into a separate module that deals with remote state?
+    module Remote
+      def self.instances(path: [], ignore_keys: [], skip_instances: {}, args: nil)
+        resource_name = path.join(" ").to_sym
+
+        Resource.list(resource_name, args).each_with_object({}) do |resource, collection|
+          skip = false
+
+          skip_instances.each do |key, value|
+            next unless resource.key?(key)
+            if resource[key] =~ value
+              skip = true
+              break
+            end
+          end
+
+          next if skip
+
+          YAML.load_file("assets/mappings/file.yml")
+
+          # FIXME: this is so keys with partial matches work..
+          file_mapping_key = path.join("::")
+          file_mapping = YAML.load_file("assets/mappings/file.yml").fetch(file_mapping_key, nil)
+
+          resource_representation_path = file_mapping.nil? ? path : file_mapping
+
+          #ap GClouder::Config::ResourceRepresentations.properties
+
+          # contains list of non-output-only remote properties
+          resource_representation = GClouder::Config::ResourceRepresentations.section(resource_representation_path)
+
+          #ap resource_representation
+
+          # FIXME: partial key matches here are bad.. i.e: [compute, networks]  matches [compute, networks, subnetworks]
+          # maps remote property names back to arguments
+          property_mappings_key = path.join("::")
+          property_mappings = YAML.load_file("assets/mappings/property.yml").fetch(property_mappings_key, [])
+
+          # Assume global, unless we can find or infer a region...
+          region = "global"
+          region = resource["region"] if resource.key?("region")
+          zone = resource["zone"] if !resource.key?("region") && resource.key?("zone")
+
+          if resource.key?("selfLink") && resource["selfLink"].match(/zones\//) && !zone
+            zone = resource["selfLink"].match(/zones\/([^\/]+)/)[1]
+          end
+
+          if zone
+            resource["zone"] = zone
+            region = zone.sub(/-[a-z]$/, "")
+          end
+
+          # 1: convert key names to snake_case
+          resource = resource.to_snake_keys
+
+          # 2: delete any keys not in resource_representations (because they're output only)
+          # FIXME: warn about deleting keys?
+          resource.delete_if do |key, _value|
+            resource_representation[key] == "OutputOnly" && key != "name"
+          end
+
+          # 3: convert the names of any keys using the mappings file
+          property_mappings.each do |argument, resource_representation|
+            # FIXME: don't overwrite arguments..
+            resource[argument] = resource.dig(*resource_representation)
+            resource.delete(resource_representation)
+          end
+
+          ignore_keys.each do |key|
+            next unless resource.key?(key)
+            resource.delete(key)
+          end
+
+          # ?: if there are any keys *not* in the resource_representations file then we have a problem
+
+          # ?: if there are any keys which dont match an argument, then we have a problem
+
+          collection[region] ||= []
+          collection[region] << resource
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/shell.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require "open3"
+
+module GClouder
+  module Shell
+    include GClouder::Logging
+
+    def self.included(klass)
+      klass.extend Shell
+    end
+
+    def shell(command, failure: true, silent: false)
+      stdout, stderr, status, failed = run(command)
+
+      if (GClouder.cli_args[:debug] || failed) && !silent
+        header(command)
+        dump_fds(stdout, stderr)
+      end
+
+      if !failed && silent
+        return true
+      end
+
+      if failed && !failure
+        return false
+      end
+
+      if failed && silent
+        return false
+      end
+
+      if failed && !silent
+        footer(status)
+      end
+
+      if silent
+        return
+      end
+
+      stdout
+    end
+
+    private
+
+    def header(command)
+      info
+      info "# #{command}"
+    end
+
+    def dump_fds(stdout, stderr)
+      dump(stdout)
+      dump(stderr)
+    end
+
+    def dump(fd)
+      return if fd.empty?
+      info fd
+    end
+
+    def footer(status)
+      fatal "there was an error running the previous shell command which exited with non-0: #{status}"
+    end
+
+    def run(command)
+      stdout, stderr, status = Open3.capture3(command)
+      failed = status.to_i > 0
+      [stdout, stderr, status, failed]
+    end
+  end
+end

data/lib/gclouder/version.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  VERSION = "0.1.1"
+end