gclouder 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0eb4fecbee428b1cdc7c390f0503ccdc4448cbff
+  data.tar.gz: 3f552005ae332e6ef91fd32cf207f827e4f2c8b6
+SHA512:
+  metadata.gz: 4ee77003e814011b94d28ab3f438973ad3e49dea34893e2f6ea20f86d132bbe5e113b08a8c7d4affb90dc1d710f0edc6d4a92430cd5ecbab87f8164485248c1b
+  data.tar.gz: 0307d6e85947977a12c5a30a5d054b0c738a71341b99691351b2e1ae1ff51eb103982821a3532fcee33a4d06c968f2f6bcd9c66f6a22cc0ea3591122339619d0

data/README.md

@@ -0,0 +1,104 @@
+## GClouder
+
+Google Cloud Resource Deployer
+
+### Usage
+
+#### Authentication
+
+Authenticate against the GCP API:
+```
+gcloud auth login
+gcloud auth application-default login
+```
+
+#### Examples
+
+```
+# output validation then exit
+./bin/gclouder -c conf/<config>.yaml --validate-only
+
+# only execute non-state-changing commands (e.g: queries)
+./bin/gclouder -c conf/<config>.yaml --dry-run
+
+# apply the config
+./bin/gclouder -c conf/<config>.yaml
+```
+
+#### Dependencies
+
+##### Google Cloud SDK
+
+Please see: https://cloud.google.com/sdk/downloads
+
+##### Ruby
+
+Requires a modern version of Ruby (>= 2.4), you can use rbenv or brew to install one, e.g:
+
+```
+git clone https://github.com/rbenv/rbenv.git ~/.rbenv
+git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
+echo 'eval "$(rbenv init -)"' >> ~/.bashrc
+. ~/.bashrc
+rbenv install 2.4.0
+rbenv global 2.4.0 # or `rbenv local 2.4.0` in gclouder dir
+```
+
+##### Gems
+
+Install `gclouder` gem dependencies:
+```
+cd gclouder
+gem install bundler
+bundle install
+```
+
+### Testing
+
+
+Test coverage is currently limited to libraries which are peripheral to the core functionality of this app, i.e: tests only exist for methods which can be independently tested of the GCP API.
+
+There are plans to add integration tests at a later date.
+
+To run the tests use one of the following:
+
+Run once:
+```
+rake
+```
+
+To monitor changes to project files during development:
+```
+rake guard
+```
+
+### Notes
+
+Each resource is designed to do the following:
+
+* validation of local configuration by:
+  * check parameters are valid arguments
+  * check required parameters are set
+  * check types are correct
+
+* create remote instances which are defined locally
+
+* check remote instance dont differs from local instance definitions
+
+* remove instances which are no longer defined locally yet exist remotely
+
+### Why?
+* Google Deployment Manager is unable to manipulate resources not managed by itself
+* It's not possible to resize things like clusters without using gcloud(1)
+* Not all resources are supported by Google Deployment Manager
+* Greater control over the magic that happens between our resource definitions and the remote resources
+* Inter-project resources
+
+### Gem
+
+To build and install a gem run:
+
+```
+rake build
+gem install pkg/gclouder-<version>.gem
+```

data/bin/gclouder

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "../lib"))
+
+require "gclouder"
+
+GClouder.run

data/lib/gclouder/config/arguments.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require "yaml"
+
+module GClouder
+  module Config
+    module Arguments
+      include GClouder::Logging
+
+      def self.arguments
+        GClouder::ConfigLoader.load("../../assets/arguments")
+      end
+
+      def self.load
+        arguments
+      end
+
+      def arguments
+        Arguments.arguments
+      end
+
+      def self.included(klass)
+        klass.extend Arguments
+      end
+
+      def self.permitted(section)
+        GClouder::ConfigSection.find(section, arguments)
+      end
+
+      def self.required(section)
+        GClouder::ConfigSection.find(section, arguments).delete_if { |key, values| ! values["required"] }
+      end
+    end
+  end
+end

data/lib/gclouder/config/cli_args.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Config
+    module CLIArgs
+      def self.cli_args
+        @cli_args ||= { debug: false }
+      end
+
+      def cli_args
+        CLIArgs.cli_args
+      end
+
+      def self.included(klass)
+        klass.extend CLIArgs
+      end
+
+      def self.valid_resources
+        GClouder.resources.map { |resource| resource[:name] }
+      end
+
+      def self.load
+        option_parser = Trollop::Parser.new do
+          banner GClouder::Header.display + "\n \n "
+
+          # required
+          opt :config,                       "path to config file\n ", type: :string, required: true
+
+          # level of operation
+          opt :dry_run,                      "passive mode"
+          opt :purge,                        "remove unmanaged resources (destructive!)\n "
+
+          # authentication / for automation
+          opt :activate_service_accounts,    "activate service account(s) (for use when running using service accounts, i.e: with CI)"
+          opt :keys_dir,                     "path to directory with service account key files (for use with --activate-service-accounts)\n ", type: :string
+
+          # which resources / actions
+          # FIXME: integrate checks for required permissions into Project module
+          opt :bootstrap,                    "create project (requires being run as non-service account)"
+          # this should be type: proc and validate that the params match one of: validate, ensure, clean
+          opt :stages,                       "which stages to run (validate,ensure,clean) [csv]", type: :string
+          opt :resources,                    "which resources to update [csv]", type: :string
+          opt :skip_cross_project_resources, "skip resources which don't reside in main project\n "
+
+          # output
+          opt :debug,                        "print commands to be executed, and stack traces"
+          opt :trace,                        "print stack traces"
+          opt :no_color,                     "disable color\n \n "
+        end
+
+        @cli_args = Trollop.with_standard_exception_handling(option_parser) do
+          raise Trollop::HelpNeeded if ARGV.empty?
+          option_parser.parse ARGV
+        end
+
+        String.disable_colorization = @cli_args[:no_color]
+
+        if @cli_args[:resources]
+          @cli_args[:resources].split(',').each do |resource|
+            unless valid_resources.include?(resource)
+              puts "valid resources: #{valid_resources.join(', ')}"
+              puts "invalid resource for --resources flag: #{resource}"
+              exit 1
+            end
+          end
+        end
+
+        check
+      end
+
+      def self.check
+        raise ArgumentError, "config file not specified" unless cli_args[:config_given]
+        raise ArgumentError, "config file not readable: #{cli_args[:config]}" unless File.readable?(cli_args[:config])
+      end
+    end
+  end
+end

data/lib/gclouder/config/cluster.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Config
+    class Cluster < DeepMergeHash
+      def context(context)
+        dup = self.dup
+
+        permitted_keys = case context
+        when :create_cluster
+          [
+            "additional_zones",
+            "async",
+            "cluster_ipv4_cidr",
+            "disable_addons",
+            "disk_size",
+            "no_enable_cloud_endpoints",
+            "no_enable_cloud_logging",
+            "no_enable_cloud_monitoring",
+            "image_type",
+            "machine_type",
+            "max_nodes_per_pool",
+            "network",
+            "num_nodes",
+            "password",
+            "scopes",
+            "subnetwork",
+            "username",
+            "wait",
+            "zone",
+          ]
+        when :create_nodepool
+          # flip value due to differing key name..
+          if self.key?("no_enable_cloud_endpoints")
+            dup[:enable_cloud_endpoints] = self[:no_enable_cloud_endpoints] ? false : true
+          end
+
+          [
+            "disk_size",
+            "enable_cloud_endpoints",
+            "image_type",
+            "machine_type",
+            "num_nodes",
+            "scopes",
+            "zone",
+            "cluster",
+          ]
+        when :resize_cluster
+          dup["size"] = self["num_nodes"] if self.key?("num_nodes")
+
+          [
+            "async",
+            "size",
+            "wait",
+            "zone",
+            "node_pool",
+          ]
+        else
+          raise StandardError, "invalid context supplied when querying config object: #{context}"
+        end
+
+        dup.delete_if { |k, _| !permitted_keys.include?(k) }
+      end
+    end
+  end
+end

data/lib/gclouder/config/defaults.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require "yaml"
+
+module GClouder
+  module Config
+    module Defaults
+      include GClouder::Logging
+
+      def self.defaults
+        GClouder::ConfigLoader.load("../../assets/defaults")
+      end
+
+      def self.load
+        defaults
+      end
+
+      def defaults
+        Defaults.defaults
+      end
+
+      def self.included(klass)
+        klass.extend Defaults
+      end
+
+      def self.section(section)
+        GClouder::ConfigSection.find(section, defaults)
+      end
+
+      def self.section?(section)
+        GClouder::ConfigSection.find(section, defaults, silent: true)
+      end
+    end
+  end
+end

data/lib/gclouder/config/files/project.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Config
+    module Files
+      module Project
+        include GClouder::Config::CLIArgs
+        include GClouder::Helpers
+
+        def self.included(klass)
+          klass.extend project
+        end
+
+        def project
+          Project.project
+        end
+
+        def self.project
+          to_deep_merge_hash(YAML.load_file(cli_args[:config]))
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/config/project.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Config
+    module Project
+      include GClouder::Logging
+      include GClouder::Helpers
+
+      def self.project
+        @project
+      end
+
+      def project
+        Project.project
+      end
+
+      def self.load
+        @project = GClouder.resources.each_with_object(GClouder::Config::Files::Project.project) do |resource, config|
+          next unless module_exists? "#{resource[:module]}::Config"
+
+          config = resource[:module]::Config.merged(config)
+        end
+
+        fatal "no project_id found in config" unless project.key?("project_id")
+      end
+
+      private
+
+      def self.included(klass)
+        klass.extend Project
+      end
+    end
+  end
+end

data/lib/gclouder/config/resource_representations.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require "yaml"
+
+module GClouder
+  module Config
+    module ResourceRepresentations
+      include GClouder::Logging
+
+      def self.properties
+        @properties ||= GClouder::ConfigLoader.load("../../assets/resource_representations")
+      end
+
+      def self.load
+        properties
+      end
+
+      def properties
+        ResourceRepresentations.properties
+      end
+
+      def self.included(klass)
+        klass.extend ResourceRepresentations
+      end
+
+      def self.section(section)
+        GClouder::ConfigSection.find(section, properties)
+      end
+    end
+  end
+end

data/lib/gclouder/config_loader.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module ConfigLoader
+    def self.load(path)
+      data = DeepMergeHash.new
+      file = File.join(File.dirname(__FILE__), path, "**/*.yml")
+
+      configs = Dir.glob(file)
+
+      configs.each do |config|
+        yaml = YAML.load_file config
+        section_path = to_path(config, path)
+        hash = section_path.reverse.inject(yaml) { |value, key| { key => value } }
+        data = data.deep_merge hash
+      end
+
+      data
+    end
+
+    def self.to_path(config, path)
+      config.gsub(/.*#{path}\//, "").gsub!(/\.yml$/, "").split("/")
+    end
+  end
+end

data/lib/gclouder/config_section.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module ConfigSection
+    include GClouder::Logging
+
+    def self.included(klass)
+      klass.extend ConfigSection
+    end
+
+    def self.find(path, data, silent: false)
+      raise StandardError, "find: path argument must be an array: #{path.inspect}" unless path.is_a?(Array)
+      raise StandardError, "find: data argument must be an hash: #{path.inspect}" unless data.is_a?(Hash)
+
+      section = data.dig(*path)
+
+      if section
+        return silent ? true : section
+      end
+
+      return false if silent
+
+      fatal "can't find key in data: #{path}"
+    end
+  end
+end

data/lib/gclouder/dependencies.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Dependencies
+    def self.check
+      %w(jq gcloud gsutil).each do |command|
+        (puts "missing dependency: #{command}"; exit 1) unless system("which #{command} > /dev/null 2>&1")
+      end
+    end
+  end
+end

data/lib/gclouder/gcloud.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module GCloud
+    include GClouder::Logging
+    include GClouder::Shell
+    include GClouder::Helpers
+    include GClouder::Config::Project
+    include GClouder::Config::CLIArgs
+
+    def self.included(klass)
+      klass.extend GCloud
+    end
+
+    def gcloud(command, force: false, failure: true, silent: false, project_id: nil)
+      project_id = verify(project_id)
+
+      GClouder::Project::ID.switch(project_id)
+
+      if cli_args[:dry_run] && !force
+        debug "# gcloud --quiet --format json --project=#{project_id} #{command}" if cli_args[:debug]
+        GClouder::Project::ID.reset
+        return
+      end
+
+      result = shell("gcloud --quiet --format json --project=#{project_id} #{command}", failure: failure, silent: silent)
+
+      GClouder::Project::ID.reset
+
+      valid_json?(result) ? JSON.parse(result.to_s) : result
+    end
+
+    def verify(project_id)
+      project_id ||= project["project_id"]
+      return project_id if project_id
+      raise StandardError, "project_id not detected"
+    end
+  end
+end

data/lib/gclouder/gsutil.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module GSUtil
+    include GClouder::Shell
+    include GClouder::Config::CLIArgs
+    include GClouder::Config::Project
+
+    def self.included(klass)
+      klass.extend GSUtil
+    end
+
+    def gsutil(command, args, force: false)
+      info "# gsutil #{command} -p #{project['project_id']} #{args}" if cli_args[:debug]
+
+      return if cli_args[:dry_run] && !force
+
+      gsutil_exec(command, args)
+    end
+
+    def gsutil_exec(command, args)
+      shell("gsutil #{command} -p #{project['project_id']} #{args}")
+    end
+  end
+end

data/lib/gclouder/header.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Header
+    def self.display
+      "\n\n  #{'g'.blue}#{'o'.red}#{'o'.yellow}#{'g'.blue}#{'l'.green}#{'e'.red} cloud platform resource deployer"
+    end
+  end
+end

data/lib/gclouder/helpers.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Helpers
+    def self.included(klass)
+      klass.extend Helpers
+    end
+
+    def hash_to_args(hash)
+      raise StandardError, "hash_to_args: input not a hash: #{hash}" unless hash.is_a?(Hash)
+      hash.map { |param, value|
+        next if param == "name"
+        to_arg(param, value)
+      }.join(" ")
+    end
+
+    def to_arg(param, value)
+      param = param.tr("_", "-")
+
+      value = case value
+      when Boolean
+        return value ? "--#{param}" : "--no-#{param}"
+      when Array
+        value.join(",")
+      else
+        value
+      end
+
+      "--#{param}='#{value}'"
+    end
+
+    def valid_json?(object)
+      JSON.parse(object.to_s)
+      return true
+    rescue JSON::ParserError
+      return false
+    end
+
+    def to_deep_merge_hash(hash, hash_type = DeepMergeHash)
+      raise StandardError, "to_deep_merge_hash: argument must be a hash" unless hash.is_a?(Hash)
+
+      hash.each do |k,v|
+        case v
+        when Hash
+          hash_to_deep_merge_hash(hash, k, v, hash_type)
+        when Array
+          array_to_deep_merge_hash(v, hash_type)
+        end
+      end
+
+      hash_type.new(hash)
+    end
+
+    def module_exists?(name, base = self.class)
+      raise StandardError, "module name must be a string" unless name.is_a?(String)
+      base.const_defined?(name) && base.const_get(name).instance_of?(::Module)
+    end
+
+    private
+
+    def hash_to_deep_merge_hash(obj, index, hash, hash_type)
+      obj[index] = hash_type.new(hash)
+      to_deep_merge_hash(obj[index], hash_type)
+    end
+
+    def array_to_deep_merge_hash(array, hash_type)
+      array.each_with_index do |e,i|
+        case e
+        when Hash
+          hash_to_deep_merge_hash(array, i, e, hash_type)
+        when Array
+          array_to_deep_merge_hash(array, hash_type)
+        end
+      end
+    end
+  end
+end