gclouder_undefined_resources 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 53f02cbd3cea848631504944efa920e5ced3574e
+  data.tar.gz: 98d661239064f19b793a6f5992cec119e37c1a7a
+SHA512:
+  metadata.gz: e4719ab0e87f18ed531b3bcdc18b7948ae0d3e1a2e1edd37dfa7d1cd756056d42ae33eeefd8de4a6ef407bf98f41c2ef55d4431062467f0a0cd5c800142fa23b
+  data.tar.gz: d29e31aef7a439ec592be10c0691942ff8e71a55d6f8a03096480967dd2eca81948006671cb43cd76257d09de7683654c0f70f48952d43df280b3558b36e429c

data/bin/gclouder_undefined_resources

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "../lib"))
+
+require "gclouder_undefined_resources"
+
+GClouderUndefinedResources.run

data/lib/gclouder_undefined_resources/cli_args.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  module CLIArgs
+    def self.cli_args
+      @cli_args ||= { debug: false }
+    end
+
+    def cli_args
+      CLIArgs.cli_args
+    end
+
+    def self.included(klass)
+      klass.extend CLIArgs
+    end
+
+    def self.load
+      option_parser = Trollop::Parser.new do
+        banner "\n  undefined gcp resources\n "
+
+        opt :local_resources,  "file containing local resource definitions", type: :string, required: true
+        #opt :remote_resources, "file containing remote resource definitions\n ", type: :string, required: true
+        opt :group_by,         "how to format output. valid values include: location, type", type: :string, default: "location"
+        opt :filter_config,    "file containing filter config", type: :string
+        opt :show_filtered,    "show filter matching output\n ", type: :bool
+        opt :verbose,          "verbose\n ", type: :bool
+      end
+
+      @cli_args = Trollop.with_standard_exception_handling(option_parser) do
+        raise Trollop::HelpNeeded if ARGV.empty?
+        option_parser.parse ARGV
+      end
+    end
+  end
+end

data/lib/gclouder_undefined_resources/local_resources.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  class LocalResources
+    extend Forwardable
+
+    def initialize(file)
+      @resources = yaml_load(file)
+    end
+
+    def_delegator :@resources, :[]
+
+    def yaml_load(file)
+      @resources = YAML.load_file(file)
+    rescue => e
+      raise StandardError, "failed to load local resource file: #{e}"
+    end
+
+    def global?(key, name)
+      #ap @resources
+      #puts "looking up: #{key}, name: #{name}"
+      match = @resources.dig(*key)
+      return false if match.nil?
+      found_resources = match.select { |hash| hash["name"] == name }
+      !found_resources.empty?
+    rescue TypeError, "type table lookup failure"
+    end
+
+    def regional?(region, key, name)
+      #ap @resources
+      #puts "looking up: #{key}, name: #{name}"
+      match = @resources.dig("regions", region, *key)
+      return false if match.nil?
+      found_resources = match.select { |hash| hash["name"] == name }
+      !found_resources.empty?
+    rescue TypeError, "type table lookup failure"
+    end
+  end
+end

data/lib/gclouder_undefined_resources/remote_resources/remote.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  class RemoteResources
+    module Remote
+      module GCloud
+        def self.fetch(project_id)
+          modules.map { |m| m.send(:fetch, project_id) }.flatten
+        end
+
+        def self.modules
+          [
+            ServiceAccounts,
+            ResourcesList
+          ]
+        end
+
+        module ServiceAccounts
+          def self.fetch(project_id)
+            json = JSON.parse(Shell.run("gcloud iam service-accounts list --format json"))
+
+            json.map do |resource|
+              resource["@type"] = "serviceAccount"
+              Resource.new(resource["email"], resource)
+            end
+          end
+        end
+
+        module ResourcesList
+          def self.fetch(project_id)
+            json = JSON.parse(Shell.run("gcloud alpha resources list --format json"))
+
+            json.select do |resource|
+              result = true
+
+              result = nil unless resource["selfLink"] =~ /projects\/#{project_id}\//
+
+              if resource["@type"] == "type.googleapis.com/google.cloud.dataproc.v1.Job"
+                puts "skipping data proc resource: #{resource['reference']}, #{resource['placement']}"
+                result = nil
+              end
+
+              result
+            end.map do |resource|
+              Resource.new(resource["name"], resource)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder_undefined_resources/remote_resources/resource.rb

@@ -0,0 +1,190 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  class RemoteResources
+    class Resource
+      extend Forwardable
+
+      attr_accessor :name, :data
+
+      def initialize(name, data)
+        if name.nil?
+          ap data
+          raise StandardError, "can't find name for resource"
+        end
+
+        @name = name
+        @data = data
+      end
+
+      def_delegator :@data, :[]
+
+      def region
+        return unless @data.key?("selfLink")
+        match = @data["selfLink"].match(/.*\/regions\/([^\/]+)/)
+        return match[1] if match
+        zone[0..-3] if zone
+      end
+
+      def zone
+        return unless @data.key?("selfLink")
+        match = @data["selfLink"].match(/.*\/zones\/([^\/]+)/)
+        match[1] if match
+      end
+
+      def global?
+        return true if zone.nil? && region.nil?
+        return unless @data.key?("selfLink")
+        !@data["selfLink"].match(/.*\/projects\/[^\/]+\/global/).nil?
+      end
+
+      # type is the Remote version of Local path.., use this to check against local data structure
+      def type
+        @type ||= get_type
+      end
+
+      # check for local regional and local global definitions
+      def defined?
+        local_resources = GClouderUndefinedResources.local_resources
+
+        if global?
+          local_resources.global?(type, name)
+        else
+          local_resources.regional?(region, type, name)
+        end
+      end
+
+      # check to see if this resource has been filtered
+      def filtered?
+        unless CLIArgs.cli_args[:filter_config_given]
+          puts "no filter config given" if CLIArgs.cli_args["verbose"]
+          return false
+        end
+
+        unless filters.key?("filters")
+          raise StandardError, "no filters key found in filter config"
+        end
+
+        original_type = @data["@type"]
+
+        filters["filters"].each do |filter|
+          if filter["region"] != region && filter["region"] != "*"
+            next
+          end
+
+          if filter["zone"] != zone && filter["zone"] != "*"
+            next
+          end
+
+          if filter["type"] != original_type && filter["type"] != "*"
+            next
+          end
+
+          if (name !~ Regexp.new(filter["name"]) ? true : false) && filter["name"] != "*"
+            next
+          end
+
+          if CLIArgs.cli_args[:show_filtered]
+            puts "filter: "
+            puts "  description: #{filter['description']}"
+            puts "  region: #{filter['region']}"
+            puts "  zone: #{filter['zone']}"
+            puts "  type: #{filter['type']}"
+            puts "  name: #{filter['name']}"
+            puts "match: "
+            puts "  region: #{region}"
+            puts "  zone: #{zone}"
+            puts "  type: #{original_type}"
+            puts "  name: #{name}"
+            puts
+          end
+
+          return true
+        end
+
+        false
+      end
+
+      private
+
+      def filters
+        @filters ||= YAML.load_file(CLIArgs.cli_args[:filter_config])
+      rescue => error
+        puts "failed to load filter config: #{error}"
+        exit 1
+      end
+
+      # FIXME - add more types
+      def get_type
+        case @data["@type"]
+        when "serviceAccount"
+          [ "service_accounts" ]
+        #when "type.googleapis.com/cloud.bigstore.api.Bucket"
+
+        when "type.googleapis.com/cloud.dns.api.ManagedZone"
+          [ "dns", "zones" ]
+
+        #when "type.googleapis.com/cloud.resourcemanager.Organization"
+
+        when "type.googleapis.com/compute.BackendBucket"
+          [ "compute", "backend-buckets" ]
+
+        when "type.googleapis.com/compute.Disk"
+          [ "compute", "disks" ]
+
+        when "type.googleapis.com/compute.Firewall"
+          [ "firewall", "rules" ]
+
+        when "type.googleapis.com/compute.HealthCheck"
+          [ "compute", "health_check" ]
+
+        when "type.googleapis.com/compute.HttpHealthCheck"
+          [ "compute", "http_health_check" ]
+
+        when "type.googleapis.com/compute.HttpsHealthCheck"
+          [ "compute", "https_health_check" ]
+
+        when "type.googleapis.com/compute.Image"
+          ["compute", "images"]
+
+        when "type.googleapis.com/compute.Instance"
+          ["compute", "instances"]
+
+        when "type.googleapis.com/compute.InstanceTemplate"
+          ["compute", "instance_templates"]
+
+        #when "type.googleapis.com/compute.InstanceTemplate"
+
+        when "type.googleapis.com/compute.Network"
+          [ "networks" ]
+
+        when "type.googleapis.com/compute.Router"
+          [ "compute", "router" ]
+
+        #when "type.googleapis.com/compute.SslCertificate"
+
+        when "type.googleapis.com/compute.Subnetwork"
+          [ "subnets" ]
+
+        #when "type.googleapis.com/compute.TargetHttpProxy"
+        #when "type.googleapis.com/compute.TargetHttpsProxy"
+
+        when "type.googleapis.com/compute.UrlMap"
+          [ "compute", "url-maps" ]
+
+        #when "type.googleapis.com/google.api.servicemanagement.v1.ManagedService"
+        #when "type.googleapis.com/google.appengine.v1.Application"
+        #when "type.googleapis.com/google.appengine.v1.Service"
+        #when "type.googleapis.com/google.appengine.v1.Version"
+        #when "type.googleapis.com/google.cloud.billing.v1.BillingAccount"
+        #when "type.googleapis.com/google.cloud.dataproc.v1.Cluster"
+        #when "type.googleapis.com/google.cloudresourcemanager.projects.v1beta1.Project"
+        else
+          puts "error: unknown or no '@type' property found for data:"
+          ap @data
+          exit 1
+        end
+      end
+    end
+  end
+end

data/lib/gclouder_undefined_resources/remote_resources.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  class RemoteResources
+    module Symbols
+      def self.tick
+        "✓".green
+      end
+
+      def self.x
+        "✗".red
+      end
+
+      def self.tick_filtered
+        "✓".yellow
+      end
+
+      def self.x_filtered
+        "✗".yellow
+      end
+    end
+
+    def initialize(project_id)
+      @project_id = project_id
+      @resources = []
+    end
+
+    def collect
+      add Remote::GCloud.fetch(@project_id)
+    end
+
+    def add(resources)
+      @resources += resources
+    end
+
+    # display: type, region, zone, name
+    def display_by_type
+      types = @resources.group_by { |resource| resource.type }
+
+      types.each do |type, resources_by_type|
+        regions = resources_by_type.group_by { |resource| resource.region }
+
+        next if regions.empty?
+
+        puts
+        puts
+        puts "  #{type.join(' / ')}"
+
+        regions.each do |region, resources_by_region|
+          zones = resources_by_region.group_by { |resource| resource.zone }
+
+          next if zones.empty?
+
+          region ||= "global"
+
+          puts
+          puts "    #{region}"
+
+          zones.each do |zone, resources_by_zone|
+            next if resources_by_zone.empty?
+
+            zone ||= "none"
+
+            puts
+            puts "      #{zone}"
+            puts
+
+            resources_by_region.each do |resource|
+              if resource.filtered?
+                status = resource.defined? ? Symbols.tick_filtered : Symbols.x_filtered
+              else
+                status = resource.defined? ? Symbols.tick : Symbols.x
+              end
+
+              # FIXME: do the skipping stuff here, or display status of the resource name..
+              # FIXME: to_s in resource could include x or tick or whatever
+              puts "        #{status} #{resource.name}"
+            end
+          end
+        end
+      end
+    end
+
+    def display_by_location
+      # display: region, zone, type, name
+      regions = @resources.group_by { |resource| resource.region }
+
+      regions.each do |region, resources_by_region|
+        region ||= "global"
+
+        puts
+        puts
+        puts "  #{region}"
+
+        zones = resources_by_region.group_by { |resource| resource.zone }
+
+        zones.each do |zone, resources_by_zone|
+          zone ||= "none"
+
+          puts
+          puts "    #{zone}"
+
+          types = resources_by_zone.group_by { |resource| resource.type }
+
+          types.each do |type, resources_by_type|
+            puts
+            puts "      #{type.join(' / ')}"
+            puts
+
+            resources_by_type.each do |resource|
+              if resource.filtered?
+                status = resource.defined? ? Symbols.tick_filtered : Symbols.x_filtered
+              else
+                status = resource.defined? ? Symbols.tick : Symbols.x
+              end
+
+              # FIXME: do the skipping stuff here, or display status of the resource name..
+              # FIXME: to_s in resource could include x or tick or whatever
+              puts "        #{status} #{resource.name}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder_undefined_resources/resource.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  module Resource
+    def self.local=(file)
+      @local_resources = YAML.load_file(file)
+    rescue => e
+      raise StandardError, "failed to load local resource file: #{e}"
+    end
+
+    def self.local
+      @local_resources
+    end
+
+    def self.global?(key, name)
+      resources = local.dig(*key)
+      return false if resources.nil?
+      found_resources = resources.select { |hash| hash["name"] == name }
+      !found_resources.empty?
+    end
+
+    def self.regional?(region, key, name)
+      resources = local.dig("regions", region, *key)
+      return false if resources.nil?
+      found_resources = resources.select { |hash| hash["name"] == name }
+      !found_resources.empty?
+    end
+  end
+end

data/lib/gclouder_undefined_resources/self_link.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  module SelfLink
+    def self.region(resource)
+      r = resource["selfLink"].match(/.*\/regions\/([^\/]+)/)
+      r.nil? ? nil : r[1]
+    end
+
+    def self.zone(resource)
+      r = resource["selfLink"].match(/.*\/zones\/([^\/]+)/)
+      r.nil? ? nil : r[1]
+    end
+
+    def self.global(resource)
+      !resource["selfLink"].match(/.*\/projects\/[^\/]+\/global/).nil?
+    end
+  end
+end

data/lib/gclouder_undefined_resources/shell.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  module Shell
+    def self.run(command)
+      puts "running command: #{command}" if CLIArgs.cli_args[:verbose]
+      stdout, stderr, status = Open3.capture3(command)
+
+      if status.to_i.nonzero?
+        raise StandardError, "command failed: #{command}\n#{stderr}\n#{stdout}"
+      end
+
+      stdout
+    rescue => error
+      puts error
+    end
+  end
+end

data/lib/gclouder_undefined_resources/version.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+module GClouderUndefinedResources
+  VERSION = "0.1.0"
+end

data/lib/gclouder_undefined_resources.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+require "yaml"
+require "json"
+require "trollop"
+require "open3"
+require "awesome_print"
+require "colorize"
+require "forwardable"
+require "gclouder_undefined_resources/cli_args"
+require "gclouder_undefined_resources/shell"
+require "gclouder_undefined_resources/local_resources"
+require "gclouder_undefined_resources/remote_resources"
+require "gclouder_undefined_resources/remote_resources/remote"
+require "gclouder_undefined_resources/remote_resources/resource"
+
+module GClouderUndefinedResources
+  def self.run
+    undefined
+  end
+
+  def self.local_resources
+    @local_resources ||= LocalResources.new(CLIArgs.cli_args[:local_resources])
+  end
+
+  def self.undefined
+    CLIArgs.load
+
+    remote_resources = RemoteResources.new(local_resources["project_id"])
+    remote_resources.collect
+
+    case CLIArgs.cli_args[:group_by].to_sym
+    when :location
+      remote_resources.display_by_location
+    when :type
+      remote_resources.display_by_type
+    else
+      puts "unknown group-by type"
+    end
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: gclouder_undefined_resources
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Rob Wilson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A tool to compare deployed resources to resources defined in gclouder(1)
+  manifests
+email: roobert@gmail.com
+executables:
+- gclouder_undefined_resources
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/gclouder_undefined_resources
+- lib/gclouder_undefined_resources.rb
+- lib/gclouder_undefined_resources/cli_args.rb
+- lib/gclouder_undefined_resources/local_resources.rb
+- lib/gclouder_undefined_resources/remote_resources.rb
+- lib/gclouder_undefined_resources/remote_resources/remote.rb
+- lib/gclouder_undefined_resources/remote_resources/resource.rb
+- lib/gclouder_undefined_resources/resource.rb
+- lib/gclouder_undefined_resources/self_link.rb
+- lib/gclouder_undefined_resources/shell.rb
+- lib/gclouder_undefined_resources/version.rb
+homepage: https://github.com/roobert/gclouder_undefined_resources
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: GClouder undefined resource detection
+test_files: []