gclouder 0.1.1

data/lib/gclouder/resources/project/iam_policy_binding.rb

@@ -0,0 +1,293 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Project
+      module IAMPolicyBinding
+        include GClouder::Config::Project
+        include GClouder::Logging
+        include GClouder::GCloud
+        include GClouder::Config::CLIArgs
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] project / iam-policy-binding", indent: 1, title: true
+        end
+
+        def self.clean
+          return if undefined.empty?
+
+          header :clean
+
+          undefined.each do |region, roles|
+            info region, indent: 2, heading: true
+            roles.each do |role|
+              info role["name"], indent: 3, heading: true
+              role["members"].each do |member|
+                message = member
+                message += " (not defined locally)"
+                warning message, indent: 4
+              end
+            end
+          end
+        end
+
+        def self.unmanaged_service_account?(member)
+          is_service_account?(member) && !member.include?(project["project_id"])
+        end
+
+        def self.is_service_account?(member)
+          member.include? "gserviceaccount.com"
+        end
+
+        def self.undefined
+          # each remote role
+          Remote.list.each_with_object({}) do |(region, remote_roles), collection|
+            # each role, {owner, ...}
+            remote_roles.each do |remote_role|
+              role_found = false
+
+              next unless remote_role.key?("members")
+
+              # for each remote member
+              remote_role["members"].each do |remote_member|
+
+                next unless Local.list.key?("global")
+
+                # see if the members role exists locally
+                Local.list["global"].each do |e|
+                  next unless e["name"] == remote_role["name"]
+
+                  role_found = true
+
+                  # if it does then check if member is in member list for role
+
+                  # member is defined, so skip it
+                  next if e["members"].include?(remote_member)
+
+                  # member is one we don't want to manage, so skip it
+                  next if unmanaged_service_account?(remote_member)
+
+                  # member is undefined so add it to collection
+
+                  collection["global"] ||= []
+
+                  # add role if it doesn't exist in collection
+                  if !resource?(collection["global"], remote_role["name"])
+                    collection["global"] <<  { "name" => remote_role["name"], "members" => [] }
+                  end
+
+                  # add memeber to role
+                  resource_array_append(collection["global"], remote_role["name"], "members", remote_member)
+                end
+              end
+
+              # if entire role is missing from local..
+              next if role_found
+              collection["global"] ||= []
+              collection["global"] << remote_role
+            end
+          end
+        end
+
+        def self.resource?(resources, resource)
+          !resources.fetch_with_default("name", resource, {}).empty?
+        end
+
+        def self.resource_array_append(resources, resource_name, resource_key, obj)
+          resource = resources.fetch_with_default("name", resource_name, {})
+          resource[resource_key] ||= []
+          resource[resource_key] << obj
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+
+          failure = false
+
+          Local.list.each do |region, roles|
+            info region, indent: 2, heading: true
+
+            next if roles.empty?
+
+            roles.each do |role|
+              if !role.is_a?(Hash)
+                bad "role type is not a Hash: #{role}"
+                failure = true
+                next
+              end
+
+              if !role.key?("name")
+                bad "missing key: name"
+                failure = true
+                next
+              end
+
+              if !role.key?("members")
+                bad "missing key: members"
+                failure = true
+                next
+              end
+
+              info role["name"], indent: 3, heading: true
+
+              unless role["members"].is_a?(Array)
+                bad "value not an array for key: #{role}", indent: 4
+                fatal "failure due to invalid config"
+              end
+
+              next unless role.key?("members")
+
+              role["members"].each do |member|
+
+                if !member.is_a?(String)
+                  bad "member isn't a String: #{member}", indent: 3
+                  failure = true
+                  next
+                end
+
+                info member, indent: 4, heading: true
+
+                good "member is a String", indent: 5
+
+                case member
+                when /^user:/
+                  good "member is a 'user'", indent: 5
+                when /^group:/
+                  good "member is a 'group'", indent: 5
+                when /^serviceAccount:/
+                  good "member is a 'serviceAccount'", indent: 5
+                when /^sink:/
+                  good "member is a 'sink'", indent: 5
+                else
+                  bad "member is an unknown type", indent: 5
+                  failure = true
+                end
+              end
+            end
+          end
+
+          fatal "config validation failure" if failure
+        end
+
+        def self.sink(member)
+          gcloud("beta logging sinks describe #{member.gsub('sink:', '')} | jq -r .writer_identity", force: true).chomp
+        rescue
+          fatal "failed to lookup writer identity for sink: #{member}"
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+
+          Local.list.each do |region, roles|
+            info region, indent: 2, heading: true
+
+            roles.each do |role|
+              info role["name"], indent: 3, heading: true
+
+              role["members"].each do |member|
+                if member.start_with?("sink:")
+                  sink_name = member
+                  member = sink(member)
+
+                  if member.empty? && cli_args[:dry_run]
+                    add "unknown - serviceAccount does not exist [#{sink_name}]", indent: 4
+                    next
+                  elsif member.empty?
+                    fatal "unable to find sink serviceAccount (writer identity) - does sink exist for name: #{sink_name}"
+                  end
+                end
+
+                if policy_member?(project_id, role["name"], member)
+                  good member, indent: 4
+                  next
+                end
+
+                if project_owner?
+                  add member, indent: 4
+                  Binding.ensure(project_id, member, role["name"])
+                  next
+                end
+
+                add "#{member} [skipping] (insufficient permissions to create user)", indent: 4
+              end
+            end
+          end
+        end
+
+        def self.policy_member?(project, role, member)
+          bindings = gcloud("--format json projects get-iam-policy #{project} | jq '.bindings[] | select(.role == \"roles/#{role}\")'", force: true)
+          return false if bindings.empty?
+          fatal "could not get policy bindings for project: #{project}" unless bindings.key?("members")
+          bindings["members"].include?(member)
+        end
+
+        def self.project_id
+          project["project_id"]
+        end
+
+        def self.executioner
+          GClouder::Project::ID.id
+        end
+
+        def self.executioner_formatted
+          "user:#{executioner.strip}"
+        end
+
+        def self.project_owner?
+          return false unless executioner
+          policy_member?(project_id, "owner", executioner_formatted)
+        end
+
+        module Local
+          include GClouder::GCloud
+          include GClouder::Config::Project
+          include GClouder::Logging
+
+          def self.list
+            return {} unless project.key?("iam")
+            { "global" => project["iam"] }
+          end
+        end
+
+        module Remote
+          include GClouder::GCloud
+          include GClouder::Config::Project
+          include GClouder::Logging
+
+          def self.list
+            resources.each_with_object({ "global" => [] }) do |data, collection|
+              data["name"] = data["role"].gsub("roles/", "")
+              data.delete("role")
+              collection["global"] << data
+            end
+          end
+
+          def self.resources
+            gcloud("--format json projects get-iam-policy #{project_id} | jq .bindings", force: true)
+          end
+
+          def self.policy_member?(project, role, member)
+            bindings = gcloud("--format json projects get-iam-policy #{project} | jq '.bindings[] | select(.role == \"roles/#{role}\")'", force: true)
+            return false if bindings.empty?
+            fatal "could not get policy bindings for project: #{project}" unless bindings.key?("members")
+            bindings["members"].include?(member["name"])
+          end
+
+          def self.project_id
+            project["project_id"]
+          end
+        end
+
+        module Binding
+          include GClouder::GCloud
+
+          def self.ensure(project_id, name, role)
+            gcloud("projects add-iam-policy-binding #{project_id} --member='#{name}' --role='roles/#{role}'")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/project.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module Project
+      include GClouder::Logging
+      include GClouder::Config::Project
+      include GClouder::GCloud
+
+      def self.header(stage = :ensure)
+        info "[#{stage}] project", title: true
+        info
+      end
+
+      # unprivileged exists? method for use by non-billing accounts
+      def self.exists?
+        Resource.resource?("projects", project["project_id"], filter_key: "project_id", silent: true)
+      end
+
+      def self.update
+        header
+        Local.ensure
+      end
+
+      module Local
+        include GClouder::Config::Project
+        include GClouder::Logging
+        include GClouder::Shell
+        include GClouder::GCloud
+        include GClouder::Config::CLIArgs
+
+        def self.ensure
+          create_project
+          link_project_to_billing_account
+        end
+
+        def self.create_project
+          if exists?
+            good project_id, indent: 2
+            return
+          end
+
+          # FIXME: wait for project to exist and apis be enabled before continuing..
+          # FIXME: enable compute engine api..
+
+          add project_id, indent: 2
+          gcloud("alpha projects create #{project_id} --enable-cloud-apis --name=#{project_id}")
+
+          # FIXME: billing account isn't listed until linked..
+          #sleep 0.5 until exists? unless cli_args[:dry_run]
+        end
+
+        def self.link_project_to_billing_account
+          if linked_to_billing_account?
+            good "linked to billing account: #{account_id}", indent: 3
+            return
+          end
+
+          add "link to billing account: #{account_id}", indent: 3
+          gcloud("alpha billing accounts projects link #{project_id} --account-id=#{account_id}")
+        end
+
+        def self.linked_to_billing_account?
+          project_data(project_id)["billingEnabled"]
+        end
+
+        def self.exists?
+          ! project_data(project_id).empty?
+        end
+
+        def self.project_data(project)
+          shell("gcloud --format json alpha billing accounts projects list #{account_id} | jq '.[] | select(.projectId == \"#{project}\")'")
+        end
+
+        def self.project_id
+          project["project_id"]
+        end
+
+        def self.account_id
+          project["account_id"]
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/project_id.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Project
+    module ID
+      include GClouder::Config::Project
+      include GClouder::Config::CLIArgs
+      include GClouder::Shell
+
+      def self.id
+        @id
+      end
+
+      def self.load
+        @id ||= current
+        switch(project["project_id"]) if @id.nil?
+      end
+
+      def self.current
+        id = shell("gcloud auth list --format json | jq -r '.[] | select(.status == \"ACTIVE\") | .account'")
+        return id if !id.empty?
+        return if cli_args[:activate_service_accounts]
+        bail
+      end
+
+      def self.bail
+        puts "not authenticated against API and --activate-service-accounts option not passed"
+        puts ""
+        puts "please either:"
+        puts ""
+        puts "  run: gcloud auth login && gcloud auth application-default login"
+        puts ""
+        puts "  or: specify --activate-service-accounts flag and make sure the relevant keys exist in the keys dir"
+        puts ""
+        exit 1
+      end
+
+      def self.switch(project_id)
+        return unless project_id
+        if cli_args[:activate_service_accounts]
+          shell("gcloud --quiet auth activate-service-account --key-file #{key_file(project_id)}")
+        end
+      end
+
+      def self.rescue
+        if @id.nil?
+          shell("gcloud config unset account")
+          return
+        end
+
+        switch(@id)
+      end
+
+      def self.reset
+        switch(project["project_id"])
+      end
+
+      def self.default
+        shell("gcloud config set account #{project['project_id']}")
+      end
+
+      def self.dir
+        cli_args[:keys_dir] || File.join(ENV["HOME"], "keys")
+      end
+
+      def self.key_file(project_id)
+        File.join(dir, "gcloud-service-key-#{project_id}.json")
+      end
+    end
+  end
+end

data/lib/gclouder/resources/pubsub/subscriptions.rb

@@ -0,0 +1,100 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module PubSub
+      module Subscriptions
+        include GClouder::Config::CLIArgs
+        include GClouder::Config::Project
+        include GClouder::Logging
+        include GClouder::Resource::Cleaner
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] pub/sub / subscriptions", indent: 1, title: true
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+
+          Local.list.each do |region, subscriptions|
+            info region, indent: 2, heading: true
+            info
+            subscriptions.each do |subscription|
+              Subscription.ensure(subscription)
+            end
+          end
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        module Local
+          include GClouder::Config::CLIArgs
+          include GClouder::Config::Project
+          include GClouder::Logging
+
+          # FIXME: improve validation
+          def self.validate
+            return if list.empty?
+
+            failure = false
+
+            list.each do |region, subscriptions|
+              info region, indent: 2, heading: true
+              subscriptions.each do |subscription|
+                info subscription["name"], indent: 3, heading: true
+                if !subscription["name"].is_a?(String)
+                  bad "#{subscription['name']} is incorrect type #{subscription['name'].class}, should be: String", indent: 4
+                  failure = true
+                end
+
+                if cli_args[:debug] || !cli_args[:output_validation]
+                  good "name is a String", indent: 4
+                end
+              end
+            end
+
+            fatal "\nerror: validation failure" if failure
+          end
+
+          def self.list
+            GClouder::Resources::Global.instances(path: %w(pubsub subscriptions))
+          end
+        end
+
+        module Remote
+          def self.list
+            { "global" => instances.fetch("global", []).map { |subscription| { "name" => subscription["subscription_id"] } } }.delete_if { |_k, v| v.empty? }
+          end
+
+          def self.instances
+            Resources::Remote.instances(
+              path: %w(beta pubsub subscriptions)
+            )
+          end
+        end
+
+        module Subscription
+          include GClouder::GCloud
+          include GClouder::Helpers
+
+          def self.args(subscription)
+            hash_to_args(subscription)
+          end
+
+          def self.ensure(subscription)
+            Resource.ensure :"beta pubsub subscriptions", subscription["name"], args(subscription), filter_key: "subscriptionId", indent: 3
+          end
+
+          def self.purge(subscription)
+            Resource.purge :"beta pubsub subscriptions", subscription["name"]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gclouder/resources/pubsub/topics.rb

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+
+module GClouder
+  module Resources
+    module PubSub
+      module Topics
+        include GClouder::Config::CLIArgs
+        include GClouder::Config::Project
+        include GClouder::Logging
+        include GClouder::Resource::Cleaner
+
+        def self.header(stage = :ensure)
+          info "[#{stage}] pub/sub / topics", indent: 1, title: true
+        end
+
+        def self.ensure
+          return if Local.list.empty?
+          header
+
+          Local.list.each do |region, topics|
+            info region, indent: 2, heading: true
+            info
+            topics.each do |topic|
+              Topic.ensure(topic["name"])
+            end
+          end
+        end
+
+        def self.validate
+          return if Local.list.empty?
+          header :validate
+          Local.validate
+        end
+
+        module Local
+          include GClouder::Config::CLIArgs
+          include GClouder::Config::Project
+          include GClouder::Logging
+
+          # FIXME: improve validation
+          def self.validate
+            return if list.empty?
+
+            failure = false
+
+            list.each do |region, topics|
+              info region, indent: 2, heading: true
+              topics.each do |topic|
+                info topic["name"], indent: 3, heading: true
+                if !topic["name"].is_a?(String)
+                  bad "#{topic['name']} is incorrect type #{topic['name'].class}, should be: String", indent: 4
+                  failure = true
+                end
+
+                if cli_args[:debug] || !cli_args[:output_validation]
+                  good "name is a String", indent: 4
+                end
+              end
+            end
+
+            fatal "\nerror: validation failure" if failure
+          end
+
+          def self.list
+            GClouder::Resources::Global.instances(path: %w(pubsub topics))
+          end
+        end
+
+        module Remote
+          def self.list
+            { "global" => instances.fetch("global", []).map { |topic| { "name" => topic["topic_id"] } } }.delete_if { |_k, v| v.empty? }
+          end
+
+          def self.instances
+            Resources::Remote.instances(
+              path: %w(beta pubsub topics)
+            )
+          end
+        end
+
+        module Topic
+          include GClouder::GCloud
+
+          def self.ensure(topic)
+            Resource.ensure :"beta pubsub topics", topic, filter_key: "topicId"
+          end
+
+          def self.purge(topic)
+            Resource.purge :"beta pubsub topics", topic
+          end
+        end
+      end
+    end
+  end
+end