seira 0.3.3 → 0.3.6

data/.rubocop.yml

@@ -53,7 +53,10 @@ Style/SignalException:
 Style/StringLiterals:
   Enabled: false
 
-Style/TrailingCommaInLiteral:
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
   Enabled: false
 
 Style/TrailingCommaInArguments:

data/lib/helpers.rb

@@ -1,5 +1,7 @@
 module Seira
   class Helpers
+    include Seira::Commands
+
     class << self
       def rails_env(context:)
         if context[:cluster] == 'internal'
@@ -11,7 +13,21 @@ module Seira
 
       def fetch_pods(filters:, app:)
         filter_string = { app: app }.merge(filters).map { |k, v| "#{k}=#{v}" }.join(',')
-        JSON.parse(`kubectl get pods --namespace=#{app} -o json --selector=#{filter_string}`)['items']
+        output = Seira::Commands.kubectl("get pods -o json --selector=#{filter_string}", context: { app: app }, return_output: true)
+        JSON.parse(output)['items']
+      end
+
+      def log_link(context:, app:, query:)
+        link = context[:settings].log_link_format
+        return nil if link.nil?
+        link.gsub! 'APP', app
+        link.gsub! 'CLUSTER', context[:cluster]
+        link.gsub! 'QUERY', query
+        link
+      end
+
+      def get_secret(app:, key:, context: {})
+        Secrets.new(app: app, action: 'get', args: [], context: context).get(key)
       end
     end
   end

data/lib/seira.rb

@@ -3,6 +3,8 @@ require 'highline/import'
 require 'colorize'
 require 'tmpdir'
 
+require 'seira/commands'
+
 require "seira/version"
 require 'helpers'
 require 'seira/app'
@@ -23,6 +25,8 @@ require 'seira/node_pools'
 # work for the command to a class in lib/seira folder.
 module Seira
   class Runner
+    include Seira::Commands
+
     CATEGORIES = {
       'secrets' => Seira::Secrets,
       'pods' => Seira::Pods,
@@ -128,7 +132,10 @@ module Seira
         cluster: cluster,
         project: project,
         settings: settings,
-        default_zone: settings.default_zone
+        default_zone: settings.default_zone,
+        app: app,
+        action: action,
+        args: args
       }
     end
 

data/lib/seira/app.rb

@@ -6,6 +6,8 @@ require 'fileutils'
 # seira staging specs app bootstrap
 module Seira
   class App
+    include Seira::Commands
+
     VALID_ACTIONS = %w[help bootstrap apply restart scale revision].freeze
     SUMMARY = "Bootstrap, scale, configure, restart, your apps.".freeze
 
@@ -53,7 +55,7 @@ module Seira
 
     def ask_cluster_for_current_revision
       tier = context[:settings].config_for_app(app)['golden_tier'] || 'web'
-      current_image = `kubectl get deployment --namespace=#{app} -l app=#{app},tier=#{tier} -o=jsonpath='{$.items[:1].spec.template.spec.containers[:1].image}'`.strip.chomp
+      current_image = kubectl("get deployment -l app=#{app},tier=#{tier} -o=jsonpath='{$.items[:1].spec.template.spec.containers[:1].image}'", context: context, return_output: true).strip.chomp
       current_revision = current_image.split(':').last
       current_revision
     end
@@ -63,7 +65,8 @@ module Seira
     def run_bootstrap
       # TODO: Verify that 00-namespace exists
       # TODO: Do conformance test on the yaml files before running anything, including that 00-namespace.yaml exists and has right name
-      system("kubectl apply -f kubernetes/#{context[:cluster]}/#{app}/00-namespace.yaml") # Create namespace before anything else
+      # Create namespace before anything else
+      kubectl("apply -f kubernetes/#{context[:cluster]}/#{app}/00-namespace.yaml", context: context)
       bootstrap_main_secret
       bootstrap_cloudsql_secret
       bootstrap_gcr_secret
@@ -113,8 +116,7 @@ module Seira
           replacement_hash: replacement_hash
         )
 
-        puts "Running 'kubectl apply -f #{destination}'"
-        system("kubectl apply -f #{destination}")
+        kubectl("apply -f #{destination}", context: context)
 
         unless async
           puts "Monitoring rollout status..."
@@ -144,7 +146,7 @@ module Seira
         end
         replicas = config['spec']['replicas'] if replicas == 'default'
         puts "scaling #{tier} to #{replicas}"
-        system("kubectl scale --namespace=#{app} --replicas=#{replicas} deployments/#{config['metadata']['name']}")
+        kubectl("scale --replicas=#{replicas} deployments/#{config['metadata']['name']}", context: context)
       end
     end
 
@@ -159,7 +161,7 @@ module Seira
       # 'internal' is a unique cluster/project "cluster". It always means production in terms of rails app.
       rails_env = Helpers.rails_env(context: context)
 
-      puts `kubectl create secret generic #{main_secret_name} --namespace #{app} --from-literal=RAILS_ENV=#{rails_env} --from-literal=RACK_ENV=#{rails_env}`
+      kubectl("create secret generic #{main_secret_name} --from-literal=RAILS_ENV=#{rails_env} --from-literal=RACK_ENV=#{rails_env}", context: context)
     end
 
     # We use a secret in our container to use a service account to connect to our cloudsql databases. The secret in 'default'

data/lib/seira/cluster.rb

@@ -5,6 +5,8 @@ require 'fileutils'
 # Example usages:
 module Seira
   class Cluster
+    include Seira::Commands
+
     VALID_ACTIONS = %w[help bootstrap upgrade-master].freeze
     SUMMARY = "For managing whole clusters.".freeze
 
@@ -49,12 +51,16 @@ module Seira
     end
 
     def self.current_cluster
-      `kubectl config current-context`.chomp.strip
+      Seira::Commands.kubectl("config current-context", context: :none, return_output: true).chomp.strip
+    end
+
+    def self.current_project
+      Seira::Commands.gcloud("config get-value project", context: :none, return_output: true).chomp.strip
     end
 
     def current
-      puts `gcloud config get-value project`
-      puts `kubectl config current-context`
+      puts current_project
+      puts current_cluster
     end
 
     private
@@ -76,7 +82,6 @@ module Seira
         exit(1)
       end
 
-      # puts `kubectl create secret generic gcr-secret --namespace default --from-file=.dockercfg=#{dockercfg_location}`
       puts `kubectl create secret docker-registry gcr-secret --docker-username=_json_key --docker-password="$(cat #{dockercfg_location})" --docker-server=https://gcr.io --docker-email=doesnotmatter@example.com`
       puts `kubectl create secret generic cloudsql-credentials --namespace default --from-file=credentials.json=#{cloudsql_credentials_location}`
     end
@@ -92,7 +97,7 @@ module Seira
       end
 
       # Ensure the specified version is supported by GKE
-      server_config = JSON.parse(`gcloud container get-server-config --format json`)
+      server_config = gcloud("container get-server-config", format: :json, context: context)
       valid_versions = server_config['validMasterVersions']
       unless valid_versions.include? new_version
         puts "Version #{new_version} is unsupported. Supported versions are:"
@@ -100,7 +105,7 @@ module Seira
         exit(1)
       end
 
-      cluster_config = JSON.parse(`gcloud container clusters describe #{cluster} --format json`)
+      cluster_config = JSON.parse(gcloud("container clusters describe #{cluster}", format: :json, context: context))
 
       # Update the master node first
       exit(1) unless Highline.agree("Are you sure you want to upgrade cluster #{cluster} master to version #{new_version}? Services should continue to run fine, but the cluster control plane will be offline.")
@@ -109,7 +114,7 @@ module Seira
       if cluster_config['currentMasterVersion'] == new_version
         # Master has already been updated; this step is not needed
         puts 'Already up to date!'
-      elsif system("gcloud container clusters upgrade #{cluster} --cluster-version=#{new_version} --master")
+      elsif gcloud("container clusters upgrade #{cluster} --cluster-version=#{new_version} --master", format: :boolean, context: context)
         puts 'Master updated successfully!'
       else
         puts 'Failed to update master.'

data/lib/seira/commands.rb

@@ -0,0 +1,22 @@
+require 'seira/commands/kubectl'
+require 'seira/commands/gcloud'
+
+module Seira
+  module Commands
+    def kubectl(command, context:, clean_output: false, return_output: false)
+      Seira::Commands.kubectl(command, context: context, clean_output: clean_output, return_output: return_output)
+    end
+
+    def self.kubectl(command, context:, clean_output: false, return_output: false)
+      Kubectl.new(command, context: context).invoke(clean_output: clean_output, return_output: return_output)
+    end
+
+    def gcloud(command, context:, clean_output: false, format: :boolean)
+      Seira::Commands.gcloud(command, context: context, clean_output: clean_output, format: format)
+    end
+
+    def self.gcloud(command, context:, clean_output: false, format: :boolean)
+      Gcloud.new(command, context: context, clean_output: clean_output, format: format).invoke
+    end
+  end
+end

data/lib/seira/commands/gcloud.rb

@@ -0,0 +1,43 @@
+module Seira
+  module Commands
+    class Gcloud
+      attr_reader :context, :command, :format, :clean_output
+
+      def initialize(command, context:, clean_output:, format:)
+        @command = command
+        @context = context
+        @format = format
+        @clean_output = clean_output
+      end
+
+      def invoke
+        puts "Calling: #{calculated_command.green}" unless clean_output
+
+        if format == :boolean
+          system(calculated_command)
+        elsif format == :json
+          `#{calculated_command}`
+        end
+      end
+
+      private
+
+      def calculated_command
+        @_calculated_command ||= begin
+          rv =
+            if format == :json
+              "gcloud #{command} --format=json"
+            else
+              "gcloud #{command}"
+            end
+
+          unless context.nil?
+            rv = "#{rv} --project=#{context[:project]}"
+          end
+
+          rv
+        end
+      end
+    end
+  end
+end

data/lib/seira/commands/kubectl.rb

@@ -0,0 +1,34 @@
+module Seira
+  module Commands
+    class Kubectl
+      attr_reader :context, :command
+
+      def initialize(command, context:)
+        @command = command
+        @context = context
+      end
+
+      def invoke(clean_output: false, return_output: false)
+        puts "Calling: #{calculated_command.green}" unless clean_output
+
+        if return_output
+          `#{calculated_command}`
+        else
+          system(calculated_command)
+        end
+      end
+
+      private
+
+      def calculated_command
+        @_calculated_command ||= begin
+          if context == :none
+            "kubectl #{command}"
+          else
+            "kubectl #{command} --namespace=#{context[:app]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/seira/db.rb

@@ -4,7 +4,9 @@ require_relative 'db/create'
 
 module Seira
   class Db
-    VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze].freeze
+    include Seira::Commands
+
+    VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze create-readonly-user].freeze
     SUMMARY = "Manage your Cloud SQL Postgres databases.".freeze
 
     attr_reader :app, :action, :args, :context
@@ -36,6 +38,8 @@ module Seira
         run_kill
       when 'analyze'
         run_analyze
+      when 'create-readonly-user'
+        run_create_readonly_user
       else
         fail "Unknown command encountered"
       end
@@ -43,7 +47,7 @@ module Seira
 
     # NOTE: Relies on the pgbouncer instance being named based on the db name, as is done in create command
     def primary_instance
-      database_url = Secrets.new(app: app, action: 'get', args: [], context: context).get('DATABASE_URL')
+      database_url = Helpers.get_secret(app: app, key: 'DATABASE_URL')
       return nil unless database_url
 
       primary_uri = URI.parse(database_url)
@@ -66,6 +70,7 @@ module Seira
       puts "ps: List running queries"
       puts "kill: Kill a query"
       puts "analyze: Display database performance information"
+      puts "create-readonly-user: Create a database user named by --username=<name> with only SELECT access privileges"
     end
 
     def run_create
@@ -74,7 +79,7 @@ module Seira
 
     def run_delete
       name = "#{app}-#{args[0]}"
-      if system("gcloud sql instances delete #{name}")
+      if gcloud("sql instances delete #{name}", context: context, format: :boolean)
         puts "Successfully deleted sql instance #{name}"
 
         # TODO: Automate the below
@@ -90,7 +95,7 @@ module Seira
 
     def run_restart
       name = "#{app}-#{args[0]}"
-      if system("gcloud sql instances restart #{name}")
+      if gcloud("sql instances restart #{name}", context: context, format: :boolean)
         puts "Successfully restarted sql instance #{name}"
       else
         puts "Failed to restart sql instance #{name}"
@@ -100,7 +105,7 @@ module Seira
     def run_connect
       name = args[0] || primary_instance
       puts "Connecting to #{name}..."
-      root_password = Secrets.new(app: app, action: 'get', args: [], context: context).get("#{name.tr('-', '_').upcase}_ROOT_PASSWORD") || "Not found in secrets"
+      root_password = Helpers.get_secret(app: app, key: "#{name.tr('-', '_').upcase}_ROOT_PASSWORD") || "Not found in secrets"
       puts "Your root password for 'postgres' user is: #{root_password}"
       system("gcloud sql connect #{name}")
     end
@@ -176,20 +181,88 @@ module Seira
       )
     end
 
-    def execute_db_command(sql_command)
+    # Example: seira staging app-name db create-readonly-user --username=readonlyuser
+    def run_create_readonly_user
+      instance_name = primary_instance # Always make user changes to primary instance, and they will propogate to replicas
+      user_name = nil
+
+      args.each do |arg|
+        if arg.start_with? '--username='
+          user_name = arg.split('=')[1]
+        else
+          puts "Warning: Unrecognized argument '#{arg}'"
+        end
+      end
+
+      if user_name.nil? || user_name.strip.chomp == ''
+        puts "Please specify the name of the read-only user to create, such as --username=testuser"
+        exit(1)
+      end
+
+      # Require that the name be alpha only for simplicity and strict but basic validation
+      if user_name.match(/\A[a-zA-Z]*\z/).nil?
+        puts "Username must be characters only"
+        exit(1)
+      end
+
+      valid_instance_names = existing_instances(remove_app_prefix: false).join(', ')
+      if instance_name.nil? || instance_name.strip.chomp == '' || !valid_instance_names.include?(instance_name)
+        puts "Could not find a valid instance name - does the DATABASE_URL have a value? Must be one of: #{valid_instance_names}"
+        exit(1)
+      end
+
+      password = SecureRandom.urlsafe_base64(32)
+      if gcloud("sql users create #{user_name} '' --instance=#{instance_name} --password=#{password}", context: context, format: :boolean)
+        puts "Created user '#{user_name}' with password #{password}"
+      else
+        puts "Failed to create user '#{user_name}'"
+        exit(1)
+      end
+
+      puts 'Setting permissions...'
+      admin_commands =
+        <<~SQL
+          REVOKE cloudsqlsuperuser FROM #{user_name};
+          ALTER ROLE #{user_name} NOCREATEDB NOCREATEROLE;
+        SQL
+      database_commands =
+        <<~SQL
+          REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM #{user_name};
+          GRANT SELECT ON ALL TABLES IN SCHEMA public TO #{user_name};
+          ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO #{user_name};
+        SQL
+      execute_db_command(admin_commands, as_admin: true)
+      execute_db_command(database_commands)
+    end
+
+    def execute_db_command(sql_command, as_admin: false)
       # TODO(josh): move pgbouncer naming logic here and in Create to a common location
-      tier = primary_instance.gsub("#{app}-", '')
+      instance_name = primary_instance
+      tier = instance_name.gsub("#{app}-", '')
       matching_pods = Helpers.fetch_pods(app: app, filters: { tier: tier })
       if matching_pods.empty?
         puts 'Could not find pgbouncer pod to connect to'
         exit 1
       end
       pod_name = matching_pods.first['metadata']['name']
-      exit 1 unless system("kubectl exec #{pod_name} --namespace #{app} -- psql -c \"#{sql_command}\"")
+      psql_command =
+        if as_admin
+          root_password = Helpers.get_secret(app: app, key: "#{instance_name.tr('-', '_').upcase}_ROOT_PASSWORD")
+          "psql postgres://postgres:#{root_password}@127.0.0.1:5432"
+        else
+          'psql'
+        end
+      exit 1 unless system("kubectl exec #{pod_name} --namespace #{app} -- #{psql_command} -c \"#{sql_command}\"")
     end
 
-    def existing_instances
-      `gcloud sql instances list --uri`.split("\n").map { |uri| uri.split('/').last }.select { |name| name.start_with? "#{app}-" }.map { |name| name.gsub(/^#{app}-/, '') }
+    def existing_instances(remove_app_prefix: true)
+      plain_list = `gcloud sql instances list --uri`.split("\n").map { |uri| uri.split('/').last }.select { |name| name.start_with? "#{app}-" }
+
+      if remove_app_prefix
+        plain_list.map { |name| name.gsub(/^#{app}-/, '') }
+      else
+        plain_list
+      end
     end
   end
 end