seira 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6bcdddd7385af9bd0c795472c6f1fe911d9586b1
-  data.tar.gz: ad65f785990edea62925605e04bba6b4d116cd93
+  metadata.gz: 95d1b86a64920650d43cdc3d5063fadc2f17be89
+  data.tar.gz: d40cbb781664aeef22dfcadd4bc048ad27a69414
 SHA512:
-  metadata.gz: a7c87cba6fa27f9debf2f11b0db52ecab602465e92e77c1a9774f875152d79c1b11b996d05dfcfbc671731a5557b2fb975031dad69c08a5709892e1b79593eb0
-  data.tar.gz: 5e8d13d180754d03d0abeb381047efae3ae65eb795bb6320f36fe3e5ce81915ac2ca530108571ae90451f1a1c0aa7dd6b760ea1b1e4ac35f9ac779acada61ae5
+  metadata.gz: 60c8eaaa70a5d8dad533e3bd1533f74bb7b59edbc7ce360ab68d43a4271d9eaf5c141e03ec2e7fbdb70ea917fcba5b8b32d5c45ab08108487fc84ce1ab2d34ef
+  data.tar.gz: aa114b74294c756197411515f4f94cd160187a00f9ef47c22f7d0fae181ff8089f2533266efe9aed62f989b5ff553c984300fa1e0af4d40ea3d0c2b68c577c94

data/lib/helpers.rb

@@ -29,6 +29,12 @@ module Seira
       def get_secret(key:, context:)
         Secrets.new(app: context[:app], action: 'get', args: [], context: context).get(key)
       end
+
+      def shell_username
+        `whoami`
+      rescue
+        'unknown'
+      end
     end
   end
 end

data/lib/seira/app.rb

@@ -197,24 +197,20 @@ module Seira
       puts "Copying source yaml from #{source} to temp folder"
       FileUtils.mkdir_p destination # Create the nested directory
       FileUtils.rm_rf("#{destination}/.", secure: true) # Clean out old files from the tmp folder
-      FileUtils.copy_entry source, destination
-      # Anything in jobs directory is not intended to be applied when deploying
-      # the app, but rather ran when needed as Job objects. Force to avoid exception if DNE.
-      FileUtils.rm_rf("#{destination}/jobs/") if File.directory?("#{destination}/jobs/")
 
       # Iterate through each yaml file and find/replace and save
       puts "Iterating temp folder files find/replace revision information"
-      Dir.foreach(destination) do |item|
+      Dir.foreach(source) do |item|
         next if item == '.' || item == '..'
 
         # If we have run into a directory item, skip it
-        next if File.directory?("#{destination}/#{item}")
+        next if File.directory?("#{source}/#{item}")
 
         # Skip any manifest file that has "seira-skip.yaml" at the end. Common use case is for Job definitions
         # to be used in "seira staging <app> jobs run"
         next if item.end_with?("seira-skip.yaml")
 
-        text = File.read("#{destination}/#{item}")
+        text = File.read("#{source}/#{item}")
 
         new_contents = text
         replacement_hash.each do |key, value|

data/lib/seira/db.rb

@@ -1,4 +1,5 @@
 require 'securerandom'
+require 'English'
 
 require_relative 'db/create'
 
@@ -6,7 +7,7 @@ module Seira
   class Db
     include Seira::Commands
 
-    VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze create-readonly-user].freeze
+    VALID_ACTIONS = %w[help create delete list restart connect ps kill analyze create-readonly-user psql table-sizes index-sizes vacuum unused-indexes unused-indices user-connections info].freeze
     SUMMARY = "Manage your Cloud SQL Postgres databases.".freeze
 
     attr_reader :app, :action, :args, :context
@@ -40,6 +41,20 @@ module Seira
         run_analyze
       when 'create-readonly-user'
         run_create_readonly_user
+      when 'psql'
+        run_psql
+      when 'table-sizes'
+        run_table_sizes
+      when 'index-sizes'
+        run_index_sizes
+      when 'vacuum'
+        run_vacuum
+      when 'unused-indexes', 'unused-indices'
+        run_unused_indexes
+      when 'user-connections'
+        run_user_connections
+      when 'info'
+        run_info
       else
         fail "Unknown command encountered"
       end
@@ -62,15 +77,24 @@ module Seira
     def run_help
       puts SUMMARY
       puts "\n"
-      puts "create: Create a new postgres instance in cloud sql. Supports creating replicas and other numerous flags."
-      puts "delete: Delete a postgres instance from cloud sql. Use with caution, and remove all kubernetes configs first."
-      puts "list: List all postgres instances."
-      puts "restart: Fully restart the database."
-      puts "connect: Open a psql command prompt. You will be shown the password needed before the prompt opens."
-      puts "ps: List running queries"
-      puts "kill: Kill a query"
-      puts "analyze: Display database performance information"
-      puts "create-readonly-user: Create a database user named by --username=<name> with only SELECT access privileges"
+      puts <<~HELPTEXT
+        analyze:              Display database performance information
+        connect:              Open a psql command prompt via gcloud connect. You will be shown the password needed before the prompt opens.
+        create:               Create a new postgres instance in cloud sql. Supports creating replicas and other numerous flags.
+        create-readonly-user: Create a database user named by --username=<name> with only SELECT access privileges
+        delete:               Delete a postgres instance from cloud sql. Use with caution, and remove all kubernetes configs first.
+        index-sizes:          List sizes of all indexes in the database
+        info:                 Summarize all database instances for the app
+        kill:                 Kill a query
+        list:                 List all postgres instances.
+        ps:                   List running queries
+        psql:                 Open a psql prompt via kubectl exec into a pgbouncer pod.
+        restart:              Fully restart the database.
+        table-sizes:          List sizes of all tables in the database
+        unused-indexes:       Show indexes with zero or low usage
+        user-connections:     List number of connections per user
+        vacuum:               Run a VACUUM ANALYZE
+      HELPTEXT
     end
 
     def run_create
@@ -120,21 +144,22 @@ module Seira
         end
       end
 
-      execute_db_command(<<~SQL
-        SELECT
-          pid,
-          state,
-          application_name AS source,
-          age(now(),query_start) AS running_for,
-          query_start,
-          wait_event IS NOT NULL AS waiting,
-          query
-        FROM pg_stat_activity
-        WHERE
-          query <> '<insufficient privilege>'
-          #{verbose ? '' : "AND state <> 'idle'"}
-          AND pid <> pg_backend_pid()
-        ORDER BY query_start DESC
+      execute_db_command(
+        <<~SQL
+          SELECT
+            pid,
+            state,
+            application_name AS source,
+            age(now(),query_start) AS running_for,
+            query_start,
+            wait_event IS NOT NULL AS waiting,
+            query
+          FROM pg_stat_activity
+          WHERE
+            query <> '<insufficient privilege>'
+            #{verbose ? '' : "AND state <> 'idle'"}
+            AND pid <> pg_backend_pid()
+          ORDER BY query_start DESC
         SQL
       )
     end
@@ -235,7 +260,148 @@ module Seira
       execute_db_command(database_commands)
     end
 
-    def execute_db_command(sql_command, as_admin: false)
+    def run_psql
+      execute_db_command(nil, interactive: true)
+    end
+
+    def run_table_sizes
+      # https://wiki.postgresql.org/wiki/Disk_Usage
+      execute_db_command(
+        <<~SQL
+          SELECT table_name
+            , row_estimate
+            , pg_size_pretty(table_bytes) AS table
+            , pg_size_pretty(index_bytes) AS index
+            , pg_size_pretty(toast_bytes) AS toast
+            , pg_size_pretty(total_bytes) AS total
+          FROM (
+            SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM (
+              SELECT relname AS table_name
+                  , c.reltuples AS row_estimate
+                  , pg_total_relation_size(c.oid) AS total_bytes
+                  , pg_indexes_size(c.oid) AS index_bytes
+                  , pg_total_relation_size(reltoastrelid) AS toast_bytes
+                FROM pg_class c
+                LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
+                WHERE relkind = 'r'
+                AND n.nspname = 'public'
+            ) a
+          ) a
+          ORDER BY total_bytes DESC;
+        SQL
+      )
+    end
+
+    def run_index_sizes
+      # https://wiki.postgresql.org/wiki/Disk_Usage
+      execute_db_command(
+        <<~SQL
+          SELECT relname AS index
+            , c.reltuples AS row_estimate
+            , pg_size_pretty(pg_relation_size(c.oid)) AS "size"
+          FROM pg_class c
+          LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)
+          WHERE relkind = 'i'
+          AND n.nspname = 'public'
+          ORDER BY pg_relation_size(c.oid) DESC;
+        SQL
+      )
+    end
+
+    def run_vacuum
+      execute_db_command(
+        <<~SQL
+          VACUUM VERBOSE ANALYZE;
+        SQL
+      )
+    end
+
+    def run_unused_indexes
+      # https://github.com/heroku/heroku-pg-extras/blob/master/commands/unused_indexes.js
+      execute_db_command(
+        <<~SQL
+          SELECT
+            schemaname || '.' || relname AS table,
+            indexrelname AS index,
+            pg_size_pretty(pg_relation_size(i.indexrelid)) AS index_size,
+            idx_scan as index_scans
+          FROM pg_stat_user_indexes ui
+          JOIN pg_index i ON ui.indexrelid = i.indexrelid
+          WHERE NOT indisunique AND idx_scan < 50 AND pg_relation_size(relid) > 5 * 8192
+          ORDER BY pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,
+          pg_relation_size(i.indexrelid) DESC;
+        SQL
+      )
+    end
+
+    def run_user_connections
+      execute_db_command(
+        <<~SQL
+          SELECT usename AS user, count(pid) FROM pg_stat_activity GROUP BY usename;
+        SQL
+      )
+    end
+
+    def run_info
+      instances = JSON.parse(gcloud("sql instances list --filter='name~\\A#{app}-'", context: context, format: :json))
+      instances.each do |instance|
+        db_info_command =
+          <<~SQL
+            COPY (SELECT pg_size_pretty(sum(pg_database_size(datname))) FROM pg_database) TO stdout;
+            COPY (SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE') TO stdout;
+            COPY (SELECT count(*) FROM pg_stat_activity) TO stdout;
+          SQL
+        db_info = execute_db_command(db_info_command, print: false)
+        data_size, table_count, connection_count = db_info.split("\n")
+        instance['data_size'] = data_size
+        instance['table_count'] = table_count
+        instance['connection_count'] = connection_count
+      end
+      instances.each do |instance|
+        # https://cloud.google.com/sql/faq
+        disk_size = instance['settings']['dataDiskSizeGb'].to_f
+        connection_limit =
+          if disk_size <= 0.6
+            25
+          elsif disk_size <= 3.75
+            50
+          elsif disk_size <= 6
+            100
+          elsif disk_size <= 7.5
+            150
+          elsif disk_size <= 15
+            200
+          elsif disk_size <= 30
+            250
+          elsif disk_size <= 60
+            300
+          elsif disk_size <= 120
+            400
+          else
+            500
+          end
+
+        backup_info = instance['settings']['backupConfiguration']['enabled'] == 'true' ? instance['settings']['backupConfiguration']['startTime'] : 'false'
+
+        puts "\n"
+        puts instance['name'].bold
+        puts <<~INFOTEXT
+          State:        #{instance['state']}
+          Tables:       #{instance['table_count']}
+          Disk Size:    #{disk_size} GB
+          Data Size:    #{instance['data_size']}
+          Auto Resize:  #{instance['settings']['storageAutoResize']}
+          Disk Type:    #{instance['settings']['dataDiskType']}
+          Tier:         #{instance['settings']['tier']}
+          Availability: #{instance['settings']['availabilityType']}
+          Version:      #{instance['databaseVersion']}
+          Backups:      #{backup_info}
+          Connections:  #{instance['connection_count']}/#{connection_limit}(?)
+        INFOTEXT
+      end
+    end
+
+    def execute_db_command(sql_command, as_admin: false, interactive: false, print: true)
       # TODO(josh): move pgbouncer naming logic here and in Create to a common location
       instance_name = primary_instance
       tier = instance_name.gsub("#{app}-", '')
@@ -252,7 +418,19 @@ module Seira
         else
           'psql'
         end
-      exit 1 unless system("kubectl exec #{pod_name} --namespace #{app} -- #{psql_command} -c \"#{sql_command}\"")
+      system_command = "kubectl exec #{pod_name} --namespace #{app}"
+      system_command += ' -ti' if interactive
+      system_command += " -- #{psql_command}"
+      system_command += " -c \"#{sql_command}\"" unless sql_command.nil?
+      if interactive
+        exit(1) unless system(system_command)
+      else
+        output = `#{system_command}`
+        success = $CHILD_STATUS.success?
+        puts output if print || !success
+        exit(1) unless success
+        output
+      end
     end
 
     def existing_instances(remove_app_prefix: true)

data/lib/seira/jobs.rb

@@ -84,7 +84,7 @@ module Seira
       replacement_hash = {
         'UNIQUE_NAME' => unique_name,
         'REVISION' => revision,
-        'COMMAND' => command.split(' ').map { |part| "\"#{part}\"" }.join(", "),
+        'COMMAND' => %("sh", "-c", "#{command}"),
         'CPU_REQUEST' => '200m',
         'CPU_LIMIT' => '500m',
         'MEMORY_REQUEST' => '500Mi',

data/lib/seira/pods.rb

@@ -105,7 +105,10 @@ module Seira
           kind: 'Pod',
           spec: spec,
           metadata: {
-            name: temp_name
+            name: temp_name,
+            annotations: {
+              owner: Helpers.shell_username
+            }
           }
         }
         # Don't restart the pod when it dies

data/lib/seira/version.rb

@@ -1,3 +1,3 @@
 module Seira
-  VERSION = "0.4.0".freeze
+  VERSION = "0.4.1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seira
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Scott Ringwelski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline