seira 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8166db4d61ddd6bc9667557e52ee2af61516e303
-  data.tar.gz: d75de14291b27b1cf5fc35d49ac50ff54c2cc6f4
+  metadata.gz: 137cf8ac1506f10dac88ef667c49c5f6a30b515b
+  data.tar.gz: f55493c0ddfc6615f0954d9672d78a2399e41bdb
 SHA512:
-  metadata.gz: 27d97e3e35d7edb6d4b61d3377be3a8b66dde379382ddc167d19b4683beb8a859f0d64017c7d2bf0d2a6c1f43ec7f127d87841a4c7d4e22dd70ea07045bcc331
-  data.tar.gz: 3f8979b56c80fd58a94220b354f74d4adabcd45d460212a4086fbca9499c8cc8cee6a890f166647be0299fcb180be832191b070e32c5fffc4f07115f2e16eda7
+  metadata.gz: 9d179c9b561dafcf2ecc0162e4c493b488a28e18e62195d8913c2aa2d211f5af805301cc5f8d6dd9f2ec59feae023d36409ce7d0e51b71a7852d140afe636a31
+  data.tar.gz: c31974b1f7b50272779f76f1628ed5aade976fe5c36035314b7774d4e3e3092ad1ba38b750ab8ebb6f1e83ecca6c86eec6f066927770c29e7371f5fbc2a16244

data/lib/seira/app.rb

@@ -41,12 +41,12 @@ module Seira
       puts "Possible actions:\n\n"
       puts "bootstrap: Create new app with main secret, cloudsql secret, and gcr secret in the new namespace."
       puts "apply: Apply the configuration in kubernetes/<cluster-name>/<app-name> using REVISION environment variable to find/replace REVISION in the YAML."
-      puts "restart: TODO."
+      puts "restart: Forces a rolling deploy for any deployment making use of RESTARTED_AT_VALUE in the deployment."
       puts "scale: Scales the given tier deployment to the specified number of instances."
     end
 
     def run_restart
-      # TODO
+      run_apply(restart: true)
     end
 
     private
@@ -60,7 +60,7 @@ module Seira
     end
 
     # Kube vanilla based upgrade
-    def run_apply
+    def run_apply(restart: false)
       destination = "tmp/#{context[:cluster]}/#{app}"
       revision = ENV['REVISION']
 
@@ -73,6 +73,10 @@ module Seira
 
       replacement_hash = { 'REVISION' => revision }
 
+      if restart
+        replacement_hash['RESTARTED_AT_VALUE'] = Time.now.to_s
+      end
+
       replacement_hash.each do |k, v|
         next unless v.nil? || v == ''
         puts "Found nil or blank value for replacement hash key #{k}. Aborting!"
@@ -144,7 +148,8 @@ module Seira
 
     def find_and_replace_revision(source:, destination:, replacement_hash:)
       puts "Copying source yaml from #{source} to #{destination}"
-      FileUtils::mkdir_p destination # Create the nested directory
+      FileUtils.mkdir_p destination # Create the nested directory
+      FileUtils.rm_rf("#{destination}/.", secure: true) # Clean out old files from the tmp folder
       FileUtils.copy_entry source, destination
 
       # Iterate through each yaml file and find/replace and save

data/lib/seira/db/create.rb

@@ -0,0 +1,327 @@
+module Seira
+  class Db
+    class Create
+      attr_reader :app, :action, :args, :context
+
+      attr_reader :name, :version, :cpu, :memory, :storage, :replica_for, :make_highly_available
+      attr_reader :root_password, :proxyuser_password
+
+      def initialize(app:, action:, args:, context:)
+        @app = app
+        @action = action
+        @args = args
+        @context = context
+
+        # We allow overriding the version, so you could specify a mysql version but much of the
+        # below assumes postgres for now
+        @version = 'POSTGRES_9_6'
+        @cpu = 1 # Number of CPUs
+        @memory = 4 # GB
+        @storage = 10 # GB
+        @replica_for = nil
+        @make_highly_available = false
+
+        @root_password = nil
+        @proxyuser_password = nil
+      end
+
+      def run(existing_instances)
+        @name = "#{app}-#{Seira::Random.unique_name(existing_instances)}"
+
+        run_create_command
+
+        if replica_for.nil?
+          update_root_password
+          create_proxy_user
+        end
+
+        set_secrets
+        write_pgbouncer_yaml
+
+        puts "To use this database, deploy the pgbouncer config file that was created and use the ENV that was set."
+        puts "To make this database the primary, promote it using the CLI and update the DATABASE_URL."
+      end
+
+      private
+
+      def run_create_command
+        # The 'beta' is needed for HA and other beta features
+        create_command = "gcloud beta sql instances create #{name}"
+
+        args.each do |arg|
+          if arg.start_with? '--version='
+            @version = arg.split('=')[1]
+          elsif arg.start_with? '--cpu='
+            @cpu = arg.split('=')[1]
+          elsif arg.start_with? '--memory='
+            @memory = arg.split('=')[1]
+          elsif arg.start_with? '--storage='
+            @storage = arg.split('=')[1]
+          elsif arg.start_with? '--primary='
+            @replica_for = arg.split('=')[1] # TODO: Read secret to get it automatically
+          elsif arg.start_with? '--highly-available'
+            @make_highly_available = true
+          elsif /^--[\w\-]+=.+$/.match? arg
+            create_command += " #{arg}"
+          else
+            puts "Warning: Unrecognized argument '#{arg}'"
+          end
+        end
+
+        if make_highly_available && !replica_for.nil?
+          puts "Cannot create an HA read-replica."
+          exit(1)
+        end
+
+        # Basic configs
+        create_command += " --database-version=#{version}"
+
+        # A read replica cannot have HA, inherits the cpu, mem and storage of its primary
+        if replica_for.nil?
+          # Make sure to do automated daily backups by default, unless it's a replica
+          create_command += " --backup"
+          create_command += " --cpu=#{cpu}"
+          create_command += " --memory=#{memory}"
+          create_command += " --storage-size=#{storage}"
+
+          # Make HA if asked for
+          create_command += " --availability-type=REGIONAL" if make_highly_available
+        else
+          create_command += " --master-instance-name=#{replica_for}"
+          # We don't need to wait for it to finish to move ahead if it's a replica, as we don't
+          # make any changes to the database itself
+          create_command += " --async"
+        end
+
+        puts "Running: #{create_command}"
+
+        # Create the sql instance with the specified/default parameters
+        if system(create_command)
+          async_additional =
+            unless replica_for.nil?
+              ". Database is still being created and may take some time to be available."
+            end
+
+          puts "Successfully created sql instance #{name}#{async_additional}"
+        else
+          puts "Failed to create sql instance"
+          exit(1)
+        end
+      end
+
+      def update_root_password
+        # Set the root user's password to something secure
+        @root_password = SecureRandom.urlsafe_base64(32)
+
+        if system("gcloud sql users set-password postgres '' --instance=#{name} --password=#{root_password}")
+          puts "Set root password to #{root_password}"
+        else
+          puts "Failed to set root password"
+          exit(1)
+        end
+      end
+
+      def create_proxy_user
+        # Create proxyuser with secure password
+        @proxyuser_password = SecureRandom.urlsafe_base64(32)
+
+        if system("gcloud sql users create proxyuser '' --instance=#{name} --password=#{proxyuser_password}")
+          puts "Created proxyuser with password #{proxyuser_password}"
+        else
+          puts "Failed to create proxyuser"
+          exit(1)
+        end
+
+        # Connect to the instance and remove some of the default group memberships and permissions
+        # from proxyuser, leaving it with only what it needs to be able to do
+        expect_script = <<~BASH
+          set timeout 90
+          spawn gcloud sql connect #{name}
+          expect "Password for user postgres:"
+          send "#{root_password}\\r"
+          expect "postgres=>"
+          send "ALTER ROLE proxyuser NOCREATEDB NOCREATEROLE;\\r"
+          expect "postgres=>"
+        BASH
+        if system("expect <<EOF\n#{expect_script}EOF")
+          puts "Successfully removed unnecessary permissions from proxyuser"
+        else
+          puts "Failed to remove unnecessary permissions from proxyuser"
+          exit(1)
+        end
+      end
+
+      def set_secrets
+        env_name = name.tr('-', '_').upcase
+
+        # If setting as primary, update relevant secrets. Only primaries have root passwords.
+        if replica_for.nil?
+          create_pgbouncer_secret(db_user: 'proxyuser', db_password: proxyuser_password)
+          Secrets.new(app: app, action: 'set', args: ["#{env_name}_ROOT_PASSWORD=#{root_password}"], context: context).run
+          write_database_env(key: "DATABASE_URL", db_user: 'proxyuser', db_password: proxyuser_password)
+          write_database_env(key: "#{env_name}_DB_URL", db_user: 'proxyuser', db_password: proxyuser_password)
+        else
+          # When creating a replica, we cannot manage users on the replica. We must manage the users on the primary, which the replica
+          # inherits. For now we will use the same credentials that the primary uses.
+          primary_uri = URI.parse(Secrets.new(app: app, action: 'get', args: [], context: context).get('DATABASE_URL'))
+          primary_user = primary_uri.user
+          primary_password = primary_uri.password
+          create_pgbouncer_secret(db_user: primary_user, db_password: primary_password)
+          write_database_env(key: "#{env_name}_DB_URL", db_user: primary_user, db_password: primary_password)
+        end
+      end
+
+      def create_pgbouncer_secret(db_user:, db_password:)
+        puts `kubectl create secret generic #{pgbouncer_secret_name} --namespace #{app} --from-literal=DB_USER=#{db_user} --from-literal=DB_PASSWORD=#{db_password}`
+      end
+
+      def write_database_env(key:, db_user:, db_password:)
+        Secrets.new(app: app, action: 'set', args: ["#{key}=postgres://#{db_user}:#{db_password}@#{pgbouncer_service_name}:6432"], context: context).run
+      end
+
+      def pgbouncer_secret_name
+        "#{name}-pgbouncer-secrets"
+      end
+
+      def pgbouncer_configs_name
+        "#{name}-pgbouncer-configs"
+      end
+
+      def pgbouncer_service_name
+        "#{name}-pgbouncer-service"
+      end
+
+      def pgbouncer_tier
+        name.gsub("handshake-", "")
+      end
+
+      def write_pgbouncer_yaml
+        # TODO: Clean this up by moving into a proper templated yaml file
+        pgbouncer_yaml = <<-FOO
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: #{pgbouncer_configs_name}
+  namespace: #{app}
+data:
+  DB_HOST: "127.0.0.1"
+  DB_PORT: "5432"
+  LISTEN_PORT: "6432"
+  LISTEN_ADDRESS: "*"
+  TCP_KEEPALIVE: "1"
+  TCP_KEEPCNT: "5"
+  TCP_KEEPIDLE: "300" # see: https://git.io/vi0Aj
+  TCP_KEEPINTVL: "300"
+  LOG_DISCONNECTIONS: "0" # spammy, not needed
+  MAX_CLIENT_CONN: "500"
+  MAX_DB_CONNECTIONS: "90"
+  DEFAULT_POOL_SIZE: "90"
+  POOL_MODE: "transaction"
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: #{name}-pgbouncer
+  namespace: #{app}
+  labels:
+    app: #{app}
+    tier: #{pgbouncer_tier}
+    database: #{name}
+spec:
+  replicas: 1
+  minReadySeconds: 20
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: #{app}
+        tier: #{pgbouncer_tier}
+        database: #{name}
+    spec:
+      containers:
+        - image: handshake/pgbouncer:0.1.2
+          name: pgbouncer
+          ports:
+            - containerPort: 6432
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: #{pgbouncer_configs_name}
+            - secretRef:
+                name: #{pgbouncer_secret_name}
+          readinessProbe:
+            tcpSocket:
+              port: 6432
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: 6432
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 100m
+              memory: 300m
+        - image: gcr.io/cloudsql-docker/gce-proxy:1.11    # Gcloud SQL Proxy
+          name: cloudsql-proxy
+          command:
+            - /cloud_sql_proxy
+            - --dir=/cloudsql
+            - -instances=#{context[:project]}:#{context[:default_zone]}:#{name}=tcp:5432
+            - -credential_file=/secrets/cloudsql/credentials.json
+          ports:
+            - containerPort: 5432
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: cloudsql-configs
+          volumeMounts:
+            - name: cloudsql-credentials
+              mountPath: /secrets/cloudsql
+              readOnly: true
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs
+            - name: cloudsql
+              mountPath: /cloudsql
+      volumes:
+        - name: cloudsql-credentials
+          secret:
+            secretName: cloudsql-credentials
+        - name: cloudsql
+          emptyDir:
+        - name: ssl-certs
+          hostPath:
+            path: /etc/ssl/certs
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: #{pgbouncer_service_name}
+  namespace: #{app}
+  labels:
+    app: #{app}
+    tier: #{pgbouncer_tier}
+spec:
+  type: NodePort
+  ports:
+  - protocol: TCP
+    port: 6432
+    targetPort: 6432
+    nodePort: 0
+  selector:
+    app: #{app}
+    tier: #{pgbouncer_tier}
+    database: #{name}
+FOO
+
+        File.write("kubernetes/#{context[:cluster]}/#{app}/pgbouncer-#{name.gsub("#{app}-", '')}.yaml", pgbouncer_yaml)
+      end
+    end
+  end
+end

data/lib/seira/db.rb

@@ -1,5 +1,7 @@
 require 'securerandom'
 
+require_relative 'db/create.rb'
+
 module Seira
   class Db
     VALID_ACTIONS = %w[help create delete list].freeze
@@ -33,104 +35,23 @@ module Seira
 
     def run_help
       puts SUMMARY
-      puts "\n\n"
-      puts "TODO"
+      puts "\n"
+      puts "create: Create a new postgres instance in cloud sql. Supports creating replicas and other numerous flags."
+      puts "delete: Delete a postgres instance from cloud sql. Use with caution, and remove all kubernetes configs first."
+      puts "list: List all postgres instances."
     end
 
     def run_create
-      # We allow overriding the version, so you could specify a mysql version but much of the
-      # below assumes postgres for now
-      version = 'POSTGRES_9_6'
-      cpu = 1 # Number of CPUs
-      memory = 4 # GB
-      storage = 10 # GB
-      set_as_primary = false
-
-      name = "#{app}-#{Seira::Random.unique_name(existing_instances)}"
-
-      create_command = "gcloud sql instances create #{name}"
-
-      args.each do |arg|
-        if arg.start_with? '--version='
-          version = arg.split('=')[1]
-        elsif arg.start_with? '--cpu='
-          cpu = arg.split('=')[1]
-        elsif arg.start_with? '--memory='
-          memory = arg.split('=')[1]
-        elsif arg.start_with? '--storage='
-          storage = arg.split('=')[1]
-        elsif arg.start_with? '--set-as-primary='
-          set_as_primary = %w[true yes t y].include?(arg.split('=')[1])
-        elsif /^--[\w\-]+=.+$/.match? arg
-          create_command += " #{arg}"
-        else
-          puts "Warning: Unrecognized argument '#{arg}'"
-        end
-      end
-
-      create_command += " --database-version=#{version}"
-      create_command += " --cpu=#{cpu}"
-      create_command += " --memory=#{memory}"
-      create_command += " --storage-size=#{storage}"
-
-      # Create the sql instance with the specified/default parameters
-      if system(create_command)
-        puts "Successfully created sql instance #{name}"
-      else
-        puts "Failed to create sql instance"
-        exit(1)
-      end
-
-      # Set the root user's password to something secure
-      root_password = SecureRandom.urlsafe_base64(32)
-      if system("gcloud sql users set-password postgres '' --instance=#{name} --password=#{root_password}")
-        puts "Set root password to #{root_password}"
-      else
-        puts "Failed to set root password"
-        exit(1)
-      end
-
-      # Create proxyuser with secure password
-      proxyuser_password = SecureRandom.urlsafe_base64(32)
-      if system("gcloud sql users create proxyuser '' --instance=#{name} --password=#{proxyuser_password}")
-        puts "Created proxyuser with password #{proxyuser_password}"
-      else
-        puts "Failed to create proxyuser"
-        exit(1)
-      end
-
-      # Connect to the instance and remove some of the default group memberships and permissions
-      # from proxyuser, leaving it with only what it needs to be able to do
-      expect_script = <<~BASH
-        set timeout 90
-        spawn gcloud sql connect #{name}
-        expect "Password for user postgres:"
-        send "#{root_password}\\r"
-        expect "postgres=>"
-        send "ALTER ROLE proxyuser NOCREATEDB NOCREATEROLE;\\r"
-        expect "postgres=>"
-      BASH
-      if system("expect <<EOF\n#{expect_script}EOF")
-        puts "Successfully removed unnecessary permissions from proxyuser"
-      else
-        puts "Failed to remove unnecessary permissions from proxyuser"
-        exit(1)
-      end
-
-      # If setting as primary, update relevant secrets
-      if set_as_primary
-        Secrets.new(app: app, action: 'create-pgbouncer-secret', args: ['proxyuser', proxyuser_password], context: context).run
-        Secrets.new(app: app, action: 'set', args: ["DATABASE_URL=postgres://proxyuser:#{proxyuser_password}@#{app}-pgbouncer-service:6432"], context: context).run
-      end
-      # Regardless of primary or not, store a URL for this db matching its unique name
-      env_name = name.tr('-', '_').upcase
-      Secrets.new(app: app, action: 'set', args: ["#{env_name}_DB_URL=postgres://proxyuser:#{proxyuser_password}@#{app}-pgbouncer-service:6432", "#{env_name}_ROOT_PASSWORD=#{root_password}"], context: context).run
+      Seira::Db::Create.new(app: app, action: action, args: args, context: context).run(existing_instances)
     end
 
     def run_delete
       name = "#{app}-#{args[0]}"
       if system("gcloud sql instances delete #{name}")
         puts "Successfully deleted sql instance #{name}"
+
+        # TODO: Automate the below
+        puts "Don't forget to delete the deployment, configmap, secret, and service for the pgbouncer instance."
       else
         puts "Failed to delete sql instance #{name}"
       end

data/lib/seira/secrets.rb

@@ -38,8 +38,6 @@ module Seira
         run_list
       when 'list-decoded'
         run_list_decoded
-      when 'create-pgbouncer-secret'
-        run_create_pgbouncer_secret
       else
         fail "Unknown command encountered"
       end
@@ -65,6 +63,11 @@ module Seira
       "#{app}-secrets"
     end
 
+    def get(key)
+      secrets = fetch_current_secrets
+      Base64.decode64(secrets['data'][key])
+    end
+
     private
 
     def run_help
@@ -88,8 +91,7 @@ module Seira
     end
 
     def run_get
-      secrets = fetch_current_secrets
-      puts "#{key}: #{Base64.decode64(secrets['data'][key])}"
+      puts "#{key}: #{get(key)}"
     end
 
     def run_set
@@ -120,12 +122,6 @@ module Seira
       end
     end
 
-    def run_create_pgbouncer_secret
-      db_user = args[0]
-      db_password = args[1]
-      puts `kubectl create secret generic #{PGBOUNCER_SECRETS_NAME} --namespace #{app} --from-literal=DB_USER=#{db_user} --from-literal=DB_PASSWORD=#{db_password}`
-    end
-
     # In the normal case the secret we are updating is just main_secret_name,
     # but in special cases we may be doing an operation on a different secret
     def write_secrets(secrets:, secret_name: main_secret_name)

data/lib/seira/settings.rb

@@ -48,6 +48,10 @@ module Seira
       nil
     end
 
+    def project_for_cluster(cluster)
+      settings['seira']['clusters'][cluster]['project']
+    end
+
     private
 
     def parse_settings

data/lib/seira/version.rb

@@ -1,3 +1,3 @@
 module Seira
-  VERSION = "0.1.4".freeze
+  VERSION = "0.1.5".freeze
 end

data/lib/seira.rb

@@ -30,7 +30,7 @@ module Seira
       'setup' => Seira::Setup
     }.freeze
 
-    attr_reader :cluster, :app, :category, :action, :args
+    attr_reader :project, :cluster, :app, :category, :action, :args
     attr_reader :settings
 
     # Pop from beginning repeatedly for the first 4 main args, and then take the remaining back to original order for
@@ -64,6 +64,7 @@ module Seira
       end
 
       @cluster = @settings.full_cluster_name_for_shorthand(cluster)
+      @project = @settings.project_for_cluster(@cluster)
     end
 
     def run
@@ -100,7 +101,9 @@ module Seira
 
     def passed_context
       {
-        cluster: cluster
+        cluster: cluster,
+        project: project,
+        default_zone: settings.default_zone
       }
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seira
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Scott Ringwelski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2017-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -106,6 +106,7 @@ files:
 - lib/seira/app.rb
 - lib/seira/cluster.rb
 - lib/seira/db.rb
+- lib/seira/db/create.rb
 - lib/seira/memcached.rb
 - lib/seira/pods.rb
 - lib/seira/proxy.rb